Stable acquire code

Author: jgg
Date: 1998-10-26 07:11:43 GMT
Stable acquire code

apt-pkg/acquire-item.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-item.cc,v 1.4 1998/10/24 04:57:56 jgg Exp $
+// $Id: acquire-item.cc,v 1.5 1998/10/26 07:11:43 jgg Exp $
 /* ######################################################################
 
    Acquire Item - Item to acquire
@@ -65,6 +65,22 @@ void pkgAcquire::Item::Done(string,unsigned long,string)
    Owner->Dequeue(this);
 }
 									/*}}}*/
+// Acquire::Item::Rename - Rename a file				/*{{{*/
+// ---------------------------------------------------------------------
+/* This helper function is used by alot of item methods as thier final
+   step */
+void pkgAcquire::Item::Rename(string From,string To)
+{
+   if (rename(From.c_str(),To.c_str()) != 0)
+   {
+      char S[300];
+      sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno),
+	      From.c_str(),To.c_str());
+      Status = StatError;
+      ErrorText = S;
+   }      
+}
+									/*}}}*/
 
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
@@ -73,6 +89,8 @@ void pkgAcquire::Item::Done(string,unsigned long,string)
 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) :
              Item(Owner), Location(Location)
 {
+   Decompression = false;
+   
    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
    DestFile += URItoFileName(Location->PackagesURI());
    
@@ -97,6 +115,57 @@ string pkgAcqIndex::Custom600Headers()
    return "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 }
 									/*}}}*/
+// AcqIndex::Done - Finished a fetch					/*{{{*/
+// ---------------------------------------------------------------------
+/* This goes through a number of states.. On the initial fetch the
+   method could possibly return an alternate filename which points
+   to the uncompressed version of the file. If this is so the file
+   is copied into the partial directory. In all other cases the file
+   is decompressed with a gzip uri. */
+void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5)
+{
+   Item::Done(Message,Size,MD5);
+
+   if (Decompression == true)
+   {
+      // Done, move it into position
+      string FinalFile = _config->FindDir("Dir::State::lists");
+      FinalFile += URItoFileName(Location->PackagesURI());
+      Rename(DestFile,FinalFile);
+      return;
+   }
+      
+   // Handle the unzipd case
+   string FileName = LookupTag(Message,"Alt-Filename");
+   if (FileName.empty() == false)
+   {
+      // The files timestamp matches
+      if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
+	 return;
+      
+      Decompression = true;
+      DestFile += ".decomp";
+      QueueURI("copy:" + FileName,string());
+      return;
+   }
+
+   FileName = LookupTag(Message,"Filename");
+   if (FileName.empty() == true)
+   {
+      Status = StatError;
+      ErrorText = "Method gave a blank filename";
+   }
+   
+   // The files timestamp matches
+   if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
+      return;
+   
+   Decompression = true;
+   DestFile += ".decomp";
+   QueueURI("gzip:" + FileName,string());
+}
+									/*}}}*/
+
 // AcqIndexRel::pkgAcqIndexRel - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* The Release file is added to the queue */
@@ -139,7 +208,12 @@ void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5)
    {
       Status = StatError;
       ErrorText = "Method gave a blank filename";
+      return;
    }
+
+   // The files timestamp matches
+   if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
+      return;
    
    // We have to copy it into place
    if (FileName != DestFile)
@@ -151,14 +225,6 @@ void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5)
    // Done, move it into position
    string FinalFile = _config->FindDir("Dir::State::lists");
    FinalFile += URItoFileName(Location->ReleaseURI());
-   
-   if (rename(DestFile.c_str(),FinalFile.c_str()) != 0)
-   {
-      char S[300];
-      sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno),
-	      DestFile.c_str(),FinalFile.c_str());
-      Status = StatError;
-      ErrorText = S;
-   }
+   Rename(DestFile,FinalFile);
 }
 									/*}}}*/

apt-pkg/acquire-item.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-item.h,v 1.3 1998/10/24 04:57:57 jgg Exp $
+// $Id: acquire-item.h,v 1.4 1998/10/26 07:11:44 jgg Exp $
 /* ######################################################################
 
    Acquire Item - Item to acquire
@@ -33,6 +33,8 @@ class pkgAcquire::Item
    inline void QueueURI(string URI,string Description) 
                  {Owner->Enqueue(this,URI,Description);};
    
+   void Rename(string From,string To);
+   
    public:
 
    // State of the item
@@ -60,9 +62,11 @@ class pkgAcqIndex : public pkgAcquire::Item
    protected:
    
    const pkgSourceList::Item *Location;
+   bool Decompression;
    
    public:
    
+   virtual void Done(string Message,unsigned long Size,string Md5Hash);   
    virtual string Custom600Headers();
 
    pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location);
@@ -77,8 +81,7 @@ class pkgAcqIndexRel : public pkgAcquire::Item
    
    public:
    
-   virtual void Done(string Message,unsigned long Size,string Md5Hash);
-   
+   virtual void Done(string Message,unsigned long Size,string Md5Hash);   
    virtual string Custom600Headers();
    
    pkgAcqIndexRel(pkgAcquire *Owner,const pkgSourceList::Item *Location);

apt-pkg/acquire-worker.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-worker.cc,v 1.6 1998/10/24 04:57:58 jgg Exp $
+// $Id: acquire-worker.cc,v 1.7 1998/10/26 07:11:45 jgg Exp $
 /* ######################################################################
 
    Acquire Worker 
@@ -106,7 +106,9 @@ bool pkgAcquire::Worker::Start()
 	 close(Pipes[I]);
       return false;
    }
-      
+   for (int I = 0; I != 4; I++)
+      SetCloseExec(Pipes[0],true);
+   
    // Fork off the process
    Process = fork();
    if (Process < 0)
@@ -122,8 +124,6 @@ bool pkgAcquire::Worker::Start()
       dup2(Pipes[1],STDOUT_FILENO);
       dup2(Pipes[2],STDIN_FILENO);
       dup2(((filebuf *)clog.rdbuf())->fd(),STDERR_FILENO);
-      for (int I = 0; I != 4; I++)
-	 close(Pipes[I]);
       SetCloseExec(STDOUT_FILENO,false);
       SetCloseExec(STDIN_FILENO,false);      
       SetCloseExec(STDERR_FILENO,false);
@@ -152,7 +152,8 @@ bool pkgAcquire::Worker::Start()
       return _error->Error("Method %s did not start correctly",Method.c_str());
 
    RunMessages();
-   SendConfiguration();
+   if (OwnerQ != 0)
+      SendConfiguration();
    
    return true;
 }
@@ -416,3 +417,35 @@ bool pkgAcquire::Worker::MethodFailure()
    return false;
 }
 									/*}}}*/
+
+// InjectConfiguration - Configuration aid for methods			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool pkgInjectConfiguration(string &Message,Configuration &Cnf)
+{
+   const char *I = Message.begin();
+   
+   unsigned int Length = strlen("Config-Item");
+   for (; I + Length < Message.end(); I++)
+   {
+      // Not a config item
+      if (I[Length] != ':' || stringcasecmp(I,I+Length,"Config-Item") != 0)
+	 continue;
+      
+      I += Length + 1;
+      
+      for (; I < Message.end() && *I == ' '; I++);
+      const char *Equals = I;
+      for (; Equals < Message.end() && *Equals != '='; Equals++);
+      const char *End = Equals;
+      for (; End < Message.end() && *End != '\n'; End++);
+      if (End == Equals)
+	 return false;
+      
+      Cnf.Set(string(I,Equals-I),string(Equals+1,End-Equals-1));
+      I = End;
+   }
+   
+   return true;
+}
+									/*}}}*/

apt-pkg/acquire-worker.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-worker.h,v 1.4 1998/10/24 04:57:59 jgg Exp $
+// $Id: acquire-worker.h,v 1.5 1998/10/26 07:11:46 jgg Exp $
 /* ######################################################################
 
    Acquire Worker - Worker process manager
@@ -13,6 +13,7 @@
 #define PKGLIB_ACQUIRE_WORKER_H
 
 #include <apt-pkg/acquire.h>
+#include <apt-pkg/configuration.h>
 
 #ifdef __GNUG__
 #pragma interface "apt-pkg/acquire-worker.h"
@@ -80,4 +81,6 @@ class pkgAcquire::Worker
    ~Worker();
 };
 
+bool pkgInjectConfiguration(string &Message,Configuration &Cnf);
+
 #endif

apt-pkg/acquire.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.cc,v 1.4 1998/10/24 04:58:01 jgg Exp $
+// $Id: acquire.cc,v 1.5 1998/10/26 07:11:47 jgg Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -33,6 +33,7 @@ pkgAcquire::pkgAcquire()
    Configs = 0;
    Workers = 0;
    ToFetch = 0;
+   Running = false;
    
    string Mode = _config->Find("Acquire::Queue-Mode","host");
    if (strcasecmp(Mode.c_str(),"host") == 0)
@@ -133,6 +134,9 @@ void pkgAcquire::Enqueue(Item *Itm,string URI,string Description)
    // Queue it into the named queue
    I->Enqueue(Itm,URI,Description);
    ToFetch++;
+   
+   if (Running == true)
+      I->Startup();
       
    // Some trace stuff
    if (Debug == true)
@@ -249,6 +253,8 @@ void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
    manage the actual fetch. */
 bool pkgAcquire::Run()
 {
+   Running = true;
+   
    for (Queue *I = Queues; I != 0; I = I->Next)
       I->Startup();
    
@@ -263,7 +269,10 @@ bool pkgAcquire::Run()
       SetFds(Highest,&RFds,&WFds);
       
       if (select(Highest+1,&RFds,&WFds,0,0) <= 0)
+      {
+	 Running = false;
 	 return _error->Errno("select","Select has failed");
+      }
       
       RunFds(&RFds,&WFds);
    }   
@@ -271,6 +280,7 @@ bool pkgAcquire::Run()
    for (Queue *I = Queues; I != 0; I = I->Next)
       I->Shutdown();
 
+   Running = false;
    return true;
 }
 									/*}}}*/

apt-pkg/acquire.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.h,v 1.4 1998/10/24 04:58:02 jgg Exp $
+// $Id: acquire.h,v 1.5 1998/10/26 07:11:48 jgg Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -66,6 +66,7 @@ class pkgAcquire
    // Configurable parameters for the schedular
    enum {QueueHost,QueueAccess} QueueMode;
    bool Debug;
+   bool Running;
    
    void Add(Item *Item);
    void Remove(Item *Item);

apt-pkg/contrib/fileutl.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: fileutl.cc,v 1.12 1998/10/24 22:15:41 jgg Exp $
+// $Id: fileutl.cc,v 1.13 1998/10/26 07:11:49 jgg Exp $
 /* ######################################################################
    
    File Utilities
@@ -29,7 +29,7 @@
 // CopyFile - Buffered copy of a file					/*{{{*/
 // ---------------------------------------------------------------------
 /* The caller is expected to set things so that failure causes erasure */
-bool CopyFile(FileFd From,FileFd To)
+bool CopyFile(FileFd &From,FileFd &To)
 {
    if (From.IsOpen() == false || To.IsOpen() == false)
       return false;

apt-pkg/contrib/fileutl.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: fileutl.h,v 1.7 1998/10/20 02:39:29 jgg Exp $
+// $Id: fileutl.h,v 1.8 1998/10/26 07:11:50 jgg Exp $
 /* ######################################################################
    
    File Utilities
@@ -59,7 +59,7 @@ class FileFd
    virtual ~FileFd();
 };
 
-bool CopyFile(FileFd From,FileFd To);
+bool CopyFile(FileFd &From,FileFd &To);
 int GetLock(string File,bool Errors = true);
 bool FileExists(string File);
 string SafeGetCWD();

doc/examples/apt.conf

@@ -1,4 +1,4 @@
-// $Id: apt.conf,v 1.5 1998/10/24 04:58:10 jgg Exp $
+// $Id: apt.conf,v 1.6 1998/10/26 07:11:51 jgg Exp $
 /* This file is an index of all APT configuration directives. It should
    NOT actually be used as a real config file, though it is a completely
    valid file.
@@ -60,3 +60,5 @@ Debug {
   pkgAcquire "false";
   pkgAcquire::Worker "true";
 }
+
+dir::state::lists "/tmp/lists/";

methods/copy.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: copy.cc,v 1.2 1998/10/25 07:07:29 jgg Exp $
+// $Id: copy.cc,v 1.3 1998/10/26 07:11:52 jgg Exp $
 /* ######################################################################
 
    Copy URI - This method takes a uri like a file: uri and copies it
@@ -121,6 +121,7 @@ int main()
 	 TimeBuf.modtime = Buf.st_mtime;
 	 if (utime(Target.c_str(),&TimeBuf) != 0)
 	 {
+	    To.OpFail();
 	    _error->Errno("utime","Failed to set modification time");
 	    Fail(URI);
 	    continue;

methods/gzip.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: gzip.cc,v 1.1 1998/10/25 07:07:30 jgg Exp $
+// $Id: gzip.cc,v 1.2 1998/10/26 07:11:53 jgg Exp $
 /* ######################################################################
 
    GZip method - Take a file URI in and decompress it into the target 
@@ -123,20 +123,25 @@ int main()
 	 {
 	    dup2(From.Fd(),STDIN_FILENO);
 	    dup2(To.Fd(),STDOUT_FILENO);
+	    From.Close();
+	    To.Close();
+	    SetCloseExec(STDIN_FILENO,false);
+	    SetCloseExec(STDOUT_FILENO,false);
 	    
 	    const char *Args[3];
-	    Args[0] = _config->FindFile("Dir::bin::gzip","gzip").c_str();
+	    Args[0] = _config->Find("Dir::bin::gzip","gzip").c_str();
 	    Args[1] = "-d";
 	    Args[2] = 0;
 	    execvp(Args[0],(char **)Args);
+	    exit(100);
 	 }
 	 From.Close();
-	 To.Close();
 	 
 	 // Wait for gzip to finish
 	 int Status;
 	 if (waitpid(Process,&Status,0) != Process)
 	 {
+	    To.OpFail();
 	    _error->Errno("wait","Waiting for gzip failed");
 	    Fail(URI);
 	    continue;
@@ -144,11 +149,14 @@ int main()
 
 	 if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0)
 	 {
+	    To.OpFail();
 	    _error->Error("gzip failed, perhaps the disk is full or the directory permissions are wrong.");
 	    Fail(URI);
 	    continue;
 	 }	 
 	 
+	 To.Close();
+	 
 	 // Transfer the modification times
 	 struct stat Buf;
 	 if (stat(File.c_str(),&Buf) != 0)