use FileFd instead of forking the compression childs by hand

apt-pkg/contrib/fileutl.cc

@@ -50,6 +50,8 @@
 #define APT_USE_ZLIB 1
 #if APT_USE_ZLIB
 #include <zlib.h>
+#else
+#warning "Usage of zlib is DISABLED!"
 #endif
 
 #ifdef WORDS_BIGENDIAN
@@ -71,7 +73,7 @@ class FileFdPrivate {
 	pid_t compressor_pid;
 	bool pipe;
 	APT::Configuration::Compressor compressor;
-	FileFd::OpenMode openmode;
+	unsigned int openmode;
 	FileFdPrivate() : gz(NULL), compressor_pid(-1), pipe(false) {};
 };
 
@@ -858,8 +860,13 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog,
    for (int J = 0; J != 2; J++)
       SetCloseExec(Pipe[J],true);
 
+   int FileFd = -1;
    if (Comp == true)
+   {
       OutFd = Pipe[1];
+      // FIXME: we should handle openmode and permission from Open() here
+      FileFd = open(FileName.c_str(), O_WRONLY, 0666);
+   }
    else
       OutFd = Pipe[0];
 
@@ -872,13 +879,14 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog,
       if (Comp == true)
       {
 	 dup2(Pipe[0],STDIN_FILENO);
+	 dup2(FileFd,STDOUT_FILENO);
 	 SetCloseExec(STDIN_FILENO,false);
       }
       else
       {
 	 dup2(Pipe[1],STDOUT_FILENO);
-	 SetCloseExec(STDOUT_FILENO,false);
       }
+      SetCloseExec(STDOUT_FILENO,false);
 
       std::vector<char const*> Args;
       Args.push_back(Prog.Binary.c_str());
@@ -887,8 +895,11 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog,
       for (std::vector<std::string>::const_iterator a = addArgs->begin();
 	   a != addArgs->end(); ++a)
 	 Args.push_back(a->c_str());
-      Args.push_back("--stdout");
-      Args.push_back(FileName.c_str());
+      if (Comp == false)
+      {
+	 Args.push_back("--stdout");
+	 Args.push_back(FileName.c_str());
+      }
       Args.push_back(NULL);
 
       execvp(Args[0],(char **)&Args[0]);
@@ -896,7 +907,10 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog,
       _exit(100);
    }
    if (Comp == true)
+   {
       close(Pipe[0]);
+      close(FileFd);
+   }
    else
       close(Pipe[1]);
 
@@ -910,7 +924,7 @@ bool ExecCompressor(APT::Configuration::Compressor const &Prog,
 // FileFd::Open - Open a file						/*{{{*/
 // ---------------------------------------------------------------------
 /* The most commonly used open mode combinations are given with Mode */
-bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long const Perms)
+bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const Perms)
 {
    if (Mode == ReadOnlyGzip)
       return Open(FileName, ReadOnly, Gzip, Perms);
@@ -934,11 +948,20 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned
    }
    else if (Compress == Extension)
    {
-      std::string ext = flExtension(FileName);
-      if (ext == FileName)
-	 ext.clear();
-      else
-	 ext = "." + ext;
+      std::string::size_type const found = FileName.find_last_of('.');
+      std::string ext;
+      if (found != std::string::npos)
+      {
+	 ext = FileName.substr(found);
+	 if (ext == ".new" || ext == ".bak")
+	 {
+	    std::string::size_type const found2 = FileName.find_last_of('.', found - 1);
+	    if (found2 != std::string::npos)
+	       ext = FileName.substr(found2, found - found2);
+	    else
+	       ext.clear();
+	 }
+      }
       for (; compressor != compressors.end(); ++compressor)
 	 if (ext == compressor->Extension)
 	    break;
@@ -960,8 +983,8 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned
       case Xz: name = "xz"; break;
       case Auto:
       case Extension:
-         // Unreachable
-         return _error->Error("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str());
+	 // Unreachable
+	 return _error->Error("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str());
       }
       for (; compressor != compressors.end(); ++compressor)
 	 if (compressor->Name == name)
@@ -974,7 +997,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned
       return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str());
    return Open(FileName, Mode, *compressor, Perms);
 }
-bool FileFd::Open(string FileName,OpenMode Mode,APT::Configuration::Compressor const &compressor, unsigned long const Perms)
+bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const Perms)
 {
    Close();
    d = new FileFdPrivate;
@@ -1015,8 +1038,35 @@ bool FileFd::Open(string FileName,OpenMode Mode,APT::Configuration::Compressor c
       if ((Mode & ReadWrite) == ReadWrite)
 	 return _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor.Name.c_str(), FileName.c_str());
 
-      if (ExecCompressor(compressor, NULL /*d->compressor_pid*/, FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false)
-         return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), FileName.c_str());
+      if ((Mode & (WriteOnly | Create)) == (WriteOnly | Create))
+      {
+	 if (TemporaryFileName.empty() == false)
+	 {
+	    if (RealFileExists(TemporaryFileName) == false)
+	    {
+	       iFd = open(TemporaryFileName.c_str(), O_WRONLY | O_CREAT, Perms);
+	       close(iFd);
+	       iFd = -1;
+	    }
+	 }
+	 else if (RealFileExists(FileName) == false)
+	 {
+	    iFd = open(FileName.c_str(), O_WRONLY | O_CREAT, Perms);
+	    close(iFd);
+	    iFd = -1;
+	 }
+      }
+
+      if (TemporaryFileName.empty() == false)
+      {
+	 if (ExecCompressor(compressor, &(d->compressor_pid), TemporaryFileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false)
+	    return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), TemporaryFileName.c_str());
+      }
+      else
+      {
+	 if (ExecCompressor(compressor, &(d->compressor_pid), FileName, iFd, ((Mode & ReadOnly) != ReadOnly)) == false)
+	    return _error->Error("Forking external compressor %s is not implemented for %s", compressor.Name.c_str(), FileName.c_str());
+      }
       d->pipe = true;
       d->compressor = compressor;
    }
@@ -1060,7 +1110,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,APT::Configuration::Compressor c
 // FileFd::OpenDescriptor - Open a filedescriptor			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose)
+bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose)
 {
    std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
    std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
@@ -1084,7 +1134,7 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool A
 
    return OpenDescriptor(Fd, Mode, *compressor, AutoClose);
 }
-bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compressor const &compressor, bool AutoClose)
+bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose)
 {
    Close();
    d = new FileFdPrivate;
@@ -1100,7 +1150,7 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compresso
    this->FileName = "";
    return true;
 }
-bool FileFd::OpenInternDescriptor(OpenMode Mode, APT::Configuration::Compressor const &compressor)
+bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor)
 {
    if (compressor.Name == ".")
       return true;
@@ -1471,8 +1521,8 @@ bool FileFd::Close()
 
    if (d != NULL)
    {
-//      if (d->compressor_pid != -1)
-//	 ExecWait(d->compressor_pid, "FileFdCompressor", true);
+      if (d->compressor_pid != -1)
+	 ExecWait(d->compressor_pid, "FileFdCompressor", true);
       delete d;
       d = NULL;
    }

apt-pkg/contrib/fileutl.h

@@ -100,14 +100,14 @@ class FileFd
 	return T;
    }
 
-   bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long const Perms = 0666);
-   bool Open(std::string FileName,OpenMode Mode,APT::Configuration::Compressor const &compressor,unsigned long const Perms = 0666);
-   inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long const Perms = 0666) {
+   bool Open(std::string FileName,unsigned int const Mode,CompressMode Compress,unsigned long const Perms = 0666);
+   bool Open(std::string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor,unsigned long const Perms = 0666);
+   inline bool Open(std::string const &FileName,unsigned int const Mode, unsigned long const Perms = 0666) {
       return Open(FileName, Mode, None, Perms);
    };
-   bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false);
-   bool OpenDescriptor(int Fd, OpenMode Mode, APT::Configuration::Compressor const &compressor, bool AutoClose=false);
-   inline bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false) {
+   bool OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose=false);
+   bool OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose=false);
+   inline bool OpenDescriptor(int Fd, unsigned int const Mode, bool AutoClose=false) {
       return OpenDescriptor(Fd, Mode, None, AutoClose);
    };
    bool Close();
@@ -126,16 +126,16 @@ class FileFd
    inline bool IsCompressed() {return (Flags & Compressed) == Compressed;};
    inline std::string &Name() {return FileName;};
    
-   FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL)
+   FileFd(std::string FileName,unsigned int const Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL)
    {
       Open(FileName,Mode, None, Perms);
    };
-   FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL)
+   FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL)
    {
       Open(FileName,Mode, Compress, Perms);
    };
    FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {};
-   FileFd(int const Fd, OpenMode Mode = ReadWrite, CompressMode Compress = None) : iFd(-1), Flags(0), d(NULL)
+   FileFd(int const Fd, unsigned int const Mode = ReadWrite, CompressMode Compress = None) : iFd(-1), Flags(0), d(NULL)
    {
       OpenDescriptor(Fd, Mode, Compress);
    };
@@ -147,7 +147,7 @@ class FileFd
 
    private:
    FileFdPrivate* d;
-   bool OpenInternDescriptor(OpenMode Mode, APT::Configuration::Compressor const &compressor);
+   bool OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor);
 };
 
 bool RunScripts(const char *Cnf);

ftparchive/multicompress.cc

@@ -91,7 +91,7 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress,
    /* Open all the temp files now so we can report any errors. File is 
       made unreable to prevent people from touching it during creating. */
    for (Files *I = Outputs; I != 0; I = I->Next)
-      I->TmpFile.Open(I->Output + ".new",FileFd::WriteEmpty,0600);
+      I->TmpFile.Open(I->Output + ".new", FileFd::WriteOnly | FileFd::Create | FileFd::Empty, FileFd::Extension, 0600);
    if (_error->PendingError() == true)
       return;
 
@@ -183,11 +183,6 @@ bool MultiCompress::Start()
       _exit(0);
    };
 
-   /* Tidy up the temp files, we open them in the constructor so as to
-      get proper error reporting. Close them now. */
-   for (Files *I = Outputs; I != 0; I = I->Next)
-      I->TmpFile.Close();
-   
    close(Pipe[0]);
    Input = fdopen(Pipe[1],"w");
    if (Input == 0)
@@ -305,14 +300,6 @@ bool MultiCompress::CloseOld(int Fd,pid_t Proc)
    is new then the temp files are renamed, otherwise they are erased. */
 bool MultiCompress::Child(int const &FD)
 {
-   // Start the compression children.
-   for (Files *I = Outputs; I != 0; I = I->Next)
-   {
-      if (ExecCompressor(I->CompressProg,&(I->CompressProc),I->TmpFile.Fd(),
-			 I->Fd,true) == false)
-	 return false;      
-   }
-
    /* Okay, now we just feed data from FD to all the other FDs. Also
       stash a hash of the data to use later. */
    SetNonBlock(FD,false);
@@ -332,25 +319,14 @@ bool MultiCompress::Child(int const &FD)
       FileSize += Res;
       for (Files *I = Outputs; I != 0; I = I->Next)
       {
-	 if (write(I->Fd,Buffer,Res) != Res)
+	 if (I->TmpFile.Write(Buffer, Res) == false)
 	 {
 	    _error->Errno("write",_("IO to subprocess/file failed"));
 	    break;
 	 }
       }      
    }   
-   
-   // Close all the writers
-   for (Files *I = Outputs; I != 0; I = I->Next)
-      close(I->Fd);
-   
-   // Wait for the compressors to exit
-   for (Files *I = Outputs; I != 0; I = I->Next)
-   {
-      if (I->CompressProc != -1)
-	 ExecWait(I->CompressProc, I->CompressProg.Binary.c_str(), false);
-   }
-   
+
    if (_error->PendingError() == true)
       return false;
    

ftparchive/multicompress.h

@@ -34,7 +34,6 @@ class MultiCompress
       FileFd TmpFile;
       pid_t CompressProc;
       time_t OldMTime;
-      int Fd;
    };
    
    Files *Outputs;