- add a ReadLine method

  - drop the explicit export of gz-compression handling

apt-pkg/contrib/fileutl.cc

@@ -44,6 +44,8 @@
 #include <set>
 #include <algorithm>
 
+#include <zlib.h>
+
 #ifdef WORDS_BIGENDIAN
 #include <inttypes.h>
 #endif
@@ -53,6 +55,12 @@
 
 using namespace std;
 
+class FileFdPrivate {
+	public:
+	gzFile gz;
+	FileFdPrivate() : gz(NULL) {};
+};
+
 // RunScripts - Run a set of scripts from a configuration subtree	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -719,11 +727,12 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap)
 // FileFd::Open - Open a file						/*{{{*/
 // ---------------------------------------------------------------------
 /* The most commonly used open mode combinations are given with Mode */
-bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long Perms)
+bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long const Perms)
 {
    if (Mode == ReadOnlyGzip)
       return Open(FileName, ReadOnly, Gzip, Perms);
    Close();
+   d = new FileFdPrivate;
    Flags = AutoClose;
 
    if (Compress == Auto && (Mode & WriteOnly) == WriteOnly)
@@ -865,6 +874,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned
 bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose)
 {
    Close();
+   d = new FileFdPrivate;
    Flags = (AutoClose) ? FileFd::AutoClose : 0;
    iFd = Fd;
    if (OpenInternDescriptor(Mode, Compress) == false)
@@ -883,13 +893,14 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress)
    else if (Compress == Gzip)
    {
       if ((Mode & ReadWrite) == ReadWrite)
-	 gz = gzdopen(iFd, "r+");
+	 d->gz = gzdopen(iFd, "r+");
       else if ((Mode & WriteOnly) == WriteOnly)
-	 gz = gzdopen(iFd, "w");
+	 d->gz = gzdopen(iFd, "w");
       else
-	 gz = gzdopen (iFd, "r");
-      if (gz == NULL)
+	 d->gz = gzdopen (iFd, "r");
+      if (d->gz == NULL)
 	 return false;
+      Flags |= Compressed;
    }
    else
       return false;
@@ -918,8 +929,8 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
    
    do
    {
-      if (gz != NULL)
-         Res = gzread(gz,To,Size);
+      if (d->gz != NULL)
+         Res = gzread(d->gz,To,Size);
       else
          Res = read(iFd,To,Size);
       if (Res < 0 && errno == EINTR)
@@ -951,6 +962,28 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
    return _error->Error(_("read, still have %llu to read but none left"), Size);
 }
 									/*}}}*/
+// FileFd::ReadLine - Read a complete line from the file		/*{{{*/
+// ---------------------------------------------------------------------
+/* Beware: This method can be quiet slow for big buffers on UNcompressed
+   files because of the naive implementation! */
+char* FileFd::ReadLine(char *To, unsigned long long const Size)
+{
+   if (d->gz != NULL)
+      return gzgets(d->gz, To, Size);
+
+   unsigned long long read = 0;
+   if (Read(To, Size, &read) == false)
+      return NULL;
+   char* c = To;
+   for (; *c != '\n' && *c != '\0' && read != 0; --read, ++c)
+      ; // find the end of the line
+   if (*c != '\0')
+      *c = '\0';
+   if (read != 0)
+      Seek(Tell() - read);
+   return To;
+}
+									/*}}}*/
 // FileFd::Write - Write to the file					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -960,8 +993,8 @@ bool FileFd::Write(const void *From,unsigned long long Size)
    errno = 0;
    do
    {
-      if (gz != NULL)
-         Res = gzwrite(gz,From,Size);
+      if (d->gz != NULL)
+         Res = gzwrite(d->gz,From,Size);
       else
          Res = write(iFd,From,Size);
       if (Res < 0 && errno == EINTR)
@@ -990,8 +1023,8 @@ bool FileFd::Write(const void *From,unsigned long long Size)
 bool FileFd::Seek(unsigned long long To)
 {
    int res;
-   if (gz)
-      res = gzseek(gz,To,SEEK_SET);
+   if (d->gz)
+      res = gzseek(d->gz,To,SEEK_SET);
    else
       res = lseek(iFd,To,SEEK_SET);
    if (res != (signed)To)
@@ -1009,8 +1042,8 @@ bool FileFd::Seek(unsigned long long To)
 bool FileFd::Skip(unsigned long long Over)
 {
    int res;
-   if (gz)
-      res = gzseek(gz,Over,SEEK_CUR);
+   if (d->gz != NULL)
+      res = gzseek(d->gz,Over,SEEK_CUR);
    else
       res = lseek(iFd,Over,SEEK_CUR);
    if (res < 0)
@@ -1027,7 +1060,7 @@ bool FileFd::Skip(unsigned long long Over)
 /* */
 bool FileFd::Truncate(unsigned long long To)
 {
-   if (gz)
+   if (d->gz != NULL)
    {
       Flags |= Fail;
       return _error->Error("Truncating gzipped files is not implemented (%s)", FileName.c_str());
@@ -1047,8 +1080,8 @@ bool FileFd::Truncate(unsigned long long To)
 unsigned long long FileFd::Tell()
 {
    off_t Res;
-   if (gz)
-     Res = gztell(gz);
+   if (d->gz != NULL)
+     Res = gztell(d->gz);
    else
      Res = lseek(iFd,0,SEEK_CUR);
    if (Res == (off_t)-1)
@@ -1076,9 +1109,9 @@ unsigned long long FileFd::Size()
    unsigned long long size = FileSize();
 
    // only check gzsize if we are actually a gzip file, just checking for
-   // "gz" is not sufficient as uncompressed files will be opened with
+   // "gz" is not sufficient as uncompressed files could be opened with
    // gzopen in "direct" mode as well
-   if (gz && !gzdirect(gz) && size > 0)
+   if (d->gz && !gzdirect(d->gz) && size > 0)
    {
        /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
 	* this ourselves; the original (uncompressed) file size is the last 32
@@ -1125,11 +1158,14 @@ time_t FileFd::ModificationTime()
 /* */
 bool FileFd::Close()
 {
+   if (iFd == -1)
+      return true;
+
    bool Res = true;
    if ((Flags & AutoClose) == AutoClose)
    {
-      if (gz != NULL) {
-	 int const e = gzclose(gz);
+      if (d != NULL && d->gz != NULL) {
+	 int const e = gzclose(d->gz);
 	 // gzdopen() on empty files always fails with "buffer error" here, ignore that
 	 if (e != 0 && e != Z_BUF_ERROR)
 	    Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str());
@@ -1147,13 +1183,17 @@ bool FileFd::Close()
    }
 
    iFd = -1;
-   gz = NULL;
 
    if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail &&
        FileName.empty() == false)
       if (unlink(FileName.c_str()) != 0)
 	 Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str());
 
+   if (d != NULL)
+   {
+      delete d;
+      d = NULL;
+   }
 
    return Res;
 }
@@ -1170,3 +1210,4 @@ bool FileFd::Sync()
    return true;
 }
 									/*}}}*/
+gzFile FileFd::gzFd() {return d->gz;};

apt-pkg/contrib/fileutl.h

@@ -31,17 +31,17 @@
 /* Define this for python-apt */
 #define APT_HAS_GZIP 1
 
+class FileFdPrivate;
 class FileFd
 {
    protected:
    int iFd;
  
    enum LocalFlags {AutoClose = (1<<0),Fail = (1<<1),DelOnFail = (1<<2),
-                    HitEof = (1<<3), Replace = (1<<4) };
+                    HitEof = (1<<3), Replace = (1<<4), Compressed = (1<<5) };
    unsigned long Flags;
    std::string FileName;
    std::string TemporaryFileName;
-   gzFile gz;
 
    public:
    enum OpenMode {
@@ -71,6 +71,7 @@ class FileFd
       return Read(To,Size);
    }   
    bool Read(void *To,unsigned long long Size,unsigned long long *Actual = 0);
+   char* ReadLine(char *To, unsigned long long const Size);
    bool Write(const void *From,unsigned long long Size);
    bool Seek(unsigned long long To);
    bool Skip(unsigned long long To);
@@ -94,8 +95,8 @@ class FileFd
 	return T;
    }
 
-   bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long Perms = 0666);
-   inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long Perms = 0666) {
+   bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long const Perms = 0666);
+   inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long const Perms = 0666) {
       return Open(FileName, Mode, None, Perms);
    };
    bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false);
@@ -108,29 +109,36 @@ class FileFd
    // Simple manipulators
    inline int Fd() {return iFd;};
    inline void Fd(int fd) {iFd = fd;};
-   inline gzFile gzFd() {return gz;};
+   __deprecated gzFile gzFd();
    inline bool IsOpen() {return iFd >= 0;};
    inline bool Failed() {return (Flags & Fail) == Fail;};
    inline void EraseOnFailure() {Flags |= DelOnFail;};
    inline void OpFail() {Flags |= Fail;};
    inline bool Eof() {return (Flags & HitEof) == HitEof;};
+   inline bool IsCompressed() {return (Flags & Compressed) == Compressed;};
    inline std::string &Name() {return FileName;};
    
-   FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), 
-            Flags(0), gz(NULL)
+   FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL)
    {
       Open(FileName,Mode, None, Perms);
    };
-   FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) :
-	    iFd(-1), Flags(0), gz(NULL)
+   FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL)
    {
       Open(FileName,Mode, Compress, Perms);
    };
-   FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose), gz(NULL) {};
-   FileFd(int Fd,bool) : iFd(Fd), Flags(0), gz(NULL) {};
+   FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {};
+   FileFd(int const Fd, OpenMode Mode = ReadWrite, CompressMode Compress = None) : iFd(-1), Flags(0), d(NULL)
+   {
+      OpenDescriptor(Fd, Mode, Compress);
+   };
+   FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL)
+   {
+      OpenDescriptor(Fd, ReadWrite, None, AutoClose);
+   };
    virtual ~FileFd();
 
    private:
+   FileFdPrivate* d;
    bool OpenInternDescriptor(OpenMode Mode, CompressMode Compress);
 };
 

apt-pkg/contrib/mmap.cc

@@ -77,7 +77,18 @@ bool MMap::Map(FileFd &Fd)
    
    if (iSize == 0)
       return _error->Error(_("Can't mmap an empty file"));
-   
+
+   // We can't mmap compressed fd's directly, so we need to read it completely
+   if (Fd.IsCompressed() == true)
+   {
+      if ((Flags & ReadOnly) != ReadOnly)
+	 return _error->Error("Compressed file %s can only be mapped readonly", Fd.Name().c_str());
+      Base = new unsigned char[iSize];
+      if (Fd.Seek(0L) == false || Fd.Read(Base, iSize) == false)
+	 return false;
+      return true;
+   }
+
    // Map it.
    Base = mmap(0,iSize,Prot,Map,Fd.Fd(),0);
    if (Base == (void *)-1)

debian/changelog

@@ -31,10 +31,12 @@ apt (0.8.16~exp9) UNRELEASED; urgency=low
     - dump the APT::Compressor settings correctly and completely
   * apt-pkg/contrib/fileutl.{h,cc}:
     - implement a ModificationTime method for FileFd
+    - add a ReadLine method
+    - drop the explicit export of gz-compression handling
   * apt-pkg/cdrom.cc:
     - support InRelease files on cdrom
 
- -- David Kalnischkies <kalnischkies@gmail.com>  Sun, 11 Dec 2011 01:30:12 +0100
+ -- David Kalnischkies <kalnischkies@gmail.com>  Sun, 11 Dec 2011 19:34:58 +0100
 
 apt (0.8.16~exp8) experimental; urgency=low
 

methods/rred.cc

@@ -37,13 +37,13 @@ class RredMethod : public pkgAcqMethod {
 	// return values
 	enum State {ED_OK, ED_ORDERING, ED_PARSER, ED_FAILURE, MMAP_FAILED};
 
-	State applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file,
+	State applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file,
 	             unsigned long &line, char *buffer, Hashes *hash) const;
 	void ignoreLineInFile(FILE *fin, char *buffer) const;
-	void ignoreLineInFile(gzFile &fin, char *buffer) const;
+	void ignoreLineInFile(FileFd &fin, char *buffer) const;
 	void copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lines,
 	                            Hashes *hash, char *buffer) const;
-	void copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines,
+	void copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines,
 	                            Hashes *hash, char *buffer) const;
 
 	State patchFile(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const;
@@ -72,10 +72,10 @@ public:
  *  \param hash the created file for correctness
  *  \return the success State of the ed command executor
  */
-RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file,
+RredMethod::State RredMethod::applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file,
 			unsigned long &line, char *buffer, Hashes *hash) const {
 	// get the current command and parse it
-	if (gzgets(ed_cmds, buffer, BUF_SIZE) == NULL) {
+	if (ed_cmds.ReadLine(buffer, BUF_SIZE) == NULL) {
 		if (Debug == true)
 			std::clog << "rred: encounter end of file - we can start patching now." << std::endl;
 		line = 0;
@@ -130,7 +130,7 @@ RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *ou
 	unsigned char mode = *idx;
 
 	// save the current position
-	unsigned const long pos = gztell(ed_cmds);
+	unsigned const long long pos = ed_cmds.Tell();
 
 	// if this is add or change then go to the next full stop
 	unsigned int data_length = 0;
@@ -164,7 +164,7 @@ RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *ou
 
 	// include data from ed script
 	if (mode == MODE_CHANGED || mode == MODE_ADDED) {
-		gzseek(ed_cmds, pos, SEEK_SET);
+		ed_cmds.Seek(pos);
 		copyLinesFromFileToFile(ed_cmds, out_file, data_length, hash, buffer);
 	}
 
@@ -190,11 +190,11 @@ void RredMethod::copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lin
 	}
 }
 										/*}}}*/
-void RredMethod::copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines,/*{{{*/
+void RredMethod::copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines,/*{{{*/
 					Hashes *hash, char *buffer) const {
 	while (0 < lines--) {
 		do {
-			gzgets(fin, buffer, BUF_SIZE);
+			fin.ReadLine(buffer, BUF_SIZE);
 			size_t const written = fwrite(buffer, 1, strlen(buffer), fout);
 			hash->Add((unsigned char*)buffer, written);
 		} while (strlen(buffer) == (BUF_SIZE - 1) &&
@@ -211,11 +211,11 @@ void RredMethod::ignoreLineInFile(FILE *fin, char *buffer) const {		/*{{{*/
 	}
 }
 										/*}}}*/
-void RredMethod::ignoreLineInFile(gzFile &fin, char *buffer) const {		/*{{{*/
-	gzgets(fin, buffer, BUF_SIZE);
+void RredMethod::ignoreLineInFile(FileFd &fin, char *buffer) const {		/*{{{*/
+	fin.ReadLine(buffer, BUF_SIZE);
 	while (strlen(buffer) == (BUF_SIZE - 1) &&
 	       buffer[BUF_SIZE - 2] != '\n') {
-		gzgets(fin, buffer, BUF_SIZE);
+		fin.ReadLine(buffer, BUF_SIZE);
 		buffer[0] = ' ';
 	}
 }
@@ -224,12 +224,11 @@ RredMethod::State RredMethod::patchFile(FileFd &Patch, FileFd &From,		/*{{{*/
 					FileFd &out_file, Hashes *hash) const {
    char buffer[BUF_SIZE];
    FILE* fFrom = fdopen(From.Fd(), "r");
-   gzFile fPatch = Patch.gzFd();
    FILE* fTo = fdopen(out_file.Fd(), "w");
 
    /* we do a tail recursion to read the commands in the right order */
    unsigned long line = -1; // assign highest possible value
-   State const result = applyFile(fPatch, fFrom, fTo, line, buffer, hash);
+   State const result = applyFile(Patch, fFrom, fTo, line, buffer, hash);
    
    /* read the rest from infile */
    if (result == ED_OK) {
@@ -258,20 +257,7 @@ struct EdCommand {
 RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From,		/*{{{*/
 					FileFd &out_file, Hashes *hash) const {
 #ifdef _POSIX_MAPPED_FILES
-	MMap ed_cmds(MMap::ReadOnly);
-	if (Patch.gzFd() != NULL) {
-		unsigned long long mapSize = Patch.Size();
-		DynamicMMap* dyn = new DynamicMMap(0, mapSize, 0);
-		if (dyn->validData() == false) {
-			delete dyn;
-			return MMAP_FAILED;
-		}
-		dyn->AddSize(mapSize);
-		gzread(Patch.gzFd(), dyn->Data(), mapSize);
-		ed_cmds = *dyn;
-	} else
-		ed_cmds = MMap(Patch, MMap::ReadOnly);
-
+	MMap ed_cmds(Patch, MMap::ReadOnly);
 	MMap in_file(From, MMap::ReadOnly);
 
 	if (ed_cmds.Size() == 0 || in_file.Size() == 0)