* first rewrite of the apt pdiff support patch finished

Makefile

@@ -17,8 +17,8 @@ all headers library clean veryclean binary program doc dirs:
 	$(MAKE) -C cmdline $@
 	$(MAKE) -C ftparchive $@
 	$(MAKE) -C dselect $@
-#	$(MAKE) -C doc $@
-#	$(MAKE) -C po $@
+	$(MAKE) -C doc $@
+	$(MAKE) -C po $@
 
 # Some very common aliases
 .PHONY: maintainer-clean dist-clean distclean pristine sanity 

apt-pkg/acquire-item.cc

@@ -24,6 +24,8 @@
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/md5.h>
+#include <apt-pkg/sha1.h>
+#include <apt-pkg/tagfile.h>
 
 #include <apti18n.h>
     
@@ -31,6 +33,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <string>
+#include <sstream>
 #include <stdio.h>
 									/*}}}*/
 
@@ -131,14 +134,251 @@ void pkgAcquire::Item::Rename(string From,string To)
 }
 									/*}}}*/
 
+// AcqIndexDiffs::AcqIndexDiffs - Constructor			
+// ---------------------------------------------------------------------
+/* The package diff is added to the queue. one object is constructed
+ * for each diff and the index
+ */
+pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
+				   string URI,string URIDesc,string ShortDesc,
+				   string ExpectedMD5, vector<string> diffs) 
+   : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), needed_files(diffs)
+{
+   
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(URI);
+
+   Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+
+   Desc.Description = URIDesc;
+   Desc.Owner = this;
+   Desc.ShortDesc = ShortDesc;
+
+   CurrentPackagesFile = _config->FindDir("Dir::State::lists");
+   CurrentPackagesFile += URItoFileName(RealURI);
+
+   if(Debug) {
+      std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " 
+		<< CurrentPackagesFile << std::endl;
+   }
+
+   if(!FileExists(CurrentPackagesFile) || 
+      !_config->FindB("Acquire::Diffs",true)) {
+      // we don't have a pkg file or we don't want to queue
+      if(Debug)
+	 std::clog << "No index file or canceld by user" << std::endl;
+      Failed("", NULL);
+      return;
+   }
+
+   if(needed_files.size() == 0)
+      QueueDiffIndex(URI);
+   else
+      QueueNextDiff();
+}
+
+void pkgAcqIndexDiffs::QueueDiffIndex(string URI)
+{
+   Desc.URI = URI + ".diff/Index";
+   Desc.Description = Description + "IndexDiff";
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(URI) + string(".IndexDiff");
+
+   if(Debug)
+      std::clog << "QueueDiffIndex: " << Desc.URI << std::endl;
+
+   QueueURI(Desc);
+}
+
+void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "Failed(): " << Desc.URI << std::endl
+		<< "Falling back to big package file" << std::endl;
+   new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, 
+		   ExpectedMD5);
+   Finish();
+}
+
+
+// helper that cleans the item out of the fetcher queue
+void pkgAcqIndexDiffs::Finish(bool allDone)
+{
+   // we restore the original name, this is required, otherwise
+   // the file will be cleaned
+   if(allDone) {
+      // this is for the "real" finish
+      DestFile = _config->FindDir("Dir::State::lists");
+      DestFile += URItoFileName(RealURI);
+      Complete = true;
+      Dequeue();
+      if(Debug)
+	 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
+      return;
+
+   }
+
+   if(Debug)
+      std::clog << "Finishing: " << Desc.URI << std::endl;
+   Complete = false;
+   Status = StatDone;
+   Dequeue();
+   return;
+}
+
+
+// this needs to be rewriten to not depend on the external ed
+bool pkgAcqIndexDiffs::ApplyDiff(string PatchFile)
+{
+   char *error;
+   int res=0;
+
+   string FinalFile = _config->FindDir("Dir::State::lists");
+   FinalFile += URItoFileName(RealURI);
+
+   int Process = ExecFork();
+   if (Process == 0)
+   {
+      chdir(_config->FindDir("Dir::State::lists").c_str());
+      string cmd = "(zcat " + PatchFile + "; echo \"wq\" ) | ed  " + FinalFile + " >/dev/null 2>/dev/null";
+      if(Debug)
+	 std::clog << "Runing: " << cmd << std::endl;
+      res = system(cmd.c_str());
+      _exit(WEXITSTATUS(res));
+   }
+   if(!ExecWait(Process, error, true)) {
+      //_error->Error("Patch failed: %s ", error);
+      return false;
+   }
+
+   return true;
+}
+
+bool pkgAcqIndexDiffs::QueueNextDiff()
+{
+   // queue diff
+   Desc.URI = string(RealURI) + string(".diff/") + needed_files[0] + string(".gz");
+   Desc.Description = Description + string("-diff");
+
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(RealURI + string(".diff/") + needed_files[0]);
+
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
+   
+   QueueURI(Desc);
+
+   return true;
+}
+
+bool pkgAcqIndexDiffs::ParseIndexDiff(string IndexDiffFile)
+{
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile 
+		<< std::endl;
+   
+   FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
+   pkgTagFile TF(&Fd);
+   if (_error->PendingError() == true)
+      return false;
+   pkgTagSection Tags;
+   if(TF.Step(Tags) == true)
+   {
+      string local_sha1;
+      string tmp = Tags.FindS("SHA1-Current");
+      std::stringstream ss(tmp);
+      ss >> ServerSha1;
+
+      FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
+      SHA1Summation SHA1;
+      SHA1.AddFD(fd.Fd(), fd.Size());
+      local_sha1 = string(SHA1.Result());
+
+      if(local_sha1 == ServerSha1) {
+	 if(Debug)
+	    std::clog << "Package file is up-to-date" << std::endl;
+	 Finish(true);
+	 return true;
+      }
+      if(Debug)
+	 std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
+
+      // check the historie and see what patches we need
+      string history = Tags.FindS("SHA1-History");     
+      std::stringstream hist(history);
+      string sha1, size, file;
+      bool found = false;
+      while(hist >> sha1 >> size >> file) {
+	 if(sha1 == local_sha1) 
+	    found=true;
+	 if(found) {
+	    if(Debug)
+	       std::clog << "Need to get diff: " << file << std::endl;
+	    needed_files.push_back(file);
+	 }
+      }
+      // no information how to get the patches, bail out
+      if(!found) {
+	 if(Debug)
+	    std::clog << "Can't find a patch in the index file" << std::endl;
+	 // Failed will queue a big package file
+	 Failed("", NULL);
+      } else {
+	 // queue the diffs
+	 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
+			      ExpectedMD5, needed_files);
+	 Finish();
+	 return true;
+      }
+   }
+
+   return false;
+}
+
+
+void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,
+			    pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
+
+   Item::Done(Message,Size,Md5Hash,Cnf);
+
+   int len = Desc.URI.size();
+   // sucess in downloading the index
+   if(Desc.URI.substr(len-strlen("Index"),len-1) == "Index") {
+      if(!ParseIndexDiff(DestFile))
+	 return Failed("", NULL);
+      else
+	 return Finish();
+   }
+
+   // sucess in downloading a diff
+   if(Desc.URI.find(".diff") != string::npos) {
+      ApplyDiff(DestFile);
+      needed_files.erase(needed_files.begin());
+
+      if(needed_files.size() > 0) {
+	 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
+			      ExpectedMD5, needed_files);
+      } else {
+	 Finish(true);
+	 return;
+      }
+   }
+
+   Finish();
+}
+
+
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* The package file is added to the queue and a second class is 
    instantiated to fetch the revision file */   
 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 			 string URI,string URIDesc,string ShortDesc,
-			 string ExpectedMD5, string comprExt) :
-   Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5)
+			 string ExpectedMD5, string comprExt)
+   : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5)
 {
    Decompression = false;
    Erase = false;
@@ -593,8 +833,8 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)
       }
       
       // Queue Packages file
-      new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
-                      (*Target)->ShortDesc, ExpectedIndexMD5);
+      new pkgAcqIndexDiffs(Owner, (*Target)->URI, (*Target)->Description,
+			   (*Target)->ShortDesc, ExpectedIndexMD5);
    }
 }
 

apt-pkg/acquire-item.h

@@ -82,6 +82,41 @@ class pkgAcquire::Item
    virtual ~Item();
 };
 
+// item for index diffs
+class pkgAcqIndexDiffs : public pkgAcquire::Item
+{
+   protected:
+   bool Debug;
+   pkgAcquire::ItemDesc Desc;
+   string RealURI;
+   string ExpectedMD5;
+   // this is the SHA-1 sum we expect after the patching
+   string ServerSha1;
+   string CurrentPackagesFile;
+   string Description;
+   vector<string> needed_files;
+   
+   public:
+   
+   // Specialized action members
+   virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
+   virtual void Done(string Message,unsigned long Size,string Md5Hash,
+		     pkgAcquire::MethodConfig *Cnf);
+   virtual string DescURI() {return RealURI + "Index";};
+
+
+   // various helpers
+   bool ParseIndexDiff(string IndexDiffFile);
+   void QueueDiffIndex(string URI);
+   bool QueueNextDiff();
+   bool ApplyDiff(string PatchFile);
+   void Finish(bool allDone=false);
+
+   pkgAcqIndexDiffs(pkgAcquire *Owner,string URI,string URIDesc,
+		    string ShortDesct, string ExpectedMD5,
+		    vector<string> diffs=vector<string>());
+};
+
 // Item class for index files
 class pkgAcqIndex : public pkgAcquire::Item
 {
@@ -92,7 +127,7 @@ class pkgAcqIndex : public pkgAcquire::Item
    pkgAcquire::ItemDesc Desc;
    string RealURI;
    string ExpectedMD5;
-   
+
    public:
    
    // Specialized action members

po/apt-all.pot

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2005-03-06 12:50+0100\n"
+"POT-Creation-Date: 2005-04-15 11:37+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -303,7 +303,7 @@ msgid ""
 "  --no-delink Enable delinking debug mode\n"
 "  --contents  Control contents file generation\n"
 "  -c=?  Read this configuration file\n"
-"  -o=?  Set an arbitary configuration option"
+"  -o=?  Set an arbitrary configuration option"
 msgstr ""
 
 #: ftparchive/apt-ftparchive.cc:757
@@ -1271,7 +1271,7 @@ msgid "File %s/%s overwrites the one in the package %s"
 msgstr ""
 
 #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:709
-#: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/acquire.cc:416 apt-pkg/clean.cc:38
+#: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/acquire.cc:420 apt-pkg/clean.cc:38
 #, c-format
 msgid "Unable to read %s"
 msgstr ""
@@ -1561,7 +1561,7 @@ msgstr ""
 msgid "Unable to accept connection"
 msgstr ""
 
-#: methods/ftp.cc:864 methods/http.cc:916 methods/rsh.cc:303
+#: methods/ftp.cc:864 methods/http.cc:953 methods/rsh.cc:303
 msgid "Problem hashing file"
 msgstr ""
 
@@ -1655,76 +1655,76 @@ msgstr ""
 msgid "Read error from %s process"
 msgstr ""
 
-#: methods/http.cc:344
+#: methods/http.cc:381
 msgid "Waiting for headers"
 msgstr ""
 
-#: methods/http.cc:490
+#: methods/http.cc:527
 #, c-format
 msgid "Got a single header line over %u chars"
 msgstr ""
 
-#: methods/http.cc:498
+#: methods/http.cc:535
 msgid "Bad header line"
 msgstr ""
 
-#: methods/http.cc:517 methods/http.cc:524
+#: methods/http.cc:554 methods/http.cc:561
 msgid "The HTTP server sent an invalid reply header"
 msgstr ""
 
-#: methods/http.cc:553
+#: methods/http.cc:590
 msgid "The HTTP server sent an invalid Content-Length header"
 msgstr ""
 
-#: methods/http.cc:568
+#: methods/http.cc:605
 msgid "The HTTP server sent an invalid Content-Range header"
 msgstr ""
 
-#: methods/http.cc:570
+#: methods/http.cc:607
 msgid "This HTTP server has broken range support"
 msgstr ""
 
-#: methods/http.cc:594
+#: methods/http.cc:631
 msgid "Unknown date format"
 msgstr ""
 
-#: methods/http.cc:737
+#: methods/http.cc:774
 msgid "Select failed"
 msgstr ""
 
-#: methods/http.cc:742
+#: methods/http.cc:779
 msgid "Connection timed out"
 msgstr ""
 
-#: methods/http.cc:765
+#: methods/http.cc:802
 msgid "Error writing to output file"
 msgstr ""
 
-#: methods/http.cc:793
+#: methods/http.cc:830
 msgid "Error writing to file"
 msgstr ""
 
-#: methods/http.cc:818
+#: methods/http.cc:855
 msgid "Error writing to the file"
 msgstr ""
 
-#: methods/http.cc:832
+#: methods/http.cc:869
 msgid "Error reading from server. Remote end closed connection"
 msgstr ""
 
-#: methods/http.cc:834
+#: methods/http.cc:871
 msgid "Error reading from server"
 msgstr ""
 
-#: methods/http.cc:1065
+#: methods/http.cc:1102
 msgid "Bad header data"
 msgstr ""
 
-#: methods/http.cc:1082
+#: methods/http.cc:1119
 msgid "Connection failed"
 msgstr ""
 
-#: methods/http.cc:1173
+#: methods/http.cc:1210
 msgid "Internal error"
 msgstr ""
 
@@ -1862,7 +1862,7 @@ msgstr ""
 msgid "Unable to stat the mount point %s"
 msgstr ""
 
-#: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:422 apt-pkg/clean.cc:44
+#: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:426 apt-pkg/clean.cc:44
 #, c-format
 msgid "Unable to change to %s"
 msgstr ""
@@ -2246,36 +2246,36 @@ msgstr ""
 msgid "IO Error saving source cache"
 msgstr ""
 
-#: apt-pkg/acquire-item.cc:126
+#: apt-pkg/acquire-item.cc:129
 #, c-format
 msgid "rename failed, %s (%s -> %s)."
 msgstr ""
 
-#: apt-pkg/acquire-item.cc:235 apt-pkg/acquire-item.cc:894
+#: apt-pkg/acquire-item.cc:475 apt-pkg/acquire-item.cc:1134
 msgid "MD5Sum mismatch"
 msgstr ""
 
-#: apt-pkg/acquire-item.cc:708
+#: apt-pkg/acquire-item.cc:948
 #, c-format
 msgid ""
 "I wasn't able to locate a file for the %s package. This might mean you need "
 "to manually fix this package. (due to missing arch)"
 msgstr ""
 
-#: apt-pkg/acquire-item.cc:761
+#: apt-pkg/acquire-item.cc:1001
 #, c-format
 msgid ""
 "I wasn't able to locate file for the %s package. This might mean you need to "
 "manually fix this package."
 msgstr ""
 
-#: apt-pkg/acquire-item.cc:797
+#: apt-pkg/acquire-item.cc:1037
 #, c-format
 msgid ""
 "The package index files are corrupted. No Filename: field for package %s."
 msgstr ""
 
-#: apt-pkg/acquire-item.cc:884
+#: apt-pkg/acquire-item.cc:1124
 msgid "Size mismatch"
 msgstr ""