support hashes for compressed pdiff files

At the moment we only have hashes for the uncompressed pdiff files, but
via the new '$HASH-Download' field in the .diff/Index hashes can be
provided for the .gz compressed pdiff file, which apt will pick up now
and use to verify the download. Now, we "just" need a buy in from the
creators of repositories…

apt-pkg/acquire-item.cc

@@ -152,14 +152,18 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const
 
 APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
 {
-   /* FIXME: We have only hashes for uncompressed pdiffs.
-      rred uncompresses them on the fly while parsing.
-      In StateFetchDiff state we also uncompress on the fly for hash check.
-      Hashes are checked while searching for (next) patch to apply. */
+   /* We don't always have the diff of the downloaded pdiff file.
+      What we have for sure is hashes for the uncompressed file,
+      but rred uncompresses them on the fly while parsing, so not handled here.
+      Hashes are (also) checked while searching for (next) patch to apply. */
+   if (State == StateFetchDiff)
+      return available_patches[0].download_hashes.empty() == false;
    return false;
 }
 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
 {
+   if (State == StateFetchDiff)
+      return available_patches[0].download_hashes;
    return HashStringList();
 }
 
@@ -168,11 +172,15 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
    /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
       we can check the rred result after all patches are applied as
       we know the expected result rather than potentially apply more patches */
+   if (State == StateFetchDiff)
+      return patch.download_hashes.empty() == false;
    return State == StateApplyDiff;
 }
 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
 {
-   if (State == StateApplyDiff)
+   if (State == StateFetchDiff)
+      return patch.download_hashes;
+   else if (State == StateApplyDiff)
       return GetExpectedHashesFor(Target->MetaKey);
    return HashStringList();
 }
@@ -1618,7 +1626,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 	 std::vector<DiffInfo>::iterator cur = available_patches.begin();
 	 for (; cur != available_patches.end(); ++cur)
 	 {
-	    if (cur->file != filename || unlikely(cur->result_size != size))
+	    if (cur->file != filename)
 	       continue;
 	    cur->result_hashes.push_back(HashString(*type, hash));
 	    break;
@@ -1630,8 +1638,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 	    DiffInfo next;
 	    next.file = filename;
 	    next.result_hashes.push_back(HashString(*type, hash));
-	    next.result_size = size;
-	    next.patch_size = 0;
+	    next.result_hashes.FileSize(size);
 	    available_patches.push_back(next);
 	 }
 	 else
@@ -1679,10 +1686,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 	 {
 	    if (cur->file != filename)
 	       continue;
-	    if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
-	       continue;
+	    if (cur->patch_hashes.empty())
+	       cur->patch_hashes.FileSize(size);
 	    cur->patch_hashes.push_back(HashString(*type, hash));
-	    cur->patch_size = size;
 	    break;
 	 }
 	 if (cur != available_patches.end())
@@ -1694,6 +1700,48 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       }
    }
 
+   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+   {
+      std::string tagname = *type;
+      tagname.append("-Download");
+      std::string const tmp = Tags.FindS(tagname.c_str());
+      if (tmp.empty() == true)
+	 continue;
+
+      string hash, filename;
+      unsigned long long size;
+      std::stringstream ss(tmp);
+
+      // FIXME: all of pdiff supports only .gz compressed patches
+      while (ss >> hash >> size >> filename)
+      {
+	 if (unlikely(hash.empty() == true || filename.empty() == true))
+	    continue;
+	 if (unlikely(APT::String::Endswith(filename, ".gz") == false))
+	    continue;
+	 filename.erase(filename.length() - 3);
+
+	 // see if we have a record for this file already
+	 std::vector<DiffInfo>::iterator cur = available_patches.begin();
+	 for (; cur != available_patches.end(); ++cur)
+	 {
+	    if (cur->file != filename)
+	       continue;
+	    if (cur->download_hashes.empty())
+	       cur->download_hashes.FileSize(size);
+	    cur->download_hashes.push_back(HashString(*type, hash));
+	    break;
+	 }
+	 if (cur != available_patches.end())
+	       continue;
+	 if (Debug == true)
+	    std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+	       << " wasn't in the list for the first parsed hash! (download)" << std::endl;
+	 break;
+      }
+   }
+
+
    bool foundStart = false;
    for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
 	 cur != available_patches.end(); ++cur)
@@ -1729,7 +1777,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
    unsigned long long patchesSize = 0;
    for (std::vector<DiffInfo>::const_iterator cur = available_patches.begin();
 	 cur != available_patches.end(); ++cur)
-      patchesSize += cur->patch_size;
+      patchesSize += cur->patch_hashes.FileSize();
    unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
    if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
    {

apt-pkg/acquire-item.h

@@ -705,17 +705,14 @@ struct APT_HIDDEN DiffInfo {						/*{{{*/
    /** The filename of the diff. */
    std::string file;
 
-   /** The hashes of the diff */
+   /** The hashes of the file after the diff is applied */
    HashStringList result_hashes;
 
-   /** The hashes of the file after the diff is applied */
+   /** The hashes of the diff */
    HashStringList patch_hashes;
 
-   /** The size of the file after the diff is applied */
-   unsigned long long result_size;
-
-   /** The size of the diff itself */
-   unsigned long long patch_size;
+   /** The hashes of the compressed diff */
+   HashStringList download_hashes;
 };
 									/*}}}*/
 /** \brief An item that is responsible for fetching client-merge patches {{{

apt-pkg/contrib/hashes.cc

@@ -188,6 +188,13 @@ unsigned long long HashStringList::FileSize() const			/*{{{*/
    return strtoull(hv.c_str(), NULL, 10);
 }
 									/*}}}*/
+bool HashStringList::FileSize(unsigned long long const Size)		/*{{{*/
+{
+   std::string size;
+   strprintf(size, "%llu", Size);
+   return push_back(HashString("Checksum-FileSize", size));
+}
+									/*}}}*/
 bool HashStringList::supported(char const * const type)			/*{{{*/
 {
    for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t)
@@ -361,9 +368,7 @@ APT_IGNORE_DEPRECATED_PUSH
    if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
       hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
 APT_IGNORE_DEPRECATED_POP
-   std::string SizeStr;
-   strprintf(SizeStr, "%llu", d->FileSize);
-   hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+   hashes.FileSize(d->FileSize);
    return hashes;
 }
 APT_IGNORE_DEPRECATED_PUSH

apt-pkg/contrib/hashes.h

@@ -96,6 +96,13 @@ class HashStringList
     */
    unsigned long long FileSize() const;
 
+   /** sets the filesize hash
+    *
+    * @param Size of the file
+    * @return @see #push_back
+    */
+   bool FileSize(unsigned long long const Size);
+
    /** check if the given hash type is supported
     *
     * @param type to check

apt-pkg/indexrecords.cc

@@ -121,9 +121,7 @@ bool indexRecords::Load(const string Filename)				/*{{{*/
             indexRecords::checkSum *Sum = new indexRecords::checkSum;
             Sum->MetaKeyFilename = Name;
             Sum->Size = Size;
-	    std::string SizeStr;
-	    strprintf(SizeStr, "%llu", Size);
-	    Sum->Hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+	    Sum->Hashes.FileSize(Size);
             APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);)
             Entries[Name] = Sum;
          }

methods/rred.cc

@@ -627,7 +627,7 @@ class RredMethod : public pkgAcqMethod {
 	    p.Close();
 	    HashStringList const hsl = patch_hash.GetHashStringList();
 	    if (hsl != I->ExpectedHashes)
-	       return _error->Error("Patch %s doesn't have the expected hashsum", patch_name.c_str());
+	       return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
 	 }
 
 	 if (Debug == true)

test/integration/test-pdiff-usage

@@ -42,6 +42,8 @@ wasmergeused() {
 
 testrun() {
 	msgmsg "Testcase: setup the base with: $*"
+	local DOWNLOADHASH=true
+	if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
 	find aptarchive -name 'Packages*' -type f -delete
 	cp ${PKGFILE} aptarchive/Packages
 	compressfile 'aptarchive/Packages'
@@ -76,6 +78,15 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+	fi
+
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	find aptarchive -name 'Packages*' -type f -delete
@@ -131,6 +142,17 @@ SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)
  $(sha256sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha1sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha256sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)" >> $PATCHINDEX
+	fi
+
 	generatereleasefiles '+2hour'
 	signreleasefiles
 	cp -a aptarchive/Packages Packages-future
@@ -150,6 +172,7 @@ SHA256-Patches:
 	mkdir -p aptarchive/Packages.diff
 	PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
 	diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true
+	cat $PATCHFILE | gzip > ${PATCHFILE}.gz
 	PATCHINDEX='aptarchive/Packages.diff/Index'
 	echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new)
 SHA1-History:
@@ -165,14 +188,22 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+	fi
 	# needs to look like a valid command, otherwise the parser will fail before hashes are checked
-	echo '1d' >> $PATCHFILE
+	echo '1d' > $PATCHFILE
 	cat $PATCHFILE | gzip > ${PATCHFILE}.gz
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	testsuccess aptget update "$@"
 	cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output
-	testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output
+	testsuccess grep 'Hash Sum mismatch' rootdir/tmp/aptgetupdate.output
 	testnopackage oldstuff
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
@@ -201,6 +232,14 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+	fi
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	#find aptarchive -name 'Packages*' -type f -delete
@@ -215,6 +254,9 @@ echo 'Debug::pkgAcquire::Diffs "true";
 Debug::Acquire::Transaction "true";
 Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
 
+testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
+testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
+
 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
 testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0