require $(HASH)-Download field in .diff/Index files

Now that we ignore SHA1-only files it makes sense to require also the
provision of hashes for the compressed patches as this was introduced in
the same patchset as support for non-SHA1 hashes in the file itself in
dak and adding support in other archive creators (if they support pdiffs
at all) will likely be in the same batch.

The reason for the change itself is simple: If you are 'scared' enough
about the security of SHA1, you shouldn't uncompress a file you haven't
verified at all – after all, it could be exploiting a bug or a zip bomb.

apt-pkg/acquire-item.cc

@@ -206,12 +206,12 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const
 
 APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
 {
-   /* We don't always have the diff of the downloaded pdiff file.
-      What we have for sure is hashes for the uncompressed file,
-      but rred uncompresses them on the fly while parsing, so not handled here.
-      Hashes are (also) checked while searching for (next) patch to apply. */
+   /* We can't check hashes of rred result as we don't know what the
+      hash of the file will be. We just know the hash of the patch(es),
+      the hash of the file they will apply on and the hash of the resulting
+      file. */
    if (State == StateFetchDiff)
-      return available_patches[0].download_hashes.empty() == false;
+      return true;
    return false;
 }
 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
@@ -227,7 +227,7 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
       we can check the rred result after all patches are applied as
       we know the expected result rather than potentially apply more patches */
    if (State == StateFetchDiff)
-      return patch.download_hashes.empty() == false;
+      return true;
    return State == StateApplyDiff;
 }
 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
@@ -2022,6 +2022,17 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       return false;
    }
 
+   for (auto const &patch: available_patches)
+      if (patch.result_hashes.usable() == false ||
+	    patch.patch_hashes.usable() == false ||
+	    patch.download_hashes.usable() == false)
+      {
+	 if (Debug)
+	    std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file
+	       << " so fallback to complete download" << std::endl;
+	 return false;
+      }
+
    // patching with too many files is rather slow compared to a fast download
    unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
    if (fileLimit != 0 && fileLimit < available_patches.size())
@@ -2036,7 +2047,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
    unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
    if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr)
    {
-      // compressed case
       unsigned long long downloadSize = std::accumulate(available_patches.begin(),
 	    available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
 	    return T + I.download_hashes.FileSize();
@@ -2065,23 +2075,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 	    return false;
 	 }
       }
-      // uncompressed case
-      downloadSize = std::accumulate(available_patches.begin(),
-	    available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
-	    return T + I.patch_hashes.FileSize();
-	    });
-      if (downloadSize != 0)
-      {
-	 unsigned long long const downloadSizeIdx = ServerSize;
-	 unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
-	 if ((sizeLimit/100) < downloadSize)
-	 {
-	    if (Debug)
-	       std::clog << "Need " << downloadSize << " uncompressed bytes (Limit is " << (sizeLimit/100) << ", "
-		  << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
-	    return false;
-	 }
-      }
    }
 
    // we have something, queue the diffs

test/integration/test-pdiff-usage

@@ -30,7 +30,7 @@ wasmergeused() {
 	if echo "$*" | grep -q -- '-o test::cannot-use-pdiff=1'; then
 		msgtest 'Check if pdiff was' 'not used'
 		cp -a rootdir/tmp/testsuccess.output rootdir/tmp/aptupdate.output
-		testsuccess --nomsg grep 'diff_Index: Did not find a good hashsum in the index' rootdir/tmp/aptupdate.output
+		testsuccess --nomsg grep "diff/Index with Message: Couldn't parse pdiff index" rootdir/tmp/aptupdate.output
 		return;
 	fi
 
@@ -51,8 +51,6 @@ wasmergeused() {
 testrun() {
 	configcompression '.' 'xz'
 	msgmsg "Testcase: setup the base with: $*"
-	local DOWNLOADHASH=true
-	if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
 	find aptarchive -name 'Packages*' -type f -delete
 	cp "${PKGFILE}" aptarchive/Packages
 	compressfile 'aptarchive/Packages'
@@ -84,12 +82,10 @@ SHA256-History:
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
- $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
- $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
 
 	generatereleasefiles '+1hour'
 	signreleasefiles
@@ -99,7 +95,7 @@ SHA256-Patches:
 	testsuccessequal "$(cat "${PKGFILE}-new")
 " aptcache show apt newstuff
 
-	msgmsg "Testcase: apply with one patch and SHA1 only: $*"
+	msgmsg "Testcase: SHA1-only patches are not used: $*"
 	find aptarchive -name 'Packages*' -type f -delete
 	cp "${PKGFILE}-new" aptarchive/Packages
 	compressfile 'aptarchive/Packages'
@@ -114,13 +110,35 @@ SHA1-History:
  $(sha1sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA1-Patches:
  7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28
- $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA1-Download:
+ $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA1-Download:
  2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
- $(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
+	generatereleasefiles '+1hour'
+	signreleasefiles
+	rm -rf rootdir/var/lib/apt/lists
+	cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists
+	wasmergeused "$@" -o test::cannot-use-pdiff=1
+	testnopackage oldstuff
+	testsuccessequal "$(cat "${PKGFILE}-new")
+" aptcache show apt newstuff
 
+	msgmsg "Testcase: no download-hashes patches are not used: $*"
+	find aptarchive -name 'Packages*' -type f -delete
+	cp "${PKGFILE}-new" aptarchive/Packages
+	compressfile 'aptarchive/Packages'
+	mkdir -p aptarchive/Packages.diff
+	PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
+	diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true
+	cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
+	PATCHINDEX='aptarchive/Packages.diff/Index'
+	echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
+SHA256-History:
+ 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
+ $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
+SHA256-Patches:
+ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	rm -rf rootdir/var/lib/apt/lists
@@ -157,13 +175,11 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
- $(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
  $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")
- $(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" > "$PATCHINDEX"
 
 	generatereleasefiles '+2hour'
 	signreleasefiles
@@ -205,12 +221,10 @@ SHA256-History:
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
- $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > $PATCHINDEX
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
- $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> $PATCHINDEX
-	fi
+ $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
 	# needs to look like a valid command, otherwise the parser will fail before hashes are checked
 	echo '1d' > "$PATCHFILE"
 	cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
@@ -236,22 +250,16 @@ SHA256-Patches:
 	diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true
 	cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
 	PATCHINDEX='aptarchive/Packages.diff/Index'
-	BIGSIZE="$(stat -c%s "$PATCHFILE")"
-	if ! $DOWNLOADHASH; then
-		BIGSIZE="${BIGSIZE}000"
-	fi
 	echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
 SHA256-History:
  01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
- $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $BIGSIZE $(basename "$PATCHFILE")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
- $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	testsuccess apt update -o Debug::pkgAcquire::Diffs=1 "$@"
@@ -266,9 +274,6 @@ Debug::Acquire::Transaction "true";
 Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
 
 testcase() {
-	testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@"
-	testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@"
-
 	testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@"
 	testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@"
 	testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0 "$@"