silently skip acquire of empty index files

There is just no point in taking the time to acquire empty files –
especially as it will be tiny non-empty compressed files usually.

apt-pkg/acquire-item.cc

@@ -1151,11 +1151,17 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify)			/*{{{*/
 	 else
 	 {
 	    auto const hashes = GetExpectedHashesFor(Target->MetaKey);
-	    if (hashes.usable() == false && hashes.empty() == false)
+	    if (hashes.empty() == false)
 	    {
-	       _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
+	       if (hashes.usable() == false)
+	       {
+		  _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
 			Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str());
-	       continue;
+		  continue;
+	       }
+	       // empty files are skipped as acquiring the very small compressed files is a waste of time
+	       else if (hashes.FileSize() == 0)
+		  continue;
 	    }
 	 }
 

test/integration/framework

@@ -566,6 +566,11 @@ forcecompressor() {
 	local CONFFILE="${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor"
 	echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; };
 Dir::Bin::uncompressed \"/does/not/exist\";" > "$CONFFILE"
+	for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
+		if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = "$COMPRESSOR" ]; then continue; fi
+		echo "Dir::Bin::${COMP} \"/does/not/exist\";" >> "$CONFFILE"
+		echo "APT::Compressor::${COMP}::Name \"${COMP}-disabled\";" >> "$CONFFILE"
+	done
 }
 
 setupsimplenativepackage() {
@@ -1843,6 +1848,7 @@ listcurrentlistsdirectory() {
 	} | sort
 }
 forallsupportedcompressors() {
+	rm -f "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor"
 	for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
 		if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi
 		"$@" "$COMP"

test/integration/test-apt-acquire-additional-files

@@ -15,6 +15,7 @@ configcompression '.' 'gz'
 LOWCOSTEXT='lz4'
 
 buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
+buildsimplenativepackage 'bar' 'all' '1' 'unstable'
 
 setupaptarchive --no-update
 changetowebserver
@@ -110,7 +111,7 @@ testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_mai
 testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*'
 testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz'
-testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
+testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-all.gz'
 
 rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT
 rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT

test/integration/test-apt-get-build-dep-file

@@ -10,6 +10,7 @@ configarchitecture 'i386'
 insertpackage 'stable' 'debhelper' 'i386' '7'
 insertpackage 'stable' 'build-essential' 'i386' '1'
 insertpackage 'stable' 'build-depends' 'i386' '1'
+insertsource 'stable' 'unrelated' 'any' '1'
 insertinstalledpackage 'build-conflict' 'i386' '1'
 
 setupaptarchive

test/integration/test-apt-update-empty-files

@@ -17,9 +17,13 @@ msgmsg 'Test with file'
 rm -rf rootdir/var/lib/apt/lists
 testsuccess apt update -o Debug::pkgAcquire::Worker=1
 cp rootdir/tmp/testsuccess.output apt.output
-testsuccess grep '%0aAlt-Filename:%20/.*/Sources%0a' apt.output
+testfailure grep '%0a\(Alt\)\?Filename:%20/.*/Sources\(\.gz\)\?%0a' apt.output
+testempty find rootdir/var/lib/apt/lists -name '*_Sources'
 
 msgmsg 'Test with http'
 changetowebserver
 rm -rf rootdir/var/lib/apt/lists
 testsuccess apt update -o Debug::pkgAcquire::Worker=1
+cp rootdir/tmp/testsuccess.output apt.output
+testfailure grep 'http:600.*Sources' apt.output
+testempty find rootdir/var/lib/apt/lists -name '*_Sources'

test/integration/test-bug-595691-empty-and-broken-archive-files

@@ -10,35 +10,28 @@ buildaptarchive
 touch aptarchive/Packages
 setupflataptarchive
 
-testaptgetupdate() {
+failureupdate() {
 	rm -rf rootdir/var/lib/apt
-	aptget update >testaptgetupdate.diff 2>&1 || true
-	sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
-	GIVEN="$1"
-	shift
-	msgtest "Test for correctness of" "apt-get update with $*"
-	if [ -z "$GIVEN" ]; then
-		echo -n "" | checkdiff - testaptgetupdate.diff && msgpass || msgfail
-	else
-		echo "$GIVEN" | checkdiff - testaptgetupdate.diff && msgpass || msgfail
-	fi
-	rm testaptgetupdate.diff
+	testfailure aptget update --allow-insecure-repositories
+	testsuccess grep "^E: Failed to fetch store:${1}  Empty files can't be valid archives$" rootdir/tmp/testfailure.output
+}
+successupdate() {
+	rm -rf rootdir/var/lib/apt
+	testwarning aptget update --allow-insecure-repositories
 }
 
 createemptyarchive() {
 	find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
 	touch aptarchive/Packages
 	echo -n "" | $COMPRESSOR_CMD > aptarchive/${1}.$COMPRESS
-	generatereleasefiles
-	signreleasefiles
+	find aptarchive -name '*Release*' -delete
 	rm -f aptarchive/Packages
 }
 
 createemptyfile() {
 	find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
 	touch aptarchive/Packages aptarchive/${1}.$COMPRESS
-	generatereleasefiles
-	signreleasefiles
+	find aptarchive -name '*Release*' -delete
 	rm -f aptarchive/Packages
 }
 
@@ -46,34 +39,21 @@ testoverfile() {
 	local APTARCHIVE="$(readlink -f ./aptarchive)"
 	forcecompressor "$1"
 
+	msgmsg 'archive over file' "Packages.$COMPRESS"
 	createemptyarchive 'Packages'
-	testaptgetupdate "Get:2 file:$APTARCHIVE  Packages []
-Reading package lists..." "empty archive Packages.$COMPRESS over file"
-
+	successupdate
 	createemptyfile 'Packages'
-	testaptgetupdate "Get:2 file:$APTARCHIVE  Packages
-Err:2 file:$APTARCHIVE  Packages
-  Empty files can't be valid archives
-Reading package lists...
-E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS})  Empty files can't be valid archives
-E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file"
+	failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS})"
 }
 
 testoverhttp() {
 	forcecompressor "$1"
 
+	msgmsg 'archive over http' "Packages.$COMPRESS"
 	createemptyarchive 'Packages'
-	testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT}  Packages []
-Reading package lists..." "empty archive Packages.$COMPRESS over http"
-
+	successupdate
 	createemptyfile 'Packages'
-	#FIXME: we should response with a good error message instead
-	testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT}  Packages
-Err:2 http://localhost:${APTHTTPPORT}  Packages
-  Empty files can't be valid archives
-Reading package lists...
-E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS})  Empty files can't be valid archives
-E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http"
+	failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS})"
 }
 
 forallsupportedcompressors testoverfile

test/integration/test-compressed-indexes

@@ -33,11 +33,9 @@ testrun() {
 	msgtest 'Check if all index files are' "${1:-uncompressed}"
 	if [ "$1" = 'compressed' ]; then
 		! test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1
-		! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
 		! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
 		! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
 		test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1
-		test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1
 		test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1
 		test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1
 		# there is no point in trying pdiff if we have compressed indexes
@@ -47,11 +45,9 @@ testrun() {
 		# clear the faked pdiff indexes so the glob below works
 		rm -f rootdir/var/lib/apt/lists/*diff_Index
 		test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1
-		test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
 		test -e rootdir/var/lib/apt/lists/*_Sources || F=1
 		test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
 		! test -e rootdir/var/lib/apt/lists/*i386_Packages.* || F=1
-		! test -e rootdir/var/lib/apt/lists/*all_Packages.* || F=1
 		! test -e rootdir/var/lib/apt/lists/*_Sources.* || F=1
 		! test -e rootdir/var/lib/apt/lists/*_Translation-en.* || F=1
 	fi
@@ -112,6 +108,7 @@ testovermethod() {
 			INDCOMP='uncompressed'
 		else
 			INDCOMP='compressed'
+			echo 'APT::Compressor::lz4::Name "lz4";' >> rootdir/etc/apt/apt.conf.d/02compressindex
 		fi
 
 		msgmsg "${1}: ${COMPRESSOR}: Test with $INDCOMP indexes gzip=$INDEX"

test/integration/test-handle-redirect-as-used-mirror-change

@@ -16,9 +16,8 @@ rewritesourceslist "http://localhost:${APTHTTPPORT}/redirectme"
 
 testsuccessequal "Get:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease [$(stat -c %s aptarchive/dists/unstable/InRelease) B]
 Get:2 http://0.0.0.0:${APTHTTPPORT} unstable/main Sources [$(stat -c %s aptarchive/dists/unstable/main/source/Sources.gz) B]
-Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main amd64 Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
-Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B]
-Get:5 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
+Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B]
+Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
 Reading package lists..." aptget update
 
 testsuccessequal "Hit:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease