condense parallel requests with the same hashes to one

It shouldn't be too common, but sometimes people have multiple mirrors
in the sources or otherwise repositories with the same content. Now that
we gracefully can handle multiple requests to the same URI, we can also
fold multiple requests with the same expected hashes into one. Note that
this isn't trying to find oppertunities for merging, but just merges if
it happens to encounter the oppertunity for it.

This is most obvious in the new testcase actually as it needs to delay
the action to give the acquire system enough time to figure out that
they can be merged.

apt-pkg/acquire.cc

@@ -680,9 +680,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
 {
    QItem **I = &Items;
    // move to the end of the queue and check for duplicates here
+   HashStringList const hsl = Item.Owner->GetExpectedHashes();
    for (; *I != 0; I = &(*I)->Next)
-      if (Item.URI == (*I)->URI) 
+      if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes())
       {
+	 if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
+	    std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
 	 (*I)->Owners.push_back(Item.Owner);
 	 Item.Owner->Status = (*I)->Owner->Status;
 	 return false;

apt-private/acqprogress.cc

@@ -116,14 +116,10 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
-   // Ignore certain kinds of transient failures (bad code)
-   if (Itm.Owner->Status == pkgAcquire::Item::StatIdle)
-      return;
-
    AssignItemID(Itm);
    clearLastLine();
 
-   if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
+   if (Itm.Owner->Status == pkgAcquire::Item::StatDone || Itm.Owner->Status == pkgAcquire::Item::StatIdle)
    {
       // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
       // which failed to download, but the error is ignored (compare "Err:")

test/integration/framework

@@ -245,7 +245,7 @@ setupenvironment() {
 	echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf
 	echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
 	echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf
-	echo "Dir::Bin::Methods \"${METHODSDIR}\";" >> aptconfig.conf
+	echo "Dir::Bin::Methods \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods\";" >> aptconfig.conf
 	# store apt-key were we can access it, even if we run it as a different user
 	# destroys coverage reporting though, so just do it for root for now
 	if [ "$(id -u)" = '0' ]; then
@@ -786,6 +786,8 @@ insertsource() {
 	local SPATH="aptarchive/dists/${RELEASE}/main/source"
 	mkdir -p $SPATH
 	local FILE="${SPATH}/Sources"
+	local DSCFILE="${NAME}_${VERSION}.dsc"
+	local TARFILE="${NAME}_${VERSION}.tar.gz"
 	echo "Package: $NAME
 Binary: $NAME
 Version: $VERSION
@@ -793,8 +795,8 @@ Maintainer: Joe Sixpack <joe@example.org>
 Architecture: $ARCH" >> $FILE
 	test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
 	echo "Files:
- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.tar.gz
+ $(echo -n "$DSCFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$DSCFILE" | wc -c) $DSCFILE
+ $(echo -n "$TARFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$TARFILE" | wc -c) $TARFILE
 " >> $FILE
 }
 

test/integration/skip-aptwebserver

@@ -1,25 +0,0 @@
-#!/bin/sh
-set -e
-
-TESTDIR=$(readlink -f $(dirname $0))
-. $TESTDIR/framework
-
-setupenvironment
-configarchitecture 'amd64'
-
-buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
-
-setupaptarchive
-changetowebserver
-
-rm -rf rootdir/var/lib/apt/lists
-aptget update -qq
-testequal 'Hit http://localhost stable InRelease
-Hit http://localhost stable/main Sources
-Hit http://localhost stable/main amd64 Packages
-Hit http://localhost stable/main Translation-en
-Reading package lists...' aptget update
-
-mv rootdir/var/lib/apt/lists/localhost* rootdir/var/lib/apt/lists/partial
-aptget update
-

test/integration/test-acquire-same-file-multiple-times

@@ -45,7 +45,7 @@ changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo
 
 httpdown() {
 	msgtest 'Downloading the same URI to different files' 'twice over http'
-	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 	cp rootdir/tmp/testsuccess.output download.log
 	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
 	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
@@ -57,7 +57,7 @@ testrun 'httpdown'
 
 httpredirectdown() {
 	msgtest 'Redirect leads' 'first URI to the second URI'
-	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 	cp rootdir/tmp/testsuccess.output download.log
 	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
 	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2

test/integration/test-acquire-same-repository-multiple-times

@@ -0,0 +1,81 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+TESTFILE="$TESTDIR/framework"
+cp $TESTFILE aptarchive/foo
+APTARCHIVE="$(readlink -f ./aptarchive)"
+
+getcodenamefromsuite() { echo "jessie"; }
+buildsimplenativepackage 'foo' 'all' '1.0' 'stable'
+setupaptarchive --no-update
+ln -s "${APTARCHIVE}/dists/stable" "${APTARCHIVE}/dists/jessie"
+for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do
+	sed 's#stable#jessie#g' $FILE > $(echo "$FILE" | sed 's#stable#jessie#g')
+done
+
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+	ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS
+done
+rm $NEWMETHODS/file
+cat >$NEWMETHODS/file <<EOF
+#!/bin/sh
+while read line; do
+	echo "\$line"
+	if [ -z "\$line" ]; then
+		sleep 0.2
+	fi
+done | $OLDMETHODS/file
+EOF
+chmod +x $NEWMETHODS/file
+
+tworepos() {
+	msgtest "Downloading the same repository twice over $1" "$3"
+	testsuccess --nomsg aptget update -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	#cat download.log
+	aptget files --format '$(FILENAME)' --no-release-info | sort > file.lst
+	testequal "$(find $(readlink -f ./rootdir/var/lib/apt/lists) -name '*_dists_*' \( ! -name '*InRelease' \) -type f | sort)" cat file.lst
+	testsuccess aptcache policy
+	testequal "foo:
+  Installed: (none)
+  Candidate: 1.0
+  Version table:
+     1.0 0
+        500 $1:$2 jessie/main amd64 Packages
+        500 $1:$2 stable/main amd64 Packages" aptcache policy foo
+	testfailure aptcache show foo/unstable
+	testsuccess aptcache show foo/stable
+	testsuccess aptcache show foo/jessie
+}
+
+tworepos 'file' "$APTARCHIVE" 'no partial'
+testequal '12' grep -c '200%20URI%20Start' ./download.log
+testequal '12' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'file' "$APTARCHIVE" 'hit'
+testequal '6' grep -c '200%20URI%20Start' ./download.log
+testequal '6' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists
+
+changetowebserver
+
+tworepos 'http' '//localhost:8080' 'no partial'
+testequal '10' grep -c '200%20URI%20Start' ./download.log
+testequal '10' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'http' '//localhost:8080' 'hit'
+testequal '2' grep -c '200%20URI%20Start' ./download.log
+testequal '4' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists

test/integration/test-apt-get-source

@@ -34,43 +34,40 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 # normal operation gets highest version number
 HEADER="Reading package lists...
 Building dependency tree..."
+DOWNLOAD1="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"
+DOWNLOAD2="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f"
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo
+$DOWNLOAD2" aptget source -q --print-uris foo
 
 # select by release: suite
 testsuccessequal "$HEADER
 Selected version '1.0' (stable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/stable
+$DOWNLOAD1" aptget source -q --print-uris foo/stable
 testsuccessequal "$HEADER
 Selected version '2.0' (unstable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/unstable
+$DOWNLOAD2" aptget source -q --print-uris foo/unstable
 
 # select by release: codename
 testsuccessequal "$HEADER
 Selected version '2.0' (sid) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/sid
+$DOWNLOAD2" aptget source -q --print-uris foo/sid
 
 # select by version
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo=1.0
+$DOWNLOAD1" aptget source -q --print-uris foo=1.0
 
 # select by release with no binary package (Bug#731102) but ensure to get
 # highest version
+DOWNLOAD01="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61"
 testsuccessequal "$HEADER
 Selected version '0.1' (wheezy) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/wheezy
+$DOWNLOAD01" aptget source -q --print-uris foo/wheezy
 
 # unavailable one
 testfailureequal "$HEADER
@@ -78,11 +75,12 @@ E: Can not find version '9.9-not-there' of package 'foo'
 E: Unable to find a source package for foo" aptget source -q --print-uris foo=9.9-not-there
 
 # version and release
+DOWNLOAD001="Need to get 0 B/29 B of source archives.
+'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 13 MD5Sum:6c819ebf0a21b1a480e1dbf6b8edfebd
+'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 16 MD5Sum:a3c7e1ac2159fc0faf522e110d6906fd"
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris -t unstable foo=0.0.1
+$DOWNLOAD001" aptget source -q --print-uris -t unstable foo=0.0.1
 
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
+Need to get 0 B/25 B of source archives.
 Fetch source foo" aptget source -q -s foo

test/integration/test-apt-get-source-arch

@@ -28,31 +28,30 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 
 HEADER="Reading package lists...
 Building dependency tree..."
+DOWNLOAD10="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"
 
 # pick :amd64
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:amd64
+$DOWNLOAD10" aptget source -q --print-uris foo:amd64
 
 # pick :i386
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f" aptget source -q --print-uris foo:i386
 
 # pick :i386 by release
 testsuccessequal "$HEADER
 Selected version '0.1' (oldstable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386/oldstable
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61" aptget source -q --print-uris foo:i386/oldstable
 
 # pick :i386 by version
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386=1.0
+$DOWNLOAD10" aptget source -q --print-uris foo:i386=1.0
 
 # error on unknown arch
 testfailureequal "$HEADER

test/integration/test-apt-get-source-multisources

@@ -20,11 +20,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 HEADER="Reading package lists...
 Building dependency tree..."
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11
 
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e"  aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b" aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113

test/integration/test-apt-get-update-unauth-warning

@@ -20,8 +20,10 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning
-testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease
+testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
+Get:2 file:$APTARCHIVE unstable Release
 Err:2 file:$APTARCHIVE unstable Release
   File not found
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
@@ -36,10 +38,41 @@ filesize() {
 	stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz"
 }
 # allow override
-testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease
+#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
+#exit
+testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
+Get:2 file:$APTARCHIVE unstable Release
 Ign:2 file:$APTARCHIVE unstable Release
   File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
 Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
 Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
 Get:5 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]

test/integration/test-apt-progress-fd

@@ -16,7 +16,6 @@ setupaptarchive
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -33,7 +32,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -50,7 +48,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -77,7 +74,6 @@ pmstatus:dpkg-exec:75:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)

test/integration/test-apt-update-ims

@@ -134,6 +134,24 @@ EXPECT="Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
 Ign:2 http://localhost:8080 unstable Release
   404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
 Hit:3 http://localhost:8080 unstable/main Sources
 Hit:4 http://localhost:8080 unstable/main amd64 Packages
 Hit:5 http://localhost:8080 unstable/main Translation-en

test/integration/test-bug-722207-print-uris-even-if-very-quiet

@@ -21,11 +21,11 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373" aptget source apt -qq --print-uris
 testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
 
-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt apt2 -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373
+'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 10 MD5Sum:4c572ce45f1e2bedbb30da7f5e1c241c
+'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 13 MD5Sum:2a96fec139f8722d93312a1ff8281232" aptget source apt apt2 -qq --print-uris

test/integration/test-bug-738785-switch-protocol

@@ -39,18 +39,15 @@ cd - >/dev/null
 testsuccess aptget install apt -y
 testdpkginstalled 'apt'
 
-# create a copy of all methods, expect https
-eval `aptconfig shell METHODS Dir::Bin::Methods/d`
-COPYMETHODS='usr/lib/apt/methods'
-mv rootdir/${COPYMETHODS} rootdir/${COPYMETHODS}.bak
-mkdir -p rootdir/$COPYMETHODS
-cd rootdir/$COPYMETHODS
-find $METHODS \! -type d | while read meth; do
-	ln -s $meth
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+	ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS
 done
-rm https
-cd - >/dev/null
-echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf
+rm $NEWMETHODS/https
 
 cd downloaded
 testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
@@ -59,8 +56,7 @@ testfailure test -e apt_1.0_all.deb
 cd - >/dev/null
 
 # revert to all methods
-rm -rf rootdir/$COPYMETHODS
-mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS}
+ln -s $OLDMETHODS/https $NEWMETHODS
 
 # check that downgrades from https to http are not allowed
 webserverconfig 'aptwebserver::support::http' 'true'