Get accurate progress reporting in apt update again

For the non-pdiff case, we have can have accurate progress
reporting because after fetching the {,In}Release files we know
how many IndexFiles will be fetched and what size they have.

Therefore init the filesize early (in pkgAcqIndex::Init) and
ensure that in Acquire::Pulse() looks at already downloaded
bits when calculating the progress in Acquire::Pulse.

Also improve debug output of Debug::acquire::progress

apt-pkg/acquire-item.cc

@@ -2610,6 +2610,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
    DestFile = GetPartialFileNameFromURI(URI);
    NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false);
 
+   // store file size of the download to ensure the fetcher gives
+   // accurate progress reporting
+   FileSize = GetExpectedHashes().FileSize();
+
    if (CurrentCompressionExtension == "uncompressed")
    {
       Desc.URI = URI;

apt-pkg/acquire.cc

@@ -1129,7 +1129,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
    CurrentBytes = 0;
    TotalItems = 0;
    CurrentItems = 0;
-   
+
    // Compute the total number of bytes to fetch
    unsigned int Unknown = 0;
    unsigned int Count = 0;
@@ -1141,16 +1141,12 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       TotalItems++;
       if ((*I)->Status == pkgAcquire::Item::StatDone)
 	 ++CurrentItems;
-      
-      // Totally ignore local items
-      if ((*I)->Local == true)
-	 continue;
 
       // see if the method tells us to expect more
       TotalItems += (*I)->ExpectedAdditionalItems;
 
       // check if there are unfetched Release files
-      if ((*I)->Complete == false && (*I)->ExpectedAdditionalItems > 0)
+      if ((*I)->Status != pkgAcquire::Item::StatDone && (*I)->ExpectedAdditionalItems > 0)
          UnfetchedReleaseFiles = true;
 
       TotalBytes += (*I)->FileSize;
@@ -1159,7 +1155,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       if ((*I)->FileSize == 0 && (*I)->Complete == false)
 	 ++Unknown;
    }
-   
+
    // Compute the current completion
    unsigned long long ResumeSize = 0;
    for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
@@ -1187,12 +1183,6 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
    if (CurrentBytes > TotalBytes)
       CurrentBytes = TotalBytes;
 
-   // debug
-   if (_config->FindB("Debug::acquire::progress", false) == true)
-      std::clog << " Bytes: " 
-                << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) 
-                << std::endl;
-   
    // Compute the CPS
    struct timeval NewTime;
    gettimeofday(&NewTime,0);
@@ -1220,6 +1210,21 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       // use both files and bytes because bytes can be unreliable
       Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) +
                  0.2 * (CurrentItems/float(TotalItems)*100.0));
+
+   // debug
+   if (_config->FindB("Debug::acquire::progress", false) == true)
+   {
+      std::clog
+         << "["
+         << std::setw(5) << std::setprecision(4) << std::showpoint << Percent 
+         << "]"
+         << " Bytes: "
+         << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes)
+         << " # Files: "
+         << CurrentItems << " / " << TotalItems
+         << std::endl;
+   }
+
    double const DiffPercent = Percent - OldPercent;
    if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
       return true;

test/integration/test-apt-progress-fd

@@ -16,7 +16,7 @@ setupaptarchive
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:20:Retrieving file 1 of 1
+dlstatus:1:100:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
 pmstatus:testing:16.6667:Preparing testing (amd64)
@@ -32,7 +32,7 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:20:Retrieving file 1 of 1
+dlstatus:1:100:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
 pmstatus:testing:16.6667:Preparing testing (amd64)
@@ -48,7 +48,7 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:20:Retrieving file 1 of 1
+dlstatus:1:100:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
 pmstatus:testing:16.6667:Preparing testing (amd64)
@@ -74,7 +74,7 @@ pmstatus:dpkg-exec:75:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:20:Retrieving file 1 of 1
+dlstatus:1:100:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
 pmstatus:testing2:16.6667:Preparing testing2 (i386)

test/integration/test-apt-update-not-modified

@@ -34,6 +34,7 @@ Reading package lists..." aptget update
 	# readd arch so its downloaded again…
 	configarchitecture 'amd64' 'i386'
 	# … but oh noes, hashsum mismatch!
+        SIZE=$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz')
 	find aptarchive/dists/unstable/main/binary-amd64/ -type f -delete
 	cat >> aptarchive/dists/unstable/main/binary-amd64/Packages <<EOF
 
@@ -43,7 +44,7 @@ Version: 1
 EOF
 	compressfile aptarchive/dists/unstable/main/binary-amd64/Packages
 	testfailureequal "Hit:1 $1 unstable InRelease
-Get:2 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Get:2 $1 unstable/main amd64 Packages [$SIZE B]
 Err:2 $1 unstable/main amd64 Packages
   Hash Sum mismatch
 Reading package lists...
@@ -92,6 +93,7 @@ Reading package lists..." aptget update
 	# readd arch so its downloaded again…
 	configarchitecture 'amd64' 'i386'
 	# … but oh noes, hashsum mismatch!
+        SIZE=$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz')
 	find aptarchive/dists/unstable/main/binary-amd64/ -type f -delete
 	cat >> aptarchive/dists/unstable/main/binary-amd64/Packages <<EOF
 
@@ -103,7 +105,7 @@ EOF
 	testfailureequal "Ign:1 $1 unstable InRelease
   404  Not Found
 Hit:2 $1 unstable Release
-Get:4 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Get:4 $1 unstable/main amd64 Packages [$SIZE B]
 Err:4 $1 unstable/main amd64 Packages
   Hash Sum mismatch
 Reading package lists...

test/integration/test-apt-update-reporting

@@ -0,0 +1,31 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+setupenvironment
+configarchitecture 'amd64'
+
+for i in $(seq 100); do
+  insertpackage 'unstable' "foo-$i" 'all' '1.0'
+  insertsource 'unstable' "foo-$i" 'all' '1.0'
+done
+
+setupaptarchive --no-update
+changetowebserver
+
+aptget update -o Debug::Acquire::Progress=1 2>progress.log >ignore.out
+
+prev_percent=0.0
+while read line; do
+    percent="$(echo "$line"|cut -b2-6)"
+    # need to cut the decimal point and digits because sh can not do
+    # float compare
+    if [ "${percent%%.*}" -lt "${prev_percent%%.*}" ]; then
+        echo "progress goes backwards"
+        cat progress.log
+        exit 1
+    fi
+    prev_percent="$percent"
+done < progress.log
+