delay progress until Release files are downloaded

Progress reporting used an "upper bound" on files we might get, expect
that this wasn't correct in case pdiff entered the picture. So instead
of calculating a value which is perhaps incorrect, we just accept that
we can't tell how many files we are going to download and just keep at
0% until we know. Additionally, if we have pdiffs we wait until we got
these (sub)index files, too.

That could all be done better by downloading all Release files first and
planing with them in hand accordingly, but one step at a time.

apt-pkg/acquire-item.cc

@@ -1188,12 +1188,12 @@ bool pkgAcqMetaBase::CheckAuthDone(string const &Message)		/*{{{*/
                 << DestFile << std::endl;
 
    // Download further indexes with verification
-   QueueIndexes(true);
+   TransactionManager->QueueIndexes(true);
 
    return true;
 }
 									/*}}}*/
-void pkgAcqMetaBase::QueueIndexes(bool const verify)			/*{{{*/
+void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 {
    // at this point the real Items are loaded in the fetcher
    ExpectedAdditionalItems = 0;
@@ -1455,7 +1455,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,	/*{{{*/
    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
 {
    // index targets + (worst case:) Release/Release.gpg
-   ExpectedAdditionalItems = IndexTargets.size() + 2;
+   ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max();
    TransactionManager->Add(this);
 }
 									/*}}}*/
@@ -1529,9 +1529,6 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c
 {
    Item::Failed(Message, Cnf);
 
-   // we failed, we will not get additional items from this method
-   ExpectedAdditionalItems = 0;
-
    if (AuthPass == false)
    {
       if (Status == StatAuthError || Status == StatTransientNetworkError)
@@ -1579,7 +1576,7 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c
 	 if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
 	    /* expired Release files are still a problem you need extra force for */;
 	 else
-	    QueueIndexes(true);
+	    TransactionManager->QueueIndexes(true);
       }
    }
 }
@@ -1604,9 +1601,6 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner,		/*{{{*/
    Desc.Owner = this;
    Desc.ShortDesc = DataTarget.ShortDesc;
    Desc.URI = DataTarget.URI;
-
-   // we expect more item
-   ExpectedAdditionalItems = IndexTargets.size();
    QueueURI(Desc);
 }
 									/*}}}*/
@@ -1641,7 +1635,7 @@ void pkgAcqMetaIndex::Failed(string const &Message,
       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
 
       // queue without any kind of hashsum support
-      QueueIndexes(false);
+      TransactionManager->QueueIndexes(false);
    }
 }
 									/*}}}*/
@@ -1787,7 +1781,7 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const
       if (MetaIndex->VerifyVendor(Message) == false)
 	 /* expired Release files are still a problem you need extra force for */;
       else
-	 MetaIndex->QueueIndexes(GoodLoad);
+	 TransactionManager->QueueIndexes(GoodLoad);
 
       TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
    }
@@ -1840,6 +1834,9 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
                                  IndexTarget const &Target)
    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
 {
+   // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
+   ExpectedAdditionalItems = 40;
+
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
    Desc.Owner = this;
@@ -1884,6 +1881,7 @@ void pkgAcqDiffIndex::QueueOnIMSHit() const				/*{{{*/
 									/*}}}*/
 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 {
+   ExpectedAdditionalItems = 0;
    // failing here is fine: our caller will take care of trying to
    // get the complete file if patching fails
    if(Debug)
@@ -2267,6 +2265,7 @@ void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig cons
 {
    pkgAcqBaseIndex::Failed(Message,Cnf);
    Status = StatDone;
+   ExpectedAdditionalItems = 0;
 
    if(Debug)
       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl

apt-pkg/acquire-item.h

@@ -416,16 +416,6 @@ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem		/*{{{*/
     */
    bool AuthPass;
 
-   /** \brief Starts downloading the individual index files.
-    *
-    *  \param verify If \b true, only indices whose expected hashsum
-    *  can be determined from the meta-index will be downloaded, and
-    *  the hashsums of indices will be checked (reporting
-    *  #StatAuthError if there is a mismatch).  If verify is \b false,
-    *  no hashsum checking will be performed.
-    */
-   void QueueIndexes(bool const verify);
-
    /** \brief Called when a file is finished being retrieved.
     *
     *  If the file was not downloaded to DestFile, a copy process is
@@ -592,6 +582,16 @@ class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex
 		     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
    virtual void Finished() APT_OVERRIDE;
 
+   /** \brief Starts downloading the individual index files.
+    *
+    *  \param verify If \b true, only indices whose expected hashsum
+    *  can be determined from the meta-index will be downloaded, and
+    *  the hashsums of indices will be checked (reporting
+    *  #StatAuthError if there is a mismatch).  If verify is \b false,
+    *  no hashsum checking will be performed.
+    */
+   void QueueIndexes(bool const verify);
+
    /** \brief Create a new pkgAcqMetaClearSig. */
    pkgAcqMetaClearSig(pkgAcquire * const Owner,
 		IndexTarget const &ClearsignedTarget,

apt-pkg/acquire.cc

@@ -1147,7 +1147,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
    // Compute the total number of bytes to fetch
    unsigned int Unknown = 0;
    unsigned int Count = 0;
-   bool UnfetchedReleaseFiles = false;
+   bool ExpectAdditionalItems = false;
    for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); 
         I != Owner->ItemsEnd();
 	++I, ++Count)
@@ -1156,12 +1156,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       if ((*I)->Status == pkgAcquire::Item::StatDone)
 	 ++CurrentItems;
 
-      // see if the method tells us to expect more
-      TotalItems += (*I)->ExpectedAdditionalItems;
-
-      // check if there are unfetched Release files
-      if ((*I)->Status != pkgAcquire::Item::StatDone && (*I)->ExpectedAdditionalItems > 0)
-         UnfetchedReleaseFiles = true;
+      // do we expect to acquire more files than we know of yet?
+      if ((*I)->ExpectedAdditionalItems > 0)
+         ExpectAdditionalItems = true;
 
       TotalBytes += (*I)->FileSize;
       if ((*I)->Complete == true)
@@ -1218,7 +1215,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
 
    double const OldPercent = Percent;
    // calculate the percentage, if we have too little data assume 1%
-   if (TotalBytes > 0 && UnfetchedReleaseFiles)
+   if (ExpectAdditionalItems)
       Percent = 0;
    else
       // use both files and bytes because bytes can be unreliable