display errortext for all Err as well as Ign logs

consistently using Item::Failed in all specializec classes helps setting
up some information bits otherwise unset, so some errors had an empty
reason as an error. Ign is upgraded to display the error message we
ignored to further help in understanding what happens.

apt-pkg/acquire-item.cc

@@ -125,7 +125,6 @@ pkgAcquire::Item::~Item()
    fetch this object */
 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
-   Status = StatIdle;
    if(ErrorText == "")
       ErrorText = LookupTag(Message,"Message");
    UsedMirror =  LookupTag(Message,"UsedMirror");
@@ -134,7 +133,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       /* This indicates that the file is not available right now but might
          be sometime later. If we do a retry cycle then this should be
 	 retried [CDROMs] */
-      if (Cnf->LocalOnly == true &&
+      if (Cnf != NULL && Cnf->LocalOnly == true &&
 	  StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
       {
 	 Status = StatIdle;
@@ -143,8 +142,11 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       }
 
       Status = StatError;
+      Complete = false;
       Dequeue();
-   }   
+   }
+   else
+      Status = StatIdle;
 
    // report mirror failure back to LP if we actually use a mirror
    string FailReason = LookupTag(Message, "FailReason");
@@ -208,11 +210,8 @@ bool pkgAcquire::Item::Rename(string From,string To)
 
 void pkgAcquire::Item::QueueURI(ItemDesc &Item)
 {
-   if (access(DestFile.c_str(), R_OK) == 0)
-   {
+   if (RealFileExists(DestFile))
       changeOwnerAndPermissionOfFile("preparePartialFile", DestFile.c_str(), "_apt", "root", 0600);
-      std::cerr << "QUEUE ITEM: " << DestFile << std::endl;
-   }
    Owner->Enqueue(Item);
 }
 void pkgAcquire::Item::Dequeue()
@@ -559,7 +558,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
    return false;
 }
 									/*}}}*/
-void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
+void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
 {
    if(Debug)
       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
@@ -567,9 +566,8 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/
 
    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
 
-   Complete = false;
+   Item::Failed(Message,Cnf);
    Status = StatDone;
-   Dequeue();
 }
 									/*}}}*/
 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,	/*{{{*/
@@ -596,7 +594,7 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList
    }
 
    if(!ParseDiffIndex(DestFile))
-      return Failed("", NULL);
+      return Failed("Message: Couldn't parse pdiff index", Cnf);
 
    // queue for final move
    string FinalFile;
@@ -713,7 +711,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff()					/*{{{*/
 
    if(!FileExists(FinalFile))
    {
-      Failed("No FinalFile " + FinalFile + " available", NULL);
+      Failed("Message: No FinalFile " + FinalFile + " available", NULL);
       return false;
    }
 
@@ -747,7 +745,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff()					/*{{{*/
    // error checking and falling back if no patch was found
    if(available_patches.empty() == true)
    {
-      Failed("No patches available", NULL);
+      Failed("Message: No patches available", NULL);
       return false;
    }
 
@@ -852,13 +850,13 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
    QueueURI(Desc);
 }
 									/*}}}*/
-void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
+void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
 {
    if(Debug)
       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
-   Complete = false;
+
+   Item::Failed(Message,Cnf);
    Status = StatDone;
-   Dequeue();
 
    // check if we are the first to fail, otherwise we are done here
    State = StateDoneDiff;
@@ -1284,7 +1282,7 @@ void pkgAcqIndex::StageDownloadDone(string Message,
       Stage = STAGE_DECOMPRESS_AND_VERIFY;
       Desc.URI = "copy:" + FileName;
       QueueURI(Desc);
-
+      SetActiveSubprocess("copy");
       return;
     }
 
@@ -1305,7 +1303,6 @@ void pkgAcqIndex::StageDownloadDone(string Message,
    DestFile += ".decomp";
    Desc.URI = decompProg + ":" + FileName;
    QueueURI(Desc);
-
    SetActiveSubprocess(decompProg);
 }
 									/*}}}*/
@@ -1383,18 +1380,15 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       return;
    }
 
+   Item::Failed(Message,Cnf);
+
    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
    if (Cnf->LocalOnly == true ||
        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
    {
       // Ignore this
       Status = StatDone;
-      Complete = false;
-      Dequeue();
-      return;
    }
-
-   Item::Failed(Message,Cnf);
 }
 									/*}}}*/
 // AcqMetaBase::Add - Add a item to the current Transaction		/*{{{*/
@@ -1639,7 +1633,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
       } else {
          _error->Error("%s", downgrade_msg.c_str());
          Rename(MetaIndexFile, MetaIndexFile+".FAILED");
-         Status = pkgAcquire::Item::StatError;
+	 Item::Failed("Message: " + downgrade_msg, Cnf);
          TransactionManager->AbortTransaction();
          return;
       }
@@ -1663,17 +1657,15 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
       _error->Warning("Use --allow-insecure-repositories to force the update");
    }
 
+   Item::Failed(Message,Cnf);
+
    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
-   if (Cnf->LocalOnly == true || 
+   if (Cnf->LocalOnly == true ||
        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
-   {      
+   {
       // Ignore this
       Status = StatDone;
-      Complete = false;
-      Dequeue();
-      return;
    }
-   Item::Failed(Message,Cnf);
 }
 									/*}}}*/
 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,			/*{{{*/
@@ -2003,9 +1995,12 @@ bool pkgAcqMetaBase::VerifyVendor(string Message, const string &RealURI)/*{{{*/
 }
 									/*}}}*/
 // pkgAcqMetaIndex::Failed - no Release file present			/*{{{*/
-void pkgAcqMetaIndex::Failed(string /*Message*/,
-                             pkgAcquire::MethodConfig * /*Cnf*/)
+void pkgAcqMetaIndex::Failed(string Message,
+                             pkgAcquire::MethodConfig * Cnf)
 {
+   pkgAcquire::Item::Failed(Message, Cnf);
+   Status = StatDone;
+
    string FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 
    _error->Warning(_("The repository '%s' does not have a Release file. "
@@ -2115,6 +2110,8 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long /*Size*/,
 									/*}}}*/
 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
 {
+   Item::Failed(Message, Cnf);
+
    // we failed, we will not get additional items from this method
    ExpectedAdditionalItems = 0;
 
@@ -2126,14 +2123,12 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*
       string FinalFile = _config->FindDir("Dir::State::lists");
       FinalFile.append(URItoFileName(RealURI));
       TransactionManager->TransactionStageRemoval(this, FinalFile);
+      Status = StatDone;
 
       new pkgAcqMetaIndex(Owner, TransactionManager,
 			MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 			MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
 			IndexTargets, MetaIndexParser);
-      if (Cnf->LocalOnly == true ||
-	  StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
-	 Dequeue();
    }
    else
    {
@@ -2149,9 +2144,11 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*
       // only allow going further if the users explicitely wants it
       if(_config->FindB("Acquire::AllowInsecureRepositories") == true)
       {
+	 Status = StatDone;
+
          /* Always move the meta index, even if gpgv failed. This ensures
           * that PackageFile objects are correctly filled in */
-         if (FileExists(DestFile)) 
+         if (FileExists(DestFile))
          {
             string FinalFile = _config->FindDir("Dir::State::lists");
             FinalFile += URItoFileName(RealURI);
@@ -2161,19 +2158,17 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*
                                       "Release");
             FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
                                           "Release");
-            
-            
+
             // Done, queue for rename on transaction finished
             TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
          }
          QueueIndexes(false);
       } else {
-         // warn if the repository is unsinged
+         // warn if the repository is unsigned
          _error->Warning("Use --allow-insecure-repositories to force the update");
          TransactionManager->AbortTransaction();
          Status = StatError;
-         return;
-      } 
+      }
    }
 }
 									/*}}}*/

apt-private/acqprogress.cc

@@ -117,6 +117,8 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
    {
       cout << _("Ign ") << Itm.Description << endl;
+      if (Itm.Owner->ErrorText.empty() == false)
+	 cout << "  " << Itm.Owner->ErrorText << endl;
    }
    else
    {

test/integration/framework

@@ -715,7 +715,7 @@ buildaptarchivefromincoming() {
 	aptftparchive -qq generate ftparchive.conf
 	cd - > /dev/null
 	msgdone "info"
-	generatereleasefiles
+	generatereleasefiles "$@"
 }
 
 buildaptarchivefromfiles() {
@@ -830,14 +830,19 @@ setupflataptarchive() {
 }
 
 setupaptarchive() {
-	buildaptarchive
+	local NOUPDATE=0
+	if [ "$1" = '--no-update' ]; then
+		NOUPDATE=1
+		shift
+	fi
+	buildaptarchive "$@"
 	if [ -e aptarchive/dists ]; then
 		setupdistsaptarchive
 	else
 		setupflataptarchive
 	fi
-	signreleasefiles
-	if [ "$1" != '--no-update' ]; then
+	signreleasefiles 'Joe Sixpack' "$@"
+	if [ "1" != "$NOUPDATE" ]; then
 		testsuccess aptget update -o Debug::pkgAcquire::Worker=true -o Debug::Acquire::gpgv=true
 	fi
 }

test/integration/test-apt-get-update-unauth-warning

@@ -20,11 +20,12 @@ rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning
 testequal "Ign file: unstable InRelease
+  File not found
 Err file: unstable Release
-  
+  File not found
 W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
 W: Use --allow-insecure-repositories to force the update
-W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release  
+W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release  File not found
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update
 
@@ -34,7 +35,9 @@ testequal "partial" ls rootdir/var/lib/apt/lists
 
 # allow override
 testequal "Ign file: unstable InRelease
+  File not found
 Ign file: unstable Release
+  File not found
 Reading package lists...
 W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories
 # ensure we can not install the package

test/integration/test-apt-update-ims

@@ -44,8 +44,9 @@ echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
 msgmsg "Release/Release.gpg"
-# with Release/Release.gpg 
+# with Release/Release.gpg
 EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
 Hit http://localhost:8080 unstable Release
 Hit http://localhost:8080 unstable Release.gpg
 Hit http://localhost:8080 unstable/main Sources
@@ -65,8 +66,10 @@ runtest
 # no Release.gpg or InRelease
 msgmsg "Release only"
 EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
 Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
+  404  Not Found
 Hit http://localhost:8080 unstable/main Sources
 Hit http://localhost:8080 unstable/main amd64 Packages
 Hit http://localhost:8080 unstable/main Translation-en

test/integration/test-apt-update-nofallback

@@ -161,7 +161,7 @@ test_inrelease_to_invalid_inrelease()
 
     testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) <joe@example.org>
 
-W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  
+W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) <joe@example.org>
 
 W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
 

test/integration/test-apt-update-rollback

@@ -19,44 +19,44 @@ create_fresh_archive()
 
     insertpackage 'unstable' 'old' 'all' '1.0'
 
-    setupaptarchive
+    setupaptarchive --no-update
 }
 
 add_new_package() {
     insertpackage "unstable" "new" "all" "1.0"
     insertsource "unstable" "new" "all" "1.0"
 
-    setupaptarchive --no-update
-
-    avoid_ims_hit
+    setupaptarchive --no-update "$@"
 }
 
 break_repository_sources_index() {
-    printf "xxx" > $APTARCHIVE/dists/unstable/main/source/Sources
-    gzip -c $APTARCHIVE/dists/unstable/main/source/Sources > \
-            $APTARCHIVE/dists/unstable/main/source/Sources.gz
-    avoid_ims_hit
+    printf 'xxx' > $APTARCHIVE/dists/unstable/main/source/Sources
+    compressfile "$APTARCHIVE/dists/unstable/main/source/Sources" "$@"
 }
 
-test_inrelease_to_new_inrelease() {
-    msgmsg "Test InRelease to new InRelease works fine"
+start_with_good_inrelease() {
     create_fresh_archive
+    testsuccess aptget update
     testequal "old/unstable 1.0 all" apt list -q
+}
 
-    add_new_package
+test_inrelease_to_new_inrelease() {
+    msgmsg 'Test InRelease to new InRelease works fine'
+    start_with_good_inrelease
 
+    add_new_package '+1hour'
     testsuccess aptget update -o Debug::Acquire::Transaction=1
-
     testequal "new/unstable 1.0 all
 old/unstable 1.0 all" apt list -q
 }
 
 test_inrelease_to_broken_hash_reverts_all() {
-    msgmsg "Test InRelease to broken InRelease reverts everything"
-    create_fresh_archive
-    add_new_package
+    msgmsg 'Test InRelease to broken InRelease reverts everything'
+    start_with_good_inrelease
+
+    add_new_package '+1hour'
     # break the Sources file
-    break_repository_sources_index
+    break_repository_sources_index '+1hour'
 
     # test the error condition
     testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources  Hash Sum mismatch
@@ -66,14 +66,14 @@ E: Some index files failed to download. They have been ignored, or old ones used
     testequal "E: Unable to locate package new" aptget install new -s -qq
 }
 
-test_inreleae_to_valid_release() {
-    msgmsg "Test InRelease to valid Release"
-    create_fresh_archive
-    add_new_package
-    # switch to a unsinged repo now
+test_inrelease_to_valid_release() {
+    msgmsg 'Test InRelease to valid Release'
+    start_with_good_inrelease
+
+    add_new_package '+1hour'
+    # switch to a unsigned repo now
     rm $APTARCHIVE/dists/unstable/InRelease
     rm $APTARCHIVE/dists/unstable/Release.gpg
-    avoid_ims_hit
 
     # update fails
     testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
@@ -85,16 +85,17 @@ test_inreleae_to_valid_release() {
     testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release
 }
 
-test_inreleae_to_release_reverts_all() {
-    msgmsg "Test InRelease to broken Release reverts everything"
-    create_fresh_archive
+test_inrelease_to_release_reverts_all() {
+    msgmsg 'Test InRelease to broken Release reverts everything'
+    start_with_good_inrelease
 
-    # switch to a unsinged repo now
-    add_new_package
+    # switch to a unsigned repo now
+    add_new_package '+1hour'
     rm $APTARCHIVE/dists/unstable/InRelease
     rm $APTARCHIVE/dists/unstable/Release.gpg
+
     # break it
-    break_repository_sources_index
+    break_repository_sources_index '+1hour'
 
     # ensure error
     testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
@@ -107,21 +108,19 @@ test_inreleae_to_release_reverts_all() {
 }
 
 test_unauthenticated_to_invalid_inrelease() {
-    msgmsg "Test UnAuthenticated to invalid InRelease reverts everything"
+    msgmsg 'Test UnAuthenticated to invalid InRelease reverts everything'
     create_fresh_archive
-    rm -rf rootdir/var/lib/apt/lists/*
     rm $APTARCHIVE/dists/unstable/InRelease
     rm $APTARCHIVE/dists/unstable/Release.gpg
-    avoid_ims_hit
-    
-    testsuccess aptget update -qq --allow-insecure-repositories
+
+    testsuccess aptget update --allow-insecure-repositories
     testequal "WARNING: The following packages cannot be authenticated!
   old
 E: There are problems and -y was used without --force-yes" aptget install -qq -y old
-    
+
     # go to authenticated but not correct
-    add_new_package
-    break_repository_sources_index
+    add_new_package '+1hour'
+    break_repository_sources_index '+1hour'
 
     testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources  Hash Sum mismatch
 
@@ -134,14 +133,14 @@ E: There are problems and -y was used without --force-yes" aptget install -qq -y
 }
 
 test_inrelease_to_unauth_inrelease() {
-    msgmsg "Test InRelease to InRelease without sig"
-    create_fresh_archive
-    signreleasefiles 'Marvin Paranoid'
-    avoid_ims_hit
-    
+    msgmsg 'Test InRelease to InRelease without good sig'
+    start_with_good_inrelease
+
+    signreleasefiles 'Marvin Paranoid' '+1hour'
+
     testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
 
-W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease  
+W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease  The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
 
 W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
 
@@ -150,13 +149,13 @@ W: Some index files failed to download. They have been ignored, or old ones used
 
 test_inrelease_to_broken_gzip() {
     msgmsg "Test InRelease to broken gzip"
-    create_fresh_archive
-    # append junk at the end of the gzip, this
+    start_with_good_inrelease
+
+    # append junk at the end of the compressed file
     echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz
-    # remove uncompressed file, otherwise apt will just fallback fetching
-    # that
+    touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz
+    # remove uncompressed file to avoid fallback
     rm $APTARCHIVE/dists/unstable/main/source/Sources
-    avoid_ims_hit
 
     testfailure aptget update
 }
@@ -174,7 +173,7 @@ ROOTDIR=${TMPWORKINGDIRECTORY}/rootdir
 APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )"
 
 # test the following cases:
-# - InRelease -> broken InRelease revert to previous state 
+# - InRelease -> broken InRelease revert to previous state
 # - empty lists dir and broken remote leaves nothing on the system
 # - InRelease -> hashsum mismatch for one file reverts all files to previous state
 # - Release/Release.gpg -> hashsum mismatch
@@ -184,13 +183,13 @@ APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )"
 # - unauthenticated -> invalid InRelease
 
 # stuff to do:
-# - ims-hit 
+# - ims-hit
 # - gzip-index tests
 
 test_inrelease_to_new_inrelease
 test_inrelease_to_broken_hash_reverts_all
-test_inreleae_to_valid_release
-test_inreleae_to_release_reverts_all
+test_inrelease_to_valid_release
+test_inrelease_to_release_reverts_all
 test_unauthenticated_to_invalid_inrelease
 test_inrelease_to_unauth_inrelease
 test_inrelease_to_broken_gzip

test/integration/test-bug-595691-empty-and-broken-archive-files

@@ -13,7 +13,7 @@ setupflataptarchive
 testaptgetupdate() {
 	rm -rf rootdir/var/lib/apt
 	aptget update 2>> testaptgetupdate.diff >> testaptgetupdate.diff || true
-	sed -i -e '/Ign / d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
+	sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
 	GIVEN="$1"
 	shift
 	msgtest "Test for correctness of" "apt-get update with $*"