implement Fallback-Of for IndexTargets

Sometimes index files are in different locations in a repository as it
is currently the case for Contents files which are per-component in
Debian, but aren't in Ubuntu. This has historic reasons and is perhaps
changed soon, but such cases of transitions can always happen in the
future again, so we should prepare:

Introduced is a new field declaring that the current item should only be
downloaded if the mentioned item wasn't allowing for transitions without
a flagday in clients and archives.

This isn't implemented 'simpler' with multiple MetaKeys as items (could)
change their descriptions and perhaps also other configuration bits with
their location.

apt-pkg/acquire-item.cc

@@ -1196,9 +1196,18 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
    // at this point the real Items are loaded in the fetcher
    ExpectedAdditionalItems = 0;
 
+   std::set<std::string> targetsSeen;
    bool const metaBaseSupportsByHash = TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
-   for (auto Target : TransactionManager->MetaIndexParser->GetIndexTargets())
+   for (auto &Target: TransactionManager->MetaIndexParser->GetIndexTargets())
    {
+      // if we have seen a target which is created-by a target this one here is declared a
+      // fallback to, we skip acquiring the fallback (but we make sure we clean up)
+      if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
+      {
+	 targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
+	 new CleanupItem(Owner, TransactionManager, Target);
+	 continue;
+      }
       // all is an implementation detail. Users shouldn't use this as arch
       // We need this support trickery here as e.g. Debian has binary-all files already,
       // but arch:all packages are still in the arch:any files, so we would waste precious
@@ -1267,6 +1276,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 	       else if (hashes.FileSize() == 0)
 	       {
 		  new CleanupItem(Owner, TransactionManager, Target);
+		  targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 		  continue;
 	       }
 	    }
@@ -1327,6 +1337,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 	    std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
 	    if (FileExists(idxfilename))
 	       new NoActionItem(Owner, Target, idxfilename);
+	    targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
 	    continue;
 	 }
 
@@ -1348,6 +1359,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
       }
 
       // Queue the Index file (Packages, Sources, Translation-$foo, …)
+      targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
       if (trypdiff)
          new pkgAcqDiffIndex(Owner, TransactionManager, Target);
       else

apt-pkg/deb/debmetaindex.cc

@@ -33,13 +33,13 @@ class APT_HIDDEN debReleaseIndexPrivate					/*{{{*/
    public:
    struct APT_HIDDEN debSectionEntry
    {
-      std::string sourcesEntry;
-      std::string Name;
-      std::vector<std::string> Targets;
-      std::vector<std::string> Architectures;
-      std::vector<std::string> Languages;
-      bool UsePDiffs;
-      std::string UseByHash;
+      std::string const sourcesEntry;
+      std::string const Name;
+      std::vector<std::string> const Targets;
+      std::vector<std::string> const Architectures;
+      std::vector<std::string> const Languages;
+      bool const UsePDiffs;
+      std::string const UseByHash;
    };
 
    std::vector<debSectionEntry> DebEntries;
@@ -172,6 +172,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 	 std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);
 	 std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
 	 std::string KeepCompressedAs = APT_T_CONFIG_STR("KeepCompressedAs", "");
+	 std::string const FallbackOf = APT_T_CONFIG_STR("Fallback-Of", "");
 #undef APT_T_CONFIG_BOOL
 #undef APT_T_CONFIG_STR
 	 if (tplMetaKey.empty())
@@ -292,6 +293,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 		  Options.insert(std::make_pair("REPO_URI", URI));
 		  Options.insert(std::make_pair("TARGET_OF", Type));
 		  Options.insert(std::make_pair("CREATED_BY", *T));
+		  Options.insert(std::make_pair("FALLBACK_OF", FallbackOf));
 		  Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));
 		  Options.insert(std::make_pair("BY_HASH", UseByHash));
 		  Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
@@ -616,7 +618,8 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/
    // special case for --print-uris
    if (GetAll)
       for (auto const &Target: GetIndexTargets())
-	 new pkgAcqIndex(Owner, TransactionManager, Target);
+	 if (Target.Option(IndexTarget::FALLBACK_OF).empty())
+	    new pkgAcqIndex(Owner, TransactionManager, Target);
 
    return true;
 }
@@ -949,6 +952,26 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	 else if (optValue == false && tarItr != mytargets.end())
 	    mytargets.erase(std::remove(mytargets.begin(), mytargets.end(), target), mytargets.end());
       }
+      // if we can't order it in a 1000 steps we give up… probably a cycle
+      for (auto i = 0; i < 1000; ++i)
+      {
+	 bool Changed = false;
+	 for (auto t = mytargets.begin(); t != mytargets.end(); ++t)
+	 {
+	    std::string const fallback = _config->Find(std::string("Acquire::IndexTargets::") + Name + "::" + *t + "::Fallback-Of");
+	    if (fallback.empty())
+	       continue;
+	    auto const faller = std::find(mytargets.begin(), mytargets.end(), fallback);
+	    if (faller == mytargets.end() || faller < t)
+	       continue;
+	    Changed = true;
+	    auto const tv = *t;
+	    mytargets.erase(t);
+	    mytargets.emplace_back(tv);
+	 }
+	 if (Changed == false)
+	    break;
+      }
 
       bool UsePDiffs = _config->FindB("Acquire::PDiffs", true);
       {

apt-pkg/indexfile.cc

@@ -139,6 +139,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(REPO_URI);
       APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
+      APT_CASE(FALLBACK_OF);
       APT_CASE(PDIFFS);
       APT_CASE(DEFAULTENABLED);
       APT_CASE(COMPRESSIONTYPES);

apt-pkg/indexfile.h

@@ -91,6 +91,7 @@ class IndexTarget							/*{{{*/
       SOURCESENTRY,
       BY_HASH,
       KEEPCOMPRESSEDAS,
+      FALLBACK_OF,
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;

doc/acquire-additional-files.txt

@@ -90,6 +90,11 @@ Additional optional properties:
   have to ensure your front-end can deal with all compressed fileformats
   supported by apt (libapt users can e.g. use FileFd, others can use
   the cat-file command of /usr/lib/apt/apt-helper).
+* Fallback-Of: Is by default not set. If it is set and specifies another
+  target name (see Created-By) which was found in the Release file the
+  download of this target will be skipped. This can be used to implement
+  fallback(chain)s to allow transitions like the rename of target files.
+  The behavior if cycles are formed with Fallback-Of is undefined!
 * flat{MetaKey,Description}: APT supports two types of repositories:
   dists-style repositories which are the default and by far the most
   common which are named after the fact that the files are in an
@@ -204,11 +209,11 @@ uppercase and wrapped in $(), as in the configuration file.
 To get all the filenames of all Translation-en files you can e.g. call:
 	apt-get indextargets --format '$(FILENAME)' "Created-By: Translations" "Language: en"
 
-The line-based filtering and the formating is rather crude and feature-
+The line-based filtering and the formatting is rather crude and feature-
 less by design: The default format is Debians standard format deb822 (in
 particular: Field names are case-insensitive and the order of fields in
 the stanza is undefined), so instead of apt reimplementing powerful
-filters and formating for this command, it is recommend to use piping
+filters and formatting for this command, it is recommend to use piping
 and dedicated tools like 'grep-dctrl' if you need more than the basics
 provided.
 

test/integration/test-apt-acquire-additional-files

@@ -158,6 +158,64 @@ rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Cont
 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 runthistest
 
+msgmsg "Contents with 3 MetaKeys, first match"
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
+Acquire::IndexTargets::deb::Contents3 {
+	MetaKey "main/Contents-all";
+	ShortDescription "Contents3";
+	Description "\$(RELEASE) all Contents3";
+	Fallback-Of "Contents2";
+};
+Acquire::IndexTargets::deb::Contents {
+	MetaKey "\$(COMPONENT)/Contents-amd64";
+	ShortDescription "Contents";
+	Description "\$(RELEASE)/\$(COMPONENT) amd64 Contents";
+};
+Acquire::IndexTargets::deb::Contents2 {
+	MetaKey "Contents-all";
+	ShortDescription "Contents2";
+	Description "\$(RELEASE) all Contents2";
+	Fallback-Of "Contents";
+};
+EOF
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-amd64.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Reading package lists..." aptget update
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64" find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readfile Contents-amd64)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents2'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents3'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64" 'aptarchive/dists/unstable/main/Contents-amd64'
+
+msgmsg "Contents with 3 MetaKeys, third match"
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+echo 'Acquire::IndexTargets::deb::Contents::MetaKey "$(COMPONENT)/Contents-i386";' >> rootdir/etc/apt/apt.conf.d/content-target.conf
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-i386.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-i386 0 " aptget update --print-uris
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Get:2 http://localhost:${APTHTTPPORT} unstable all Contents3 [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]
+Reading package lists..." aptget update
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readfile Contents-all)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents3'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents2'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" 'aptarchive/dists/unstable/main/Contents-all'
+
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all
 rm -f rootdir/etc/apt/apt.conf.d/content-target.conf
 msgmsg "No Contents file"