configureable acquire targets to download additional files

First pass at making the acquire system capable of downloading files
based on configuration rather than hardcoded entries. It is now possible
to instruct 'deb' and 'deb-src' sources.list lines to download more than
just Packages/Translation-* and Sources files. Details on how to do that
can be found in the included documentation file.

apt-pkg/contrib/configuration.cc

@@ -259,7 +259,7 @@ vector<string> Configuration::FindVector(const char *Name) const
    return FindVector(Name, "");
 }
 #endif
-vector<string> Configuration::FindVector(const char *Name, std::string const &Default) const
+vector<string> Configuration::FindVector(const char *Name, std::string const &Default, bool const Keys) const
 {
    vector<string> Vec;
    const Item *Top = Lookup(Name);
@@ -272,7 +272,7 @@ vector<string> Configuration::FindVector(const char *Name, std::string const &De
    Item *I = Top->Child;
    while(I != NULL)
    {
-      Vec.push_back(I->Value);
+      Vec.push_back(Keys ? I->Tag : I->Value);
       I = I->Next;
    }
    if (Vec.empty() == true)

apt-pkg/contrib/configuration.h

@@ -84,15 +84,8 @@ class Configuration
     *
     * \param Name of the parent node
     * \param Default list of values separated by commas */
-#if APT_PKG_ABI >= 413
-   std::vector<std::string> FindVector(const char *Name, std::string const &Default = "") const;
-   std::vector<std::string> FindVector(std::string const &Name, std::string const &Default = "") const { return FindVector(Name.c_str(), Default); };
-#else
-   std::vector<std::string> FindVector(const char *Name, std::string const &Default) const;
-   std::vector<std::string> FindVector(std::string const &Name, std::string const &Default) const { return FindVector(Name.c_str(), Default); };
-   std::vector<std::string> FindVector(const char *Name) const;
-   std::vector<std::string> FindVector(std::string const &Name) const { return FindVector(Name.c_str(), ""); };
-#endif
+   std::vector<std::string> FindVector(const char *Name, std::string const &Default = "", bool const Keys = false) const;
+   std::vector<std::string> FindVector(std::string const &Name, std::string const &Default = "", bool const Keys = false) const { return FindVector(Name.c_str(), Default, Keys); };
 
    int FindI(const char *Name,int const &Default = 0) const;
    int FindI(std::string const &Name,int const &Default = 0) const {return FindI(Name.c_str(),Default);};

apt-pkg/deb/debmetaindex.cc

@@ -23,25 +23,6 @@
 
 using namespace std;
 
-string debReleaseIndex::Info(const char *Type, string const &Section, string const &Arch) const
-{
-   string Info = ::URI::SiteOnly(URI) + ' ';
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Info += Dist;
-   }
-   else
-   {
-      Info += Dist + '/' + Section;
-      if (Arch.empty() != true)
-	 Info += " " + Arch;
-   }
-   Info += " ";
-   Info += Type;
-   return Info;
-}
-
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
 {
    string Info = ::URI::SiteOnly(URI) + ' ';
@@ -92,81 +73,6 @@ std::string debReleaseIndex::LocalFileName() const
    return "";
 }
 
-string debReleaseIndex::IndexURISuffix(const char *Type, string const &Section, string const &Arch) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-   {
-      if (Arch == "native")
-	 Res += Section + "/binary-" + _config->Find("APT::Architecture") + '/';
-      else
-	 Res += Section + "/binary-" + Arch + '/';
-   }
-   return Res + Type;
-}
-   
-
-string debReleaseIndex::IndexURI(const char *Type, string const &Section, string const &Arch) const
-{
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      string Res;
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + '/' + IndexURISuffix(Type, Section, Arch);
- }
-
-string debReleaseIndex::SourceIndexURISuffix(const char *Type, const string &Section) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-      Res += Section + "/source/";
-   return Res + Type;
-}
-
-string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section) const
-{
-   string Res;
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section);
-}
-
-string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-      Res += Section + "/i18n/Translation-";
-   return Res + Type;
-}
-
-string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const
-{
-   string Res;
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section);
-}
-
 debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) :
 					metaIndex(URI, Dist, "deb"), Trusted(CHECK_TRUST)
 {}
@@ -184,73 +90,161 @@ debReleaseIndex::~debReleaseIndex() {
 			delete *S;
 }
 
-vector <IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
-	vector <IndexTarget *>* IndexTargets = new vector <IndexTarget *>;
+vector <IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const
+{
+   vector <IndexTarget *>* IndexTargets = new vector <IndexTarget *>;
 
-	map<string, vector<debSectionEntry const*> >::const_iterator const src = ArchEntries.find("source");
-	if (src != ArchEntries.end()) {
-		vector<debSectionEntry const*> const SectionEntries = src->second;
-		for (vector<debSectionEntry const*>::const_iterator I = SectionEntries.begin();
-		     I != SectionEntries.end(); ++I) {
-			char const * const ShortDesc = "Sources";
-			IndexTarget * const Target = new IndexTarget(
-			      SourceIndexURISuffix(ShortDesc, (*I)->Section),
-			      ShortDesc,
-			      Info(ShortDesc, (*I)->Section),
-			      SourceIndexURI(ShortDesc, (*I)->Section)
-			      );
-			IndexTargets->push_back (Target);
-		}
-	}
+   bool const flatArchive = (Dist[Dist.length() - 1] == '/');
+   std::string baseURI = URI;
+   if (flatArchive)
+   {
+      if (Dist != "/")
+         baseURI += Dist;
+   }
+   else
+      baseURI += "dists/" + Dist + "/";
+   std::string const Release = (Dist == "/") ? "" : Dist;
+   std::string const Site = ::URI::SiteOnly(URI);
+   std::vector<std::string> lang = APT::Configuration::getLanguages(true);
+   if (lang.empty())
+      lang.push_back("none");
+   map<string, vector<debSectionEntry const*> >::const_iterator const src = ArchEntries.find("source");
+   if (src != ArchEntries.end())
+   {
+      std::vector<std::string> const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true);
+      for (std::vector<std::string>::const_iterator T = targets.begin(); T != targets.end(); ++T)
+      {
+#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X))
+	 std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+	 std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
+	 std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
+	 bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true);
+#undef APT_T_CONFIG
+	 if (URI.empty())
+	    continue;
 
-	// Only source release
-	if (IndexTargets->empty() == false && ArchEntries.size() == 1)
-		return IndexTargets;
+	 vector<debSectionEntry const*> const SectionEntries = src->second;
+	 for (vector<debSectionEntry const*>::const_iterator I = SectionEntries.begin();
+	       I != SectionEntries.end(); ++I)
+	 {
+	    for (vector<std::string>::const_iterator l = lang.begin(); l != lang.end(); ++l)
+	    {
+	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+		  continue;
+
+	       struct SubstVar subst[] = {
+		  { "$(SITE)", &Site },
+		  { "$(RELEASE)", &Release },
+		  { "$(COMPONENT)", &((*I)->Section) },
+		  { "$(LANGUAGE)", &(*l) },
+		  { NULL, NULL }
+	       };
+	       std::string const name = SubstVar(URI, subst);
+	       IndexTarget * Target;
+	       if (IsOptional == true)
+	       {
+		  Target = new OptionalIndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       else
+	       {
+		  Target = new IndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       IndexTargets->push_back(Target);
+
+	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+		  break;
+	    }
 
-	std::set<std::string> sections;
-	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
-	     a != ArchEntries.end(); ++a) {
-		if (a->first == "source")
-			continue;
-		for (vector <const debSectionEntry *>::const_iterator I = a->second.begin();
-		     I != a->second.end(); ++I) {
-			char const * const ShortDesc = "Packages";
-			IndexTarget * const Target = new IndexTarget(
-			      IndexURISuffix(ShortDesc, (*I)->Section, a->first),
-			      ShortDesc,
-			      Info (ShortDesc, (*I)->Section, a->first),
-			      IndexURI(ShortDesc, (*I)->Section, a->first)
-			      );
-			IndexTargets->push_back (Target);
-			sections.insert((*I)->Section);
-		}
-	}
+	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	       break;
+	 }
+      }
+   }
 
-	std::vector<std::string> lang = APT::Configuration::getLanguages(true);
-	std::vector<std::string>::iterator lend = std::remove(lang.begin(), lang.end(), "none");
-	if (lend != lang.end())
-		lang.erase(lend);
-
-	if (lang.empty() == true)
-		return IndexTargets;
-
-	// get the Translation-* files, later we will skip download of non-existent if we have an index
-	for (std::set<std::string>::const_iterator s = sections.begin();
-	     s != sections.end(); ++s) {
-		for (std::vector<std::string>::const_iterator l = lang.begin();
-		     l != lang.end(); ++l) {
-			std::string const ShortDesc = "Translation-" + *l;
-			IndexTarget * const Target = new OptionalIndexTarget(
-			      TranslationIndexURISuffix(l->c_str(), *s),
-			      ShortDesc,
-			      Info (ShortDesc.c_str(), *s),
-			      TranslationIndexURI(l->c_str(), *s)
-			      );
-			IndexTargets->push_back(Target);
-		}
-	}
+   // Only source release
+   if (IndexTargets->empty() == false && ArchEntries.size() == 1)
+      return IndexTargets;
 
-	return IndexTargets;
+   std::vector<std::string> const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true);
+   for (std::vector<std::string>::const_iterator T = targets.begin(); T != targets.end(); ++T)
+   {
+#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X))
+      std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+      std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
+      std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
+      bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true);
+#undef APT_T_CONFIG
+      if (URI.empty())
+	 continue;
+
+      for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
+	    a != ArchEntries.end(); ++a)
+      {
+	 if (a->first == "source")
+	    continue;
+
+	 for (vector <const debSectionEntry *>::const_iterator I = a->second.begin();
+	       I != a->second.end(); ++I) {
+
+	    for (vector<std::string>::const_iterator l = lang.begin(); l != lang.end(); ++l)
+	    {
+	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+		  continue;
+
+	       struct SubstVar subst[] = {
+		  { "$(SITE)", &Site },
+		  { "$(RELEASE)", &Release },
+		  { "$(COMPONENT)", &((*I)->Section) },
+		  { "$(LANGUAGE)", &(*l) },
+		  { "$(ARCHITECTURE)", &(a->first) },
+		  { NULL, NULL }
+	       };
+	       std::string const name = SubstVar(URI, subst);
+	       IndexTarget * Target;
+	       if (IsOptional == true)
+	       {
+		  Target = new OptionalIndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       else
+	       {
+		  Target = new IndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       IndexTargets->push_back(Target);
+
+	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+		  break;
+	    }
+
+	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	       break;
+	 }
+
+	 if (URI.find("$(ARCHITECTURE)") == std::string::npos)
+	    break;
+      }
+   }
+
+   return IndexTargets;
 }
 									/*}}}*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
@@ -361,17 +355,6 @@ void debReleaseIndex::PushSectionEntry(string const &Arch, const debSectionEntry
 	ArchEntries[Arch].push_back(Entry);
 }
 
-void debReleaseIndex::PushSectionEntry(const debSectionEntry *Entry) {
-	if (Entry->IsSrc == true)
-		PushSectionEntry("source", Entry);
-	else {
-		for (map<string, vector<const debSectionEntry *> >::iterator a = ArchEntries.begin();
-		     a != ArchEntries.end(); ++a) {
-			a->second.push_back(Entry);
-		}
-	}
-}
-
 debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section,
 		bool const &IsSrc): Section(Section), IsSrc(IsSrc)
 {}

apt-pkg/deb/debmetaindex.h

@@ -47,7 +47,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual std::string ArchiveURI(std::string const &File) const {return URI + File;};
    virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const;
    std::vector <IndexTarget *>* ComputeIndexTargets() const;
-   std::string Info(const char *Type, std::string const &Section, std::string const &Arch="") const;
 
    std::string MetaIndexInfo(const char *Type) const;
    std::string MetaIndexFile(const char *Types) const;
@@ -58,12 +57,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
 #endif
    std::string LocalFileName() const;
 
-   std::string IndexURI(const char *Type, std::string const &Section, std::string const &Arch="native") const;
-   std::string IndexURISuffix(const char *Type, std::string const &Section, std::string const &Arch="native") const;
-   std::string SourceIndexURI(const char *Type, const std::string &Section) const;
-   std::string SourceIndexURISuffix(const char *Type, const std::string &Section) const;
-   std::string TranslationIndexURI(const char *Type, const std::string &Section) const;
-   std::string TranslationIndexURISuffix(const char *Type, const std::string &Section) const;
    virtual std::vector <pkgIndexFile *> *GetIndexFiles();
 
    void SetTrusted(bool const Trusted);
@@ -71,7 +64,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
 
    void PushSectionEntry(std::vector<std::string> const &Archs, const debSectionEntry *Entry);
    void PushSectionEntry(std::string const &Arch, const debSectionEntry *Entry);
-   void PushSectionEntry(const debSectionEntry *Entry);
 };
 
 class APT_HIDDEN debDebFileMetaIndex : public metaIndex

apt-pkg/init.cc

@@ -101,8 +101,26 @@ bool pkgInitConfig(Configuration &Cnf)
    // The default user we drop to in the methods
    Cnf.CndSet("APT::Sandbox::User", "_apt");
 
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::URI", "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatURI", "Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::ShortDescription", "Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Description", "$(SITE) $(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatDescription", "$(SITE) $(RELEASE) Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Optional", false);
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::URI", "$(COMPONENT)/i18n/Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatURI", "$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::ShortDescription", "Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatDescription", "$(SITE) $(RELEASE) Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::URI", "$(COMPONENT)/source/Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatURI", "Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::ShortDescription", "Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatDescription", "$(SITE) $(RELEASE) Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Optional", false);
+
    bool Res = true;
-   
+
    // Read an alternate config file
    const char *Cfg = getenv("APT_CONFIG");
    if (Cfg != 0 && strlen(Cfg) != 0)

doc/acquire-additional-files.txt

@@ -0,0 +1,142 @@
+# Acquire additional files in 'update' operations
+
+The download and verification of data from multiple sources in different
+compression formats, with partial downloads and patches is an involved
+process which is hard to implement correctly and securely.
+
+APT frontends share the code and binaries to make this happen in libapt
+with the Acquire system, supported by helpers shipped in the apt package
+itself and additional transports in individual packages like
+apt-transport-https.
+
+For its own operation libapt needs or can make use of Packages, Sources
+and Translation-* files, which it will acquire by default, but
+a repository might contain more data files (e.g.  Contents) a frontend
+might want to use and would therefore need to be downloaded as well
+(e.g. apt-file).
+
+This file describes the configuration scheme such a frontend can use to
+instruct the Acquire system to download those additional files.
+
+Note that you can't download files from other sources. It must be files
+in the same repository and below the Release file. The Release file must
+also contain hashes for the file itself as well as for the compressed
+file if wanted, otherwise a download isn't even tried!
+
+
+# The Configuration Stanza
+
+The Acquire system uses the same configuration settings to implement the
+files it downloads by default. These settings are the default, but if
+they would be written in a configuration file the configuration
+instructing the Acquire system to download the Packages files would look
+like this (see also apt.conf(5) manpage for configuration file syntax):
+
+	APT::Acquire::Targets::deb::Packages {
+		URI "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages";
+		ShortDescription "Packages";
+		Description "$(SITE) $(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages";
+
+		flatURI "Packages";
+		flatDescription "$(SITE) $(RELEASE) Packages";
+
+		Optional "false";
+	};
+
+All files which should be downloaded (nicknamed 'Targets') are mentioned
+below the APT::Acquire::Targets scope. 'deb' is here the type of the
+sources.list entry the file should be acquired for. The only other
+supported value is hence 'deb-src'. Beware: You can't specify multiple
+types here and you can't download the same URI for multiple types!
+
+After the type you can pick any valid and unique string which preferable
+refers to the file it downloads (In the example we picked 'Packages').
+This string is never shown or used.
+
+All targets have three main properties you can define:
+* URI: The identifier of the file to be downloaded as used in the
+  Release file.  It is also the relative location of the file from the
+  Release file.  You can neither download from a different server
+  entirely (absolute URI) nor should you try to access directories above
+  the Release file (e.g. "../../").
+* ShortDescription: Very short string intended to be displayed to the
+  user e.g.  while reporting progress. apt will e.g. use this string in
+  the last line to indicate progress of e.g. the download of a specific
+  item.
+* Description: A preferable human understandable and readable identifier
+  of which file is acquired exactly. Mainly used for progress reporting
+  and error messages. apt will e.g. use this string in the Get/Hit/Err
+  progress lines.
+
+Additional optional properties:
+* flat{URI,Description}: APT supports two types of repositories:
+  dists-style repositories which are the default and by far the most
+  common which are named after the fact that the files are in an
+  elaborated directory structure.  In contrast a flat-style repositories
+  lumps all files together in one directory.  Support for these flat
+  repositories exists mainly for legacy purposes only.  It is therefore
+  recommend to not set these values.
+* Optional: The default value is 'true' and should be kept at this
+  value.  If enabled the acquire system will skip the download if the
+  file isn't mentioned in the Release file. Otherwise this is treated as
+  a hard error and the update process fails.
+
+
+Note that the acquire system will automatically choose to download
+a compressed file if it is available and uncompress it for you, just as
+it will also use pdiff patching if provided by the repository and
+enabled by the user. You only have to ensure that the Release file
+contains the information about the compressed files/pdiffs to make this
+happen. NO properties have to be set to enable this.
+
+# More examples
+
+The stanzas for Translation-* files as well as for Sources files would
+look like this:
+
+APT::Acquire::Targets {
+	deb::Translations {
+		URI "$(COMPONENT)/i18n/Translation-$(LANGUAGE)";
+		ShortDescription "Translation-$(LANGUAGE)";
+		Description "$(SITE) $(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)";
+
+		flatURI "$(LANGUAGE)";
+		flatDescription "$(SITE) $(RELEASE) Translation-$(LANGUAGE)";
+	};
+
+	deb-src::Sources {
+		URI "$(COMPONENT)/source/Sources";
+		ShortDescription "Sources";
+		Description "$(SITE) $(RELEASE)/$(COMPONENT) Sources";
+
+		flatURI "Sources";
+		flatDescription "$(SITE) $(RELEASE) Sources";
+
+		Optional "false";
+	};
+};
+
+# Substitution variables
+
+As seen in the examples, properties can contain placeholders filled in
+by the acquire system. The following variables are known; note that
+unknown variables have no default value nor are they touched: They are
+printed literally.
+
+* $(SITE): An identifier of the site we access, e.g.
+  "http://example.org/".
+* $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
+  "stretch".  Note that flat-style repositories do not have a archive-
+  or codename per-se, so the value might very well be just "/" or so.
+* $(COMPONENT): as given in the sources.list, e.g. "main", "non-free" or
+  "universe".  Note that flat-style repositories again do not really
+  have a meaningful value here.
+* $(LANGUAGE): Values are all entries (expect "none") of configuration
+  option Acquire::Languages, e.g. "en", "de" or "de_AT".
+
+These values are defined both for 'deb' as well as 'deb-src' targets.
+'deb' targets additional have the variable:
+
+* $(ARCHITECTURE): Values are all entries of configuration option
+  APT::Architectures (potentially modified by sources.list options),
+  e.g. "amd64", "i386" or "armel".

test/integration/test-apt-acquire-additional-files

@@ -0,0 +1,54 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'amd64'
+configcompression '.' 'gz'
+
+buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
+
+setupaptarchive --no-update
+changetowebserver
+
+testsuccessequal "Get:1 http://localhost:8080 unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B]
+Get:2 http://localhost:8080 unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B]
+Get:3 http://localhost:8080 unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
+Get:4 http://localhost:8080 unstable/main Translation-en [$(stat -c%s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
+Reading package lists..." aptget update
+
+testempty find rootdir/var/lib/apt/lists -name '*Contents*'
+
+cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
+APT::Acquire::Targets::deb::Contents {
+	URI "\$(COMPONENT)/Contents-\$(ARCHITECTURE)";
+	ShortDescription "Contents";
+	Description "\$(SITE) \$(RELEASE)/\$(COMPONENT) \$(ARCHITECTURE) Contents";
+};
+EOF
+
+testsuccessequal "Hit http://localhost:8080 unstable InRelease
+Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Reading package lists..." aptget update
+
+testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*'
+testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'
+
+# no automatic uncompress based on the name please,
+# only if we downloaded a compressed file, but target was uncompressed
+cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
+APT::Acquire::Targets::deb::Contents {
+	URI "\$(COMPONENT)/Contents-\$(ARCHITECTURE).gz";
+	ShortDescription "Contents.gz";
+	Description "\$(SITE) \$(RELEASE)/\$(COMPONENT) \$(ARCHITECTURE) Contents.gz";
+};
+EOF
+
+testsuccessequal "Hit http://localhost:8080 unstable InRelease
+Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Reading package lists..." aptget update
+
+testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
+testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'

test/integration/test-apt-by-hash-update

@@ -11,8 +11,6 @@ insertpackage 'unstable' 'foo' 'all' '1.0'
 
 setupaptarchive --no-update
 
-APTARCHIVE=$(readlink -f ./aptarchive)
-
 # make Packages *only* accessable by-hash for this test
 mkdir -p aptarchive/dists/unstable/main/binary-i386/by-hash/SHA512
 (cd  aptarchive/dists/unstable/main/binary-i386/by-hash/SHA512 && 
@@ -26,7 +24,7 @@ mkdir -p aptarchive/dists/unstable/main/source/by-hash/SHA512
 )
 
 # we moved the Packages file away, normal update won't work
-testfailure aptget upate
+testfailure aptget update
 
 # ensure we do not know about "foo"
 testfailureequal "Reading package lists...
@@ -36,14 +34,18 @@ E: Unable to locate package foo" aptget install -q -s foo
 # ensure we can apt-get update by hash
 testsuccess aptget update -o APT::Acquire::By-Hash=1 -o Acquire::Languages=none
 
-# ensure it works
-testsuccessequal "Inst foo (1.0 unstable [all])
+ensureitworks() {
+	testsuccessequal "Inst foo (1.0 unstable [all])
 Conf foo (1.0 unstable [all])" aptget install -qq -s foo
+}
+ensureitworks
 
 # add magic string to Release file ...
 MAGIC="Acquire-By-Hash: true"
 sed -i "s#Suite: unstable#Suite: unstable\n$MAGIC#" aptarchive/dists/unstable/Release
 signreleasefiles
 # ... and verify that it fetches by hash now
+rm -rf rootdir/var/lib/apt/lists
 testsuccess aptget update -o Acquire::Languages=none
 
+ensureitworks