implement 'apt-get files' to access index targets

Downloading additional files is only half the job. We still need a way
to allow external tools to know where the files are they requested for
download given that we don't want them to choose their own location.

'apt-get files' is our answer to this showing by default in a deb822
format information about each IndexTarget with the potential to filter
the records based on lines and an option to change the output format.

The command serves also as an example on how to get to this information
via libapt.

apt-pkg/deb/debmetaindex.cc

@@ -136,11 +136,14 @@ void foreachTarget(std::string const URI, std::string const Dist,
 	       std::map<std::string, std::string> Options;
 	       Options.insert(std::make_pair("SITE", Site));
 	       Options.insert(std::make_pair("RELEASE", Release));
-	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
-	       Options.insert(std::make_pair("LANGUAGE", *l));
+	       if (MetaKey.find("$(COMPONENT)") != std::string::npos)
+		  Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       if (MetaKey.find("$(LANGUAGE)") != std::string::npos)
+		  Options.insert(std::make_pair("LANGUAGE", *l));
 	       Options.insert(std::make_pair("ARCHITECTURE", "source"));
 	       Options.insert(std::make_pair("BASE_URI", baseURI));
 	       Options.insert(std::make_pair("REPO_URI", URI));
+	       Options.insert(std::make_pair("TARGET_OF", "deb-src"));
 	       Options.insert(std::make_pair("CREATED_BY", *T));
 	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
 
@@ -183,11 +186,15 @@ void foreachTarget(std::string const URI, std::string const Dist,
 	       std::map<std::string, std::string> Options;
 	       Options.insert(std::make_pair("SITE", Site));
 	       Options.insert(std::make_pair("RELEASE", Release));
-	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
-	       Options.insert(std::make_pair("LANGUAGE", *l));
-	       Options.insert(std::make_pair("ARCHITECTURE", a->first));
+	       if (MetaKey.find("$(COMPONENT)") != std::string::npos)
+		  Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       if (MetaKey.find("$(LANGUAGE)") != std::string::npos)
+		  Options.insert(std::make_pair("LANGUAGE", *l));
+	       if (MetaKey.find("$(ARCHITECTURE)") != std::string::npos)
+		  Options.insert(std::make_pair("ARCHITECTURE", a->first));
 	       Options.insert(std::make_pair("BASE_URI", baseURI));
 	       Options.insert(std::make_pair("REPO_URI", URI));
+	       Options.insert(std::make_pair("TARGET_OF", "deb"));
 	       Options.insert(std::make_pair("CREATED_BY", *T));
 	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
 

apt-pkg/indexfile.cc

@@ -133,8 +133,10 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(ARCHITECTURE);
       APT_CASE(BASE_URI);
       APT_CASE(REPO_URI);
+      APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
 #undef APT_CASE
+      case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
    }
    std::map<std::string,std::string>::const_iterator const M = Options.find(Key);
    if (M == Options.end())
@@ -142,6 +144,20 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
    return M->second;
 }
 									/*}}}*/
+std::string IndexTarget::Format(std::string format) const		/*{{{*/
+{
+   for (std::map<std::string, std::string>::const_iterator O = Options.begin(); O != Options.end(); ++O)
+   {
+      format = SubstVar(format, std::string("$(") + O->first + ")", O->second);
+   }
+   format = SubstVar(format, "$(METAKEY)", MetaKey);
+   format = SubstVar(format, "$(SHORTDESC)", ShortDesc);
+   format = SubstVar(format, "$(DESCRIPTION)", Description);
+   format = SubstVar(format, "$(URI)", URI);
+   format = SubstVar(format, "$(FILENAME)", Option(IndexTarget::FILENAME));
+   return format;
+}
+									/*}}}*/
 
 pkgIndexTargetFile::pkgIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/
    pkgIndexFile(Trusted), Target(Target)
@@ -162,7 +178,7 @@ std::string pkgIndexTargetFile::Describe(bool Short) const		/*{{{*/
 									/*}}}*/
 std::string pkgIndexTargetFile::IndexFileName() const			/*{{{*/
 {
-   std::string const s =_config->FindDir("Dir::State::lists") + URItoFileName(Target.URI);
+   std::string const s = Target.Option(IndexTarget::FILENAME);
    if (FileExists(s))
       return s;
 

apt-pkg/indexfile.h

@@ -79,8 +79,11 @@ class IndexTarget							/*{{{*/
       BASE_URI,
       REPO_URI,
       CREATED_BY,
+      TARGET_OF,
+      FILENAME,
    };
    std::string Option(OptionKeys const Key) const;
+   std::string Format(std::string format) const;
 };
 									/*}}}*/
 

apt-private/private-cmndline.cc

@@ -163,6 +163,10 @@ static bool addArgumentsAPTGet(std::vector<CommandLine::Args> &Args, char const
       // once sbuild is fixed, this option can be removed
       addArg('f', "fix-broken", "APT::Get::Fix-Broken", 0);
    }
+   else if (CmdMatches("files"))
+   {
+      addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);
+   }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
 	    CmdMatches("markauto", "unmarkauto")) // deprecated commands
       ;

cmdline/apt-get.cc

@@ -86,6 +86,7 @@
 #include <algorithm>
 #include <fstream>
 #include <iostream>
+#include <sstream>
 #include <set>
 #include <string>
 #include <vector>
@@ -1602,6 +1603,91 @@ static bool DoChangelog(CommandLine &CmdL)
    return true;
 }
 									/*}}}*/
+// DoFiles - Lists all IndexTargets					/*{{{*/
+static std::string format_key(std::string key)
+{
+   // deb822 is case-insensitive, but the human eye prefers candy
+   std::transform(key.begin(), key.end(), key.begin(), ::tolower);
+   key[0] = ::toupper(key[0]);
+   size_t found = key.find("_uri");
+   if (found != std::string::npos)
+      key.replace(found, 4, "-URI");
+   while ((found = key.find('_')) != std::string::npos)
+   {
+      key[found] = '-';
+      key[found + 1] = ::toupper(key[found + 1]);
+   }
+   return key;
+}
+static bool DoFiles(CommandLine &CmdL)
+{
+   pkgCacheFile CacheFile;
+   pkgSourceList *SrcList = CacheFile.GetSourceList();
+
+   if (SrcList == NULL)
+      return false;
+
+   std::string const Format = _config->Find("APT::Get::Files::Format");
+   bool Filtered = CmdL.FileSize() > 1;
+   for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
+   {
+      std::vector<IndexTarget> const targets = (*S)->GetIndexTargets();
+      std::map<std::string, string> AddOptions;
+      AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+
+      for (std::vector<IndexTarget>::const_iterator T = targets.begin(); T != targets.end(); ++T)
+      {
+	 std::ostringstream stanza;
+	 if (Filtered || Format.empty())
+	 {
+	    stanza << "MetaKey: " << T->MetaKey << "\n"
+	       << "ShortDesc: " << T->ShortDesc << "\n"
+	       << "Description: " << T->Description << "\n"
+	       << "URI: " << T->URI << "\n"
+	       << "Filename: " << T->Option(IndexTarget::FILENAME) << "\n"
+	       << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
+	    for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
+	       stanza << format_key(O->first) << ": " << O->second << "\n";
+	    for (std::map<std::string,std::string>::const_iterator O = T->Options.begin(); O != T->Options.end(); ++O)
+	       stanza << format_key(O->first) << ": " << O->second << "\n";
+	    stanza << "\n";
+
+	    if (Filtered)
+	    {
+	       // that is a bit crude, but good enough for now
+	       bool found = true;
+	       std::string haystack = std::string("\n") + stanza.str() + "\n";
+	       std::transform(haystack.begin(), haystack.end(), haystack.begin(), ::tolower);
+	       size_t const filesize = CmdL.FileSize() - 1;
+	       for (size_t i = 0; i != filesize; ++i)
+	       {
+		  std::string needle = std::string("\n") + CmdL.FileList[i + 1] + "\n";
+		  std::transform(needle.begin(), needle.end(), needle.begin(), ::tolower);
+		  if (haystack.find(needle) != std::string::npos)
+		     continue;
+		  found = false;
+		  break;
+	       }
+	       if (found == false)
+		  continue;
+	    }
+	 }
+
+	 if (Format.empty())
+	    cout << stanza.str();
+	 else
+	 {
+	    std::string out = T->Format(Format);
+	    for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
+	       out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
+	    cout << out << std::endl;
+	 }
+      }
+   }
+
+   return true;
+}
+									/*}}}*/
 // ShowHelp - Show a help screen					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -1716,6 +1802,7 @@ int main(int argc,const char *argv[])					/*{{{*/
 				   {"source",&DoSource},
                                    {"download",&DoDownload},
                                    {"changelog",&DoChangelog},
+				   {"files",&DoFiles},
 				   {"moo",&DoMoo},
 				   {"help",&ShowHelp},
                                    {0,0}};

doc/acquire-additional-files.txt

@@ -47,11 +47,13 @@ All files which should be downloaded (nicknamed 'Targets') are mentioned
 below the APT::Acquire::Targets scope. 'deb' is here the type of the
 sources.list entry the file should be acquired for. The only other
 supported value is hence 'deb-src'. Beware: You can't specify multiple
-types here and you can't download the same MetaKey form multiple types!
+types here and you can't download the same (evaluated) MetaKey from
+multiple types!
 
 After the type you can pick any valid and unique string which preferable
 refers to the file it downloads (In the example we picked 'Packages').
-This string is never shown or used.
+This string is used as identifier for the target class and accessible as
+'Created-By' e.g. in the "apt-get files" output as detailed below.
 
 All targets have three main properties you can define:
 * MetaKey: The identifier of the file to be downloaded as used in the
@@ -123,11 +125,12 @@ by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are
 printed literally.
 
-* $(SITE): An identifier of the site we access, e.g.
-  "http://example.org/".
+* $(SITE): An identifier of the site we access as seen in sources.list,
+  e.g. "http://example.org/debian" or "file:/path/to/a/repository".
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.
+  Again, as seen in the sources.list.
 * $(COMPONENT): as given in the sources.list, e.g. "main", "non-free" or
   "universe".  Note that flat-style repositories again do not really
   have a meaningful value here.
@@ -137,3 +140,79 @@ printed literally.
   APT::Architectures (potentially modified by sources.list options),
   e.g. "amd64", "i386" or "armel" for the 'deb' type. In type 'deb-src'
   this variable has the value "source".
+
+Note that while more variables might exist in the implementation, these
+are to be considered undefined and their usage strongly discouraged. If
+you have a need for another variable contact us instead.
+
+# Accessing files
+
+Do NOT hardcode specific file locations, names or compression types in
+your application! You will notice that the configuration options give
+you no choice over where the downloaded files will be stored. This is by
+design so multiple applications can download and use the same file
+rather than each and every one of them potentially downloads and uses
+its own copy somewhere on disk.
+
+"apt-get files" can be used to get the location as well as other
+information about all files downloaded (aka: you will see Packages,
+Sources and Translation-* files here as well). Provide a line of the
+default output format as parameter to filter out all entries which do
+not have such a line. With --format, you can further more define your
+own output style. The variables are what you see in the output, just all
+uppercase and wrapped in $(), as in the configuration file.
+
+To get all the filenames of all Translation-en files you can e.g. call:
+	apt-get files --format '$(FILENAME)' "Created-By: Translations" "Language: en"
+
+Accessing this information via libapt is done by reading the
+sources.lists (pkgSourceList), iterating over the metaIndex objects this
+creates and calling GetIndexTargets() on them. See the sourcecode of
+"apt-get files" for a complete example.
+
+Remarks on the available fields:
+* MetaKey, ShortDesc, Description, Site, Release: as defined
+  by the configuration and described further above.
+* Created-By: configuration entity responsible for this target
+* Target-Of: type of the sources.list entry
+* URI, Repo-URI: avoid using. Contains potentially username/password.
+  Prefer 'Site', especially for display.
+* Filename: The mentioned file doesn't need to exist, e.g. because the
+  file wasn't downloaded yet – or it does exist compressed. libapt users
+  are encouraged to use FileFd to open such files as it can handle
+  decompression transparently.
+* Trusted: As of the last 'apt-get update' call denoting if e.g. apt-get
+  would print the unauthenticated warning while installing packages
+  mentioned in this file (example assumes Packages file) [Not really
+  a property of the target itself, but of the metaIndex].
+* Optional: Decodes the option of the same name from the configuration.
+  Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
+* Language, Architecture, Component: as defined further above, but with
+  the catch that they might be missing if they don't effect the target
+  (aka: They weren't used while evaluating the MetaKey template).
+
+Again, additional fields might be visible in certain implementation, but
+you should avoid using them and instead talk to us about a portable
+implementation.
+
+# Multiple application requiring the same files
+
+It is highly encouraged that applications talk to each other and to us
+about which files they require. It is usually best to have a common
+package ship the configuration needed to get the files, but specific
+needs might require specific solutions. Again: talk to us.
+
+# Acquiring files not mentioned in the Release file
+
+You can't. This is by design as these files couldn't be verified to not
+be modified in transit, corrupted by the download process or simple if
+they are present at all on the server, which would require apt to probe
+for them. APT did this in the past for legacy reasons, we do not intend
+to go back to these dark times.
+
+This is also why you can't request files from a different server. It
+would have the additional problem that this server might not even be
+accessible (e.g. proxy settings) or that local sources (file:/, cdrom:/)
+start requesting online files…
+
+In other words: We would be opening Pandora's box.

doc/apt-get.8.xml

@@ -243,6 +243,21 @@
        </listitem>
      </varlistentry>
 
+     <varlistentry><term><option>files</option></term>
+	<listitem><para>Displays by default a deb822 formatted listing of
+	      information about all data files <command>apt-get
+		 update</command> would download. Supports a
+	      <option>--format</option> option to modify the output format as
+	      well as accepts lines of the default output to filter the records
+	      by. The command is mainly used as an interface for external tools
+	      working with APT to get information as well as filenames for
+	      downloaded files so they can use them as well instead of
+	      downloading them again on their own. Detailed documentation is
+	      omitted here and can instead be found in the source tree in
+	      <literal><filename>doc/acquire-additional-files.txt</filename></literal>.
+	   </para>
+	</listitem>
+     </varlistentry>
 
 
    </variablelist>

test/integration/test-apt-acquire-additional-files

@@ -48,6 +48,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'
 
 # no automatic uncompress based on the name please,
@@ -72,6 +73,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarch
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
 
 rm -f rootdir/etc/apt/apt.conf.d/content-target.conf