implement default apt-get file --release-info mode

Selecting targets based on the Release they belong to isn't to
unrealistic. In fact, it is assumed to be the most used case so it is
made the default especially as this allows to bundle another thing we
have to be careful with: Filenames and only showing targets we have
acquired.

Closes: 752702

apt-pkg/deb/debmetaindex.cc

@@ -422,7 +422,7 @@ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/
 }
 									/*}}}*/
 // ReleaseIndex::FindInCache - Find this index				/*{{{*/
-pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool const ModifyCheck) const
 {
    std::string ReleaseFile;
    bool const releaseExists = ReleaseFileName(this, ReleaseFile);
@@ -434,7 +434,7 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
 	 continue;
 
        // empty means the file does not exist by "design"
-       if (releaseExists == false && File->Size == 0)
+       if (ModifyCheck == false || (releaseExists == false && File->Size == 0))
 	  return File;
 
       struct stat St;

apt-pkg/deb/debmetaindex.h

@@ -51,7 +51,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual std::vector<IndexTarget> GetIndexTargets() const;
 
    virtual std::string Describe() const;
-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    std::string MetaIndexInfo(const char *Type) const;

apt-pkg/indexfile.cc

@@ -137,6 +137,18 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(CREATED_BY);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
+      case EXISTING_FILENAME:
+	 std::string const filename = Option(FILENAME);
+	 std::vector<std::string> const types = APT::Configuration::getCompressionTypes();
+	 for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
+	 {
+	    if (t->empty())
+	       continue;
+	    std::string const file = (*t == "uncompressed") ? filename : (filename + "." + *t);
+	    if (FileExists(file))
+	       return file;
+	 }
+	 return "";
    }
    std::map<std::string,std::string>::const_iterator const M = Options.find(Key);
    if (M == Options.end())

apt-pkg/indexfile.h

@@ -81,6 +81,7 @@ class IndexTarget							/*{{{*/
       CREATED_BY,
       TARGET_OF,
       FILENAME,
+      EXISTING_FILENAME,
    };
    std::string Option(OptionKeys const Key) const;
    std::string Format(std::string format) const;

apt-pkg/metaindex.cc

@@ -28,7 +28,7 @@ std::string metaIndex::Describe() const
    return "Release";
 }
 
-pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache, bool const) const
 {
    return pkgCache::RlsFileIterator(Cache);
 }

apt-pkg/metaindex.h

@@ -54,7 +54,7 @@ class metaIndex
    virtual bool IsTrusted() const = 0;
 
    virtual std::string Describe() const;
-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    metaIndex(std::string const &URI, std::string const &Dist,

apt-pkg/pkgcachegen.cc

@@ -1224,7 +1224,7 @@ static bool CheckValidity(const string &CacheFile,
    {
       if (Debug == true)
 	 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
-      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache);
+      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
       if (RlsFile.end() == true)
       {
 	 if (Debug == true)
@@ -1348,7 +1348,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    {
       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
       {
-	 if ((*i)->FindInCache(Gen.GetCache()).end() == false)
+	 if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
 	 {
 	    _error->Warning("Duplicate sources.list entry %s",
 		  (*i)->Describe().c_str());

apt-private/private-cmndline.cc

@@ -166,6 +166,7 @@ static bool addArgumentsAPTGet(std::vector<CommandLine::Args> &Args, char const
    else if (CmdMatches("files"))
    {
       addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);
+      addArg(0,"release-info","APT::Get::Files::ReleaseInfo", 0);
    }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
 	    CmdMatches("markauto", "unmarkauto")) // deprecated commands

cmdline/apt-get.cc

@@ -1628,15 +1628,34 @@ static bool DoFiles(CommandLine &CmdL)
       return false;
 
    std::string const Format = _config->Find("APT::Get::Files::Format");
+   bool const ReleaseInfo = _config->FindB("APT::Get::Files::ReleaseInfo", true);
    bool Filtered = CmdL.FileSize() > 1;
    for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
    {
       std::vector<IndexTarget> const targets = (*S)->GetIndexTargets();
       std::map<std::string, string> AddOptions;
-      AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+      if (ReleaseInfo)
+      {
+	 AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+	 pkgCache &Cache = *CacheFile.GetPkgCache();
+	 pkgCache::RlsFileIterator const RlsFile = (*S)->FindInCache(Cache, false);
+	 if (RlsFile.end())
+	    continue;
+#define APT_RELEASE(X,Y) if (RlsFile.Y() != NULL) AddOptions.insert(std::make_pair(X, RlsFile.Y()))
+	 APT_RELEASE("CODENAME", Codename);
+	 APT_RELEASE("SUITE", Archive);
+	 APT_RELEASE("VERSION", Version);
+	 APT_RELEASE("ORIGIN", Origin);
+	 APT_RELEASE("LABEL", Label);
+#undef APT_RELEASE
+      }
 
       for (std::vector<IndexTarget>::const_iterator T = targets.begin(); T != targets.end(); ++T)
       {
+	 std::string filename = T->Option(ReleaseInfo ? IndexTarget::EXISTING_FILENAME : IndexTarget::FILENAME);
+	 if (filename.empty())
+	    continue;
+
 	 std::ostringstream stanza;
 	 if (Filtered || Format.empty())
 	 {
@@ -1644,7 +1663,7 @@ static bool DoFiles(CommandLine &CmdL)
 	       << "ShortDesc: " << T->ShortDesc << "\n"
 	       << "Description: " << T->Description << "\n"
 	       << "URI: " << T->URI << "\n"
-	       << "Filename: " << T->Option(IndexTarget::FILENAME) << "\n"
+	       << "Filename: " << filename << "\n"
 	       << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
 	    for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
 	       stanza << format_key(O->first) << ": " << O->second << "\n";
@@ -1677,7 +1696,8 @@ static bool DoFiles(CommandLine &CmdL)
 	    cout << stanza.str();
 	 else
 	 {
-	    std::string out = T->Format(Format);
+	    std::string out = SubstVar(Format, "$(FILENAME)", filename);
+	    out = T->Format(out);
 	    for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
 	       out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
 	    cout << out << std::endl;

doc/acquire-additional-files.txt

@@ -18,12 +18,6 @@ might want to use and would therefore need to be downloaded as well
 This file describes the configuration scheme such a frontend can use to
 instruct the Acquire system to download those additional files.
 
-Note that you can't download files from other sources. It must be files
-in the same repository and below the Release file. The Release file must
-also contain hashes for the file itself as well as for the compressed
-file if wanted, otherwise a download isn't even tried!
-
-
 # The Configuration Stanza
 
 The Acquire system uses the same configuration settings to implement the
@@ -81,15 +75,17 @@ Additional optional properties:
 * Optional: The default value is 'true' and should be kept at this
   value.  If enabled the acquire system will skip the download if the
   file isn't mentioned in the Release file. Otherwise this is treated as
-  a hard error and the update process fails.
+  a hard error and the update process fails. Note that failures while
+  downloading (e.g. 404 or hash verification errors) are failures,
+  regardless of this setting.
 
 
-Note that the acquire system will automatically choose to download
-a compressed file if it is available and uncompress it for you, just as
-it will also use pdiff patching if provided by the repository and
-enabled by the user. You only have to ensure that the Release file
-contains the information about the compressed files/pdiffs to make this
-happen. NO properties have to be set to enable this.
+The acquire system will automatically choose to download a compressed
+file if it is available and uncompress it for you, just as it will also
+use pdiff patching if provided by the repository and enabled by the
+user. You only have to ensure that the Release file contains the
+information about the compressed files/pdiffs to make this happen.
+NO properties have to be set to enable this.
 
 # More examples
 
@@ -123,10 +119,12 @@ APT::Acquire::Targets {
 As seen in the examples, properties can contain placeholders filled in
 by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are
-printed literally.
+printed as-is.
 
 * $(SITE): An identifier of the site we access as seen in sources.list,
-  e.g. "http://example.org/debian" or "file:/path/to/a/repository".
+  e.g. "http://example.org/debian" or "file:/path/to/a/repository". You
+  can't use this field in {,flat}MetaKey, it is for description proposes
+  only.
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.
@@ -143,7 +141,7 @@ printed literally.
 
 Note that while more variables might exist in the implementation, these
 are to be considered undefined and their usage strongly discouraged. If
-you have a need for another variable contact us instead.
+you have a need for other variables contact us.
 
 # Accessing files
 
@@ -170,29 +168,42 @@ sources.lists (pkgSourceList), iterating over the metaIndex objects this
 creates and calling GetIndexTargets() on them. See the sourcecode of
 "apt-get files" for a complete example.
 
-Remarks on the available fields:
+Note that by default targets are not listed if they weren't downloaded.
+If you want to see all targets, you can use the --no-release-info, which
+also removes the Codename, Suite, Version, Origin, Label and Trusted
+fields from the output as these also display data which needs to be
+downloaded first and could hence be inaccurate [on the pro-side: This
+mode is faster as it doesn't require a valid binary cache to operate].
+The most notable difference perhaps is in the Filename field through: By
+default it indicates an existing file, potentially compressed (Hint:
+libapt users can use FileFd to open compressed files transparently). In
+the --no-release-info mode the indicated file doesn't need to exist and
+it will always refer to an uncompressed file, even if the index would be
+(or is) stored compressed.
+
+Remarks on fields only available in (default) --release-info mode:
+* Trusted: Denotes with a 'yes' or 'no' if the data in this file is
+  authenticated by a trustchain rooted in a trusted gpg key. You should
+  be careful with untrusted data and warn the user if you use it.
+* Codename, Suite, Version, Origin and Label are fields from the Release
+  file, are only present if they are present in the Release file and
+  contain the same data.
+
+Remarks on other available fields:
 * MetaKey, ShortDesc, Description, Site, Release: as defined
   by the configuration and described further above.
 * Created-By: configuration entity responsible for this target
 * Target-Of: type of the sources.list entry
 * URI, Repo-URI: avoid using. Contains potentially username/password.
   Prefer 'Site', especially for display.
-* Filename: The mentioned file doesn't need to exist, e.g. because the
-  file wasn't downloaded yet – or it does exist compressed. libapt users
-  are encouraged to use FileFd to open such files as it can handle
-  decompression transparently.
-* Trusted: As of the last 'apt-get update' call denoting if e.g. apt-get
-  would print the unauthenticated warning while installing packages
-  mentioned in this file (example assumes Packages file) [Not really
-  a property of the target itself, but of the metaIndex].
 * Optional: Decodes the option of the same name from the configuration.
   Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
 * Language, Architecture, Component: as defined further above, but with
   the catch that they might be missing if they don't effect the target
   (aka: They weren't used while evaluating the MetaKey template).
 
-Again, additional fields might be visible in certain implementation, but
-you should avoid using them and instead talk to us about a portable
+Again, additional fields might be visible in certain implementations,
+but you should avoid using them and instead talk to us about a portable
 implementation.
 
 # Multiple application requiring the same files

test/integration/test-apt-acquire-additional-files

@@ -37,6 +37,12 @@ APT::Acquire::Targets::deb::Contents {
 };
 EOF
 
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --no-release-info --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+# lets fake the existence of a compressed Contents file
+touch ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
+
 testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
 'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0