add by-hash sources.list option and document all of by-hash

This changes the semantics of the option (which is renamed too) to be a
yes/no value with the special additional value "force" as this allows
by-hash to be disabled even if the repository indicates it would be
supported and is more in line with our other yes/no options like pdiff
which disable themselves if no support can be detected.

The feature wasn't documented so far and hasn't reached a (un)stable
release yet, so changing it without trying too hard to keep
compatibility seems okay.

apt-pkg/acquire-item.cc

@@ -1247,7 +1247,7 @@ string pkgAcqMetaClearSig::Custom600Headers() const
    return Header;
 }
 									/*}}}*/
-bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,
+bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,		/*{{{*/
 	 pkgAcquire::MethodConfig const * const Cnf)
 {
    Item::VerifyDone(Message, Cnf);
@@ -1257,6 +1257,7 @@ bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,
 
    return true;
 }
+									/*}}}*/
 // pkgAcqMetaClearSig::Done - We got a file				/*{{{*/
 void pkgAcqMetaClearSig::Done(std::string const &Message,
                               HashStringList const &Hashes,
@@ -2480,13 +2481,9 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
 // AcqIndex::AdjustForByHash - modify URI for by-hash support		/*{{{*/
 void pkgAcqIndex::InitByHashIfNeeded()
 {
-   // TODO:
-   //  - (maybe?) add support for by-hash into the sources.list as flag
-   //  - make apt-ftparchive generate the hashes (and expire?)
-   std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
-   if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
-      _config->FindB(HostKnob, false) == true ||
-      TransactionManager->MetaIndexParser->GetSupportsAcquireByHash())
+   std::string const useByHash = Target.Option(IndexTarget::BY_HASH);
+   if(useByHash == "force" || (StringToBool(useByHash) == true &&
+	    TransactionManager->MetaIndexParser->GetSupportsAcquireByHash()))
    {
       HashStringList const Hashes = GetExpectedHashes();
       if(Hashes.usable())

apt-pkg/deb/debmetaindex.cc

@@ -39,6 +39,7 @@ class APT_HIDDEN debReleaseIndexPrivate					/*{{{*/
       std::vector<std::string> Architectures;
       std::vector<std::string> Languages;
       bool UsePDiffs;
+      std::string UseByHash;
    };
 
    std::vector<debSectionEntry> DebEntries;
@@ -149,6 +150,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 	 bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex);
 	 bool const DefaultEnabled = APT_T_CONFIG_BOOL("DefaultEnabled", true);
 	 bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);
+	 std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);
 	 std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
 #undef APT_T_CONFIG_BOOL
 #undef APT_T_CONFIG_STR
@@ -245,6 +247,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 	       Options.insert(std::make_pair("TARGET_OF", Type));
 	       Options.insert(std::make_pair("CREATED_BY", *T));
 	       Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));
+	       Options.insert(std::make_pair("BY_HASH", UseByHash));
 	       Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
 	       Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
 	       Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry));
@@ -286,12 +289,12 @@ void debReleaseIndex::AddComponent(std::string const &sourcesEntry,	/*{{{*/
 	 std::vector<std::string> const &Targets,
 	 std::vector<std::string> const &Architectures,
 	 std::vector<std::string> Languages,
-	 bool const usePDiffs)
+	 bool const usePDiffs, std::string const &useByHash)
 {
    if (Languages.empty() == true)
       Languages.push_back("none");
    debReleaseIndexPrivate::debSectionEntry const entry = {
-      sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs
+      sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs, useByHash
    };
    if (isSrc)
       d->DebSrcEntries.push_back(entry);
@@ -822,6 +825,17 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	    UsePDiffs = StringToBool(opt->second);
       }
 
+      std::string UseByHash = _config->Find("APT::Acquire::By-Hash", "yes");
+      UseByHash = _config->Find("Acquire::By-Hash", UseByHash);
+      {
+	 std::string const host = ::URI(URI).Host;
+	 UseByHash = _config->Find("APT::Acquire::" + host + "::By-Hash", UseByHash);
+	 UseByHash = _config->Find("Acquire::" + host + "::By-Hash", UseByHash);
+	 std::map<std::string, std::string>::const_iterator const opt = Options.find("by-hash");
+	 if (opt != Options.end())
+	    UseByHash = opt->second;
+      }
+
       auto const entry = Options.find("sourceslist-entry");
       Deb->AddComponent(
 	    entry->second,
@@ -830,7 +844,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	    mytargets,
 	    parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()),
 	    parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)),
-	    UsePDiffs
+	    UsePDiffs,
+	    UseByHash
 	    );
 
       if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false ||

apt-pkg/deb/debmetaindex.h

@@ -64,7 +64,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
 	 std::vector<std::string> const &Targets,
 	 std::vector<std::string> const &Architectures,
 	 std::vector<std::string> Languages,
-	 bool const usePDiffs);
+	 bool const usePDiffs, std::string const &useByHash);
 };
 
 #endif

apt-pkg/indexfile.cc

@@ -148,6 +148,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(DEFAULTENABLED);
       APT_CASE(COMPRESSIONTYPES);
       APT_CASE(SOURCESENTRY);
+      APT_CASE(BY_HASH);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       case EXISTING_FILENAME:

apt-pkg/indexfile.h

@@ -89,6 +89,7 @@ class IndexTarget							/*{{{*/
       COMPRESSIONTYPES,
       DEFAULTENABLED,
       SOURCESENTRY,
+      BY_HASH,
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;

apt-pkg/sourcelist.cc

@@ -108,6 +108,7 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,	/*{{{*/
    mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
    mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
    mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
+   mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));
 
    for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
       if (Tags.Exists(m->first))

doc/acquire-additional-files.txt

@@ -114,6 +114,9 @@ aren't accidentally used by frontends:
   Defaults to the value of Acquire::PDiffs which is true by default.
   Can be overridden per-source by the sources.list option of the same
   name. See the documentation for both of these for details.
+* By-Hash: controls if apt will try to use an URI constructed from
+  a hashsum of the file to download. See the documentation for config
+  option Acquire::By-Hash and sources.list option By-Hash for details.
 * CompressionTypes: The default value is a space separated list of
   compression types supported by apt (see Acquire::CompressionTypes).
   You can set this option to prevent apt from downloading a compression

doc/apt.conf.5.xml

@@ -347,6 +347,16 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
 	 </para></listitem>
      </varlistentry>
 
+     <varlistentry><term><option>By-Hash</option></term>
+	 <listitem><para>Try to download indexes via an URI constructed from a
+	 hashsum of the expected file rather than downloaded via a well-known
+	 stable filename. True by default, but automatically disabled if the
+	 source indicates no support for it. Usage can be forced with the special
+	 value "force". Preferably, this can be set for specific &sources-list; entries
+	 or index files by using the <option>By-Hash</option> option there.
+	 </para></listitem>
+     </varlistentry>
+
      <varlistentry><term><option>Queue-Mode</option></term>
      <listitem><para>Queuing mode; <literal>Queue-Mode</literal> can be one of <literal>host</literal> or 
      <literal>access</literal> which determines how  APT parallelizes outgoing 

doc/sources.list.5.xml

@@ -239,6 +239,22 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [.
 		<literal>yes</literal>.
 	  </para></listitem>
 
+	  <listitem><para><option>By-Hash</option> (<option>by-hash</option>)
+		can have the value "yes", "no" or "force" and controls if APT
+		should try to acquire indexes via an URI constructed from a
+		hashsum of the expected file instead of using the well-known
+		stable filename of the index. Using this can avoid hashsum
+		mismatches, but requires a supporting mirror. The value
+		"yes"/"no" activates/disables the use of this feature if this
+		source indicates support for it, while "force" will enable the
+		feature regardless of what the source indicates.
+		Defaults to the value of the option of the same name for a
+		specific index file defined in the
+		<option>Acquire::IndexTargets</option> scope, which itself
+		defaults to the value of configuration option
+		<option>Acquire::By-Hash</option> which defaults to
+		<literal>yes</literal>.
+	  </para></listitem>
 
        </itemizedlist>
 

test/integration/test-apt-by-hash-update

@@ -5,8 +5,9 @@ TESTDIR=$(readlink -f $(dirname $0))
 . $TESTDIR/framework
 
 setupenvironment
-configarchitecture "i386"
+configarchitecture 'i386'
 confighashes 'SHA512'
+configcompression '.' 'gz'
 
 insertpackage 'unstable' 'foo' 'all' '1.0'
 
@@ -24,29 +25,47 @@ mkdir -p aptarchive/dists/unstable/main/source/by-hash/SHA512
      ln -s ../../Sources.gz  $(sha512sum ../../Sources.gz|cut -f1 -d' ') 
 )
 
-# we moved the Packages file away, normal update won't work
-testfailure aptget update
-
-# ensure we do not know about "foo"
-testfailureequal "Reading package lists...
+ensureitsbroken() {
+	rm -rf rootdir/var/lib/apt/lists
+	# we moved the Packages file away, normal update won't work
+	testfailure aptget update "$@"
+	# ensure we do not know about "foo"
+	testfailureequal "Reading package lists...
 Building dependency tree...
 E: Unable to locate package foo" aptget install -q -s foo
-
-# ensure we can apt-get update by hash
-testsuccess aptget update -o APT::Acquire::By-Hash=1 -o Acquire::Languages=none
+}
+ensureitsbroken
+ensureitsbroken -o Acquire::By-Hash=1
 
 ensureitworks() {
+	rm -rf rootdir/var/lib/apt/lists
+	testsuccess aptget update -o Acquire::Languages=none "$@"
 	testsuccessequal "Inst foo (1.0 unstable [all])
 Conf foo (1.0 unstable [all])" aptget install -qq -s foo
 }
-ensureitworks
+msgmsg 'Test by-hash via' 'config option'
+ensureitworks -o Acquire::By-Hash=force
 
+msgmsg 'Test by-hash via' 'release option'
+cp -a aptarchive/dists aptarchive/dists.bak
 # add magic string to Release file ...
-MAGIC="Acquire-By-Hash: true"
-sed -i "s#Suite: unstable#Suite: unstable\n$MAGIC#" aptarchive/dists/unstable/Release
+sed -i '/^Suite: / a \
+Acquire-By-Hash: yes' aptarchive/dists/unstable/Release
 signreleasefiles
-# ... and verify that it fetches by hash now
-rm -rf rootdir/var/lib/apt/lists
-testsuccess aptget update -o Acquire::Languages=none
+ensureitworks
+ensureitsbroken -o Acquire::By-Hash=0
+
+msgmsg 'Test by-hash via' 'sources option'
+sed -i "s#^\(deb\(-src\)\?\) #\1 [by-hash=yes] #" rootdir/etc/apt/sources.list.d/*
+ensureitworks
+#ensureitsbroken -o Acquire::By-Hash=0
+
+rm -rf aptarchive/dists
+cp -a aptarchive/dists.bak aptarchive/dists
+#ensureitworks -o Acquire::By-Hash=force
+ensureitsbroken -o Acquire::By-Hash=1
+ensureitsbroken -o Acquire::By-Hash=0
 
+sed -i "s#^\(deb\(-src\)\?\) \[by-hash=yes\] #\1 [by-hash=force] #" rootdir/etc/apt/sources.list.d/*
 ensureitworks
+#ensureitsbroken -o Acquire::By-Hash=0