allow individual targets to be kept compressed

There is an option to keep all targets (Packages, Sources, …) compressed
for a while now, but the all-or-nothing approach is a bit limited for
our purposes with additional targets as some of them are very big
(Contents) and rarely used in comparison, so keeping them compressed by
default can make sense, while others are still unpacked.

Most interesting is the copy-change maybe: Copy is used by the acquire
system as an uncompressor and it is hence expected that it returns the
hashes for the "output", not the input. Now, in the case of keeping a
file compressed, the output is never written to disk, but generated in
memory and we should still validated it, so for compressed files copy is
expected to return the hashes of the uncompressed file. We used to use
the config option to enable on-the-fly decompress in the method, but in
reality copy is never used in a way where it shouldn't decompress a
compressed file to get its hashes, so we can save us the trouble of
sending this information to the method and just do it always.

apt-pkg/acquire-item.cc

@@ -80,18 +80,18 @@ static std::string GetFinalFileNameFromURI(std::string const &uri)	/*{{{*/
    return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
 }
 									/*}}}*/
-static std::string GetCompressedFileName(std::string const &URI, std::string const &Name, std::string const &Ext) /*{{{*/
+static std::string GetCompressedFileName(IndexTarget const &Target, std::string const &Name, std::string const &Ext) /*{{{*/
 {
    if (Ext.empty() || Ext == "uncompressed")
       return Name;
 
    // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
    // file when its doing the indexcopy
-   if (URI.substr(0,6) == "cdrom:")
+   if (Target.URI.substr(0,6) == "cdrom:")
       return Name;
 
    // adjust DestFile if its compressed on disk
-   if (_config->FindB("Acquire::GzipIndexes",false) == true)
+   if (Target.KeepCompressed == true)
       return Name + '.' + Ext;
    return Name;
 }
@@ -269,7 +269,7 @@ std::string pkgAcqDiffIndex::GetFinalFilename() const
 std::string pkgAcqIndex::GetFinalFilename() const
 {
    std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
-   return GetCompressedFileName(Target.URI, FinalFile, CurrentCompressionExtension);
+   return GetCompressedFileName(Target, FinalFile, CurrentCompressionExtension);
 }
 std::string pkgAcqMetaSig::GetFinalFilename() const
 {
@@ -2456,7 +2456,7 @@ void pkgAcqIndex::ReverifyAfterIMS()
 {
    // update destfile to *not* include the compression extension when doing
    // a reverify (as its uncompressed on disk already)
-   DestFile = GetCompressedFileName(Target.URI, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);
+   DestFile = GetCompressedFileName(Target, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);
 
    // copy FinalFile into partial/ so that we check the hash again
    string FinalFile = GetFinalFilename();
@@ -2532,8 +2532,8 @@ void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const
       return;
    }
 
-   // If we have compressed indexes enabled, queue for hash verification
-   if (_config->FindB("Acquire::GzipIndexes",false))
+   // If we want compressed indexes, just copy in place for hash verification
+   if (Target.KeepCompressed == true)
    {
       DestFile = GetPartialFileNameFromURI(Target.URI + '.' + CurrentCompressionExtension);
       EraseFileName = "";

apt-pkg/deb/debmetaindex.cc

@@ -123,6 +123,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
    std::string const Release = (Dist == "/") ? "" : Dist;
    std::string const Site = ::URI::ArchiveOnly(URI);
 
+   bool const GzipIndex = _config->FindB("Acquire::GzipIndexes", false);
    for (std::vector<debReleaseIndexPrivate::debSectionEntry>::const_iterator E = entries.begin(); E != entries.end(); ++E)
    {
       for (std::vector<std::string>::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T)
@@ -131,7 +132,8 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 	 std::string const tplMetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey");
 	 std::string const tplShortDesc = APT_T_CONFIG("ShortDescription");
 	 std::string const tplLongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
-	 bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true);
+	 bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::Optional", true);
+	 bool const KeepCompressed = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::KeepCompressed", GzipIndex);
 #undef APT_T_CONFIG
 	 if (tplMetaKey.empty())
 	    continue;
@@ -173,6 +175,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
 		     LongDesc,
 		     Options.find("BASE_URI")->second + MetaKey,
 		     IsOptional,
+		     KeepCompressed,
 		     Options
 		     );
 	       IndexTargets.push_back(Target);
@@ -413,7 +416,7 @@ bool debReleaseIndex::parseSumData(const char *&Start, const char *End,	/*{{{*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/
 {
    std::vector<IndexTarget> const targets = GetIndexTargets();
-#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map<std::string,std::string>())
+#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, false, std::map<std::string,std::string>())
    pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner,
 	 APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"),
 	 targets, this);

apt-pkg/indexfile.cc

@@ -115,8 +115,9 @@ APT_DEPRECATED std::string pkgIndexFile::LanguageCode() {
 // IndexTarget - Constructor						/*{{{*/
 IndexTarget::IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
       std::string const &LongDesc, std::string const &URI, bool const IsOptional,
-      std::map<std::string, std::string> const &Options) :
-   URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey), IsOptional(IsOptional), Options(Options)
+      bool const KeepCompressed, std::map<std::string, std::string> const &Options) :
+   URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey),
+   IsOptional(IsOptional), KeepCompressed(KeepCompressed), Options(Options)
 {
 }
 									/*}}}*/

apt-pkg/indexfile.h

@@ -60,6 +60,9 @@ class IndexTarget							/*{{{*/
    /** \brief Is it okay if the file isn't found in the meta index */
    bool IsOptional;
 
+   /** \brief If the file is downloaded compressed, do not unpack it */
+   bool KeepCompressed;
+
    /** \brief options with which this target was created
        Prefer the usage of #Option if at all possible.
        Beware: Not all of these options are intended for public use */
@@ -67,7 +70,7 @@ class IndexTarget							/*{{{*/
 
    IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
 	 std::string const &LongDesc, std::string const &URI, bool const IsOptional,
-	 std::map<std::string, std::string> const &Options);
+	 bool const KeepCompressed, std::map<std::string, std::string> const &Options);
 
    enum OptionKeys {
       SITE,

methods/copy.cc

@@ -38,11 +38,7 @@ class CopyMethod : public pkgAcqMethod
 void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
 {
    Hashes Hash(Itm->ExpectedHashes);
-   FileFd::CompressMode CompressMode = FileFd::None;
-   if (_config->FindB("Acquire::GzipIndexes", false) == true)
-      CompressMode = FileFd::Extension;
-
-   FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode);
+   FileFd Fd(Res.Filename, FileFd::ReadOnly, FileFd::Extension);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
 }
@@ -53,7 +49,7 @@ void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
 bool CopyMethod::Fetch(FetchItem *Itm)
 {
    // this ensures that relative paths work in copy
-   std::string File = Itm->Uri.substr(Itm->Uri.find(':')+1);
+   std::string const File = Itm->Uri.substr(Itm->Uri.find(':')+1);
 
    // Stat the file and send a start message
    struct stat Buf;

test/integration/test-apt-acquire-additional-files

@@ -57,7 +57,23 @@ testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents
 testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'
 
-# no automatic uncompress based on the name please,
+rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+
+# if we asked for keeping it compressed, keep it
+echo 'APT::Acquire::Targets::deb::Contents::KeepCompressed "true";' >> rootdir/etc/apt/apt.conf.d/content-target.conf
+testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
+Get:2 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Reading package lists..." aptget update
+
+testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
+testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+
+rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+
+# and no automatic uncompress based on the name please,
 # only if we downloaded a compressed file, but target was uncompressed
 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
 APT::Acquire::Targets::deb::Contents {

test/integration/test-compressed-indexes

@@ -60,7 +60,7 @@ testrun() {
 	msgtest 'Check if package is downloadable'
 	cd downloaded
 	testsuccess --nomsg aptget download testpkg
-	msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
+	msgtest 'deb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
 	rm -f testpkg_1.0_i386.deb
 	cd - >/dev/null
 	testsuccessequal 'Reading package lists...