store Release files data in the Cache

We used to read the Release file for each Packages file and store the
data in the PackageFile struct even through potentially many Packages
(and Translation-*) files could use the same data. The point of the
exercise isn't the duplicated data through. Having the Release files as
first-class citizens in the Cache allows us to properly track their
state as well as allows us to use the information also for files which
aren't in the cache, but where we know to which Release file they
belong (Sources are an example for this).

This modifies the pkgCache structs, especially the PackagesFile struct
which depending on how libapt users access the data in these structs can
mean huge breakage or no visible change. As a single data point:
aptitude seems to be fine with this. Even if there is breakage it is
trivial to fix in a backportable way while avoiding breakage for
everyone would be a huge pain for us.

Note that not all PackageFile structs have a corresponding ReleaseFile.
In particular the dpkg/status file as well as *.deb files have not. As
these have only a Archive property need, the Component property takes
over this duty and the ReleaseFile remains zero. This is also the reason
why it isn't needed nor particularily recommended to change from
PackagesFile to ReleaseFile blindly. Sticking with the earlier is
usually the better option.

apt-pkg/acquire-item.cc

@@ -2568,7 +2568,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour
    // Skip not source sources, they do not have file fields.
    for (; Vf.end() == false; ++Vf)
    {
-      if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
+      if (Vf.File().Flagged(pkgCache::Flag::NotSource))
 	 continue;
       break;
    }
@@ -2636,14 +2636,14 @@ bool pkgAcqArchive::QueueNext()
    {
       pkgCache::PkgFileIterator const PkgF = Vf.File();
       // Ignore not source sources
-      if ((PkgF->Flags & pkgCache::Flag::NotSource) != 0)
+      if (PkgF.Flagged(pkgCache::Flag::NotSource))
 	 continue;
 
       // Try to cross match against the source list
       pkgIndexFile *Index;
       if (Sources->FindIndex(PkgF, Index) == false)
 	    continue;
-      LocalSource = (PkgF->Flags & pkgCache::Flag::LocalSource) == pkgCache::Flag::LocalSource;
+      LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);
 
       // only try to get a trusted package from another source if that source
       // is also trusted

apt-pkg/cacheiterators.h

@@ -367,27 +367,61 @@ class pkgCache::PrvIterator : public Iterator<Provides, PrvIterator> {
 	}
 };
 									/*}}}*/
-// Package file								/*{{{*/
-class pkgCache::PkgFileIterator : public Iterator<PackageFile, PkgFileIterator> {
+// Release file								/*{{{*/
+class pkgCache::RlsFileIterator : public Iterator<ReleaseFile, RlsFileIterator> {
 	protected:
-	inline PackageFile* OwnerPointer() const {
-		return (Owner != 0) ? Owner->PkgFileP : 0;
+	inline ReleaseFile* OwnerPointer() const {
+		return (Owner != 0) ? Owner->RlsFileP : 0;
 	}
 
 	public:
 	// Iteration
-	void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;}
+	void operator ++(int) {if (S != Owner->RlsFileP) S = Owner->RlsFileP + S->NextFile;}
 	inline void operator ++() {operator ++(0);}
 
 	// Accessors
 	inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;}
 	inline const char *Archive() const {return S->Archive == 0?0:Owner->StrP + S->Archive;}
-	inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;}
 	inline const char *Version() const {return S->Version == 0?0:Owner->StrP + S->Version;}
 	inline const char *Origin() const {return S->Origin == 0?0:Owner->StrP + S->Origin;}
 	inline const char *Codename() const {return S->Codename ==0?0:Owner->StrP + S->Codename;}
 	inline const char *Label() const {return S->Label == 0?0:Owner->StrP + S->Label;}
 	inline const char *Site() const {return S->Site == 0?0:Owner->StrP + S->Site;}
+	inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return (S->Flags & flag) == flag; }
+
+	bool IsOk();
+	std::string RelStr();
+
+	// Constructors
+	inline RlsFileIterator() : Iterator<ReleaseFile, RlsFileIterator>() {}
+	inline RlsFileIterator(pkgCache &Owner) : Iterator<ReleaseFile, RlsFileIterator>(Owner, Owner.RlsFileP) {}
+	inline RlsFileIterator(pkgCache &Owner,ReleaseFile *Trg) : Iterator<ReleaseFile, RlsFileIterator>(Owner, Trg) {}
+};
+									/*}}}*/
+// Package file								/*{{{*/
+class pkgCache::PkgFileIterator : public Iterator<PackageFile, PkgFileIterator> {
+	protected:
+	inline PackageFile* OwnerPointer() const {
+		return (Owner != 0) ? Owner->PkgFileP : 0;
+	}
+
+	public:
+	// Iteration
+	void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;}
+	inline void operator ++() {operator ++(0);}
+
+	// Accessors
+	inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;}
+	inline pkgCache::RlsFileIterator ReleaseFile() const {return RlsFileIterator(*Owner, Owner->RlsFileP + S->Release);}
+	inline const char *Archive() const {return S->Release == 0 ? Component() : ReleaseFile().Archive();}
+	inline const char *Version() const {return S->Release == 0 ? NULL : ReleaseFile().Version();}
+	inline const char *Origin() const {return S->Release == 0 ? NULL : ReleaseFile().Origin();}
+	inline const char *Codename() const {return S->Release == 0 ? NULL : ReleaseFile().Codename();}
+	inline const char *Label() const {return S->Release == 0 ? NULL : ReleaseFile().Label();}
+	inline const char *Site() const {return S->Release == 0 ? NULL : ReleaseFile().Site();}
+	inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return S->Release== 0 ? false : ReleaseFile().Flagged(flag);}
+	inline bool Flagged(pkgCache::Flag::PkgFFlags const flag) const {return (S->Flags & flag) == flag;}
+	inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;}
 	inline const char *Architecture() const {return S->Architecture == 0?0:Owner->StrP + S->Architecture;}
 	inline const char *IndexType() const {return S->IndexType == 0?0:Owner->StrP + S->IndexType;}
 

apt-pkg/clean.cc

@@ -106,7 +106,7 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache)
 		 J.end() == false; ++J)
 	    {
 	       if (CleanInstalled == true &&
-		   (J.File()->Flags & pkgCache::Flag::NotSource) != 0)
+		   J.File().Flagged(pkgCache::Flag::NotSource))
 		  continue;
 	       IsFetchable = true;
 	       break;

apt-pkg/deb/debindexfile.cc

@@ -128,7 +128,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    if (Dist.empty())
       Dist = "/";
    ::URI Tmp(URI);
-   if (Gen.SelectFile(PackageFile,Tmp.Host,*this) == false)
+   if (Gen.SelectFile(PackageFile, *this, Target.Option(IndexTarget::COMPONENT)) == false)
       return _error->Error("Problem with SelectFile %s",PackageFile.c_str());
 
    // Store the IMS information
@@ -136,31 +136,10 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    pkgCacheGenerator::Dynamic<pkgCache::PkgFileIterator> DynFile(File);
    File->Size = Pkg.FileSize();
    File->mtime = Pkg.ModificationTime();
-   
+
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeList %s",PackageFile.c_str());
 
-   // Check the release file
-   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease");
-   bool releaseExists = false;
-   if (FileExists(ReleaseFile) == true)
-      releaseExists = true;
-   else
-      ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
-
-   if (releaseExists == true || FileExists(ReleaseFile) == true)
-   {
-      FileFd Rel;
-      // Beware: The 'Release' file might be clearsigned in case the
-      // signature for an 'InRelease' file couldn't be checked
-      if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
-	 return false;
-
-      if (_error->PendingError() == true)
-	 return false;
-      Parser.LoadReleaseInfo(File, Rel, Target.Option(IndexTarget::COMPONENT));
-   }
-   
    return true;
 }
 									/*}}}*/
@@ -221,17 +200,17 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
      debTranslationsParser TransParser(&Trans);
      if (_error->PendingError() == true)
        return false;
-     
+
      if (Prog != NULL)
 	Prog->SubProgress(0, Target.Description);
-     if (Gen.SelectFile(TranslationFile,string(),*this) == false)
+     if (Gen.SelectFile(TranslationFile, *this, Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource) == false)
        return _error->Error("Problem with SelectFile %s",TranslationFile.c_str());
 
      // Store the IMS information
      pkgCache::PkgFileIterator TransFile = Gen.GetCurFile();
      TransFile->Size = Trans.FileSize();
      TransFile->mtime = Trans.ModificationTime();
-   
+
      if (Gen.MergeList(TransParser) == false)
        return _error->Error("Problem with MergeList %s",TranslationFile.c_str());
    }
@@ -305,18 +284,17 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
    if (Prog != NULL)
       Prog->SubProgress(0,File);
-   if (Gen.SelectFile(File,string(),*this,pkgCache::Flag::NotSource) == false)
+   if (Gen.SelectFile(File, *this, "now", pkgCache::Flag::NotSource) == false)
       return _error->Error("Problem with SelectFile %s",File.c_str());
 
    // Store the IMS information
    pkgCache::PkgFileIterator CFile = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic<pkgCache::PkgFileIterator> DynFile(CFile);
    CFile->Size = Pkg.FileSize();
    CFile->mtime = Pkg.ModificationTime();
-   map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "now");
-   CFile->Archive = storage;
-   
+
    if (Gen.MergeList(Parser) == false)
-      return _error->Error("Problem with MergeList %s",File.c_str());   
+      return _error->Error("Problem with MergeList %s",File.c_str());
    return true;
 }
 									/*}}}*/
@@ -431,7 +409,7 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const
 
    // and give it to the list parser
    debDebFileParser Parser(DebControl, DebFile);
-   if(Gen.SelectFile(DebFile, "local", *this, pkgCache::Flag::LocalSource) == false)
+   if(Gen.SelectFile(DebFile, *this, "now", pkgCache::Flag::LocalSource) == false)
       return _error->Error("Problem with SelectFile %s", DebFile.c_str());
 
    pkgCache::PkgFileIterator File = Gen.GetCurFile();

apt-pkg/deb/debindexfile.h

@@ -45,7 +45,6 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile
    virtual bool HasPackages() const {return true;};
    virtual unsigned long Size() const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
-   bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog, unsigned long const Flag) const;
    virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const;
 
    debStatusIndex(std::string File);

apt-pkg/deb/deblistparser.cc

@@ -951,43 +951,6 @@ bool debListParser::Step()
    return false;
 }
 									/*}}}*/
-// ListParser::LoadReleaseInfo - Load the release information		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,
-				    FileFd &File, string component)
-{
-   // apt-secure does no longer download individual (per-section) Release
-   // file. to provide Component pinning we use the section name now
-   map_stringitem_t const storage = StoreString(pkgCacheGenerator::MIXED, component);
-   FileI->Component = storage;
-
-   pkgTagFile TagFile(&File, File.Size());
-   pkgTagSection Section;
-   if (_error->PendingError() == true || TagFile.Step(Section) == false)
-      return false;
-
-   std::string data;
-   #define APT_INRELEASE(TYPE, TAG, STORE) \
-   data = Section.FindS(TAG); \
-   if (data.empty() == false) \
-   { \
-      map_stringitem_t const storage = StoreString(pkgCacheGenerator::TYPE, data); \
-      STORE = storage; \
-   }
-   APT_INRELEASE(MIXED, "Suite", FileI->Archive)
-   APT_INRELEASE(MIXED, "Component", FileI->Component)
-   APT_INRELEASE(VERSIONNUMBER, "Version", FileI->Version)
-   APT_INRELEASE(MIXED, "Origin", FileI->Origin)
-   APT_INRELEASE(MIXED, "Codename", FileI->Codename)
-   APT_INRELEASE(MIXED, "Label", FileI->Label)
-   #undef APT_INRELEASE
-   Section.FindFlag("NotAutomatic", FileI->Flags, pkgCache::Flag::NotAutomatic);
-   Section.FindFlag("ButAutomaticUpgrades", FileI->Flags, pkgCache::Flag::ButAutomaticUpgrades);
-
-   return !_error->PendingError();
-}
-									/*}}}*/
 // ListParser::GetPrio - Convert the priority from a string		/*{{{*/
 // ---------------------------------------------------------------------
 /* */

apt-pkg/deb/deblistparser.h

@@ -80,9 +80,9 @@ class APT_HIDDEN debListParser : public pkgCacheGenerator::ListParser
    virtual map_filesize_t Size() {return Section.size();};
 
    virtual bool Step();
-   
-   bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File,
-			std::string section);
+
+   bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File,
+			std::string const &section);
 
    APT_PUBLIC static const char *ParseDepends(const char *Start,const char *Stop,
 	 std::string &Package,std::string &Ver,unsigned int &Op);

apt-pkg/deb/debmetaindex.cc

@@ -1,5 +1,6 @@
 #include <config.h>
 
+#include <apt-pkg/error.h>
 #include <apt-pkg/debmetaindex.h>
 #include <apt-pkg/debindexfile.h>
 #include <apt-pkg/strutl.h>
@@ -10,10 +11,12 @@
 #include <apt-pkg/indexrecords.h>
 #include <apt-pkg/sourcelist.h>
 #include <apt-pkg/hashes.h>
-#include <apt-pkg/macros.h>
 #include <apt-pkg/metaindex.h>
+#include <apt-pkg/pkgcachegen.h>
+#include <apt-pkg/tagfile.h>
+#include <apt-pkg/gpgv.h>
+#include <apt-pkg/macros.h>
 
-#include <string.h>
 #include <map>
 #include <string>
 #include <utility>
@@ -21,6 +24,11 @@
 #include <set>
 #include <algorithm>
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <string.h>
+
 using namespace std;
 
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
@@ -37,6 +45,10 @@ string debReleaseIndex::MetaIndexInfo(const char *Type) const
    Info += Type;
    return Info;
 }
+std::string debReleaseIndex::Describe() const
+{
+   return MetaIndexInfo("Release");
+}
 
 string debReleaseIndex::MetaIndexFile(const char *Type) const
 {
@@ -339,6 +351,114 @@ debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section,
 		bool const &IsSrc): Section(Section), IsSrc(IsSrc)
 {}
 
+static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile)
+{
+   ReleaseFile = That->MetaIndexFile("InRelease");
+   bool releaseExists = false;
+   if (FileExists(ReleaseFile) == true)
+      releaseExists = true;
+   else
+   {
+      ReleaseFile = That->MetaIndexFile("Release");
+      if (FileExists(ReleaseFile))
+	 releaseExists = true;
+   }
+   return releaseExists;
+}
+
+bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   ::URI Tmp(URI);
+   if (Gen.SelectReleaseFile(ReleaseFile, Tmp.Host) == false)
+      return _error->Error("Problem with SelectReleaseFile %s", ReleaseFile.c_str());
+
+   if (releaseExists == false)
+      return true;
+
+   FileFd Rel;
+   // Beware: The 'Release' file might be clearsigned in case the
+   // signature for an 'InRelease' file couldn't be checked
+   if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
+      return false;
+   if (_error->PendingError() == true)
+      return false;
+
+   // Store the IMS information
+   pkgCache::RlsFileIterator File = Gen.GetCurRlsFile();
+   pkgCacheGenerator::Dynamic<pkgCache::RlsFileIterator> DynFile(File);
+   // Rel can't be used as this is potentially a temporary file
+   struct stat Buf;
+   if (stat(ReleaseFile.c_str(), &Buf) != 0)
+      return _error->Errno("fstat", "Unable to stat file %s", ReleaseFile.c_str());
+   File->Size = Buf.st_size;
+   File->mtime = Buf.st_mtime;
+
+   pkgTagFile TagFile(&Rel, Rel.Size());
+   pkgTagSection Section;
+   if (_error->PendingError() == true || TagFile.Step(Section) == false)
+      return false;
+
+   std::string data;
+   #define APT_INRELEASE(TYPE, TAG, STORE) \
+   data = Section.FindS(TAG); \
+   if (data.empty() == false) \
+   { \
+      map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::TYPE, data); \
+      STORE = storage; \
+   }
+   APT_INRELEASE(MIXED, "Suite", File->Archive)
+   APT_INRELEASE(VERSIONNUMBER, "Version", File->Version)
+   APT_INRELEASE(MIXED, "Origin", File->Origin)
+   APT_INRELEASE(MIXED, "Codename", File->Codename)
+   APT_INRELEASE(MIXED, "Label", File->Label)
+   #undef APT_INRELEASE
+   Section.FindFlag("NotAutomatic", File->Flags, pkgCache::Flag::NotAutomatic);
+   Section.FindFlag("ButAutomaticUpgrades", File->Flags, pkgCache::Flag::ButAutomaticUpgrades);
+
+   return !_error->PendingError();
+}
+									/*}}}*/
+// ReleaseIndex::FindInCache - Find this index				/*{{{*/
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   pkgCache::RlsFileIterator File = Cache.RlsFileBegin();
+   for (; File.end() == false; ++File)
+   {
+       if (File->FileName == 0 || ReleaseFile != File.FileName())
+	 continue;
+
+       // empty means the file does not exist by "design"
+       if (releaseExists == false && File->Size == 0)
+	  return File;
+
+      struct stat St;
+      if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - stat failed on " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - size (" << St.st_size << " <> " << File->Size
+			<< ") or mtime (" << St.st_mtime << " <> " << File->mtime
+			<< ") doesn't match for " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      return File;
+   }
+
+   return File;
+}
+									/*}}}*/
+
 class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type
 {
    protected:

apt-pkg/deb/debmetaindex.h

@@ -20,6 +20,8 @@ class pkgAcquire;
 class pkgIndexFile;
 class debDebPkgFileIndex;
 class IndexTarget;
+class pkgCacheGenerator;
+class OpProgress;
 
 class APT_HIDDEN debReleaseIndex : public metaIndex {
    public:
@@ -48,6 +50,10 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const;
    virtual std::vector<IndexTarget> GetIndexTargets() const;
 
+   virtual std::string Describe() const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
+
    std::string MetaIndexInfo(const char *Type) const;
    std::string MetaIndexFile(const char *Types) const;
    std::string MetaIndexURI(const char *Type) const;

apt-pkg/deb/dpkgpm.cc

@@ -205,8 +205,10 @@ pkgCache::VerIterator FindNowVersion(const pkgCache::PkgIterator &Pkg)
    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
       for (pkgCache::VerFileIterator Vf = Ver.FileList(); Vf.end() == false; ++Vf)
 	 for (pkgCache::PkgFileIterator F = Vf.File(); F.end() == false; ++F)
-	    if (F->Archive != 0 && strcmp(F.Archive(), "now") == 0)
+	 {
+	    if (F.Archive() != 0 && strcmp(F.Archive(), "now") == 0)
 	       return Ver;
+	 }
    return Ver;
 }
 									/*}}}*/

apt-pkg/depcache.cc

@@ -1685,13 +1685,13 @@ pkgCache::VerIterator pkgDepCache::Policy::GetCandidateVer(PkgIterator const &Pk
       
       for (VerFileIterator J = I.FileList(); J.end() == false; ++J)
       {
-	 if ((J.File()->Flags & Flag::NotSource) != 0)
+	 if (J.File().Flagged(Flag::NotSource))
 	    continue;
 
 	 /* Stash the highest version of a not-automatic source, we use it
 	    if there is nothing better */
-	 if ((J.File()->Flags & Flag::NotAutomatic) != 0 ||
-	     (J.File()->Flags & Flag::ButAutomaticUpgrades) != 0)
+	 if (J.File().Flagged(Flag::NotAutomatic) ||
+	     J.File().Flagged(Flag::ButAutomaticUpgrades))
 	 {
 	    if (Last.end() == true)
 	       Last = I;

apt-pkg/edsp.cc

@@ -129,7 +129,7 @@ void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgI
       signed short const p = Cache.GetPolicy().GetPriority(File);
       if (Pin < p)
 	 Pin = p;
-      if ((File->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource) {
+      if (File.Flagged(pkgCache::Flag::NotSource) == false) {
 	 string Release = File.RelStr();
 	 if (!Release.empty())
 	    Releases.insert(Release);

apt-pkg/edsp/edspindexfile.cc

@@ -49,15 +49,14 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
    if (Prog != NULL)
       Prog->SubProgress(0,File);
-   if (Gen.SelectFile(File,std::string(),*this) == false)
+   if (Gen.SelectFile(File, *this, "edsp") == false)
       return _error->Error("Problem with SelectFile %s",File.c_str());
 
    // Store the IMS information
    pkgCache::PkgFileIterator CFile = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic<pkgCache::PkgFileIterator> DynFile(CFile);
    CFile->Size = Pkg.FileSize();
    CFile->mtime = Pkg.ModificationTime();
-   map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "edsp::scenario");
-   CFile->Archive = storage;
 
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeList %s",File.c_str());

apt-pkg/edsp/edsplistparser.cc

@@ -86,8 +86,8 @@ bool edspListParser::ParseStatus(pkgCache::PkgIterator &Pkg,
 }
 									/*}}}*/
 // ListParser::LoadReleaseInfo - Load the release information		/*{{{*/
-APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::PkgFileIterator & /*FileI*/,
-				    FileFd & /*File*/, std::string /*component*/)
+APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::RlsFileIterator & /*FileI*/,
+				    FileFd & /*File*/, std::string const &/*component*/)
 {
    return true;
 }

apt-pkg/edsp/edsplistparser.h

@@ -34,8 +34,8 @@ class APT_HIDDEN edspListParser : public debListParser
    virtual MD5SumValue Description_md5();
    virtual unsigned short VersionHash();
 
-   bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File,
-			std::string section);
+   bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File,
+			std::string const &section);
 
    edspListParser(FileFd *File, std::string const &Arch = "");
 

apt-pkg/metaindex.cc

@@ -1,4 +1,5 @@
 // Include Files                                                       /*{{{*/
+#include <apt-pkg/pkgcachegen.h>
 #include <apt-pkg/indexfile.h>
 #include <apt-pkg/metaindex.h>
 
@@ -22,6 +23,22 @@ std::string metaIndex::LocalFileName() const
 }
 #endif
 
+std::string metaIndex::Describe() const
+{
+   return "Release";
+}
+
+pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const
+{
+   return pkgCache::RlsFileIterator(Cache);
+}
+
+bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const
+{
+   return Gen.SelectReleaseFile("", "");
+}
+
+
 metaIndex::metaIndex(std::string const &URI, std::string const &Dist,
       char const * const Type)
 : Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(false)

apt-pkg/metaindex.h

@@ -23,6 +23,8 @@ using std::string;
 
 class pkgAcquire;
 class IndexTarget;
+class pkgCacheGenerator;
+class OpProgress;
 
 class metaIndex
 {
@@ -51,6 +53,10 @@ class metaIndex
    virtual std::vector<pkgIndexFile *> *GetIndexFiles() = 0;
    virtual bool IsTrusted() const = 0;
 
+   virtual std::string Describe() const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
+
    metaIndex(std::string const &URI, std::string const &Dist,
              char const * const Type);
    virtual ~metaIndex();

apt-pkg/pkgcache.cc

@@ -61,6 +61,7 @@ pkgCache::Header::Header()
    HeaderSz = sizeof(pkgCache::Header);
    GroupSz = sizeof(pkgCache::Group);
    PackageSz = sizeof(pkgCache::Package);
+   ReleaseFileSz = sizeof(pkgCache::ReleaseFile);
    PackageFileSz = sizeof(pkgCache::PackageFile);
    VersionSz = sizeof(pkgCache::Version);
    DescriptionSz = sizeof(pkgCache::Description);
@@ -74,6 +75,7 @@ pkgCache::Header::Header()
    VersionCount = 0;
    DescriptionCount = 0;
    DependsCount = 0;
+   ReleaseFileCount = 0;
    PackageFileCount = 0;
    VerFileCount = 0;
    DescFileCount = 0;
@@ -82,6 +84,7 @@ pkgCache::Header::Header()
    MaxDescFileSize = 0;
    
    FileList = 0;
+   RlsFileList = 0;
 #if APT_PKG_ABI < 413
    APT_IGNORE_DEPRECATED(StringList = 0;)
 #endif
@@ -102,6 +105,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const
    if (HeaderSz == Against.HeaderSz &&
        GroupSz == Against.GroupSz &&
        PackageSz == Against.PackageSz &&
+       ReleaseFileSz == Against.ReleaseFileSz &&
        PackageFileSz == Against.PackageFileSz &&
        VersionSz == Against.VersionSz &&
        DescriptionSz == Against.DescriptionSz &&
@@ -140,6 +144,7 @@ bool pkgCache::ReMap(bool const &Errorchecks)
    PkgP = (Package *)Map.Data();
    VerFileP = (VerFile *)Map.Data();
    DescFileP = (DescFile *)Map.Data();
+   RlsFileP = (ReleaseFile *)Map.Data();
    PkgFileP = (PackageFile *)Map.Data();
    VerP = (Version *)Map.Data();
    DescP = (Description *)Map.Data();
@@ -814,7 +819,7 @@ APT_PURE bool pkgCache::VerIterator::Downloadable() const
 {
    VerFileIterator Files = FileList();
    for (; Files.end() == false; ++Files)
-      if ((Files.File()->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource)
+      if (Files.File().Flagged(pkgCache::Flag::NotSource) == false)
 	 return true;
    return false;
 }
@@ -828,7 +833,7 @@ APT_PURE bool pkgCache::VerIterator::Automatic() const
    VerFileIterator Files = FileList();
    for (; Files.end() == false; ++Files)
       // Do not check ButAutomaticUpgrades here as it is kind of automatic…
-      if ((Files.File()->Flags & pkgCache::Flag::NotAutomatic) != pkgCache::Flag::NotAutomatic)
+      if (Files.File().Flagged(pkgCache::Flag::NotAutomatic) == false)
 	 return true;
    return false;
 }
@@ -861,27 +866,27 @@ string pkgCache::VerIterator::RelStr() const
    for (pkgCache::VerFileIterator I = this->FileList(); I.end() == false; ++I)
    {
       // Do not print 'not source' entries'
-      pkgCache::PkgFileIterator File = I.File();
-      if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
+      pkgCache::PkgFileIterator const File = I.File();
+      if (File.Flagged(pkgCache::Flag::NotSource))
 	 continue;
 
       // See if we have already printed this out..
       bool Seen = false;
       for (pkgCache::VerFileIterator J = this->FileList(); I != J; ++J)
       {
-	 pkgCache::PkgFileIterator File2 = J.File();
-	 if (File2->Label == 0 || File->Label == 0)
+	 pkgCache::PkgFileIterator const File2 = J.File();
+	 if (File2.Label() == 0 || File.Label() == 0)
 	    continue;
 
 	 if (strcmp(File.Label(),File2.Label()) != 0)
 	    continue;
 	 
-	 if (File2->Version == File->Version)
+	 if (File2.Version() == File.Version())
 	 {
 	    Seen = true;
 	    break;
 	 }
-	 if (File2->Version == 0 || File->Version == 0)
+	 if (File2.Version() == 0 || File.Version() == 0)
 	    break;
 	 if (strcmp(File.Version(),File2.Version()) == 0)
 	    Seen = true;
@@ -895,12 +900,12 @@ string pkgCache::VerIterator::RelStr() const
       else
 	 First = false;
       
-      if (File->Label != 0)
+      if (File.Label() != 0)
 	 Res = Res + File.Label() + ':';
 
-      if (File->Archive != 0)
+      if (File.Archive() != 0)
       {
-	 if (File->Version == 0)
+	 if (File.Version() == 0)
 	    Res += File.Archive();
 	 else
 	    Res = Res + File.Version() + '/' +  File.Archive();
@@ -908,7 +913,7 @@ string pkgCache::VerIterator::RelStr() const
       else
       {
 	 // No release file, print the host name that this came from
-	 if (File->Site == 0 || File.Site()[0] == 0)
+	 if (File.Site() == 0 || File.Site()[0] == 0)
 	    Res += "localhost";
 	 else
 	    Res += File.Site();
@@ -931,12 +936,12 @@ const char * pkgCache::VerIterator::MultiArchType() const
    return "none";
 }
 									/*}}}*/
-// PkgFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
+// RlsFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
 // ---------------------------------------------------------------------
 /* This stats the file and compares its stats with the ones that were
-   stored during generation. Date checks should probably also be 
+   stored during generation. Date checks should probably also be
    included here. */
-bool pkgCache::PkgFileIterator::IsOk()
+bool pkgCache::RlsFileIterator::IsOk()
 {
    struct stat Buf;
    if (stat(FileName(),&Buf) != 0)
@@ -948,10 +953,8 @@ bool pkgCache::PkgFileIterator::IsOk()
    return true;
 }
 									/*}}}*/
-// PkgFileIterator::RelStr - Return the release string			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-string pkgCache::PkgFileIterator::RelStr()
+// RlsFileIterator::RelStr - Return the release string			/*{{{*/
+string pkgCache::RlsFileIterator::RelStr()
 {
    string Res;
    if (Version() != 0)
@@ -964,8 +967,40 @@ string pkgCache::PkgFileIterator::RelStr()
       Res = Res + (Res.empty() == true?"n=":",n=")  + Codename();
    if (Label() != 0)
       Res = Res + (Res.empty() == true?"l=":",l=")  + Label();
-   if (Component() != 0)
-      Res = Res + (Res.empty() == true?"c=":",c=")  + Component();
+   return Res;
+}
+									/*}}}*/
+// PkgFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
+// ---------------------------------------------------------------------
+/* This stats the file and compares its stats with the ones that were
+   stored during generation. Date checks should probably also be
+   included here. */
+bool pkgCache::PkgFileIterator::IsOk()
+{
+   struct stat Buf;
+   if (stat(FileName(),&Buf) != 0)
+      return false;
+
+   if (Buf.st_size != (signed)S->Size || Buf.st_mtime != S->mtime)
+      return false;
+
+   return true;
+}
+									/*}}}*/
+string pkgCache::PkgFileIterator::RelStr()				/*{{{*/
+{
+   std::string Res;
+   if (ReleaseFile() == 0)
+   {
+      if (Component() != 0)
+	 Res = Res + (Res.empty() == true?"a=":",a=")  + Component();
+   }
+   else
+   {
+      Res = ReleaseFile().RelStr();
+      if (Component() != 0)
+	 Res = Res + (Res.empty() == true?"c=":",c=")  + Component();
+   }
    if (Architecture() != 0)
       Res = Res + (Res.empty() == true?"b=":",b=")  + Architecture();
    return Res;

apt-pkg/pkgcache.h

@@ -133,6 +133,7 @@ class pkgCache								/*{{{*/
    struct Header;
    struct Group;
    struct Package;
+   struct ReleaseFile;
    struct PackageFile;
    struct Version;
    struct Description;
@@ -150,6 +151,7 @@ class pkgCache								/*{{{*/
    class DescIterator;
    class DepIterator;
    class PrvIterator;
+   class RlsFileIterator;
    class PkgFileIterator;
    class VerFileIterator;
    class DescFileIterator;
@@ -192,9 +194,11 @@ class pkgCache								/*{{{*/
       enum PkgFlags {Auto=(1<<0),Essential=(1<<3),Important=(1<<4)};
       enum PkgFFlags {
 	 NotSource=(1<<0), /*!< packages can't be fetched from here, e.g. dpkg/status file */
-	 NotAutomatic=(1<<1), /*!< archive has a default pin of 1 */
-	 ButAutomaticUpgrades=(1<<2), /*!< (together with the previous) archive has a default pin of 100 */
-	 LocalSource=(1<<3), /*!< local sources can't and will not be verified by hashes */
+	 LocalSource=(1<<1), /*!< local sources can't and will not be verified by hashes */
+      };
+      enum ReleaseFileFlags {
+	 NotAutomatic=(1<<0), /*!< archive has a default pin of 1 */
+	 ButAutomaticUpgrades=(1<<1), /*!< (together with the previous) archive has a default pin of 100 */
       };
    };
    
@@ -215,6 +219,7 @@ class pkgCache								/*{{{*/
    Package *PkgP;
    VerFile *VerFileP;
    DescFile *DescFileP;
+   ReleaseFile *RlsFileP;
    PackageFile *PkgFileP;
    Version *VerP;
    Description *DescP;
@@ -247,6 +252,8 @@ class pkgCache								/*{{{*/
    inline PkgIterator PkgEnd();
    inline PkgFileIterator FileBegin();
    inline PkgFileIterator FileEnd();
+   inline RlsFileIterator RlsFileBegin();
+   inline RlsFileIterator RlsFileEnd();
 
    inline bool MultiArchCache() const { return MultiArchEnabled; }
    inline char const * NativeArch();
@@ -296,6 +303,7 @@ struct pkgCache::Header
    unsigned short HeaderSz;
    unsigned short GroupSz;
    unsigned short PackageSz;
+   unsigned short ReleaseFileSz;
    unsigned short PackageFileSz;
    unsigned short VersionSz;
    unsigned short DescriptionSz;
@@ -314,6 +322,7 @@ struct pkgCache::Header
    map_id_t VersionCount;
    map_id_t DescriptionCount;
    map_id_t DependsCount;
+   map_fileid_t ReleaseFileCount;
    map_fileid_t PackageFileCount;
    map_fileid_t VerFileCount;
    map_fileid_t DescFileCount;
@@ -324,6 +333,9 @@ struct pkgCache::Header
        The PackageFile structures are singly linked lists that represent
        all package files that have been merged into the cache. */
    map_pointer_t FileList;
+   /** \brief index of the first ReleaseFile structure */
+   map_pointer_t RlsFileList;
+
 #if APT_PKG_ABI < 413
    APT_DEPRECATED map_pointer_t StringList;
 #endif
@@ -482,15 +494,14 @@ struct pkgCache::Package
    unsigned long Flags;
 };
 									/*}}}*/
-// Package File structure						/*{{{*/
-/** \brief stores information about the files used to generate the cache
+// Release File structure						/*{{{*/
+/** \brief stores information about the release files used to generate the cache
 
-    Package files are referenced by Version structures to be able to know
-    after the generation still from which Packages file includes this Version
-    as we need this information later on e.g. for pinning. */
-struct pkgCache::PackageFile
+    PackageFiles reference ReleaseFiles as we need to keep record of which
+    version belongs to which release e.g. for pinning. */
+struct pkgCache::ReleaseFile
 {
-   /** \brief physical disk file that this PackageFile represents */
+   /** \brief physical disk file that this ReleaseFile represents */
    map_stringitem_t FileName;
    /** \brief the release information
 
@@ -498,13 +509,47 @@ struct pkgCache::PackageFile
        release information means. */
    map_stringitem_t Archive;
    map_stringitem_t Codename;
-   map_stringitem_t Component;
    map_stringitem_t Version;
    map_stringitem_t Origin;
    map_stringitem_t Label;
-   map_stringitem_t Architecture;
    /** \brief The site the index file was fetched from */
    map_stringitem_t Site;
+
+   /** \brief Size of the file
+
+       Used together with the modification time as a
+       simple check to ensure that the Packages
+       file has not been altered since Cache generation. */
+   map_filesize_t Size;
+   /** \brief Modification time for the file */
+   time_t mtime;
+
+   /** @TODO document PackageFile::Flags */
+   unsigned long Flags;
+
+   // Linked list
+   /** \brief Link to the next ReleaseFile in the Cache */
+   map_pointer_t NextFile;
+   /** \brief unique sequel ID */
+   should_be_map_fileid_t ID;
+};
+									/*}}}*/
+// Package File structure						/*{{{*/
+/** \brief stores information about the files used to generate the cache
+
+    Package files are referenced by Version structures to be able to know
+    after the generation still from which Packages file includes this Version
+    as we need this information later on e.g. for pinning. */
+struct pkgCache::PackageFile
+{
+   /** \brief physical disk file that this PackageFile represents */
+   map_stringitem_t FileName;
+   /** \brief the release information */
+   map_pointer_t Release;
+
+   map_stringitem_t Component;
+   map_stringitem_t Architecture;
+
    /** \brief indicates what sort of index file this is
 
        @TODO enumerate at least the possible indexes */
@@ -744,6 +789,11 @@ inline pkgCache::PkgFileIterator pkgCache::FileBegin()
        {return PkgFileIterator(*this,PkgFileP + HeaderP->FileList);}
 inline pkgCache::PkgFileIterator pkgCache::FileEnd()
        {return PkgFileIterator(*this,PkgFileP);}
+inline pkgCache::RlsFileIterator pkgCache::RlsFileBegin()
+       {return RlsFileIterator(*this,RlsFileP + HeaderP->RlsFileList);}
+inline pkgCache::RlsFileIterator pkgCache::RlsFileEnd()
+       {return RlsFileIterator(*this,RlsFileP);}
+
 
 // Oh I wish for Real Name Space Support
 class pkgCache::Namespace						/*{{{*/
@@ -755,6 +805,7 @@ class pkgCache::Namespace						/*{{{*/
    typedef pkgCache::DescIterator DescIterator;
    typedef pkgCache::DepIterator DepIterator;
    typedef pkgCache::PrvIterator PrvIterator;
+   typedef pkgCache::RlsFileIterator RlsFileIterator;
    typedef pkgCache::PkgFileIterator PkgFileIterator;
    typedef pkgCache::VerFileIterator VerFileIterator;
    typedef pkgCache::Version Version;

apt-pkg/pkgcachegen.cc

@@ -54,10 +54,8 @@ using std::string;
 /* We set the dirty flag and make sure that is written to the disk */
 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
 		    Map(*pMap), Cache(pMap,false), Progress(Prog),
-		    FoundFileDeps(0)
+		     CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0)
 {
-   CurrentFile = 0;
-
    if (_error->PendingError() == true)
       return;
 
@@ -145,6 +143,7 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    Cache.ReMap(false);
 
    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
+   CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 
    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
@@ -167,6 +166,9 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
       (*i)->ReMap(oldMap, newMap);
+   for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
+	i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
+      (*i)->ReMap(oldMap, newMap);
 }									/*}}}*/
 // CacheGenerator::WriteStringInMap					/*{{{*/
 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
@@ -1072,13 +1074,47 @@ bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*
    return Hash == Ver->Hash;
 }
 									/*}}}*/
+// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to	/*{{{*/
+bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
+				   unsigned long Flags)
+{
+   if (File.empty() && Site.empty())
+   {
+      CurrentRlsFile = NULL;
+      return true;
+   }
+
+   // Get some space for the structure
+   map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
+   if (unlikely(idxFile == 0))
+      return false;
+   CurrentRlsFile = Cache.RlsFileP + idxFile;
+
+   // Fill it in
+   map_stringitem_t const idxFileName = WriteStringInMap(File);
+   map_stringitem_t const idxSite = StoreString(MIXED, Site);
+   if (unlikely(idxFileName == 0 || idxSite == 0))
+      return false;
+   CurrentRlsFile->FileName = idxFileName;
+   CurrentRlsFile->Site = idxSite;
+   CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
+   CurrentRlsFile->Flags = Flags;
+   CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
+   RlsFileName = File;
+   Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
+   Cache.HeaderP->ReleaseFileCount++;
+
+   return true;
+}
+									/*}}}*/
 // CacheGenerator::SelectFile - Select the current file being parsed	/*{{{*/
 // ---------------------------------------------------------------------
 /* This is used to select which file is to be associated with all newly
    added versions. The caller is responsible for setting the IMS fields. */
-bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
-				   const pkgIndexFile &Index,
-				   unsigned long Flags)
+bool pkgCacheGenerator::SelectFile(std::string const &File,
+				   pkgIndexFile const &Index,
+				   std::string const &Component,
+				   unsigned long const Flags)
 {
    // Get some space for the structure
    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
@@ -1088,18 +1124,24 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 
    // Fill it in
    map_stringitem_t const idxFileName = WriteStringInMap(File);
-   map_stringitem_t const idxSite = StoreString(MIXED, Site);
-   if (unlikely(idxFileName == 0 || idxSite == 0))
+   if (unlikely(idxFileName == 0))
       return false;
    CurrentFile->FileName = idxFileName;
-   CurrentFile->Site = idxSite;
    CurrentFile->NextFile = Cache.HeaderP->FileList;
-   CurrentFile->Flags = Flags;
    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
    if (unlikely(idxIndexType == 0))
       return false;
    CurrentFile->IndexType = idxIndexType;
+   map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
+   if (unlikely(component == 0))
+      return false;
+   CurrentFile->Component = component;
+   CurrentFile->Flags = Flags;
+   if (CurrentRlsFile != NULL)
+      CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
+   else
+      CurrentFile->Release = 0;
    PkgFileName = File;
    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
    Cache.HeaderP->PackageFileCount++;
@@ -1174,35 +1216,59 @@ static bool CheckValidity(const string &CacheFile,
       _error->Discard();
       return false;
    }
-   
+
+   SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
+   memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
+   std::vector<pkgIndexFile *> Files;
+   for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
+   {
+      if (Debug == true)
+	 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
+      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache);
+      if (RlsFile.end() == true)
+      {
+	 if (Debug == true)
+	    std::clog << "FindInCache returned end-Pointer" << std::endl;
+	 return false;
+      }
+
+      RlsVisited[RlsFile->ID] = true;
+      if (Debug == true)
+	 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
+
+      std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
+      for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
+	 if ((*j)->HasPackages())
+	    Files.push_back (*j);
+   }
+   for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
+      if (RlsVisited[I] == false)
+      {
+	 if (Debug == true)
+	    std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
+	 return false;
+      }
+
+   for (; Start != End; ++Start)
+      Files.push_back(*Start);
+
    /* Now we check every index file, see if it is in the cache,
       verify the IMS data and check that it is on the disk too.. */
    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
-   for (; Start != End; ++Start)
+   for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
    {
       if (Debug == true)
-	 std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
-      if ((*Start)->HasPackages() == false)
-      {
-         if (Debug == true)
-	    std::clog << "Has NO packages" << std::endl;
-	 continue;
-      }
-    
-      if ((*Start)->Exists() == false)
+	 std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
+      if ((*PkgFile)->Exists() == false)
       {
-#if 0 // mvo: we no longer give a message here (Default Sources spec)
-	 _error->WarningE("stat",_("Couldn't stat source package list %s"),
-			  (*Start)->Describe().c_str());
-#endif
          if (Debug == true)
 	    std::clog << "file doesn't exist" << std::endl;
 	 continue;
       }
 
       // FindInCache is also expected to do an IMS check.
-      pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
+      pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
       if (File.end() == true)
       {
 	 if (Debug == true)
@@ -1214,15 +1280,15 @@ static bool CheckValidity(const string &CacheFile,
       if (Debug == true)
 	 std::clog << "with ID " << File->ID << " is valid" << std::endl;
    }
-   
+
    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
       if (Visited[I] == false)
       {
 	 if (Debug == true)
-	    std::clog << "File with ID" << I << " wasn't visited" << std::endl;
+	    std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
 	 return false;
       }
-   
+
    if (_error->PendingError() == true)
    {
       if (Debug == true)
@@ -1243,9 +1309,20 @@ static bool CheckValidity(const string &CacheFile,
 // ---------------------------------------------------------------------
 /* Size is kind of an abstract notion that is only used for the progress
    meter */
-static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
+static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
 {
    map_filesize_t TotalSize = 0;
+   if (List !=  NULL)
+   {
+      for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+      {
+	 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
+	 for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
+	    if ((*j)->HasPackages() == true)
+	       TotalSize += (*j)->Size();
+      }
+   }
+
    for (; Start < End; ++Start)
    {
       if ((*Start)->HasPackages() == false)
@@ -1261,11 +1338,63 @@ static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
 static bool BuildCache(pkgCacheGenerator &Gen,
 		       OpProgress *Progress,
 		       map_filesize_t &CurrentSize,map_filesize_t TotalSize,
+		       pkgSourceList const * const List,
 		       FileIterator Start, FileIterator End)
 {
+   std::vector<pkgIndexFile *> Files;
+   bool const HasFileDeps = Gen.HasFileDeps();
+
+   if (List !=  NULL)
+   {
+      for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+      {
+	 if ((*i)->FindInCache(Gen.GetCache()).end() == false)
+	 {
+	    _error->Warning("Duplicate sources.list entry %s",
+		  (*i)->Describe().c_str());
+	    continue;
+	 }
+
+	 if ((*i)->Merge(Gen, Progress) == false)
+	    return false;
+
+	 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
+	 for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
+	 {
+	    if (HasFileDeps)
+	       Files.push_back(*I);
+
+	    if ((*I)->HasPackages() == false)
+	       continue;
+
+	    if ((*I)->Exists() == false)
+	       continue;
+
+	    if ((*I)->FindInCache(Gen.GetCache()).end() == false)
+	    {
+	       _error->Warning("Duplicate sources.list entry %s",
+		     (*I)->Describe().c_str());
+	       continue;
+	    }
+
+	    map_filesize_t Size = (*I)->Size();
+	    if (Progress != NULL)
+	       Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
+	    CurrentSize += Size;
+
+	    if ((*I)->Merge(Gen,Progress) == false)
+	       return false;
+	 }
+      }
+   }
+
+   Gen.SelectReleaseFile("", "");
    FileIterator I;
    for (I = Start; I != End; ++I)
    {
+      if (HasFileDeps)
+	 Files.push_back(*I);
+
       if ((*I)->HasPackages() == false)
 	 continue;
       
@@ -1288,13 +1417,13 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 	 return false;
    }   
 
-   if (Gen.HasFileDeps() == true)
+   if (HasFileDeps == true)
    {
       if (Progress != NULL)
 	 Progress->Done();
-      TotalSize = ComputeSize(Start, End);
+      TotalSize = ComputeSize(List, Start, End);
       CurrentSize = 0;
-      for (I = Start; I != End; ++I)
+      for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
       {
 	 map_filesize_t Size = (*I)->Size();
 	 if (Progress != NULL)
@@ -1339,6 +1468,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 
    std::vector<pkgIndexFile *> Files;
+   /*
    for (std::vector<metaIndex *>::const_iterator i = List.begin();
         i != List.end();
         ++i)
@@ -1349,8 +1479,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 	   ++j)
          Files.push_back (*j);
    }
-   
-   map_filesize_t const EndOfSource = Files.size();
+*/
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
@@ -1442,8 +1571,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    // Lets try the source cache.
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
-   if (CheckValidity(SrcCacheFile, List, Files.begin(),
-		     Files.begin()+EndOfSource) == true)
+   if (CheckValidity(SrcCacheFile, List, Files.end(),
+		     Files.end()) == true)
    {
       if (Debug == true)
 	 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
@@ -1455,28 +1584,28 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 				SCacheF.Size()) == false)
 	 return false;
 
-      TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
+      TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
 
       // Build the status cache
       pkgCacheGenerator Gen(Map.Get(),Progress);
       if (_error->PendingError() == true)
 	 return false;
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin()+EndOfSource,Files.end()) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
+		     Files.begin(),Files.end()) == false)
 	 return false;
    }
    else
    {
       if (Debug == true)
 	 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
-      TotalSize = ComputeSize(Files.begin(),Files.end());
+      TotalSize = ComputeSize(&List, Files.begin(),Files.end());
       
       // Build the source cache
       pkgCacheGenerator Gen(Map.Get(),Progress);
       if (_error->PendingError() == true)
 	 return false;
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin(),Files.begin()+EndOfSource) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
+		     Files.end(),Files.end()) == false)
 	 return false;
       
       // Write it back
@@ -1503,8 +1632,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
       }
       
       // Build the status cache
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin()+EndOfSource,Files.end()) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
+		     Files.begin(), Files.end()) == false)
 	 return false;
    }
    if (Debug == true)
@@ -1536,7 +1665,6 @@ APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **Ou
 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
 {
    std::vector<pkgIndexFile *> Files;
-   map_filesize_t EndOfSource = Files.size();
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
@@ -1544,7 +1672,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
    
-   TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
+   TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
    
    // Build the status cache
    if (Progress != NULL)
@@ -1552,8 +1680,8 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    pkgCacheGenerator Gen(Map.Get(),Progress);
    if (_error->PendingError() == true)
       return false;
-   if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		  Files.begin()+EndOfSource,Files.end()) == false)
+   if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
+		  Files.begin(), Files.end()) == false)
       return false;
 
    if (_error->PendingError() == true)

apt-pkg/pkgcachegen.h

@@ -69,7 +69,9 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    DynamicMMap ⤅
    pkgCache Cache;
    OpProgress *Progress;
-   
+
+   std::string RlsFileName;
+   pkgCache::ReleaseFile *CurrentRlsFile;
    std::string PkgFileName;
    pkgCache::PackageFile *CurrentFile;
 
@@ -100,12 +102,14 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    inline map_stringitem_t StoreString(enum StringType const type, const std::string &S) {return StoreString(type, S.c_str(),S.length());};
 
    void DropProgress() {Progress = 0;};
-   bool SelectFile(const std::string &File,const std::string &Site,pkgIndexFile const &Index,
-		   unsigned long Flags = 0);
+   bool SelectFile(const std::string &File,pkgIndexFile const &Index, std::string const &Component, unsigned long Flags = 0);
+   bool SelectReleaseFile(const std::string &File, const std::string &Site, unsigned long Flags = 0);
    bool MergeList(ListParser &List,pkgCache::VerIterator *Ver = 0);
    inline pkgCache &GetCache() {return Cache;};
    inline pkgCache::PkgFileIterator GetCurFile() 
          {return pkgCache::PkgFileIterator(Cache,CurrentFile);};
+   inline pkgCache::RlsFileIterator GetCurRlsFile() 
+         {return pkgCache::RlsFileIterator(Cache,CurrentRlsFile);};
 
    bool HasFileDeps() {return FoundFileDeps;};
    bool MergeFileProvides(ListParser &List);

apt-pkg/policy.cc

@@ -63,9 +63,9 @@ pkgPolicy::pkgPolicy(pkgCache *Owner) : Pins(0), PFPriority(0), Cache(Owner)
       pkgVersionMatch vm("", pkgVersionMatch::None);
       for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
       {
-	 if ((F->Archive != 0 && vm.ExpressionMatches(DefRel, F.Archive()) == true) ||
-	     (F->Codename != 0 && vm.ExpressionMatches(DefRel, F.Codename()) == true) ||
-	     (F->Version != 0 && vm.ExpressionMatches(DefRel, F.Version()) == true) ||
+	 if (vm.ExpressionMatches(DefRel, F.Archive()) ||
+	     vm.ExpressionMatches(DefRel, F.Codename()) ||
+	     vm.ExpressionMatches(DefRel, F.Version()) ||
 	     (DefRel.length() > 2 && DefRel[1] == '='))
 	    found = true;
       }
@@ -86,11 +86,11 @@ bool pkgPolicy::InitDefaults()
    for (pkgCache::PkgFileIterator I = Cache->FileBegin(); I != Cache->FileEnd(); ++I)
    {
       PFPriority[I->ID] = 500;
-      if ((I->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
+      if (I.Flagged(pkgCache::Flag::NotSource))
 	 PFPriority[I->ID] = 100;
-      else if ((I->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades)
+      else if (I.Flagged(pkgCache::Flag::ButAutomaticUpgrades))
 	 PFPriority[I->ID] = 100;
-      else if ((I->Flags & pkgCache::Flag::NotAutomatic) == pkgCache::Flag::NotAutomatic)
+      else if (I.Flagged(pkgCache::Flag::NotAutomatic))
 	 PFPriority[I->ID] = 1;
    }
 
@@ -170,8 +170,7 @@ pkgCache::VerIterator pkgPolicy::GetCandidateVer(pkgCache::PkgIterator const &Pk
 	    then it is not a candidate for installation, ever. This weeds
 	    out bogus entries that may be due to config-file states, or
 	    other. */
-	 if ((VF.File()->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource &&
-	     instVer == false)
+	 if (VF.File().Flagged(pkgCache::Flag::NotSource) && instVer == false)
 	    continue;
 
 	 signed Prio = PFPriority[VF.File()->ID];

apt-pkg/versionmatch.cc

@@ -137,7 +137,10 @@ pkgVersionMatch::pkgVersionMatch(string Data,MatchType Type) : Type(Type)
 // ---------------------------------------------------------------------
 /* */
 bool pkgVersionMatch::MatchVer(const char *A,string B,bool Prefix)
-{   
+{
+   if (A == NULL)
+      return false;
+
    const char *Ab = A;
    const char *Ae = Ab + strlen(A);
    
@@ -178,13 +181,16 @@ pkgCache::VerIterator pkgVersionMatch::Find(pkgCache::PkgIterator Pkg)
    // This will be Ended by now.
    return Ver;
 }
+									/*}}}*/
 
 #ifndef FNM_CASEFOLD
 #define FNM_CASEFOLD 0
 #endif
 
-bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)
+bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)/*{{{*/
 {
+   if (pattern == NULL || string == NULL)
+      return false;
    if (pattern[0] == '/') {
       size_t length = strlen(pattern);
       if (pattern[length - 1] == '/') {
@@ -230,38 +236,30 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File)
 	 return false;
 
       if (RelVerStr.empty() == false)
-	 if (File->Version == 0 ||
-	     (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false &&
-	      ExpressionMatches(RelVerStr, File.Version()) == false))
+	 if (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false &&
+	       ExpressionMatches(RelVerStr, File.Version()) == false)
 	    return false;
       if (RelOrigin.empty() == false)
-	 if (File->Origin == 0 || !ExpressionMatches(RelOrigin,File.Origin()))
+	 if (!ExpressionMatches(RelOrigin,File.Origin()))
 	    return false;
       if (RelArchive.empty() == false)
-	 if (File->Archive == 0 ||
-	     !ExpressionMatches(RelArchive,File.Archive()))
+	 if (!ExpressionMatches(RelArchive,File.Archive()))
             return false;
       if (RelCodename.empty() == false)
-	 if (File->Codename == 0 ||
-	     !ExpressionMatches(RelCodename,File.Codename()))
+	 if (!ExpressionMatches(RelCodename,File.Codename()))
             return false;
       if (RelRelease.empty() == false)
-	 if ((File->Archive == 0 ||
-	     !ExpressionMatches(RelRelease,File.Archive())) &&
-             (File->Codename == 0 ||
-	      !ExpressionMatches(RelRelease,File.Codename())))
+	 if (!ExpressionMatches(RelRelease,File.Archive()) &&
+             !ExpressionMatches(RelRelease,File.Codename()))
 	       return false;
       if (RelLabel.empty() == false)
-	 if (File->Label == 0 ||
-	     !ExpressionMatches(RelLabel,File.Label()))
+	 if (!ExpressionMatches(RelLabel,File.Label()))
 	    return false;
       if (RelComponent.empty() == false)
-	 if (File->Component == 0 ||
-	     !ExpressionMatches(RelComponent,File.Component()))
+	 if (!ExpressionMatches(RelComponent,File.Component()))
 	    return false;
       if (RelArchitecture.empty() == false)
-	 if (File->Architecture == 0 ||
-	     !ExpressionMatches(RelArchitecture,File.Architecture()))
+	 if (!ExpressionMatches(RelArchitecture,File.Architecture()))
 	    return false;
       return true;
    }
@@ -269,11 +267,14 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File)
    if (Type == Origin)
    {
       if (OrSite.empty() == false) {
-	 if (File->Site == 0)
+	 if (File.Site() == NULL)
 	    return false;
       } else // so we are talking about file:// or status file
-	 if (strcmp(File.Site(),"") == 0 && File->Archive != 0 && strcmp(File.Archive(),"now") == 0) // skip the status file
+      {
+	 pkgCache::RlsFileIterator const RlsFile = File.ReleaseFile();
+	 if (strcmp(File.Site(),"") == 0 && RlsFile->Archive != 0 && strcmp(RlsFile.Archive(),"now") == 0) // skip the status file
 	    return false;
+      }
       return (ExpressionMatches(OrSite, File.Site())); /* both strings match */
    }
 

apt-pkg/versionmatch.h

@@ -70,8 +70,8 @@ class pkgVersionMatch
    enum MatchType {None = 0,Version,Release,Origin} Type;
 
    bool MatchVer(const char *A,std::string B,bool Prefix) APT_PURE;
-   bool ExpressionMatches(const char *pattern, const char *string);
-   bool ExpressionMatches(const std::string& pattern, const char *string);
+   static bool ExpressionMatches(const char *pattern, const char *string);
+   static bool ExpressionMatches(const std::string& pattern, const char *string);
    bool FileMatch(pkgCache::PkgFileIterator File);
    pkgCache::VerIterator Find(pkgCache::PkgIterator Pkg);
 

cmdline/apt-cache.cc

@@ -413,17 +413,21 @@ static bool Stats(CommandLine &)
 	    stritems.insert(Prv->ProvideVersion);
       }
    }
-   for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
+   for (pkgCache::RlsFileIterator F = Cache->RlsFileBegin(); F != Cache->RlsFileEnd(); ++F)
    {
       stritems.insert(F->FileName);
       stritems.insert(F->Archive);
       stritems.insert(F->Codename);
-      stritems.insert(F->Component);
       stritems.insert(F->Version);
       stritems.insert(F->Origin);
       stritems.insert(F->Label);
-      stritems.insert(F->Architecture);
       stritems.insert(F->Site);
+   }
+   for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
+   {
+      stritems.insert(F->FileName);
+      stritems.insert(F->Architecture);
+      stritems.insert(F->Component);
       stritems.insert(F->IndexType);
    }
    unsigned long Size = 0;
@@ -446,6 +450,7 @@ static bool Stats(CommandLine &)
       APT_CACHESIZE(VersionCount, VersionSz) +
       APT_CACHESIZE(DescriptionCount, DescriptionSz) +
       APT_CACHESIZE(DependsCount, DependencySz) +
+      APT_CACHESIZE(ReleaseFileCount, ReleaseFileSz) +
       APT_CACHESIZE(PackageFileCount, PackageFileSz) +
       APT_CACHESIZE(VerFileCount, VerFileSz) +
       APT_CACHESIZE(DescFileCount, DescFileSz) +

test/integration/framework

@@ -137,7 +137,14 @@ dpkgcheckbuilddeps() {
 	command dpkg-checkbuilddeps --admindir=${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg "$@"
 }
 gdb() {
-	local CMD="$1"
+	local CMD
+	case "$1" in
+	aptget) CMD="apt-get";;
+	aptcache) CMD="apt-cache";;
+	aptmark) CMD="apt-mark";;
+	apthelper) CMD="apt-helper";;
+	*) CMD="$1";;
+	esac
 	shift
 	runapt command gdb --quiet -ex run "${BUILDDIRECTORY}/$CMD" --args "${BUILDDIRECTORY}/$CMD" "$@"
 }

test/integration/test-policy-pinning

@@ -22,7 +22,7 @@ testequalpolicy() {
      release a=now
  $(echo "$AP" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}  Packages
      release c=
-Pinned packages:" aptcache policy $*
+Pinned packages:" aptcache policy "$@"
 }
 
 testglobalpolicy() {