Devel acquire module

Author: jgg
Date: 1998-10-15 06:59:59 GMT
Devel acquire module

apt-pkg/acquire-item.cc

@@ -0,0 +1,85 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: acquire-item.cc,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Item - Item to acquire
+
+   Each item can download to exactly one file at a time. This means you
+   cannot create an item that fetches two uri's to two files at the same 
+   time. The pkgAcqIndex class creates a second class upon instantiation
+   to fetch the other index files because of this.
+   
+   ##################################################################### */
+									/*}}}*/
+// Include Files							/*{{{*/
+#ifdef __GNUG__
+#pragma implementation "apt-pkg/acquire-item.h"
+#endif
+#include <apt-pkg/acquire-item.h>
+#include <apt-pkg/configuration.h>
+#include <strutl.h>
+									/*}}}*/
+
+// Acquire::Item::Item - Constructor					/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), QueueCounter(0)
+{
+   Owner->Add(this);
+}
+									/*}}}*/
+// Acquire::Item::~Item - Destructor					/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::Item::~Item()
+{
+   Owner->Remove(this);
+}
+									/*}}}*/
+
+// AcqIndex::AcqIndex - Constructor					/*{{{*/
+// ---------------------------------------------------------------------
+/* The package file is added to the queue and a second class is 
+   instantiated to fetch the revision file */
+pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) :
+             Item(Owner), Location(Location)
+{
+   QueueURI(Location->PackagesURI() + ".gz");
+   Description = Location->PackagesInfo();
+   
+   new pkgAcqIndexRel(Owner,Location);
+}
+									/*}}}*/
+// pkgAcqIndex::ToFile - File to write the download to			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+string pkgAcqIndex::ToFile()
+{
+   string PartialDir = _config->FindDir("Dir::State::lists") + "/partial/";
+   
+   return PartialDir + URItoFileName(Location->PackagesURI());
+}
+									/*}}}*/
+
+// AcqIndexRel::pkgAcqIndexRel - Constructor				/*{{{*/
+// ---------------------------------------------------------------------
+/* The Release file is added to the queue */
+pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
+			       const pkgSourceList::Item *Location) :
+                Item(Owner), Location(Location)
+{
+   QueueURI(Location->ReleaseURI());
+   Description = Location->ReleaseInfo();
+}
+									/*}}}*/
+// AcqIndexRel::ToFile - File to write the download to			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+string pkgAcqIndexRel::ToFile()
+{
+   string PartialDir = _config->FindDir("Dir::State::lists") + "/partial/";
+   
+   return PartialDir + URItoFileName(Location->ReleaseURI());
+}
+									/*}}}*/

apt-pkg/acquire-item.h

@@ -0,0 +1,76 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: acquire-item.h,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Item - Item to acquire
+
+   When an item is instantiated it will add it self to the local list in
+   the Owner Acquire class. Derived classes will then call QueueURI to 
+   register all the URI's they wish to fetch for at the initial moment.   
+   
+   Two item classes are provided to provide functionality for downloading
+   of Index files and downloading of Packages.
+   
+   ##################################################################### */
+									/*}}}*/
+#ifndef PKGLIB_ACQUIRE_ITEM_H
+#define PKGLIB_ACQUIRE_ITEM_H
+
+#include <apt-pkg/acquire.h>
+#include <apt-pkg/sourcelist.h>
+
+#ifdef __GNUG__
+#pragma interface "apt-pkg/acquire-item.h"
+#endif 
+
+// Item to acquire
+class pkgAcquire::Item
+{  
+   protected:
+   
+   pkgAcquire *Owner;
+   inline void QueueURI(string URI) {Owner->Enqueue(this,URI);};
+   
+   public:
+
+   unsigned int QueueCounter;
+   string Description;
+   
+   virtual string ToFile() = 0;
+   virtual void Failed() {};
+   
+   Item(pkgAcquire *Owner);
+   virtual ~Item();
+};
+
+// Item class for index files
+class pkgAcqIndex : public pkgAcquire::Item
+{
+   protected:
+   
+   const pkgSourceList::Item *Location;
+   
+   public:
+   
+   virtual string ToFile();
+
+   pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location);
+};
+
+// Item class for index files
+class pkgAcqIndexRel : public pkgAcquire::Item
+{
+   protected:
+   
+   const pkgSourceList::Item *Location;
+   
+   public:
+   
+   virtual string ToFile();
+
+   pkgAcqIndexRel(pkgAcquire *Owner,const pkgSourceList::Item *Location);
+};
+
+
+#endif

apt-pkg/acquire-worker.cc

@@ -0,0 +1,15 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: acquire-worker.cc,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Worker 
+
+   ##################################################################### */
+									/*}}}*/
+// Include Files							/*{{{*/
+#ifdef __GNUG__
+#pragma implementation "apt-pkg/acquire-worker.h"
+#endif       
+#include <apt-pkg/acquire-worker.h>
+									/*}}}*/

apt-pkg/acquire-worker.h

@@ -0,0 +1,39 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: acquire-worker.h,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Worker - Worker process manager
+   
+   ##################################################################### */
+									/*}}}*/
+#ifndef PKGLIB_ACQUIRE_WORKER_H
+#define PKGLIB_ACQUIRE_WORKER_H
+
+#include <apt-pkg/acquire.h>
+
+#ifdef __GNUG__
+#pragma interface "apt-pkg/acquire-worker.h"
+#endif 
+
+// Interfacing to the method process
+class pkgAcquire::Worker
+{
+   protected:
+   
+   Queue *OwnerQ;
+   MethodConfig *Config;   
+   Worker *Next;
+   
+   friend Queue;
+   
+   public:
+   
+   bool Create();
+   
+   Worker(Queue *OwnerQ);
+   Worker(MethodConfig *Config);
+   ~Worker();
+};
+
+#endif

apt-pkg/acquire.cc

@@ -0,0 +1,66 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: acquire.cc,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire - File Acquiration
+
+   ##################################################################### */
+									/*}}}*/
+// Include Files							/*{{{*/
+#ifdef __GNUG__
+#pragma implementation "apt-pkg/acquire.h"
+#endif       
+#include <apt-pkg/acquire.h>
+#include <apt-pkg/acquire-item.h>
+#include <apt-pkg/acquire-worker.h>
+									/*}}}*/
+
+// Acquire::pkgAcquire - Constructor					/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::pkgAcquire()
+{
+   Queues = 0;
+   Configs = 0;
+}
+									/*}}}*/
+// Acquire::~pkgAcquire	- Destructor					/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::~pkgAcquire()
+{
+   while (Items.size() != 0)
+      delete Items[0];
+}
+									/*}}}*/
+// Acquire::Add - Add a new item					/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void pkgAcquire::Add(Item *Itm)
+{
+   Items.push_back(Itm);
+}
+									/*}}}*/
+// Acquire::Remove - Remove a item					/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void pkgAcquire::Remove(Item *Itm)
+{
+   for (vector<Item *>::iterator I = Items.begin(); I < Items.end(); I++)
+   {
+      if (*I == Itm)
+	 Items.erase(I);
+   }   
+}
+									/*}}}*/
+// Acquire::Enqueue - Queue an URI for fetching				/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void pkgAcquire::Enqueue(Item *Item,string URI)
+{
+   cout << "Fetching " << URI << endl;
+   cout << "   to " << Item->ToFile() << endl;
+}
+									/*}}}*/
+

apt-pkg/acquire.h

@@ -0,0 +1,97 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: acquire.h,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire - File Acquiration
+   
+   This module contians the Acquire system. It is responsible for bringing
+   files into the local pathname space. It deals with URIs for files and
+   URI handlers responsible for downloading or finding the URIs.
+   
+   Each file to download is represented by an Acquire::Item class subclassed
+   into a specialization. The Item class can add itself to several URI
+   acquire queues each prioritized by the download scheduler. When the 
+   system is run the proper URI handlers are spawned and the the acquire 
+   queues are fed into the handlers by the schedular until the queues are
+   empty. This allows for an Item to be downloaded from an alternate source
+   if the first try turns out to fail. It also alows concurrent downloading
+   of multiple items from multiple sources as well as dynamic balancing
+   of load between the sources.
+   
+   Schedualing of downloads is done on a first ask first get basis. This
+   preserves the order of the download as much as possible. And means the
+   fastest source will tend to process the largest number of files.
+   
+   Internal methods and queues for performing gzip decompression,
+   md5sum hashing and file copying are provided to allow items to apply
+   a number of transformations to the data files they are working with.
+   
+   ##################################################################### */
+									/*}}}*/
+#ifndef PKGLIB_ACQUIRE_H
+#define PKGLIB_ACQUIRE_H
+
+#include <vector>
+#include <string>
+
+#ifdef __GNUG__
+#pragma interface "apt-pkg/acquire.h"
+#endif 
+
+class pkgAcquire
+{   
+   public:
+   
+   class Item;
+   class Queue;
+   class Worker;
+   struct MethodConfig;
+   friend Item;
+   
+   protected:
+   
+   vector<Item *> Items;
+   Queue *Queues;
+   MethodConfig *Configs;
+   
+   void Add(Item *Item);
+   void Remove(Item *Item);
+   void Enqueue(Item *Item,string URI);
+   
+   public:
+   
+   pkgAcquire();
+   ~pkgAcquire();
+};
+
+// List of possible items queued for download.
+class pkgAcquire::Queue
+{
+   friend pkgAcquire;
+   Queue *Next;
+   
+   protected:
+   
+   string Access;
+   string URIMatch;
+   
+   vector<Item *> Items;
+   
+   public:
+};
+
+// Configuration information from each method
+struct pkgAcquire::MethodConfig
+{
+   string Access;
+
+   string Version;
+   bool SingleInstance;
+   bool PreScan;
+   
+   MethodConfig();
+   ~MethodConfig();
+};
+
+#endif

apt-pkg/makefile

@@ -22,8 +22,9 @@ SOURCE = contrib/mmap.cc contrib/error.cc contrib/strutl.cc \
 # Source code for the main library
 SOURCE+= pkgcache.cc version.cc fileutl.cc pkgcachegen.cc depcache.cc \
          orderlist.cc tagfile.cc sourcelist.cc packagemanager.cc \
-	 pkgrecords.cc algorithms.cc init.cc templates.cc
-	 
+	 pkgrecords.cc algorithms.cc acquire.cc acquire-item.cc \
+	 acquire-worker.cc init.cc templates.cc
+
 # Source code for the debian specific components	 
 SOURCE+= deb/deblistparser.cc deb/debrecords.cc
 
@@ -31,7 +32,9 @@ SOURCE+= deb/deblistparser.cc deb/debrecords.cc
 HEADERS = algorithms.h depcache.h mmap.h pkgcachegen.h cacheiterators.h \
           error.h orderlist.h sourcelist.h configuration.h fileutl.h \
           packagemanager.h tagfile.h deblistparser.h init.h pkgcache.h \
-          version.h progress.h pkgrecords.h debrecords.h cmndline.h
+          version.h progress.h pkgrecords.h debrecords.h cmndline.h \
+	  acquire.h acquire-worker.h acquire-item.h
+
 HEADERS := $(addprefix apt-pkg/,$(HEADERS))
 
 # Private header files

apt-pkg/pkgcachegen.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: pkgcachegen.cc,v 1.18 1998/10/02 04:39:47 jgg Exp $
+// $Id: pkgcachegen.cc,v 1.19 1998/10/15 07:00:00 jgg Exp $
 /* ######################################################################
    
    Package Cache Generator - Generator for the cache structure.
@@ -487,10 +487,31 @@ bool pkgPkgCacheCheck(string CacheFile)
       return false;
    }
 
+   // Status files that must be in the cache
+   string Status[3];
+   Status[0] = _config->FindDir("Dir::State::xstatus");
+   Status[1]= _config->FindDir("Dir::State::userstatus");
+   Status[2] = _config->FindDir("Dir::State::status");
+   
    // Cheack each file
    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
+   {
       if (F.IsOk() == false)
 	 return false;
+      
+      // See if this is one of the status files
+      for (int I = 0; I != 3; I++)
+	 if (F.FileName() == Status[I])
+	    Status[I] = string();
+   }
+   
+   // Make sure all the status files are loaded.
+   for (int I = 0; I != 3; I++)
+   {
+      if (Status[I].empty() == false && FileExists(Status[I]) == true)
+	 return false;
+   }   
+   
    return true;
 }
 									/*}}}*/

apt-pkg/sourcelist.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: sourcelist.cc,v 1.5 1998/07/26 04:49:33 jgg Exp $
+// $Id: sourcelist.cc,v 1.6 1998/10/15 07:00:01 jgg Exp $
 /* ######################################################################
 
    List of Sources
@@ -195,6 +195,48 @@ string pkgSourceList::Item::PackagesInfo() const
    return Res;
 }
 									/*}}}*/
+// SourceList::Item::ReleaseURI - Returns a URI to the release file	/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+string pkgSourceList::Item::ReleaseURI() const
+{
+   string Res;
+   switch (Type)
+   {
+      case Deb:
+      if (Dist[Dist.size() - 1] == '/')
+	 Res = URI + Dist;
+      else
+	 Res = URI + "dists/" + Dist + '/' + Section +
+	 "/binary-" + _config->Find("APT::Architecture") + '/';
+      
+      Res += "Release";
+      break;
+   };
+   return Res;
+}
+									/*}}}*/
+// SourceList::Item::ReleaseInfo - Shorter version of the URI		/*{{{*/
+// ---------------------------------------------------------------------
+/* This is a shorter version that is designed to be < 60 chars or so */
+string pkgSourceList::Item::ReleaseInfo() const
+{
+   string Res;
+   switch (Type)
+   {
+      case Deb:
+      Res += SiteOnly(URI) + ' ';
+      if (Dist[Dist.size() - 1] == '/')
+	 Res += Dist;
+      else
+	 Res += Dist + '/' + Section;
+      
+      Res += " Release";
+      break;
+   };
+   return Res;
+}
+									/*}}}*/
 // SourceList::Item::ArchiveInfo - Shorter version of the archive spec	/*{{{*/
 // ---------------------------------------------------------------------
 /* This is a shorter version that is designed to be < 60 chars or so */

apt-pkg/sourcelist.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: sourcelist.h,v 1.5 1998/07/19 21:24:15 jgg Exp $
+// $Id: sourcelist.h,v 1.6 1998/10/15 07:00:02 jgg Exp $
 /* ######################################################################
 
    SourceList - Manage a list of sources
@@ -47,6 +47,8 @@ class pkgSourceList
       bool SetURI(string S);
       string PackagesURI() const;
       string PackagesInfo() const;      
+      string ReleaseURI() const;
+      string ReleaseInfo() const;      
       string SiteOnly(string URI) const;
       string ArchiveInfo(pkgCache::VerIterator Ver) const;
       string ArchiveURI(string File) const;
@@ -67,7 +69,7 @@ class pkgSourceList
    inline const_iterator end() const {return List.end();};
    inline unsigned int size() const {return List.size();};
    inline bool empty() const {return List.empty();};
-   
+
    pkgSourceList();
    pkgSourceList(string File);   
 };