Almost done now

Author: jgg
Date: 1998-11-11 06:54:13 GMT
Almost done now

apt-pkg/acquire-item.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-item.cc,v 1.8 1998/11/09 01:09:19 jgg Exp $
+// $Id: acquire-item.cc,v 1.9 1998/11/11 06:54:13 jgg Exp $
 /* ######################################################################
 
    Acquire Item - Item to acquire
@@ -31,7 +31,7 @@
 // ---------------------------------------------------------------------
 /* */
 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
-                       Complete(false), QueueCounter(0)
+                       Mode(0), ID(0), Complete(false), QueueCounter(0)
 {
    Owner->Add(this);
    Status = StatIdle;
@@ -73,8 +73,16 @@ void pkgAcquire::Item::Start(string Message,unsigned long Size)
 // Acquire::Item::Done - Item downloaded OK				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-void pkgAcquire::Item::Done(string,unsigned long,string)
+void pkgAcquire::Item::Done(string Message,unsigned long Size,string)
 {
+   // We just downloaded something..
+   string FileName = LookupTag(Message,"Filename");
+   if (Complete == false && FileName == DestFile)
+   {
+      if (Owner->Log != 0)
+	 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
+   }
+   
    Status = StatDone;
    ErrorText = string();
    Owner->Dequeue(this);
@@ -186,6 +194,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5)
       DestFile += ".decomp";
       Desc.URI = "copy:" + FileName;
       QueueURI(Desc);
+      Mode = "copy";
       return;
    }
 
@@ -209,6 +218,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5)
    DestFile += ".decomp";
    Desc.URI = "gzip:" + FileName,Location->PackagesInfo();
    QueueURI(Desc);
+   Mode = "gzip";
 }
 									/*}}}*/
 

apt-pkg/acquire-item.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-item.h,v 1.6 1998/11/09 01:09:21 jgg Exp $
+// $Id: acquire-item.h,v 1.7 1998/11/11 06:54:14 jgg Exp $
 /* ######################################################################
 
    Acquire Item - Item to acquire
@@ -41,6 +41,8 @@ class pkgAcquire::Item
    enum {StatIdle, StatFetching, StatDone, StatError} Status;
    string ErrorText;
    unsigned long FileSize;
+   char *Mode;
+   unsigned long ID;
    bool Complete;
    
    // Number of queues we are inserted into

apt-pkg/acquire-method.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-method.cc,v 1.5 1998/11/09 01:09:22 jgg Exp $
+// $Id: acquire-method.cc,v 1.6 1998/11/11 06:54:15 jgg Exp $
 /* ######################################################################
 
    Acquire Method
@@ -141,6 +141,10 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt)
    if (Res.MD5Sum.empty() == false)
       End += snprintf(End,sizeof(S) - (End - S),"MD5Sum: %s\n",Res.MD5Sum.c_str());
 
+   if (Res.ResumePoint != 0)
+      End += snprintf(End,sizeof(S) - (End - S),"Resume-Point: %u\n",
+		      Res.ResumePoint);
+
    if (Res.IMSHit == true)
       strcat(End,"IMS-Hit: true\n");
    End = S + strlen(S);
@@ -256,7 +260,9 @@ int pkgAcqMethod::Run(bool Single)
 	    
 	    Tmp->Uri = LookupTag(Message,"URI");
 	    Tmp->DestFile = LookupTag(Message,"FileName");
-	    StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified);
+	    if (StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified) == false)
+	       Tmp->LastModified = 0;
+	    
 	    Tmp->Next = 0;
 	    
 	    // Append it to the list

apt-pkg/acquire.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.cc,v 1.10 1998/11/09 01:09:25 jgg Exp $
+// $Id: acquire.cc,v 1.11 1998/11/11 06:54:16 jgg Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -194,7 +194,7 @@ string pkgAcquire::QueueName(string Uri)
    /* Single-Instance methods get exactly one queue per URI. This is
       also used for the Access queue method  */
    if (Config->SingleInstance == true || QueueMode == QueueAccess)
-      return U.Access;
+       return U.Access;
 
    return U.Access + ':' + U.Host;
 }
@@ -276,6 +276,9 @@ bool pkgAcquire::Run()
    for (Queue *I = Queues; I != 0; I = I->Next)
       I->Startup();
    
+   if (Log != 0)
+      Log->Start();
+   
    // Run till all things have been acquired
    struct timeval tv;
    tv.tv_sec = 0;
@@ -311,6 +314,9 @@ bool pkgAcquire::Run()
       }      
    }   
 
+   if (Log != 0)
+      Log->Stop();
+   
    // Shut down the acquire bits
    Running = false;
    for (Queue *I = Queues; I != 0; I = I->Next)
@@ -514,3 +520,123 @@ void pkgAcquire::Queue::Bump()
 {
 }
 									/*}}}*/
+
+// AcquireStatus::pkgAcquireStatus - Constructor			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquireStatus::pkgAcquireStatus()
+{
+   Start();
+}
+									/*}}}*/
+// AcquireStatus::Pulse - Called periodically				/*{{{*/
+// ---------------------------------------------------------------------
+/* This computes some internal state variables for the derived classes to
+   use. It generates the current downloaded bytes and total bytes to download
+   as well as the current CPS estimate. */
+void pkgAcquireStatus::Pulse(pkgAcquire *Owner)
+{
+   TotalBytes = 0;
+   CurrentBytes = 0;
+   
+   // Compute the total number of bytes to fetch
+   unsigned int Unknown = 0;
+   unsigned int Count = 0;
+   for (pkgAcquire::Item **I = Owner->ItemsBegin(); I != Owner->ItemsEnd(); 
+	I++, Count++)
+   {
+      TotalBytes += (*I)->FileSize;
+      if ((*I)->Complete == true)
+	 CurrentBytes += (*I)->FileSize;
+      if ((*I)->FileSize == 0 && (*I)->Complete == false)
+	 Unknown++;
+   }
+   
+   // Compute the current completion
+   for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
+	I = Owner->WorkerStep(I))
+      if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
+	 CurrentBytes += I->CurrentSize;
+      
+   // Normalize the figures and account for unknown size downloads
+   if (TotalBytes <= 0)
+      TotalBytes = 1;
+   if (Unknown == Count)
+      TotalBytes = Unknown;
+   else
+      TotalBytes += TotalBytes/(Count - Unknown)*Unknown;
+   
+   // Compute the CPS
+   struct timeval NewTime;
+   gettimeofday(&NewTime,0);
+   if (NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec ||
+       NewTime.tv_sec - Time.tv_sec > 6)
+   {    
+      // Compute the delta time with full accuracy
+      long usdiff = NewTime.tv_usec - Time.tv_usec;
+      long sdiff = NewTime.tv_sec - Time.tv_sec;
+      
+      // Borrow
+      if (usdiff < 0)
+      {	 
+	 usdiff += 1000000;
+	 sdiff--;
+      }
+            
+      // Compute the CPS value
+      CurrentCPS = (CurrentBytes - LastBytes)/(sdiff + usdiff/1000000.0);
+      LastBytes = CurrentBytes;
+      ElapsedTime = NewTime.tv_sec - StartTime.tv_sec;
+      Time = NewTime;
+   }
+}
+									/*}}}*/
+// AcquireStatus::Start - Called when the download is started		/*{{{*/
+// ---------------------------------------------------------------------
+/* We just reset the counters */
+void pkgAcquireStatus::Start()
+{
+   gettimeofday(&Time,0);
+   gettimeofday(&StartTime,0);
+   LastBytes = 0;
+   CurrentCPS = 0;
+   CurrentBytes = 0;
+   TotalBytes = 0;
+   FetchedBytes = 0;
+   ElapsedTime = 0;
+}
+									/*}}}*/
+// pkgAcquireStatus::Stop - Finished downloading			/*{{{*/
+// ---------------------------------------------------------------------
+/* This accurately computes the elapsed time and the total overall CPS. */
+void pkgAcquireStatus::Stop()
+{
+   // Compute the CPS and elapsed time
+   struct timeval NewTime;
+   gettimeofday(&NewTime,0);
+   
+   // Compute the delta time with full accuracy
+   long usdiff = NewTime.tv_usec - StartTime.tv_usec;
+   long sdiff = NewTime.tv_sec - StartTime.tv_sec;
+   
+   // Borrow
+   if (usdiff < 0)
+   {	 
+      usdiff += 1000000;
+      sdiff--;
+   }
+   
+   // Compute the CPS value
+   CurrentCPS = FetchedBytes/(sdiff + usdiff/1000000.0);
+   LastBytes = CurrentBytes;
+   ElapsedTime = sdiff;
+}
+									/*}}}*/
+// AcquireStatus::Fetched - Called when a byte set has been fetched	/*{{{*/
+// ---------------------------------------------------------------------
+/* This is used to get accurate final transfer rate reporting. */
+void pkgAcquireStatus::Fetched(unsigned long Size,unsigned long Resume)
+{
+   FetchedBytes += Size - Resume;
+}
+									/*}}}*/

apt-pkg/acquire.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.h,v 1.9 1998/11/09 01:09:26 jgg Exp $
+// $Id: acquire.h,v 1.10 1998/11/11 06:54:17 jgg Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -39,6 +39,7 @@
 #pragma interface "apt-pkg/acquire.h"
 #endif 
 
+#include <sys/time.h>
 #include <unistd.h>
 
 class pkgAcquireStatus;
@@ -180,18 +181,34 @@ struct pkgAcquire::MethodConfig
 
 class pkgAcquireStatus
 {
+   protected:
+   
+   struct timeval Time;
+   struct timeval StartTime;
+   unsigned long LastBytes;
+   double CurrentCPS;
+   unsigned long CurrentBytes;
+   unsigned long TotalBytes;
+   unsigned long FetchedBytes;
+   unsigned long ElapsedTime;
+   
    public:
 
    bool Update;
    
+   // Called by items when they have finished a real download
+   virtual void Fetched(unsigned long Size,unsigned long ResumePoint);
+   
    // Each of these is called by the workers when an event occures
    virtual void IMSHit(pkgAcquire::ItemDesc &Itm) {};
    virtual void Fetch(pkgAcquire::ItemDesc &Itm) {};
    virtual void Done(pkgAcquire::ItemDesc &Itm) {};
    virtual void Fail(pkgAcquire::ItemDesc &Itm) {};   
-   virtual void Pulse(pkgAcquire *Owner) {};
-   
-   pkgAcquireStatus() : Update(false) {};
+   virtual void Pulse(pkgAcquire *Owner);
+   virtual void Start();
+   virtual void Stop();
+
+   pkgAcquireStatus();
    virtual ~pkgAcquireStatus() {};
 };
 

doc/examples/apt.conf

@@ -1,4 +1,4 @@
-// $Id: apt.conf,v 1.10 1998/11/11 06:09:34 jgg Exp $
+// $Id: apt.conf,v 1.11 1998/11/11 06:54:18 jgg Exp $
 /* This file is an index of all APT configuration directives. It should
    NOT actually be used as a real config file, though it is a completely
    valid file.
@@ -40,6 +40,7 @@ Acquire
 /*  http 
   {
     Proxy "http://127.0.0.1:3128";
+    Proxy::http.us.debian.org "DIRECT";  // Specific per-host setting 
   };*/
 };
 

methods/gzip.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: gzip.cc,v 1.5 1998/11/05 07:21:47 jgg Exp $
+// $Id: gzip.cc,v 1.6 1998/11/11 06:54:19 jgg Exp $
 /* ######################################################################
 
    GZip method - Take a file URI in and decompress it into the target 
@@ -37,6 +37,10 @@ bool GzipMethod::Fetch(FetchItem *Itm)
 {
    URI Get = Itm->Uri;
 
+   FetchResult Res;
+   Res.Filename = Itm->DestFile;
+   URIStart(Res);
+   
    // Open the source and destintation files
    FileFd From(Get.Path,FileFd::ReadOnly);
    FileFd To(Itm->DestFile,FileFd::WriteEmpty);   
@@ -96,9 +100,7 @@ bool GzipMethod::Fetch(FetchItem *Itm)
       return _error->Errno("utime","Failed to set modification time");
 
    // Return a Done response
-   FetchResult Res;
    Res.LastModified = Buf.st_mtime;
-   Res.Filename = Itm->DestFile;
    URIDone(Res);
    
    return true;

methods/http.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: http.cc,v 1.4 1998/11/05 07:21:48 jgg Exp $
+// $Id: http.cc,v 1.5 1998/11/11 06:54:21 jgg Exp $
 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.
@@ -613,7 +613,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 
    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
-//   cout << Req << endl;
+//   cerr << Req << endl;
    
    Out.Read(Req);
 }

methods/http.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: http.h,v 1.3 1998/11/04 07:10:50 jgg Exp $
+// $Id: http.h,v 1.4 1998/11/11 06:54:22 jgg Exp $
 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.
@@ -133,7 +133,7 @@ class HttpMethod : public pkgAcqMethod
    
    int Loop();
    
-   HttpMethod() : pkgAcqMethod("1.2",SingleInstance | Pipeline | SendConfig) 
+   HttpMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig) 
    {
       Depth = 0;
       File = 0;

test/scratch.cc

@@ -1,9 +1,202 @@
 #include <apt-pkg/acquire-item.h>
+#include <apt-pkg/acquire-worker.h>
 #include <apt-pkg/init.h>
 #include <apt-pkg/error.h>
-#include <signal.h>
 #include <strutl.h>
 
+#include <signal.h>
+#include <stdio.h>
+
+class AcqTextStatus : public pkgAcquireStatus
+{
+   unsigned int ScreenWidth;
+   char BlankLine[300];
+   unsigned long ID;
+   
+   public:
+   
+   virtual void IMSHit(pkgAcquire::ItemDesc &Itm);
+   virtual void Fetch(pkgAcquire::ItemDesc &Itm);
+   virtual void Done(pkgAcquire::ItemDesc &Itm);
+   virtual void Fail(pkgAcquire::ItemDesc &Itm);
+   virtual void Start() {pkgAcquireStatus::Start(); BlankLine[0] = 0; ID = 1;};
+   virtual void Stop();
+   
+   void Pulse(pkgAcquire *Owner);
+};
+
+// AcqTextStatus::IMSHit - Called when an item got a HIT response	/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
+{
+   cout << '\r' << BlankLine << '\r';   
+   cout << "Hit " << Itm.Description;
+   if (Itm.Owner->FileSize != 0)
+      cout << " [" << SizeToStr(Itm.Owner->FileSize) << ']';
+   cout << endl;
+   Update = true;
+};
+									/*}}}*/
+// AcqTextStatus::Fetch - An item has started to download		/*{{{*/
+// ---------------------------------------------------------------------
+/* This prints out the short description and the expected size */
+void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
+{
+   Update = true;
+   if (Itm.Owner->Complete == true)
+      return;
+   
+   Itm.Owner->ID = ID++;
+   
+   cout << '\r' << BlankLine << '\r';   
+   cout << hex << Itm.Owner->ID << dec << " Get " << Itm.Description;
+   if (Itm.Owner->FileSize != 0)
+      cout << " [" << SizeToStr(Itm.Owner->FileSize) << ']';
+   cout << endl;
+};
+									/*}}}*/
+// AcqTextStatus::Done - Completed a download				/*{{{*/
+// ---------------------------------------------------------------------
+/* We don't display anything... */
+void AcqTextStatus::Done(pkgAcquire::ItemDesc &Itm)
+{
+   Update = true;
+};
+									/*}}}*/
+// AcqTextStatus::Fail - Called when an item fails to download		/*{{{*/
+// ---------------------------------------------------------------------
+/* We print out the error text  */
+void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
+{
+   cout << '\r' << BlankLine << '\r';   
+   cout << "Err " << Itm.Description << endl;
+   cout << "  " << Itm.Owner->ErrorText << endl;
+   Update = true;
+};
+									/*}}}*/
+// AcqTextStatus::Stop - Finished downloading				/*{{{*/
+// ---------------------------------------------------------------------
+/* This prints out the bytes downloaded and the overall average line
+   speed */
+void AcqTextStatus::Stop()
+{
+   pkgAcquireStatus::Stop();
+   cout << '\r' << BlankLine << '\r';
+   
+   if (FetchedBytes == 0)
+      cout << flush;
+   else
+      cout << "Fetched " << SizeToStr(FetchedBytes) << " in " << 
+         TimeToStr(ElapsedTime) << " (" << SizeToStr(CurrentCPS) << 
+         "/s)" << endl;
+}
+									/*}}}*/
+// AcqTextStatus::Pulse - Regular event pulse				/*{{{*/
+// ---------------------------------------------------------------------
+/* This draws the current progress. Each line has an overall percent
+   meter and a per active item status meter along with an overall 
+   bandwidth and ETA indicator. */
+void AcqTextStatus::Pulse(pkgAcquire *Owner)
+{
+   pkgAcquireStatus::Pulse(Owner);
+   
+   enum {Long = 0,Medium,Short} Mode = Long;
+   
+   ScreenWidth = 140;
+   char Buffer[300];
+   char *End = Buffer + sizeof(Buffer);
+   char *S = Buffer;
+   
+   // Put in the percent done
+   sprintf(S,"%ld%%",long(double(CurrentBytes*100.0)/double(TotalBytes)));   
+   
+   for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
+	I = Owner->WorkerStep(I))
+   {
+      S += strlen(S);
+      
+      // There is no item running 
+      if (I->CurrentItem == 0)
+      {
+	 if (I->Status.empty() == false)
+	    snprintf(S,End-S," [%s]",I->Status.c_str());
+	 continue;
+      }
+
+      // Add in the short description
+      if (I->CurrentItem->Owner->ID != 0)
+	 snprintf(S,End-S," [%x %s",I->CurrentItem->Owner->ID,
+		  I->CurrentItem->ShortDesc.c_str());
+      else
+	 snprintf(S,End-S," [%s",I->CurrentItem->ShortDesc.c_str());
+      S += strlen(S);
+
+      // Show the short mode string
+      if (I->CurrentItem->Owner->Mode != 0)
+      {
+	 snprintf(S,End-S," %s",I->CurrentItem->Owner->Mode);
+	 S += strlen(S);
+      }
+            
+      // Add the current progress
+      if (Mode == Long)
+	 snprintf(S,End-S," %u",I->CurrentSize);
+      else
+      {
+	 if (Mode == Medium || I->TotalSize == 0)
+	    snprintf(S,End-S," %s",SizeToStr(I->CurrentSize).c_str());
+      }
+      S += strlen(S);
+      
+      // Add the total size and percent
+      if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false)
+      {
+	 if (Mode == Long)
+	    snprintf(S,End-S,"/%u %u%%",I->TotalSize,
+		     long(double(I->CurrentSize*100.0)/double(I->TotalSize)));
+	 else
+	 {
+	    if (Mode == Medium)
+	       snprintf(S,End-S,"/%s %u%%",SizeToStr(I->TotalSize).c_str(),
+			long(double(I->CurrentSize*100.0)/double(I->TotalSize)));
+	    else
+	       snprintf(S,End-S," %u%%",
+			long(double(I->CurrentSize*100.0)/double(I->TotalSize)));
+	 }      
+      }      
+      S += strlen(S);
+      snprintf(S,End-S,"]");
+   }
+
+   // Put in the ETA and cps meter
+   if (CurrentCPS != 0)
+   {      
+      char Tmp[300];
+      unsigned long ETA = (unsigned long)((TotalBytes - CurrentBytes)/CurrentCPS);
+      sprintf(Tmp," %s/s %s",SizeToStr(CurrentCPS).c_str(),TimeToStr(ETA).c_str());
+      unsigned int Len = strlen(Buffer);
+      unsigned int LenT = strlen(Tmp);
+      if (Len + LenT < ScreenWidth)
+      {	 
+	 memset(Buffer + Len,' ',ScreenWidth - Len);
+	 strcpy(Buffer + ScreenWidth - LenT,Tmp);
+      }      
+   }
+   Buffer[ScreenWidth] = 0;
+   
+   // Draw the current status
+   if (strlen(Buffer) == strlen(BlankLine))
+      cout << '\r' << Buffer << flush;
+   else
+      cout << '\r' << BlankLine << '\r' << Buffer << flush;
+   memset(BlankLine,' ',strlen(Buffer));
+   BlankLine[strlen(Buffer)] = 0;
+   
+   Update = false;
+}
+									/*}}}*/
+
 int main(int argc,char *argv[])
 {
    signal(SIGPIPE,SIG_IGN);
@@ -21,7 +214,8 @@ int main(int argc,char *argv[])
    pkgInitialize(*_config);
    
    pkgSourceList List;
-   pkgAcquire Fetcher;
+   AcqTextStatus Stat;
+   pkgAcquire Fetcher(&Stat);
    List.ReadMainList();
    
    pkgSourceList::const_iterator I;
@@ -31,7 +225,7 @@ int main(int argc,char *argv[])
       if (_error->PendingError() == true)
 	 break;
    }
-
+   
    Fetcher.Run();
    
    _error->DumpErrors();