HTTP pipelining

Author: jgg
Date: 1998-12-05 04:19:01 GMT
HTTP pipelining

apt-pkg/acquire-method.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-method.cc,v 1.13 1998/12/05 01:45:19 jgg Exp $
+// $Id: acquire-method.cc,v 1.14 1998/12/05 04:19:01 jgg Exp $
 /* ######################################################################
 
    Acquire Method
@@ -47,8 +47,9 @@ pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags)
       exit(100);
 
    SetNonBlock(STDIN_FILENO,true);
-   
+
    Queue = 0;
+   QueueBack = 0;
 }
 									/*}}}*/
 // AcqMethod::Fail - A fetch has failed					/*{{{*/
@@ -78,6 +79,8 @@ void pkgAcqMethod::Fail(string Err,bool Transient)
       FetchItem *Tmp = Queue;
       Queue = Queue->Next;
       delete Tmp;
+      if (Tmp == QueueBack)
+	 QueueBack = Queue;
    }
    else
       snprintf(S,sizeof(S),"400 URI Failure\nURI: <UNKNOWN>\n"
@@ -183,6 +186,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt)
    FetchItem *Tmp = Queue;
    Queue = Queue->Next;
    delete Tmp;
+   if (Tmp == QueueBack)
+      QueueBack = Queue;
 }
 									/*}}}*/
 // AcqMethod::MediaFail - Syncronous request for new media		/*{{{*/
@@ -328,7 +333,9 @@ int pkgAcqMethod::Run(bool Single)
 	    FetchItem **I = &Queue;
 	    for (; *I != 0; I = &(*I)->Next);
 	    *I = Tmp;
-
+	    if (QueueBack == 0)
+	       QueueBack = Tmp;
+	    
 	    // Notify that this item is to be fetched.
 	    if (Fetch(Tmp) == false)
 	       Fail();

apt-pkg/acquire-method.h

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-method.h,v 1.6 1998/12/04 22:56:51 jgg Exp $
+// $Id: acquire-method.h,v 1.7 1998/12/05 04:19:02 jgg Exp $
 /* ######################################################################
 
    Acquire Method - Method helper class + functions
@@ -48,6 +48,7 @@ class pkgAcqMethod
    // State
    vector<string> Messages;
    FetchItem *Queue;
+   FetchItem *QueueBack;
       
    // Handlers for messages
    virtual bool Configuration(string Message);

apt-pkg/acquire.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.cc,v 1.19 1998/11/29 01:24:19 jgg Exp $
+// $Id: acquire.cc,v 1.20 1998/12/05 04:19:03 jgg Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -514,6 +514,17 @@ bool pkgAcquire::Queue::Startup()
    if (Workers->Start() == false)
       return false;
    
+   /* When pipelining we commit 10 items. This needs to change when we
+      added other source retry to have cycle maintain a pipeline depth
+      on its own. */
+   if (Cnf->Pipeline == true)
+   {
+      bool Res = true;
+      for (int I = 0; I != 10 && Res == true; I++)
+	 Res &= Cycle();
+      return Res;
+   }
+   
    return Cycle();
 }
 									/*}}}*/

methods/http.cc

@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: http.cc,v 1.8 1998/11/28 20:50:10 jgg Exp $
+// $Id: http.cc,v 1.9 1998/12/05 04:19:05 jgg Exp $
 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.
@@ -880,6 +880,38 @@ void HttpMethod::SigTerm(int)
    exit(100);
 }
 									/*}}}*/
+// HttpMethod::Fetch - Fetch an item					/*{{{*/
+// ---------------------------------------------------------------------
+/* This adds an item to the pipeline. We keep the pipeline at a fixed
+   depth. */
+bool HttpMethod::Fetch(FetchItem *)
+{
+   if (Server == 0)
+      return true;
+   
+   // Queue the requests
+   int Depth = -1;
+   bool Tail = false;
+   for (FetchItem *I = Queue; I != 0 && Depth < 5; I = I->Next, Depth++)
+   {
+      // Make sure we stick with the same server
+      if (Server->Comp(I->Uri) == false)
+	 break;
+      
+      if (QueueBack == I)
+	 Tail = true;
+      if (Tail == true)
+      {
+	 Depth++;
+	 QueueBack = I->Next;
+	 SendReq(I,Server->Out);
+	 continue;
+      }	 
+   }
+   
+   return true;
+};
+									/*}}}*/
 // HttpMethod::Loop - Main loop						/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -888,7 +920,7 @@ int HttpMethod::Loop()
    signal(SIGTERM,SigTerm);
    signal(SIGINT,SigTerm);
    
-   ServerState *Server = 0;
+   Server = 0;
    
    int FailCounter = 0;
    while (1)
@@ -926,10 +958,10 @@ int HttpMethod::Loop()
 	 Fail();
 	 continue;
       }
-      
-      // Queue the request
-      SendReq(Queue,Server->Out);
 
+      // Fill the pipeline.
+      Fetch(0);
+      
       // Fetch the next URL header data from the server.
       switch (Server->RunHeaders())
       {
@@ -954,7 +986,7 @@ int HttpMethod::Loop()
 	    continue;
 	 }
       };
-      
+
       // Decide what to do.
       FetchResult Res;
       Res.Filename = Queue->DestFile;

methods/http.h

@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
-// Description								/*{{{*/
-// $Id: http.h,v 1.4 1998/11/11 06:54:22 jgg Exp $
+// Description								/*{{{*/// $Id: http.h,v 1.5 1998/12/05 04:19:06 jgg Exp $
 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.
@@ -118,6 +117,8 @@ class HttpMethod : public pkgAcqMethod
    bool Flush(ServerState *Srv);
    bool ServerDie(ServerState *Srv);
    int DealWithHeaders(FetchResult &Res,ServerState *Srv);
+
+   bool Fetch(FetchItem *);
    
    // In the event of a fatal signal this file will be closed and timestamped.
    static string FailFile;
@@ -128,16 +129,15 @@ class HttpMethod : public pkgAcqMethod
    public:
    friend ServerState;
 
-   int Depth;
    FileFd *File;
+   ServerState *Server;
    
    int Loop();
    
    HttpMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig) 
    {
-      Depth = 0;
       File = 0;
-      Depth = 0;
+      Server = 0;
    };
 };