acquire.cc 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876
  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. // $Id: acquire.cc,v 1.50 2004/03/17 05:17:11 mdz Exp $
  4. /* ######################################################################
  5. Acquire - File Acquiration
  6. The core element for the schedual system is the concept of a named
  7. queue. Each queue is unique and each queue has a name derived from the
  8. URI. The degree of paralization can be controled by how the queue
  9. name is derived from the URI.
  10. ##################################################################### */
  11. /*}}}*/
  12. // Include Files /*{{{*/
  13. #ifdef __GNUG__
  14. #pragma implementation "apt-pkg/acquire.h"
  15. #endif
  16. #include <apt-pkg/acquire.h>
  17. #include <apt-pkg/acquire-item.h>
  18. #include <apt-pkg/acquire-worker.h>
  19. #include <apt-pkg/configuration.h>
  20. #include <apt-pkg/error.h>
  21. #include <apt-pkg/strutl.h>
  22. #include <apti18n.h>
  23. #include <iostream>
  24. #include <sstream>
  25. #include <dirent.h>
  26. #include <sys/time.h>
  27. #include <errno.h>
  28. #include <sys/stat.h>
  29. /*}}}*/
  30. using namespace std;
  31. // Acquire::pkgAcquire - Constructor /*{{{*/
  32. // ---------------------------------------------------------------------
  33. /* We grab some runtime state from the configuration space */
  34. pkgAcquire::pkgAcquire(pkgAcquireStatus *Log) : Log(Log)
  35. {
  36. Queues = 0;
  37. Configs = 0;
  38. Workers = 0;
  39. ToFetch = 0;
  40. Running = false;
  41. string Mode = _config->Find("Acquire::Queue-Mode","host");
  42. if (strcasecmp(Mode.c_str(),"host") == 0)
  43. QueueMode = QueueHost;
  44. if (strcasecmp(Mode.c_str(),"access") == 0)
  45. QueueMode = QueueAccess;
  46. Debug = _config->FindB("Debug::pkgAcquire",false);
  47. // This is really a stupid place for this
  48. struct stat St;
  49. if (stat((_config->FindDir("Dir::State::lists") + "partial/").c_str(),&St) != 0 ||
  50. S_ISDIR(St.st_mode) == 0)
  51. _error->Error(_("Lists directory %spartial is missing."),
  52. _config->FindDir("Dir::State::lists").c_str());
  53. if (stat((_config->FindDir("Dir::Cache::Archives") + "partial/").c_str(),&St) != 0 ||
  54. S_ISDIR(St.st_mode) == 0)
  55. _error->Error(_("Archive directory %spartial is missing."),
  56. _config->FindDir("Dir::Cache::Archives").c_str());
  57. }
  58. /*}}}*/
  59. // Acquire::~pkgAcquire - Destructor /*{{{*/
  60. // ---------------------------------------------------------------------
  61. /* Free our memory, clean up the queues (destroy the workers) */
  62. pkgAcquire::~pkgAcquire()
  63. {
  64. Shutdown();
  65. while (Configs != 0)
  66. {
  67. MethodConfig *Jnk = Configs;
  68. Configs = Configs->Next;
  69. delete Jnk;
  70. }
  71. }
  72. /*}}}*/
  73. // Acquire::Shutdown - Clean out the acquire object /*{{{*/
  74. // ---------------------------------------------------------------------
  75. /* */
  76. void pkgAcquire::Shutdown()
  77. {
  78. while (Items.size() != 0)
  79. {
  80. if (Items[0]->Status == Item::StatFetching)
  81. Items[0]->Status = Item::StatError;
  82. delete Items[0];
  83. }
  84. while (Queues != 0)
  85. {
  86. Queue *Jnk = Queues;
  87. Queues = Queues->Next;
  88. delete Jnk;
  89. }
  90. }
  91. /*}}}*/
  92. // Acquire::Add - Add a new item /*{{{*/
  93. // ---------------------------------------------------------------------
  94. /* This puts an item on the acquire list. This list is mainly for tracking
  95. item status */
  96. void pkgAcquire::Add(Item *Itm)
  97. {
  98. Items.push_back(Itm);
  99. }
  100. /*}}}*/
  101. // Acquire::Remove - Remove a item /*{{{*/
  102. // ---------------------------------------------------------------------
  103. /* Remove an item from the acquire list. This is usually not used.. */
  104. void pkgAcquire::Remove(Item *Itm)
  105. {
  106. Dequeue(Itm);
  107. for (ItemIterator I = Items.begin(); I != Items.end();)
  108. {
  109. if (*I == Itm)
  110. {
  111. Items.erase(I);
  112. I = Items.begin();
  113. }
  114. else
  115. I++;
  116. }
  117. }
  118. /*}}}*/
  119. // Acquire::Add - Add a worker /*{{{*/
  120. // ---------------------------------------------------------------------
  121. /* A list of workers is kept so that the select loop can direct their FD
  122. usage. */
  123. void pkgAcquire::Add(Worker *Work)
  124. {
  125. Work->NextAcquire = Workers;
  126. Workers = Work;
  127. }
  128. /*}}}*/
  129. // Acquire::Remove - Remove a worker /*{{{*/
  130. // ---------------------------------------------------------------------
  131. /* A worker has died. This can not be done while the select loop is running
  132. as it would require that RunFds could handling a changing list state and
  133. it cant.. */
  134. void pkgAcquire::Remove(Worker *Work)
  135. {
  136. if (Running == true)
  137. abort();
  138. Worker **I = &Workers;
  139. for (; *I != 0;)
  140. {
  141. if (*I == Work)
  142. *I = (*I)->NextAcquire;
  143. else
  144. I = &(*I)->NextAcquire;
  145. }
  146. }
  147. /*}}}*/
  148. // Acquire::Enqueue - Queue an URI for fetching /*{{{*/
  149. // ---------------------------------------------------------------------
  150. /* This is the entry point for an item. An item calls this function when
  151. it is constructed which creates a queue (based on the current queue
  152. mode) and puts the item in that queue. If the system is running then
  153. the queue might be started. */
  154. void pkgAcquire::Enqueue(ItemDesc &Item)
  155. {
  156. // Determine which queue to put the item in
  157. const MethodConfig *Config;
  158. string Name = QueueName(Item.URI,Config);
  159. if (Name.empty() == true)
  160. return;
  161. // Find the queue structure
  162. Queue *I = Queues;
  163. for (; I != 0 && I->Name != Name; I = I->Next);
  164. if (I == 0)
  165. {
  166. I = new Queue(Name,this);
  167. I->Next = Queues;
  168. Queues = I;
  169. if (Running == true)
  170. I->Startup();
  171. }
  172. // See if this is a local only URI
  173. if (Config->LocalOnly == true && Item.Owner->Complete == false)
  174. Item.Owner->Local = true;
  175. Item.Owner->Status = Item::StatIdle;
  176. // Queue it into the named queue
  177. I->Enqueue(Item);
  178. ToFetch++;
  179. // Some trace stuff
  180. if (Debug == true)
  181. {
  182. clog << "Fetching " << Item.URI << endl;
  183. clog << " to " << Item.Owner->DestFile << endl;
  184. clog << " Queue is: " << Name << endl;
  185. }
  186. }
  187. /*}}}*/
  188. // Acquire::Dequeue - Remove an item from all queues /*{{{*/
  189. // ---------------------------------------------------------------------
  190. /* This is called when an item is finished being fetched. It removes it
  191. from all the queues */
  192. void pkgAcquire::Dequeue(Item *Itm)
  193. {
  194. Queue *I = Queues;
  195. bool Res = false;
  196. for (; I != 0; I = I->Next)
  197. Res |= I->Dequeue(Itm);
  198. if (Debug == true)
  199. clog << "Dequeuing " << Itm->DestFile << endl;
  200. if (Res == true)
  201. ToFetch--;
  202. }
  203. /*}}}*/
  204. // Acquire::QueueName - Return the name of the queue for this URI /*{{{*/
  205. // ---------------------------------------------------------------------
  206. /* The string returned depends on the configuration settings and the
  207. method parameters. Given something like http://foo.org/bar it can
  208. return http://foo.org or http */
  209. string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
  210. {
  211. URI U(Uri);
  212. Config = GetConfig(U.Access);
  213. if (Config == 0)
  214. return string();
  215. /* Single-Instance methods get exactly one queue per URI. This is
  216. also used for the Access queue method */
  217. if (Config->SingleInstance == true || QueueMode == QueueAccess)
  218. return U.Access;
  219. return U.Access + ':' + U.Host;
  220. }
  221. /*}}}*/
  222. // Acquire::GetConfig - Fetch the configuration information /*{{{*/
  223. // ---------------------------------------------------------------------
  224. /* This locates the configuration structure for an access method. If
  225. a config structure cannot be found a Worker will be created to
  226. retrieve it */
  227. pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
  228. {
  229. // Search for an existing config
  230. MethodConfig *Conf;
  231. for (Conf = Configs; Conf != 0; Conf = Conf->Next)
  232. if (Conf->Access == Access)
  233. return Conf;
  234. // Create the new config class
  235. Conf = new MethodConfig;
  236. Conf->Access = Access;
  237. Conf->Next = Configs;
  238. Configs = Conf;
  239. // Create the worker to fetch the configuration
  240. Worker Work(Conf);
  241. if (Work.Start() == false)
  242. return 0;
  243. return Conf;
  244. }
  245. /*}}}*/
  246. // Acquire::SetFds - Deal with readable FDs /*{{{*/
  247. // ---------------------------------------------------------------------
  248. /* Collect FDs that have activity monitors into the fd sets */
  249. void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
  250. {
  251. for (Worker *I = Workers; I != 0; I = I->NextAcquire)
  252. {
  253. if (I->InReady == true && I->InFd >= 0)
  254. {
  255. if (Fd < I->InFd)
  256. Fd = I->InFd;
  257. FD_SET(I->InFd,RSet);
  258. }
  259. if (I->OutReady == true && I->OutFd >= 0)
  260. {
  261. if (Fd < I->OutFd)
  262. Fd = I->OutFd;
  263. FD_SET(I->OutFd,WSet);
  264. }
  265. }
  266. }
  267. /*}}}*/
  268. // Acquire::RunFds - Deal with active FDs /*{{{*/
  269. // ---------------------------------------------------------------------
  270. /* Dispatch active FDs over to the proper workers. It is very important
  271. that a worker never be erased while this is running! The queue class
  272. should never erase a worker except during shutdown processing. */
  273. void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
  274. {
  275. for (Worker *I = Workers; I != 0; I = I->NextAcquire)
  276. {
  277. if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0)
  278. I->InFdReady();
  279. if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0)
  280. I->OutFdReady();
  281. }
  282. }
  283. /*}}}*/
  284. // Acquire::Run - Run the fetch sequence /*{{{*/
  285. // ---------------------------------------------------------------------
  286. /* This runs the queues. It manages a select loop for all of the
  287. Worker tasks. The workers interact with the queues and items to
  288. manage the actual fetch. */
  289. pkgAcquire::RunResult pkgAcquire::Run(int PulseIntervall)
  290. {
  291. Running = true;
  292. for (Queue *I = Queues; I != 0; I = I->Next)
  293. I->Startup();
  294. if (Log != 0)
  295. Log->Start();
  296. bool WasCancelled = false;
  297. // Run till all things have been acquired
  298. struct timeval tv;
  299. tv.tv_sec = 0;
  300. tv.tv_usec = PulseIntervall;
  301. while (ToFetch > 0)
  302. {
  303. fd_set RFds;
  304. fd_set WFds;
  305. int Highest = 0;
  306. FD_ZERO(&RFds);
  307. FD_ZERO(&WFds);
  308. SetFds(Highest,&RFds,&WFds);
  309. int Res;
  310. do
  311. {
  312. Res = select(Highest+1,&RFds,&WFds,0,&tv);
  313. }
  314. while (Res < 0 && errno == EINTR);
  315. if (Res < 0)
  316. {
  317. _error->Errno("select","Select has failed");
  318. break;
  319. }
  320. RunFds(&RFds,&WFds);
  321. if (_error->PendingError() == true)
  322. break;
  323. // Timeout, notify the log class
  324. if (Res == 0 || (Log != 0 && Log->Update == true))
  325. {
  326. tv.tv_usec = PulseIntervall;
  327. for (Worker *I = Workers; I != 0; I = I->NextAcquire)
  328. I->Pulse();
  329. if (Log != 0 && Log->Pulse(this) == false)
  330. {
  331. WasCancelled = true;
  332. break;
  333. }
  334. }
  335. }
  336. if (Log != 0)
  337. Log->Stop();
  338. // Shut down the acquire bits
  339. Running = false;
  340. for (Queue *I = Queues; I != 0; I = I->Next)
  341. I->Shutdown(false);
  342. // Shut down the items
  343. for (ItemIterator I = Items.begin(); I != Items.end(); I++)
  344. (*I)->Finished();
  345. if (_error->PendingError())
  346. return Failed;
  347. if (WasCancelled)
  348. return Cancelled;
  349. return Continue;
  350. }
  351. /*}}}*/
  352. // Acquire::Bump - Called when an item is dequeued /*{{{*/
  353. // ---------------------------------------------------------------------
  354. /* This routine bumps idle queues in hopes that they will be able to fetch
  355. the dequeued item */
  356. void pkgAcquire::Bump()
  357. {
  358. for (Queue *I = Queues; I != 0; I = I->Next)
  359. I->Bump();
  360. }
  361. /*}}}*/
  362. // Acquire::WorkerStep - Step to the next worker /*{{{*/
  363. // ---------------------------------------------------------------------
  364. /* Not inlined to advoid including acquire-worker.h */
  365. pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I)
  366. {
  367. return I->NextAcquire;
  368. };
  369. /*}}}*/
  370. // Acquire::Clean - Cleans a directory /*{{{*/
  371. // ---------------------------------------------------------------------
  372. /* This is a bit simplistic, it looks at every file in the dir and sees
  373. if it is part of the download set. */
  374. bool pkgAcquire::Clean(string Dir)
  375. {
  376. DIR *D = opendir(Dir.c_str());
  377. if (D == 0)
  378. return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
  379. string StartDir = SafeGetCWD();
  380. if (chdir(Dir.c_str()) != 0)
  381. {
  382. closedir(D);
  383. return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str());
  384. }
  385. for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
  386. {
  387. // Skip some files..
  388. if (strcmp(Dir->d_name,"lock") == 0 ||
  389. strcmp(Dir->d_name,"partial") == 0 ||
  390. strcmp(Dir->d_name,".") == 0 ||
  391. strcmp(Dir->d_name,"..") == 0)
  392. continue;
  393. // Look in the get list
  394. ItemCIterator I = Items.begin();
  395. for (; I != Items.end(); I++)
  396. if (flNotDir((*I)->DestFile) == Dir->d_name)
  397. break;
  398. // Nothing found, nuke it
  399. if (I == Items.end())
  400. unlink(Dir->d_name);
  401. };
  402. chdir(StartDir.c_str());
  403. closedir(D);
  404. return true;
  405. }
  406. /*}}}*/
  407. // Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/
  408. // ---------------------------------------------------------------------
  409. /* This is the total number of bytes needed */
  410. double pkgAcquire::TotalNeeded()
  411. {
  412. double Total = 0;
  413. for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++)
  414. Total += (*I)->FileSize;
  415. return Total;
  416. }
  417. /*}}}*/
  418. // Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/
  419. // ---------------------------------------------------------------------
  420. /* This is the number of bytes that is not local */
  421. double pkgAcquire::FetchNeeded()
  422. {
  423. double Total = 0;
  424. for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++)
  425. if ((*I)->Local == false)
  426. Total += (*I)->FileSize;
  427. return Total;
  428. }
  429. /*}}}*/
  430. // Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/
  431. // ---------------------------------------------------------------------
  432. /* This is the number of bytes that is not local */
  433. double pkgAcquire::PartialPresent()
  434. {
  435. double Total = 0;
  436. for (ItemCIterator I = ItemsBegin(); I != ItemsEnd(); I++)
  437. if ((*I)->Local == false)
  438. Total += (*I)->PartialSize;
  439. return Total;
  440. }
  441. // Acquire::UriBegin - Start iterator for the uri list /*{{{*/
  442. // ---------------------------------------------------------------------
  443. /* */
  444. pkgAcquire::UriIterator pkgAcquire::UriBegin()
  445. {
  446. return UriIterator(Queues);
  447. }
  448. /*}}}*/
  449. // Acquire::UriEnd - End iterator for the uri list /*{{{*/
  450. // ---------------------------------------------------------------------
  451. /* */
  452. pkgAcquire::UriIterator pkgAcquire::UriEnd()
  453. {
  454. return UriIterator(0);
  455. }
  456. /*}}}*/
  457. // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/
  458. // ---------------------------------------------------------------------
  459. /* */
  460. pkgAcquire::MethodConfig::MethodConfig()
  461. {
  462. SingleInstance = false;
  463. Pipeline = false;
  464. SendConfig = false;
  465. LocalOnly = false;
  466. Removable = false;
  467. Next = 0;
  468. }
  469. /*}}}*/
  470. // Queue::Queue - Constructor /*{{{*/
  471. // ---------------------------------------------------------------------
  472. /* */
  473. pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : Name(Name),
  474. Owner(Owner)
  475. {
  476. Items = 0;
  477. Next = 0;
  478. Workers = 0;
  479. MaxPipeDepth = 1;
  480. PipeDepth = 0;
  481. }
  482. /*}}}*/
  483. // Queue::~Queue - Destructor /*{{{*/
  484. // ---------------------------------------------------------------------
  485. /* */
  486. pkgAcquire::Queue::~Queue()
  487. {
  488. Shutdown(true);
  489. while (Items != 0)
  490. {
  491. QItem *Jnk = Items;
  492. Items = Items->Next;
  493. delete Jnk;
  494. }
  495. }
  496. /*}}}*/
  497. // Queue::Enqueue - Queue an item to the queue /*{{{*/
  498. // ---------------------------------------------------------------------
  499. /* */
  500. void pkgAcquire::Queue::Enqueue(ItemDesc &Item)
  501. {
  502. QItem **I = &Items;
  503. for (; *I != 0; I = &(*I)->Next);
  504. // Create a new item
  505. QItem *Itm = new QItem;
  506. *Itm = Item;
  507. Itm->Next = 0;
  508. *I = Itm;
  509. Item.Owner->QueueCounter++;
  510. if (Items->Next == 0)
  511. Cycle();
  512. }
  513. /*}}}*/
  514. // Queue::Dequeue - Remove an item from the queue /*{{{*/
  515. // ---------------------------------------------------------------------
  516. /* We return true if we hit something */
  517. bool pkgAcquire::Queue::Dequeue(Item *Owner)
  518. {
  519. if (Owner->Status == pkgAcquire::Item::StatFetching)
  520. return _error->Error("Tried to dequeue a fetching object");
  521. bool Res = false;
  522. QItem **I = &Items;
  523. for (; *I != 0;)
  524. {
  525. if ((*I)->Owner == Owner)
  526. {
  527. QItem *Jnk= *I;
  528. *I = (*I)->Next;
  529. Owner->QueueCounter--;
  530. delete Jnk;
  531. Res = true;
  532. }
  533. else
  534. I = &(*I)->Next;
  535. }
  536. return Res;
  537. }
  538. /*}}}*/
  539. // Queue::Startup - Start the worker processes /*{{{*/
  540. // ---------------------------------------------------------------------
  541. /* It is possible for this to be called with a pre-existing set of
  542. workers. */
  543. bool pkgAcquire::Queue::Startup()
  544. {
  545. if (Workers == 0)
  546. {
  547. URI U(Name);
  548. pkgAcquire::MethodConfig *Cnf = Owner->GetConfig(U.Access);
  549. if (Cnf == 0)
  550. return false;
  551. Workers = new Worker(this,Cnf,Owner->Log);
  552. Owner->Add(Workers);
  553. if (Workers->Start() == false)
  554. return false;
  555. /* When pipelining we commit 10 items. This needs to change when we
  556. added other source retry to have cycle maintain a pipeline depth
  557. on its own. */
  558. if (Cnf->Pipeline == true)
  559. MaxPipeDepth = 10;
  560. else
  561. MaxPipeDepth = 1;
  562. }
  563. return Cycle();
  564. }
  565. /*}}}*/
  566. // Queue::Shutdown - Shutdown the worker processes /*{{{*/
  567. // ---------------------------------------------------------------------
  568. /* If final is true then all workers are eliminated, otherwise only workers
  569. that do not need cleanup are removed */
  570. bool pkgAcquire::Queue::Shutdown(bool Final)
  571. {
  572. // Delete all of the workers
  573. pkgAcquire::Worker **Cur = &Workers;
  574. while (*Cur != 0)
  575. {
  576. pkgAcquire::Worker *Jnk = *Cur;
  577. if (Final == true || Jnk->GetConf()->NeedsCleanup == false)
  578. {
  579. *Cur = Jnk->NextQueue;
  580. Owner->Remove(Jnk);
  581. delete Jnk;
  582. }
  583. else
  584. Cur = &(*Cur)->NextQueue;
  585. }
  586. return true;
  587. }
  588. /*}}}*/
  589. // Queue::FindItem - Find a URI in the item list /*{{{*/
  590. // ---------------------------------------------------------------------
  591. /* */
  592. pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner)
  593. {
  594. for (QItem *I = Items; I != 0; I = I->Next)
  595. if (I->URI == URI && I->Worker == Owner)
  596. return I;
  597. return 0;
  598. }
  599. /*}}}*/
  600. // Queue::ItemDone - Item has been completed /*{{{*/
  601. // ---------------------------------------------------------------------
  602. /* The worker signals this which causes the item to be removed from the
  603. queue. If this is the last queue instance then it is removed from the
  604. main queue too.*/
  605. bool pkgAcquire::Queue::ItemDone(QItem *Itm)
  606. {
  607. PipeDepth--;
  608. if (Itm->Owner->Status == pkgAcquire::Item::StatFetching)
  609. Itm->Owner->Status = pkgAcquire::Item::StatDone;
  610. if (Itm->Owner->QueueCounter <= 1)
  611. Owner->Dequeue(Itm->Owner);
  612. else
  613. {
  614. Dequeue(Itm->Owner);
  615. Owner->Bump();
  616. }
  617. return Cycle();
  618. }
  619. /*}}}*/
  620. // Queue::Cycle - Queue new items into the method /*{{{*/
  621. // ---------------------------------------------------------------------
  622. /* This locates a new idle item and sends it to the worker. If pipelining
  623. is enabled then it keeps the pipe full. */
  624. bool pkgAcquire::Queue::Cycle()
  625. {
  626. if (Items == 0 || Workers == 0)
  627. return true;
  628. if (PipeDepth < 0)
  629. return _error->Error("Pipedepth failure");
  630. // Look for a queable item
  631. QItem *I = Items;
  632. while (PipeDepth < (signed)MaxPipeDepth)
  633. {
  634. for (; I != 0; I = I->Next)
  635. if (I->Owner->Status == pkgAcquire::Item::StatIdle)
  636. break;
  637. // Nothing to do, queue is idle.
  638. if (I == 0)
  639. return true;
  640. I->Worker = Workers;
  641. I->Owner->Status = pkgAcquire::Item::StatFetching;
  642. PipeDepth++;
  643. if (Workers->QueueItem(I) == false)
  644. return false;
  645. }
  646. return true;
  647. }
  648. /*}}}*/
  649. // Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
  650. // ---------------------------------------------------------------------
  651. /* This is called when an item in multiple queues is dequeued */
  652. void pkgAcquire::Queue::Bump()
  653. {
  654. Cycle();
  655. }
  656. /*}}}*/
  657. // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/
  658. // ---------------------------------------------------------------------
  659. /* */
  660. pkgAcquireStatus::pkgAcquireStatus() : Update(true), MorePulses(false)
  661. {
  662. Start();
  663. }
  664. /*}}}*/
  665. // AcquireStatus::Pulse - Called periodically /*{{{*/
  666. // ---------------------------------------------------------------------
  667. /* This computes some internal state variables for the derived classes to
  668. use. It generates the current downloaded bytes and total bytes to download
  669. as well as the current CPS estimate. */
  670. bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
  671. {
  672. TotalBytes = 0;
  673. CurrentBytes = 0;
  674. TotalItems = 0;
  675. CurrentItems = 0;
  676. // Compute the total number of bytes to fetch
  677. unsigned int Unknown = 0;
  678. unsigned int Count = 0;
  679. for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); I != Owner->ItemsEnd();
  680. I++, Count++)
  681. {
  682. TotalItems++;
  683. if ((*I)->Status == pkgAcquire::Item::StatDone)
  684. CurrentItems++;
  685. // Totally ignore local items
  686. if ((*I)->Local == true)
  687. continue;
  688. TotalBytes += (*I)->FileSize;
  689. if ((*I)->Complete == true)
  690. CurrentBytes += (*I)->FileSize;
  691. if ((*I)->FileSize == 0 && (*I)->Complete == false)
  692. Unknown++;
  693. }
  694. // Compute the current completion
  695. unsigned long ResumeSize = 0;
  696. for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
  697. I = Owner->WorkerStep(I))
  698. if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false)
  699. {
  700. CurrentBytes += I->CurrentSize;
  701. ResumeSize += I->ResumePoint;
  702. // Files with unknown size always have 100% completion
  703. if (I->CurrentItem->Owner->FileSize == 0 &&
  704. I->CurrentItem->Owner->Complete == false)
  705. TotalBytes += I->CurrentSize;
  706. }
  707. // Normalize the figures and account for unknown size downloads
  708. if (TotalBytes <= 0)
  709. TotalBytes = 1;
  710. if (Unknown == Count)
  711. TotalBytes = Unknown;
  712. // Wha?! Is not supposed to happen.
  713. if (CurrentBytes > TotalBytes)
  714. CurrentBytes = TotalBytes;
  715. // Compute the CPS
  716. struct timeval NewTime;
  717. gettimeofday(&NewTime,0);
  718. if (NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec ||
  719. NewTime.tv_sec - Time.tv_sec > 6)
  720. {
  721. double Delta = NewTime.tv_sec - Time.tv_sec +
  722. (NewTime.tv_usec - Time.tv_usec)/1000000.0;
  723. // Compute the CPS value
  724. if (Delta < 0.01)
  725. CurrentCPS = 0;
  726. else
  727. CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/Delta;
  728. LastBytes = CurrentBytes - ResumeSize;
  729. ElapsedTime = (unsigned long)Delta;
  730. Time = NewTime;
  731. }
  732. int fd = _config->FindI("APT::Status-Fd",-1);
  733. if(fd > 0)
  734. {
  735. ostringstream status;
  736. char msg[200];
  737. long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems;
  738. unsigned long ETA =
  739. (unsigned long)((TotalBytes - CurrentBytes) / CurrentCPS);
  740. snprintf(msg,sizeof(msg), _("Downloading file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
  741. // build the status str
  742. status << "dlstatus:" << i
  743. << ":" << (CurrentBytes/float(TotalBytes)*100.0)
  744. << ":" << msg
  745. << endl;
  746. write(fd, status.str().c_str(), status.str().size());
  747. }
  748. return true;
  749. }
  750. /*}}}*/
  751. // AcquireStatus::Start - Called when the download is started /*{{{*/
  752. // ---------------------------------------------------------------------
  753. /* We just reset the counters */
  754. void pkgAcquireStatus::Start()
  755. {
  756. gettimeofday(&Time,0);
  757. gettimeofday(&StartTime,0);
  758. LastBytes = 0;
  759. CurrentCPS = 0;
  760. CurrentBytes = 0;
  761. TotalBytes = 0;
  762. FetchedBytes = 0;
  763. ElapsedTime = 0;
  764. TotalItems = 0;
  765. CurrentItems = 0;
  766. }
  767. /*}}}*/
  768. // AcquireStatus::Stop - Finished downloading /*{{{*/
  769. // ---------------------------------------------------------------------
  770. /* This accurately computes the elapsed time and the total overall CPS. */
  771. void pkgAcquireStatus::Stop()
  772. {
  773. // Compute the CPS and elapsed time
  774. struct timeval NewTime;
  775. gettimeofday(&NewTime,0);
  776. double Delta = NewTime.tv_sec - StartTime.tv_sec +
  777. (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
  778. // Compute the CPS value
  779. if (Delta < 0.01)
  780. CurrentCPS = 0;
  781. else
  782. CurrentCPS = FetchedBytes/Delta;
  783. LastBytes = CurrentBytes;
  784. ElapsedTime = (unsigned int)Delta;
  785. }
  786. /*}}}*/
  787. // AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
  788. // ---------------------------------------------------------------------
  789. /* This is used to get accurate final transfer rate reporting. */
  790. void pkgAcquireStatus::Fetched(unsigned long Size,unsigned long Resume)
  791. {
  792. FetchedBytes += Size - Resume;
  793. }
  794. /*}}}*/