http.cc 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846
  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
  4. /* ######################################################################
  5. HTTP Acquire Method - This is the HTTP acquire method for APT.
  6. It uses HTTP/1.1 and many of the fancy options there-in, such as
  7. pipelining, range, if-range and so on.
  8. It is based on a doubly buffered select loop. A groupe of requests are
  9. fed into a single output buffer that is constantly fed out the
  10. socket. This provides ideal pipelining as in many cases all of the
  11. requests will fit into a single packet. The input socket is buffered
  12. the same way and fed into the fd for the file (may be a pipe in future).
  13. This double buffering provides fairly substantial transfer rates,
  14. compared to wget the http method is about 4% faster. Most importantly,
  15. when HTTP is compared with FTP as a protocol the speed difference is
  16. huge. In tests over the internet from two sites to llug (via ATM) this
  17. program got 230k/s sustained http transfer rates. FTP on the other
  18. hand topped out at 170k/s. That combined with the time to setup the
  19. FTP connection makes HTTP a vastly superior protocol.
  20. ##################################################################### */
  21. /*}}}*/
  22. // Include Files /*{{{*/
  23. #include <config.h>
  24. #include <apt-pkg/fileutl.h>
  25. #include <apt-pkg/acquire-method.h>
  26. #include <apt-pkg/configuration.h>
  27. #include <apt-pkg/error.h>
  28. #include <apt-pkg/hashes.h>
  29. #include <apt-pkg/netrc.h>
  30. #include <apt-pkg/strutl.h>
  31. #include <stddef.h>
  32. #include <stdlib.h>
  33. #include <sys/select.h>
  34. #include <cstring>
  35. #include <sys/stat.h>
  36. #include <sys/time.h>
  37. #include <unistd.h>
  38. #include <stdio.h>
  39. #include <errno.h>
  40. #include <iostream>
  41. #include <sstream>
  42. #include "config.h"
  43. #include "connect.h"
  44. #include "http.h"
  45. #include <apti18n.h>
  46. /*}}}*/
  47. using namespace std;
  48. unsigned long long CircleBuf::BwReadLimit=0;
  49. unsigned long long CircleBuf::BwTickReadData=0;
  50. struct timeval CircleBuf::BwReadTick={0,0};
  51. const unsigned int CircleBuf::BW_HZ=10;
  52. // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
  53. // ---------------------------------------------------------------------
  54. /* */
  55. CircleBuf::CircleBuf(unsigned long long Size)
  56. : Size(Size), Hash(0), TotalWriten(0)
  57. {
  58. Buf = new unsigned char[Size];
  59. Reset();
  60. CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  61. }
  62. /*}}}*/
  63. // CircleBuf::Reset - Reset to the default state /*{{{*/
  64. // ---------------------------------------------------------------------
  65. /* */
  66. void CircleBuf::Reset()
  67. {
  68. InP = 0;
  69. OutP = 0;
  70. StrPos = 0;
  71. TotalWriten = 0;
  72. MaxGet = (unsigned long long)-1;
  73. OutQueue = string();
  74. if (Hash != 0)
  75. {
  76. delete Hash;
  77. Hash = new Hashes;
  78. }
  79. }
  80. /*}}}*/
  81. // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
  82. // ---------------------------------------------------------------------
  83. /* This fills up the buffer with as much data as is in the FD, assuming it
  84. is non-blocking.. */
  85. bool CircleBuf::Read(int Fd)
  86. {
  87. while (1)
  88. {
  89. // Woops, buffer is full
  90. if (InP - OutP == Size)
  91. return true;
  92. // what's left to read in this tick
  93. unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
  94. if(CircleBuf::BwReadLimit) {
  95. struct timeval now;
  96. gettimeofday(&now,0);
  97. unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
  98. now.tv_usec-CircleBuf::BwReadTick.tv_usec;
  99. if(d > 1000000/BW_HZ) {
  100. CircleBuf::BwReadTick = now;
  101. CircleBuf::BwTickReadData = 0;
  102. }
  103. if(CircleBuf::BwTickReadData >= BwReadMax) {
  104. usleep(1000000/BW_HZ);
  105. return true;
  106. }
  107. }
  108. // Write the buffer segment
  109. ssize_t Res;
  110. if(CircleBuf::BwReadLimit) {
  111. Res = read(Fd,Buf + (InP%Size),
  112. BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
  113. } else
  114. Res = read(Fd,Buf + (InP%Size),LeftRead());
  115. if(Res > 0 && BwReadLimit > 0)
  116. CircleBuf::BwTickReadData += Res;
  117. if (Res == 0)
  118. return false;
  119. if (Res < 0)
  120. {
  121. if (errno == EAGAIN)
  122. return true;
  123. return false;
  124. }
  125. if (InP == 0)
  126. gettimeofday(&Start,0);
  127. InP += Res;
  128. }
  129. }
  130. /*}}}*/
  131. // CircleBuf::Read - Put the string into the buffer /*{{{*/
  132. // ---------------------------------------------------------------------
  133. /* This will hold the string in and fill the buffer with it as it empties */
  134. bool CircleBuf::Read(string Data)
  135. {
  136. OutQueue += Data;
  137. FillOut();
  138. return true;
  139. }
  140. /*}}}*/
  141. // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
  142. // ---------------------------------------------------------------------
  143. /* */
  144. void CircleBuf::FillOut()
  145. {
  146. if (OutQueue.empty() == true)
  147. return;
  148. while (1)
  149. {
  150. // Woops, buffer is full
  151. if (InP - OutP == Size)
  152. return;
  153. // Write the buffer segment
  154. unsigned long long Sz = LeftRead();
  155. if (OutQueue.length() - StrPos < Sz)
  156. Sz = OutQueue.length() - StrPos;
  157. memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
  158. // Advance
  159. StrPos += Sz;
  160. InP += Sz;
  161. if (OutQueue.length() == StrPos)
  162. {
  163. StrPos = 0;
  164. OutQueue = "";
  165. return;
  166. }
  167. }
  168. }
  169. /*}}}*/
  170. // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
  171. // ---------------------------------------------------------------------
  172. /* This empties the buffer into the FD. */
  173. bool CircleBuf::Write(int Fd)
  174. {
  175. while (1)
  176. {
  177. FillOut();
  178. // Woops, buffer is empty
  179. if (OutP == InP)
  180. return true;
  181. if (OutP == MaxGet)
  182. return true;
  183. // Write the buffer segment
  184. ssize_t Res;
  185. Res = write(Fd,Buf + (OutP%Size),LeftWrite());
  186. if (Res == 0)
  187. return false;
  188. if (Res < 0)
  189. {
  190. if (errno == EAGAIN)
  191. return true;
  192. return false;
  193. }
  194. TotalWriten += Res;
  195. if (Hash != 0)
  196. Hash->Add(Buf + (OutP%Size),Res);
  197. OutP += Res;
  198. }
  199. }
  200. /*}}}*/
  201. // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
  202. // ---------------------------------------------------------------------
  203. /* This copies till the first empty line */
  204. bool CircleBuf::WriteTillEl(string &Data,bool Single)
  205. {
  206. // We cheat and assume it is unneeded to have more than one buffer load
  207. for (unsigned long long I = OutP; I < InP; I++)
  208. {
  209. if (Buf[I%Size] != '\n')
  210. continue;
  211. ++I;
  212. if (Single == false)
  213. {
  214. if (I < InP && Buf[I%Size] == '\r')
  215. ++I;
  216. if (I >= InP || Buf[I%Size] != '\n')
  217. continue;
  218. ++I;
  219. }
  220. Data = "";
  221. while (OutP < I)
  222. {
  223. unsigned long long Sz = LeftWrite();
  224. if (Sz == 0)
  225. return false;
  226. if (I - OutP < Sz)
  227. Sz = I - OutP;
  228. Data += string((char *)(Buf + (OutP%Size)),Sz);
  229. OutP += Sz;
  230. }
  231. return true;
  232. }
  233. return false;
  234. }
  235. /*}}}*/
  236. // CircleBuf::Stats - Print out stats information /*{{{*/
  237. // ---------------------------------------------------------------------
  238. /* */
  239. void CircleBuf::Stats()
  240. {
  241. if (InP == 0)
  242. return;
  243. struct timeval Stop;
  244. gettimeofday(&Stop,0);
  245. /* float Diff = Stop.tv_sec - Start.tv_sec +
  246. (float)(Stop.tv_usec - Start.tv_usec)/1000000;
  247. clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
  248. }
  249. /*}}}*/
  250. CircleBuf::~CircleBuf()
  251. {
  252. delete [] Buf;
  253. delete Hash;
  254. }
  255. // HttpServerState::HttpServerState - Constructor /*{{{*/
  256. HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024)
  257. {
  258. TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
  259. Reset();
  260. }
  261. /*}}}*/
  262. // HttpServerState::Open - Open a connection to the server /*{{{*/
  263. // ---------------------------------------------------------------------
  264. /* This opens a connection to the server. */
  265. bool HttpServerState::Open()
  266. {
  267. // Use the already open connection if possible.
  268. if (ServerFd != -1)
  269. return true;
  270. Close();
  271. In.Reset();
  272. Out.Reset();
  273. Persistent = true;
  274. // Determine the proxy setting
  275. string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
  276. if (!SpecificProxy.empty())
  277. {
  278. if (SpecificProxy == "DIRECT")
  279. Proxy = "";
  280. else
  281. Proxy = SpecificProxy;
  282. }
  283. else
  284. {
  285. string DefProxy = _config->Find("Acquire::http::Proxy");
  286. if (!DefProxy.empty())
  287. {
  288. Proxy = DefProxy;
  289. }
  290. else
  291. {
  292. char* result = getenv("http_proxy");
  293. Proxy = result ? result : "";
  294. }
  295. }
  296. // Parse no_proxy, a , separated list of domains
  297. if (getenv("no_proxy") != 0)
  298. {
  299. if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
  300. Proxy = "";
  301. }
  302. // Determine what host and port to use based on the proxy settings
  303. int Port = 0;
  304. string Host;
  305. if (Proxy.empty() == true || Proxy.Host.empty() == true)
  306. {
  307. if (ServerName.Port != 0)
  308. Port = ServerName.Port;
  309. Host = ServerName.Host;
  310. }
  311. else
  312. {
  313. if (Proxy.Port != 0)
  314. Port = Proxy.Port;
  315. Host = Proxy.Host;
  316. }
  317. // Connect to the remote server
  318. if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
  319. return false;
  320. return true;
  321. }
  322. /*}}}*/
  323. // HttpServerState::Close - Close a connection to the server /*{{{*/
  324. // ---------------------------------------------------------------------
  325. /* */
  326. bool HttpServerState::Close()
  327. {
  328. close(ServerFd);
  329. ServerFd = -1;
  330. return true;
  331. }
  332. /*}}}*/
  333. // HttpServerState::RunData - Transfer the data from the socket /*{{{*/
  334. bool HttpServerState::RunData(FileFd * const File)
  335. {
  336. State = Data;
  337. // Chunked transfer encoding is fun..
  338. if (Encoding == Chunked)
  339. {
  340. while (1)
  341. {
  342. // Grab the block size
  343. bool Last = true;
  344. string Data;
  345. In.Limit(-1);
  346. do
  347. {
  348. if (In.WriteTillEl(Data,true) == true)
  349. break;
  350. }
  351. while ((Last = Go(false, File)) == true);
  352. if (Last == false)
  353. return false;
  354. // See if we are done
  355. unsigned long long Len = strtoull(Data.c_str(),0,16);
  356. if (Len == 0)
  357. {
  358. In.Limit(-1);
  359. // We have to remove the entity trailer
  360. Last = true;
  361. do
  362. {
  363. if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
  364. break;
  365. }
  366. while ((Last = Go(false, File)) == true);
  367. if (Last == false)
  368. return false;
  369. return !_error->PendingError();
  370. }
  371. // Transfer the block
  372. In.Limit(Len);
  373. while (Go(true, File) == true)
  374. if (In.IsLimit() == true)
  375. break;
  376. // Error
  377. if (In.IsLimit() == false)
  378. return false;
  379. // The server sends an extra new line before the next block specifier..
  380. In.Limit(-1);
  381. Last = true;
  382. do
  383. {
  384. if (In.WriteTillEl(Data,true) == true)
  385. break;
  386. }
  387. while ((Last = Go(false, File)) == true);
  388. if (Last == false)
  389. return false;
  390. }
  391. }
  392. else
  393. {
  394. /* Closes encoding is used when the server did not specify a size, the
  395. loss of the connection means we are done */
  396. if (Encoding == Closes)
  397. In.Limit(-1);
  398. else
  399. In.Limit(Size - StartPos);
  400. // Just transfer the whole block.
  401. do
  402. {
  403. if (In.IsLimit() == false)
  404. continue;
  405. In.Limit(-1);
  406. return !_error->PendingError();
  407. }
  408. while (Go(true, File) == true);
  409. }
  410. return Owner->Flush() && !_error->PendingError();
  411. }
  412. /*}}}*/
  413. bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
  414. {
  415. return In.WriteTillEl(Data);
  416. }
  417. /*}}}*/
  418. bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
  419. {
  420. return Go(ToFile, File);
  421. }
  422. /*}}}*/
  423. bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
  424. {
  425. return Out.Read(Data);
  426. }
  427. /*}}}*/
  428. APT_PURE bool HttpServerState::IsOpen() /*{{{*/
  429. {
  430. return (ServerFd != -1);
  431. }
  432. /*}}}*/
  433. bool HttpServerState::InitHashes(FileFd &File) /*{{{*/
  434. {
  435. delete In.Hash;
  436. In.Hash = new Hashes;
  437. // Set the expected size and read file for the hashes
  438. File.Truncate(StartPos);
  439. return In.Hash->AddFD(File, StartPos);
  440. }
  441. /*}}}*/
  442. APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/
  443. {
  444. return In.Hash;
  445. }
  446. /*}}}*/
  447. // HttpServerState::Die - The server has closed the connection. /*{{{*/
  448. bool HttpServerState::Die(FileFd &File)
  449. {
  450. unsigned int LErrno = errno;
  451. // Dump the buffer to the file
  452. if (State == ServerState::Data)
  453. {
  454. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  455. // can't be set
  456. if (File.Name() != "/dev/null")
  457. SetNonBlock(File.Fd(),false);
  458. while (In.WriteSpace() == true)
  459. {
  460. if (In.Write(File.Fd()) == false)
  461. return _error->Errno("write",_("Error writing to the file"));
  462. // Done
  463. if (In.IsLimit() == true)
  464. return true;
  465. }
  466. }
  467. // See if this is because the server finished the data stream
  468. if (In.IsLimit() == false && State != HttpServerState::Header &&
  469. Encoding != HttpServerState::Closes)
  470. {
  471. Close();
  472. if (LErrno == 0)
  473. return _error->Error(_("Error reading from server. Remote end closed connection"));
  474. errno = LErrno;
  475. return _error->Errno("read",_("Error reading from server"));
  476. }
  477. else
  478. {
  479. In.Limit(-1);
  480. // Nothing left in the buffer
  481. if (In.WriteSpace() == false)
  482. return false;
  483. // We may have got multiple responses back in one packet..
  484. Close();
  485. return true;
  486. }
  487. return false;
  488. }
  489. /*}}}*/
  490. // HttpServerState::Flush - Dump the buffer into the file /*{{{*/
  491. // ---------------------------------------------------------------------
  492. /* This takes the current input buffer from the Server FD and writes it
  493. into the file */
  494. bool HttpServerState::Flush(FileFd * const File)
  495. {
  496. if (File != NULL)
  497. {
  498. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  499. // can't be set
  500. if (File->Name() != "/dev/null")
  501. SetNonBlock(File->Fd(),false);
  502. if (In.WriteSpace() == false)
  503. return true;
  504. while (In.WriteSpace() == true)
  505. {
  506. if (In.Write(File->Fd()) == false)
  507. return _error->Errno("write",_("Error writing to file"));
  508. if (In.IsLimit() == true)
  509. return true;
  510. }
  511. if (In.IsLimit() == true || Encoding == ServerState::Closes)
  512. return true;
  513. }
  514. return false;
  515. }
  516. /*}}}*/
  517. // HttpServerState::Go - Run a single loop /*{{{*/
  518. // ---------------------------------------------------------------------
  519. /* This runs the select loop over the server FDs, Output file FDs and
  520. stdin. */
  521. bool HttpServerState::Go(bool ToFile, FileFd * const File)
  522. {
  523. // Server has closed the connection
  524. if (ServerFd == -1 && (In.WriteSpace() == false ||
  525. ToFile == false))
  526. return false;
  527. fd_set rfds,wfds;
  528. FD_ZERO(&rfds);
  529. FD_ZERO(&wfds);
  530. /* Add the server. We only send more requests if the connection will
  531. be persisting */
  532. if (Out.WriteSpace() == true && ServerFd != -1
  533. && Persistent == true)
  534. FD_SET(ServerFd,&wfds);
  535. if (In.ReadSpace() == true && ServerFd != -1)
  536. FD_SET(ServerFd,&rfds);
  537. // Add the file
  538. int FileFD = -1;
  539. if (File != NULL)
  540. FileFD = File->Fd();
  541. if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
  542. FD_SET(FileFD,&wfds);
  543. // Add stdin
  544. if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
  545. FD_SET(STDIN_FILENO,&rfds);
  546. // Figure out the max fd
  547. int MaxFd = FileFD;
  548. if (MaxFd < ServerFd)
  549. MaxFd = ServerFd;
  550. // Select
  551. struct timeval tv;
  552. tv.tv_sec = TimeOut;
  553. tv.tv_usec = 0;
  554. int Res = 0;
  555. if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
  556. {
  557. if (errno == EINTR)
  558. return true;
  559. return _error->Errno("select",_("Select failed"));
  560. }
  561. if (Res == 0)
  562. {
  563. _error->Error(_("Connection timed out"));
  564. return Die(*File);
  565. }
  566. // Handle server IO
  567. if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
  568. {
  569. errno = 0;
  570. if (In.Read(ServerFd) == false)
  571. return Die(*File);
  572. }
  573. if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
  574. {
  575. errno = 0;
  576. if (Out.Write(ServerFd) == false)
  577. return Die(*File);
  578. }
  579. // Send data to the file
  580. if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
  581. {
  582. if (In.Write(FileFD) == false)
  583. return _error->Errno("write",_("Error writing to output file"));
  584. }
  585. if (ExpectedSize > 0 && In.TotalWriten > ExpectedSize)
  586. return _error->Error("Writing more data than expected (%llu > %llu)",
  587. In.TotalWriten, ExpectedSize);
  588. // Handle commands from APT
  589. if (FD_ISSET(STDIN_FILENO,&rfds))
  590. {
  591. if (Owner->Run(true) != -1)
  592. exit(100);
  593. }
  594. return true;
  595. }
  596. /*}}}*/
  597. // HttpMethod::SendReq - Send the HTTP request /*{{{*/
  598. // ---------------------------------------------------------------------
  599. /* This places the http request in the outbound buffer */
  600. void HttpMethod::SendReq(FetchItem *Itm)
  601. {
  602. URI Uri = Itm->Uri;
  603. // The HTTP server expects a hostname with a trailing :port
  604. std::stringstream Req;
  605. string ProperHost;
  606. if (Uri.Host.find(':') != string::npos)
  607. ProperHost = '[' + Uri.Host + ']';
  608. else
  609. ProperHost = Uri.Host;
  610. /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
  611. but while its a must for all servers to accept absolute URIs,
  612. it is assumed clients will sent an absolute path for non-proxies */
  613. std::string requesturi;
  614. if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
  615. requesturi = Uri.Path;
  616. else
  617. requesturi = Itm->Uri;
  618. // The "+" is encoded as a workaround for a amazon S3 bug
  619. // see LP bugs #1003633 and #1086997.
  620. requesturi = QuoteString(requesturi, "+~ ");
  621. /* Build the request. No keep-alive is included as it is the default
  622. in 1.1, can cause problems with proxies, and we are an HTTP/1.1
  623. client anyway.
  624. C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
  625. Req << "GET " << requesturi << " HTTP/1.1\r\n";
  626. if (Uri.Port != 0)
  627. Req << "Host: " << ProperHost << ":" << Uri.Port << "\r\n";
  628. else
  629. Req << "Host: " << ProperHost << "\r\n";
  630. // generate a cache control header (if needed)
  631. if (_config->FindB("Acquire::http::No-Cache",false) == true)
  632. Req << "Cache-Control: no-cache\r\n"
  633. << "Pragma: no-cache\r\n";
  634. else if (Itm->IndexFile == true)
  635. Req << "Cache-Control: max-age=" << _config->FindI("Acquire::http::Max-Age",0) << "\r\n";
  636. else if (_config->FindB("Acquire::http::No-Store",false) == true)
  637. Req << "Cache-Control: no-store\r\n";
  638. // If we ask for uncompressed files servers might respond with content-
  639. // negotiation which lets us end up with compressed files we do not support,
  640. // see 657029, 657560 and co, so if we have no extension on the request
  641. // ask for text only. As a sidenote: If there is nothing to negotate servers
  642. // seem to be nice and ignore it.
  643. if (_config->FindB("Acquire::http::SendAccept", true) == true)
  644. {
  645. size_t const filepos = Itm->Uri.find_last_of('/');
  646. string const file = Itm->Uri.substr(filepos + 1);
  647. if (flExtension(file) == file)
  648. Req << "Accept: text/*\r\n";
  649. }
  650. // Check for a partial file and send if-queries accordingly
  651. struct stat SBuf;
  652. if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
  653. Req << "Range: bytes=" << SBuf.st_size << "-\r\n"
  654. << "If-Range: " << TimeRFC1123(SBuf.st_mtime) << "\r\n";
  655. else if (Itm->LastModified != 0)
  656. Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified).c_str() << "\r\n";
  657. if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
  658. Req << "Proxy-Authorization: Basic "
  659. << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
  660. maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
  661. if (Uri.User.empty() == false || Uri.Password.empty() == false)
  662. Req << "Authorization: Basic "
  663. << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
  664. Req << "User-Agent: " << _config->Find("Acquire::http::User-Agent",
  665. "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
  666. Req << "\r\n";
  667. if (Debug == true)
  668. cerr << Req << endl;
  669. Server->WriteResponse(Req.str());
  670. }
  671. /*}}}*/
  672. // HttpMethod::Configuration - Handle a configuration message /*{{{*/
  673. // ---------------------------------------------------------------------
  674. /* We stash the desired pipeline depth */
  675. bool HttpMethod::Configuration(string Message)
  676. {
  677. if (ServerMethod::Configuration(Message) == false)
  678. return false;
  679. AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
  680. PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
  681. PipelineDepth);
  682. Debug = _config->FindB("Debug::Acquire::http",false);
  683. // Get the proxy to use
  684. AutoDetectProxy();
  685. return true;
  686. }
  687. /*}}}*/
  688. // HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/
  689. // ---------------------------------------------------------------------
  690. /* */
  691. bool HttpMethod::AutoDetectProxy()
  692. {
  693. // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
  694. // name without the dash ("-")
  695. AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect",
  696. _config->Find("Acquire::http::ProxyAutoDetect"));
  697. if (AutoDetectProxyCmd.empty())
  698. return true;
  699. if (Debug)
  700. clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl;
  701. int Pipes[2] = {-1,-1};
  702. if (pipe(Pipes) != 0)
  703. return _error->Errno("pipe", "Failed to create Pipe");
  704. pid_t Process = ExecFork();
  705. if (Process == 0)
  706. {
  707. close(Pipes[0]);
  708. dup2(Pipes[1],STDOUT_FILENO);
  709. SetCloseExec(STDOUT_FILENO,false);
  710. const char *Args[2];
  711. Args[0] = AutoDetectProxyCmd.c_str();
  712. Args[1] = 0;
  713. execv(Args[0],(char **)Args);
  714. cerr << "Failed to exec method " << Args[0] << endl;
  715. _exit(100);
  716. }
  717. char buf[512];
  718. int InFd = Pipes[0];
  719. close(Pipes[1]);
  720. int res = read(InFd, buf, sizeof(buf)-1);
  721. ExecWait(Process, "ProxyAutoDetect", true);
  722. if (res < 0)
  723. return _error->Errno("read", "Failed to read");
  724. if (res == 0)
  725. return _error->Warning("ProxyAutoDetect returned no data");
  726. // add trailing \0
  727. buf[res] = 0;
  728. if (Debug)
  729. clog << "auto detect command returned: '" << buf << "'" << endl;
  730. if (strstr(buf, "http://") == buf)
  731. _config->Set("Acquire::http::proxy", _strstrip(buf));
  732. return true;
  733. }
  734. /*}}}*/
  735. ServerState * HttpMethod::CreateServerState(URI uri) /*{{{*/
  736. {
  737. return new HttpServerState(uri, this);
  738. }
  739. /*}}}*/
  740. void HttpMethod::RotateDNS() /*{{{*/
  741. {
  742. ::RotateDNS();
  743. }
  744. /*}}}*/