http.cc 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838
  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
  4. /* ######################################################################
  5. HTTP Acquire Method - This is the HTTP acquire method for APT.
  6. It uses HTTP/1.1 and many of the fancy options there-in, such as
  7. pipelining, range, if-range and so on.
  8. It is based on a doubly buffered select loop. A groupe of requests are
  9. fed into a single output buffer that is constantly fed out the
  10. socket. This provides ideal pipelining as in many cases all of the
  11. requests will fit into a single packet. The input socket is buffered
  12. the same way and fed into the fd for the file (may be a pipe in future).
  13. This double buffering provides fairly substantial transfer rates,
  14. compared to wget the http method is about 4% faster. Most importantly,
  15. when HTTP is compared with FTP as a protocol the speed difference is
  16. huge. In tests over the internet from two sites to llug (via ATM) this
  17. program got 230k/s sustained http transfer rates. FTP on the other
  18. hand topped out at 170k/s. That combined with the time to setup the
  19. FTP connection makes HTTP a vastly superior protocol.
  20. ##################################################################### */
  21. /*}}}*/
  22. // Include Files /*{{{*/
  23. #include <config.h>
  24. #include <apt-pkg/fileutl.h>
  25. #include <apt-pkg/acquire-method.h>
  26. #include <apt-pkg/configuration.h>
  27. #include <apt-pkg/error.h>
  28. #include <apt-pkg/hashes.h>
  29. #include <apt-pkg/netrc.h>
  30. #include <apt-pkg/strutl.h>
  31. #include <stddef.h>
  32. #include <stdlib.h>
  33. #include <sys/select.h>
  34. #include <cstring>
  35. #include <sys/stat.h>
  36. #include <sys/time.h>
  37. #include <unistd.h>
  38. #include <stdio.h>
  39. #include <errno.h>
  40. #include <iostream>
  41. #include <sstream>
  42. #include "config.h"
  43. #include "connect.h"
  44. #include "http.h"
  45. #include <apti18n.h>
  46. /*}}}*/
  47. using namespace std;
  48. unsigned long long CircleBuf::BwReadLimit=0;
  49. unsigned long long CircleBuf::BwTickReadData=0;
  50. struct timeval CircleBuf::BwReadTick={0,0};
  51. const unsigned int CircleBuf::BW_HZ=10;
  52. // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
  53. // ---------------------------------------------------------------------
  54. /* */
  55. CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0)
  56. {
  57. Buf = new unsigned char[Size];
  58. Reset();
  59. CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  60. }
  61. /*}}}*/
  62. // CircleBuf::Reset - Reset to the default state /*{{{*/
  63. // ---------------------------------------------------------------------
  64. /* */
  65. void CircleBuf::Reset()
  66. {
  67. InP = 0;
  68. OutP = 0;
  69. StrPos = 0;
  70. MaxGet = (unsigned long long)-1;
  71. OutQueue = string();
  72. if (Hash != 0)
  73. {
  74. delete Hash;
  75. Hash = new Hashes;
  76. }
  77. }
  78. /*}}}*/
  79. // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
  80. // ---------------------------------------------------------------------
  81. /* This fills up the buffer with as much data as is in the FD, assuming it
  82. is non-blocking.. */
  83. bool CircleBuf::Read(int Fd)
  84. {
  85. while (1)
  86. {
  87. // Woops, buffer is full
  88. if (InP - OutP == Size)
  89. return true;
  90. // what's left to read in this tick
  91. unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
  92. if(CircleBuf::BwReadLimit) {
  93. struct timeval now;
  94. gettimeofday(&now,0);
  95. unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
  96. now.tv_usec-CircleBuf::BwReadTick.tv_usec;
  97. if(d > 1000000/BW_HZ) {
  98. CircleBuf::BwReadTick = now;
  99. CircleBuf::BwTickReadData = 0;
  100. }
  101. if(CircleBuf::BwTickReadData >= BwReadMax) {
  102. usleep(1000000/BW_HZ);
  103. return true;
  104. }
  105. }
  106. // Write the buffer segment
  107. ssize_t Res;
  108. if(CircleBuf::BwReadLimit) {
  109. Res = read(Fd,Buf + (InP%Size),
  110. BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
  111. } else
  112. Res = read(Fd,Buf + (InP%Size),LeftRead());
  113. if(Res > 0 && BwReadLimit > 0)
  114. CircleBuf::BwTickReadData += Res;
  115. if (Res == 0)
  116. return false;
  117. if (Res < 0)
  118. {
  119. if (errno == EAGAIN)
  120. return true;
  121. return false;
  122. }
  123. if (InP == 0)
  124. gettimeofday(&Start,0);
  125. InP += Res;
  126. }
  127. }
  128. /*}}}*/
  129. // CircleBuf::Read - Put the string into the buffer /*{{{*/
  130. // ---------------------------------------------------------------------
  131. /* This will hold the string in and fill the buffer with it as it empties */
  132. bool CircleBuf::Read(string Data)
  133. {
  134. OutQueue += Data;
  135. FillOut();
  136. return true;
  137. }
  138. /*}}}*/
  139. // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
  140. // ---------------------------------------------------------------------
  141. /* */
  142. void CircleBuf::FillOut()
  143. {
  144. if (OutQueue.empty() == true)
  145. return;
  146. while (1)
  147. {
  148. // Woops, buffer is full
  149. if (InP - OutP == Size)
  150. return;
  151. // Write the buffer segment
  152. unsigned long long Sz = LeftRead();
  153. if (OutQueue.length() - StrPos < Sz)
  154. Sz = OutQueue.length() - StrPos;
  155. memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
  156. // Advance
  157. StrPos += Sz;
  158. InP += Sz;
  159. if (OutQueue.length() == StrPos)
  160. {
  161. StrPos = 0;
  162. OutQueue = "";
  163. return;
  164. }
  165. }
  166. }
  167. /*}}}*/
  168. // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
  169. // ---------------------------------------------------------------------
  170. /* This empties the buffer into the FD. */
  171. bool CircleBuf::Write(int Fd)
  172. {
  173. while (1)
  174. {
  175. FillOut();
  176. // Woops, buffer is empty
  177. if (OutP == InP)
  178. return true;
  179. if (OutP == MaxGet)
  180. return true;
  181. // Write the buffer segment
  182. ssize_t Res;
  183. Res = write(Fd,Buf + (OutP%Size),LeftWrite());
  184. if (Res == 0)
  185. return false;
  186. if (Res < 0)
  187. {
  188. if (errno == EAGAIN)
  189. return true;
  190. return false;
  191. }
  192. if (Hash != 0)
  193. Hash->Add(Buf + (OutP%Size),Res);
  194. OutP += Res;
  195. }
  196. }
  197. /*}}}*/
  198. // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
  199. // ---------------------------------------------------------------------
  200. /* This copies till the first empty line */
  201. bool CircleBuf::WriteTillEl(string &Data,bool Single)
  202. {
  203. // We cheat and assume it is unneeded to have more than one buffer load
  204. for (unsigned long long I = OutP; I < InP; I++)
  205. {
  206. if (Buf[I%Size] != '\n')
  207. continue;
  208. ++I;
  209. if (Single == false)
  210. {
  211. if (I < InP && Buf[I%Size] == '\r')
  212. ++I;
  213. if (I >= InP || Buf[I%Size] != '\n')
  214. continue;
  215. ++I;
  216. }
  217. Data = "";
  218. while (OutP < I)
  219. {
  220. unsigned long long Sz = LeftWrite();
  221. if (Sz == 0)
  222. return false;
  223. if (I - OutP < Sz)
  224. Sz = I - OutP;
  225. Data += string((char *)(Buf + (OutP%Size)),Sz);
  226. OutP += Sz;
  227. }
  228. return true;
  229. }
  230. return false;
  231. }
  232. /*}}}*/
  233. // CircleBuf::Stats - Print out stats information /*{{{*/
  234. // ---------------------------------------------------------------------
  235. /* */
  236. void CircleBuf::Stats()
  237. {
  238. if (InP == 0)
  239. return;
  240. struct timeval Stop;
  241. gettimeofday(&Stop,0);
  242. /* float Diff = Stop.tv_sec - Start.tv_sec +
  243. (float)(Stop.tv_usec - Start.tv_usec)/1000000;
  244. clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
  245. }
  246. /*}}}*/
  247. CircleBuf::~CircleBuf()
  248. {
  249. delete [] Buf;
  250. delete Hash;
  251. }
  252. // HttpServerState::HttpServerState - Constructor /*{{{*/
  253. HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024)
  254. {
  255. TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
  256. Reset();
  257. }
  258. /*}}}*/
  259. // HttpServerState::Open - Open a connection to the server /*{{{*/
  260. // ---------------------------------------------------------------------
  261. /* This opens a connection to the server. */
  262. bool HttpServerState::Open()
  263. {
  264. // Use the already open connection if possible.
  265. if (ServerFd != -1)
  266. return true;
  267. Close();
  268. In.Reset();
  269. Out.Reset();
  270. Persistent = true;
  271. // Determine the proxy setting
  272. string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
  273. if (!SpecificProxy.empty())
  274. {
  275. if (SpecificProxy == "DIRECT")
  276. Proxy = "";
  277. else
  278. Proxy = SpecificProxy;
  279. }
  280. else
  281. {
  282. string DefProxy = _config->Find("Acquire::http::Proxy");
  283. if (!DefProxy.empty())
  284. {
  285. Proxy = DefProxy;
  286. }
  287. else
  288. {
  289. char* result = getenv("http_proxy");
  290. Proxy = result ? result : "";
  291. }
  292. }
  293. // Parse no_proxy, a , separated list of domains
  294. if (getenv("no_proxy") != 0)
  295. {
  296. if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
  297. Proxy = "";
  298. }
  299. // Determine what host and port to use based on the proxy settings
  300. int Port = 0;
  301. string Host;
  302. if (Proxy.empty() == true || Proxy.Host.empty() == true)
  303. {
  304. if (ServerName.Port != 0)
  305. Port = ServerName.Port;
  306. Host = ServerName.Host;
  307. }
  308. else
  309. {
  310. if (Proxy.Port != 0)
  311. Port = Proxy.Port;
  312. Host = Proxy.Host;
  313. }
  314. // Connect to the remote server
  315. if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
  316. return false;
  317. return true;
  318. }
  319. /*}}}*/
  320. // HttpServerState::Close - Close a connection to the server /*{{{*/
  321. // ---------------------------------------------------------------------
  322. /* */
  323. bool HttpServerState::Close()
  324. {
  325. close(ServerFd);
  326. ServerFd = -1;
  327. return true;
  328. }
  329. /*}}}*/
  330. // HttpServerState::RunData - Transfer the data from the socket /*{{{*/
  331. bool HttpServerState::RunData(FileFd * const File)
  332. {
  333. State = Data;
  334. // Chunked transfer encoding is fun..
  335. if (Encoding == Chunked)
  336. {
  337. while (1)
  338. {
  339. // Grab the block size
  340. bool Last = true;
  341. string Data;
  342. In.Limit(-1);
  343. do
  344. {
  345. if (In.WriteTillEl(Data,true) == true)
  346. break;
  347. }
  348. while ((Last = Go(false, File)) == true);
  349. if (Last == false)
  350. return false;
  351. // See if we are done
  352. unsigned long long Len = strtoull(Data.c_str(),0,16);
  353. if (Len == 0)
  354. {
  355. In.Limit(-1);
  356. // We have to remove the entity trailer
  357. Last = true;
  358. do
  359. {
  360. if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
  361. break;
  362. }
  363. while ((Last = Go(false, File)) == true);
  364. if (Last == false)
  365. return false;
  366. return !_error->PendingError();
  367. }
  368. // Transfer the block
  369. In.Limit(Len);
  370. while (Go(true, File) == true)
  371. if (In.IsLimit() == true)
  372. break;
  373. // Error
  374. if (In.IsLimit() == false)
  375. return false;
  376. // The server sends an extra new line before the next block specifier..
  377. In.Limit(-1);
  378. Last = true;
  379. do
  380. {
  381. if (In.WriteTillEl(Data,true) == true)
  382. break;
  383. }
  384. while ((Last = Go(false, File)) == true);
  385. if (Last == false)
  386. return false;
  387. }
  388. }
  389. else
  390. {
  391. /* Closes encoding is used when the server did not specify a size, the
  392. loss of the connection means we are done */
  393. if (Encoding == Closes)
  394. In.Limit(-1);
  395. else
  396. In.Limit(Size - StartPos);
  397. // Just transfer the whole block.
  398. do
  399. {
  400. if (In.IsLimit() == false)
  401. continue;
  402. In.Limit(-1);
  403. return !_error->PendingError();
  404. }
  405. while (Go(true, File) == true);
  406. }
  407. return Owner->Flush() && !_error->PendingError();
  408. }
  409. /*}}}*/
  410. bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
  411. {
  412. return In.WriteTillEl(Data);
  413. }
  414. /*}}}*/
  415. bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
  416. {
  417. return Go(ToFile, File);
  418. }
  419. /*}}}*/
  420. bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
  421. {
  422. return Out.Read(Data);
  423. }
  424. /*}}}*/
  425. APT_PURE bool HttpServerState::IsOpen() /*{{{*/
  426. {
  427. return (ServerFd != -1);
  428. }
  429. /*}}}*/
  430. bool HttpServerState::InitHashes(FileFd &File) /*{{{*/
  431. {
  432. delete In.Hash;
  433. In.Hash = new Hashes;
  434. // Set the expected size and read file for the hashes
  435. File.Truncate(StartPos);
  436. return In.Hash->AddFD(File, StartPos);
  437. }
  438. /*}}}*/
  439. APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/
  440. {
  441. return In.Hash;
  442. }
  443. /*}}}*/
  444. // HttpServerState::Die - The server has closed the connection. /*{{{*/
  445. bool HttpServerState::Die(FileFd &File)
  446. {
  447. unsigned int LErrno = errno;
  448. // Dump the buffer to the file
  449. if (State == ServerState::Data)
  450. {
  451. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  452. // can't be set
  453. if (File.Name() != "/dev/null")
  454. SetNonBlock(File.Fd(),false);
  455. while (In.WriteSpace() == true)
  456. {
  457. if (In.Write(File.Fd()) == false)
  458. return _error->Errno("write",_("Error writing to the file"));
  459. // Done
  460. if (In.IsLimit() == true)
  461. return true;
  462. }
  463. }
  464. // See if this is because the server finished the data stream
  465. if (In.IsLimit() == false && State != HttpServerState::Header &&
  466. Encoding != HttpServerState::Closes)
  467. {
  468. Close();
  469. if (LErrno == 0)
  470. return _error->Error(_("Error reading from server. Remote end closed connection"));
  471. errno = LErrno;
  472. return _error->Errno("read",_("Error reading from server"));
  473. }
  474. else
  475. {
  476. In.Limit(-1);
  477. // Nothing left in the buffer
  478. if (In.WriteSpace() == false)
  479. return false;
  480. // We may have got multiple responses back in one packet..
  481. Close();
  482. return true;
  483. }
  484. return false;
  485. }
  486. /*}}}*/
  487. // HttpServerState::Flush - Dump the buffer into the file /*{{{*/
  488. // ---------------------------------------------------------------------
  489. /* This takes the current input buffer from the Server FD and writes it
  490. into the file */
  491. bool HttpServerState::Flush(FileFd * const File)
  492. {
  493. if (File != NULL)
  494. {
  495. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  496. // can't be set
  497. if (File->Name() != "/dev/null")
  498. SetNonBlock(File->Fd(),false);
  499. if (In.WriteSpace() == false)
  500. return true;
  501. while (In.WriteSpace() == true)
  502. {
  503. if (In.Write(File->Fd()) == false)
  504. return _error->Errno("write",_("Error writing to file"));
  505. if (In.IsLimit() == true)
  506. return true;
  507. }
  508. if (In.IsLimit() == true || Encoding == ServerState::Closes)
  509. return true;
  510. }
  511. return false;
  512. }
  513. /*}}}*/
  514. // HttpServerState::Go - Run a single loop /*{{{*/
  515. // ---------------------------------------------------------------------
  516. /* This runs the select loop over the server FDs, Output file FDs and
  517. stdin. */
  518. bool HttpServerState::Go(bool ToFile, FileFd * const File)
  519. {
  520. // Server has closed the connection
  521. if (ServerFd == -1 && (In.WriteSpace() == false ||
  522. ToFile == false))
  523. return false;
  524. fd_set rfds,wfds;
  525. FD_ZERO(&rfds);
  526. FD_ZERO(&wfds);
  527. /* Add the server. We only send more requests if the connection will
  528. be persisting */
  529. if (Out.WriteSpace() == true && ServerFd != -1
  530. && Persistent == true)
  531. FD_SET(ServerFd,&wfds);
  532. if (In.ReadSpace() == true && ServerFd != -1)
  533. FD_SET(ServerFd,&rfds);
  534. // Add the file
  535. int FileFD = -1;
  536. if (File != NULL)
  537. FileFD = File->Fd();
  538. if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
  539. FD_SET(FileFD,&wfds);
  540. // Add stdin
  541. if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
  542. FD_SET(STDIN_FILENO,&rfds);
  543. // Figure out the max fd
  544. int MaxFd = FileFD;
  545. if (MaxFd < ServerFd)
  546. MaxFd = ServerFd;
  547. // Select
  548. struct timeval tv;
  549. tv.tv_sec = TimeOut;
  550. tv.tv_usec = 0;
  551. int Res = 0;
  552. if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
  553. {
  554. if (errno == EINTR)
  555. return true;
  556. return _error->Errno("select",_("Select failed"));
  557. }
  558. if (Res == 0)
  559. {
  560. _error->Error(_("Connection timed out"));
  561. return Die(*File);
  562. }
  563. // Handle server IO
  564. if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
  565. {
  566. errno = 0;
  567. if (In.Read(ServerFd) == false)
  568. return Die(*File);
  569. }
  570. if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
  571. {
  572. errno = 0;
  573. if (Out.Write(ServerFd) == false)
  574. return Die(*File);
  575. }
  576. // Send data to the file
  577. if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
  578. {
  579. if (In.Write(FileFD) == false)
  580. return _error->Errno("write",_("Error writing to output file"));
  581. }
  582. // Handle commands from APT
  583. if (FD_ISSET(STDIN_FILENO,&rfds))
  584. {
  585. if (Owner->Run(true) != -1)
  586. exit(100);
  587. }
  588. return true;
  589. }
  590. /*}}}*/
  591. // HttpMethod::SendReq - Send the HTTP request /*{{{*/
  592. // ---------------------------------------------------------------------
  593. /* This places the http request in the outbound buffer */
  594. void HttpMethod::SendReq(FetchItem *Itm)
  595. {
  596. URI Uri = Itm->Uri;
  597. // The HTTP server expects a hostname with a trailing :port
  598. std::stringstream Req;
  599. string ProperHost;
  600. if (Uri.Host.find(':') != string::npos)
  601. ProperHost = '[' + Uri.Host + ']';
  602. else
  603. ProperHost = Uri.Host;
  604. /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
  605. but while its a must for all servers to accept absolute URIs,
  606. it is assumed clients will sent an absolute path for non-proxies */
  607. std::string requesturi;
  608. if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
  609. requesturi = Uri.Path;
  610. else
  611. requesturi = Itm->Uri;
  612. // The "+" is encoded as a workaround for a amazon S3 bug
  613. // see LP bugs #1003633 and #1086997.
  614. requesturi = QuoteString(requesturi, "+~ ");
  615. /* Build the request. No keep-alive is included as it is the default
  616. in 1.1, can cause problems with proxies, and we are an HTTP/1.1
  617. client anyway.
  618. C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
  619. Req << "GET " << requesturi << " HTTP/1.1\r\n";
  620. if (Uri.Port != 0)
  621. Req << "Host: " << ProperHost << ":" << Uri.Port << "\r\n";
  622. else
  623. Req << "Host: " << ProperHost << "\r\n";
  624. // generate a cache control header (if needed)
  625. if (_config->FindB("Acquire::http::No-Cache",false) == true)
  626. Req << "Cache-Control: no-cache\r\n"
  627. << "Pragma: no-cache\r\n";
  628. else if (Itm->IndexFile == true)
  629. Req << "Cache-Control: max-age=" << _config->FindI("Acquire::http::Max-Age",0) << "\r\n";
  630. else if (_config->FindB("Acquire::http::No-Store",false) == true)
  631. Req << "Cache-Control: no-store\r\n";
  632. // If we ask for uncompressed files servers might respond with content-
  633. // negotiation which lets us end up with compressed files we do not support,
  634. // see 657029, 657560 and co, so if we have no extension on the request
  635. // ask for text only. As a sidenote: If there is nothing to negotate servers
  636. // seem to be nice and ignore it.
  637. if (_config->FindB("Acquire::http::SendAccept", true) == true)
  638. {
  639. size_t const filepos = Itm->Uri.find_last_of('/');
  640. string const file = Itm->Uri.substr(filepos + 1);
  641. if (flExtension(file) == file)
  642. Req << "Accept: text/*\r\n";
  643. }
  644. // Check for a partial file and send if-queries accordingly
  645. struct stat SBuf;
  646. if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
  647. Req << "Range: bytes=" << SBuf.st_size << "-\r\n"
  648. << "If-Range: " << TimeRFC1123(SBuf.st_mtime) << "\r\n";
  649. else if (Itm->LastModified != 0)
  650. Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified).c_str() << "\r\n";
  651. if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
  652. Req << "Proxy-Authorization: Basic "
  653. << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
  654. maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
  655. if (Uri.User.empty() == false || Uri.Password.empty() == false)
  656. Req << "Authorization: Basic "
  657. << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
  658. Req << "User-Agent: " << _config->Find("Acquire::http::User-Agent",
  659. "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
  660. Req << "\r\n";
  661. if (Debug == true)
  662. cerr << Req.str() << endl;
  663. Server->WriteResponse(Req.str());
  664. }
  665. /*}}}*/
  666. // HttpMethod::Configuration - Handle a configuration message /*{{{*/
  667. // ---------------------------------------------------------------------
  668. /* We stash the desired pipeline depth */
  669. bool HttpMethod::Configuration(string Message)
  670. {
  671. if (ServerMethod::Configuration(Message) == false)
  672. return false;
  673. AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
  674. PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
  675. PipelineDepth);
  676. Debug = _config->FindB("Debug::Acquire::http",false);
  677. // Get the proxy to use
  678. AutoDetectProxy();
  679. return true;
  680. }
  681. /*}}}*/
  682. // HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/
  683. // ---------------------------------------------------------------------
  684. /* */
  685. bool HttpMethod::AutoDetectProxy()
  686. {
  687. // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
  688. // name without the dash ("-")
  689. AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect",
  690. _config->Find("Acquire::http::ProxyAutoDetect"));
  691. if (AutoDetectProxyCmd.empty())
  692. return true;
  693. if (Debug)
  694. clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl;
  695. int Pipes[2] = {-1,-1};
  696. if (pipe(Pipes) != 0)
  697. return _error->Errno("pipe", "Failed to create Pipe");
  698. pid_t Process = ExecFork();
  699. if (Process == 0)
  700. {
  701. close(Pipes[0]);
  702. dup2(Pipes[1],STDOUT_FILENO);
  703. SetCloseExec(STDOUT_FILENO,false);
  704. const char *Args[2];
  705. Args[0] = AutoDetectProxyCmd.c_str();
  706. Args[1] = 0;
  707. execv(Args[0],(char **)Args);
  708. cerr << "Failed to exec method " << Args[0] << endl;
  709. _exit(100);
  710. }
  711. char buf[512];
  712. int InFd = Pipes[0];
  713. close(Pipes[1]);
  714. int res = read(InFd, buf, sizeof(buf)-1);
  715. ExecWait(Process, "ProxyAutoDetect", true);
  716. if (res < 0)
  717. return _error->Errno("read", "Failed to read");
  718. if (res == 0)
  719. return _error->Warning("ProxyAutoDetect returned no data");
  720. // add trailing \0
  721. buf[res] = 0;
  722. if (Debug)
  723. clog << "auto detect command returned: '" << buf << "'" << endl;
  724. if (strstr(buf, "http://") == buf)
  725. _config->Set("Acquire::http::proxy", _strstrip(buf));
  726. return true;
  727. }
  728. /*}}}*/
  729. ServerState * HttpMethod::CreateServerState(URI uri) /*{{{*/
  730. {
  731. return new HttpServerState(uri, this);
  732. }
  733. /*}}}*/
  734. void HttpMethod::RotateDNS() /*{{{*/
  735. {
  736. ::RotateDNS();
  737. }
  738. /*}}}*/