http.cc 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864
  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
  4. /* ######################################################################
  5. HTTP Acquire Method - This is the HTTP acquire method for APT.
  6. It uses HTTP/1.1 and many of the fancy options there-in, such as
  7. pipelining, range, if-range and so on.
  8. It is based on a doubly buffered select loop. A groupe of requests are
  9. fed into a single output buffer that is constantly fed out the
  10. socket. This provides ideal pipelining as in many cases all of the
  11. requests will fit into a single packet. The input socket is buffered
  12. the same way and fed into the fd for the file (may be a pipe in future).
  13. This double buffering provides fairly substantial transfer rates,
  14. compared to wget the http method is about 4% faster. Most importantly,
  15. when HTTP is compared with FTP as a protocol the speed difference is
  16. huge. In tests over the internet from two sites to llug (via ATM) this
  17. program got 230k/s sustained http transfer rates. FTP on the other
  18. hand topped out at 170k/s. That combined with the time to setup the
  19. FTP connection makes HTTP a vastly superior protocol.
  20. ##################################################################### */
  21. /*}}}*/
  22. // Include Files /*{{{*/
  23. #include <config.h>
  24. #include <apt-pkg/fileutl.h>
  25. #include <apt-pkg/acquire-method.h>
  26. #include <apt-pkg/configuration.h>
  27. #include <apt-pkg/error.h>
  28. #include <apt-pkg/hashes.h>
  29. #include <apt-pkg/netrc.h>
  30. #include <apt-pkg/strutl.h>
  31. #include <stddef.h>
  32. #include <stdlib.h>
  33. #include <sys/select.h>
  34. #include <cstring>
  35. #include <sys/stat.h>
  36. #include <sys/time.h>
  37. #include <unistd.h>
  38. #include <stdio.h>
  39. #include <errno.h>
  40. #include <iostream>
  41. #include "config.h"
  42. #include "connect.h"
  43. #include "http.h"
  44. #include <apti18n.h>
  45. /*}}}*/
  46. using namespace std;
  47. unsigned long long CircleBuf::BwReadLimit=0;
  48. unsigned long long CircleBuf::BwTickReadData=0;
  49. struct timeval CircleBuf::BwReadTick={0,0};
  50. const unsigned int CircleBuf::BW_HZ=10;
  51. // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
  52. // ---------------------------------------------------------------------
  53. /* */
  54. CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0)
  55. {
  56. Buf = new unsigned char[Size];
  57. Reset();
  58. CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  59. }
  60. /*}}}*/
  61. // CircleBuf::Reset - Reset to the default state /*{{{*/
  62. // ---------------------------------------------------------------------
  63. /* */
  64. void CircleBuf::Reset()
  65. {
  66. InP = 0;
  67. OutP = 0;
  68. StrPos = 0;
  69. MaxGet = (unsigned long long)-1;
  70. OutQueue = string();
  71. if (Hash != 0)
  72. {
  73. delete Hash;
  74. Hash = new Hashes;
  75. }
  76. }
  77. /*}}}*/
  78. // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
  79. // ---------------------------------------------------------------------
  80. /* This fills up the buffer with as much data as is in the FD, assuming it
  81. is non-blocking.. */
  82. bool CircleBuf::Read(int Fd)
  83. {
  84. while (1)
  85. {
  86. // Woops, buffer is full
  87. if (InP - OutP == Size)
  88. return true;
  89. // what's left to read in this tick
  90. unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
  91. if(CircleBuf::BwReadLimit) {
  92. struct timeval now;
  93. gettimeofday(&now,0);
  94. unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
  95. now.tv_usec-CircleBuf::BwReadTick.tv_usec;
  96. if(d > 1000000/BW_HZ) {
  97. CircleBuf::BwReadTick = now;
  98. CircleBuf::BwTickReadData = 0;
  99. }
  100. if(CircleBuf::BwTickReadData >= BwReadMax) {
  101. usleep(1000000/BW_HZ);
  102. return true;
  103. }
  104. }
  105. // Write the buffer segment
  106. ssize_t Res;
  107. if(CircleBuf::BwReadLimit) {
  108. Res = read(Fd,Buf + (InP%Size),
  109. BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
  110. } else
  111. Res = read(Fd,Buf + (InP%Size),LeftRead());
  112. if(Res > 0 && BwReadLimit > 0)
  113. CircleBuf::BwTickReadData += Res;
  114. if (Res == 0)
  115. return false;
  116. if (Res < 0)
  117. {
  118. if (errno == EAGAIN)
  119. return true;
  120. return false;
  121. }
  122. if (InP == 0)
  123. gettimeofday(&Start,0);
  124. InP += Res;
  125. }
  126. }
  127. /*}}}*/
  128. // CircleBuf::Read - Put the string into the buffer /*{{{*/
  129. // ---------------------------------------------------------------------
  130. /* This will hold the string in and fill the buffer with it as it empties */
  131. bool CircleBuf::Read(string Data)
  132. {
  133. OutQueue += Data;
  134. FillOut();
  135. return true;
  136. }
  137. /*}}}*/
  138. // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
  139. // ---------------------------------------------------------------------
  140. /* */
  141. void CircleBuf::FillOut()
  142. {
  143. if (OutQueue.empty() == true)
  144. return;
  145. while (1)
  146. {
  147. // Woops, buffer is full
  148. if (InP - OutP == Size)
  149. return;
  150. // Write the buffer segment
  151. unsigned long long Sz = LeftRead();
  152. if (OutQueue.length() - StrPos < Sz)
  153. Sz = OutQueue.length() - StrPos;
  154. memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
  155. // Advance
  156. StrPos += Sz;
  157. InP += Sz;
  158. if (OutQueue.length() == StrPos)
  159. {
  160. StrPos = 0;
  161. OutQueue = "";
  162. return;
  163. }
  164. }
  165. }
  166. /*}}}*/
  167. // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
  168. // ---------------------------------------------------------------------
  169. /* This empties the buffer into the FD. */
  170. bool CircleBuf::Write(int Fd)
  171. {
  172. while (1)
  173. {
  174. FillOut();
  175. // Woops, buffer is empty
  176. if (OutP == InP)
  177. return true;
  178. if (OutP == MaxGet)
  179. return true;
  180. // Write the buffer segment
  181. ssize_t Res;
  182. Res = write(Fd,Buf + (OutP%Size),LeftWrite());
  183. if (Res == 0)
  184. return false;
  185. if (Res < 0)
  186. {
  187. if (errno == EAGAIN)
  188. return true;
  189. return false;
  190. }
  191. if (Hash != 0)
  192. Hash->Add(Buf + (OutP%Size),Res);
  193. OutP += Res;
  194. }
  195. }
  196. /*}}}*/
  197. // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
  198. // ---------------------------------------------------------------------
  199. /* This copies till the first empty line */
  200. bool CircleBuf::WriteTillEl(string &Data,bool Single)
  201. {
  202. // We cheat and assume it is unneeded to have more than one buffer load
  203. for (unsigned long long I = OutP; I < InP; I++)
  204. {
  205. if (Buf[I%Size] != '\n')
  206. continue;
  207. ++I;
  208. if (Single == false)
  209. {
  210. if (I < InP && Buf[I%Size] == '\r')
  211. ++I;
  212. if (I >= InP || Buf[I%Size] != '\n')
  213. continue;
  214. ++I;
  215. }
  216. Data = "";
  217. while (OutP < I)
  218. {
  219. unsigned long long Sz = LeftWrite();
  220. if (Sz == 0)
  221. return false;
  222. if (I - OutP < Sz)
  223. Sz = I - OutP;
  224. Data += string((char *)(Buf + (OutP%Size)),Sz);
  225. OutP += Sz;
  226. }
  227. return true;
  228. }
  229. return false;
  230. }
  231. /*}}}*/
  232. // CircleBuf::Stats - Print out stats information /*{{{*/
  233. // ---------------------------------------------------------------------
  234. /* */
  235. void CircleBuf::Stats()
  236. {
  237. if (InP == 0)
  238. return;
  239. struct timeval Stop;
  240. gettimeofday(&Stop,0);
  241. /* float Diff = Stop.tv_sec - Start.tv_sec +
  242. (float)(Stop.tv_usec - Start.tv_usec)/1000000;
  243. clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
  244. }
  245. /*}}}*/
  246. CircleBuf::~CircleBuf()
  247. {
  248. delete [] Buf;
  249. delete Hash;
  250. }
  251. // HttpServerState::HttpServerState - Constructor /*{{{*/
  252. HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024)
  253. {
  254. TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
  255. Reset();
  256. }
  257. /*}}}*/
  258. // HttpServerState::Open - Open a connection to the server /*{{{*/
  259. // ---------------------------------------------------------------------
  260. /* This opens a connection to the server. */
  261. bool HttpServerState::Open()
  262. {
  263. // Use the already open connection if possible.
  264. if (ServerFd != -1)
  265. return true;
  266. Close();
  267. In.Reset();
  268. Out.Reset();
  269. Persistent = true;
  270. // Determine the proxy setting
  271. string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
  272. if (!SpecificProxy.empty())
  273. {
  274. if (SpecificProxy == "DIRECT")
  275. Proxy = "";
  276. else
  277. Proxy = SpecificProxy;
  278. }
  279. else
  280. {
  281. string DefProxy = _config->Find("Acquire::http::Proxy");
  282. if (!DefProxy.empty())
  283. {
  284. Proxy = DefProxy;
  285. }
  286. else
  287. {
  288. char* result = getenv("http_proxy");
  289. Proxy = result ? result : "";
  290. }
  291. }
  292. // Parse no_proxy, a , separated list of domains
  293. if (getenv("no_proxy") != 0)
  294. {
  295. if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
  296. Proxy = "";
  297. }
  298. // Determine what host and port to use based on the proxy settings
  299. int Port = 0;
  300. string Host;
  301. if (Proxy.empty() == true || Proxy.Host.empty() == true)
  302. {
  303. if (ServerName.Port != 0)
  304. Port = ServerName.Port;
  305. Host = ServerName.Host;
  306. }
  307. else
  308. {
  309. if (Proxy.Port != 0)
  310. Port = Proxy.Port;
  311. Host = Proxy.Host;
  312. }
  313. // Connect to the remote server
  314. if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
  315. return false;
  316. return true;
  317. }
  318. /*}}}*/
  319. // HttpServerState::Close - Close a connection to the server /*{{{*/
  320. // ---------------------------------------------------------------------
  321. /* */
  322. bool HttpServerState::Close()
  323. {
  324. close(ServerFd);
  325. ServerFd = -1;
  326. return true;
  327. }
  328. /*}}}*/
  329. // HttpServerState::RunData - Transfer the data from the socket /*{{{*/
  330. bool HttpServerState::RunData(FileFd * const File)
  331. {
  332. State = Data;
  333. // Chunked transfer encoding is fun..
  334. if (Encoding == Chunked)
  335. {
  336. while (1)
  337. {
  338. // Grab the block size
  339. bool Last = true;
  340. string Data;
  341. In.Limit(-1);
  342. do
  343. {
  344. if (In.WriteTillEl(Data,true) == true)
  345. break;
  346. }
  347. while ((Last = Go(false, File)) == true);
  348. if (Last == false)
  349. return false;
  350. // See if we are done
  351. unsigned long long Len = strtoull(Data.c_str(),0,16);
  352. if (Len == 0)
  353. {
  354. In.Limit(-1);
  355. // We have to remove the entity trailer
  356. Last = true;
  357. do
  358. {
  359. if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
  360. break;
  361. }
  362. while ((Last = Go(false, File)) == true);
  363. if (Last == false)
  364. return false;
  365. return !_error->PendingError();
  366. }
  367. // Transfer the block
  368. In.Limit(Len);
  369. while (Go(true, File) == true)
  370. if (In.IsLimit() == true)
  371. break;
  372. // Error
  373. if (In.IsLimit() == false)
  374. return false;
  375. // The server sends an extra new line before the next block specifier..
  376. In.Limit(-1);
  377. Last = true;
  378. do
  379. {
  380. if (In.WriteTillEl(Data,true) == true)
  381. break;
  382. }
  383. while ((Last = Go(false, File)) == true);
  384. if (Last == false)
  385. return false;
  386. }
  387. }
  388. else
  389. {
  390. /* Closes encoding is used when the server did not specify a size, the
  391. loss of the connection means we are done */
  392. if (Encoding == Closes)
  393. In.Limit(-1);
  394. else
  395. In.Limit(Size - StartPos);
  396. // Just transfer the whole block.
  397. do
  398. {
  399. if (In.IsLimit() == false)
  400. continue;
  401. In.Limit(-1);
  402. return !_error->PendingError();
  403. }
  404. while (Go(true, File) == true);
  405. }
  406. return Owner->Flush() && !_error->PendingError();
  407. }
  408. /*}}}*/
  409. bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
  410. {
  411. return In.WriteTillEl(Data);
  412. }
  413. /*}}}*/
  414. bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
  415. {
  416. return Go(ToFile, File);
  417. }
  418. /*}}}*/
  419. bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
  420. {
  421. return Out.Read(Data);
  422. }
  423. /*}}}*/
  424. APT_PURE bool HttpServerState::IsOpen() /*{{{*/
  425. {
  426. return (ServerFd != -1);
  427. }
  428. /*}}}*/
  429. bool HttpServerState::InitHashes(FileFd &File) /*{{{*/
  430. {
  431. delete In.Hash;
  432. In.Hash = new Hashes;
  433. // Set the expected size and read file for the hashes
  434. File.Truncate(StartPos);
  435. return In.Hash->AddFD(File, StartPos);
  436. }
  437. /*}}}*/
  438. APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/
  439. {
  440. return In.Hash;
  441. }
  442. /*}}}*/
  443. // HttpServerState::Die - The server has closed the connection. /*{{{*/
  444. bool HttpServerState::Die(FileFd &File)
  445. {
  446. unsigned int LErrno = errno;
  447. // Dump the buffer to the file
  448. if (State == ServerState::Data)
  449. {
  450. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  451. // can't be set
  452. if (File.Name() != "/dev/null")
  453. SetNonBlock(File.Fd(),false);
  454. while (In.WriteSpace() == true)
  455. {
  456. if (In.Write(File.Fd()) == false)
  457. return _error->Errno("write",_("Error writing to the file"));
  458. // Done
  459. if (In.IsLimit() == true)
  460. return true;
  461. }
  462. }
  463. // See if this is because the server finished the data stream
  464. if (In.IsLimit() == false && State != HttpServerState::Header &&
  465. Encoding != HttpServerState::Closes)
  466. {
  467. Close();
  468. if (LErrno == 0)
  469. return _error->Error(_("Error reading from server. Remote end closed connection"));
  470. errno = LErrno;
  471. return _error->Errno("read",_("Error reading from server"));
  472. }
  473. else
  474. {
  475. In.Limit(-1);
  476. // Nothing left in the buffer
  477. if (In.WriteSpace() == false)
  478. return false;
  479. // We may have got multiple responses back in one packet..
  480. Close();
  481. return true;
  482. }
  483. return false;
  484. }
  485. /*}}}*/
  486. // HttpServerState::Flush - Dump the buffer into the file /*{{{*/
  487. // ---------------------------------------------------------------------
  488. /* This takes the current input buffer from the Server FD and writes it
  489. into the file */
  490. bool HttpServerState::Flush(FileFd * const File)
  491. {
  492. if (File != NULL)
  493. {
  494. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  495. // can't be set
  496. if (File->Name() != "/dev/null")
  497. SetNonBlock(File->Fd(),false);
  498. if (In.WriteSpace() == false)
  499. return true;
  500. while (In.WriteSpace() == true)
  501. {
  502. if (In.Write(File->Fd()) == false)
  503. return _error->Errno("write",_("Error writing to file"));
  504. if (In.IsLimit() == true)
  505. return true;
  506. }
  507. if (In.IsLimit() == true || Encoding == ServerState::Closes)
  508. return true;
  509. }
  510. return false;
  511. }
  512. /*}}}*/
  513. // HttpServerState::Go - Run a single loop /*{{{*/
  514. // ---------------------------------------------------------------------
  515. /* This runs the select loop over the server FDs, Output file FDs and
  516. stdin. */
  517. bool HttpServerState::Go(bool ToFile, FileFd * const File)
  518. {
  519. // Server has closed the connection
  520. if (ServerFd == -1 && (In.WriteSpace() == false ||
  521. ToFile == false))
  522. return false;
  523. fd_set rfds,wfds;
  524. FD_ZERO(&rfds);
  525. FD_ZERO(&wfds);
  526. /* Add the server. We only send more requests if the connection will
  527. be persisting */
  528. if (Out.WriteSpace() == true && ServerFd != -1
  529. && Persistent == true)
  530. FD_SET(ServerFd,&wfds);
  531. if (In.ReadSpace() == true && ServerFd != -1)
  532. FD_SET(ServerFd,&rfds);
  533. // Add the file
  534. int FileFD = -1;
  535. if (File != NULL)
  536. FileFD = File->Fd();
  537. if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
  538. FD_SET(FileFD,&wfds);
  539. // Add stdin
  540. if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
  541. FD_SET(STDIN_FILENO,&rfds);
  542. // Figure out the max fd
  543. int MaxFd = FileFD;
  544. if (MaxFd < ServerFd)
  545. MaxFd = ServerFd;
  546. // Select
  547. struct timeval tv;
  548. tv.tv_sec = TimeOut;
  549. tv.tv_usec = 0;
  550. int Res = 0;
  551. if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
  552. {
  553. if (errno == EINTR)
  554. return true;
  555. return _error->Errno("select",_("Select failed"));
  556. }
  557. if (Res == 0)
  558. {
  559. _error->Error(_("Connection timed out"));
  560. return Die(*File);
  561. }
  562. // Handle server IO
  563. if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
  564. {
  565. errno = 0;
  566. if (In.Read(ServerFd) == false)
  567. return Die(*File);
  568. }
  569. if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
  570. {
  571. errno = 0;
  572. if (Out.Write(ServerFd) == false)
  573. return Die(*File);
  574. }
  575. // Send data to the file
  576. if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
  577. {
  578. if (In.Write(FileFD) == false)
  579. return _error->Errno("write",_("Error writing to output file"));
  580. }
  581. // Handle commands from APT
  582. if (FD_ISSET(STDIN_FILENO,&rfds))
  583. {
  584. if (Owner->Run(true) != -1)
  585. exit(100);
  586. }
  587. return true;
  588. }
  589. /*}}}*/
  590. // HttpMethod::SendReq - Send the HTTP request /*{{{*/
  591. // ---------------------------------------------------------------------
  592. /* This places the http request in the outbound buffer */
  593. void HttpMethod::SendReq(FetchItem *Itm)
  594. {
  595. URI Uri = Itm->Uri;
  596. // The HTTP server expects a hostname with a trailing :port
  597. char Buf[1000];
  598. string ProperHost;
  599. if (Uri.Host.find(':') != string::npos)
  600. ProperHost = '[' + Uri.Host + ']';
  601. else
  602. ProperHost = Uri.Host;
  603. if (Uri.Port != 0)
  604. {
  605. sprintf(Buf,":%u",Uri.Port);
  606. ProperHost += Buf;
  607. }
  608. // Just in case.
  609. if (Itm->Uri.length() >= sizeof(Buf))
  610. abort();
  611. /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
  612. but while its a must for all servers to accept absolute URIs,
  613. it is assumed clients will sent an absolute path for non-proxies */
  614. std::string requesturi;
  615. if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
  616. requesturi = Uri.Path;
  617. else
  618. requesturi = Itm->Uri;
  619. // The "+" is encoded as a workaround for a amazon S3 bug
  620. // see LP bugs #1003633 and #1086997.
  621. requesturi = QuoteString(requesturi, "+~ ");
  622. /* Build the request. No keep-alive is included as it is the default
  623. in 1.1, can cause problems with proxies, and we are an HTTP/1.1
  624. client anyway.
  625. C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
  626. sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
  627. requesturi.c_str(),ProperHost.c_str());
  628. // generate a cache control header (if needed)
  629. if (_config->FindB("Acquire::http::No-Cache",false) == true)
  630. {
  631. strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
  632. }
  633. else
  634. {
  635. if (Itm->IndexFile == true)
  636. {
  637. sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
  638. _config->FindI("Acquire::http::Max-Age",0));
  639. }
  640. else
  641. {
  642. if (_config->FindB("Acquire::http::No-Store",false) == true)
  643. strcat(Buf,"Cache-Control: no-store\r\n");
  644. }
  645. }
  646. // If we ask for uncompressed files servers might respond with content-
  647. // negotiation which lets us end up with compressed files we do not support,
  648. // see 657029, 657560 and co, so if we have no extension on the request
  649. // ask for text only. As a sidenote: If there is nothing to negotate servers
  650. // seem to be nice and ignore it.
  651. if (_config->FindB("Acquire::http::SendAccept", true) == true)
  652. {
  653. size_t const filepos = Itm->Uri.find_last_of('/');
  654. string const file = Itm->Uri.substr(filepos + 1);
  655. if (flExtension(file) == file)
  656. strcat(Buf,"Accept: text/*\r\n");
  657. }
  658. string Req = Buf;
  659. // Check for a partial file
  660. struct stat SBuf;
  661. if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
  662. {
  663. // In this case we send an if-range query with a range header
  664. sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size,
  665. TimeRFC1123(SBuf.st_mtime).c_str());
  666. Req += Buf;
  667. }
  668. else
  669. {
  670. if (Itm->LastModified != 0)
  671. {
  672. sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
  673. Req += Buf;
  674. }
  675. }
  676. if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
  677. Req += string("Proxy-Authorization: Basic ") +
  678. Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) + "\r\n";
  679. maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
  680. if (Uri.User.empty() == false || Uri.Password.empty() == false)
  681. {
  682. Req += string("Authorization: Basic ") +
  683. Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
  684. }
  685. Req += "User-Agent: " + _config->Find("Acquire::http::User-Agent",
  686. "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") + "\r\n\r\n";
  687. if (Debug == true)
  688. cerr << Req << endl;
  689. Server->WriteResponse(Req);
  690. }
  691. /*}}}*/
  692. // HttpMethod::Configuration - Handle a configuration message /*{{{*/
  693. // ---------------------------------------------------------------------
  694. /* We stash the desired pipeline depth */
  695. bool HttpMethod::Configuration(string Message)
  696. {
  697. if (ServerMethod::Configuration(Message) == false)
  698. return false;
  699. AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
  700. PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
  701. PipelineDepth);
  702. Debug = _config->FindB("Debug::Acquire::http",false);
  703. // Get the proxy to use
  704. AutoDetectProxy();
  705. return true;
  706. }
  707. /*}}}*/
  708. // HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/
  709. // ---------------------------------------------------------------------
  710. /* */
  711. bool HttpMethod::AutoDetectProxy()
  712. {
  713. // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
  714. // name without the dash ("-")
  715. AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect",
  716. _config->Find("Acquire::http::ProxyAutoDetect"));
  717. if (AutoDetectProxyCmd.empty())
  718. return true;
  719. if (Debug)
  720. clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl;
  721. int Pipes[2] = {-1,-1};
  722. if (pipe(Pipes) != 0)
  723. return _error->Errno("pipe", "Failed to create Pipe");
  724. pid_t Process = ExecFork();
  725. if (Process == 0)
  726. {
  727. close(Pipes[0]);
  728. dup2(Pipes[1],STDOUT_FILENO);
  729. SetCloseExec(STDOUT_FILENO,false);
  730. const char *Args[2];
  731. Args[0] = AutoDetectProxyCmd.c_str();
  732. Args[1] = 0;
  733. execv(Args[0],(char **)Args);
  734. cerr << "Failed to exec method " << Args[0] << endl;
  735. _exit(100);
  736. }
  737. char buf[512];
  738. int InFd = Pipes[0];
  739. close(Pipes[1]);
  740. int res = read(InFd, buf, sizeof(buf)-1);
  741. ExecWait(Process, "ProxyAutoDetect", true);
  742. if (res < 0)
  743. return _error->Errno("read", "Failed to read");
  744. if (res == 0)
  745. return _error->Warning("ProxyAutoDetect returned no data");
  746. // add trailing \0
  747. buf[res] = 0;
  748. if (Debug)
  749. clog << "auto detect command returned: '" << buf << "'" << endl;
  750. if (strstr(buf, "http://") == buf)
  751. _config->Set("Acquire::http::proxy", _strstrip(buf));
  752. return true;
  753. }
  754. /*}}}*/
  755. ServerState * HttpMethod::CreateServerState(URI uri) /*{{{*/
  756. {
  757. return new HttpServerState(uri, this);
  758. }
  759. /*}}}*/
  760. void HttpMethod::RotateDNS() /*{{{*/
  761. {
  762. ::RotateDNS();
  763. }
  764. /*}}}*/