server.cc 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839
  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. /* ######################################################################
  4. HTTP and HTTPS share a lot of common code and these classes are
  5. exactly the dumping ground for this common code
  6. ##################################################################### */
  7. /*}}}*/
  8. // Include Files /*{{{*/
  9. #include <config.h>
  10. #include <apt-pkg/configuration.h>
  11. #include <apt-pkg/error.h>
  12. #include <apt-pkg/fileutl.h>
  13. #include <apt-pkg/strutl.h>
  14. #include <ctype.h>
  15. #include <signal.h>
  16. #include <stdio.h>
  17. #include <stdlib.h>
  18. #include <sys/stat.h>
  19. #include <sys/time.h>
  20. #include <time.h>
  21. #include <unistd.h>
  22. #include <iostream>
  23. #include <limits>
  24. #include <map>
  25. #include <string>
  26. #include <vector>
  27. #include "server.h"
  28. #include <apti18n.h>
  29. /*}}}*/
  30. using namespace std;
  31. string ServerMethod::FailFile;
  32. int ServerMethod::FailFd = -1;
  33. time_t ServerMethod::FailTime = 0;
  34. // ServerState::RunHeaders - Get the headers before the data /*{{{*/
  35. // ---------------------------------------------------------------------
  36. /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  37. parse error occurred */
  38. ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
  39. const std::string &Uri)
  40. {
  41. Reset(false);
  42. Owner->Status(_("Waiting for headers"));
  43. do
  44. {
  45. string Data;
  46. if (ReadHeaderLines(Data) == false)
  47. continue;
  48. if (Owner->Debug == true)
  49. clog << "Answer for: " << Uri << endl << Data;
  50. for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  51. {
  52. string::const_iterator J = I;
  53. for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  54. if (HeaderLine(string(I,J)) == false)
  55. return RUN_HEADERS_PARSE_ERROR;
  56. I = J;
  57. }
  58. // 100 Continue is a Nop...
  59. if (Result == 100)
  60. continue;
  61. // Tidy up the connection persistence state.
  62. if (Encoding == Closes && HaveContent == true)
  63. Persistent = false;
  64. return RUN_HEADERS_OK;
  65. }
  66. while (LoadNextResponse(false, File) == true);
  67. return RUN_HEADERS_IO_ERROR;
  68. }
  69. /*}}}*/
  70. // ServerState::HeaderLine - Process a header line /*{{{*/
  71. // ---------------------------------------------------------------------
  72. /* */
  73. bool ServerState::HeaderLine(string Line)
  74. {
  75. if (Line.empty() == true)
  76. return true;
  77. if (Line.size() > 4 && stringcasecmp(Line.data(), Line.data()+4, "HTTP") == 0)
  78. {
  79. // Evil servers return no version
  80. if (Line[4] == '/')
  81. {
  82. int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
  83. if (elements == 3)
  84. {
  85. Code[0] = '\0';
  86. if (Owner != NULL && Owner->Debug == true)
  87. clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl;
  88. }
  89. else if (elements != 4)
  90. return _error->Error(_("The HTTP server sent an invalid reply header"));
  91. }
  92. else
  93. {
  94. Major = 0;
  95. Minor = 9;
  96. if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
  97. return _error->Error(_("The HTTP server sent an invalid reply header"));
  98. }
  99. /* Check the HTTP response header to get the default persistence
  100. state. */
  101. if (Major < 1)
  102. Persistent = false;
  103. else
  104. {
  105. if (Major == 1 && Minor == 0)
  106. {
  107. Persistent = false;
  108. }
  109. else
  110. {
  111. Persistent = true;
  112. if (PipelineAllowed)
  113. Pipeline = true;
  114. }
  115. }
  116. return true;
  117. }
  118. // Blah, some servers use "connection:closes", evil.
  119. // and some even send empty header fields…
  120. string::size_type Pos = Line.find(':');
  121. if (Pos == string::npos)
  122. return _error->Error(_("Bad header line"));
  123. ++Pos;
  124. // Parse off any trailing spaces between the : and the next word.
  125. string::size_type Pos2 = Pos;
  126. while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
  127. Pos2++;
  128. string const Tag(Line,0,Pos);
  129. string const Val(Line,Pos2);
  130. if (stringcasecmp(Tag,"Content-Length:") == 0)
  131. {
  132. if (Encoding == Closes)
  133. Encoding = Stream;
  134. HaveContent = true;
  135. unsigned long long * DownloadSizePtr = &DownloadSize;
  136. if (Result == 416 || (Result >= 300 && Result < 400))
  137. DownloadSizePtr = &JunkSize;
  138. *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
  139. if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
  140. return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
  141. else if (*DownloadSizePtr == 0)
  142. HaveContent = false;
  143. // On partial content (206) the Content-Length less than the real
  144. // size, so do not set it here but leave that to the Content-Range
  145. // header instead
  146. if(Result != 206 && TotalFileSize == 0)
  147. TotalFileSize = DownloadSize;
  148. return true;
  149. }
  150. if (stringcasecmp(Tag,"Content-Type:") == 0)
  151. {
  152. HaveContent = true;
  153. return true;
  154. }
  155. if (stringcasecmp(Tag,"Content-Range:") == 0)
  156. {
  157. HaveContent = true;
  158. // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
  159. if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
  160. ; // we got the expected filesize which is all we wanted
  161. else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
  162. return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
  163. if ((unsigned long long)StartPos > TotalFileSize)
  164. return _error->Error(_("This HTTP server has broken range support"));
  165. // figure out what we will download
  166. DownloadSize = TotalFileSize - StartPos;
  167. return true;
  168. }
  169. if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
  170. {
  171. HaveContent = true;
  172. if (stringcasecmp(Val,"chunked") == 0)
  173. Encoding = Chunked;
  174. return true;
  175. }
  176. if (stringcasecmp(Tag,"Connection:") == 0)
  177. {
  178. if (stringcasecmp(Val,"close") == 0)
  179. {
  180. Persistent = false;
  181. Pipeline = false;
  182. /* Some servers send error pages (as they are dynamically generated)
  183. for simplicity via a connection close instead of e.g. chunked,
  184. so assuming an always closing server only if we get a file + close */
  185. if (Result >= 200 && Result < 300)
  186. PipelineAllowed = false;
  187. }
  188. else if (stringcasecmp(Val,"keep-alive") == 0)
  189. Persistent = true;
  190. return true;
  191. }
  192. if (stringcasecmp(Tag,"Last-Modified:") == 0)
  193. {
  194. if (RFC1123StrToTime(Val.c_str(), Date) == false)
  195. return _error->Error(_("Unknown date format"));
  196. return true;
  197. }
  198. if (stringcasecmp(Tag,"Location:") == 0)
  199. {
  200. Location = Val;
  201. return true;
  202. }
  203. if (stringcasecmp(Tag, "Accept-Ranges:") == 0)
  204. {
  205. std::string ranges = ',' + Val + ',';
  206. ranges.erase(std::remove(ranges.begin(), ranges.end(), ' '), ranges.end());
  207. if (ranges.find(",bytes,") == std::string::npos)
  208. RangesAllowed = false;
  209. return true;
  210. }
  211. return true;
  212. }
  213. /*}}}*/
  214. // ServerState::ServerState - Constructor /*{{{*/
  215. ServerState::ServerState(URI Srv, ServerMethod *Owner) :
  216. DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner)
  217. {
  218. Reset();
  219. }
  220. /*}}}*/
  221. bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
  222. {
  223. File.Truncate(StartPos);
  224. return GetHashes()->AddFD(File, StartPos);
  225. }
  226. /*}}}*/
  227. void ServerState::Reset(bool const Everything) /*{{{*/
  228. {
  229. Major = 0; Minor = 0; Result = 0; Code[0] = '\0';
  230. TotalFileSize = 0; JunkSize = 0; StartPos = 0;
  231. Encoding = Closes; time(&Date); HaveContent = false;
  232. State = Header; MaximumSize = 0;
  233. if (Everything)
  234. {
  235. Persistent = false; Pipeline = false; PipelineAllowed = true;
  236. RangesAllowed = true;
  237. }
  238. }
  239. /*}}}*/
  240. // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
  241. // ---------------------------------------------------------------------
  242. /* We look at the header data we got back from the server and decide what
  243. to do. Returns DealWithHeadersResult (see http.h for details).
  244. */
  245. ServerMethod::DealWithHeadersResult
  246. ServerMethod::DealWithHeaders(FetchResult &Res)
  247. {
  248. // Not Modified
  249. if (Server->Result == 304)
  250. {
  251. RemoveFile("server", Queue->DestFile);
  252. Res.IMSHit = true;
  253. Res.LastModified = Queue->LastModified;
  254. Res.Size = 0;
  255. return IMS_HIT;
  256. }
  257. /* Redirect
  258. *
  259. * Note that it is only OK for us to treat all redirection the same
  260. * because we *always* use GET, not other HTTP methods. There are
  261. * three redirection codes for which it is not appropriate that we
  262. * redirect. Pass on those codes so the error handling kicks in.
  263. */
  264. if (AllowRedirect
  265. && (Server->Result > 300 && Server->Result < 400)
  266. && (Server->Result != 300 // Multiple Choices
  267. && Server->Result != 304 // Not Modified
  268. && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
  269. {
  270. if (Server->Location.empty() == true)
  271. ;
  272. else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
  273. {
  274. URI Uri = Queue->Uri;
  275. if (Uri.Host.empty() == false)
  276. NextURI = URI::SiteOnly(Uri);
  277. else
  278. NextURI.clear();
  279. NextURI.append(DeQuoteString(Server->Location));
  280. if (Queue->Uri == NextURI)
  281. {
  282. SetFailReason("RedirectionLoop");
  283. _error->Error("Redirection loop encountered");
  284. if (Server->HaveContent == true)
  285. return ERROR_WITH_CONTENT_PAGE;
  286. return ERROR_UNRECOVERABLE;
  287. }
  288. return TRY_AGAIN_OR_REDIRECT;
  289. }
  290. else
  291. {
  292. NextURI = DeQuoteString(Server->Location);
  293. URI tmpURI = NextURI;
  294. if (tmpURI.Access.find('+') != std::string::npos)
  295. {
  296. _error->Error("Server tried to trick us into using a specific implementation: %s", tmpURI.Access.c_str());
  297. if (Server->HaveContent == true)
  298. return ERROR_WITH_CONTENT_PAGE;
  299. return ERROR_UNRECOVERABLE;
  300. }
  301. URI Uri = Queue->Uri;
  302. if (Binary.find('+') != std::string::npos)
  303. {
  304. auto base = Binary.substr(0, Binary.find('+'));
  305. if (base != tmpURI.Access)
  306. {
  307. tmpURI.Access = base + '+' + tmpURI.Access;
  308. if (tmpURI.Access == Binary)
  309. {
  310. std::string tmpAccess = Uri.Access;
  311. std::swap(tmpURI.Access, Uri.Access);
  312. NextURI = tmpURI;
  313. std::swap(tmpURI.Access, Uri.Access);
  314. }
  315. else
  316. NextURI = tmpURI;
  317. }
  318. }
  319. if (Queue->Uri == NextURI)
  320. {
  321. SetFailReason("RedirectionLoop");
  322. _error->Error("Redirection loop encountered");
  323. if (Server->HaveContent == true)
  324. return ERROR_WITH_CONTENT_PAGE;
  325. return ERROR_UNRECOVERABLE;
  326. }
  327. Uri.Access = Binary;
  328. // same protocol redirects are okay
  329. if (tmpURI.Access == Uri.Access)
  330. return TRY_AGAIN_OR_REDIRECT;
  331. // as well as http to https
  332. else if ((Uri.Access == "http" || Uri.Access == "https+http") && tmpURI.Access == "https")
  333. return TRY_AGAIN_OR_REDIRECT;
  334. else
  335. {
  336. auto const tmpplus = tmpURI.Access.find('+');
  337. if (tmpplus != std::string::npos && tmpURI.Access.substr(tmpplus + 1) == "https")
  338. {
  339. auto const uriplus = Uri.Access.find('+');
  340. if (uriplus == std::string::npos)
  341. {
  342. if (Uri.Access == tmpURI.Access.substr(0, tmpplus)) // foo -> foo+https
  343. return TRY_AGAIN_OR_REDIRECT;
  344. }
  345. else if (Uri.Access.substr(uriplus + 1) == "http" &&
  346. Uri.Access.substr(0, uriplus) == tmpURI.Access.substr(0, tmpplus)) // foo+http -> foo+https
  347. return TRY_AGAIN_OR_REDIRECT;
  348. }
  349. }
  350. _error->Error("Redirection from %s to '%s' is forbidden", Uri.Access.c_str(), NextURI.c_str());
  351. }
  352. /* else pass through for error message */
  353. }
  354. // retry after an invalid range response without partial data
  355. else if (Server->Result == 416)
  356. {
  357. struct stat SBuf;
  358. if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
  359. {
  360. bool partialHit = false;
  361. if (Queue->ExpectedHashes.usable() == true)
  362. {
  363. Hashes resultHashes(Queue->ExpectedHashes);
  364. FileFd file(Queue->DestFile, FileFd::ReadOnly);
  365. Server->TotalFileSize = file.FileSize();
  366. Server->Date = file.ModificationTime();
  367. resultHashes.AddFD(file);
  368. HashStringList const hashList = resultHashes.GetHashStringList();
  369. partialHit = (Queue->ExpectedHashes == hashList);
  370. }
  371. else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
  372. partialHit = true;
  373. if (partialHit == true)
  374. {
  375. // the file is completely downloaded, but was not moved
  376. if (Server->HaveContent == true)
  377. {
  378. // nuke the sent error page
  379. Server->RunDataToDevNull();
  380. Server->HaveContent = false;
  381. }
  382. Server->StartPos = Server->TotalFileSize;
  383. Server->Result = 200;
  384. }
  385. else if (RemoveFile("server", Queue->DestFile))
  386. {
  387. NextURI = Queue->Uri;
  388. return TRY_AGAIN_OR_REDIRECT;
  389. }
  390. }
  391. }
  392. /* We have a reply we don't handle. This should indicate a perm server
  393. failure */
  394. if (Server->Result < 200 || Server->Result >= 300)
  395. {
  396. if (_error->PendingError() == false)
  397. {
  398. std::string err;
  399. strprintf(err, "HttpError%u", Server->Result);
  400. SetFailReason(err);
  401. _error->Error("%u %s", Server->Result, Server->Code);
  402. }
  403. if (Server->HaveContent == true)
  404. return ERROR_WITH_CONTENT_PAGE;
  405. return ERROR_UNRECOVERABLE;
  406. }
  407. // This is some sort of 2xx 'data follows' reply
  408. Res.LastModified = Server->Date;
  409. Res.Size = Server->TotalFileSize;
  410. return FILE_IS_OPEN;
  411. }
  412. /*}}}*/
  413. // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
  414. // ---------------------------------------------------------------------
  415. /* This closes and timestamps the open file. This is necessary to get
  416. resume behavoir on user abort */
  417. void ServerMethod::SigTerm(int)
  418. {
  419. if (FailFd == -1)
  420. _exit(100);
  421. struct timeval times[2];
  422. times[0].tv_sec = FailTime;
  423. times[1].tv_sec = FailTime;
  424. times[0].tv_usec = times[1].tv_usec = 0;
  425. utimes(FailFile.c_str(), times);
  426. close(FailFd);
  427. _exit(100);
  428. }
  429. /*}}}*/
  430. // ServerMethod::Fetch - Fetch an item /*{{{*/
  431. // ---------------------------------------------------------------------
  432. /* This adds an item to the pipeline. We keep the pipeline at a fixed
  433. depth. */
  434. bool ServerMethod::Fetch(FetchItem *)
  435. {
  436. if (Server == nullptr || QueueBack == nullptr)
  437. return true;
  438. // If pipelining is disabled, we only queue 1 request
  439. auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0;
  440. // how deep is our pipeline currently?
  441. decltype(PipelineDepth) CurrentDepth = 0;
  442. for (FetchItem const *I = Queue; I != QueueBack; I = I->Next)
  443. ++CurrentDepth;
  444. if (CurrentDepth > AllowedDepth)
  445. return true;
  446. do {
  447. // Make sure we stick with the same server
  448. if (Server->Comp(QueueBack->Uri) == false)
  449. break;
  450. bool const UsableHashes = QueueBack->ExpectedHashes.usable();
  451. // if we have no hashes, do at most one such request
  452. // as we can't fixup pipeling misbehaviors otherwise
  453. if (CurrentDepth != 0 && UsableHashes == false)
  454. break;
  455. if (UsableHashes && FileExists(QueueBack->DestFile))
  456. {
  457. FileFd partial(QueueBack->DestFile, FileFd::ReadOnly);
  458. Hashes wehave(QueueBack->ExpectedHashes);
  459. if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize())
  460. {
  461. if (wehave.AddFD(partial) &&
  462. wehave.GetHashStringList() == QueueBack->ExpectedHashes)
  463. {
  464. FetchResult Res;
  465. Res.Filename = QueueBack->DestFile;
  466. Res.ResumePoint = QueueBack->ExpectedHashes.FileSize();
  467. URIStart(Res);
  468. // move item to the start of the queue as URIDone will
  469. // always dequeued the first item in the queue
  470. if (Queue != QueueBack)
  471. {
  472. FetchItem *Prev = Queue;
  473. for (; Prev->Next != QueueBack; Prev = Prev->Next)
  474. /* look for the previous queue item */;
  475. Prev->Next = QueueBack->Next;
  476. QueueBack->Next = Queue;
  477. Queue = QueueBack;
  478. QueueBack = Prev->Next;
  479. }
  480. Res.TakeHashes(wehave);
  481. URIDone(Res);
  482. continue;
  483. }
  484. else
  485. RemoveFile("Fetch-Partial", QueueBack->DestFile);
  486. }
  487. }
  488. auto const Tmp = QueueBack;
  489. QueueBack = QueueBack->Next;
  490. SendReq(Tmp);
  491. ++CurrentDepth;
  492. } while (CurrentDepth <= AllowedDepth && QueueBack != nullptr);
  493. return true;
  494. }
  495. /*}}}*/
  496. // ServerMethod::Loop - Main loop /*{{{*/
  497. int ServerMethod::Loop()
  498. {
  499. signal(SIGTERM,SigTerm);
  500. signal(SIGINT,SigTerm);
  501. Server = 0;
  502. int FailCounter = 0;
  503. while (1)
  504. {
  505. // We have no commands, wait for some to arrive
  506. if (Queue == 0)
  507. {
  508. if (WaitFd(STDIN_FILENO) == false)
  509. return 0;
  510. }
  511. /* Run messages, we can accept 0 (no message) if we didn't
  512. do a WaitFd above.. Otherwise the FD is closed. */
  513. int Result = Run(true);
  514. if (Result != -1 && (Result != 0 || Queue == 0))
  515. {
  516. if(FailReason.empty() == false ||
  517. ConfigFindB("DependOnSTDIN", true) == true)
  518. return 100;
  519. else
  520. return 0;
  521. }
  522. if (Queue == 0)
  523. continue;
  524. // Connect to the server
  525. if (Server == 0 || Server->Comp(Queue->Uri) == false)
  526. {
  527. Server = CreateServerState(Queue->Uri);
  528. setPostfixForMethodNames(::URI(Queue->Uri).Host.c_str());
  529. AllowRedirect = ConfigFindB("AllowRedirect", true);
  530. PipelineDepth = ConfigFindI("Pipeline-Depth", 10);
  531. Debug = DebugEnabled();
  532. }
  533. /* If the server has explicitly said this is the last connection
  534. then we pre-emptively shut down the pipeline and tear down
  535. the connection. This will speed up HTTP/1.0 servers a tad
  536. since we don't have to wait for the close sequence to
  537. complete */
  538. if (Server->Persistent == false)
  539. Server->Close();
  540. // Reset the pipeline
  541. if (Server->IsOpen() == false)
  542. QueueBack = Queue;
  543. // Connnect to the host
  544. if (Server->Open() == false)
  545. {
  546. Fail(true);
  547. Server = nullptr;
  548. continue;
  549. }
  550. // Fill the pipeline.
  551. Fetch(0);
  552. // Fetch the next URL header data from the server.
  553. switch (Server->RunHeaders(File, Queue->Uri))
  554. {
  555. case ServerState::RUN_HEADERS_OK:
  556. break;
  557. // The header data is bad
  558. case ServerState::RUN_HEADERS_PARSE_ERROR:
  559. {
  560. _error->Error(_("Bad header data"));
  561. Fail(true);
  562. Server->Close();
  563. RotateDNS();
  564. continue;
  565. }
  566. // The server closed a connection during the header get..
  567. default:
  568. case ServerState::RUN_HEADERS_IO_ERROR:
  569. {
  570. FailCounter++;
  571. _error->Discard();
  572. Server->Close();
  573. Server->Pipeline = false;
  574. Server->PipelineAllowed = false;
  575. if (FailCounter >= 2)
  576. {
  577. Fail(_("Connection failed"),true);
  578. FailCounter = 0;
  579. }
  580. RotateDNS();
  581. continue;
  582. }
  583. };
  584. // Decide what to do.
  585. FetchResult Res;
  586. Res.Filename = Queue->DestFile;
  587. switch (DealWithHeaders(Res))
  588. {
  589. // Ok, the file is Open
  590. case FILE_IS_OPEN:
  591. {
  592. URIStart(Res);
  593. // Run the data
  594. bool Result = true;
  595. // ensure we don't fetch too much
  596. // we could do "Server->MaximumSize = Queue->MaximumSize" here
  597. // but that would break the clever pipeline messup detection
  598. // so instead we use the size of the biggest item in the queue
  599. Server->MaximumSize = FindMaximumObjectSizeInQueue();
  600. if (Server->HaveContent)
  601. Result = Server->RunData(File);
  602. /* If the server is sending back sizeless responses then fill in
  603. the size now */
  604. if (Res.Size == 0)
  605. Res.Size = File->Size();
  606. // Close the file, destroy the FD object and timestamp it
  607. FailFd = -1;
  608. delete File;
  609. File = 0;
  610. // Timestamp
  611. struct timeval times[2];
  612. times[0].tv_sec = times[1].tv_sec = Server->Date;
  613. times[0].tv_usec = times[1].tv_usec = 0;
  614. utimes(Queue->DestFile.c_str(), times);
  615. // Send status to APT
  616. if (Result == true)
  617. {
  618. Hashes * const resultHashes = Server->GetHashes();
  619. HashStringList const hashList = resultHashes->GetHashStringList();
  620. if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
  621. {
  622. // we did not get the expected hash… mhhh:
  623. // could it be that server/proxy messed up pipelining?
  624. FetchItem * BeforeI = Queue;
  625. for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
  626. {
  627. if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
  628. {
  629. // yes, he did! Disable pipelining and rewrite queue
  630. if (Server->Pipeline == true)
  631. {
  632. Warning(_("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
  633. Server->Pipeline = false;
  634. Server->PipelineAllowed = false;
  635. // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
  636. }
  637. Rename(Res.Filename, I->DestFile);
  638. Res.Filename = I->DestFile;
  639. BeforeI->Next = I->Next;
  640. I->Next = Queue;
  641. Queue = I;
  642. break;
  643. }
  644. BeforeI = I;
  645. }
  646. }
  647. Res.TakeHashes(*resultHashes);
  648. URIDone(Res);
  649. }
  650. else
  651. {
  652. if (Server->IsOpen() == false)
  653. {
  654. FailCounter++;
  655. _error->Discard();
  656. Server->Close();
  657. if (FailCounter >= 2)
  658. {
  659. Fail(_("Connection failed"),true);
  660. FailCounter = 0;
  661. }
  662. QueueBack = Queue;
  663. }
  664. else
  665. {
  666. Server->Close();
  667. Fail(true);
  668. }
  669. }
  670. break;
  671. }
  672. // IMS hit
  673. case IMS_HIT:
  674. {
  675. URIDone(Res);
  676. break;
  677. }
  678. // Hard server error, not found or something
  679. case ERROR_UNRECOVERABLE:
  680. {
  681. Fail();
  682. break;
  683. }
  684. // Hard internal error, kill the connection and fail
  685. case ERROR_NOT_FROM_SERVER:
  686. {
  687. delete File;
  688. File = 0;
  689. Fail();
  690. RotateDNS();
  691. Server->Close();
  692. break;
  693. }
  694. // We need to flush the data, the header is like a 404 w/ error text
  695. case ERROR_WITH_CONTENT_PAGE:
  696. {
  697. Server->RunDataToDevNull();
  698. Fail();
  699. break;
  700. }
  701. // Try again with a new URL
  702. case TRY_AGAIN_OR_REDIRECT:
  703. {
  704. // Clear rest of response if there is content
  705. if (Server->HaveContent)
  706. Server->RunDataToDevNull();
  707. Redirect(NextURI);
  708. break;
  709. }
  710. default:
  711. Fail(_("Internal error"));
  712. break;
  713. }
  714. FailCounter = 0;
  715. }
  716. return 0;
  717. }
  718. /*}}}*/
  719. unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
  720. {
  721. unsigned long long MaxSizeInQueue = 0;
  722. for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
  723. MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
  724. return MaxSizeInQueue;
  725. }
  726. /*}}}*/
  727. ServerMethod::ServerMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/
  728. aptMethod(std::move(Binary), Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
  729. AllowRedirect(false), Debug(false)
  730. {
  731. }
  732. /*}}}*/
  733. bool ServerMethod::Configuration(std::string Message) /*{{{*/
  734. {
  735. if (aptMethod::Configuration(Message) == false)
  736. return false;
  737. _config->CndSet("Acquire::tor::Proxy",
  738. "socks5h://apt-transport-tor@localhost:9050");
  739. return true;
  740. }
  741. /*}}}*/
  742. bool ServerMethod::AddProxyAuth(URI &Proxy, URI const &Server) const /*{{{*/
  743. {
  744. if (std::find(methodNames.begin(), methodNames.end(), "tor") != methodNames.end() &&
  745. Proxy.User == "apt-transport-tor" && Proxy.Password.empty())
  746. {
  747. std::string pass = Server.Host;
  748. pass.erase(std::remove_if(pass.begin(), pass.end(), [](char const c) { return std::isalnum(c) == 0; }), pass.end());
  749. if (pass.length() > 255)
  750. Proxy.Password = pass.substr(0, 255);
  751. else
  752. Proxy.Password = std::move(pass);
  753. }
  754. // FIXME: should we support auth.conf for proxies?
  755. return true;
  756. }
  757. /*}}}*/