retry without partial data after a 416 response

If we get a 416 from the server it means the Range we asked for is above
the real filesize of the file on the server. Mostly this happens if the
server isn't supporting If-Range, but regardless of how we end up with
the partial data, the data is invalid so we discard it and retry with a
fresh plate and hope for the best.

Old behavior was to consider 416 an error and retry with a different
compression until we ran out of compression and requested the
uncompressed file (which doesn't exist on most mirrors) with an accept
line which server answered with "406 Not Acceptable".

Closes: 710924

methods/http.cc

@@ -604,6 +604,8 @@ bool ServerState::HeaderLine(string Line)
       Size = strtoull(Val.c_str(), NULL, 10);
       if (Size >= std::numeric_limits<unsigned long long>::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
+      else if (Size == 0)
+	 HaveContent = false;
       return true;
    }
 
@@ -616,8 +618,14 @@ bool ServerState::HeaderLine(string Line)
    if (stringcasecmp(Tag,"Content-Range:") == 0)
    {
       HaveContent = true;
-      
-      if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
+
+      // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
+      if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
+      {
+	 StartPos = 1; // ignore Content-Length, it would override Size
+	 HaveContent = false;
+      }
+      else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
 	 return _error->Error(_("This HTTP server has broken range support"));
@@ -995,6 +1003,13 @@ HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
       }
       /* else pass through for error message */
    }
+   // retry after an invalid range response without partial data
+   else if (Srv->Result == 416 && FileExists(Queue->DestFile) == true &&
+	 unlink(Queue->DestFile.c_str()) == 0)
+   {
+      NextURI = Queue->Uri;
+      return TRY_AGAIN_OR_REDIRECT;
+   }
  
    /* We have a reply we dont handle. This should indicate a perm server
       failure */

test/integration/test-releasefile-verification

@@ -11,18 +11,24 @@ buildaptarchive
 setupflataptarchive
 changetowebserver
 
+downloadfile "http://localhost:8080/_config/set/aptwebserver::support::range/false" '/dev/null' >/dev/null
+
 prepare() {
 	local DATE="${2:-now}"
-	if [ "$DATE" = 'now' -a "$1" = "${PKGFILE}-new" ]; then
-		DATE='now + 6 days'
+	if [ "$DATE" = 'now' ]; then
+		if [ "$1" = "${PKGFILE}-new" ]; then
+			DATE='now - 1 day'
+		else
+			DATE='now - 7 day'
+		fi
 	fi
 	for release in $(find rootdir/var/lib/apt/lists 2> /dev/null); do
-		touch -d 'now - 6 hours' $release
+		touch -d 'now - 1 year' $release
 	done
 	aptget clean
 	cp $1 aptarchive/Packages
 	find aptarchive -name 'Release' -delete
-	compressfile 'aptarchive/Packages'
+	compressfile 'aptarchive/Packages' "$DATE"
 	generatereleasefiles "$DATE"
 }
 
@@ -85,13 +91,34 @@ touch aptarchive/apt.deb
 
 PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
 
+updatesuccess() {
+	local LOG='update.log'
+	if aptget update >$LOG 2>&1 || grep -q -E '^(W|E): ' $LOG; then
+		msgpass
+	else
+		cat $LOG
+		msgfail
+	fi
+}
+
+updatefailure() {
+	local LOG='update.log'
+	aptget update >$LOG 2>&1 || true
+	if grep -q -E "$1" $LOG; then
+		msgpass
+	else
+		cat $LOG
+		msgfail
+	fi
+}
+
 runtest() {
 	prepare ${PKGFILE}
 	rm -rf rootdir/var/lib/apt/lists
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Joe Sixpack'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 	testequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -100,7 +127,7 @@ runtest() {
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Joe Sixpack'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 	testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
@@ -111,7 +138,7 @@ runtest() {
 	signreleasefiles 'Rex Expired'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Rex Expired'
-	aptget update 2>&1 | grep -E '^W: .* KEYEXPIRED' > /dev/null && msgpass || msgfail
+	updatefailure '^W: .* KEYEXPIRED'
 	testequal "$(cat ${PKGFILE})
 " aptcache show apt
 	failaptold
@@ -122,7 +149,7 @@ runtest() {
 	signreleasefiles 'Marvin Paranoid'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Marvin Paranoid'
-	aptget update 2>&1 | grep -E '^W: .* NO_PUBKEY' > /dev/null && msgpass || msgfail
+	updatefailure '^W: .* NO_PUBKEY'
 	testequal "$(cat ${PKGFILE})
 " aptcache show apt
 	failaptold
@@ -136,7 +163,7 @@ runtest() {
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Bad warm archive signed by' 'Joe Sixpack'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 	testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
@@ -147,7 +174,7 @@ runtest() {
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Joe Sixpack'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 	testequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -156,7 +183,7 @@ runtest() {
 	signreleasefiles 'Marvin Paranoid'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Marvin Paranoid'
-	aptget update 2>&1 | grep -E '^W: .* NO_PUBKEY' > /dev/null && msgpass || msgfail
+	updatefailure '^W: .* NO_PUBKEY'
 	testequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -166,7 +193,7 @@ runtest() {
 	signreleasefiles 'Rex Expired'
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Rex Expired'
-	aptget update 2>&1 | grep -E '^W: .* KEYEXPIRED' > /dev/null && msgpass || msgfail
+	updatefailure '^W: .* KEYEXPIRED'
 	testequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -176,7 +203,7 @@ runtest() {
 	signreleasefiles
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Joe Sixpack'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 	testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
@@ -187,7 +214,7 @@ runtest2() {
 	rm -rf rootdir/var/lib/apt/lists
 	signreleasefiles 'Joe Sixpack'
 	msgtest 'Cold archive signed by' 'Joe Sixpack'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 
 	# New .deb but now an unsigned archive. For example MITM to circumvent
 	# package verification.
@@ -195,7 +222,7 @@ runtest2() {
 	find aptarchive/ -name InRelease -delete
 	find aptarchive/ -name Release.gpg -delete
 	msgtest 'Warm archive signed by' 'nobody'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 	testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	failaptnew
@@ -203,7 +230,7 @@ runtest2() {
 	# Unsigned archive from the beginning must also be detected.
 	rm -rf rootdir/var/lib/apt/lists
 	msgtest 'Cold archive signed by' 'nobody'
-	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	updatesuccess
 	testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	failaptnew