select kernels to protect from autoremove based on Debian version

This is basically a rewrite of the script with the general idea of
finding the Debian version of the installed kernels – as multiple
flavours will have the same Debian version – select the two newest of
them and translate them back to versions found in package names.

This way we avoid e.g. kernel and kernel-rt to use up the protected
slots even through they are basically the same kernel (just a different
flavour) so it is likely that if kernel doesn't work for some reason,
kernel-rt will not either.

This also deals with foreign kernel packages, kernels on hold and partly
installed kernels (in case multiple kernels are installed in the same
apt run) in a hopefully sensible way.

Closes: 787827

debian/apt.auto-removal.sh

@@ -1,72 +1,45 @@
 #!/bin/sh
 set -e
-
-# Author: Steve Langasek <steve.langasek@canonical.com>
-#
 # Mark as not-for-autoremoval those kernel packages that are:
 #  - the currently booted version
 #  - the kernel version we've been called for
-#  - the latest kernel version (determined using rules copied from the grub
-#    package for deciding which kernel to boot)
-#  - the second-latest kernel version, if the booted kernel version is
-#    already the latest and this script is called for that same version,
-#    to ensure a fallback remains available in the event the newly-installed
-#    kernel at this ABI fails to boot
-# In the common case, this results in exactly two kernels saved, but it can
-# result in three kernels being saved.  It's better to err on the side of
-# saving too many kernels than saving too few.
+#  - the latest kernel version (as determined by debian version number)
+#  - the second-latest kernel version
 #
-# We generate this list and save it to /etc/apt/apt.conf.d instead of marking
-# packages in the database because this runs from a postinst script, and apt
-# will overwrite the db when it exits.
-
+# In the common case this results in two kernels saved (booted into the
+# second-latest kernel, we install the latest kernel in an upgrade), but
+# can save up to four. Kernel refers here to a distinct release, which can
+# potentially be installed in multiple flavours counting as one kernel.
 
 eval $(apt-config shell APT_CONF_D Dir::Etc::parts/d)
 test -n "${APT_CONF_D}" || APT_CONF_D="/etc/apt/apt.conf.d"
-config_file=${APT_CONF_D}/01autoremove-kernels
+config_file="${APT_CONF_D}/01autoremove-kernels"
 
 eval $(apt-config shell DPKG Dir::bin::dpkg/f)
 test -n "$DPKG" || DPKG="/usr/bin/dpkg"
 
-installed_version="$1"
-running_version="$(uname -r)"
 
+list="$(${DPKG} -l | awk '/^[ih][^nc][ ]+(linux|kfreebsd|gnumach)-image-[0-9]+\./ && $2 !~ /-dbg$/ && $2 !~ /-dbgsym$/ { print $2,$3; }' \
+   | sed -e 's#^\(linux\|kfreebsd\|gnumach\)-image-##' -e 's#:[^:]\+ # #')"
+debverlist="$(echo "$list" | cut -d' ' -f 2 | sort --unique --reverse --version-sort)"
 
-version_test_gt ()
-{
-	local version_test_gt_sedexp="s/[._-]\(pre\|rc\|test\|git\|old\|trunk\)/~\1/g"
-	local version_a="`echo "$1" | sed -e "$version_test_gt_sedexp"`"
-	local version_b="`echo "$2" | sed -e "$version_test_gt_sedexp"`"
-	$DPKG --compare-versions "$version_a" gt "$version_b"
-	return "$?"
-}
-
-list="$(${DPKG} -l | awk '/^ii[ ]+(linux|kfreebsd|gnumach)-image-[0-9]+\./ && $2 !~ /-dbg$/ { print $2 }' | sed -e 's#\(linux\|kfreebsd\|gnumach\)-image-##')"
-
-latest_version=""
-previous_version=""
-for i in $list; do
-	if version_test_gt "$i" "$latest_version"; then
-		previous_version="$latest_version"
-		latest_version="$i"
-	elif version_test_gt "$i" "$previous_version"; then
-		previous_version="$i"
-	fi
-done
-
-if [ "$latest_version" != "$installed_version" ] \
-   || [ "$latest_version" != "$running_version" ] \
-   || [ "$installed_version" != "$running_version" ]
-then
-	# We have at least two kernels that we have reason to think the
-	# user wants, so don't save the second-newest version.
-	previous_version=
+if [ -n "$1" ]; then
+	installed_version="$(echo "$list" | awk "\$1 == \"$1\" { print \$2;exit; }")"
+fi
+unamer="$(uname -r)"
+if [ -n "$unamer" ]; then
+	running_version="$(echo "$list" | awk "\$1 == \"$unamer\" { print \$2;exit; }")"
 fi
+latest_version="$(echo "$debverlist" | sed -n 1p)"
+previous_version="$(echo "$debverlist" | sed -n 2p)"
 
-kernels="$(echo "$latest_version
+debkernels="$(echo "$latest_version
 $installed_version
 $running_version
-$previous_version" | sort -u | sed -e 's#\.#\\.#g' )"
+$previous_version" | sort -u | sed -e '/^$/ d')"
+kernels="$( (echo "$1
+$unamer"; for deb in $debkernels; do echo "$list" | awk "\$2 == \"$deb\" { print \$1; }"; done; ) \
+   | sed -e 's#\.#\\.#g' -e '/^$/ d' | sort -u)"
 
 generateconfig() {
 	cat <<EOF
@@ -83,3 +56,4 @@ EOF
 }
 generateconfig > "${config_file}.dpkg-new"
 mv "${config_file}.dpkg-new" "$config_file"
+chmod 444 "$config_file"

test/integration/framework

@@ -1470,6 +1470,10 @@ msgfailoutput() {
 			if expr match "$1" "$FILEFLAGS" >/dev/null; then
 				echo "#### stat(2) of file: $2 ####"
 				stat "$2" || true
+				if test -e "$2"; then
+					echo "#### Complete file: $2 ####"
+					cat >&2 "$2" || true
+				fi
 			fi
 		}
 		msgfailoutputstatfile "$2" "$3"

test/integration/test-kernel-helper-autoremove

@@ -6,53 +6,51 @@ TESTDIR=$(readlink -f $(dirname $0))
 setupenvironment
 configarchitecture 'amd64'
 
-# the executed script would use the installed apt-config,
-# which is outside of our control
-msgtest 'Check that the installed apt-config supports' '--no-empty'
-if /usr/bin/apt-config dump --no-empty >/dev/null 2>&1; then
-	msgpass
-else
-	msgskip
-	exit 0
-fi
-
 CURRENTKERNEL="linux-image-$(uname -r)"
-insertinstalledpackage "$CURRENTKERNEL" 'amd64' '1'
+insertinstalledpackage "$CURRENTKERNEL" 'amd64' '5-1'
+# debug packages do not need our protection
+insertinstalledpackage "${CURRENTKERNEL}-dbg" 'amd64' '5-1'
+# but other kernel flavours should be protected
+insertinstalledpackage "${CURRENTKERNEL}-686-pae" 'i386' '5-1'
+insertinstalledpackage "${CURRENTKERNEL}-rt" 'amd64' '5-1'
+# some more versions
 insertinstalledpackage 'linux-image-1.0.0-2-generic' 'amd64' '1.0.0-2'
 insertinstalledpackage 'linux-image-100.0.0-1-generic' 'amd64' '100.0.0-1'
-insertinstalledpackage 'linux-image-amd64' 'amd64' '100.0.0-1'
+# kernel metapackages should be ignored
+insertinstalledpackage 'linux-image-amd64' 'amd64' '200-1'
+insertinstalledpackage 'linux-image-686-pae' 'i386' '300-1'
 # ensure that the '.' is really a dot and not a wildcard
 insertinstalledpackage 'linux-headers-1000000-1-generic' 'amd64' '100.0.0-1'
 
-testsuccess aptmark auto "$CURRENTKERNEL" 'linux-image-1.0.0-2-generic' 'linux-image-100.0.0-1-generic' 'linux-headers-1000000-1-generic'
-
-# install fake-dpkg into it
-catfail() {
-	echo >&2
-	echo >&2 '### List of protected kernels:'
-	cat >&2 protected.list
-	msgfail
-}
+testsuccess aptmark auto "$CURRENTKERNEL" "${CURRENTKERNEL}-dbg" "${CURRENTKERNEL}-686-pae:i386" "${CURRENTKERNEL}-rt" \
+	'linux-image-1.0.0-2-generic' 'linux-image-100.0.0-1-generic' 'linux-headers-1000000-1-generic'
+testsuccess aptmark hold "${CURRENTKERNEL}-rt"
 
 testprotected() {
 	rm -f rootdir/etc/apt/apt.conf.d/01autoremove-kernels protected.list
 
 	testsuccess runapt sh ${TESTDIR}/../../debian/apt.auto-removal.sh "$@"
+	testfailure test -s rootdir/tmp/testsuccess.output
 
 	msgtest 'Check kernel autoremoval protection list' 'is created'
-	test -e rootdir/etc/apt/apt.conf.d/01autoremove-kernels && msgpass || msgfail
+	testsuccess --nomsg test -e rootdir/etc/apt/apt.conf.d/01autoremove-kernels
+	testfilestats 'rootdir/etc/apt/apt.conf.d/01autoremove-kernels' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:444"
 
 	msgtest 'Check kernel autoremoval protection list' 'can be dumped'
-	aptconfig dump --no-empty --format '%v%n' 'APT::NeverAutoRemove' >protected.list 2>&1 && msgpass || catfail
+	testsuccess --nomsg aptconfig dump --no-empty --format '%v%n' 'APT::NeverAutoRemove'
+	cp rootdir/tmp/testsuccess.output protected.list
 
 	msgtest 'Check kernel autoremoval protection list' 'can be parsed'
-	grep -q '^[A-Z]: ' protected.list && catfail || msgpass
+	testfailure --nomsg grep '^[A-Z]: ' protected.list
 
 	msgtest 'Check kernel autoremoval protection list includes' 'most recent kernel'
-	grep -q '^\^linux-image-100\\\.0\\\.0-1-generic\$$' protected.list && msgpass || catfail
+	testsuccess --nomsg grep '^\^linux-image-100\\\.0\\\.0-1-generic\$$' protected.list
 
 	msgtest 'Check kernel autoremoval protection list includes' 'running kernel'
-	grep -q "^\\^linux-image-$(uname -r | sed -e 's#\.#\\\\.#g')\\\$\$" protected.list && msgpass || catfail
+	testsuccess --nomsg grep "^\\^linux-image-$(uname -r | sed -e 's#\.#\\\\.#g')\\\$\$" protected.list
+
+	msgtest 'Check kernel autoremoval protection list does not include' 'metapackages'
+	testfailure --nomsg grep -e '^\^linux-image-amd64\$$' -e '^\^linux-image-686-pae\$$' -e ':i386' protected.list
 }
 
 testsuccessequal "Reading package lists...
@@ -62,13 +60,28 @@ The following packages were automatically installed and are no longer required:
    linux-headers-1000000-1-generic (100.0.0-1)
    linux-image-1.0.0-2-generic (1.0.0-2)
    linux-image-100.0.0-1-generic (100.0.0-1)
-   $CURRENTKERNEL (1)
+   $CURRENTKERNEL (5-1)
+   ${CURRENTKERNEL}-686-pae:i386 (5-1)
+   ${CURRENTKERNEL}-dbg (5-1)
 Use 'apt-get autoremove' to remove them.
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget install -sV
+testsuccessequal "Reading package lists...
+Building dependency tree...
+Reading state information...
+The following packages were automatically installed and are no longer required:
+   linux-headers-1000000-1-generic (100.0.0-1)
+   linux-image-1.0.0-2-generic (1.0.0-2)
+   linux-image-100.0.0-1-generic (100.0.0-1)
+   $CURRENTKERNEL (5-1)
+   ${CURRENTKERNEL}-686-pae:i386 (5-1)
+   ${CURRENTKERNEL}-dbg (5-1)
+   ${CURRENTKERNEL}-rt (5-1)
+Use 'apt-get autoremove' to remove them.
+0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget install -sV --ignore-hold
 testequal "Reading package lists...
 Building dependency tree...
 Reading state information...
-4 packages were automatically installed and are no longer required.
+6 packages were automatically installed and are no longer required.
 Use 'apt-get autoremove' to remove them.
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget install -s -o APT::Get::HideAutoRemove=small
 testequal "Reading package lists...
@@ -78,33 +91,72 @@ The following packages will be REMOVED:
    linux-headers-1000000-1-generic (100.0.0-1)
    linux-image-1.0.0-2-generic (1.0.0-2)
    linux-image-100.0.0-1-generic (100.0.0-1)
-   $CURRENTKERNEL (1)
-0 upgraded, 0 newly installed, 4 to remove and 0 not upgraded.
+   $CURRENTKERNEL (5-1)
+   ${CURRENTKERNEL}-686-pae:i386 (5-1)
+   ${CURRENTKERNEL}-dbg (5-1)
+0 upgraded, 0 newly installed, 6 to remove and 0 not upgraded.
 Remv linux-headers-1000000-1-generic [100.0.0-1]
 Remv linux-image-1.0.0-2-generic [1.0.0-2]
 Remv linux-image-100.0.0-1-generic [100.0.0-1]
-Remv $CURRENTKERNEL [1]" aptget autoremove -sV
+Remv $CURRENTKERNEL [5-1]
+Remv ${CURRENTKERNEL}-686-pae:i386 [5-1]
+Remv ${CURRENTKERNEL}-dbg [5-1]" aptget autoremove -sV
 
+msgmsg "run without parameter"
 testprotected
 msgtest 'Check kernel autoremoval protection list does not include' 'old kernel'
-grep -q '^\^linux-image-1\\\.0\\\.0-2-generic\$$' protected.list && catfail || msgpass
+testfailure --nomsg grep '^\^linux-image-1\\\.0\\\.0-2-generic\$$' protected.list
+testsuccessequal "Reading package lists...
+Building dependency tree...
+Reading state information...
+The following packages will be REMOVED:
+  linux-headers-1000000-1-generic linux-image-1.0.0-2-generic
+  ${CURRENTKERNEL}-dbg
+0 upgraded, 0 newly installed, 3 to remove and 0 not upgraded.
+Remv linux-headers-1000000-1-generic [100.0.0-1]
+Remv linux-image-1.0.0-2-generic [1.0.0-2]
+Remv ${CURRENTKERNEL}-dbg [5-1]" aptget autoremove -s
 
-testsuccessequal 'Reading package lists...
+msgmsg "install unknown kernel"
+# even if installed/uname reports a kernel which we can't find via dpkg,
+# ensure that we still protect it just in case as these are kernels we
+# know for sure without complicated detection mechanisms
+testprotected 1.0.0-2-ungeneric
+msgtest 'Check kernel autoremoval protection list does not include' 'old kernel'
+testfailure --nomsg grep '^\^linux-image-1\\\.0\\\.0-2-generic\$$' protected.list
+msgtest 'Check kernel autoremoval protection list does include' 'unknown installed kernel'
+testsuccess --nomsg grep '^\^linux-image-1\\\.0\\\.0-2-ungeneric\$$' protected.list
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
   linux-headers-1000000-1-generic linux-image-1.0.0-2-generic
-0 upgraded, 0 newly installed, 2 to remove and 0 not upgraded.
+  ${CURRENTKERNEL}-dbg
+0 upgraded, 0 newly installed, 3 to remove and 0 not upgraded.
 Remv linux-headers-1000000-1-generic [100.0.0-1]
-Remv linux-image-1.0.0-2-generic [1.0.0-2]' aptget autoremove -s
+Remv linux-image-1.0.0-2-generic [1.0.0-2]
+Remv ${CURRENTKERNEL}-dbg [5-1]" aptget autoremove -s
 
+msgmsg "install an old kernel"
 testprotected 1.0.0-2-generic
 msgtest 'Check kernel autoremoval protection list includes' 'installed kernel'
-grep -q '^\^linux-image-1\\\.0\\\.0-2-generic\$$' protected.list && msgpass || catfail
-testsuccessequal 'Reading package lists...
+testsuccess --nomsg grep '^\^linux-image-1\\\.0\\\.0-2-generic\$$' protected.list
+testsuccessequal "Reading package lists...
+Building dependency tree...
+Reading state information...
+The following packages will be REMOVED:
+  linux-headers-1000000-1-generic ${CURRENTKERNEL}-dbg
+0 upgraded, 0 newly installed, 2 to remove and 0 not upgraded.
+Remv linux-headers-1000000-1-generic [100.0.0-1]
+Remv ${CURRENTKERNEL}-dbg [5-1]" aptget autoremove -s
+
+# rt kernel was put on hold while the protected list was generated
+testsuccess aptmark unhold "${CURRENTKERNEL}-rt"
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
-  linux-headers-1000000-1-generic
-0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded.
-Remv linux-headers-1000000-1-generic [100.0.0-1]' aptget autoremove -s
+  linux-headers-1000000-1-generic ${CURRENTKERNEL}-dbg
+0 upgraded, 0 newly installed, 2 to remove and 0 not upgraded.
+Remv linux-headers-1000000-1-generic [100.0.0-1]
+Remv ${CURRENTKERNEL}-dbg [5-1]" aptget autoremove -s