eipp: enable xz-compressed scenario logging

In 385d9f2f23057bc5808b5e013e77ba16d1c94da4 I implemented the storage of
scenario files based on enabling this by default for EIPP, but I
implemented it first optionally for EDSP to have it independent.

The reasons mentioned in the earlier commit (debugging and bugreports)
obviously apply here, especially as EIPP solutions aren't user approved,
nearly impossible to verify before starting the execution and at the
time of error the scenario has changed already, so that reproducing the
issue becomes hard(er).

apt-pkg/edsp.cc

@@ -1074,6 +1074,20 @@ bool EDSP::ResolveExternal(const char* const solver, pkgDepCache &Cache,
 bool EIPP::OrderInstall(char const * const solver, pkgPackageManager * const PM,	/*{{{*/
 			 unsigned int const flags, OpProgress * const Progress)
 {
+   if (strcmp(solver, "internal") == 0)
+   {
+      auto const dumpfile = _config->FindFile("Dir::Log::Planer");
+      if (dumpfile.empty())
+	 return false;
+      auto const dumpdir = flNotFile(dumpfile);
+      FileFd output;
+      if (CreateAPTDirectoryIfNeeded(dumpdir, dumpdir) == false ||
+	    output.Open(dumpfile, FileFd::WriteOnly | FileFd::Exclusive | FileFd::Create, FileFd::Extension, 0644) == false)
+	 return _error->WarningE("EIPP::OrderInstall", _("Could not open file '%s'"), dumpfile.c_str());
+      bool Okay = EIPP::WriteRequest(PM->Cache, output, flags, nullptr);
+      return Okay && EIPP::WriteScenario(PM->Cache, output, nullptr);
+   }
+
    int solver_in, solver_out;
    pid_t const solver_pid = ExecuteExternal("planer", solver, "Dir::Bin::Planers", &solver_in, &solver_out);
    if (solver_pid == 0)
@@ -1081,7 +1095,7 @@ bool EIPP::OrderInstall(char const * const solver, pkgPackageManager * const PM,
 
    FileFd output;
    if (output.OpenDescriptor(solver_in, FileFd::WriteOnly | FileFd::BufferedWrite, true) == false)
-      return _error->Errno("OrderInstall", "Opening planer %s stdin on fd %d for writing failed", solver, solver_in);
+      return _error->Errno("EIPP::OrderInstall", "Opening planer %s stdin on fd %d for writing failed", solver, solver_in);
 
    bool Okay = output.Failed() == false;
    if (Progress != NULL)

apt-pkg/init.cc

@@ -76,6 +76,7 @@ bool pkgInitConfig(Configuration &Cnf)
    Cnf.CndSet("Dir::Log","var/log/apt");
    Cnf.CndSet("Dir::Log::Terminal","term.log");
    Cnf.CndSet("Dir::Log::History","history.log");
+   Cnf.CndSet("Dir::Log::Planer","eipp.log.xz");
 
    Cnf.Set("Dir::Ignore-Files-Silently::", "~$");
    Cnf.Set("Dir::Ignore-Files-Silently::", "\\.disabled$");

apt-pkg/packagemanager.cc

@@ -1033,26 +1033,21 @@ pkgPackageManager::OrderResult pkgPackageManager::OrderInstall()
       return Failed;
 
    Reset();
-   
+
    if (Debug == true)
       clog << "Beginning to order" << endl;
 
    std::string const planer = _config->Find("APT::Planer", "internal");
+   unsigned int flags = 0;
+   if (_config->FindB("APT::Immediate-Configure", true) == false)
+      flags |= EIPP::Request::NO_IMMEDIATE_CONFIGURATION;
+   else if (_config->FindB("APT::Immediate-Configure-All", false))
+      flags |= EIPP::Request::IMMEDIATE_CONFIGURATION_ALL;
+   else if (_config->FindB("APT::Force-LoopBreak", false))
+      flags |= EIPP::Request::ALLOW_TEMPORARY_REMOVE_OF_ESSENTIALS;
+   auto const ret = EIPP::OrderInstall(planer.c_str(), this, flags, nullptr);
    if (planer != "internal")
-   {
-      unsigned int flags = 0;
-      if (_config->FindB("APT::Immediate-Configure", true) == false)
-	 flags |= EIPP::Request::NO_IMMEDIATE_CONFIGURATION;
-      else if (_config->FindB("APT::Immediate-Configure-All", false))
-	 flags |= EIPP::Request::IMMEDIATE_CONFIGURATION_ALL;
-      else if (_config->FindB("APT::Force-LoopBreak", false))
-	 flags |= EIPP::Request::ALLOW_TEMPORARY_REMOVE_OF_ESSENTIALS;
-
-      if (EIPP::OrderInstall(planer.c_str(), this, flags, nullptr))
-	 return Completed;
-      else
-	 return Failed;
-   }
+      return ret ? Completed : Failed;
 
    bool const ordering =
 	_config->FindB("PackageManager::UnpackAll",true) ?
@@ -1062,7 +1057,7 @@ pkgPackageManager::OrderResult pkgPackageManager::OrderInstall()
       _error->Error("Internal ordering error");
       return Failed;
    }
-   
+
    if (Debug == true)
       clog << "Done ordering" << endl;
 

test/integration/framework

@@ -309,8 +309,8 @@ setupenvironment() {
 		ln -s "${BUILDDIRECTORY}/apt-dump-solver" usr/lib/apt/planers/dump
 		ln -s "${BUILDDIRECTORY}/apt-internal-solver" usr/lib/apt/solvers/apt
 		ln -s "${BUILDDIRECTORY}/apt-internal-planer" usr/lib/apt/planers/apt
-		echo "Dir::Bin::Solvers \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/solvers\";" > etc/apt/apt.conf.d/externalsolver.conf
-		echo "Dir::Bin::Planers \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/planers\";" > etc/apt/apt.conf.d/externalplaner.conf
+		echo "Dir::Bin::Solvers \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/solvers\";" >> ../aptconfig.conf
+		echo "Dir::Bin::Planers \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/planers\";" >> ../aptconfig.conf
 	fi
         # use the autoremove from the BUILDDIRECTORY if its there, otherwise
         # system
@@ -333,7 +333,7 @@ setupenvironment() {
 	chmod 644 keys/*
 	ln -s "${TMPWORKINGDIRECTORY}/keys/joesixpack.pub" rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 
-	echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf
+	echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" >> aptconfig.conf
 	echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
 	echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf
 	echo "Dir::Bin::Methods \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods\";" >> aptconfig.conf
@@ -1571,12 +1571,13 @@ msgfailoutput() {
 	local MSG="$1"
 	local OUTPUT="$2"
 	shift 2
-	if [ "$1" = 'grep' ]; then
+	local CMD="$1"
+	if [ "$1" = 'grep' -o "$1" = 'tail' -o "$1" = 'head' ]; then
 		echo >&2
 		while [ -n "$2" ]; do shift; done
 		echo "#### Complete file: $1 ####"
 		catfile "$1"
-		echo '#### grep output ####'
+		echo "#### $CMD output ####"
 	elif [ "$1" = 'test' ]; then
 		echo >&2
 		# doesn't support ! or non-file flags

test/integration/test-external-installation-planer-protocol

@@ -20,12 +20,56 @@ insertinstalledpackage 'unrelated-1' 'all' '1'
 
 setupaptarchive --no-update
 
+EIPPLOG="${TMPWORKINGDIRECTORY}/rootdir/var/log/apt/eipp.log"
+echo "Dir::Log::Planer \"$EIPPLOG\";" > ./rootdir/etc/apt/apt.conf.d/eipp-logging
+
 testsuccess apt update
 export APT_EDSP_DUMP_FILENAME="${TMPWORKINGDIRECTORY}/eipp.dump"
+testfailure test -r "$EIPPLOG"
 testfailure aptget install foo --planer dump -y
+testfailure test -r "$EIPPLOG"
 testfailure grep 'unrelated-2' "$APT_EDSP_DUMP_FILENAME"
 testsuccessequal '2' grep -c '^Package: foo$' "$APT_EDSP_DUMP_FILENAME"
 testsuccessequal '1' grep -c '^Package: libfoo$' "$APT_EDSP_DUMP_FILENAME"
-#less "$APT_EDSP_DUMP_FILENAME"
+testsuccessequal 'Planer: dump' grep '^Planer: ' "$APT_EDSP_DUMP_FILENAME"
+
+testsuccess aptget install foo -s
+testsuccess aptget install foo -y
+testsuccess test -r "$EIPPLOG"
+testsuccessequal 'Request: EIPP 0.1
+Architecture: amd64
+Architectures: amd64
+Remove: bar:amd64
+Install: libfoo:amd64 foo:amd64
+Planer: internal' head -n 6 "$EIPPLOG"
+aptinternalplaner < "$EIPPLOG" > planer.log || true
+testsuccessequal 'Remove: 6
+Unpack: 2
+Unpack: 4
+Configure: 2
+Configure: 4' grep -e '^Unpack:' -e '^Install:' -e '^Configure:' -e '^Remove:' planer.log
+
+rm -f "$EIPPLOG"
+testsuccess aptget install foo -s --reinstall
+testsuccess aptget install foo -y --reinstall
+testsuccess test -r "$EIPPLOG"
+testsuccessequal 'Request: EIPP 0.1
+Architecture: amd64
+Architectures: amd64
+ReInstall: foo:amd64
+Planer: internal' head -n 5 "$EIPPLOG"
+aptinternalplaner < "$EIPPLOG" > planer.log || true
+testsuccessequal 'Unpack: 4
+Configure: 4' grep -e '^Unpack:' -e '^Install:' -e '^Configure:' -e '^Remove:' planer.log
 
-aptget install foo -ys #--planer apt
+rm -f "$EIPPLOG"
+testsuccess aptget purge foo -s
+testsuccess aptget purge foo -y
+testsuccess test -r "$EIPPLOG"
+testsuccessequal 'Request: EIPP 0.1
+Architecture: amd64
+Architectures: amd64
+Remove: foo:amd64
+Planer: internal' head -n 5 "$EIPPLOG"
+aptinternalplaner < "$EIPPLOG" > planer.log || true
+testsuccessequal 'Remove: 4' grep -e '^Unpack:' -e '^Install:' -e '^Configure:' -e '^Remove:' planer.log