From 4e93b8eddcee4f8e2a8d4b3bed24ed794a936c0e Mon Sep 17 00:00:00 2001
From: Michael Prokop <mika@grml.org>
Date: Wed, 10 Mar 2021 14:59:50 +0100
Subject: [PATCH] GRMLBASE/01-packages: also detect and report unknown package
 names

If FAI doesn't know package names during its dirinstall run, it reports:

| WARNING: These unknown packages are removed from the installation list

... in fai.log and software.log. This happens e.g. if a package was
removed from a Debian repository and is no longer known therefore. What
FAI reports through its install_packages.list are only packages that
were asked for installation *and* exist as such (so not a "unknown
package package"), but failed to install - so this sadly doesn't help
us.

One option would be to set FAI_DISABLE_PACKAGE_NAME_CHECK=1 (which
corresponds to FAI install_packages's -N cmdline option), but then the
installation fails hard with unknown packages, which isn't what we want
as default behavior. But provide this disabled(!) configuration option
in GRMLBASE.var, just in case someone is looking for it.

As a solution for our use case, let's handle this via our existing
GRMLBASE/01-packages script, which checks for missing packages already.
Now we also check FAI's software.log for the 'These unknown packages'
error message and report them via package_errors.log.

This work was funded by Grml-Forensic.
---
 etc/grml/fai/config/class/GRMLBASE.var           |  3 ++
 etc/grml/fai/config/scripts/GRMLBASE/01-packages | 43 +++++++++++++++---------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/etc/grml/fai/config/class/GRMLBASE.var b/etc/grml/fai/config/class/GRMLBASE.var
index 5ac489c..caf5098 100644
--- a/etc/grml/fai/config/class/GRMLBASE.var
+++ b/etc/grml/fai/config/class/GRMLBASE.var
@@ -20,3 +20,6 @@ ROOTPW='x'
 # needs to be quite high so we can override installation
 # of specific packages through the IGNORE class.
 MAXPACKAGES=3000
+
+# Do not check package names whether they are valid, but report failure instead.
+# FAI_DISABLE_PACKAGE_NAME_CHECK=1
diff --git a/etc/grml/fai/config/scripts/GRMLBASE/01-packages b/etc/grml/fai/config/scripts/GRMLBASE/01-packages
index bf92ae9..aae58ae 100755
--- a/etc/grml/fai/config/scripts/GRMLBASE/01-packages
+++ b/etc/grml/fai/config/scripts/GRMLBASE/01-packages
@@ -9,12 +9,24 @@
 set -u
 set -e
 
-PACKAGE_LIST=/var/log/install_packages.list
+echo -n > "${LOGDIR}"/package_errors.log  # ensure we start with an empty file
 
-if ! [ -r "$target/${PACKAGE_LIST}" ] ; then
-  echo "No $target/${PACKAGE_LIST} found, will not run package validation check."
+if ! [ -e "${LOGDIR}"/software.log ] ; then
+  echo "Warning: no ${LOGDIR}/software.log found (build/update run?), skipping check for unknown packages."
 else
-  printf "Validating package list: "
+  if grep -q 'These unknown packages' "${LOGDIR}"/software.log ; then
+    echo "Identified unknown packages in ${LOGDIR}/software.log"
+    grep 'These unknown packages' "${LOGDIR}"/software.log | \
+      sed 's/.*These unknown packages.*: //; s/ / not_installable\n/g' >> "${LOGDIR}/package_errors.log"
+  fi
+fi
+
+PACKAGE_LIST=/var/log/install_packages.list
+# shellcheck disable=SC2154
+if ! [ -r "${target}/${PACKAGE_LIST}" ] ; then
+  echo "No ${target}/${PACKAGE_LIST} found, will not run package validation check."
+else
+  echo "Validating package list against dpkg state..."
 
   TMPSTDOUT=$(mktemp)
   TMPSTDERR=$(mktemp)
@@ -25,23 +37,24 @@ else
   # for packages unknown to dpkg on stderr
   # NOTE: 'grep -v -- '-$' ignores packages in FAI's package list that are
   # marked for removal
-  $ROOTCMD dpkg --list $(grep -v '^#' $target/${PACKAGE_LIST} | grep -v -- '-$') 2>"$TMPSTDERR" | \
-    grep -e '^[urph][ncufhWt]' > "$TMPSTDOUT" || true
+  # shellcheck disable=SC2046
+  ${ROOTCMD} dpkg --list $(grep -v '^#' "${target}/${PACKAGE_LIST}" | grep -v -- '-$') 2>"${TMPSTDERR}" | \
+    grep -e '^[urph][ncufhWt]' > "${TMPSTDOUT}" || true
 
   # extract packages from stdout
-  awk '/^un/ {print $2 " not_installable"}' "$TMPSTDOUT" > "$LOGDIR/package_errors.log"
+  awk '/^un/ {print $2 " not_installable"}' "${TMPSTDOUT}" >> "${LOGDIR}/package_errors.log"
 
   # extract packages from stderr
-  grep 'packages found matching' "$TMPSTDERR" | \
-    sed 's/dpkg-query: [Nn]o packages found matching \(.*\)/\1 not_installable/' >> "$LOGDIR/package_errors.log"
+  grep 'packages found matching' "${TMPSTDERR}" | \
+    sed 's/dpkg-query: [Nn]o packages found matching \(.*\)/\1 not_installable/' >> "${LOGDIR}/package_errors.log"
 
-  if [ -s "$LOGDIR/package_errors.log" ] ; then
-    printf "failed (there have been errors, find them at $LOGDIR/package_errors.log)\n"
-  else
-    printf "done - no errors found\n"
-  fi
+  rm -f "${TMPSTDOUT}" "${TMPSTDERR}"
+fi
 
-  rm -f "$TMPSTDOUT" "$TMPSTDERR"
+if [ -s "${LOGDIR}/package_errors.log" ] ; then
+  echo "Warning: failed (there have been errors, find them at ${LOGDIR}/package_errors.log)."
+else
+  echo "Done - no errors found."
 fi
 
 ## END OF FILE #################################################################
-- 
2.1.4