travis: run tests from inside VM + support test execution outside of Travis CI

This is a big improvement, because we can now communicate with
the running VM, including executing processes inside the VM, like
running goss directly from inside the VM.

While at it also adjust travis/execute.sh to be able to run it
*locally* as well. We now don't have to necessarily push to Travis
CI, which also serves to be useful when developing new features.

Thanks: Christian Hofstaedtler <zeha@debian.org> for helping me with the serial console handling

diff --git a/travis/execute.sh b/travis/execute.sh
index 52e3d26..77016ea 100755 (executable)
--- a/travis/execute.sh
+++ b/travis/execute.sh
@@ -1,55 +1,172 @@
 #!/bin/bash
 
 set -eu -o pipefail
 #!/bin/bash
 
 set -eu -o pipefail

+set -x
+
+if [ -z "${TRAVIS:-}" ] ; then
+  echo "Running outside of Travis."
+
+  if [ "$#" -ne 1 ] ; then
+    echo "Usage: $(basename "$0") ./grml-debootstrap*.deb" >&2
+    exit 1
+  else
+    GRML_DEBOOTSTRAP_DEB="$1"
+    if [ "$(dirname "$(realpath "$GRML_DEBOOTSTRAP_DEB")")" != "$(pwd)" ] ; then
+      echo "Error: the grml-debootstrap*.deb needs to be inside $(pwd) to be shared with docker container." >&2
+      exit 1
+    fi
+  fi
+fi

 
 RELEASE="${RELEASE:-stretch}"
 export RELEASE
 
 
 RELEASE="${RELEASE:-stretch}"
 export RELEASE
 

+TARGET="${TARGET:-qemu.img}"
+
+bailout() {
+  if [ -n "${QEMU_PID:-}" ] ; then
+    # shellcheck disable=SC2009
+    ps --pid="${QEMU_PID}" -o pid= | grep -q '.' && kill "${QEMU_PID:-}"
+  fi
+
+  if [ -f "${TARGET:-}" ] ; then
+    sudo kpartx -dv "$(realpath "${TARGET}")"
+  fi
+
+  if [ -n "${LOOP_DISK:-}" ] ; then
+    if sudo dmsetup ls | grep -q "${LOOP_DISK}"; then
+      sudo kpartx -d "/dev/${LOOP_DISK}"
+    fi
+  fi
+
+  local loopmount
+  loopmount="$(sudo losetup -a | grep "$(realpath "${TARGET}")" | cut -f1 -d: || true)"
+
+  if [ -n "${loopmount:-}" ] ; then
+    sudo losetup -d "${loopmount}"
+  fi
+
+  [ -n "${1:-}" ] && EXIT_CODE="$1" || EXIT_CODE=1
+  exit "$EXIT_CODE"
+}
+trap bailout 1 2 3 6 14 15
+

 # run shellcheck tests
 docker run koalaman/shellcheck:stable --version
 # run shellcheck tests
 docker run koalaman/shellcheck:stable --version

-docker run -v "$(pwd)":/code koalaman/shellcheck:stable -e SC2181 /code/chroot-script /code/grml-debootstrap
+docker run --rm -v "$(pwd)":/code koalaman/shellcheck:stable -e SC2181 /code/chroot-script /code/grml-debootstrap

 
 # build Debian package
 
 # build Debian package

-wget -O- https://travis.debian.net/script.sh | sh -
-
-if ! [ "${TRAVIS_DEBIAN_DISTRIBUTION:-}" = "unstable" ] ; then
-  echo "TRAVIS_DEBIAN_DISTRIBUTION is $TRAVIS_DEBIAN_DISTRIBUTION and not unstable, skipping VM build tests."
-  exit 0
+if [ -z "${TRAVIS:-}" ] ; then
+  echo "Not running under Travis, installing local grml-debootstrap package ${GRML_DEBOOTSTRAP_DEB}."
+else
+  if ! [ "${TRAVIS_DEBIAN_DISTRIBUTION:-}" = "unstable" ] ; then
+    echo "TRAVIS_DEBIAN_DISTRIBUTION is $TRAVIS_DEBIAN_DISTRIBUTION and not unstable, skipping VM build tests."
+    exit 0
+  fi
+  wget -O- https://travis.debian.net/script.sh | sh -
+  # copy only the binary from the TRAVIS_DEBIAN_INCREMENT_VERSION_NUMBER=true build
+  cp ../grml-debootstrap_*travis*deb .

 fi
 
 fi
 

-# copy only the binary from the TRAVIS_DEBIAN_INCREMENT_VERSION_NUMBER=true build
-cp ../grml-debootstrap_*travis*deb .
-

 # we need to run in privileged mode to be able to use loop devices
 docker run --privileged -v "$(pwd)":/code --rm -i -t debian:stretch /code/travis/build-vm.sh
 
 [ -x ./goss ] || curl -fsSL https://goss.rocks/install | GOSS_DST="$(pwd)" sh
 
 # we need to run in privileged mode to be able to use loop devices
 docker run --privileged -v "$(pwd)":/code --rm -i -t debian:stretch /code/travis/build-vm.sh
 
 [ -x ./goss ] || curl -fsSL https://goss.rocks/install | GOSS_DST="$(pwd)" sh
 

+# Ubuntu trusty (14.04LTS) doesn't have realpath in coreutils yet
+if ! command -v realpath &>/dev/null ; then
+  REALPATH_PACKAGE=realpath
+fi
+

 sudo apt-get update
 sudo apt-get update

-sudo apt-get -y install qemu-system-x86
+sudo apt-get -y install qemu-system-x86 kpartx python-pexpect python-serial ${REALPATH_PACKAGE:-}

 
 

-# sudo timeout --preserve-status --foreground 120 qemu-system-x86_64 -hda qemu.img -serial stdio -display none | tee -a qemu.log
-sudo qemu-system-x86_64 -hda qemu.img -serial stdio -display none | tee -a qemu.log &
+# run tests from inside Debian system
+DEVINFO=$(sudo kpartx -asv "${TARGET}")
+LOOP_PART="${DEVINFO##add map }"
+LOOP_PART="${LOOP_PART// */}"
+LOOP_DISK="${LOOP_PART%p*}"
+IMG_FILE="/dev/mapper/$LOOP_PART"

 
 

-timeout=120
+MNTPOINT="$(mktemp -d)"
+sudo mount "$IMG_FILE" "${MNTPOINT}"
+
+sudo cp ./goss "${MNTPOINT}"/usr/local/bin/goss
+sudo cp ./travis/goss.yaml "${MNTPOINT}"/root/goss.yaml
+
+sudo umount "${MNTPOINT}"
+sudo kpartx -dv "$(realpath "${TARGET}")"
+if sudo dmsetup ls | grep -q "${LOOP_DISK}"; then
+  sudo kpartx -d "/dev/${LOOP_DISK}"
+fi
+
+rmdir "$MNTPOINT"
+
+sudo chown "$(id -un)" qemu.img
+rm -f ./serial0
+mkfifo ./serial0
+qemu-system-x86_64 -hda qemu.img -display none -vnc :0 \
+                   -device virtio-serial-pci \
+                   -chardev pipe,id=ch0,path=./serial0 \
+                   -device virtserialport,chardev=ch0,name=serial0 \
+                   -serial pty &>qemu.log &
+QEMU_PID="$!"
+
+timeout=30

 success=0
 success=0

-while [ "$timeout" -gt 0 ]; do
+while [ "$timeout" -gt 0 ] ; do

   ((timeout--))
   ((timeout--))

-  if ./goss --gossfile ./travis/goss.yaml validate --format nagios ; then
+  if grep -q 'char device redirected to ' qemu.log ; then

     success=1
     sleep 1
     break
   else
     success=1
     sleep 1
     break
   else

-    echo "Tests didn't pass YET, will retry again [$timeout retries left]"
+    echo "No serial console from Qemu found yet [$timeout retries left]"

     sleep 1
   fi
 done
 
 if [ "$success" = "1" ] ; then
     sleep 1
   fi
 done
 
 if [ "$success" = "1" ] ; then

-  echo "All tests passed! (◕‿◕)"
+  serial_port=$(awk '/char device redirected/ {print $5}' qemu.log)

 else
 else

-  echo "Reached timeout after $timeout seconds with failing tests. ¯\(º o)/¯ ☂" >&2
-  echo "Latest test run results:"
-  ./goss --gossfile ./travis/goss.yaml validate
+  echo "Error: Failed to identify serial console port." >&2
+  exit 1
+fi
+
+timeout=30
+success=0
+while [ "$timeout" -gt 0 ] ; do
+  ((timeout--))
+  if [ -c "$serial_port" ] ; then
+    success=1
+    sleep 1
+    break
+  else
+    echo "No block device for serial console found yet [$timeout retries left]"
+    sleep 1
+  fi
+done
+
+if [ "$success" = "0" ] ; then
+  echo "Error: can't access serial console block device." >&2

   exit 1
 fi
   exit 1
 fi

+
+sudo chown "$(id -un)" "$serial_port"
+./travis/serial-console-connection --port "$serial_port" --hostname "$RELEASE" --pipefile "serial0" --vmoutput "vm-output.txt"
+
+cat vm-output.txt
+
+RC=0
+if grep -q '^failure_exit' vm-output.txt ; then
+  echo "We noticed failing tests."
+  RC=1
+else
+  echo "All tests passed."
+fi
+
+echo "Finished serial console connection [timeout=${timeout}]."
+
+bailout $RC
+
+# EOF

diff --git a/travis/goss.yaml b/travis/goss.yaml
index 9e8e673..a504d1f 100644 (file)
--- a/travis/goss.yaml
+++ b/travis/goss.yaml
@@ -1,5 +1,3 @@
-command:
-  grep -qe "Debian GNU/Linux .* {{ .Env.RELEASE }} ttyS0" qemu.log:
-    exit-status: 0
-  grep -qe "{{ .Env.RELEASE }} login:" qemu.log:
-    exit-status: 0
+process:
+  sshd:
+    running: true

diff --git a/travis/serial-console-connection b/travis/serial-console-connection
new file mode 100755 (executable)
index 0000000..46c2f02
--- /dev/null
+++ b/travis/serial-console-connection
@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import argparse
+import pexpect
+import serial
+import sys
+import time
+from pexpect import fdpexpect
+
+parser = argparse.ArgumentParser(description='Connect to serial console ' +
+                                 'to execute stuff')
+parser.add_argument('--port', required=True,
+                    help='serial console device to connect ' +
+                    'to (e.g. /dev/pts/X)')
+parser.add_argument('--hostname', default="stretch",
+                    help='hostname of the system for login process ' +
+                    '(default: stretch)')
+parser.add_argument('--pipefile', default="./serial0",
+                    help='file name for named pipe file (for ' +
+                    'interacting between host + VM via QEMU ' +
+                    '(default: ./serial0)')
+parser.add_argument('--vmoutput', default="vm-output.log",
+                    help='filename for VM output (default: vm-output.log)')
+parser.add_argument('--user', default="root",
+                    help='user name to use for login (default: root)')
+parser.add_argument('--password', default="grml",
+                    help='password for login (default: grml)')
+args = parser.parse_args()
+
+
+def execute(port, hostname, user, baudrate=115200, timeout=5):
+    ser = serial.Serial(port, baudrate)
+    ser.flushInput()
+    ser.flushOutput()
+    ser.write("\n")
+    ser.flush()
+
+    child = fdpexpect.fdspawn(ser.fileno())
+    child.sendline("")
+    try:
+        print("Begin of execution inside VM")
+        child.expect("%s@%s" % (user, hostname), timeout=timeout)
+        child.sendline("/usr/local/bin/goss --gossfile /root/goss.yaml " +
+                       "validate --format tap > /root/goss.tap ; " +
+                       "echo $? > /root/goss.exitcode\n")
+        # NOTE - the serial0 is hardcoded here
+        child.sendline("cat /root/goss.tap > /dev/virtio-ports/serial0\n")
+        child.sendline("grep -q '^0' /root/goss.exitcode && " +
+                       "echo clean_exit > /dev/virtio-ports/serial0\n")
+        child.sendline("grep -q '^0' /root/goss.exitcode || " +
+                       "echo failure_exit > /dev/virtio-ports/serial0\n")
+        child.sendline("poweroff\n")
+        print("End of execution inside VM")
+    except Exception as except_inst:
+        print("Execution inside VM failed: ", except_inst)
+
+
+def login(port, hostname, user, password,
+          baudrate=115200, timeout=5):
+    ser = serial.Serial(port, baudrate)
+    ser.flushInput()
+    ser.flushOutput()
+
+    child = fdpexpect.fdspawn(ser.fileno())
+    child.sendline("\n")
+
+    try:
+        child.expect("root@%s" % hostname, timeout=timeout)
+        return
+    except:
+        pass
+
+    print("Checking for login prompt...")
+    child.expect("%s login:" % hostname, timeout=timeout)
+    ser.write("%s\n" % user)
+    ser.flush()
+    time.sleep(1)
+    ser.write("%s\n" % password)
+    ser.flush()
+    time.sleep(1)
+    print("login ok...")
+
+
+if __name__ == "__main__":
+    hostname = args.hostname
+    password = args.password
+    pipefile = args.pipefile
+    port = args.port
+    user = args.user
+    vmoutput = args.vmoutput
+
+    with open(pipefile, 'r') as output_pipe:
+        success = False
+        for i in range(12):
+            try:
+                print("Logging into {0} via serial "
+                      "console [try {1}]".format(port, i))
+                login(port, hostname, user, password)
+                success = True
+                break
+            except Exception as except_inst:
+                print("Login failure (try {0}):".format(i),
+                      except_inst, file=sys.stderr)
+                time.sleep(5)
+
+        if success:
+            execute(port, hostname, user)
+            with open(vmoutput, 'w') as fp:
+                output = output_pipe.read()
+                print(output)
+                fp.write(output)