From: Michael Prokop Date: Wed, 14 Nov 2018 21:12:40 +0000 (+0100) Subject: Merge remote-tracking branch 'origin/pr/111' X-Git-Tag: v0.83~3 X-Git-Url: https://git.grml.org/?p=grml-debootstrap.git;a=commitdiff_plain;h=be5760d9fb97ac3663c1e6e3bd1787e9e8584676;hp=fc0a6d8c09bd56cdd17ef2586c45a919356fbd88 Merge remote-tracking branch 'origin/pr/111' --- diff --git a/.travis.yml b/.travis.yml index fe39908..0a3f432 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,7 @@ env: - TRAVIS_DEBIAN_DISTRIBUTION=unstable TRAVIS_DEBIAN_INCREMENT_VERSION_NUMBER=true script: - - docker run koalaman/shellcheck:stable --version - - docker run -v "$(pwd)":/code koalaman/shellcheck:stable -e SC2181 /code/chroot-script /code/grml-debootstrap - - wget -O- http://travis.debian.net/script.sh | sh - + - ./travis/execute.sh matrix: fast_finish: true diff --git a/Makefile b/Makefile index 235c916..8f790f8 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,6 @@ install: mkdir -p $(DESTDIR)/etc/debootstrap/extrapackages mkdir -p $(DESTDIR)/usr/sbin/ mkdir -p $(DESTDIR)/etc/zsh/completion.d/ - mkdir -p $(DESTDIR)/usr/share/grml-debootstrap/functions/ install -m 644 config $(DESTDIR)/etc/debootstrap/ install -m 644 devices.tar.gz $(DESTDIR)/etc/debootstrap/ install -m 644 locale.gen $(DESTDIR)/etc/debootstrap/ diff --git a/chroot-script b/chroot-script index 766a59d..4a20fa7 100755 --- a/chroot-script +++ b/chroot-script @@ -465,8 +465,12 @@ default_locales() { # adjust timezone {{{ timezone() { if [ -n "$TIMEZONE" ] ; then - echo "Adjusting /etc/localtime" - ln -sf "/usr/share/zoneinfo/$TIMEZONE" /etc/localtime + echo "Adjusting /etc/localtime" + ln -sf "/usr/share/zoneinfo/$TIMEZONE" /etc/localtime + + echo "Setting /etc/timezone to $TIMEZONE" + printf '%s\n' "$TIMEZONE" > /etc/timezone + fi } # }}} diff --git a/config b/config index c4d3b8e..6638a44 100644 --- a/config +++ b/config @@ -187,7 +187,7 @@ # Default: 'en_US:en' # DEFAULT_LANGUAGE='en_US:en' -# Use /usr/share/zoneinfo/$TIMEZONE for /etc/localtime. +# Use /usr/share/zoneinfo/$TIMEZONE for /etc/localtime + set /etc/timezone. # Default: 'Europe/Vienna' # TIMEZONE='Europe/Vienna' diff --git a/grml-debootstrap b/grml-debootstrap index 96f4a45..73e69dd 100755 --- a/grml-debootstrap +++ b/grml-debootstrap @@ -298,7 +298,13 @@ cleanup() { if [ -n "${ORIG_TARGET}" ] ; then einfo "Removing loopback mount of file ${ORIG_TARGET}." - kpartx -d "${ORIG_TARGET}" ; eend $? + kpartx -d "${ORIG_TARGET}" + # Workaround for a bug in kpartx which doesn't clean up properly, + # see Debian Bug #891077 and Github-PR grml/grml-debootstrap#112 + if dmsetup ls | grep -q "^${LOOP_PART} "; then + kpartx -d "/dev/${LOOP_DISK}" >/dev/null + fi + eend $? fi } @@ -1428,6 +1434,7 @@ prepare_vm() { # hopefully this always works as expected LOOP_PART="${DEVINFO##add map }" # 'loop0p1 (254:5): 0 20477 linear 7:0 3' LOOP_PART="${LOOP_PART// */}" # 'loop0p1' + LOOP_DISK="${LOOP_PART%p*}" # 'loop0' export TARGET="/dev/mapper/$LOOP_PART" # '/dev/mapper/loop1p1' if [ -z "$TARGET" ] ; then @@ -1512,6 +1519,11 @@ fi try_umount 3 "${MNTPOINT}"/dev umount "${MNTPOINT}" kpartx -d "${ORIG_TARGET}" >/dev/null + # Workaround for a bug in kpartx which doesn't clean up properly, + # see Debian Bug #891077 and Github-PR grml/grml-debootstrap#112 + if dmsetup ls | grep -q "^${LOOP_PART} "; then + kpartx -d "/dev/${LOOP_DISK}" >/dev/null + fi } # }}} diff --git a/travis/build-vm.sh b/travis/build-vm.sh new file mode 100755 index 0000000..9387d76 --- /dev/null +++ b/travis/build-vm.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eu -o pipefail + +TARGET="${TARGET:-/code/qemu.img}" +RELEASE="${RELEASE:-stretch}" + +cd "$(dirname "$TARGET")" +apt update +apt -y install ./grml-debootstrap*.deb + +grml-debootstrap \ + --force \ + --vmfile \ + --vmsize 3G \ + --target "$TARGET" \ + --bootappend "console=ttyS0,115200 console=tty0 vga=791" \ + --password grml \ + --release "$RELEASE" \ + --hostname "$RELEASE" \ + # EOF diff --git a/travis/execute.sh b/travis/execute.sh new file mode 100755 index 0000000..77016ea --- /dev/null +++ b/travis/execute.sh @@ -0,0 +1,172 @@ +#!/bin/bash + +set -eu -o pipefail +set -x + +if [ -z "${TRAVIS:-}" ] ; then + echo "Running outside of Travis." + + if [ "$#" -ne 1 ] ; then + echo "Usage: $(basename "$0") ./grml-debootstrap*.deb" >&2 + exit 1 + else + GRML_DEBOOTSTRAP_DEB="$1" + if [ "$(dirname "$(realpath "$GRML_DEBOOTSTRAP_DEB")")" != "$(pwd)" ] ; then + echo "Error: the grml-debootstrap*.deb needs to be inside $(pwd) to be shared with docker container." >&2 + exit 1 + fi + fi +fi + +RELEASE="${RELEASE:-stretch}" +export RELEASE + +TARGET="${TARGET:-qemu.img}" + +bailout() { + if [ -n "${QEMU_PID:-}" ] ; then + # shellcheck disable=SC2009 + ps --pid="${QEMU_PID}" -o pid= | grep -q '.' && kill "${QEMU_PID:-}" + fi + + if [ -f "${TARGET:-}" ] ; then + sudo kpartx -dv "$(realpath "${TARGET}")" + fi + + if [ -n "${LOOP_DISK:-}" ] ; then + if sudo dmsetup ls | grep -q "${LOOP_DISK}"; then + sudo kpartx -d "/dev/${LOOP_DISK}" + fi + fi + + local loopmount + loopmount="$(sudo losetup -a | grep "$(realpath "${TARGET}")" | cut -f1 -d: || true)" + + if [ -n "${loopmount:-}" ] ; then + sudo losetup -d "${loopmount}" + fi + + [ -n "${1:-}" ] && EXIT_CODE="$1" || EXIT_CODE=1 + exit "$EXIT_CODE" +} +trap bailout 1 2 3 6 14 15 + +# run shellcheck tests +docker run koalaman/shellcheck:stable --version +docker run --rm -v "$(pwd)":/code koalaman/shellcheck:stable -e SC2181 /code/chroot-script /code/grml-debootstrap + +# build Debian package +if [ -z "${TRAVIS:-}" ] ; then + echo "Not running under Travis, installing local grml-debootstrap package ${GRML_DEBOOTSTRAP_DEB}." +else + if ! [ "${TRAVIS_DEBIAN_DISTRIBUTION:-}" = "unstable" ] ; then + echo "TRAVIS_DEBIAN_DISTRIBUTION is $TRAVIS_DEBIAN_DISTRIBUTION and not unstable, skipping VM build tests." + exit 0 + fi + wget -O- https://travis.debian.net/script.sh | sh - + # copy only the binary from the TRAVIS_DEBIAN_INCREMENT_VERSION_NUMBER=true build + cp ../grml-debootstrap_*travis*deb . +fi + +# we need to run in privileged mode to be able to use loop devices +docker run --privileged -v "$(pwd)":/code --rm -i -t debian:stretch /code/travis/build-vm.sh + +[ -x ./goss ] || curl -fsSL https://goss.rocks/install | GOSS_DST="$(pwd)" sh + +# Ubuntu trusty (14.04LTS) doesn't have realpath in coreutils yet +if ! command -v realpath &>/dev/null ; then + REALPATH_PACKAGE=realpath +fi + +sudo apt-get update +sudo apt-get -y install qemu-system-x86 kpartx python-pexpect python-serial ${REALPATH_PACKAGE:-} + +# run tests from inside Debian system +DEVINFO=$(sudo kpartx -asv "${TARGET}") +LOOP_PART="${DEVINFO##add map }" +LOOP_PART="${LOOP_PART// */}" +LOOP_DISK="${LOOP_PART%p*}" +IMG_FILE="/dev/mapper/$LOOP_PART" + +MNTPOINT="$(mktemp -d)" +sudo mount "$IMG_FILE" "${MNTPOINT}" + +sudo cp ./goss "${MNTPOINT}"/usr/local/bin/goss +sudo cp ./travis/goss.yaml "${MNTPOINT}"/root/goss.yaml + +sudo umount "${MNTPOINT}" +sudo kpartx -dv "$(realpath "${TARGET}")" +if sudo dmsetup ls | grep -q "${LOOP_DISK}"; then + sudo kpartx -d "/dev/${LOOP_DISK}" +fi + +rmdir "$MNTPOINT" + +sudo chown "$(id -un)" qemu.img +rm -f ./serial0 +mkfifo ./serial0 +qemu-system-x86_64 -hda qemu.img -display none -vnc :0 \ + -device virtio-serial-pci \ + -chardev pipe,id=ch0,path=./serial0 \ + -device virtserialport,chardev=ch0,name=serial0 \ + -serial pty &>qemu.log & +QEMU_PID="$!" + +timeout=30 +success=0 +while [ "$timeout" -gt 0 ] ; do + ((timeout--)) + if grep -q 'char device redirected to ' qemu.log ; then + success=1 + sleep 1 + break + else + echo "No serial console from Qemu found yet [$timeout retries left]" + sleep 1 + fi +done + +if [ "$success" = "1" ] ; then + serial_port=$(awk '/char device redirected/ {print $5}' qemu.log) +else + echo "Error: Failed to identify serial console port." >&2 + exit 1 +fi + +timeout=30 +success=0 +while [ "$timeout" -gt 0 ] ; do + ((timeout--)) + if [ -c "$serial_port" ] ; then + success=1 + sleep 1 + break + else + echo "No block device for serial console found yet [$timeout retries left]" + sleep 1 + fi +done + +if [ "$success" = "0" ] ; then + echo "Error: can't access serial console block device." >&2 + exit 1 +fi + +sudo chown "$(id -un)" "$serial_port" +./travis/serial-console-connection --port "$serial_port" --hostname "$RELEASE" --pipefile "serial0" --vmoutput "vm-output.txt" + +cat vm-output.txt + +RC=0 +if grep -q '^failure_exit' vm-output.txt ; then + echo "We noticed failing tests." + RC=1 +else + echo "All tests passed." +fi + +echo "Finished serial console connection [timeout=${timeout}]." + +bailout $RC + +# EOF diff --git a/travis/goss.yaml b/travis/goss.yaml new file mode 100644 index 0000000..97b75d8 --- /dev/null +++ b/travis/goss.yaml @@ -0,0 +1,11 @@ +process: + sshd: + running: true +file: + /etc/timezone: + exists: true + contains: ["Europe/Vienna"] + /etc/localtime: + filetype: symlink + exists: true + linked-to: /usr/share/zoneinfo/Europe/Vienna diff --git a/travis/serial-console-connection b/travis/serial-console-connection new file mode 100755 index 0000000..46c2f02 --- /dev/null +++ b/travis/serial-console-connection @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +from __future__ import print_function +import argparse +import pexpect +import serial +import sys +import time +from pexpect import fdpexpect + +parser = argparse.ArgumentParser(description='Connect to serial console ' + + 'to execute stuff') +parser.add_argument('--port', required=True, + help='serial console device to connect ' + + 'to (e.g. /dev/pts/X)') +parser.add_argument('--hostname', default="stretch", + help='hostname of the system for login process ' + + '(default: stretch)') +parser.add_argument('--pipefile', default="./serial0", + help='file name for named pipe file (for ' + + 'interacting between host + VM via QEMU ' + + '(default: ./serial0)') +parser.add_argument('--vmoutput', default="vm-output.log", + help='filename for VM output (default: vm-output.log)') +parser.add_argument('--user', default="root", + help='user name to use for login (default: root)') +parser.add_argument('--password', default="grml", + help='password for login (default: grml)') +args = parser.parse_args() + + +def execute(port, hostname, user, baudrate=115200, timeout=5): + ser = serial.Serial(port, baudrate) + ser.flushInput() + ser.flushOutput() + ser.write("\n") + ser.flush() + + child = fdpexpect.fdspawn(ser.fileno()) + child.sendline("") + try: + print("Begin of execution inside VM") + child.expect("%s@%s" % (user, hostname), timeout=timeout) + child.sendline("/usr/local/bin/goss --gossfile /root/goss.yaml " + + "validate --format tap > /root/goss.tap ; " + + "echo $? > /root/goss.exitcode\n") + # NOTE - the serial0 is hardcoded here + child.sendline("cat /root/goss.tap > /dev/virtio-ports/serial0\n") + child.sendline("grep -q '^0' /root/goss.exitcode && " + + "echo clean_exit > /dev/virtio-ports/serial0\n") + child.sendline("grep -q '^0' /root/goss.exitcode || " + + "echo failure_exit > /dev/virtio-ports/serial0\n") + child.sendline("poweroff\n") + print("End of execution inside VM") + except Exception as except_inst: + print("Execution inside VM failed: ", except_inst) + + +def login(port, hostname, user, password, + baudrate=115200, timeout=5): + ser = serial.Serial(port, baudrate) + ser.flushInput() + ser.flushOutput() + + child = fdpexpect.fdspawn(ser.fileno()) + child.sendline("\n") + + try: + child.expect("root@%s" % hostname, timeout=timeout) + return + except: + pass + + print("Checking for login prompt...") + child.expect("%s login:" % hostname, timeout=timeout) + ser.write("%s\n" % user) + ser.flush() + time.sleep(1) + ser.write("%s\n" % password) + ser.flush() + time.sleep(1) + print("login ok...") + + +if __name__ == "__main__": + hostname = args.hostname + password = args.password + pipefile = args.pipefile + port = args.port + user = args.user + vmoutput = args.vmoutput + + with open(pipefile, 'r') as output_pipe: + success = False + for i in range(12): + try: + print("Logging into {0} via serial " + "console [try {1}]".format(port, i)) + login(port, hostname, user, password) + success = True + break + except Exception as except_inst: + print("Login failure (try {0}):".format(i), + except_inst, file=sys.stderr) + time.sleep(5) + + if success: + execute(port, hostname, user) + with open(vmoutput, 'w') as fp: + output = output_pipe.read() + print(output) + fp.write(output)