From d761bd6cdc6b54b67883e295a0dd1694e492bd7b Mon Sep 17 00:00:00 2001 From: Michael Prokop Date: Sun, 13 Dec 2009 15:32:47 +0100 Subject: [PATCH] Support for logging to database via grml-live-db. Changed logic of ZERO_LOGFILE to PRESERVE_LOGFILE. grml-live-db provides a feature to log build information into a sqlite database. --- db/db-to-fai | 49 +++++++++++++++ db/dpkg-to-db | 142 ++++++++++++++++++++++++++++++++++++++++++ debian/control | 10 +++ debian/grml-live-db.doc-base | 10 +++ debian/grml-live-db.docs | 1 + debian/grml-live-db.install | 3 + debian/overrides.grml-live-db | 1 + debian/rules | 22 ++++++- docs/Makefile | 6 +- docs/grml-live-db.txt | 104 +++++++++++++++++++++++++++++++ etc/grml/grml-live.conf | 19 ++++-- grml-live | 54 +++++++++++++++- 12 files changed, 412 insertions(+), 9 deletions(-) create mode 100755 db/db-to-fai create mode 100755 db/dpkg-to-db create mode 100644 debian/grml-live-db.doc-base create mode 100644 debian/grml-live-db.docs create mode 100644 debian/grml-live-db.install create mode 100644 debian/overrides.grml-live-db create mode 100644 docs/grml-live-db.txt diff --git a/db/db-to-fai b/db/db-to-fai new file mode 100755 index 0000000..074cca3 --- /dev/null +++ b/db/db-to-fai @@ -0,0 +1,49 @@ +#!/bin/sh +# Filename: db-to-fai +# Purpose: convert output of grml-live's sqlite database for use within FAI +# Authors: grml-team (grml.org) +# Bug-Reports: see http://grml.org/bugs/ +# License: This file is licensed under the GPL v2 or any later version. +################################################################################ + +if [ -z "$2" ] ; then + echo "Usage: $0 /path/to/grml-live.db " + exit 1 +fi + +DB="$1" +BUILD_ID="$2" + +if ! [ -r "$DB" ] ; then + echo "Error: can not access database ${DB}.">&2 + bailout 1 +fi + +TMPFILE=$(mktemp) + +bailout() { + rm -f "$TMPFILE" + [ -n "$1" ] && exit "$1" || exit 0 +} + +# get information from db: +if ! echo "select package,version FROM packages, build WHERE build.id = $BUILD_ID AND packages.build = build.id and status = 'ii';" | sqlite3 $DB > $TMPFILE ; then + echo "Error retrieving values from database ${DB}." >&2 + bailout 1 +else + # make sure we god some matches: + if ! grep -q '^[a-zA-Z]*' "$TMPFILE" ; then + echo "No packages retrieved from build id $BUILD_ID - wrong id?" >&2 + bailout 1 + fi + + # write fai header and package information to stdout: + echo "# package list of build $BUILD_ID from database $DB:" + echo "PACKAGES aptitude" + awk -F\| '{print $1"="$2}' "$TMPFILE" +fi + +# clean exit: +bailout + +## END OF FILE ################################################################# diff --git a/db/dpkg-to-db b/db/dpkg-to-db new file mode 100755 index 0000000..6510d12 --- /dev/null +++ b/db/dpkg-to-db @@ -0,0 +1,142 @@ +#!/usr/bin/perl -w +# Filename: dpkg-to-db +# Purpose: add grml build information into a sqlite database +# Authors: grml-team (grml.org) +# Bug-Reports: see http://grml.org/bugs/ +# License: This file is licensed under the GPL v2 or any later version. +################################################################################ +# Requires the following Debian packages (handled via grml-live-db depends): +# libdbd-sqlite3-perl libdbi-perl libtimedate-perl perl-doc sqlite3 +################################################################################ + +use strict; + +use warnings; +use Getopt::Long; +use Pod::Usage; +use DBI; +use Date::Format; + + +my ($db, $logfile, $flavour, $help); +my $rc = GetOptions ( + 'db|d=s' => \$db, + 'logfile|l=s' => \$logfile, + 'flavour|f=s' => \$flavour, + 'help|h' => \$help, + ); + +pod2usage(1) if $help; + +pod2usage(-message => "$0: Need a sqlite database.\n") unless $db; +pod2usage(-message => "$0: Need a logfile to insert.\n") unless $logfile; +pod2usage(-message => "$0: Need the flavour information\n") unless $flavour; + +my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") or die "Could not connect to database: " . $DBI::err; + +# We use foreign key - beware this needs sqlite > 3.6.19 +$dbh->do("PRAGMA foreign_keys = ON"); + +open (my $fh, '<', $logfile) or die "Could not open $logfile: $!"; + +# read content of log file - please do not try this at home :) +my $log = do { local $/; <$fh> }; + +my $identifier = "$flavour-". time2str('%Y%m%d%H%M%S', time()); + +# Prepare tables if not yet present {{{ +my $create_table_build = $dbh->prepare(" +CREATE TABLE if not exists build ( id integer primary key autoincrement, +identifier varchar(30), +flavour varchar(30), +date varchar(30), +logfile blob); +") + or die "Could not create tables: " . $dbh->errstr."\n"; + +$create_table_build->execute() + or die "Can't execute SQL statement: " . $dbh->errstr."\n"; + +my $create_table_packages = $dbh->prepare(" +CREATE TABLE if not exists packages ( id integer primary key autoincrement, +package varchar(30), +status varchar(2), +version varchar(30), +build integer, +FOREIGN KEY(build) REFERENCES build(id)); +") + or die "Could not create tables: " . $dbh->errstr."\n"; + +$create_table_packages->execute() + or die "Can't execute SQL statement: " . $dbh->errstr."\n"; +# }}} + + +# Write information to database {{{ +my $sth = $dbh->prepare("INSERT into build ('identifier','flavour','date','logfile') VALUES (?,?,?,?)") + or die "Could not prepare db statement: " . $dbh->errstr; + +# Execute the query +$sth->execute($identifier, $flavour, time(), $log) + or die "Could not add build to db: " . $sth->errstr; + +$sth = $dbh->prepare("SELECT id from build where identifier = ?"); +$sth->execute($identifier) or die "Couldn't execute statement: " . $sth->errstr; +my $row = $sth->fetch; +my $id = $row->[0]; + +die "No id?" unless $id; + +$sth = $dbh->prepare("INSERT into packages (package, status, version, build) VALUES (?,?,?,?)") + or die "Could not prepare db statement: " . $dbh->errstr; + +while (my $line = <>) { + next unless $line =~ /^[a-z]{2} /; + # remove new lines + my ($status, $package, $version, $desc) = split (/\s+/, $line, 4); + $sth->execute($package, $status, $version, $id) + or die "Couldn't execute statement: " . $sth->errstr; + +} +# }}} + +print "recorded buildinformation with identifier $identifier as id $id\n"; + +# perldoc -F ./dpkg-to-db + +__END__ + +=head1 dpkg-to-db + +dpkg-to-db - add grml build information into a sqlite database + +=head1 SYNOPSIS + +dpkg-to-db [options] + + Options: + --help brief help message + --db database file + --logfile logfile which should be added + --flavour which flavour the build is + +=head1 OPTIONS + +=over 8 + +=item B<-help> + +Print a brief help message and exits. + +=back + +=head1 DESCRIPTION + +B will read the given input file(s) which holds output of +`dpkg --list' and writes the information to the specified database. + +=head1 USAGE EXAMPLES + +Please see B for further information. + +=cut diff --git a/debian/control b/debian/control index 84324b6..2e91104 100644 --- a/debian/control +++ b/debian/control @@ -27,3 +27,13 @@ Description: templates/boot/addons for grml-live This package provides the boot addons known as the features providing bootoptions bsd, dos, grub and hdt. You can safely skip this package if you don't need the addons. + +Package: grml-live-db +Architecture: all +Depends: ${misc:Depends}, grml-live, libdbd-sqlite3-perl, libdbi-perl, libtimedate-perl, sqlite3 +Recommends: perl-doc +Description: log package build information of grml-live to database + This package provides a database layer for storing build + information about grml-live builds in a sqlite3 database. + More details are available in the provided grml-live-db manpage + and /usr/share/doc/grml-live-db/grml-live-db.html diff --git a/debian/grml-live-db.doc-base b/debian/grml-live-db.doc-base new file mode 100644 index 0000000..2eab618 --- /dev/null +++ b/debian/grml-live-db.doc-base @@ -0,0 +1,10 @@ +Document: grml-live-db +Title: Documentation for the database wrapper of grml-live +Author: Michael Prokop +Abstract: grml-live-db provides a database layer for storing build + information about grml-live builds in a sqlite3 database. +Section: Debian + +Format: HTML +Index: /usr/share/doc/grml-live-db/grml-live-db.html +Files: /usr/share/doc/grml-live-db/grml-live-db.html diff --git a/debian/grml-live-db.docs b/debian/grml-live-db.docs new file mode 100644 index 0000000..1443038 --- /dev/null +++ b/debian/grml-live-db.docs @@ -0,0 +1 @@ +docs/grml-live-db.html diff --git a/debian/grml-live-db.install b/debian/grml-live-db.install new file mode 100644 index 0000000..ff4810d --- /dev/null +++ b/debian/grml-live-db.install @@ -0,0 +1,3 @@ +db/db-to-fai usr/share/grml-live-db/scripts/ +db/dpkg-to-db usr/share/grml-live-db/scripts/ +docs/grml-live-db.8 usr/share/man/man8/ diff --git a/debian/overrides.grml-live-db b/debian/overrides.grml-live-db new file mode 100644 index 0000000..5946edd --- /dev/null +++ b/debian/overrides.grml-live-db @@ -0,0 +1 @@ +grml-live-db: unknown-section grml diff --git a/debian/rules b/debian/rules index 905916b..a2e847f 100755 --- a/debian/rules +++ b/debian/rules @@ -78,11 +78,31 @@ grml-live-addons: install dh_md5sums -p$@ -P$(b)/$@ dh_builddeb -p$@ -P$(b)/$@ +grml-live-db: install + @echo "--- Building: $@" + dh_installdirs -p$@ -P$(b)/$@ + dh_link -p$@ -P$(b)/$@ + dh_installdocs -p$@ -P$(b)/$@ + dh_installchangelogs -p$@ -P$(b)/$@ + dh_install -p$@ -P$(b)/$@ + dh_strip -p$@ -P$(b)/$@ + dh_compress -p$@ -P$(b)/$@ + mkdir -p $(b)/$(@)/usr/share/lintian/overrides/ + cp -av debian/overrides.$(@) $(b)/$(@)/usr/share/lintian/overrides/$(@) + dh_fixperms -p$@ -P$(b)/$@ + dh_makeshlibs -p$@ -P$(b)/$@ -V + dh_installdeb -p$@ -P$(b)/$@ + dh_shlibdeps -p$@ -P$(b)/$@ + dh_installdebconf -p$@ -P$(b)/$@ + dh_gencontrol -p$@ -P$(b)/$@ + dh_md5sums -p$@ -P$(b)/$@ + dh_builddeb -p$@ -P$(b)/$@ + # Build architecture-dependent files here. binary-all: build install # Build architecture-independent files here. -binary-indep: build install grml-live grml-live-addons +binary-indep: build install grml-live grml-live-addons grml-live-db binary: binary-indep binary-arch .PHONY: build clean binary-indep binary-arch binary install diff --git a/docs/Makefile b/docs/Makefile index 9771ab2..a64b80b 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -4,9 +4,10 @@ doc: doc_man doc_html doc_html: html-stamp -html-stamp: grml-live.txt grml-live-remaster.txt +html-stamp: grml-live.txt grml-live-remaster.txt grml-live-db.txt asciidoc -b xhtml11 -a icons -a toc -a numbered grml-live.txt asciidoc -b xhtml11 -a icons grml-live-remaster.txt + asciidoc -b xhtml11 -a icons grml-live-db.txt touch html-stamp doc_man: man-stamp @@ -16,6 +17,8 @@ man-stamp: grml-live.txt grml-live-remaster.txt xsltproc /usr/share/xml/docbook/stylesheet/nwalsh/manpages/docbook.xsl grml-live.xml asciidoc -d manpage -b docbook grml-live-remaster.txt xsltproc /usr/share/xml/docbook/stylesheet/nwalsh/manpages/docbook.xsl grml-live-remaster.xml + asciidoc -d manpage -b docbook grml-live-db.txt + xsltproc /usr/share/xml/docbook/stylesheet/nwalsh/manpages/docbook.xsl grml-live-db.xml touch man-stamp online: doc_html @@ -24,3 +27,4 @@ online: doc_html clean: rm -rf grml-live-remaster.html grml-live-remaster.xml grml-live-remaster.8 rm -rf grml-live.html grml-live.xml grml-live.8 html-stamp man-stamp + rm -rf grml-live-db.html grml-live-db.xml grml-live-db.8 html-stamp man-stamp diff --git a/docs/grml-live-db.txt b/docs/grml-live-db.txt new file mode 100644 index 0000000..a52e74e --- /dev/null +++ b/docs/grml-live-db.txt @@ -0,0 +1,104 @@ +grml-live-db(8) +=============== + +Name +---- +grml-live-db - log package build information of grml-live to database + +Synopsis +-------- +dpkg-to-db [ options ] || db-to-fai /path/to/grml-live.db | + +Introduction +------------ + +The grml-live-db Debian package provides a simple way to put build information +of grml-live into a database. By default you have to do nothing but install +grml-live-db and during each invocation of grml-live you'll get an additional +entry in the sqlite3 database /var/log/grml-live.db. If you want to customize +the database logging check out the following sections in this manpage. + +Provided scripts +---------------- + +/usr/share/grml-live-db/scripts/dpkg-to-db adds grml-live build information +(output of 'dpkg --list') and (optionally) a logfile into a sqlite3 database. +This script is used by default if grml-live-db is installed (no configuration +needed by default). + +/usr/share/grml-live-db/scripts/db-to-fai converts output of grml-live's sqlite +database for use within FAI. This script is useful if you want to reproduce a +certain build with specific package versions. Please note that you need the +according Debian mirrors providing all the specific package versions of course. + +Options +------- + +dpkg-to-db supports the following options: + + --help + +Brief help message. + + --db + +Use specified database file. + + --logfile + +Logfile thath should be added to the database entry. + + --flavour + +Name of the grml-live flavour that was being built. + +db-to-fai does not support any options but needs to be invoked +with path to the grml-live database and the build id. + +Configuration and using custom database wrapper scripts +------------------------------------------------------- + +The following configuration variables are available and can be adjusted: + + DPKG_DATABASE=/var/log/grml-live.db + +Path to the database file that should be used for storing the build information. +This database is ysed within dpkg-to-db by default. + + DPKG_DBSCRIPT=/usr/share/grml-live-db/scripts/dpkg-to-db + +The database wrapper script that's used for storing the build information. +If you do not want to log to the sqlite3 database but instead use your own +abstraction layer just point this variable to your favourite script. + + DPKG_DBOPTIONS="-d $DPKG_DATABASE --logfile $LOGFILE --flavour $GRML_NAME < $DPKG_LIST" + +If the database script ($DPKG_DBSCRIPT) requires any command line options +specifiy it through this variable. + +Usage Examples +-------------- + +How dpkg-to-db is being used inside grml-live: + + /usr/share/grml-live-db/scripts/dpkg-to-db -d /var/log/grml-live.db --logfile /var/log/grml-live.log --flavour $GRML_NAME < /var/log/fai/$HOSTNAME/last/dpkg.list + +Manually insert data to database: + + # /usr/share/grml-live-db/scripts/dpkg-to-db -d ./grml-live.db --logfile /tmp/logfile --flavour grml-full < ./dpkg.list + + # dpkg -l | ./dpkg-to-db --db ./grml-live.db --logfile /tmp/logfile --flavour grml-full + +Retrieve build information of a specific build for use within FAI: + + # /usr/share/grml-live-db/scripts/db-to-fai /var/log/grml-live.db 6 > /etc/grml/fai/config/package_config/REPRODUCE + +Describe schema of database: + + # echo '.schema' | sqlite3 /var/log/grml-live.db + +Query database: + + # echo 'SELECT package,version,status,build.flavour,build.identifier FROM packages, build WHERE build.identifier = "grml-full-20091213012517" AND packages.build = build.id ; ' | sqlite3 /var/log/grml-live.db + + # echo 'SELECT package,version,status,build.flavour,build.identifier FROM packages, build WHERE build.id = 7 AND packages.build = build.id ; ' | sqlite3 /var/log/grml-live.db diff --git a/etc/grml/grml-live.conf b/etc/grml/grml-live.conf index 6a4671b..493bb2d 100644 --- a/etc/grml/grml-live.conf +++ b/etc/grml/grml-live.conf @@ -24,11 +24,20 @@ # Where do you want to find the final ISO? # ISO_OUTPUT="$OUTPUT/grml_isos" -# Do you want to zero / clean up the logfile on each grml-live execution? -# Especially useful if you are using an autobuild setup where you want -# store /var/log/grml-live.log after each invocation of grml-live. -# Default: unset (so do not zero the logfile) -# ZERO_LOGFILE='1' +# Do you want to preserve the logfile from being cleaned after each execution +# of grml-live? By default the logfile is cleaned so the log doesn't fill up. +# If you want to store your logs permanently it's recommended to use grml-live-db. +# PRESERVE_LOGFILE='1' + +# If package grml-live-db is installed the package selection and grml-live.log +# are being logged to to a sqlite database.Defaults to /var/log/grml-live.db +# DPKG_DATABASE=/var/log/grml-live.db + +# Use your own database wrapper script for grml-live-db: +# DPKG_DBSCRIPT=/usr/share/grml-live-db/scripts/dpkg-to-db + +# Use your own database script cmdline options for grml-live-db: +# DPKG_DBOPTIONS="-d $DPKG_DATABASE --logfile $LOGFILE --flavour $GRML_NAME < $DPKG_LIST" # Do you want to zero / clean up / remove the previous logfiles of FAI # before executing grml-live? Otherwise keep all the logfiles inside diff --git a/grml-live b/grml-live index 9057b04..7d8bb82 100755 --- a/grml-live +++ b/grml-live @@ -123,6 +123,8 @@ else eerror() { echo " [!] $*">&2 ;} ewarn() { echo " [x] $*" ;} eend() { return 0 ;} + eindent() { return 0 ;} + eoutdent() { return 0 ;} fi # source main configuration file: @@ -287,7 +289,9 @@ ISO_OUTPUT="$OUTPUT/grml_isos" # }}} # clean/zero grml-live logfile {{{ -if [ -n "$ZERO_LOGFILE" ] ; then +if [ -n "$PRESERVE_LOGFILE" ] ; then + echo "Preserving logfile $LOGFILE as requested via \$PRESERVE_LOGFILE" +else echo -n > $LOGFILE fi # }}} @@ -1118,10 +1122,56 @@ else fi # }}} +# log build information to database if grml-live-db is installed and enabled {{{ +dpkg_to_db() { +if [ -d /usr/share/grml-live-db ] ; then + + # safe defaults + DPKG_LIST="/var/log/fai/$HOSTNAME/last/dpkg.list" # the dpkg --list output of the chroot: + [ -n "$DPKG_DATABASE" ] || DPKG_DATABASE=/var/log/grml-live.db + [ -n "$DPKG_DBSCRIPT" ] || DPKG_DBSCRIPT=/usr/share/grml-live-db/scripts/dpkg-to-db + [ -n "$DPKG_DBOPTIONS" ] || DPKG_DBOPTIONS="-d $DPKG_DATABASE --logfile $LOGFILE --flavour $GRML_NAME < $DPKG_LIST" + + if ! [ -x "$DPKG_DBSCRIPT" ] ; then + eerror "Error: $DPKG_DBSCRIPT is not executable, can not log dpkg information." ; eend 1 + bailout 14 + fi + + # disable by default for now, not sure whether really everyone is using a local db file + #if ! touch "$DPKG_DATABASE" ; then + # eerror "Error: can not write to ${DPKG_DATABASE}, can not log dpkg information." ; eend 1 + # bailout 14 + #fi + + if ! [ -r "$DPKG_LIST" ] ; then + eerror "Error reading $DPKG_LIST - can not provide information to $DPKG_DBSCRIPT" ; eend 1 + bailout 14 + else + einfo "Logging $DPKG_LIST to database $DPKG_DATABASE" + eindent + + if DB_INFO=$("$DPKG_DBSCRIPT" $DPKG_DBOPTIONS 2>&1) ; then + einfo "$DB_INFO" + eend 0 + else + eerror "$DB_INFO" + eend 1 + fi + + eoutdent + fi + +fi +} +# }}} + # finalize {{{ [ -n "$start_seconds" ] && SECONDS="$[$(cut -d . -f 1 /proc/uptime)-$start_seconds]" || SECONDS="unknown" -einfo "Successfully finished execution of $PN [$(date) - running ${SECONDS} seconds]" ; eend 0 log "Successfully finished execution of $PN [$(date) - running ${SECONDS} seconds]" + +dpkg_to_db # make sure we catch the last log line as well, therefore execute between log + einfo + +einfo "Successfully finished execution of $PN [$(date) - running ${SECONDS} seconds]" ; eend 0 bailout 0 # }}} -- 2.1.4