From 29a686688674dc875775305312513405fa396a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Fri, 5 Jan 2018 17:15:41 +0100 Subject: daemon: Add gzip log compression. * nix/nix-daemon/guix-daemon.cc (GUIX_OPT_LOG_COMPRESSION): New macro. (options): Mark "disable-log-compression" as hidden and add "log-compression". (parse_opt): Handle GUIX_OPT_LOG_COMPRESSION. * nix/libstore/build.cc (DerivationGoal): Add 'gzLogFile'. (openLogFile): Initialize it when 'logCompression' is COMPRESSION_GZIP. (closeLogFile, handleChildOutput): Honor 'gzLogFile'. * nix/libstore/globals.hh (Settings)[compressLog]: Remove. [logCompression]: New field. (CompressionType): New enum. * nix/libstore/globals.cc (Settings::Settings): Initialize it. (update): Remove '_get' call for 'compressLog'. * nix/local.mk (guix_daemon_LDADD, guix_register_LDADD): Add -lz. * guix/store.scm (log-file): Handle '.gz' log files. * tests/guix-daemon.sh: Add test with '--log-compression=gzip'. * doc/guix.texi (Invoking guix-daemon): Adjust accordingly. * config-daemon.ac: Check for libz and zlib.h. --- config-daemon.ac | 6 ++++++ doc/guix.texi | 9 +++++---- guix/store.scm | 6 ++++-- nix/libstore/build.cc | 45 +++++++++++++++++++++++++++++++++++++++---- nix/libstore/globals.cc | 4 ++-- nix/libstore/globals.hh | 8 +++++++- nix/local.mk | 6 +++--- nix/nix-daemon/guix-daemon.cc | 25 +++++++++++++++++++++--- tests/guix-daemon.sh | 38 +++++++++++++++++++++++++++++++++++- 9 files changed, 127 insertions(+), 20 deletions(-) diff --git a/config-daemon.ac b/config-daemon.ac index 42b59819d3..59f6f2713f 100644 --- a/config-daemon.ac +++ b/config-daemon.ac @@ -18,6 +18,12 @@ if test "x$guix_build_daemon" = "xyes"; then dnl Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE + dnl Look for zlib, a required dependency. + AC_CHECK_LIB([z], [gzdopen], [true], + [AC_MSG_ERROR([Guix requires zlib. See http://www.zlib.net/.])]) + AC_CHECK_HEADERS([zlib.h], [true], + [AC_MSG_ERROR([Guix requires zlib. See http://www.zlib.net/.])]) + dnl Look for libbz2, a required dependency. AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], [AC_MSG_ERROR([Guix requires libbz2, which is part of bzip2. See http://www.bzip.org/.])]) diff --git a/doc/guix.texi b/doc/guix.texi index f64f1e0476..ca3f614042 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -13,7 +13,7 @@ @set OPENPGP-SIGNING-KEY-ID 3CE464558A84FDC69DB40CFB090B11993D9AEBB5 @copying -Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès@* +Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès@* Copyright @copyright{} 2013, 2014, 2016 Andreas Enge@* Copyright @copyright{} 2013 Nikita Karetnikov@* Copyright @copyright{} 2014, 2015, 2016 Alex Kost@* @@ -1235,12 +1235,13 @@ processes to gain access to undeclared dependencies. It is necessary, though, when @command{guix-daemon} is running under an unprivileged user account. -@item --disable-log-compression -Disable compression of the build logs. +@item --log-compression=@var{type} +Compress build logs according to @var{type}, one of @code{gzip}, +@code{bzip2}, or @code{none}. Unless @code{--lose-logs} is used, all the build logs are kept in the @var{localstatedir}. To save space, the daemon automatically compresses -them with bzip2 by default. This option disables that. +them with bzip2 by default. @item --disable-deduplication @cindex deduplication diff --git a/guix/store.scm b/guix/store.scm index e6e45ba89c..89db46b8e6 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -1567,8 +1567,10 @@ must be an absolute store file name, or a derivation file name." "/log/guix/drvs/" (string-take base 2) "/" (string-drop base 2))) + (log.gz (string-append log ".gz")) (log.bz2 (string-append log ".bz2"))) - (cond ((file-exists? log.bz2) log.bz2) + (cond ((file-exists? log.gz) log.gz) + ((file-exists? log.bz2) log.bz2) ((file-exists? log) log) (else #f)))) (else diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index d68e8b2bc0..5bf3e3aacb 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -31,6 +31,7 @@ #include #include +#include #include /* Includes required for chroot support. */ @@ -744,6 +745,7 @@ private: /* File descriptor for the log file. */ FILE * fLogFile; + gzFile gzLogFile; BZFILE * bzLogFile; AutoCloseFD fdLogFile; @@ -892,6 +894,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut , needRestart(false) , retrySubstitution(false) , fLogFile(0) + , gzLogFile(0) , bzLogFile(0) , useChroot(false) , buildMode(buildMode) @@ -2599,8 +2602,25 @@ Path DerivationGoal::openLogFile() Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); createDirs(dir); - if (settings.compressLog) { + switch (settings.logCompression) + { + case COMPRESSION_GZIP: { + Path logFileName = (format("%1%/%2%.gz") % dir % string(baseName, 2)).str(); + AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName); + closeOnExec(fd); + + /* Note: FD will be closed by 'gzclose'. */ + if (!(gzLogFile = gzdopen(fd.borrow(), "w"))) + throw Error(format("cannot open compressed log file `%1%'") % logFileName); + + gzbuffer(gzLogFile, 32768); + gzsetparams(gzLogFile, Z_BEST_COMPRESSION, Z_DEFAULT_STRATEGY); + return logFileName; + } + + case COMPRESSION_BZIP2: { Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str(); AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName); @@ -2614,20 +2634,30 @@ Path DerivationGoal::openLogFile() throw Error(format("cannot open compressed log file `%1%'") % logFileName); return logFileName; + } - } else { + case COMPRESSION_NONE: { Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str(); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); closeOnExec(fdLogFile); return logFileName; + } } + + abort(); } void DerivationGoal::closeLogFile() { - if (bzLogFile) { + if (gzLogFile) { + int err; + err = gzclose(gzLogFile); + gzLogFile = NULL; + if (err != Z_OK) throw Error(format("cannot close compressed log file (gzip error = %1%)") % err); + } + else if (bzLogFile) { int err; BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0); bzLogFile = 0; @@ -2695,7 +2725,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (verbosity >= settings.buildVerbosity) writeToStderr(data); - if (bzLogFile) { + + if (gzLogFile) { + if (data.size() > 0) { + int count, err; + count = gzwrite(gzLogFile, data.data(), data.size()); + if (count == 0) throw Error(format("cannot write to compressed log file (gzip error = %1%)") % gzerror(gzLogFile, &err)); + } + } else if (bzLogFile) { int err; BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc index 65dad24d91..82d528dc98 100644 --- a/nix/libstore/globals.cc +++ b/nix/libstore/globals.cc @@ -45,7 +45,7 @@ Settings::Settings() useSshSubstituter = false; impersonateLinux26 = false; keepLog = true; - compressLog = true; + logCompression = COMPRESSION_BZIP2; maxLogSize = 0; cacheFailure = false; pollInterval = 5; @@ -162,7 +162,7 @@ void Settings::update() _get(useChroot, "build-use-chroot"); _get(impersonateLinux26, "build-impersonate-linux-26"); _get(keepLog, "build-keep-log"); - _get(compressLog, "build-compress-log"); + // _get(logCompression, "build-log-compression"); _get(maxLogSize, "build-max-log-size"); _get(cacheFailure, "build-cache-failure"); _get(pollInterval, "build-poll-interval"); diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh index 7beb1a55ca..81cf2f52d4 100644 --- a/nix/libstore/globals.hh +++ b/nix/libstore/globals.hh @@ -8,6 +8,12 @@ namespace nix { +enum CompressionType +{ + COMPRESSION_NONE = 0, + COMPRESSION_GZIP = 1, + COMPRESSION_BZIP2 = 2 +}; struct Settings { @@ -169,7 +175,7 @@ struct Settings { bool keepLog; /* Whether to compress logs. */ - bool compressLog; + enum CompressionType logCompression; /* Maximum number of bytes a builder can write to stdout/stderr before being killed (0 means no limit). */ diff --git a/nix/local.mk b/nix/local.mk index 9e0c457bec..d802da6170 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -1,5 +1,5 @@ # GNU Guix --- Functional package management for GNU -# Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès +# Copyright © 2012, 2013, 2014, 2015, 2016, 2018 Ludovic Courtès # Copyright © 2016 Mathieu Lirzin # # This file is part of GNU Guix. @@ -132,7 +132,7 @@ guix_daemon_CPPFLAGS = \ -I$(top_srcdir)/%D%/libstore guix_daemon_LDADD = \ - libstore.a libutil.a libformat.a -lbz2 \ + libstore.a libutil.a libformat.a -lz -lbz2 \ $(SQLITE3_LIBS) $(LIBGCRYPT_LIBS) guix_daemon_headers = \ @@ -149,7 +149,7 @@ guix_register_CPPFLAGS = \ # XXX: Should we start using shared libs? guix_register_LDADD = \ - libstore.a libutil.a libformat.a -lbz2 \ + libstore.a libutil.a libformat.a -lz -lbz2 \ $(SQLITE3_LIBS) $(LIBGCRYPT_LIBS) diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc index 7963358202..a1ef90dfdc 100644 --- a/nix/nix-daemon/guix-daemon.cc +++ b/nix/nix-daemon/guix-daemon.cc @@ -1,5 +1,5 @@ /* GNU Guix --- Functional package management for GNU - Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès + Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès Copyright (C) 2006, 2010, 2012, 2014 Eelco Dolstra This file is part of GNU Guix. @@ -88,6 +88,7 @@ builds derivations on behalf of its clients."); #define GUIX_OPT_BUILD_ROUNDS 17 #define GUIX_OPT_TIMEOUT 18 #define GUIX_OPT_MAX_SILENT_TIME 19 +#define GUIX_OPT_LOG_COMPRESSION 20 static const struct argp_option options[] = { @@ -120,8 +121,11 @@ static const struct argp_option options[] = n_("build each derivation N times in a row") }, { "lose-logs", GUIX_OPT_LOSE_LOGS, 0, 0, n_("do not keep build logs") }, - { "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0, 0, + { "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0, + OPTION_HIDDEN, // deprecated n_("disable compression of the build logs") }, + { "log-compression", GUIX_OPT_LOG_COMPRESSION, "TYPE", 0, + n_("use the specified compression type for build logs") }, /* '--disable-deduplication' was known as '--disable-store-optimization' up to Guix 0.7 included, so keep the alias around. */ @@ -197,8 +201,21 @@ parse_opt (int key, char *arg, struct argp_state *state) settings.set("build-extra-chroot-dirs", chroot_dirs); break; } + case GUIX_OPT_LOG_COMPRESSION: + if (strcmp (arg, "none") == 0) + settings.logCompression = COMPRESSION_NONE; + else if (strcmp (arg, "gzip") == 0) + settings.logCompression = COMPRESSION_GZIP; + else if (strcmp (arg, "bzip2") == 0) + settings.logCompression = COMPRESSION_BZIP2; + else + { + fprintf (stderr, _("error: %s: unknown compression type\n"), arg); + exit (EXIT_FAILURE); + } + break; case GUIX_OPT_DISABLE_LOG_COMPRESSION: - settings.compressLog = false; + settings.logCompression = COMPRESSION_NONE; break; case GUIX_OPT_BUILD_USERS_GROUP: settings.buildUsersGroup = arg; @@ -487,6 +504,8 @@ main (int argc, char *argv[]) /* Effect all the changes made via 'settings.set'. */ settings.update (); + printMsg(lvlDebug, + format ("build log compression: %1%") % settings.logCompression); if (settings.useSubstitutes) { diff --git a/tests/guix-daemon.sh b/tests/guix-daemon.sh index 7212e3eb68..6f91eb58bf 100644 --- a/tests/guix-daemon.sh +++ b/tests/guix-daemon.sh @@ -1,5 +1,5 @@ # GNU Guix --- Functional package management for GNU -# Copyright © 2012, 2014, 2015, 2016, 2017 Ludovic Courtès +# Copyright © 2012, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès # # This file is part of GNU Guix. # @@ -193,3 +193,39 @@ do GUIX_DAEMON_SOCKET="$socket" guile -c "$client_code" kill "$daemon_pid" done + +# Log compression. + +guix-daemon --listen="$socket" --disable-chroot --debug --log-compression=gzip & +daemon_pid=$! + +stamp="compressed-build-log-test-$$-`date +%H%M%S`" +client_code=" + (use-modules (guix) (gnu packages bootstrap)) + + (with-store store + (run-with-store store + (mlet %store-monad ((drv (lower-object + (computed-file \"compressed-log-test\" + #~(begin + (display \"$stamp\") + (newline) + (mkdir #\$output)) + #:guile %bootstrap-guile)))) + (display (derivation-file-name drv)) + (newline) + (return #t)))) +" + +GUIX_DAEMON_SOCKET="$socket" +export GUIX_DAEMON_SOCKET + +drv=`guile -c "$client_code"` +guix build "$drv" + +log=`guix build "$drv" --log-file` +test -f "$log" +case "$log" in + *.gz) test "`gunzip -c < "$log"`" = "$stamp" ;; + *) false ;; +esac -- cgit v1.2.3