daemon: Add gzip log compression.
* nix/nix-daemon/guix-daemon.cc (GUIX_OPT_LOG_COMPRESSION): New macro. (options): Mark "disable-log-compression" as hidden and add "log-compression". (parse_opt): Handle GUIX_OPT_LOG_COMPRESSION. * nix/libstore/build.cc (DerivationGoal): Add 'gzLogFile'. (openLogFile): Initialize it when 'logCompression' is COMPRESSION_GZIP. (closeLogFile, handleChildOutput): Honor 'gzLogFile'. * nix/libstore/globals.hh (Settings)[compressLog]: Remove. [logCompression]: New field. (CompressionType): New enum. * nix/libstore/globals.cc (Settings::Settings): Initialize it. (update): Remove '_get' call for 'compressLog'. * nix/local.mk (guix_daemon_LDADD, guix_register_LDADD): Add -lz. * guix/store.scm (log-file): Handle '.gz' log files. * tests/guix-daemon.sh: Add test with '--log-compression=gzip'. * doc/guix.texi (Invoking guix-daemon): Adjust accordingly. * config-daemon.ac: Check for libz and zlib.h.master
Ludovic Courtès
parent
896fec476f
commit
29a6866886
|
|
@ -18,6 +18,12 @@ if test "x$guix_build_daemon" = "xyes"; then
|
|||
dnl Use 64-bit file system calls so that we can support files > 2 GiB.
|
||||
AC_SYS_LARGEFILE
|
||||
|
||||
dnl Look for zlib, a required dependency.
|
||||
AC_CHECK_LIB([z], [gzdopen], [true],
|
||||
[AC_MSG_ERROR([Guix requires zlib. See http://www.zlib.net/.])])
|
||||
AC_CHECK_HEADERS([zlib.h], [true],
|
||||
[AC_MSG_ERROR([Guix requires zlib. See http://www.zlib.net/.])])
|
||||
|
||||
dnl Look for libbz2, a required dependency.
|
||||
AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
|
||||
[AC_MSG_ERROR([Guix requires libbz2, which is part of bzip2. See http://www.bzip.org/.])])
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
@set OPENPGP-SIGNING-KEY-ID 3CE464558A84FDC69DB40CFB090B11993D9AEBB5
|
||||
|
||||
@copying
|
||||
Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès@*
|
||||
Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès@*
|
||||
Copyright @copyright{} 2013, 2014, 2016 Andreas Enge@*
|
||||
Copyright @copyright{} 2013 Nikita Karetnikov@*
|
||||
Copyright @copyright{} 2014, 2015, 2016 Alex Kost@*
|
||||
|
|
@ -1235,12 +1235,13 @@ processes to gain access to undeclared dependencies. It is necessary,
|
|||
though, when @command{guix-daemon} is running under an unprivileged user
|
||||
account.
|
||||
|
||||
@item --disable-log-compression
|
||||
Disable compression of the build logs.
|
||||
@item --log-compression=@var{type}
|
||||
Compress build logs according to @var{type}, one of @code{gzip},
|
||||
@code{bzip2}, or @code{none}.
|
||||
|
||||
Unless @code{--lose-logs} is used, all the build logs are kept in the
|
||||
@var{localstatedir}. To save space, the daemon automatically compresses
|
||||
them with bzip2 by default. This option disables that.
|
||||
them with bzip2 by default.
|
||||
|
||||
@item --disable-deduplication
|
||||
@cindex deduplication
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
;;; GNU Guix --- Functional package management for GNU
|
||||
;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
|
||||
;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
|
||||
;;;
|
||||
;;; This file is part of GNU Guix.
|
||||
;;;
|
||||
|
|
@ -1567,8 +1567,10 @@ must be an absolute store file name, or a derivation file name."
|
|||
"/log/guix/drvs/"
|
||||
(string-take base 2) "/"
|
||||
(string-drop base 2)))
|
||||
(log.gz (string-append log ".gz"))
|
||||
(log.bz2 (string-append log ".bz2")))
|
||||
(cond ((file-exists? log.bz2) log.bz2)
|
||||
(cond ((file-exists? log.gz) log.gz)
|
||||
((file-exists? log.bz2) log.bz2)
|
||||
((file-exists? log) log)
|
||||
(else #f))))
|
||||
(else
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@
|
|||
#include <pwd.h>
|
||||
#include <grp.h>
|
||||
|
||||
#include <zlib.h>
|
||||
#include <bzlib.h>
|
||||
|
||||
/* Includes required for chroot support. */
|
||||
|
|
@ -744,6 +745,7 @@ private:
|
|||
|
||||
/* File descriptor for the log file. */
|
||||
FILE * fLogFile;
|
||||
gzFile gzLogFile;
|
||||
BZFILE * bzLogFile;
|
||||
AutoCloseFD fdLogFile;
|
||||
|
||||
|
|
@ -892,6 +894,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
|
|||
, needRestart(false)
|
||||
, retrySubstitution(false)
|
||||
, fLogFile(0)
|
||||
, gzLogFile(0)
|
||||
, bzLogFile(0)
|
||||
, useChroot(false)
|
||||
, buildMode(buildMode)
|
||||
|
|
@ -2599,8 +2602,25 @@ Path DerivationGoal::openLogFile()
|
|||
Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str();
|
||||
createDirs(dir);
|
||||
|
||||
if (settings.compressLog) {
|
||||
switch (settings.logCompression)
|
||||
{
|
||||
case COMPRESSION_GZIP: {
|
||||
Path logFileName = (format("%1%/%2%.gz") % dir % string(baseName, 2)).str();
|
||||
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
|
||||
if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
|
||||
closeOnExec(fd);
|
||||
|
||||
/* Note: FD will be closed by 'gzclose'. */
|
||||
if (!(gzLogFile = gzdopen(fd.borrow(), "w")))
|
||||
throw Error(format("cannot open compressed log file `%1%'") % logFileName);
|
||||
|
||||
gzbuffer(gzLogFile, 32768);
|
||||
gzsetparams(gzLogFile, Z_BEST_COMPRESSION, Z_DEFAULT_STRATEGY);
|
||||
|
||||
return logFileName;
|
||||
}
|
||||
|
||||
case COMPRESSION_BZIP2: {
|
||||
Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str();
|
||||
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
|
||||
if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
|
||||
|
|
@ -2614,20 +2634,30 @@ Path DerivationGoal::openLogFile()
|
|||
throw Error(format("cannot open compressed log file `%1%'") % logFileName);
|
||||
|
||||
return logFileName;
|
||||
}
|
||||
|
||||
} else {
|
||||
case COMPRESSION_NONE: {
|
||||
Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str();
|
||||
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
|
||||
if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
|
||||
closeOnExec(fdLogFile);
|
||||
return logFileName;
|
||||
}
|
||||
}
|
||||
|
||||
abort();
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::closeLogFile()
|
||||
{
|
||||
if (bzLogFile) {
|
||||
if (gzLogFile) {
|
||||
int err;
|
||||
err = gzclose(gzLogFile);
|
||||
gzLogFile = NULL;
|
||||
if (err != Z_OK) throw Error(format("cannot close compressed log file (gzip error = %1%)") % err);
|
||||
}
|
||||
else if (bzLogFile) {
|
||||
int err;
|
||||
BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0);
|
||||
bzLogFile = 0;
|
||||
|
|
@ -2695,7 +2725,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
|
|||
}
|
||||
if (verbosity >= settings.buildVerbosity)
|
||||
writeToStderr(data);
|
||||
if (bzLogFile) {
|
||||
|
||||
if (gzLogFile) {
|
||||
if (data.size() > 0) {
|
||||
int count, err;
|
||||
count = gzwrite(gzLogFile, data.data(), data.size());
|
||||
if (count == 0) throw Error(format("cannot write to compressed log file (gzip error = %1%)") % gzerror(gzLogFile, &err));
|
||||
}
|
||||
} else if (bzLogFile) {
|
||||
int err;
|
||||
BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
|
||||
if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err);
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ Settings::Settings()
|
|||
useSshSubstituter = false;
|
||||
impersonateLinux26 = false;
|
||||
keepLog = true;
|
||||
compressLog = true;
|
||||
logCompression = COMPRESSION_BZIP2;
|
||||
maxLogSize = 0;
|
||||
cacheFailure = false;
|
||||
pollInterval = 5;
|
||||
|
|
@ -162,7 +162,7 @@ void Settings::update()
|
|||
_get(useChroot, "build-use-chroot");
|
||||
_get(impersonateLinux26, "build-impersonate-linux-26");
|
||||
_get(keepLog, "build-keep-log");
|
||||
_get(compressLog, "build-compress-log");
|
||||
// _get(logCompression, "build-log-compression");
|
||||
_get(maxLogSize, "build-max-log-size");
|
||||
_get(cacheFailure, "build-cache-failure");
|
||||
_get(pollInterval, "build-poll-interval");
|
||||
|
|
|
|||
|
|
@ -8,6 +8,12 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
enum CompressionType
|
||||
{
|
||||
COMPRESSION_NONE = 0,
|
||||
COMPRESSION_GZIP = 1,
|
||||
COMPRESSION_BZIP2 = 2
|
||||
};
|
||||
|
||||
struct Settings {
|
||||
|
||||
|
|
@ -169,7 +175,7 @@ struct Settings {
|
|||
bool keepLog;
|
||||
|
||||
/* Whether to compress logs. */
|
||||
bool compressLog;
|
||||
enum CompressionType logCompression;
|
||||
|
||||
/* Maximum number of bytes a builder can write to stdout/stderr
|
||||
before being killed (0 means no limit). */
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# GNU Guix --- Functional package management for GNU
|
||||
# Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org>
|
||||
# Copyright © 2012, 2013, 2014, 2015, 2016, 2018 Ludovic Courtès <ludo@gnu.org>
|
||||
# Copyright © 2016 Mathieu Lirzin <mthl@gnu.org>
|
||||
#
|
||||
# This file is part of GNU Guix.
|
||||
|
|
@ -132,7 +132,7 @@ guix_daemon_CPPFLAGS = \
|
|||
-I$(top_srcdir)/%D%/libstore
|
||||
|
||||
guix_daemon_LDADD = \
|
||||
libstore.a libutil.a libformat.a -lbz2 \
|
||||
libstore.a libutil.a libformat.a -lz -lbz2 \
|
||||
$(SQLITE3_LIBS) $(LIBGCRYPT_LIBS)
|
||||
|
||||
guix_daemon_headers = \
|
||||
|
|
@ -149,7 +149,7 @@ guix_register_CPPFLAGS = \
|
|||
|
||||
# XXX: Should we start using shared libs?
|
||||
guix_register_LDADD = \
|
||||
libstore.a libutil.a libformat.a -lbz2 \
|
||||
libstore.a libutil.a libformat.a -lz -lbz2 \
|
||||
$(SQLITE3_LIBS) $(LIBGCRYPT_LIBS)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* GNU Guix --- Functional package management for GNU
|
||||
Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
|
||||
Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
|
||||
Copyright (C) 2006, 2010, 2012, 2014 Eelco Dolstra <e.dolstra@tudelft.nl>
|
||||
|
||||
This file is part of GNU Guix.
|
||||
|
|
@ -88,6 +88,7 @@ builds derivations on behalf of its clients.");
|
|||
#define GUIX_OPT_BUILD_ROUNDS 17
|
||||
#define GUIX_OPT_TIMEOUT 18
|
||||
#define GUIX_OPT_MAX_SILENT_TIME 19
|
||||
#define GUIX_OPT_LOG_COMPRESSION 20
|
||||
|
||||
static const struct argp_option options[] =
|
||||
{
|
||||
|
|
@ -120,8 +121,11 @@ static const struct argp_option options[] =
|
|||
n_("build each derivation N times in a row") },
|
||||
{ "lose-logs", GUIX_OPT_LOSE_LOGS, 0, 0,
|
||||
n_("do not keep build logs") },
|
||||
{ "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0, 0,
|
||||
{ "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0,
|
||||
OPTION_HIDDEN, // deprecated
|
||||
n_("disable compression of the build logs") },
|
||||
{ "log-compression", GUIX_OPT_LOG_COMPRESSION, "TYPE", 0,
|
||||
n_("use the specified compression type for build logs") },
|
||||
|
||||
/* '--disable-deduplication' was known as '--disable-store-optimization'
|
||||
up to Guix 0.7 included, so keep the alias around. */
|
||||
|
|
@ -197,8 +201,21 @@ parse_opt (int key, char *arg, struct argp_state *state)
|
|||
settings.set("build-extra-chroot-dirs", chroot_dirs);
|
||||
break;
|
||||
}
|
||||
case GUIX_OPT_LOG_COMPRESSION:
|
||||
if (strcmp (arg, "none") == 0)
|
||||
settings.logCompression = COMPRESSION_NONE;
|
||||
else if (strcmp (arg, "gzip") == 0)
|
||||
settings.logCompression = COMPRESSION_GZIP;
|
||||
else if (strcmp (arg, "bzip2") == 0)
|
||||
settings.logCompression = COMPRESSION_BZIP2;
|
||||
else
|
||||
{
|
||||
fprintf (stderr, _("error: %s: unknown compression type\n"), arg);
|
||||
exit (EXIT_FAILURE);
|
||||
}
|
||||
break;
|
||||
case GUIX_OPT_DISABLE_LOG_COMPRESSION:
|
||||
settings.compressLog = false;
|
||||
settings.logCompression = COMPRESSION_NONE;
|
||||
break;
|
||||
case GUIX_OPT_BUILD_USERS_GROUP:
|
||||
settings.buildUsersGroup = arg;
|
||||
|
|
@ -487,6 +504,8 @@ main (int argc, char *argv[])
|
|||
|
||||
/* Effect all the changes made via 'settings.set'. */
|
||||
settings.update ();
|
||||
printMsg(lvlDebug,
|
||||
format ("build log compression: %1%") % settings.logCompression);
|
||||
|
||||
if (settings.useSubstitutes)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# GNU Guix --- Functional package management for GNU
|
||||
# Copyright © 2012, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
|
||||
# Copyright © 2012, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
|
||||
#
|
||||
# This file is part of GNU Guix.
|
||||
#
|
||||
|
|
@ -193,3 +193,39 @@ do
|
|||
GUIX_DAEMON_SOCKET="$socket" guile -c "$client_code"
|
||||
kill "$daemon_pid"
|
||||
done
|
||||
|
||||
# Log compression.
|
||||
|
||||
guix-daemon --listen="$socket" --disable-chroot --debug --log-compression=gzip &
|
||||
daemon_pid=$!
|
||||
|
||||
stamp="compressed-build-log-test-$$-`date +%H%M%S`"
|
||||
client_code="
|
||||
(use-modules (guix) (gnu packages bootstrap))
|
||||
|
||||
(with-store store
|
||||
(run-with-store store
|
||||
(mlet %store-monad ((drv (lower-object
|
||||
(computed-file \"compressed-log-test\"
|
||||
#~(begin
|
||||
(display \"$stamp\")
|
||||
(newline)
|
||||
(mkdir #\$output))
|
||||
#:guile %bootstrap-guile))))
|
||||
(display (derivation-file-name drv))
|
||||
(newline)
|
||||
(return #t))))
|
||||
"
|
||||
|
||||
GUIX_DAEMON_SOCKET="$socket"
|
||||
export GUIX_DAEMON_SOCKET
|
||||
|
||||
drv=`guile -c "$client_code"`
|
||||
guix build "$drv"
|
||||
|
||||
log=`guix build "$drv" --log-file`
|
||||
test -f "$log"
|
||||
case "$log" in
|
||||
*.gz) test "`gunzip -c < "$log"`" = "$stamp" ;;
|
||||
*) false ;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in New Issue