From 94d92c7796a3dd50c27d532315f7d497ac99f08e Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 20 Jul 2015 04:30:16 +0200 Subject: daemon: Add "builtin:download" derivation builder. This ensures that 1) the derivation doesn't change when Guix changes; 2) the derivation closure doesn't contain Guix and its dependencies; 3) we don't have to rely on ugly chroot hacks. Adapted from Nix commit 0a2bee307b20411f5b0dda0c662b1f9bb9e0e131. * nix/libstore/build.cc (DerivationGoal::runChild): Add special case for 'isBuiltin(drv)'. Disable chroot when 'isBuiltin(drv)'. * nix/libstore/builtins.cc, nix/libstore/builtins.hh, nix/scripts/download.in, guix/scripts/perform-download.scm: New files. * guix/ui.scm (show-guix-help)[internal?]: Add 'perform-download'. * nix/local.mk (libstore_a_SOURCES): Add builtins.cc. (libstore_headers): Add builtins.hh. (nodist_pkglibexec_SCRIPTS): Add 'scripts/download'. * config-daemon.ac: Emit 'scripts/download'. * Makefile.am (MODULES): Add 'guix/scripts/perform-download.scm'. * tests/derivations.scm ("unknown built-in builder") ("'download' built-in builder") ("'download' built-in builder, invalid hash") ("'download' built-in builder, not found") ("'download' built-in builder, not fixed-output"): New tests. Co-authored-by: Eelco Dolstra --- nix/libstore/build.cc | 36 ++++++++++++++++++++----- nix/libstore/builtins.cc | 69 ++++++++++++++++++++++++++++++++++++++++++++++++ nix/libstore/builtins.hh | 41 ++++++++++++++++++++++++++++ nix/local.mk | 5 +++- nix/scripts/download.in | 11 ++++++++ 5 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 nix/libstore/builtins.cc create mode 100644 nix/libstore/builtins.hh create mode 100644 nix/scripts/download.in (limited to 'nix') diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index ae78e65199..889ee3d2bd 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -8,6 +8,7 @@ #include "util.hh" #include "archive.hh" #include "affinity.hh" +#include "builtins.hh" #include #include @@ -2047,7 +2048,12 @@ void DerivationGoal::runChild() commonChildInit(builderOut); #if CHROOT_ENABLED - if (useChroot) { + /* Note: built-in builders are *not* running in a chroot environment + so that we can easily implement them in Guile without having it as + a derivation input (they are running under a separate build user, + though). */ + + if (useChroot && !isBuiltin(drv)) { /* Initialise the loopback interface. */ AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); if (fd == -1) throw SysError("cannot open IP socket"); @@ -2255,6 +2261,28 @@ void DerivationGoal::runChild() throw SysError("setuid failed"); } + restoreSIGPIPE(); + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, "\n"); + + /* Execute the program. This should not return. */ + if (isBuiltin(drv)) { + try { + logType = ltFlat; + + auto buildDrv = lookupBuiltinBuilder(drv.builder); + if (buildDrv != NULL) + buildDrv(drv, drvPath); + else + throw Error(format("unsupported builtin function '%1%'") % string(drv.builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + _exit(1); + } + } + /* Fill in the arguments. */ Strings args; string builderBasename = baseNameOf(drv.builder); @@ -2262,12 +2290,6 @@ void DerivationGoal::runChild() foreach (Strings::iterator, i, drv.args) args.push_back(rewriteHashes(*i, rewritesToTmp)); - restoreSIGPIPE(); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, "\n"); - - /* Execute the program. This should not return. */ execve(drv.builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); throw SysError(format("executing `%1%'") % drv.builder); diff --git a/nix/libstore/builtins.cc b/nix/libstore/builtins.cc new file mode 100644 index 0000000000..605e44079a --- /dev/null +++ b/nix/libstore/builtins.cc @@ -0,0 +1,69 @@ +/* GNU Guix --- Functional package management for GNU + Copyright (C) 2016 Ludovic Courtès + + This file is part of GNU Guix. + + GNU Guix is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or (at + your option) any later version. + + GNU Guix is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNU Guix. If not, see . */ + +#include +#include +#include + +#include + +namespace nix { + +static void builtinDownload(const Derivation &drv, + const std::string &drvPath) +{ + /* Invoke 'guix perform-download'. */ + Strings args; + args.push_back("perform-download"); + args.push_back(drvPath); + + /* Close all other file descriptors. */ + closeMostFDs(set()); + + const char *const argv[] = { "download", drvPath.c_str(), NULL }; + + /* XXX: Hack our way to use the 'download' script from 'LIBEXECDIR/guix' + or just 'LIBEXECDIR', depending on whether we're running uninstalled or + not. */ + const string subdir = getenv("GUIX_UNINSTALLED") != NULL + ? "" : "/guix"; + + const string program = settings.nixLibexecDir + subdir + "/download"; + execv(program.c_str(), (char *const *) argv); + + throw SysError(format("failed to run download program '%1%'") % program); +} + +static const std::map builtins = +{ + { "download", builtinDownload } +}; + +derivationBuilder lookupBuiltinBuilder(const std::string & name) +{ + if (name.substr(0, 8) == "builtin:") + { + auto realName = name.substr(8); + auto builder = builtins.find(realName); + return builder == builtins.end() ? NULL : builder->second; + } + else + return NULL; +} + +} diff --git a/nix/libstore/builtins.hh b/nix/libstore/builtins.hh new file mode 100644 index 0000000000..0c6db651ab --- /dev/null +++ b/nix/libstore/builtins.hh @@ -0,0 +1,41 @@ +/* GNU Guix --- Functional package management for GNU + Copyright (C) 2016 Ludovic Courtès + + This file is part of GNU Guix. + + GNU Guix is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or (at + your option) any later version. + + GNU Guix is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNU Guix. If not, see . */ + +/* Interface to built-in derivation builders. */ + +#pragma once + +#include +#include +#include + +namespace nix { + + inline bool isBuiltin(const Derivation & drv) + { + return string(drv.builder, 0, 8) == "builtin:"; + } + + /* Build DRV, which lives at DRVPATH. */ + typedef void (*derivationBuilder) (const Derivation &drv, + const std::string &drvPath); + + /* Return the built-in builder called BUILDER, or NULL if none was + found. */ + derivationBuilder lookupBuiltinBuilder(const std::string &builder); +} diff --git a/nix/local.mk b/nix/local.mk index c666edd033..86ef769549 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -87,6 +87,7 @@ libstore_a_SOURCES = \ %D%/libstore/build.cc \ %D%/libstore/pathlocks.cc \ %D%/libstore/derivations.cc \ + %D%/libstore/builtins.cc \ %D%/libstore/sqlite.cc libstore_headers = \ @@ -98,6 +99,7 @@ libstore_headers = \ %D%/libstore/misc.hh \ %D%/libstore/local-store.hh \ %D%/libstore/sqlite.hh \ + %D%/libstore/builtins.hh \ %D%/libstore/store-api.hh libstore_a_CPPFLAGS = \ @@ -166,7 +168,8 @@ noinst_HEADERS = \ nodist_pkglibexec_SCRIPTS = \ %D%/scripts/list-runtime-roots \ - %D%/scripts/substitute + %D%/scripts/substitute \ + %D%/scripts/download if BUILD_DAEMON_OFFLOAD diff --git a/nix/scripts/download.in b/nix/scripts/download.in new file mode 100644 index 0000000000..4d7088a993 --- /dev/null +++ b/nix/scripts/download.in @@ -0,0 +1,11 @@ +#!@SHELL@ +# A shorthand for "guix perform-download", for use by the daemon. + +if test "x$GUIX_UNINSTALLED" = "x" +then + prefix="@prefix@" + exec_prefix="@exec_prefix@" + exec "@bindir@/guix" perform-download "$@" +else + exec guix perform-download "$@" +fi -- cgit v1.2.3 From f9aefa2d5fb3f6aad25a907939ee872c828b33d0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 13 Nov 2016 22:59:54 +0100 Subject: daemon: Add 'built-in-builders' RPC. * nix/libstore/builtins.cc (builtinBuilderNames): New function. * nix/libstore/builtins.hh (builtinBuilderNames): New declaration. * nix/libstore/worker-protocol.hh (PROTOCOL_VERSION): Bump to 0x160. (WorkerOp)[wopBuiltinBuilders]: New value. * nix/nix-daemon/nix-daemon.cc (performOp): Handle it. * guix/store.scm (operation-id)[built-in-builders]: New value. * guix/store.scm (read-arg): Add 'string-list'. (built-in-builders): New procedure. * tests/derivations.scm ("built-in-builders"): New test. --- guix/store.scm | 25 +++++++++++++++++++++++-- nix/libstore/builtins.cc | 10 ++++++++++ nix/libstore/builtins.hh | 3 +++ nix/libstore/worker-protocol.hh | 5 +++-- nix/nix-daemon/nix-daemon.cc | 9 +++++++++ tests/derivations.scm | 4 ++++ 6 files changed, 52 insertions(+), 4 deletions(-) (limited to 'nix') diff --git a/guix/store.scm b/guix/store.scm index 43cfda9214..3047dc39b9 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -95,6 +95,7 @@ (define-module (guix store) path-info-registration-time path-info-nar-size + built-in-builders references references/substitutes requisites @@ -187,7 +188,8 @@ (define-enumerate-type operation-id (query-substitutable-paths 32) (query-valid-derivers 33) (optimize-store 34) - (verify-store 35)) + (verify-store 35) + (built-in-builders 80)) (define-enumerate-type hash-algo ;; hash.hh @@ -283,7 +285,7 @@ (define-syntax write-arg (write-string (bytevector->base16-string arg) p)))) (define-syntax read-arg - (syntax-rules (integer boolean string store-path store-path-list + (syntax-rules (integer boolean string store-path store-path-list string-list substitutable-path-list path-info base16) ((_ integer p) (read-int p)) @@ -295,6 +297,8 @@ (define-syntax read-arg (read-store-path p)) ((_ store-path-list p) (read-store-path-list p)) + ((_ string-list p) + (read-string-list p)) ((_ substitutable-path-list p) (read-substitutable-path-list p)) ((_ path-info p) @@ -914,6 +918,23 @@ (define substitutable-path-info order of PATHS." substitutable-path-list)) +(define built-in-builders + (let ((builders (operation (built-in-builders) + "Return the built-in builders." + string-list))) + (lambda (store) + "Return the names of the supported built-in derivation builders +supported by STORE." + ;; Check whether STORE's version supports this RPC and built-in + ;; derivation builders in general, which appeared in Guix > 0.11.0. + ;; Return the empty list if it doesn't. Note that this RPC does not + ;; exist in 'nix-daemon'. + (if (or (> (nix-server-major-version store) #x100) + (and (= (nix-server-major-version store) #x100) + (>= (nix-server-minor-version store) #x60))) + (builders store) + '())))) + (define-operation (optimize-store) "Optimize the store by hard-linking identical files (\"deduplication\".) Return #t on success." diff --git a/nix/libstore/builtins.cc b/nix/libstore/builtins.cc index 605e44079a..32af767dc4 100644 --- a/nix/libstore/builtins.cc +++ b/nix/libstore/builtins.cc @@ -66,4 +66,14 @@ derivationBuilder lookupBuiltinBuilder(const std::string & name) return NULL; } +std::list builtinBuilderNames() +{ + std::list result; + for(auto&& iter: builtins) + { + result.push_back(iter.first); + } + return result; +} + } diff --git a/nix/libstore/builtins.hh b/nix/libstore/builtins.hh index 0c6db651ab..79171fcb6c 100644 --- a/nix/libstore/builtins.hh +++ b/nix/libstore/builtins.hh @@ -38,4 +38,7 @@ namespace nix { /* Return the built-in builder called BUILDER, or NULL if none was found. */ derivationBuilder lookupBuiltinBuilder(const std::string &builder); + + /* Return the list of supported built-in builder names. */ + std::list builtinBuilderNames(); } diff --git a/nix/libstore/worker-protocol.hh b/nix/libstore/worker-protocol.hh index 7b7be4a8a0..bdeaca2e3a 100644 --- a/nix/libstore/worker-protocol.hh +++ b/nix/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10f +#define PROTOCOL_VERSION 0x160 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -43,7 +43,8 @@ typedef enum { wopQuerySubstitutablePaths = 32, wopQueryValidDerivers = 33, wopOptimiseStore = 34, - wopVerifyStore = 35 + wopVerifyStore = 35, + wopBuiltinBuilders = 80 } WorkerOp; diff --git a/nix/nix-daemon/nix-daemon.cc b/nix/nix-daemon/nix-daemon.cc index 35c284f7e1..9b29b3e91d 100644 --- a/nix/nix-daemon/nix-daemon.cc +++ b/nix/nix-daemon/nix-daemon.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "affinity.hh" #include "globals.hh" +#include "builtins.hh" #include @@ -671,6 +672,14 @@ static void performOp(bool trusted, unsigned int clientVersion, break; } + case wopBuiltinBuilders: { + startWork(); + auto names = builtinBuilderNames(); + stopWork(); + writeStrings(names, to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } diff --git a/tests/derivations.scm b/tests/derivations.scm index 449fb47832..2b5aa796d4 100644 --- a/tests/derivations.scm +++ b/tests/derivations.scm @@ -211,6 +211,10 @@ (define prefix-len (string-length dir)) (= (stat:ino (lstat file1)) (stat:ino (lstat file2)))))))) +(test-equal "built-in-builders" + '("download") + (built-in-builders %store)) + (test-assert "unknown built-in builder" (let ((drv (derivation %store "ohoh" "builtin:does-not-exist" '()))) (guard (c ((nix-protocol-error? c) -- cgit v1.2.3 From 332d7903f52c2bf3741b04ac2d01cd9018b70800 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Thu, 24 Nov 2016 09:23:11 +0100 Subject: Add system start-up files for "guix publish". * .gitignore: add etc/guix-publish.conf and /etc/guix-publish.service. * etc/guix-publish.conf.in: New file. * etc/guix-publish.service.in: New file. * nix/local.mk (etc/guix-%.service, etc/guix-%.conf): Generalized former build-rules for by using patterns. (nodist_systemdservice_DATA): Add etc/guix-publish.service, update comment. (nodist_upstartjob_DATA): Add etc/guix-publish.conf, update comment. * doc/guix.texi (Invoking guix publish): Add description for enabling "guix publish" on host distros using the new files. --- .gitignore | 2 ++ doc/guix.texi | 24 ++++++++++++++++++++++++ etc/guix-publish.conf.in | 12 ++++++++++++ etc/guix-publish.service.in | 19 +++++++++++++++++++ nix/local.mk | 16 ++++++++-------- 5 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 etc/guix-publish.conf.in create mode 100644 etc/guix-publish.service.in (limited to 'nix') diff --git a/.gitignore b/.gitignore index 329d489713..b64f5ef4b0 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,8 @@ /emacs/guix-helper.scm /etc/guix-daemon.conf /etc/guix-daemon.service +/etc/guix-publish.conf +/etc/guix-publish.service /guix-daemon /guix-register /guix/config.scm diff --git a/doc/guix.texi b/doc/guix.texi index 347361ca74..0055d094e8 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -6055,6 +6055,30 @@ add a call to @code{guix-publish-service} in the @code{services} field of the @code{operating-system} declaration (@pxref{guix-publish-service, @code{guix-publish-service}}). +If you are instead running Guix on a ``foreign distro'', follow these +instructions:” + +@itemize +@item +If your host distro uses the systemd init system: + +@example +# ln -s ~root/.guix-profile/lib/systemd/system/guix-publish.service \ + /etc/systemd/system/ +# systemctl start guix-publish && systemctl enable guix-publish +@end example + +@item +If your host distro uses the Upstart init system: + +@example +# ln -s ~root/.guix-profile/lib/upstart/system/guix-publish.conf /etc/init/ +# start guix-publish +@end example + +@item +Otherwise, proceed similarly with your distro's init system. +@end itemize @node Invoking guix challenge @section Invoking @command{guix challenge} diff --git a/etc/guix-publish.conf.in b/etc/guix-publish.conf.in new file mode 100644 index 0000000000..498fa295be --- /dev/null +++ b/etc/guix-publish.conf.in @@ -0,0 +1,12 @@ +# This is a "job" for the Upstart init system to launch 'guix-daemon'. +# Drop it in /etc/init to have 'guix-daemon' automatically started. + +description "Publish the GNU Guix store" + +start on runlevel [2345] + +stop on runlevel [016] + +task + +exec @bindir@/guix publish --user=nobody --port=8181 diff --git a/etc/guix-publish.service.in b/etc/guix-publish.service.in new file mode 100644 index 0000000000..fc4e3c21f3 --- /dev/null +++ b/etc/guix-publish.service.in @@ -0,0 +1,19 @@ +# This is a "service unit file" for the systemd init system to launch +# 'guix publish'. Drop it in /etc/systemd/system or similar to have +# 'guix publish' automatically started. + +[Unit] +Description=Publish the GNU Guix store + +[Service] +ExecStart=@bindir@/guix publish --user=nobody --port=8181 +Environment=GUIX_LOCPATH=/root/.guix-profile/lib/locale +RemainAfterExit=yes +StandardOutput=syslog +StandardError=syslog + +# See . +TasksMax=1024 + +[Install] +WantedBy=multi-user.target diff --git a/nix/local.mk b/nix/local.mk index 86ef769549..79667ed49e 100644 --- a/nix/local.mk +++ b/nix/local.mk @@ -183,26 +183,26 @@ endif BUILD_DAEMON_OFFLOAD nodist_libexec_SCRIPTS = \ %D%/scripts/guix-authenticate -# The '.service' file for systemd. +# The '.service' files for systemd. systemdservicedir = $(libdir)/systemd/system -nodist_systemdservice_DATA = etc/guix-daemon.service +nodist_systemdservice_DATA = etc/guix-daemon.service etc/guix-publish.service -etc/guix-daemon.service: etc/guix-daemon.service.in \ +etc/guix-%.service: etc/guix-%.service.in \ $(top_builddir)/config.status $(AM_V_GEN)$(MKDIR_P) "`dirname $@`"; \ $(SED) -e 's|@''bindir''@|$(bindir)|' < \ - "$(srcdir)/etc/guix-daemon.service.in" > "$@.tmp"; \ + "$(srcdir)/$<" > "$@.tmp"; \ mv "$@.tmp" "$@" -# The '.conf' job for Upstart. +# The '.conf' jobs for Upstart. upstartjobdir = $(libdir)/upstart/system -nodist_upstartjob_DATA = etc/guix-daemon.conf +nodist_upstartjob_DATA = etc/guix-daemon.conf etc/guix-publish.conf -etc/guix-daemon.conf: etc/guix-daemon.conf.in \ +etc/guix-%.conf: etc/guix-%.conf.in \ $(top_builddir)/config.status $(AM_V_GEN)$(MKDIR_P) "`dirname $@`"; \ $(SED) -e 's|@''bindir''@|$(bindir)|' < \ - "$(srcdir)/etc/guix-daemon.conf.in" > "$@.tmp"; \ + "$(srcdir)/$<" > "$@.tmp"; \ mv "$@.tmp" "$@" EXTRA_DIST += \ -- cgit v1.2.3