build-system/go: Build with a filesystem union of Go dependencies.

This basically changes (guix build-system go) so that instead of looking
up its dependencies in a list of directories in $GOPATH, all the
Go dependencies are symlinked into a single directory.

Fixes <https://bugs.gnu.org/33620>.

* guix/build/go-build-system.scm (setup-go-environment): New variable.
(setup-environment, install-source): Remove variables.
(unpack): Unpack the source relative to $GOPATH.
(install): Do not install the compiled objects in the 'pkg' directory.
Install the source code in this phase, and only install the source of
the package named by IMPORT-PATH.
* doc/guix.texi (Build Systems): Adjust accordingly.
* gnu/packages/docker.scm (docker): Import (guix build union) on the build side
and adjust to build phase name changes in (guix build-system go).
* gnu/packages/shellutils.scm (direnv): Likewise.
* gnu/packages/databases.scm (mongo-tools)[arguments]:
Set '#:install-source #f'.
* gnu/packages/music.scm (demlo)[arguments]: Move the 'install-scripts'
phase after the 'install' phase.

doc/guix.texi

@@ -5791,8 +5791,8 @@ some cases, you will need to unpack the package's source code to a
 different directory structure than the one indicated by the import path,
 and @code{#:unpack-path} should be used in such cases.
 
-Packages that provide Go libraries should be installed along with their
+Packages that provide Go libraries should install their source code into
-source code.  The key @code{#:install-source?}, which defaults to
+the built output.  The key @code{#:install-source?}, which defaults to
 @code{#t}, controls whether or not the source code is installed.  It can
 be set to @code{#f} for packages that only provide executable files.
 @end defvr

gnu/packages/databases.scm

@@ -2610,6 +2610,7 @@ transforms idiomatic python function calls to well-formed SQL queries.")
        #:modules ((srfi srfi-1)
                   (guix build go-build-system)
                   (guix build utils))
+       #:install-source? #f
        #:phases
        (let ((all-tools
               '("bsondump" "mongodump" "mongoexport" "mongofiles"
@@ -2629,8 +2630,6 @@ transforms idiomatic python function calls to well-formed SQL queries.")
                  (("skipping restore of system.profile collection\", db)")
                   "skipping restore of system.profile collection\")"))
                #t))
-           ;; We don't need to install the source code for end-user applications
-           (delete 'install-source)
            (replace 'build
              (lambda _
                (for-each (lambda (tool)

gnu/packages/docker.scm

@@ -249,9 +249,11 @@ network attachments.")
      `(#:modules
        ((guix build gnu-build-system)
         ((guix build go-build-system) #:prefix go:)
+        (guix build union)
         (guix build utils))
        #:imported-modules
        (,@%gnu-build-system-modules
+        (guix build union)
         (guix build go-build-system))
        #:phases
        (modify-phases %standard-phases
@@ -412,8 +414,8 @@ network attachments.")
              ;; Make build faster
              (setenv "GOCACHE" "/tmp")
              #t))
-         (add-before 'build 'setup-environment
+         (add-before 'build 'setup-go-environment
-           (assoc-ref go:%standard-phases 'setup-environment))
+           (assoc-ref go:%standard-phases 'setup-go-environment))
          (replace 'build
            (lambda _
              ;; Our LD doesn't like the statically linked relocatable things

gnu/packages/music.scm

@@ -4361,7 +4361,7 @@ console music players.")
                                             dir "/sbin"))
                            (list ffmpeg chromaprint))))
                  #t)))
-           (add-after 'install-source 'install-scripts
+           (add-after 'install 'install-scripts
              (lambda* (#:key outputs #:allow-other-keys)
                (let* ((out (assoc-ref outputs "out"))
                       (root (string-append out "/src/gitlab.com/ambrevar/demlo"))

gnu/packages/shellutils.scm

@@ -118,15 +118,17 @@ are already there.")
        #:make-flags (list (string-append "DESTDIR=" (assoc-ref %outputs "out")))
        #:modules ((guix build gnu-build-system)
                   ((guix build go-build-system) #:prefix go:)
+                  (guix build union)
                   (guix build utils))
        #:imported-modules (,@%gnu-build-system-modules
+                            (guix build union)
                             (guix build go-build-system))
        #:phases
        (modify-phases %standard-phases
          (delete 'configure)
          ;; Help the build scripts find the Go language dependencies.
-         (add-after 'unpack 'setup-go-environment
+         (add-before 'unpack 'setup-go-environment
-           (assoc-ref go:%standard-phases 'setup-environment)))))
+           (assoc-ref go:%standard-phases 'setup-go-environment)))))
     (inputs
      `(("go" ,go)
        ("go-github-com-burntsushi-toml" ,go-github-com-burntsushi-toml)

guix/build-system/go.scm

@@ -39,6 +39,7 @@
 (define %go-build-system-modules
   ;; Build-side modules imported and used by default.
   `((guix build go-build-system)
+    (guix build union)
     ,@%gnu-build-system-modules))
 
 (define (default-go)
@@ -87,6 +88,7 @@
                    (guile #f)
                    (imported-modules %go-build-system-modules)
                    (modules '((guix build go-build-system)
+                              (guix build union)
                               (guix build utils))))
   (define builder
    `(begin

guix/build/go-build-system.scm

@@ -1,6 +1,6 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2016 Petter <petter@mykolab.ch>
-;;; Copyright © 2017 Leo Famulari <leo@famulari.name>
+;;; Copyright © 2017, 2019 Leo Famulari <leo@famulari.name>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -19,6 +19,7 @@
 
 (define-module (guix build go-build-system)
   #:use-module ((guix build gnu-build-system) #:prefix gnu:)
+  #:use-module (guix build union)
   #:use-module (guix build utils)
   #:use-module (ice-9 match)
   #:use-module (srfi srfi-1)
@@ -38,24 +39,26 @@
 ;; results. [0]
 
 ;; Go software is developed and built within a particular file system hierarchy
-;; structure called a 'workspace' [1].  This workspace is found by Go
+;; structure called a 'workspace' [1].  This workspace can be found by Go via
-;; via the GOPATH environment variable.  Typically, all Go source code
+;; the GOPATH environment variable.  Typically, all Go source code and compiled
-;; and compiled objects are kept in a single workspace, but it is
+;; objects are kept in a single workspace, but GOPATH may be a list of
-;; possible for GOPATH to contain a list of directories, and that is
+;; directories [2].  In this go-build-system we create a filesystem union of
-;; what we do in this go-build-system. [2]
+;; the Go-language dependencies. Previously, we made GOPATH a list of store
+;; directories, but stopped because Go programs started keeping references to
+;; these directories in Go 1.11:
+;; <https://bugs.gnu.org/33620>.
 ;;
-;; Go software, whether a package or a command, is uniquely named using
+;; Go software, whether a package or a command, is uniquely named using an
-;; an 'import path'.  The import path is based on the URL of the
+;; 'import path'.  The import path is based on the URL of the software's source.
-;; software's source.  Since most source code is provided over the
+;; Because most source code is provided over the internet, the import path is
-;; internet, the import path is typically a combination of the remote
+;; typically a combination of the remote URL and the source repository's file
-;; URL and the source repository's file system structure. For example,
+;; system structure. For example, the Go port of the common `du` command is
-;; the Go port of the common `du` command is hosted on github.com, at
+;; hosted on github.com, at <https://github.com/calmh/du>.  Thus, the import
-;; <https://github.com/calmh/du>.  Thus, the import path is
+;; path is <github.com/calmh/du>. [3]
-;; <github.com/calmh/du>. [3]
 ;;
-;; It may be possible to programatically guess a package's import path
+;; It may be possible to automatically guess a package's import path based on
-;; based on the source URL, but we don't try that in this revision of
+;; the source URL, but we don't try that in this revision of the
-;; the go-build-system.
+;; go-build-system.
 ;;
 ;; Modules of modular Go libraries are named uniquely with their
 ;; file system paths.  For example, the supplemental but "standardized"
@@ -75,6 +78,17 @@
 ;; file system union of the required modules of such libraries.  I think
 ;; this could be improved in future revisions of the go-build-system.
 ;;
+;; TODO:
+;; * Avoid copying dependencies into the build environment and / or avoid using
+;; a tmpdir when creating the inputs union.
+;; * Use Go modules [4]
+;; * Re-use compiled packages [5]
+;; * Avoid the go-inputs hack
+;; * Stop needing remove-go-references (-trimpath ? )
+;; * Remove module packages, only offering the full Git repos? This is
+;; more idiomatic, I think, because Go downloads Git repos, not modules.
+;; What are the trade-offs?
+;;
 ;; [0] `go build`:
 ;; https://golang.org/cmd/go/#hdr-Compile_packages_and_dependencies
 ;; `go install`:
@@ -107,18 +121,44 @@
 ;;
 ;; [2] https://golang.org/doc/code.html#GOPATH
 ;; [3] https://golang.org/doc/code.html#ImportPaths
+;; [4] https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more
+;; [5] https://bugs.gnu.org/32919
 ;;
 ;; Code:
 
+(define* (setup-go-environment #:key inputs outputs #:allow-other-keys)
+  "Prepare a Go build environment for INPUTS and OUTPUTS.  Build a filesystem
+union of INPUTS.  Export GOPATH, which helps the compiler find the source code
+of the package being built and its dependencies, and GOBIN, which determines
+where executables (\"commands\") are installed to.  This phase is sometimes used
+by packages that use (guix build-system gnu) but have a handful of Go
+dependencies, so it should be self-contained."
+  ;; Using the current working directory as GOPATH makes it easier for packagers
+  ;; who need to manipulate the unpacked source code.
+  (setenv "GOPATH" (getcwd))
+  (setenv "GOBIN" (string-append (assoc-ref outputs "out") "/bin"))
+  (let ((tmpdir (tmpnam)))
+    (match (go-inputs inputs)
+      (((names . directories) ...)
+       (union-build tmpdir (filter directory-exists? directories)
+                    #:create-all-directories? #t
+                    #:log-port (%make-void-port "w"))))
+    ;; XXX A little dance because (guix build union) doesn't use mkdir-p.
+    (copy-recursively tmpdir
+                      (string-append (getenv "GOPATH"))
+                      #:keep-mtime? #t)
+    (delete-file-recursively tmpdir))
+  #t)
+
 (define* (unpack #:key source import-path unpack-path #:allow-other-keys)
-  "Unpack SOURCE in the UNPACK-PATH, or the IMPORT-PATH is the UNPACK-PATH is
+  "Relative to $GOPATH, unpack SOURCE in the UNPACK-PATH, or the IMPORT-PATH is
-unset.  When SOURCE is a directory, copy it instead of unpacking."
+the UNPACK-PATH is unset.  When SOURCE is a directory, copy it instead of
+unpacking."
   (if (string-null? import-path)
       ((display "WARNING: The Go import path is unset.\n")))
   (if (string-null? unpack-path)
       (set! unpack-path import-path))
-  (mkdir "src")
+  (let ((dest (string-append (getenv "GOPATH") "/src/" unpack-path)))
-  (let ((dest (string-append "src/" unpack-path)))
     (mkdir-p dest)
     (if (file-is-directory? source)
       (begin
@@ -128,15 +168,6 @@ unset.  When SOURCE is a directory, copy it instead of unpacking."
         (invoke "unzip" "-d" dest source)
         (invoke "tar" "-C" dest "-xvf" source)))))
 
-(define* (install-source #:key install-source? outputs #:allow-other-keys)
-  "Install the source code to the output directory."
-  (let* ((out (assoc-ref outputs "out"))
-         (source "src")
-         (dest (string-append out "/" source)))
-    (when install-source?
-      (copy-recursively source dest #:keep-mtime? #t))
-    #t))
-
 (define (go-package? name)
   (string-prefix? "go-" name))
 
@@ -155,27 +186,6 @@ unset.  When SOURCE is a directory, copy it instead of unpacking."
                 (_ #f))
               inputs))))
 
-(define* (setup-environment #:key inputs outputs #:allow-other-keys)
-  "Export the variables GOPATH and GOBIN, which are based on INPUTS and OUTPUTS,
-respectively."
-  (let ((out (assoc-ref outputs "out")))
-    ;; GOPATH is where Go looks for the source code of the build's dependencies.
-    (set-path-environment-variable "GOPATH"
-                                   ;; XXX Matching "." hints that we could do
-                                   ;; something simpler here...
-                                   (list ".")
-                                   (match (go-inputs inputs)
-                                     (((_ . dir) ...)
-                                      dir)))
-
-    ;; Add the source code of the package being built to GOPATH.
-    (if (getenv "GOPATH")
-      (setenv "GOPATH" (string-append (getcwd) ":" (getenv "GOPATH")))
-      (setenv "GOPATH" (getcwd)))
-    ;; Where to install compiled executable files ('commands' in Go parlance').
-    (setenv "GOBIN" (string-append out "/bin"))
-    #t))
-
 (define* (build #:key import-path #:allow-other-keys)
   "Build the package named by IMPORT-PATH."
   (with-throw-handler
@@ -193,22 +203,26 @@ respectively."
                               "Here are the results of `go env`:\n"))
       (invoke "go" "env"))))
 
+;; Can this also install commands???
 (define* (check #:key tests? import-path #:allow-other-keys)
   "Run the tests for the package named by IMPORT-PATH."
   (when tests?
     (invoke "go" "test" import-path))
   #t)
 
-(define* (install #:key outputs #:allow-other-keys)
+(define* (install #:key install-source? outputs import-path unpack-path #:allow-other-keys)
-  "Install the compiled libraries. `go install` installs these files to
+  "Install the source code of IMPORT-PATH to the primary output directory.
-$GOPATH/pkg, so we have to copy them into the output directory manually.
+Compiled executable files (Go \"commands\") should have already been installed
-Compiled executable files should have already been installed to the store based
+to the store based on $GOBIN in the build phase.
-on $GOBIN in the build phase."
+XXX We can't make us of compiled libraries (Go \"packages\")."
-  ;; TODO: From go-1.10 onward, the pkg folder should not be needed (see
+  (when install-source?
-  ;; https://lists.gnu.org/archive/html/guix-devel/2018-11/msg00208.html).
+    (if (string-null? import-path)
-  ;; Remove it?
+        ((display "WARNING: The Go import path is unset.\n")))
-  (when (file-exists? "pkg")
+    (let* ((out (assoc-ref outputs "out"))
-    (copy-recursively "pkg" (string-append (assoc-ref outputs "out") "/pkg")))
+           (source (string-append (getenv "GOPATH") "/src/" import-path))
+           (dest (string-append out "/src/" import-path)))
+      (mkdir-p dest)
+      (copy-recursively source dest #:keep-mtime? #t)))
   #t)
 
 (define* (remove-store-reference file file-name
@@ -269,9 +283,8 @@ files in OUTPUTS."
     (delete 'bootstrap)
     (delete 'configure)
     (delete 'patch-generated-file-shebangs)
+    (add-before 'unpack 'setup-go-environment setup-go-environment)
     (replace 'unpack unpack)
-    (add-after 'unpack 'install-source install-source)
-    (add-before 'build 'setup-environment setup-environment)
     (replace 'build build)
     (replace 'check check)
     (replace 'install install)