guix-devel/guix/download.scm

298 lines
12 KiB
Scheme
Raw Normal View History

;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2012, 2013, 2014 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2013, 2014 Andreas Enge <andreas@enge.fr>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
(define-module (guix download)
#:use-module (ice-9 match)
#:use-module (guix derivations)
#:use-module (guix packages)
#:use-module ((guix store) #:select (derivation-path? add-to-store))
Use #:prefix instead of #:renamer with 'symbol-prefix-proc'. * gnu/packages/abiword.scm, gnu/packages/admin.scm, gnu/packages/aidc.scm, gnu/packages/backup.scm, gnu/packages/bittorrent.scm, gnu/packages/boost.scm, gnu/packages/compression.scm, gnu/packages/cryptsetup.scm, gnu/packages/curl.scm, gnu/packages/cyrus-sasl.scm, gnu/packages/dc.scm, gnu/packages/emacs.scm, gnu/packages/fonts.scm, gnu/packages/fontutils.scm, gnu/packages/geeqie.scm, gnu/packages/ghostscript.scm, gnu/packages/gimp.scm, gnu/packages/gkrellm.scm, gnu/packages/gl.scm, gnu/packages/glib.scm, gnu/packages/gnome.scm, gnu/packages/gnunet.scm, gnu/packages/gnupg.scm, gnu/packages/gnutls.scm, gnu/packages/gnuzilla.scm, gnu/packages/gsasl.scm, gnu/packages/gtk.scm, gnu/packages/gv.scm, gnu/packages/image.scm, gnu/packages/indent.scm, gnu/packages/inkscape.scm, gnu/packages/lesstif.scm, gnu/packages/links.scm, gnu/packages/lisp.scm, gnu/packages/lsh.scm, gnu/packages/lsof.scm, gnu/packages/maths.scm, gnu/packages/mp3.scm, gnu/packages/openssl.scm, gnu/packages/ots.scm, gnu/packages/pciutils.scm, gnu/packages/pcre.scm, gnu/packages/pdf.scm, gnu/packages/pulseaudio.scm, gnu/packages/python.scm, gnu/packages/slim.scm, gnu/packages/ssh.scm, gnu/packages/texlive.scm, gnu/packages/vim.scm, gnu/packages/vpn.scm, gnu/packages/web.scm, gnu/packages/wv.scm, gnu/packages/xiph.scm, gnu/packages/xml.scm, gnu/packages/xorg.scm, gnu/packages/yasm.scm, gnu/packages/zip.scm, guix/build/cmake-build-system.scm, guix/build/perl-build-system.scm, guix/build/python-build-system.scm, guix/download.scm: Replaces uses of #:renamer and 'symbol-prefix-proc' with #:prefix.
2014-09-23 22:44:22 +02:00
#:use-module ((guix build download) #:prefix build:)
#:use-module (guix monads)
#:use-module (guix gexp)
#:use-module (guix utils)
#:use-module (web uri)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-26)
#:export (%mirrors
url-fetch
download-to-store))
;;; Commentary:
;;;
;;; Produce fixed-output derivations with data fetched over HTTP or FTP.
;;;
;;; Code:
(define %mirrors
;; Mirror lists used when `mirror://' URLs are passed.
(let* ((gnu-mirrors
'(;; This one redirects to a (supposedly) nearby and (supposedly)
;; up-to-date mirror.
"http://ftpmirror.gnu.org/"
"ftp://ftp.cs.tu-berlin.de/pub/gnu/"
"ftp://ftp.funet.fi/pub/mirrors/ftp.gnu.org/gnu/"
;; This one is the master repository, and thus it's always
;; up-to-date.
"http://ftp.gnu.org/pub/gnu/")))
`((gnu ,@gnu-mirrors)
(gcc
"ftp://ftp.nluug.nl/mirror/languages/gcc/"
"ftp://ftp.fu-berlin.de/unix/languages/gcc/"
"ftp://ftp.irisa.fr/pub/mirrors/gcc.gnu.org/gcc/"
"ftp://gcc.gnu.org/pub/gcc/"
,@(map (cut string-append <> "/gcc") gnu-mirrors))
(gnupg
"ftp://gd.tuwien.ac.at/privacy/gnupg/"
"ftp://mirrors.dotsrc.org/gcrypt/"
"ftp://mirror.cict.fr/gnupg/"
"http://artfiles.org/gnupg.org"
"ftp://ftp.franken.de/pub/crypt/mirror/ftp.gnupg.org/gcrypt/"
"ftp://ftp.freenet.de/pub/ftp.gnupg.org/gcrypt/"
"http://www.crysys.hu/"
"ftp://ftp.hi.is/pub/mirrors/gnupg/"
"ftp://ftp.heanet.ie/mirrors/ftp.gnupg.org/gcrypt/"
"ftp://ftp.bit.nl/mirror/gnupg/"
"ftp://ftp.surfnet.nl/pub/security/gnupg/"
"ftp://ftp.iasi.roedu.net/pub/mirrors/ftp.gnupg.org/"
"ftp://ftp.sunet.se/pub/security/gnupg/"
"ftp://mirror.switch.ch/mirror/gnupg/"
"ftp://mirror.tje.me.uk/pub/mirrors/ftp.gnupg.org/"
"ftp://ftp.mirrorservice.org/sites/ftp.gnupg.org/gcrypt/"
"ftp://ftp.ring.gr.jp/pub/net/gnupg/"
"ftp://ftp.gnupg.org/gcrypt/")
(gnome
"http://ftp.belnet.be/ftp.gnome.org/"
"http://ftp.linux.org.uk/mirrors/ftp.gnome.org/"
"http://ftp.gnome.org/pub/GNOME/"
"http://mirror.yandex.ru/mirrors/ftp.gnome.org/")
(savannah
"http://download.savannah.gnu.org/releases/"
"ftp://ftp.twaren.net/Unix/NonGNU/"
"ftp://mirror.csclub.uwaterloo.ca/nongnu/"
"ftp://mirror.publicns.net/pub/nongnu/"
"ftp://savannah.c3sl.ufpr.br/"
"http://ftp.cc.uoc.gr/mirrors/nongnu.org/"
"http://ftp.twaren.net/Unix/NonGNU/"
"http://mirror.csclub.uwaterloo.ca/nongnu/"
"http://nongnu.askapache.com/"
"http://savannah.c3sl.ufpr.br/"
"http://www.centervenus.com/mirrors/nongnu/"
"http://download.savannah.gnu.org/releases-noredirect/")
(sourceforge
"http://prdownloads.sourceforge.net/"
"http://heanet.dl.sourceforge.net/sourceforge/"
"http://surfnet.dl.sourceforge.net/sourceforge/"
"http://dfn.dl.sourceforge.net/sourceforge/"
"http://mesh.dl.sourceforge.net/sourceforge/"
"http://ovh.dl.sourceforge.net/sourceforge/"
"http://osdn.dl.sourceforge.net/sourceforge/")
(kernel.org
"http://www.all.kernel.org/pub/"
"http://ramses.wh2.tu-dresden.de/pub/mirrors/kernel.org/"
"http://linux-kernel.uio.no/pub/"
"http://kernel.osuosl.org/pub/"
"ftp://ftp.funet.fi/pub/mirrors/ftp.kernel.org/pub/"
"http://ftp.be.debian.org/pub/"
"http://mirror.linux.org.au/")
(apache ; from http://www.apache.org/mirrors/dist.html
"http://www.eu.apache.org/dist/"
"http://www.us.apache.org/dist/"
"ftp://gd.tuwien.ac.at/pub/infosys/servers/http/apache/dist/"
"http://apache.belnet.be/"
"http://mirrors.ircam.fr/pub/apache/"
"http://apache-mirror.rbc.ru/pub/apache/"
;; As a last resort, try the archive.
"http://archive.apache.org/dist/")
(xorg ; from http://www.x.org/wiki/Releases/Download
"http://www.x.org/releases/" ; main mirrors
"ftp://mirror.csclub.uwaterloo.ca/x.org/" ; North America
"ftp://xorg.mirrors.pair.com/"
"http://mirror.csclub.uwaterloo.ca/x.org/"
"http://xorg.mirrors.pair.com/"
"http://mirror.us.leaseweb.net/xorg/"
"ftp://artfiles.org/x.org/" ; Europe
"ftp://ftp.chg.ru/pub/X11/x.org/"
"ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/"
"ftp://ftp.gwdg.de/pub/x11/x.org/"
"ftp://ftp.mirrorservice.org/sites/ftp.x.org/"
"ftp://ftp.ntua.gr/pub/X11/"
"ftp://ftp.piotrkosoft.net/pub/mirrors/ftp.x.org/"
"ftp://ftp.portal-to-web.de/pub/mirrors/x.org/"
"ftp://ftp.solnet.ch/mirror/x.org/"
"ftp://gd.tuwien.ac.at/X11/"
"ftp://mi.mirror.garr.it/mirrors/x.org/"
"ftp://mirror.cict.fr/x.org/"
"ftp://mirror.switch.ch/mirror/X11/"
"ftp://mirrors.ircam.fr/pub/x.org/"
"ftp://x.mirrors.skynet.be/pub/ftp.x.org/"
"ftp://ftp.cs.cuhk.edu.hk/pub/X11" ; East Asia
"ftp://ftp.u-aizu.ac.jp/pub/x11/x.org/"
"ftp://ftp.yz.yamagata-u.ac.jp/pub/X11/x.org/"
"ftp://ftp.kaist.ac.kr/x.org/"
"ftp://mirrors.go-part.com/xorg/"
"http://x.cs.pu.edu.tw/"
"ftp://ftp.is.co.za/pub/x.org") ; South Africa
(cpan ; from http://www.cpan.org/SITES.html
"http://cpan.enstimac.fr/"
"ftp://ftp.ciril.fr/pub/cpan/"
"ftp://artfiles.org/cpan.org/"
"http://www.cpan.org/"
"ftp://cpan.rinet.ru/pub/mirror/CPAN/"
"http://cpan.cu.be/"
"ftp://cpan.inode.at/"
"ftp://cpan.iht.co.il/"
"ftp://ftp.osuosl.org/pub/CPAN/"
"ftp://ftp.nara.wide.ad.jp/pub/CPAN/"
"http://mirrors.163.com/cpan/"
"ftp://cpan.mirror.ac.za/")
(imagemagick
;; from http://www.imagemagick.org/script/download.php
;; (without mirrors that are unavailable or not up to date)
;; mirrors keeping old versions at the top level
"ftp://sunsite.icm.edu.pl/packages/ImageMagick/"
;; mirrors moving old versions to "legacy"
"http://mirrors-au.go-parts.com/mirrors/ImageMagick/"
"ftp://mirror.aarnet.edu.au/pub/imagemagick/"
"http://mirror.checkdomain.de/imagemagick/"
"ftp://ftp.kddlabs.co.jp/graphics/ImageMagick/"
"ftp://ftp.u-aizu.ac.jp/pub/graphics/image/ImageMagick/imagemagick.org/"
"ftp://ftp.nluug.nl/pub/ImageMagick/"
"http://ftp.surfnet.nl/pub/ImageMagick/"
"http://mirror.searchdaimon.com/ImageMagick"
"ftp://ftp.tpnet.pl/pub/graphics/ImageMagick/"
"http://mirrors-ru.go-parts.com/mirrors/ImageMagick/"
"http://mirror.is.co.za/pub/imagemagick/"
"http://mirrors-uk.go-parts.com/mirrors/ImageMagick/"
"http://mirrors-usa.go-parts.com/mirrors/ImageMagick/"
"ftp://ftp.fifi.org/pub/ImageMagick/"
"http://www.imagemagick.org/download/"
;; one legacy location as a last resort
"http://www.imagemagick.org/download/legacy/")
(debian
"http://ftp.de.debian.org/debian/"
"http://ftp.fr.debian.org/debian/"
"http://ftp.debian.org/debian/"))))
(define (gnutls-package)
"Return the default GnuTLS package."
(let ((module (resolve-interface '(gnu packages gnutls))))
(module-ref module 'gnutls)))
(define* (url-fetch store url hash-algo hash
#:optional name
#:key (system (%current-system)) guile
(mirrors %mirrors))
"Return the path of a fixed-output derivation in STORE that fetches
URL (a string, or a list of strings denoting alternate URLs), which is
expected to have hash HASH of type HASH-ALGO (a symbol). By default,
the file name is the base name of URL; optionally, NAME can specify a
different file name.
When one of the URL starts with mirror://, then its host part is
interpreted as the name of a mirror scheme, taken from MIRRORS; MIRRORS
must be a list of symbol/URL-list pairs."
(define guile-for-build
(package-derivation store
(or guile
gnu: Split (gnu packages base), adding (gnu packages commencement). * gnu/packages/base.scm (gnu-make-boot0, diffutils-boot0, findutils-boot0, %boot0-inputs, nix-system->gnu-triplet, boot-triplet, binutils-boot0, gcc-boot0, perl-boot0, linux-libre-headers-boot0, texinfo-boot0, %boot1-inputs, glibc-final-with-bootstrap-bash, cross-gcc-wrapper, static-bash-for-glibc, glibc-final, gcc-boot0-wrapped, %boot2-inputs, binutils-final, libstdc++, gcc-final, ld-wrapper-boot3, %boot3-inputs, bash-final, %boot4-inputs, guile-final, gnu-make-final, ld-wrapper, coreutils-final, grep-final, %boot5-inputs, %final-inputs, canonical-package, gcc-toolchain, gcc-toolchain-4.8, gcc-toolchain-4.9): Move to... * gnu/packages/commencement.scm: ... here. New file. * gnu-system.am (GNU_SYSTEM_MODULES): Add it. * build-aux/check-final-inputs-self-contained.scm: Adjust accordingly. * gnu/packages/cross-base.scm: Likewise. * gnu/packages/make-bootstrap.scm: Likewise. * guix/build-system/cmake.scm (cmake-build): Likewise. * guix/build-system/gnu.scm (standard-packages, gnu-build, gnu-cross-build): Likewise. * guix/build-system/perl.scm (perl-build): Likewise. * guix/build-system/python.scm (python-build): Likewise. * guix/build-system/trivial.scm (guile-for-build): Likewise. * guix/download.scm (url-fetch): Likewise. * guix/gexp.scm (default-guile): Likewise. * guix/git-download.scm (git-fetch): Likewise. * guix/monads.scm (run-with-store): Likewise. * guix/packages.scm (default-guile): Likewise. * guix/scripts/package.scm (guix-package): Likewise. * guix/scripts/refresh.scm: Likewise. * guix/svn-download.scm (svn-fetch): Likewise. * tests/builders.scm (%bootstrap-inputs, %bootstrap-search-paths): Likewise. * tests/packages.scm ("GNU Make, bootstrap"): Likewise. * tests/guix-package.sh: Likewise. * gnu/services/base.scm: Use 'canonical-package' instead of xxx-final. * gnu/services/xorg.scm: Likewise. * gnu/system/vm.scm: Likewise. * guix/scripts/pull.scm (guix-pull): Likewise.
2014-08-27 00:25:17 +02:00
(let ((distro (resolve-interface
'(gnu packages commencement))))
(module-ref distro 'guile-final)))
system))
(define file-name
(match url
((head _ ...)
(basename head))
(_
(basename url))))
(define need-gnutls?
;; True if any of the URLs need TLS support.
(let ((https? (cut string-prefix? "https://" <>)))
(match url
((? string?)
(https? url))
((url ...)
(any https? url)))))
(define builder
#~(begin
#$(if need-gnutls?
;; Add GnuTLS to the inputs and to the load path.
#~(eval-when (load expand eval)
(set! %load-path
(cons (string-append #$(gnutls-package)
"/share/guile/site/"
(effective-version))
%load-path)))
#~#t)
(use-modules (guix build download))
(url-fetch '#$url #$output
#:mirrors '#$mirrors)))
(let ((uri (and (string? url) (string->uri url))))
(if (or (and (string? url) (not uri))
(and uri (memq (uri-scheme uri) '(#f file))))
(add-to-store store (or name file-name)
#f "sha256" (if uri (uri-path uri) url))
(run-with-store store
(gexp->derivation (or name file-name) builder
#:system system
#:hash-algo hash-algo
#:hash hash
#:modules '((guix build download)
(guix build utils)
(guix ftp-client))
#:guile-for-build guile-for-build
;; In general, offloading downloads is not a good idea.
;;#:local-build? #t
;; FIXME: The above would also disable use of
;; substitutes, so comment it out; see
;; <https://bugs.gnu.org/18747>.
)
#:guile-for-build guile-for-build
#:system system))))
(define* (download-to-store store url #:optional (name (basename url))
#:key (log (current-error-port)))
"Download from URL to STORE, either under NAME or URL's basename if
omitted. Write progress reports to LOG."
(define uri
(string->uri url))
(if (or (not uri) (memq (uri-scheme uri) '(file #f)))
(add-to-store store name #f "sha256"
(if uri (uri-path uri) url))
(call-with-temporary-output-file
(lambda (temp port)
(let ((result
(parameterize ((current-output-port log))
(build:url-fetch url temp #:mirrors %mirrors))))
(close port)
(and result
(add-to-store store name #f "sha256" temp)))))))
;;; download.scm ends here