From ac906cb7bb2ec77821ddec291db4857cc812599d Mon Sep 17 00:00:00 2001 From: Ricardo Wurmus Date: Thu, 30 Aug 2018 15:12:07 +0200 Subject: [PATCH] import: pypi: Support recursive importing. * guix/import/pypi.scm (guess-requirements): Use upstream names. (compute-inputs): Return the upstream dependency names as an additional value. (make-pypi-sexp): Likewise. (pypi->guix-package): Memoize it. (pypi-recursive-import): New procedure. * guix/scripts/import/pypi.scm (show-help, %options): Accept "recursive" option. (guix-import-pypi): Use pypi-recursive-import. * doc/guix.texi (Invoking guix import): Document it. --- doc/guix.texi | 8 +++ guix/import/pypi.scm | 119 ++++++++++++++++++++--------------- guix/scripts/import/pypi.scm | 28 +++++++-- 3 files changed, 99 insertions(+), 56 deletions(-) diff --git a/doc/guix.texi b/doc/guix.texi index 1e17c294b6..8611059791 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -6442,6 +6442,14 @@ package: guix import pypi itsdangerous @end example +@table @code +@item --recursive +@itemx -r +Traverse the dependency graph of the given upstream package recursively +and generate package expressions for all those packages that are not yet +in Guix. +@end table + @item gem @cindex gem Import metadata from @uref{https://rubygems.org/, diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index 51c2d5264a..87b047bdac 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -25,6 +25,7 @@ #:use-module (ice-9 match) #:use-module (ice-9 pretty-print) #:use-module (ice-9 regex) + #:use-module (ice-9 receive) #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) @@ -47,6 +48,7 @@ #:use-module ((guix licenses) #:prefix license:) #:use-module (guix build-system python) #:export (guix-package->pypi-name + pypi-recursive-import pypi->guix-package %pypi-updater)) @@ -162,7 +164,7 @@ cannot determine package dependencies")) ((or (string-null? line) (comment? line)) (loop result)) (else - (loop (cons (python->package-name (clean-requirement line)) + (loop (cons (clean-requirement line) result)))))))))) (define (read-wheel-metadata wheel-archive) @@ -182,9 +184,7 @@ cannot determine package dependencies")) (hash-ref (list-ref run_requires 0) "requires") '()))) - (map (lambda (r) - (python->package-name (clean-requirement r))) - requirements))))) + (map clean-requirement requirements))))) (lambda () (delete-file json-file) (rmdir dirname)))))) @@ -237,16 +237,21 @@ cannot determine package dependencies")) (define (compute-inputs source-url wheel-url tarball) "Given the SOURCE-URL of an already downloaded TARBALL, return a list of -name/variable pairs describing the required inputs of this package." - (sort - (map (lambda (input) - (list input (list 'unquote (string->symbol input)))) - (remove (cut string=? "python-argparse" <>) - (guess-requirements source-url wheel-url tarball))) - (lambda args - (match args - (((a _ ...) (b _ ...)) - (string-ci) + (guess-requirements source-url wheel-url tarball)))) + (values (sort + (map (lambda (input) + (let ((guix-name (python->package-name input))) + (list guix-name (list 'unquote (string->symbol guix-name))))) + dependencies) + (lambda args + (match args + (((a _ ...) (b _ ...)) + (string-cipackage-name name)) - (version ,version) - (source (origin - (method url-fetch) + (receive (input-package-names upstream-dependency-names) + (compute-inputs source-url wheel-url temp) + (values + `(package + (name ,(python->package-name name)) + (version ,version) + (source (origin + (method url-fetch) - ;; Sometimes 'pypi-uri' doesn't quite work due to mixed - ;; cases in NAME, for instance, as is the case with - ;; "uwsgi". In that case, fall back to a full URL. - (uri (pypi-uri ,(string-downcase name) version)) - (sha256 - (base32 - ,(guix-hash-url temp))))) - (build-system python-build-system) - ,@(maybe-inputs (compute-inputs source-url wheel-url temp)) - (home-page ,home-page) - (synopsis ,synopsis) - (description ,description) - (license ,(license->symbol license))))))) + ;; Sometimes 'pypi-uri' doesn't quite work due to mixed + ;; cases in NAME, for instance, as is the case with + ;; "uwsgi". In that case, fall back to a full URL. + (uri (pypi-uri ,(string-downcase name) version)) + (sha256 + (base32 + ,(guix-hash-url temp))))) + (build-system python-build-system) + ,@(maybe-inputs input-package-names) + (home-page ,home-page) + (synopsis ,synopsis) + (description ,description) + (license ,(license->symbol license))) + upstream-dependency-names)))))) -(define (pypi->guix-package package-name) - "Fetch the metadata for PACKAGE-NAME from pypi.org, and return the +(define pypi->guix-package + (memoize + (lambda* (package-name) + "Fetch the metadata for PACKAGE-NAME from pypi.org, and return the `package' s-expression corresponding to that package, or #f on failure." - (let ((package (pypi-fetch package-name))) - (and package - (guard (c ((missing-source-error? c) - (let ((package (missing-source-error-package c))) - (leave (G_ "no source release for pypi package ~a ~a~%") - (assoc-ref* package "info" "name") - (assoc-ref* package "info" "version"))))) - (let ((name (assoc-ref* package "info" "name")) - (version (assoc-ref* package "info" "version")) - (release (assoc-ref (latest-source-release package) "url")) - (wheel (assoc-ref (latest-wheel-release package) "url")) - (synopsis (assoc-ref* package "info" "summary")) - (description (assoc-ref* package "info" "summary")) - (home-page (assoc-ref* package "info" "home_page")) - (license (string->license (assoc-ref* package "info" "license")))) - (make-pypi-sexp name version release wheel home-page synopsis - description license)))))) + (let ((package (pypi-fetch package-name))) + (and package + (guard (c ((missing-source-error? c) + (let ((package (missing-source-error-package c))) + (leave (G_ "no source release for pypi package ~a ~a~%") + (assoc-ref* package "info" "name") + (assoc-ref* package "info" "version"))))) + (let ((name (assoc-ref* package "info" "name")) + (version (assoc-ref* package "info" "version")) + (release (assoc-ref (latest-source-release package) "url")) + (wheel (assoc-ref (latest-wheel-release package) "url")) + (synopsis (assoc-ref* package "info" "summary")) + (description (assoc-ref* package "info" "summary")) + (home-page (assoc-ref* package "info" "home_page")) + (license (string->license (assoc-ref* package "info" "license")))) + (make-pypi-sexp name version release wheel home-page synopsis + description license)))))))) + +(define (pypi-recursive-import package-name) + (recursive-import package-name #f + #:repo->guix-package (lambda (name repo) + (pypi->guix-package name)) + #:guix-name python->package-name)) (define (string->license str) "Convert the string STR into a license object." diff --git a/guix/scripts/import/pypi.scm b/guix/scripts/import/pypi.scm index 59a925a3ca..7bd83818ba 100644 --- a/guix/scripts/import/pypi.scm +++ b/guix/scripts/import/pypi.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson +;;; Copyright © 2018 Ricardo Wurmus ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,6 +26,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-37) + #:use-module (srfi srfi-41) #:use-module (ice-9 match) #:use-module (ice-9 format) #:export (guix-import-pypi)) @@ -43,6 +45,8 @@ Import and convert the PyPI package for PACKAGE-NAME.\n")) (display (G_ " -h, --help display this help and exit")) (display (G_ " + -r, --recursive import packages recursively")) + (display (G_ " -V, --version display version information and exit")) (newline) (show-bug-report-information)) @@ -56,6 +60,9 @@ Import and convert the PyPI package for PACKAGE-NAME.\n")) (option '(#\V "version") #f #f (lambda args (show-version-and-exit "guix import pypi"))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) %standard-import-options)) @@ -81,11 +88,22 @@ Import and convert the PyPI package for PACKAGE-NAME.\n")) (reverse opts)))) (match args ((package-name) - (let ((sexp (pypi->guix-package package-name))) - (unless sexp - (leave (G_ "failed to download meta-data for package '~a'~%") - package-name)) - sexp)) + (if (assoc-ref opts 'recursive) + ;; Recursive import + (map (match-lambda + ((and ('package ('name name) . rest) pkg) + `(define-public ,(string->symbol name) + ,pkg)) + (_ #f)) + (reverse + (stream->list + (pypi-recursive-import package-name)))) + ;; Single import + (let ((sexp (pypi->guix-package package-name))) + (unless sexp + (leave (G_ "failed to download meta-data for package '~a'~%") + package-name)) + sexp))) (() (leave (G_ "too few arguments~%"))) ((many ...)