diff --git a/gnu/packages/julia-xyz.scm b/gnu/packages/julia-xyz.scm index 50be99ba6f..50e6c8e0f9 100644 --- a/gnu/packages/julia-xyz.scm +++ b/gnu/packages/julia-xyz.scm @@ -839,3 +839,33 @@ effectively integers that are interpreted as being scaled by a constant factor. Consequently, they have a fixed number of digits (bits) after the decimal (radix) point.") (license license:expat))) + +(define-public julia-tokenize + (package + (name "julia-tokenize") + (version "0.5.6") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/JuliaLang/Tokenize.jl") + (commit (string-append "v" version)))) + (file-name "Tokenize") + (sha256 + (base32 "1i05y9wvy5lii082bsn6qqs65nc9lvhw3mvl6jqmpdm5xqxw5mrg")))) + (build-system julia-build-system) + (home-page "https://github.com/JuliaLang/Tokenize.jl") + (synopsis "Tokenization for Julia source code") + (description "@code{Tokenize.jl} is a Julia package that serves a similar +purpose and API as the tokenize module in Python but for Julia. This is to +take a string or buffer containing Julia code, perform lexical analysis and +return a stream of tokens. + +The goals of this package is to be: +@enumerate +@item Fast +@item Round trippable +@item Non error throwing +@end enumerate +") + (license license:expat)))