commit 573ac9aa5e17b49ac10a32ae8f72854bafef35f0 Author: nixo Date: Fri Oct 25 03:21:21 2019 +0200 Init code The project is in a good state. Session support is wonderful, async evaluation works well, it supports inline images in html export... I need to write the documentation and unit tests diff --git a/init.jl b/init.jl new file mode 100644 index 0000000..7af40b2 --- /dev/null +++ b/init.jl @@ -0,0 +1,262 @@ +const supported_packages = [:DataFrames, :NamedArrays, :Plots] + +# Generic fallback +orgshow(io::IO, Any, i; kwargs...) = show(io, i) +orgshow(io::IO, ::MIME"text/org", i; kwargs...) = show(io, i) +# Overload types +orgshow(io::IO, ::MIME"text/org", t::Tuple; kwargs...) = print(io, join(t, ',')) +orgshow(io::IO, ::MIME"text/org", ::Nothing; kwargs...) = print(io, "") +orgshow(io::IO, ::MIME"text/org", a::Array{T,1}; kwargs...) where T <: Any = print(io, join(a, '\n')) + +# You can override this with a better one that uses some available module +function orgshow(io::IO, ::MIME"text/html", i::Array{T,2}; kwargs...) where T <: Any + width = get(Dict(kwargs), :width, "100") + print(io, """""") + content = eachrow(i) |> x -> string("", + join([string("")) + print(io, content, "
", join(l, "")) + for l in x], "
") +end + +function orgshow(io::IO, ::MIME"text/org", i::Array{T,2}; kwargs...) where T <: Any + out = eachrow(i) |> x -> join([join(l, ',') for l in x], '\n') + print(io, out) +end + +function orgshow(io::IO, ::MIME"text/csv", i::Array{T,2}; kwargs...) where T <: Any + orgshow(io, MIME("text/org"), i; kwargs...) +end + +# The comma is needed to allow export as table +orgshow(io::IO, ::MIME"text/org", e::Exception; kwargs...) = print(io, "ERROR,", e) + +function orgshow(io::IO, ::MIME"text/org", t::NamedTuple) + print(io, join(string.(keys(t)), ',')) + println(io) + print(io, join(t, ',')) +end + +function orgshow(io::IO, ::MIME"text/org", + ta::Vector{<:NamedTuple}) + "This assume keys are the same. A better NamedTuple export is provided by + the DataFrames (DataFrame(ta))" + length(ta) <= 0 && return "" + println(io, join(keys(first(ta)), ',')) + for t in ta + print(io, join(string.(values(t)), ',')) + println(io) + end +end + +function define_Plots() + # Fallback: we will try to plot any image/png or image/svg + Main.@eval function orgshow(io::IO, + m::MIME"image/png", + any; kwargs...) + show(io, MIME("image/png"), plot(any; kwargs...)) + end + Main.@eval function orgshow(io::IO, + m::MIME"image/svg+xml", + any; kwargs...) + show(io, MIME("image/svg+xml"), plot(any; kwargs...)) + end + Main.@eval function orgshow(io::IO, + m::MIME"image/png", + p::Plots.Plot; kwargs...) + show(io, MIME("image/png"), plot(p; kwargs...)) + end + Main.@eval function orgshow(io::IO, ::MIME"image/png", e::Exception; kwargs...) + let p = plot(showaxis = false, grid = false, bg = :yellow) + annotate!([0.5], [0.5], (string("ERROR: ", e), :red)) + orgshow(io, MIME("image/png"), p; kwargs...) + end + end + Main.@eval function orgshow(io::IO, ::MIME"image/svg+xml", e::Exception; kwargs...) + let p = plot(showaxis = false, grid = false, bg = :yellow) + annotate!([0.5], [0.5], (string("ERROR: ", e), :red)) + orgshow(io, MIME("image/svg+xml"), p; kwargs...) + end + end + Main.@eval function orgshow(io::IO, ::MIME"text/html", p::Plots.Plot; kwargs...) + Plots._show(io, MIME("text/html"), plot(p; kwargs...)) + end + Main.@eval function orgshow(io::IO, ::MIME"image/svg+xml", p::Plots.Plot; kwargs...) + show(io, MIME("image/svg+xml"), plot(p; kwargs...)) + end + Main.@eval function orgshow(io::IO, ::MIME"application/pdf", p::Plots.Plot; kwargs...) + # ps, eps, tex or pdf. I think extra packages are required for all but pdf + show(io, MIME("application/pdf"), plot(p; kwargs...)) + end + Main.@eval function orgshow(io::IO, ::MIME"application/postscript", p::Plots.Plot; kwargs...) + show(io, MIME("application/postscript"), plot(p; kwargs...)) + end + Main.@eval function orgshow(io::IO, ::MIME"image/eps", p::Plots.Plot; + kwargs...) + show(io, MIME("image/eps"), plot(p; kwargs...)) + end + Main.@eval function orgshow(io::IO, ::MIME"application/x-tex", p::Plots.Plot; + kwargs...) + show(io, MIME("application/x-tex"), plot(p; kwargs...)) + end + Main.@eval function orgshow(io::IO, ::MIME"text/org", p::Plots.Plot; kwargs...) + # png or svg + p.attr[:html_output_format] = "png" + orgshow(io::IO, MIME("text/html"), p::Plots.Plot; kwargs...) + end +end + +function define_DataFrames() + Main.@eval function orgshow(io::IO, ::MIME"text/csv", d::DataFrames.DataFrame) + orgshow(io, MIME("text/org"), d) + end + Main.@eval function orgshow(io::IO, ::MIME"text/org", d::DataFrames.DataFrame) + out = join(string.(names(d)), ',') * '\n' + out *= join([join(x, ',') for x in eachrow(d) .|> collect],'\n') + print(io, out) + end +end + +function define_NamedArrays() + Main.@eval function orgshow(io::IO, ::MIME"text/org", + na::NamedArrays.NamedArray{T,2} where T <: Any) + n = names(na) + a = collect(na) + print(io, join(string.(na.dimnames), "/") * ',') + print(io, join(n[2], ','') * '\n') + print(io, join([join([string(n[1][i], ','), + join([a[i,j] + for j in 1:size(na,2) + ], ',')]) + for i in 1:size(na,1) + ], '\n')) + end + Main.@eval function orgshow(io::IO, ::MIME"text/org", na::NamedArrays.NamedArray{T,1} where T <: Any) + n = names(na) + a = collect(na) + print(io, string(na.dimnames[1], ',', '\n')) + print(io, join([join([n[1][i], a[i]], ',') + for i in 1:length(n[1])], '\n')) + end +end + +define_package_functions(pkg::Symbol) = (@eval $pkg)() + +function OrgBabelImport(_imports; forced = false) + "Load dependencies. Do this before calling OrgBabelReload()" + # Reload this module, so that if new packages have been imported, + # we can use them to save the output + !forced && isempty(_imports) && return + try + Main.eval(Meta.parse( + """begin + $_imports + end""")) + true + catch e + @show e + end +end + +function OrgBabelReload() + "Defines show method based on loaded packages" + for pkg in supported_packages + if isdefined(Main, pkg) && (isa(getfield(Main, pkg), Module) || + isa(getfield(Main, pkg), UnionAll)) + define_package_functions(Symbol("define_", pkg)) + end + end +end + +const Mimes = Dict(:org => "text/org", + :csv => "text/csv", + :png => "image/png", + :svg => "image/svg+xml", + :pdf => "application/pdf", + :html => "text/html", + :auto => "text/org", + :ps => "application/postscript", + :eps => "image/eps", + :tex => "application/x-tex") + +function OrgBabelFormat(output_type::Symbol, + output_file, + dir, vars, + src_file, + silently::Bool, + kwargs) + content = read(src_file, String) + # Fake a prompt with the current input + try + println(string("\njulia> ", content)) + catch + end + # Dispatch on output type + if output_type == :value + # Run the code + result = cd(dir) do + try + # Variable assignment + Main.eval(Meta.parse(vars)) + # src block evaluation + Main.eval(Meta.parse("begin $content end")) + catch e + e + end + end + if !silently + try + print(result) + catch e + println("Error $e while showing results") + end + end + # Decide output type. + # If the output has an extension, use it. + # else, use the exporter format. Fallback to text/org + output_ext = replace(splitext(output_file)[2], "." => "") + required_format = isempty(output_ext) ? :auto : Symbol(output_ext) + mime = get(Mimes, required_format, "text/org") + temporary_output = IOBuffer() + # Output directly to org (no :file, -> save to output_file) + try + orgshow(temporary_output, + MIME(mime), result; Base.eval(Meta.parse(kwargs))...) + catch e + @error "Probable ob-julia error! Please report to the author!" + @error "Error: $e" + print(temporary_output, + "Probable ob-julia error! Please report to the author!", + "Error: $e") + end + write(output_file, take!(temporary_output)) + elseif output_type == :output + temporary_output_file = tempname() + open(temporary_output_file, create = true, write = true) do f + redirect_stdout(f) do + redirect_stderr(f) do + cd(dir) do + try + # Variable assignment + Main.eval(Meta.parse(vars)) + # src block evaluation + Main.eval(Meta.parse("begin $content end")) + catch e + println(e) + end + end + end + end + end + if !silently + # It's stupid to write and read it but I don't know how to + # save redirect_stdout to IOBuffer or similar + print(read(temporary_output_file, String)) + end + mv(temporary_output_file, output_file, force = true) + else + "ERROR: invalid ouput type" + end + return nothing +end + +OrgBabelReload() diff --git a/ob-julia.el b/ob-julia.el new file mode 100644 index 0000000..cdf8287 --- /dev/null +++ b/ob-julia.el @@ -0,0 +1,535 @@ +;;; ob-julia --- Org Mode babel support for julia, using ESS +;;; Commentary: +;; This package adds Julia support to Org Mode src block evaluation +;;; Code: +(require 'ob) +(require 'seq) +(eval-when-compile (require 'cl)) + +(defcustom org-babel-julia-command "julia" + "Name of command to use for executing julia code." + :group 'org-babel + :version "24.1" + :type 'string) + +(defcustom org-babel-julia-startup-script + (concat (file-name-directory (or load-file-name (buffer-file-name))) + "init.jl") + "Julia file path to run at startup. Must be absolute." + :group 'org-babel + :version "24.1" + :type 'string) + +(defcustom org-babel-julia-table-as-dict nil + "If t, tables are imported as Dictionary, else as NamedTuple. +In both cases, if you use DataFrames you can pass them to +`DataFrame'. +Importing NamedTuple is slower (more data) but they preserve the column order." + :group 'org-babel + :version "24.1" + :type 'boolean) + +(defcustom org-babel-julia-silent-repl nil + "Disable printing results in julia REPL. + +When non-nil, do not print org-src evaluation result in julia +session REPL. Since printing results require extra +compuatations, if you never look at the REPL setting this non-nil +this might be desired. +There's no effect in non-session evaluations" + :group 'org-babel + :version "24.1" + :type 'boolean) + +(defcustom org-babel-julia-debug nil + "Enable sending messages with debugging information." + :group 'org-babel + :version "24.1" + :type 'boolean) + +(defconst org-babel-header-args:julia + '((width . :any) + (height . :any) + (size . :any) + (inline . :any) + (import . :any) + (using . :any) + (async . :any) + (results . ((file + matrix table + list + ;; vector table scalar verbatim + ) + (raw html latex org + ;; code pp drawer + ) + (replace silent none append prepend) + (output value)))) + "Julia-specific header arguments.") + +(defvar org-babel-default-header-args:julia '()) +(defvar org-babel-julia-default-session "*julia*") + +(defvar ess-ask-for-ess-directory nil) ; dynamically scoped +(defvar org-babel-julia-session-directory) + +(defun org-babel-prep-session:julia (session params) + "Prepare SESSION according to the header arguments specified in PARAMS." + (let ((dir (or (cdr (assoc :dir params)) + (inferior-ess--maybe-prompt-startup-directory + org-babel-julia-command "julia")))) + (set (make-local-variable 'org-babel-julia-session-directory) dir) + (save-window-excursion + (require 'ess) + (julia) + (rename-buffer + (if (bufferp session) + (buffer-name session) + (if (stringp session) + session + (buffer-name)))) + ;; Register the async callback. Important to do this before + ;; running the command + (set-process-filter (get-buffer-process + (org-babel-comint-buffer-livep session)) + 'org-julia-async-process-filter) + ;; Initialization + (let ((julia-init + (with-temp-buffer + (insert-file-contents org-babel-julia-startup-script) + (buffer-string)))) + (ess-send-string (ess-get-process) julia-init nil)) + (current-buffer)))) + +(defun org-babel-julia-get-session-name (params) + "Extract the session name from the PARAMS. + +If session should not be used, return nil. + + session can be: + - (:session) :: param passed, empty, use default + - (:session name) :: param passed, with a name, use it + - (:session none) :: param not passed, do not use the session" + (let ((session (cdr (assoc :session params)))) + (cond + ((null session) org-babel-julia-default-session) + ((string-equal session "none") nil) + (t session)))) + +(defun org-julia-async-process-filter (process output) + "Replace julia-async: tags with async results. +Takes OUTPUT from PROCESS, tries to extract from the +ob_julia_async the `uuid' in the `org-mode' buffer name. Then, +searches for the `uuid' in the `org-mode' buffer, and replaces it +with the output file content. + +This function is used for all async processing with and without session." + (if (string-match "ob_julia_async_\\([0-9a-z\\-]+\\)_\\(.+\\)" output) + ;; capture ob-julia ouptut + (progn + (let ((uuid (match-string-no-properties 1 output)) + (org-buffer (match-string-no-properties 2 output)) + new-hash results params cache info) + (save-window-excursion + (save-excursion + (switch-to-buffer org-buffer) + (save-restriction + ;; If it's narrowed, substitution would fail + (widen) + ;; search the matching src block + (goto-char (point-max)) + (when (search-backward (concat "julia-async:" uuid) nil t) + ;; get output file name (stored in the buffer + (setq results + (let ((line (buffer-substring-no-properties + (line-beginning-position) + (line-end-position)))) + (when (string-match "julia-async:.+:\\([^\s]*\\)" + line) + (match-string-no-properties 1 line)))) + ;; remove results + (search-backward "#+end_src") + (setq info (org-babel-get-src-block-info 'light)) + ;; This will evaluate the code again + ;; (cl-callf org-babel-process-params (nth 2 info)) + (setq params (nth 2 info)) + (setq cache (let ((c (cdr (assq :cache params)))) + (and c (string= "yes" c)))) + ;; pass info to have a different hash + (setq new-hash (if cache (org-babel-sha1-hash) nil)) + (org-babel-remove-result) + ;; insert new one + (org-babel-insert-result + (org-babel-julia-process-results results params 'callback) + (cdr (assq :result-params params)) + info new-hash "julia"))))) + (inferior-ess-output-filter process "\n"))) + ;; This is the standard + (inferior-ess-output-filter process output))) + +(defun org-babel-julia-evaluate-external-process (block outfile params buffer) + "Evaluate julia SRC code, according to PARAMS. +Does not rely on an ESS session." + (let* ((uuid (org-id-uuid)) + (command (format + "%s;println(string(\"ob_julia_async_\", %S, \"_\", %S))" + block uuid buffer)) + (tmpfile (make-temp-file "ob-julia" nil ".jl" block))) + (if (and (org-babel-julia-async-p params) + (org-babel-julia-really-async-p)) + (progn + (make-process :name "*julia-async-process*" + :filter #'org-julia-async-process-filter + :command `(,org-babel-julia-command + "--load" ,org-babel-julia-startup-script + "--eval" + ,(format "include(%S);%s" tmpfile command))) + (concat "julia-async:" uuid ":" outfile)) + (progn + (shell-command + (format "%s --load %s %s" org-babel-julia-command + org-babel-julia-startup-script tmpfile)) + outfile)))) + +(defun org-babel-julia-assign-to-var-or-array (var) + "" + (if (listp (cdr var)) + (org-babel-julia-assign-to-array (car var) (cdr var)) + (org-babel-julia-assign-to-var (car var) (cdr var)))) + +(defun org-babel-julia-assign-to-array (name matrix) + "Create a Matrix (Vector{Any,2} from `MATRIX' and assign it to `NAME'" + (format "%s = [%s]" name + (mapconcat (lambda (line) (mapconcat (lambda (e) + (format "%s" e)) + line " ")) matrix ";"))) + +(defun org-babel-julia-assign-to-var (name value) + "Assign `VALUE' to a variable called `NAME'." + (format "%s = %S" name value)) + +(defun org-babel-julia-assign-to-dict (name column-names values) + "Create a Dict with lists as values. +Create a Dict where keys are Symbol from `COLUMN-NAMES', +values are Array taken from `VALUES', and assign it to `NAME'" + (format "%s = Dict(%s)" name + (mapconcat + (lambda (i) + (format ":%s => [%s]" (nth i column-names) + (mapconcat + (lambda (line) (format "%S" (nth i line))) + values + ","))) + (number-sequence 0 (1- (length column-names))) + ","))) + +(defun org-babel-julia-assign-to-named-tuple (name column-names values) + "Create a NamedTuple" + (format "%s = [%s]" name + (mapconcat + (lambda (i) + (concat + "(" (mapconcat + (lambda (j) + (format "%s=%S" + (nth j column-names) + (nth j (nth i values)))) + (number-sequence 0 (1- (length column-names))) + ",") + ")")) + (number-sequence 0 (1- (length values))) ", "))) + +(defun org-babel-variable-assignments:julia (params) + "Return list of julia statements assigning the block's variables." + (let ((vars (org-babel--get-vars params)) + (colnames (cdr (assoc :colname-names params)))) + (mapcar (lambda (i) + (let* ((var (nth i vars)) + (column-names + (car (seq-filter + (lambda (cols) + (eq (car cols) (car var))) + colnames)))) + (if column-names + (if org-babel-julia-table-as-dict + (org-babel-julia-assign-to-dict + (car var) (cdr column-names) (cdr var)) + (org-babel-julia-assign-to-named-tuple + (car var) (cdr column-names) (cdr var))) + (org-babel-julia-assign-to-var-or-array var)))) + (number-sequence 0 (1- (length vars)))))) + +(defun org-babel-julia-make-kwargs (args) + "" + (format "(%s)" (mapconcat (lambda (arg) + (format "%s=%s," + (car arg) + (cdr arg))) + (seq-filter (lambda (arg) (cdr arg)) args) ""))) + +(defun org-babel-julia-block-expand (params srcfile outfile) + "Takes BODY, apply required PARAMS and return the Julia code. + +OUTFILE and FILE can either be a string or nil. +If FILE is defined, output is _save()_d to a file with that name. +else OUTFILE is used, and data is _write()_ to it." + (let* ((vars (org-babel-variable-assignments:julia params)) + (dir (or (cdr (assoc :dir params)) default-directory)) + (using-param (cdr (assoc :using params))) + (using (if using-param (split-string using-param) nil)) + (import-param (cdr (assoc :import params))) + (import (if import-param (split-string import-param ";") nil)) + (result-type (cdr (assoc :result-type params))) + (output-type (case result-type (value ":value") (output ":output"))) + ;; kwargs + (size (cdr (assoc :size params))) + (width (cdr (assoc :width params))) + (height (cdr (assoc :height params)))) + (concat + (if (or using import) + (format "OrgBabelImport(%S);OrgBabelReload();" + (concat (if using (mapconcat (lambda (x) (concat "using " x)) + using "\n") "") + (if import (mapconcat (lambda (x) (concat "import " x)) + import "\n") ""))) + "") + (format + "OrgBabelFormat(%s,%S,%S,%S,%S,%s,%S);" + output-type outfile + dir + (mapconcat 'concat vars ";") srcfile + (if org-babel-julia-silent-repl + "true" "false") + (org-babel-julia-make-kwargs `((width . ,width) + (height . ,height) + (size . ,size))))))) + +(defun org-babel-execute:julia-async (buffer session body block output params) + (let* ((uuid (org-id-uuid)) + ;; The whole line must be printed in as single statement + ;; (ob_julia_async...) or you can receive only a portion of + ;; it. But cannot be joined together (else it will trigger + ;; immediately). That's why I'm using string(..) + (command + (format "%s;println(string(\"ob_julia_async_\", %S, \"_\", %S))" block + uuid buffer))) + (progn + (org-babel-remove-result) + (process-send-string session (concat command "\n")) + ;; store the command in input history! + (with-current-buffer session + (comint-add-to-input-history body))) + (concat "julia-async:" uuid ":" output))) + +(defun org-babel-execute:julia-sync (session body block output params) + "Run FILE, in session `SESSION`, synchronously. +PARAMS are passed" + (org-babel-comint-eval-invisibly-and-wait-for-file + session output block 0.1) + (with-current-buffer session + (comint-add-to-input-history body)) + output) + +(defun org-babel-julia-process-value-result (results type) + "Insert hline if needed (combining info from RESULT and TYPE." + ;; add an hline if the result seems to be a table + ;; always obay explicit type + (if (or (eq type 'table) + (and (eq type 'auto) + (listp results) ; a table must be a list + (listp (car results)) ; of lists + (stringp (caar results)))) ; with strings as first line + (cons (car results) (cons 'hline (cdr results))) + results)) + +(defun org-babel-julia-process-results (results params &optional callback) + "Decides what to insert as result. +If PARAMS is :async, insert a link, unless CALLBACK is true." + (let ((result-type (org-babel-julia-parse-result-type params)) + (file (cdr (assoc :file params))) + (inlined (org-babel-julia-get-inline-type params)) + (async (org-babel-julia-async-p params)) + (session (org-babel-julia-get-session-name params)) + (res (cdr (assoc :results params)))) + (if (and async + (not callback) + (org-babel-julia-really-async-p)) + results + (unless file ; do not process files + (when org-babel-julia-debug + (message (format "Processing results %s" results))) + (if inlined + (with-temp-buffer + (when (bound-and-true-p org-export-current-backend) + (insert (format "@@%s:" + (if org-export-current-backend + org-export-current-backend + inlined)))) + (insert-file-contents results) + (when (bound-and-true-p org-export-current-backend) + (goto-char (point-max)) + (insert "@@")) + (buffer-string)) + (org-babel-result-cond (if res (split-string res) nil) + (with-temp-buffer + (when org-babel-julia-debug (message res)) + (insert-file-contents results) + (buffer-string)) + (org-babel-julia-process-value-result + (org-babel-import-elisp-from-file results '(4)) + result-type))))))) + +(defun org-babel-julia-parse-result-type (params) + "Decide how to parse results. Default is \"auto\" +(results can be anything. If \"table\", force parsing as a +table. To force a matrix, use matrix" + (let* ((results (cdr (assoc :results params))) + (results (if (stringp results) (split-string results) nil))) + (cond + ((member "table" results) 'table) + ((member "matrix" results) 'matrix) + ((member "raw" results) 'raw) + (t 'auto)))) + +(defun org-babel-julia-async-p (params) + "Check whether the session should be async or not." + (let* ((res (cdr (assoc :results params))) + (async (assoc :async params))) + (and async + (or + (not (cdr async)) + (string= "t" (cdr async)) + (string= "yes" (cdr async))) + (not (and res (stringp res) (member "silent" (split-string res))))))) + +(defun org-babel-julia-really-async-p () + ;; (let* + ;; ((head (org-babel-where-is-src-block-head)) + ;; (async (and (not (bound-and-true-p org-export-current-backend)) + ;; head + ;; org-babel-current-src-block-location + ;; (equal org-babel-current-src-block-location + ;; head)))) + ;; async) + ;; Disable async on export + (not (bound-and-true-p org-export-current-backend))) + +;; Copied from ob-python +(defun org-babel-julia-with-earmuffs (session) + (let ((name (if (stringp session) session (format "%s" session)))) + (if (and (string= "*" (substring name 0 1)) + (string= "*" (substring name (- (length name) 1)))) + name + (format "*%s*" name)))) + +(defun org-babel-julia-get-inline-type (params) + "Parse the :inline header from PARAMS. +Returns t, nil or the output format." + (let ((inlined (assoc :inline params))) + (if inlined + (if (and + (cdr inlined) + (not (string= (cdr inlined) "no"))) + (cdr inlined) + (if (bound-and-true-p org-export-current-backend) + (format "%s" org-export-current-backend) + nil)) + nil))) + +;; (defun org-babel-execute:julia (body params) +;; "Execute a block of julia code. +;; This function is called by `org-babel-execute-src-block'. +;; BODY is the content of the src block +;; PARAMS are the parameter passed to the block" +;; ;; org-babel-current-src-block-location ; this variable does not work >.< +;; (save-excursion +;; (let* ((buffer (buffer-name)) +;; (session (org-babel-julia-get-session-name params)) +;; (async (org-babel-julia-async-p params)) +;; (file (cdr (assoc :file params))) +;; (inlined (org-babel-julia-get-inline-type params)) +;; (outfile (org-babel-process-file-name +;; (if file (concat default-directory file) +;; (org-babel-temp-file +;; "julia-" (if inlined (format ".%s" inlined) ""))))) +;; (src (make-temp-file "ob-julia" nil ".jl" body)) +;; (block (org-babel-julia-block-expand params src outfile))) +;; (when org-babel-julia-debug (message block)) +;; (if session +;; (progn +;; ;; TODO: check if session exists, if it does, make it like +;; ;; *session:$N* (where N is the first number available) +;; (setq session (org-babel-julia-with-earmuffs session)) +;; (when (not (org-babel-comint-buffer-livep session)) +;; (org-babel-prep-session:julia session params)) +;; (if (and async +;; (org-babel-julia-really-async-p)) +;; (progn +;; (when org-babel-julia-debug (message "async export")) +;; (org-babel-julia-process-results +;; (org-babel-execute:julia-async buffer session body +;; block outfile params) +;; params)) +;; (progn +;; (when org-babel-julia-debug (message "sync export")) +;; (org-babel-julia-process-results +;; (org-babel-execute:julia-sync session body block outfile +;; params) +;; params)))) +;; (let ((res (org-babel-julia-evaluate-external-process +;; block outfile params buffer))) +;; (if (and async (org-babel-julia-really-async-p)) +;; res +;; (org-babel-julia-process-results res params))))))) + +(defun org-babel-execute:julia (body params) + "Execute a block of julia code. +This function is called by `org-babel-execute-src-block'. +BODY is the content of the src block +PARAMS are the parameter passed to the block" + ;; org-babel-current-src-block-location ; this variable does not work >.< + (message (format "body: %s, params: %s" body params)) + (save-excursion + (let* ((buffer (buffer-name)) + (session (org-babel-julia-get-session-name params)) + (async (org-babel-julia-async-p params)) + (file (cdr (assoc :file params))) + (inlined (org-babel-julia-get-inline-type params)) + (outfile (org-babel-process-file-name + (if file (concat default-directory file) + (org-babel-temp-file + "julia-" (if inlined (format ".%s" inlined) ""))))) + (src (make-temp-file "ob-julia" nil ".jl" body)) + (block (org-babel-julia-block-expand params src outfile))) + (when org-babel-julia-debug (message block)) + (if session + (progn + ;; TODO: check if session exists, if it does, make it like + ;; *session:$N* (where N is the first number available) + (setq session (org-babel-julia-with-earmuffs session)) + (when (not (org-babel-comint-buffer-livep session)) + (org-babel-prep-session:julia session params)) + (if (and async + (org-babel-julia-really-async-p)) + (progn + (when org-babel-julia-debug (message "async export")) + (org-babel-julia-process-results + (org-babel-execute:julia-async buffer session body + block outfile params) + params)) + (progn + (when org-babel-julia-debug (message "sync export")) + (org-babel-julia-process-results + (org-babel-execute:julia-sync session body block outfile + params) + params)))) + (let ((res (org-babel-julia-evaluate-external-process + block outfile params buffer))) + (if (and async (org-babel-julia-really-async-p)) + res + (org-babel-julia-process-results res params))))))) + +(add-to-list 'org-babel-tangle-lang-exts '("julia" . "jl")) + +(provide 'ob-julia) +;;; ob-julia.el ends here diff --git a/readme.org b/readme.org new file mode 100644 index 0000000..04ab6c7 --- /dev/null +++ b/readme.org @@ -0,0 +1,551 @@ +#+title: ob-julia: high quality julia org-mode support +#+author: Nicolò Balzarotti +#+property: header-args:julia :exports both +#+html_head: + +* ob-julia + +See [[Implemented features]] for more details. + +* How it works + +1. Code block is saved to a temporary file (under /tmp) +2. Decide whether we need to start a new julia process or not + 1. If session is "none", don't use a session (code under /tmp will + be passed to =julia -L initfile src-block.jl=). This does not + require ess. The command is customized by + =org-babel-julia-command=) + 2. If session is nil, use default session name + (customized by =org-babel-julia-default-session=) + 3. If session has a values, use it's name +3. Check if we want to use a session or not. Check if the session + exists. Start the session accordingly. +4. Is the evaluation async? + 1. YES: + 1. Register a filter function + 2. Return immediately, printing a ref to the evaluation. + The ref is in the form: =julia-async:$uuid:$tmpfile= + - uuid: identifier of this evaluation, used to find out where to + insert again results + - tmpfile: the path where the output will be saved. This is + useful both for debugging purposes and so that we do not need + to store an object that maps computations to files. The + process-filter look for the uuid, and then inserts =$tmpfile=. + 2. NO: Run the code, wait for the results. Insert the results. + +* Implemented features + +** Session (=:session none=, =:session=, =:session session-name=) + +By default code is executed without a session. The advantage is that +you do not requires =emacs-ess= to run julia code with ob-julia. But +sessions (especially for julia, where speed comes from compiled code) +are available. The same behaviour is obtained by setting the =:session= +header to =none=. + +You can enable sessions with the =:session= argument. Without +parameters, the session used is named after +=org-babel-julia-default-session= (=*julia*= by default). With a +parameter, the name is earmuffed (a star is prepended and appended). + +The REPL is kept sane. There's no cluttering (you don't see all the +code executed that is required by ob-julia to have results), history +is preserved (you can ~C-S-up~ to see the list of org-src block +evaluated), and results are shown (this can be customized by +=org-babel-julia-silent-repl=). + +** Async (=:async= =:async yes=, =:async t=) + +Async works both with and without =:session=. + +The best combination of features is combining session with +=:async=. Async allows code evaluation in background (you can continue +using emacs while julia compute results). + +You can change buffer or even narrow it while the evaluation +completes: the result is added automatically as soon as julia +finishes. Multiple evaluation are queued automatically (thanks to +ess). Cache is supported (evaluating the same code-block twice does +not re-trigger evaluation, even if the code is still running). + +It's not possible to have async blocks with =:results silent=. I'm +currently using this to distinguish between active src block and +variable resolution (when a =:var= refer to an async block, the block +cannot be executed asynchonously. So we need to distinguish between +the two. This is the only way I was able to find, if you know better +please tell me). + +#+begin_src julia :session :async t +sleep(1) +"It works!" +#+end_src + +#+begin_src julia :session :async yes :cache yes +sleep(1) +"It works!" +#+end_src + +Here the same, without the session +#+begin_src julia :async +sleep(1) +"It works!" +#+end_src + +#+RESULTS: +: It works! + +#+begin_src julia :async :session +sleep(1) +"It works!" +#+end_src + +#+RESULTS: +: It works! + + +Asynchronous evaluation is automatically disabled on export, or when a +code block depends on one (=:var=) + +** Variables input (=:var=), Standard output + +*** Inputs + +Those are example inputs that will be used later, to check whether +import+export pipeline works as expected. + +A table + +#+name: table +| a | b | +|---+---| +| 1 | 1 | +| 2 | 2 | +| | | +| 4 | 4 | + +A matrix (no hline) + +#+name: matrix +| 1 | 2 | 3 | 4 | +| 1 | 2 | 3 | 4 | + +A column + +#+name: column +| 1 | +| 2 | +| 3 | +| 4 | + +A row + +#+name: row +| 1 | 2 | 3 | 4 | + +A list + +#+name: list +- 1 +- 2 +- 3 +- 4 + +**** Table + +#+begin_src julia :session *julia-test-variables* :var table=table +table +#+end_src + +As you can see, the table automatically adds the hline after the +header. This is a heuristic that might fail (might be triggered for +matrix, might not trigger on tables), so you can manually +force/disable it with the =:results table= or =:results matrix= param. + +#+begin_src julia :session *julia-test-variables* :var table=table :async :results matrix +table +#+end_src + +**** Row + +Column, Rows, and Matrix export works just fine (tests in session sync, session async +and without session). + +#+name: sync-row +#+begin_src julia :session *julia-test-variables* :var row=row +row +#+end_src + +#+name: async-row +#+begin_src julia :session *julia-test-variables* :var row=row :async +row +#+end_src + +#+name: no-session-row +#+begin_src julia :var row=row :async +row +#+end_src + +**** Column + +Works both with synchronous evaluation + +#+name: sync-column +#+begin_src julia :session *julia-test-variables* :var column=column +column +#+end_src + +asynchronous evaluation + +#+name: async-column +#+begin_src julia :session *julia-test-variables* :var column=column :async +column +#+end_src + +and without a session + +#+name: no-session-column +#+begin_src julia :var column=column +column +#+end_src + +**** Matrix + +Sync + +#+name: sync-matrix +#+begin_src julia :session *julia-test-variables* :var matrix=matrix +matrix +#+end_src + +Just like for tables, you can control header hline line with the +results param. + +#+begin_src julia :session *julia-test-variables* :var matrix=matrix :results table +matrix +#+end_src + +Async + +#+name: async-matrix +#+begin_src julia :session *julia-test-variables* :var matrix=matrix :async +matrix +#+end_src + +No session + +#+name: no-session-matrix +#+begin_src julia :var matrix=matrix :results table :async +matrix +#+end_src + +**** List + +List are parsed as columns + +#+begin_src emacs-lisp :var list=list +list +#+end_src + +=:results list= return the list (just like R). It's not perfect with +#+begin_src julia :var list=list :async :results list +list +#+end_src + +**** Table + +There are two ways in which tables can be passed to Julia: +- Array{NamedTuple} +- Dictionary + +I like the NamedTuple approach, but if you don't like it you can +customize the variable =org-babel-julia-table-as-dict=. In both cases, +if you [[id:5a0042fc-1cf2-4f11-823f-658e30776931][:import]] DataFrames, you can construct a DataFrame from both. + +TOOD: I miss the julia code for printing Array{NamedTuple}. + +#+begin_src julia :var table=table :async :session last +table +#+end_src + +Also, it's nice that a single NamedTuple can represent a table: +#+begin_src julia :var table=table :async :session last +table[2] +#+end_src + +** Directory (=:dir=) + +Each source block is evaluated in it's :dir param + +#+begin_src julia :session *julia-test-change-dir* :dir "/tmp" +pwd() +#+end_src + +#+begin_src julia :session *julia-test-change-dir* :dir "/" +pwd() +#+end_src + +If unspecified, the directory is session's one +#+begin_src julia :session *julia-test-change-dir* +pwd() +#+end_src + +Changing dir from julia code still works +#+begin_src julia :session *julia-test-change-dir* +cd("/") +realpath(".") +#+end_src + +but is ephemeral (like fort the =:dir= param) +#+begin_src julia :session *julia-test-change-dir* +realpath(".") +#+end_src + +This is obtained by wrapping the src block in a =cd()= call: +#+begin_src julia :eval never :exports code +cd(folder) do + block +end +#+end_src + +** Error management + +If the block errors out, + +#+name: undef-variable +#+begin_src julia :session julia-error-handling +x +#+end_src + +#+name: method-error +#+begin_src julia :session julia-error-handling +1 + "ciao" +#+end_src + +#+RESULTS: +| ERROR | MethodError(+ | (1 | ciao) | 0x0000000000006420) | +|-------+---------------+----+-------+---------------------| + +It works in async +#+begin_src julia :session julia-error-handling :async +x +#+end_src + +On external process (sync) +#+begin_src julia :async +x +#+end_src + +and on external process (async) +#+begin_src julia :async +x +#+end_src + +Error management can still be improved for helping with debug (see +scimax). + +** Using (=:using=) and Import (=:import=) +:PROPERTIES: +:ID: 5a0042fc-1cf2-4f11-823f-658e30776931 +:END: + +To include dependencies, you can use =:using= and =:import=. + +Because of how the julia code is actually implemented, in order to use +specialized exports (e.g., DataFrames, see ) you need the +modules to be available _before_ the block gets evaluated. The problem +can be solved in 2 (or 3 ways): +- Evaluating a block with using/import, then the other block +- Using the header arguments +- Fixing the Julia code :D + +to use =:import=, you need to pass arguments quoted: +#+begin_example +:using DataFrames Query :import "FileIO: load" "Plots: plot" +#+end_example + +** Results (=:results output=, =:results file=, ) + +The default is to return the value: + +#+begin_src julia :async :results value :session julia-results +10 +#+end_src + +If results is output, it's included the stdout (what's printed in the +terminal). (This part still needs some work to be useful.) + +#+begin_src julia :async :results output :session julia-results +10 +#+end_src + +#+begin_src julia :async :results output :session julia-results +println(10) +#+end_src + +#+begin_src julia :async :results output :session julia-results +println("a") + +"10" + +println("b") +#+end_src + +Error (results output) + +#+begin_src julia :session error-output :results output +This will throw an error +#+end_src + +Another error (result ouptut) +#+begin_src julia :session error-output :results output +print(one(3,3)) +#+end_src + +A matrix +#+begin_src julia :session :results output +print(ones(3,3)) +#+end_src + +** Supported Types +:PROPERTIES: +:ID: 99d2531c-9810-4069-94a3-ac8bca9f6c23 +:END: + +Adding new types is easy (you need to define an =orgshow()= function for +your type. See [[file+emacs:init.jl::orgshow][init.jl]]). There's a simple mechanism that allows to +define method on non-yet-existing types [[file+emacs:init.jl::function%20define_][example]]. + +The current version supports a couple of useful type: DataFrames and +Plots. ob-julia needs community support to add more types: please help! + +** File output & Inlining + +There's native support for writing output to file. For unkown file +types, instead of inserting the output in the buffer it's written to the file. + +#+begin_src julia :session :file readme/output.html +zeros(3,3) +#+end_src + +#+begin_src julia :session :file readme/output.csv :async +zeros(3,3) +#+end_src + +#+begin_src julia :session :file readme/output.csv :async +Dict(10 => 10) +#+end_src + +Saving plots requires the Plots library. You can require it with the +=:using= [[id:5a0042fc-1cf2-4f11-823f-658e30776931][header]]. There's the custom =:size "tuple"= header argument for +specifying the output size. It _must_ be placed inside parentheses, and +it's evaluated as julia object (that means it can contains variables +and expressions). + +#+begin_src julia :session :file readme/output-plot.png :async :using Plots :var matrix=matrix :size "let unit = 200; (unit, 2unit); end" +plot(matrix) +#+end_src + +Matrix also has an automatic conversion (when Plots is loaded), so you +don't even need to pass it to the =plot()= function (there's a generic +fallback that tries to plot anything saved to png or svg). + +#+begin_src julia :session :file readme/output-matrix.svg :async :using Plots :var matrix=matrix +matrix +#+end_src + +Plots can also manage errors (in a visually-disturbing way). + +#+begin_src julia :session :file readme/output-undef.svg :using Plots +this_is_undefined +#+end_src + +#+begin_src julia :session :file readme/output-undef.png :using Plots :async +another_undefined_but_async +#+end_src + +#+name: dataframe +#+begin_src julia :session :using DataFrames :async +DataFrame(x = 1:10, y = (0:9) .+ 'a') +#+end_src + +#+begin_src julia :session :async :using DataFrames :var data=dataframe table=table :file readme/output-table.csv +DataFrame(table) +#+end_src + +#+begin_src julia :session :async :using DataFrames :var data=dataframe +data +#+end_src + +*** Inlining (=:inline no=, =:inline=, =:inline format=) +Output to file and Inlining are different things but nicely fit +together to solve the same problem: inserting images or other objects +inside the buffer. + +If your type can be represented inside the exported format (like +images as svg/base-64 encoded pngs inside =.html=, tex plots in a =.tex= +file), the =:inline= header is what you need. The behaviour changes +based depending on your interactive use and on the desired output +format. + +TODO: right now :results raw is required on export. How do we fix it? + +Examples: :inline keyword alone, in interactive evaluation, the output +inserted in the buffer is the usual. +#+begin_src julia :session :inline :async :var matrix=matrix :results raw +matrix +#+end_src + +But when you export the output to html, the output will be processed +_by julia_, and inserted in the buffer (or a different representation +for different export formats). This is not of much use with tables +(even if you can customize the export, e.g. by passing the :width +keyword), but is wonderful for pictures. If a result can be inserted +in multiple ways (picture in html can be both inline png or svg), you +can specify the desired format by passing the argument to the :inline +keyword (like =:inline svg=). In this case, the processed output is +inserted also in interactive sessions. + +# Here we should fix the way we escape the params +#+begin_src julia :session :inline html :async :var matrix=matrix :results raw :width 10 +matrix +#+end_src + +Plots default to inline png + +#+begin_src julia :session :inline :var matrix=matrix :using Plots :async :results raw +plot(matrix) +#+end_src + + +But you can also force svg (Since it's multiline, :wrap it with =:wrap html=) + +#+begin_src julia :session :inline svg :results raw :async +plot(matrix) +#+end_src + + +* Issues and Missing features + +- No automated tests yet +- Not tested with remote host +- Variable resolution of src-block is synchronous. If your :async src + block depends on another :async src block, the latter is evaluated + synchronously, then former asynchonously. This could be implemented + by using a simple queue, where next item is started in the + process-filter and where variables starting with julia-async: are + replaced. Right now I don't feel the need (if results are cached, + things already feels good). +- For async evaluation to work the session must be started from + ob-julia (or you must register the filter function manually, + undocumented). +- =:results output= is implemented but untested. I rarely find it + useful. +- import/using errors are not reported + +* Credits + +This project originally started as a fork of +[[https://github.com/astahlman/ob-async/][astahlman/ob-async]]. However, because of changes in new version of Org +Mode, julia 1.0 release and unsupported features, I decided to start +over. +