Merge pull request #481 from dlfivefifty/patch-2

Mention Quarto in Related Packages

Project.toml

@@ -1,6 +1,6 @@
 name = "Weave"
 uuid = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9"
-version = "0.10.6"
+version = "0.10.12"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -12,15 +12,17 @@ Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
 [compat]
-Highlights = "0.3.1, 0.4"
+Highlights = "0.3.1, 0.4, 0.5"
 JSON = "0.21"
 Mustache = "0.4.1, 0.5, 1"
 Plots = "0.28, 0.29, 1.0"
+RelocatableFolders = "0.1,0.2,0.3,1"
 Requires = "1.0"
 YAML = "0.3, 0.4"
 julia = "1.2"

README.md

@@ -97,3 +97,8 @@ Thanks for the important additions, fixes and comments.
 
 - [DiffEqTutorials.jl](https://github.com/JuliaDiffEq/DiffEqTutorials.jl) uses Weave to output tutorials (`.jmd` documents) to html, pdf and Jupyter notebooks.
 - [TuringTutorials](https://github.com/TuringLang/TuringTutorials) uses Weave to convert notebooks to html.
+
+## Related packages
+
+- [Literate.jl](https://github.com/fredrikekre/Literate.jl) can be used to generate Markdown and Jupyter notebooks directly from Julia source files with markdown in comments.
+- [Quarto](https://quarto.org) can generate Jupyter notebooks, HTML, or PDF directly from a Markdown format containing Julia code blocks, and also works with R and Python.

doc/src/usage.md

@@ -130,7 +130,7 @@ e.g. to include a Plots figure in markdown you can use:
 ```
 or to produce any HTML output:
 ```
-! display("text/html", "Header from julia");
+! display("text/html", HTML("Header from julia"));
 ```
 
 ### Script Format

src/Weave.jl

@@ -1,14 +1,13 @@
 module Weave
 
-using Highlights, Mustache, Requires, Pkg, REPL
-
+using Highlights, Mustache, Requires, Pkg, REPL, RelocatableFolders, Base64
 
 # directories
 const PKG_DIR = normpath(@__DIR__, "..")
-const TEMPLATE_DIR = normpath(PKG_DIR, "templates")
-const STYLESHEET_DIR = normpath(PKG_DIR, "stylesheets")
+const TEMPLATE_DIR = @path joinpath(PKG_DIR, "templates")
+const STYLESHEET_DIR = @path joinpath(PKG_DIR, "stylesheets")
 # keeps paths of sample documents for easy try
-const EXAMPLE_FOLDER = normpath(PKG_DIR, "examples")
+const EXAMPLE_FOLDER = @path joinpath(PKG_DIR, "examples")
 
 # constant names
 const WEAVE_OPTION_NAME = "weave_options"
@@ -244,8 +243,16 @@ function specific_options!(weave_options, doctype)
 end
 
 get_out_path(doc, out_path, ext::Nothing = nothing) = get_out_path(doc, out_path, doc.format.extension)
-get_out_path(doc, out_path, ext) = abspath(get_cwd(doc, out_path), string(doc.basename , '.', ext))
-
+function get_out_path(doc, out_path, ext)
+    if (out_path === :doc) || (out_path === :pwd)
+        abspath(get_cwd(doc, out_path), string(doc.basename, '.', ext))
+    elseif isempty(splitext(out_path)[2]) # directory given
+        abspath(get_cwd(doc, out_path), string(doc.basename, '.', ext))
+    else
+        # out_path is given, but if extension is explitly provided override this will override the extension
+        abspath(string(splitext(out_path)[1], '.', ext))
+    end
+end
 """
     notebook(source::AbstractString; kwargs...)
 

src/WeaveMarkdown/markdown.jl

@@ -5,9 +5,11 @@ using ..Weave: isnothing, take2string!
 using Markdown
 import Markdown: @trigger, @breaking, Code, MD, withstream, startswith, LaTeX
 
-# Note that this definition causes a "Method overwritten" warning,
-# but defining this function in __init__() is not legal in julia v1.5
-function Markdown.latex(io::IO, tex::Markdown.LaTeX)
+# HACK: that this definition is type-piracy. It is required since `Markdown`
+# does not have a built in system for contextual rendering by users. `io` here
+# should always be either `IOBuffer` or `IOContext` since it is reached via
+# `sprint` in all user-facing code paths in `Markdown`.
+function Markdown.latex(io::Union{IOBuffer,IOContext}, tex::Markdown.LaTeX)
     math_envs = ["align", "equation", "eqnarray"]
     use_dollars =
         !any([occursin("\\begin{$me", tex.formula) for me in math_envs])

src/plots.jl

@@ -1,6 +1,6 @@
 module WeavePlots
 
-using Base64, ..Plots, ..Weave
+using ..Base64, ..Plots, ..Weave
 
 
 # Pre-execute hooks to set the plot size for the chunk

src/reader/markdown.jl

@@ -5,7 +5,7 @@ function parse_markdown(document_body; is_pandoc = false)
         r"^<<(?<options>.*?)>>=\s*$",
         r"^@\s*$"
     else
-        r"^[`~]{3}(\{?)julia\s*([;,]?)\s*(?<options>.*?)(\}|\s*)$",
+        r"^[`~]{3}(\{?)julia\s*([;,\{]?)\s*(?<options>.*?)(\}|\s*)$",
         r"^[`~]{3}\s*$"
     end
     return header, parse_markdown_body(document_body, code_start, code_end, offset)

src/rendering/exportformat.jl

@@ -0,0 +1,24 @@
+abstract type ExportFormat <: WeaveFormat end
+
+function Base.getproperty(sf::T, s::Symbol) where {T<:ExportFormat}
+    hasfield(T, s) && return getfield(sf, s)
+    return getproperty(sf.primaryformat, s)
+end
+function Base.setproperty!(sf::T, s::Symbol, v) where {T<:ExportFormat}
+    if hasfield(T, s)
+        setfield!(sf, s, v)
+    else
+        setproperty!(sf.primaryformat, s, v)
+    end
+end
+function Base.hasproperty(sf::T, s::Symbol) where {T<:ExportFormat}
+    hasfield(T, s) || hasproperty(sf.primaryformat, s)
+end
+
+render_doc(df::ExportFormat, body, doc) = render_doc(df.primaryformat, body, doc)
+
+render_chunk(df::ExportFormat, chunk) = render_chunk(df.primaryformat, chunk)
+# Need to define these to avoid ambiguities
+render_chunk(df::ExportFormat, chunk::DocChunk) = render_chunk(df.primaryformat, chunk)
+render_chunk(df::ExportFormat, chunk::CodeChunk) = render_chunk(df.primaryformat, chunk)
+render_output(df::ExportFormat, output) = render_output(df.primaryformat, output)

src/rendering/htmlformats.jl

@@ -124,23 +124,9 @@ end
 # Pandoc
 # ------
 
-Base.@kwdef mutable struct Pandoc2HTML <: HTMLFormat
+Base.@kwdef mutable struct Pandoc2HTML <: ExportFormat
     description = "HTML via intermediate Pandoc Markdown (requires Pandoc 2)"
-    extension = "md"
-    codestart = '\n'
-    codeend = '\n'
-    termstart = codestart
-    termend = codeend
-    outputstart = '\n'
-    outputend = '\n'
-    mimetypes = ["image/png", "image/svg+xml", "image/jpg", "text/html", "text/markdown", "text/plain"]
-    fig_ext = ".png"
-    out_width = nothing
-    out_height = nothing
-    fig_pos = nothing
-    fig_env = nothing
-    # specials
-    preserve_header = true
+    primaryformat = Pandoc()
     template_path = nothing
     stylesheet_path = nothing
     highlight_theme = nothing

src/rendering/pandocformats.jl

@@ -54,28 +54,14 @@ Base.@kwdef mutable struct Pandoc <: PandocFormat
 end
 register_format!("pandoc", Pandoc())
 
+
 const DEFAULT_PANDOC_OPTIONS = String[]
 
-Base.@kwdef mutable struct Pandoc2PDF <: PandocFormat
+Base.@kwdef mutable struct Pandoc2PDF <: ExportFormat
     description = "PDF via intermediate Pandoc Markdown"
-    extension = "md"
-    codestart = "~~~~{.julia}"
-    codeend = "~~~~~~~~~~~~~\n"
-    termstart = codestart
-    termend = codeend
-    outputstart = "~~~~"
-    outputend = "~~~~\n\n"
-    # Prefer png figures for markdown conversion, svg doesn't work with latex
-    mimetypes = ["image/png", "image/jpg", "image/svg+xml", "text/markdown", "text/plain"]
-    fig_ext = ".png"
-    out_width = nothing
-    out_height = nothing
-    fig_pos = nothing
-    fig_env = nothing
-    # specials
-    preserve_header = true
-    header_template = normpath(TEMPLATE_DIR, "pandoc2pdf_header.txt")
+    primaryformat = Pandoc()
     pandoc_options = DEFAULT_PANDOC_OPTIONS
+    header_template = normpath(TEMPLATE_DIR, "pandoc2pdf_header.txt")
 end
 register_format!("pandoc2pdf", Pandoc2PDF())
 

src/rendering/rendering.jl

@@ -23,9 +23,9 @@ function render_doc(doc::WeaveDoc)
     return render_doc(docformat, body, doc)
 end
 
-
+include("exportformat.jl")
 include("common.jl")
+include("pandocformats.jl")
 include("htmlformats.jl")
 include("texformats.jl")
-include("pandocformats.jl")
 include("miscformats.jl")

src/rendering/texformats.jl

@@ -34,7 +34,10 @@ render_output(docformat::LaTeXFormat, output) = unicode2latex(docformat, output,
 
 render_code(docformat::LaTeXFormat, code) = unicode2latex(docformat, code, true)
 
-render_termchunk(docformat::LaTeXFormat, chunk) = string(docformat.termstart, chunk.output, docformat.termend, "\n")
+render_termchunk(docformat::LaTeXFormat, chunk) =
+    string(docformat.termstart,
+            unicode2latex(docformat, chunk.output, true),
+            docformat.termend, "\n")
 
 # from julia symbols (e.g. "\bfhoge") to valid latex
 const UNICODE2LATEX = let
@@ -149,11 +152,11 @@ end
 Base.@kwdef mutable struct LaTeXMinted <: LaTeXFormat
     description = "LaTeX using minted package for code highlighting"
     extension = "tex"
-    codestart = "\\begin{minted}[escapeinside=||, mathescape, fontsize=\\small, xleftmargin=0.5em]{julia}"
+    codestart = "\\begin{minted}[texcomments = true, mathescape, fontsize=\\small, xleftmargin=0.5em]{julia}"
     codeend = "\\end{minted}"
-    termstart = "\\begin{minted}[escapeinside=||, mathescape, fontsize=\\footnotesize, xleftmargin=0.5em]{jlcon}"
+    termstart = "\\begin{minted}[texcomments = true, mathescape, fontsize=\\footnotesize, xleftmargin=0.5em]{jlcon}"
     termend = "\\end{minted}"
-    outputstart = "\\begin{minted}[escapeinside=||, mathescape, fontsize=\\small, xleftmargin=0.5em, frame = leftline]{text}"
+    outputstart = "\\begin{minted}[texcomments = true, mathescape, fontsize=\\small, xleftmargin=0.5em, frame = leftline]{text}"
     outputend = "\\end{minted}"
     mimetypes = ["application/pdf", "image/png", "text/latex", "text/plain"]
     fig_ext = ".pdf"
@@ -198,8 +201,14 @@ function render_code(docformat::WeaveLaTeXFormat, code)
     unicode2latex(docformat, ret, false)
 end
 
-render_termchunk(docformat::WeaveLaTeXFormat, chunk) =
-    should_render(chunk) ? highlight_term(MIME("text/latex"), chunk.output, docformat.highlight_theme) : ""
+function render_termchunk(docformat::WeaveLaTeXFormat, chunk)
+    if should_render(chunk)
+        ret = highlight_term(MIME("text/latex"), chunk.output, docformat.highlight_theme)
+        unicode2latex(docformat, ret, true)
+    else
+        ""
+    end
+end
 
 function render_doc(docformat::WeaveLaTeXFormat, body, doc)
     return Mustache.render(
@@ -233,44 +242,23 @@ Base.@kwdef mutable struct WeaveLaTeX <: WeaveLaTeXFormat
     tex_deps = ""
     # how to escape latex in verbatim/code environment
     escape_starter = "(*@"
-    escape_closer = reverse(escape_starter)
+    escape_closer = "@*)"
 end
 register_format!("md2tex", WeaveLaTeX())
 
 # will be used by `write_doc`
 const DEFAULT_LATEX_CMD = ["xelatex", "-shell-escape", "-halt-on-error"]
 
-Base.@kwdef mutable struct WeaveLaTeX2PDF <: WeaveLaTeXFormat
-    description = "PDF via Weave-styled LaTeX"
-    extension = "tex"
-    codestart = ""
-    codeend = ""
-    termstart = codestart
-    termend = codeend
-    outputstart = "\\begin{lstlisting}"
-    outputend = "\\end{lstlisting}\n"
-    mimetypes = ["application/pdf", "image/png", "image/jpg", "text/latex", "text/markdown", "text/plain"]
-    fig_ext = ".pdf"
-    out_width = "\\linewidth"
-    out_height = nothing
-    fig_pos = nothing
-    fig_env = nothing
-    # specials
-    highlight_theme = nothing
-    template = nothing
-    keep_unicode = false
-    tex_deps = ""
-    latex_cmd = DEFAULT_LATEX_CMD
-    # how to escape latex in verbatim/code environment
-    escape_starter = "(*@"
-    escape_closer = reverse(escape_starter)
-end
-register_format!("md2pdf", WeaveLaTeX2PDF())
 
-function set_format_options!(docformat::WeaveLaTeX2PDF; template = nothing, highlight_theme = nothing, keep_unicode = false, latex_cmd = DEFAULT_LATEX_CMD, _kwargs...)
-    docformat.template =
-        get_mustache_template(isnothing(template) ? normpath(TEMPLATE_DIR, "md2pdf.tpl") : template)
-    docformat.highlight_theme = get_highlight_theme(highlight_theme)
-    docformat.keep_unicode |= keep_unicode
-    docformat.latex_cmd = latex_cmd
+Base.@kwdef mutable struct LaTeX2PDF <: ExportFormat
+    primaryformat = WeaveLaTeX()
+    description = "PDF via LaTeX"
+    latex_cmd = DEFAULT_LATEX_CMD
+end
+register_format!("md2pdf", LaTeX2PDF())
+register_format!("minted2pdf", LaTeX2PDF(primaryformat=LaTeXMinted()))
+
+function set_format_options!(docformat::LaTeX2PDF; latex_cmd = DEFAULT_LATEX_CMD, _kwargs...)
+    docformat.latex_cmd = latex_cmd
+    set_format_options!(docformat.primaryformat; _kwargs...)
 end

src/run.jl

@@ -39,7 +39,7 @@ function run_doc(
 
     # New sandbox for each document with args exposed
     isnothing(mod) && (mod = sandbox = Core.eval(Main, :(module $(gensym(:WeaveSandBox)) end))::Module)
-    Core.eval(mod, :(const WEAVE_ARGS = $(args)))
+    Core.eval(mod, :(WEAVE_ARGS = $(args)))
 
     mimetypes = doc.format.mimetypes
 
@@ -90,6 +90,8 @@ function run_doc(
         @info "Weaved all chunks" progress=1 _id=PROGRESS_ID
         cd_back()
         popdisplay(report) # ensure display pops out even if internal error occurs
+        # Temporary fig_path is not automatically removed because it contains files so...
+        !isnothing(fig_path) && startswith(fig_path, "jl_") && rm(normpath(cwd, fig_path), force=true, recursive=true)
     end
 
     return doc
@@ -140,27 +142,17 @@ function embed_figures!(chunk::CodeChunk, cwd)
         chunk.figures[i] = img2base64(fig, cwd)
     end
 end
-
-function embed_figures!(chunks::Vector{CodeChunk}, cwd)
-    for chunk in chunks
-        embed_figures!(chunk, cwd)
-    end
-end
+embed_figures!(chunks, cwd) = embed_figures!.(chunks, Ref(cwd))
 
 function img2base64(fig, cwd)
     ext = splitext(fig)[2]
     f = open(joinpath(cwd, fig), "r")
     raw = read(f)
     close(f)
-    if ext == ".png"
-        return "data:image/png;base64," * stringmime(MIME("image/png"), raw)
-    elseif ext == ".svg"
-        return "data:image/svg+xml;base64," * stringmime(MIME("image/svg"), raw)
-    elseif ext == ".gif"
-        return "data:image/gif;base64," * stringmime(MIME("image/gif"), raw)
-    else
-        return (fig)
-    end
+    return ext == ".png" ? "data:image/png;base64," * stringmime(MIME("image/png"), raw) :
+           ext == ".svg" ? "data:image/svg+xml;base64," * stringmime(MIME("image/svg"), raw) :
+           ext == ".gif" ? "data:image/gif;base64," * stringmime(MIME("image/gif"), raw) :
+           fig
 end
 
 function run_chunk(chunk::DocChunk, doc, report, mod)
@@ -274,15 +266,9 @@ function eval_chunk(doc::WeaveDoc, chunk::CodeChunk, report::Report, mod::Module
 
     execute_posthooks!(chunk)
 
-    chunks = if chunk.options[:term]
-        collect_term_results(chunk)
-    elseif chunk.options[:hold]
-        collect_hold_results(chunk)
-    else
-        collect_results(chunk)
-    end
-
-    return chunks
+    return chunk.options[:term] ? collect_term_results(chunk) :
+           chunk.options[:hold] ? collect_hold_results(chunk) :
+           collect_results(chunk)
 end
 
 # Hooks to run before and after chunks, this is form IJulia,
@@ -293,7 +279,11 @@ function pop_preexecution_hook!(f::Function)
     isnothing(i) && error("this function has not been registered in the pre-execution hook yet")
     return splice!(preexecution_hooks, i)
 end
-execute_prehooks!(chunk::CodeChunk) = for prehook in preexecution_hooks; Base.invokelatest(prehook, chunk); end
+function execute_prehooks!(chunk::CodeChunk)
+    for prehook in preexecution_hooks
+        Base.invokelatest(prehook, chunk)
+    end
+end
 
 const postexecution_hooks = Function[]
 push_postexecution_hook!(f::Function) = push!(postexecution_hooks, f)
@@ -302,7 +292,11 @@ function pop_postexecution_hook!(f::Function)
     isnothing(i) && error("this function has not been registered in the post-execution hook yet")
     return splice!(postexecution_hooks, i)
 end
-execute_posthooks!(chunk::CodeChunk) = for posthook in postexecution_hooks; Base.invokelatest(posthook, chunk); end
+function execute_posthooks!(chunk::CodeChunk)
+    for posthook in postexecution_hooks
+        Base.invokelatest(posthook, chunk)
+    end
+end
 
 """
     clear_module!(mod::Module)
@@ -341,11 +335,7 @@ function get_figname(report::Report, chunk; fignum = nothing, ext = nothing)
 end
 
 function set_rc_params(doc::WeaveDoc, fig_path, fig_ext)
-    if isnothing(fig_ext)
-        doc.chunk_defaults[:fig_ext] = doc.format.fig_ext
-    else
-        doc.chunk_defaults[:fig_ext] = fig_ext
-    end
+    doc.chunk_defaults[:fig_ext] = isnothing(fig_ext) ? doc.format.fig_ext : fig_ext
     doc.chunk_defaults[:fig_path] = fig_path
 end
 
@@ -363,9 +353,9 @@ function collect_results(chunk::CodeChunk)
                 chunk.optionstring,
                 copy(chunk.options),
             )
-            rchunk.figures = r.figures
             rchunk.output = r.stdout
             rchunk.rich_output = r.rich_output
+            rchunk.figures = r.figures
             push!(result_chunks, rchunk)
             content = ""
         end

src/writer/latex.jl

@@ -1,4 +1,4 @@
-function write_doc(docformat::WeaveLaTeX2PDF, doc, rendered, out_path)
+function write_doc(docformat::LaTeX2PDF, doc, rendered, out_path)
     cd_back = let d = pwd(); () -> cd(d); end
     cd(doc.cwd)
     try

src/writer/pandoc.jl

@@ -61,7 +61,7 @@ function write_doc(docformat::Pandoc2PDF, doc, rendered, out_path)
         cmd = `pandoc -f markdown+raw_tex -s  --pdf-engine=xelatex --highlight-style=tango
          $filt $citeproc $(docformat.pandoc_options)
          --include-in-header=$(docformat.header_template)
-         -V fontsize=12pt -o $(out)`
+         -o $(out)`
         proc = open(cmd, "r+")
         println(proc.in, rendered)
         close(proc.in)

test/end2end/test_simple.jl

@@ -3,7 +3,7 @@ using Weave.Dates
 
 test_doctypes = filter(first.(Weave.list_out_formats())) do doctype
     # don't test doctypes which need external programs
-    doctype ∉ ("pandoc2html", "pandoc2pdf", "md2pdf")
+    doctype ∉ ("pandoc2html", "pandoc2pdf", "md2pdf", "minted2pdf")
 end
 
 function test_func(body)
@@ -24,7 +24,6 @@ using Dates
 Date(now())
 ```
 """
-
 for doctype in test_doctypes
     test_mock_weave(test_func, julia_markdown_body; informat = "markdown", doctype = doctype)
 end

test/reader/test_chunk_options.jl

@@ -72,6 +72,10 @@ let opts = get_options("```julia, opt1 = 1, opt2 = \"2\"\n```")
     @test (:opt1 => 1) in opts
     @test (:opt2 => "2") in opts
 end
+let opts = get_options("```julia{opt1 = 1, opt2 = \"2\"}\n```")
+    @test (:opt1 => 1) in opts
+    @test (:opt2 => "2") in opts
+end
 
 end
 

test/run/test_module.jl

@@ -46,4 +46,14 @@ Core.eval(m, :(const a = $ary))
 Weave.clear_module!(m)
 @test_broken Base.summarysize(m) < size
 
+julia_markdown_body = """
+this is just to test the `out_path` option
+"""
+
+f_in = tempname()
+f_out = tempname() * ".md"
+write(f_in, julia_markdown_body)
+f = weave(f_in; out_path=f_out)
+@test isfile(f_out)
+
 end # @testset "clear_module!"

test/test_display.jl

@@ -6,18 +6,18 @@
 doc = mock_run("""
 ```julia
 using DataFrames
-DataFrame(rand(10,3))
+DataFrame(a=rand(10))
 ```
 """; doctype = "md2html")
 @test isdefined(doc.chunks[1], :rich_output)
-@test count("<tr>", doc.chunks[1].rich_output) == 12 # additonal 2 for name and type row
+@test count("<tr", doc.chunks[1].rich_output) == 12 # additonal 2 for name and type row
 
 # limit
 n = 100000
 doc = mock_run("""
 ```julia
 using DataFrames
-DataFrame(rand($n,3))
+DataFrame(a=rand($n))
 ```
 """; doctype = "md2html")
 @test isdefined(doc.chunks[1], :rich_output)