Merge pull request #362 from JonasIsensee/texpipeline

Improvements to the tex rendering pipeline

src/Weave.jl

@@ -1,6 +1,6 @@
 module Weave
 
-using Highlights, Mustache, Requires, Pkg
+using Highlights, Mustache, Requires, Pkg, REPL
 
 
 # directories
@@ -32,6 +32,8 @@ end
 take2string!(io) = String(take!(io))
 joinlines(lines) = join(lines, '\n')
 
+get_format(doctype::AbstractString) = FORMATS[doctype]
+
 """
     list_out_formats()
 

src/rendering/rendering.jl

@@ -5,7 +5,6 @@
 # - 4. Document Interface
 
 using Mustache, Highlights, .WeaveMarkdown, Markdown, Dates, Printf
-using REPL.REPLCompletions: latex_symbols
 
 
 const FORMATS = Dict{String,WeaveFormat}()

src/rendering/texformats.jl

@@ -3,7 +3,10 @@
 
 abstract type TexFormat <: WeaveFormat end
 
-set_rendering_options!(docformat::TexFormat; keep_unicode = false, kwargs...) = docformat.keep_unicode |= keep_unicode
+function set_rendering_options!(docformat::TexFormat; keep_unicode = false, template=nothing, kwargs...)
+    docformat.keep_unicode |= keep_unicode
+    docformat.template = get_tex_template(template)
+end
 
 function formatfigures(chunk, docformat::TexFormat)
     fignames = chunk.figures
@@ -73,40 +76,17 @@ function md_length_to_latex(def, reference)
     return def
 end
 
-# plain Tex
-# ---------
-
-Base.@kwdef mutable struct Tex <: TexFormat
-    description = "Latex with custom code environments"
-    extension = "tex"
-    codestart = "\\begin{juliacode}"
-    codeend = "\\end{juliacode}"
-    termstart = "\\begin{juliaterm}"
-    termend = "\\end{juliaterm}"
-    outputstart = "\\begin{juliaout}"
-    outputend = "\\end{juliaout}"
-    mimetypes = ["application/pdf", "image/png", "text/latex", "text/plain"]
-    fig_ext = ".pdf"
-    out_width = "\\linewidth"
-    out_height = nothing
-    fig_pos = "htpb"
-    fig_env = "figure"
-    # specials
-    keep_unicode = false
-end
-register_format!("tex", Tex())
-
 # minted Tex
 # ----------
 
 Base.@kwdef mutable struct TexMinted <: TexFormat
     description = "Latex using minted for highlighting"
     extension = "tex"
-    codestart = "\\begin{minted}[mathescape, fontsize=\\small, xleftmargin=0.5em]{julia}"
+    codestart = "\\begin{minted}[escapeinside=||, mathescape, fontsize=\\small, xleftmargin=0.5em]{julia}"
     codeend = "\\end{minted}"
-    termstart = "\\begin{minted}[fontsize=\\footnotesize, xleftmargin=0.5em, mathescape]{jlcon}"
+    termstart = "\\begin{minted}[escapeinside=||, mathescape, fontsize=\\footnotesize, xleftmargin=0.5em]{jlcon}"
     termend = "\\end{minted}"
-    outputstart = "\\begin{minted}[fontsize=\\small, xleftmargin=0.5em, mathescape, frame = leftline]{text}"
+    outputstart = "\\begin{minted}[escapeinside=||, mathescape, fontsize=\\small, xleftmargin=0.5em, frame = leftline]{text}"
     outputend = "\\end{minted}"
     mimetypes = ["application/pdf", "image/png", "text/latex", "text/plain"]
     fig_ext = ".pdf"
@@ -116,6 +96,11 @@ Base.@kwdef mutable struct TexMinted <: TexFormat
     fig_env = "figure"
     # specials
     keep_unicode = false
+    template = nothing
+    tex_deps = "\\usepackage{minted}"
+    # how to escape latex in verbatim/code environment
+    escape_starter = "|\$"
+    escape_closer = "\$|"
 end
 register_format!("texminted", TexMinted())
 
@@ -142,6 +127,10 @@ Base.@kwdef mutable struct JMarkdown2tex <: TexFormat
     highlight_theme = nothing
     template = nothing
     keep_unicode = false
+    tex_deps = ""
+    # how to escape latex in verbatim/code environment
+    escape_starter = "(*@"
+    escape_closer = "@*)"
 end
 register_format!("md2tex", JMarkdown2tex())
 register_format!("md2pdf", JMarkdown2tex())
@@ -155,17 +144,28 @@ end
 get_tex_template(::Nothing) = get_template(normpath(TEMPLATE_DIR, "md2pdf.tpl"))
 get_tex_template(x) = get_template(x)
 
+function render_doc(docformat::TexFormat, body, doc)
+    return Mustache.render(
+        docformat.template;
+        body = body,
+        highlight = "",
+        tex_deps = docformat.tex_deps,
+        [Pair(Symbol(k), v) for (k, v) in doc.header]...,
+    )
+end
+
 function render_doc(docformat::JMarkdown2tex, body, doc)
     return Mustache.render(
         docformat.template;
         body = body,
         highlight = get_highlight_stylesheet(MIME("text/latex"), docformat.highlight_theme),
+        tex_deps = docformat.tex_deps,
         [Pair(Symbol(k), v) for (k, v) in doc.header]...,
     )
 end
 
 # very similar to export to html
-function format_chunk(chunk::DocChunk, docformat::JMarkdown2tex)
+function format_chunk(chunk::DocChunk, docformat::TexFormat)
     out = IOBuffer()
     io = IOBuffer()
     for inline in chunk.content
@@ -182,7 +182,11 @@ function format_chunk(chunk::DocChunk, docformat::JMarkdown2tex)
     end
     clear_buffer_and_format!(io, out, WeaveMarkdown.latex)
     out = take2string!(out)
-    return docformat.keep_unicode ? out : uc2tex(out)
+    return unicode2latex(docformat, out)
+end
+
+function format_output(result, docformat::TexFormat)
+    return unicode2latex(docformat, result, true)
 end
 
 function format_output(result, docformat::JMarkdown2tex)
@@ -192,43 +196,48 @@ function format_output(result, docformat::JMarkdown2tex)
             Highlights.Format.escape(io, MIME("text/latex"), x, charescape = true),
         result,
     )
-    docformat.keep_unicode || return uc2tex(result_escaped, true)
-    return result_escaped
+    return unicode2latex(docformat, result_escaped, true)
 end
 
+function format_code(code, docformat::TexFormat)
+    return unicode2latex(docformat, code, true)
+end
 function format_code(code, docformat::JMarkdown2tex)
     ret = highlight_code(MIME("text/latex"), code, docformat.highlight_theme)
-    docformat.keep_unicode || return uc2tex(ret)
-    return ret
+    unicode2latex(docformat, ret, false)
 end
 
-# Convert unicode to tex, escape listings if needed
-function uc2tex(s, escape = false)
-    for key in keys(latex_symbols)
-        if escape
-            s = replace(s, latex_symbols[key] => "(*@\\ensuremath{$(texify(key))}@*)")
+# from julia symbols (e.g. "\bfhoge") to valid latex
+const UNICODE2LATEX = let
+    function texify(s)
+        return if occursin(r"^\\bf[A-Z]$", s)
+            replace(s, "\\bf" => "\\bm{\\mathrm{") * "}}"
+        elseif startswith(s, "\\bfrak")
+            replace(s, "\\bfrak" => "\\bm{\\mathfrak{") * "}}"
+        elseif startswith(s, "\\bf")
+            replace(s, "\\bf" => "\\bm{\\") * "}"
+        elseif startswith(s, "\\frak")
+            replace(s, "\\frak" => "\\mathfrak{") * "}"
         else
-            s = replace(s, latex_symbols[key] => "\\ensuremath{$(texify(key))}")
+            s
         end
     end
+    Dict(unicode => texify(sym) for (sym, unicode) in REPL.REPLCompletions.latex_symbols)
+end
+
+function unicode2latex(docformat::TexFormat, s, escape = false)
+    # Check whether to convert at all and return input if not
+    docformat.keep_unicode && return s
+    for (unicode, latex) in UNICODE2LATEX
+        body = "\\ensuremath{$(latex)}"
+        target = escape ? string(docformat.escape_starter, body, docformat.escape_closer) : body
+        s = replace(s, unicode => target)
+    end
     return s
 end
 
-# should_render(chunk) ? highlight_term(MIME("text/latex"), , docformat.highlight_theme) : ""
+format_termchunk(chunk, docformat::TexFormat) =
+    string(docformat.termstart, chunk.output, docformat.termend, "\n")
+
 format_termchunk(chunk, docformat::JMarkdown2tex) =
     should_render(chunk) ? highlight_term(MIME("text/latex"), chunk.output, docformat.highlight_theme) : ""
-
-# Make julia symbols (\bf* etc.) valid latex
-function texify(s)
-    return if occursin(r"^\\bf[A-Z]$", s)
-        replace(s, "\\bf" => "\\bm{\\mathrm{") * "}}"
-    elseif startswith(s, "\\bfrak")
-        replace(s, "\\bfrak" => "\\bm{\\mathfrak{") * "}}"
-    elseif startswith(s, "\\bf")
-        replace(s, "\\bf" => "\\bm{\\") * "}"
-    elseif startswith(s, "\\frak")
-        replace(s, "\\frak" => "\\mathfrak{") * "}"
-    else
-        s
-    end
-end

src/run.jl

@@ -17,7 +17,7 @@ function run_doc(
     # cache :all, :user, :off, :refresh
 
     doc.doctype = isnothing(doctype) ? (doctype = detect_doctype(doc.source)) : doctype
-    doc.format = deepcopy(FORMATS[doctype])
+    doc.format = deepcopy(get_format(doctype))
 
     cwd = doc.cwd = get_cwd(doc, out_path)
     isdir(cwd) || mkdir(cwd)

templates/md2pdf.tpl

@@ -7,7 +7,9 @@
 \usepackage{graphicx}
 \usepackage{microtype}
 \usepackage{hyperref}
-
+{{#:tex_deps}}
+{{{ :tex_deps }}}
+{{/:tex_deps}}
 \setlength{\parindent}{0pt}
 \setlength{\parskip}{1.2ex}
 

test/figureformatter_test.jl

@@ -1,4 +1,4 @@
-test_formatfigures(chunk, format) = Weave.formatfigures(chunk, Weave.FORMATS[format])
+test_formatfigures(chunk, format) = Weave.formatfigures(chunk, get_format(format))
 
 
 # Make a dummy codehunk with figure
@@ -9,7 +9,6 @@ chunk.figures = ["figs/figures_plot1.png"]
 
 
 @test test_formatfigures(chunk, "md2tex") == "\\includegraphics{figs/figures_plot1.png}\n"
-@test test_formatfigures(chunk, "tex") == "\\includegraphics{figs/figures_plot1.png}\n"
 @test test_formatfigures(chunk, "texminted") == "\\includegraphics{figs/figures_plot1.png}\n"
 @test test_formatfigures(chunk, "pandoc") == "![](figs/figures_plot1.png)\\ \n\n"
 @test test_formatfigures(chunk, "github") == "![](figs/figures_plot1.png)\n"
@@ -24,7 +23,6 @@ chunk.options[:out_width] = "100%"
 
 
 chunk.options[:fig_cap] = "Nice plot"
-@test test_formatfigures(chunk, "tex") == "\\begin{figure}[!h]\n\\center\n\\includegraphics[width=1.0\\linewidth]{figs/figures_plot1.png}\n\\caption{Nice plot}\n\\end{figure}\n"
 @test test_formatfigures(chunk, "pandoc") == "![Nice plot](figs/figures_plot1.png){width=100%}\n"
 @test test_formatfigures(chunk, "md2tex") == "\\begin{figure}[!h]\n\\center\n\\includegraphics[width=1.0\\linewidth]{figs/figures_plot1.png}\n\\caption{Nice plot}\n\\end{figure}\n"
 @test test_formatfigures(chunk, "md2html") == "<figure>\n<img src=\"figs/figures_plot1.png\" width=\"100%\" />\n<figcaption>Nice plot</figcaption>\n</figure>\n"
@@ -35,21 +33,3 @@ chunk.options[:fig_cap] = "Nice plot"
 
 chunk.options[:label] = "somefig"
 @test test_formatfigures(chunk, "pandoc") == "![Nice plot](figs/figures_plot1.png){width=100% #fig:somefig}\n"
-@test test_formatfigures(chunk, "tex") == "\\begin{figure}[!h]\n\\center\n\\includegraphics[width=1.0\\linewidth]{figs/figures_plot1.png}\n\\caption{Nice plot}\n\\label{fig:somefig}\n\\end{figure}\n"
-
-
-chunk.options[:label] = nothing
-chunk.options[:fig_cap] = nothing
-chunk.options[:fig_env] = "center"
-chunk.options[:fig_pos] = ""
-@test test_formatfigures(chunk, "tex") == "\\begin{center}\n\\includegraphics[width=1.0\\linewidth]{figs/figures_plot1.png}\n\\end{center}\n"
-
-
-chunk.options[:out_width] = "50%"
-chunk.options[:out_height] = "75 %"
-@test test_formatfigures(chunk, "tex") == "\\begin{center}\n\\includegraphics[width=0.5\\linewidth,height=0.75\\paperheight]{figs/figures_plot1.png}\n\\end{center}\n"
-
-
-chunk.options[:out_width] = "A%"
-chunk.options[:out_height] = "0.5\\textwidth"
-@test test_formatfigures(chunk, "tex") == "\\begin{center}\n\\includegraphics[width=A%,height=0.5\\textwidth]{figs/figures_plot1.png}\n\\end{center}\n"

test/formatter_test.jl

@@ -1,36 +0,0 @@
-# Test disable escaping of unicode
-@testset "escape/unescape unicode characters" begin
-
-content = """
-# Test chunk
-α
-"""
-chunk = Weave.DocChunk(content, 1, 1)
-fmt = deepcopy(Weave.FORMATS["md2tex"])
-
-f = Weave.format_chunk(chunk, fmt)
-@test f == "\\section{Test chunk}\n\\ensuremath{\\alpha}\n\n"
-
-fmt.keep_unicode = true
-f = Weave.format_chunk(chunk, fmt)
-@test f == "\\section{Test chunk}\nα\n\n"
-
-
-str = """
-```julia
-α = 10
-```
-"""
-doc = mock_run(str; doctype = "md2tex")
-Weave.set_rendering_options!(doc.format)
-doc = Weave.render_doc(doc)
-@test occursin(Weave.uc2tex("α"), doc)
-@test !occursin("α", doc)
-
-doc = mock_run(str; doctype = "md2tex")
-Weave.set_rendering_options!(doc.format; keep_unicode = true)
-doc = Weave.render_doc(doc)
-@test occursin("α", doc)
-@test !occursin(Weave.uc2tex("α"), doc)
-
-end

test/render/texformats.jl

@@ -0,0 +1,30 @@
+@testset "unicode to latex conversion" begin
+    unicode2latex(args...) = Weave.unicode2latex(get_format("md2tex"), args...)
+
+    # unit test
+    let
+        s = unicode2latex("α = 10")
+        @test !occursin("α", s)
+        @test occursin("alpha", s)
+    end
+
+    # end2end
+    let
+        str = """
+        ```julia
+        α = 10
+        ```
+        """
+        doc = mock_run(str; doctype = "md2tex")
+        Weave.set_rendering_options!(doc.format)
+        rendered = Weave.render_doc(doc)
+        @test occursin("alpha", rendered)
+        @test !occursin("α", rendered)
+
+        doc = mock_run(str; doctype = "md2tex")
+        Weave.set_rendering_options!(doc.format; keep_unicode = true)
+        rendered = Weave.render_doc(doc)
+        @test !occursin("alpha", rendered)
+        @test occursin("α", rendered)
+    end
+end  # @testset "rendering tex formats"

test/runtests.jl

@@ -7,7 +7,7 @@
 
 # %%
 using Weave, Test
-using Weave: WeaveDoc, run_doc
+using Weave: WeaveDoc, run_doc, get_format
 
 
 function mock_doc(str, informat = "markdown")
@@ -49,6 +49,10 @@ end
         include("run/test_error.jl")
     end
 
+    @testset "render" begin
+        include("render/texformats.jl")
+    end
+
     @testset "conversions" begin
         include("test_converter.jl")
     end
@@ -58,7 +62,6 @@ end
     end
 
     @testset "legacy" begin
-        include("formatter_test.jl")
         include("markdown_test.jl")
         include("figureformatter_test.jl")
         include("cache_test.jl")