dispatch-base documentation generation

pull/374/head
Shuhei Kadowaki 2020-06-14 16:36:51 +09:00
parent 29546b7716
commit 7e9c4fb99d
8 changed files with 288 additions and 219 deletions

View File

@ -32,6 +32,19 @@ end
take2string!(io) = String(take!(io))
joinlines(lines) = join(lines, '\n')
include("types.jl")
include("config.jl")
include("WeaveMarkdown/markdown.jl")
include("display_methods.jl")
include("reader/reader.jl")
include("run.jl")
include("cache.jl")
include("rendering/rendering.jl")
include("writer/writer.jl")
include("converter.jl")
get_format(doctype::AbstractString) = FORMATS[doctype]
"""
@ -103,7 +116,7 @@ Weave an input document to output file.
- `css::Union{Nothing,AbstractString} = nothing`: Path of a CSS file used for md2html format
- `highlight_theme::Union{Nothing,Type{<:Highlights.AbstractTheme}} = nothing`: Theme used for syntax highlighting (defaults to `Highlights.Themes.DefaultTheme`)
- `pandoc_options::Vector{<:AbstractString} = String[]`: `String`s of options to pass to pandoc for `pandoc2html` and `pandoc2pdf` formats, e.g. `["--toc", "-N"]`
- `latex_cmd::AbstractString = "xelatex"`: The command used to make PDF file from .tex
- `latex_cmd::Vector{<:AbstractString} = $(DEFAULT_LATEX_CMD)`: The command used to make PDF file from .tex
- `keep_unicode::Bool = false`: If `true`, do not convert unicode characters to their respective latex representation. This is especially useful if a font and tex-engine with support for unicode characters are used
!!! note
@ -124,7 +137,7 @@ function weave(
css::Union{Nothing,AbstractString} = nothing, # TODO: rename to `stylesheet`
highlight_theme::Union{Nothing,Type{<:Highlights.AbstractTheme}} = nothing,
pandoc_options::Vector{<:AbstractString} = String[],
latex_cmd::AbstractString = "xelatex",
latex_cmd::Vector{<:AbstractString} = DEFAULT_LATEX_CMD,
keep_unicode::Bool = false,
)
doc = WeaveDoc(source, informat)
@ -173,10 +186,7 @@ function weave(
cache = cache,
)
# render document
# ---------------
# overwrites options with those specified in header, that are needed for rendering document
# overwrites options with those specified in header, that are needed for rendering/writing document
# NOTE: these YAML options can be given dynamically
if !isnothing(weave_options)
if haskey(weave_options, "template")
@ -190,38 +200,31 @@ function weave(
css isa AbstractString && (css = normpath(dirname(source), css))
end
highlight_theme = get(weave_options, "highlight_theme", highlight_theme)
latex_cmd = get(weave_options, "latex_cmd", latex_cmd)
keep_unicode = get(weave_options, "keep_unicode", keep_unicode)
end
set_format_options!(doc; template = template, highlight_theme = highlight_theme, css = css, keep_unicode = keep_unicode)
rendered = render_doc(doc)
out_path = get_out_path(doc, out_path)
write(out_path, rendered)
# document generation via external programs
# -----------------------------------------
if !isnothing(weave_options)
latex_cmd = get(weave_options, "latex_cmd", latex_cmd)
pandoc_options = get(weave_options, "pandoc_options", pandoc_options)
end
doctype = doc.doctype
if doctype == "pandoc2html"
intermediate = out_path
out_path = get_out_path(doc, out_path, "html")
pandoc2html(rendered, doc, highlight_theme, out_path, pandoc_options)
rm(intermediate)
elseif doctype == "pandoc2pdf"
intermediate = out_path
out_path = get_out_path(doc, out_path, "pdf")
pandoc2pdf(rendered, doc, out_path, pandoc_options)
rm(intermediate)
elseif doctype == "md2pdf"
run_latex(doc, out_path, latex_cmd)
out_path = get_out_path(doc, out_path, "pdf")
end
set_format_options!(
doc;
# general
template = template,
highlight_theme = highlight_theme,
css = css,
# pandoc
pandoc_options = pandoc_options,
# latex
keep_unicode = keep_unicode,
latex_cmd = latex_cmd,
)
# render document
# ---------------
rendered = render_doc(doc)
# write documents
# ---------------
out_path = write_doc(doc, rendered, get_out_path(doc, out_path))
@info "Weaved to $(out_path)"
return out_path
@ -320,16 +323,6 @@ end
include_weave(source, informat = nothing) = include_weave(Main, source, informat)
include("types.jl")
include("config.jl")
include("WeaveMarkdown/markdown.jl")
include("display_methods.jl")
include("reader/reader.jl")
include("run.jl")
include("cache.jl")
include("rendering/rendering.jl")
include("pandoc.jl")
include("converter.jl")
export weave,
list_out_formats,

View File

@ -1,103 +0,0 @@
function pandoc2html(rendered, doc, highlight_theme, out_path, pandoc_options)
template_path = normpath(TEMPLATE_DIR, "pandoc2html.html")
stylesheet_path = normpath(STYLESHEET_DIR, "pandoc2html_skeleton.css")
highlight_stylesheet = get_highlight_stylesheet(MIME("text/html"), highlight_theme)
_, weave_source = splitdir(abspath(doc.source))
weave_version, weave_date = weave_info()
# Header is inserted from displayed plots
header_script = doc.header_script
self_contained = (header_script "") ? [] : "--self-contained"
if haskey(doc.header, "bibliography")
filt = "--filter"
citeproc = "pandoc-citeproc"
else
filt = []
citeproc = []
end
# Change path for pandoc
cd_back = let d = pwd(); () -> cd(d); end
cd(doc.cwd)
out_path = basename(out_path)
try
cmd = `pandoc -f markdown+raw_html -s --mathjax=""
$filt $citeproc $pandoc_options
--template $template_path
-H $stylesheet_path
$self_contained
-V highlight_stylesheet=$highlight_stylesheet
-V weave_version=$weave_version
-V weave_date=$weave_date
-V weave_source=$weave_source
-V headerscript=$header_script
-o $out_path`
proc = open(cmd, "r+")
println(proc.in, rendered)
close(proc.in)
proc_output = read(proc.out, String)
catch
@warn "Error converting document to HTML"
rethrow() # TODO: just show error content instead of rethrow the err
finally
cd_back()
end
end
function pandoc2pdf(rendered, doc, out_path, pandoc_options)
header_template = normpath(TEMPLATE_DIR, "pandoc2pdf_header.txt")
out_path = basename(out_path)
# Change path for pandoc
cd_back = let d = pwd(); () -> cd(d); end
cd(doc.cwd)
if haskey(doc.header, "bibliography")
filt = "--filter"
citeproc = "pandoc-citeproc"
else
filt = []
citeproc = []
end
@info "Done executing code. Running xelatex"
try
cmd = `pandoc -f markdown+raw_tex -s --pdf-engine=xelatex --highlight-style=tango
$filt $citeproc $pandoc_options
--include-in-header=$header_template
-V fontsize=12pt -o $out_path`
proc = open(cmd, "r+")
println(proc.in, rendered)
close(proc.in)
proc_output = read(proc.out, String)
catch
@warn "Error converting document to pdf"
rethrow()
finally
cd_back()
end
end
function run_latex(doc::WeaveDoc, out_path, latex_cmd = "xelatex")
cd_back = let d = pwd(); () -> cd(d); end
cd(doc.cwd)
xname = basename(out_path)
@info "Weaved code to $out_path . Running $latex_cmd" # space before '.' added for link to be clickable in Juno terminal
textmp = mktempdir(".")
try
cmd = `$latex_cmd -shell-escape $xname -aux-directory $textmp -include-directory $(doc.cwd)`
run(cmd); run(cmd) # XXX: is twice enough for every case ?
catch
@warn "Error converting document to pdf. Try running latex manually"
rethrow()
finally
rm(xname)
rm(textmp, recursive = true)
cd_back()
end
end

View File

@ -36,17 +36,13 @@ end
register_format!("md2html", JMarkdown2HTML())
function set_format_options!(docformat::JMarkdown2HTML; template = nothing, css = nothing, highlight_theme = nothing, _kwargs...)
docformat.template = get_html_template(template)
docformat.stylesheet = get_stylesheet(css)
template_path = isnothing(template) ? normpath(TEMPLATE_DIR, "md2html.tpl") : template
docformat.template = get_mustache_template(template_path)
stylesheet_path = isnothing(css) ? normpath(STYLESHEET_DIR, "skeleton.css") : css
docformat.stylesheet = read(stylesheet_path, String)
docformat.highlight_theme = get_highlight_theme(highlight_theme)
end
get_html_template(::Nothing) = get_mustache_template(normpath(TEMPLATE_DIR, "md2html.tpl"))
get_html_template(x) = get_mustache_template(x)
get_stylesheet(::Nothing) = get_stylesheet(normpath(STYLESHEET_DIR, "skeleton.css"))
get_stylesheet(path::AbstractString) = read(path, String)
# very similar to tex version of function
function format_chunk(chunk::DocChunk, docformat::JMarkdown2HTML)
out = IOBuffer()
@ -137,14 +133,27 @@ Base.@kwdef mutable struct Pandoc2HTML <: HTMLFormat
termend = codeend
outputstart = "\n"
outputend = "\n"
mimetypes = ["image/png", "image/svg+xml", "image/jpg",
"text/html", "text/markdown", "text/plain"]
mimetypes = ["image/png", "image/svg+xml", "image/jpg", "text/html", "text/markdown", "text/plain"]
fig_ext = ".png"
out_width = nothing
out_height = nothing
fig_pos = nothing
fig_env = nothing
# specials
template_path = nothing
stylesheet_path = nothing
highlight_theme = nothing
pandoc_options = String[]
end
register_format!("pandoc2html", Pandoc2HTML())
function set_format_options!(docformat::Pandoc2HTML; template = nothing, css = nothing, highlight_theme = nothing, pandoc_options = String[], _kwargs...)
docformat.template_path =
isnothing(template) ? normpath(TEMPLATE_DIR, "pandoc2html.html") : template
docformat.stylesheet_path =
isnothing(css) ? normpath(STYLESHEET_DIR, "pandoc2html_skeleton.css") : css
docformat.highlight_theme = get_highlight_theme(highlight_theme)
docformat.pandoc_options = pandoc_options
end
formatfigures(chunk, docformat::Pandoc2HTML) = formatfigures(chunk, Pandoc())

View File

@ -141,28 +141,9 @@ end
# pandoc
# ------
Base.@kwdef mutable struct Pandoc <: MarkdownFormat
description = "Pandoc markdown"
extension = "md"
codestart = "~~~~{.julia}"
codeend = "~~~~~~~~~~~~~\n\n"
termstart = codestart
termend = codeend
outputstart = "~~~~"
outputend = "~~~~\n\n"
# Prefer png figures for markdown conversion, svg doesn't work with latex
mimetypes = ["image/png", "image/jpg", "image/svg+xml",
"text/markdown", "text/plain"]
fig_ext = ".png"
out_width = nothing
out_height = nothing
fig_pos = nothing
fig_env = nothing
end
register_format!("pandoc", Pandoc())
register_format!("pandoc2pdf", Pandoc())
abstract type PandocFormat <: MarkdownFormat end
function formatfigures(chunk, docformat::Pandoc)
function formatfigures(chunk, docformat::PandocFormat)
fignames = chunk.figures
length(fignames) > 0 || (return "")
@ -194,3 +175,48 @@ function formatfigures(chunk, docformat::Pandoc)
end
return result
end
Base.@kwdef mutable struct Pandoc <: PandocFormat
description = "Pandoc markdown"
extension = "md"
codestart = "~~~~{.julia}"
codeend = "~~~~~~~~~~~~~\n\n"
termstart = codestart
termend = codeend
outputstart = "~~~~"
outputend = "~~~~\n\n"
# Prefer png figures for markdown conversion, svg doesn't work with latex
mimetypes = ["image/png", "image/jpg", "image/svg+xml", "text/markdown", "text/plain"]
fig_ext = ".png"
out_width = nothing
out_height = nothing
fig_pos = nothing
fig_env = nothing
end
register_format!("pandoc", Pandoc())
Base.@kwdef mutable struct Pandoc2PDF <: PandocFormat
description = "Pandoc markdown to PDF"
extension = "md"
codestart = "~~~~{.julia}"
codeend = "~~~~~~~~~~~~~\n\n"
termstart = codestart
termend = codeend
outputstart = "~~~~"
outputend = "~~~~\n\n"
# Prefer png figures for markdown conversion, svg doesn't work with latex
mimetypes = ["image/png", "image/jpg", "image/svg+xml", "text/markdown", "text/plain"]
fig_ext = ".png"
out_width = nothing
out_height = nothing
fig_pos = nothing
fig_env = nothing
# specials
header_template = normpath(TEMPLATE_DIR, "pandoc2pdf_header.txt")
pandoc_options = String[]
end
register_format!("pandoc2pdf", Pandoc2PDF())
function set_format_options!(docformat::Pandoc2PDF; pandoc_options = String[], _kwargs...)
docformat.pandoc_options = pandoc_options
end

View File

@ -3,14 +3,12 @@
abstract type TexFormat <: WeaveFormat end
function set_format_options!(docformat::TexFormat; keep_unicode = false, template=nothing, _kwargs...)
function set_format_options!(docformat::TexFormat; keep_unicode = false, template = nothing, _kwargs...)
docformat.keep_unicode |= keep_unicode
docformat.template = get_tex_template(template)
docformat.template =
get_mustache_template(isnothing(template) ? normpath(TEMPLATE_DIR, "md2pdf.tpl") : template)
end
get_tex_template(::Nothing) = get_mustache_template(normpath(TEMPLATE_DIR, "md2pdf.tpl"))
get_tex_template(x) = get_mustache_template(x)
# very similar to export to html
function format_chunk(chunk::DocChunk, docformat::TexFormat)
out = IOBuffer()
@ -31,12 +29,13 @@ function format_chunk(chunk::DocChunk, docformat::TexFormat)
out = take2string!(out)
return unicode2latex(docformat, out)
end
format_termchunk(chunk, docformat::TexFormat) = string(docformat.termstart, chunk.output, docformat.termend, "\n")
format_output(result, docformat::TexFormat) = unicode2latex(docformat, result, true)
format_code(code, docformat::TexFormat) = unicode2latex(docformat, code, true)
format_termchunk(chunk, docformat::TexFormat) = string(docformat.termstart, chunk.output, docformat.termend, "\n")
# from julia symbols (e.g. "\bfhoge") to valid latex
const UNICODE2LATEX = let
function texify(s)
@ -175,8 +174,45 @@ register_format!("texminted", TexMinted())
# Tex (directly to PDF)
# ---------------------
Base.@kwdef mutable struct JMarkdown2PDF <: TexFormat
description = "Julia markdown to latex"
abstract type JMarkdownTexFormat <: TexFormat end
function set_format_options!(docformat::JMarkdownTexFormat; template = nothing, highlight_theme = nothing, keep_unicode = false, _kwargs...)
docformat.template =
get_mustache_template(isnothing(template) ? normpath(TEMPLATE_DIR, "md2pdf.tpl") : template)
docformat.highlight_theme = get_highlight_theme(highlight_theme)
docformat.keep_unicode |= keep_unicode
end
function format_output(result, docformat::JMarkdownTexFormat)
# Highligts has some extra escaping defined, eg of $, ", ...
result_escaped = sprint(
(io, x) ->
Highlights.Format.escape(io, MIME("text/latex"), x, charescape = true),
result,
)
return unicode2latex(docformat, result_escaped, true)
end
function format_code(code, docformat::JMarkdownTexFormat)
ret = highlight_code(MIME("text/latex"), code, docformat.highlight_theme)
unicode2latex(docformat, ret, false)
end
format_termchunk(chunk, docformat::JMarkdownTexFormat) =
should_render(chunk) ? highlight_term(MIME("text/latex"), chunk.output, docformat.highlight_theme) : ""
function render_doc(docformat::JMarkdownTexFormat, body, doc)
return Mustache.render(
docformat.template;
body = body,
highlight = get_highlight_stylesheet(MIME("text/latex"), docformat.highlight_theme),
tex_deps = docformat.tex_deps,
[Pair(Symbol(k), v) for (k, v) in doc.header]...,
)
end
Base.@kwdef mutable struct JMarkdown2Tex <: JMarkdownTexFormat
description = "Julia markdown to LaTeX"
extension = "tex"
codestart = ""
codeend = ""
@ -184,8 +220,7 @@ Base.@kwdef mutable struct JMarkdown2PDF <: TexFormat
termend = codeend
outputstart = "\\begin{lstlisting}"
outputend = "\\end{lstlisting}\n"
mimetypes = ["application/pdf", "image/png", "image/jpg",
"text/latex", "text/markdown", "text/plain"]
mimetypes = ["application/pdf", "image/png", "image/jpg", "text/latex", "text/markdown", "text/plain"]
fig_ext = ".pdf"
out_width = "\\linewidth"
out_height = nothing
@ -200,39 +235,42 @@ Base.@kwdef mutable struct JMarkdown2PDF <: TexFormat
escape_starter = "(*@"
escape_closer = reverse(escape_starter)
end
register_format!("md2tex", JMarkdown2PDF())
register_format!("md2tex", JMarkdown2Tex())
# will be used by `write_doc`
const DEFAULT_LATEX_CMD = ["xelatex", "-shell-escape", "-halt-on-error"]
Base.@kwdef mutable struct JMarkdown2PDF <: JMarkdownTexFormat
description = "Julia markdown to LaTeX"
extension = "tex"
codestart = ""
codeend = ""
termstart = codestart
termend = codeend
outputstart = "\\begin{lstlisting}"
outputend = "\\end{lstlisting}\n"
mimetypes = ["application/pdf", "image/png", "image/jpg", "text/latex", "text/markdown", "text/plain"]
fig_ext = ".pdf"
out_width = "\\linewidth"
out_height = nothing
fig_pos = nothing
fig_env = nothing
# specials
highlight_theme = nothing
template = nothing
keep_unicode = false
tex_deps = ""
latex_cmd = DEFAULT_LATEX_CMD
# how to escape latex in verbatim/code environment
escape_starter = "(*@"
escape_closer = reverse(escape_starter)
end
register_format!("md2pdf", JMarkdown2PDF())
function set_format_options!(docformat::JMarkdown2PDF; template = nothing, highlight_theme = nothing, keep_unicode = false, _kwargs...)
docformat.template = get_tex_template(template)
function set_format_options!(docformat::JMarkdown2PDF; template = nothing, highlight_theme = nothing, keep_unicode = false, latex_cmd = DEFAULT_LATEX_CMD, _kwargs...)
docformat.template =
get_mustache_template(isnothing(template) ? normpath(TEMPLATE_DIR, "md2pdf.tpl") : template)
docformat.highlight_theme = get_highlight_theme(highlight_theme)
docformat.keep_unicode |= keep_unicode
docformat.latex_cmd = latex_cmd
end
function render_doc(docformat::JMarkdown2PDF, body, doc)
return Mustache.render(
docformat.template;
body = body,
highlight = get_highlight_stylesheet(MIME("text/latex"), docformat.highlight_theme),
tex_deps = docformat.tex_deps,
[Pair(Symbol(k), v) for (k, v) in doc.header]...,
)
end
function format_output(result, docformat::JMarkdown2PDF)
# Highligts has some extra escaping defined, eg of $, ", ...
result_escaped = sprint(
(io, x) ->
Highlights.Format.escape(io, MIME("text/latex"), x, charescape = true),
result,
)
return unicode2latex(docformat, result_escaped, true)
end
function format_code(code, docformat::JMarkdown2PDF)
ret = highlight_code(MIME("text/latex"), code, docformat.highlight_theme)
unicode2latex(docformat, ret, false)
end
format_termchunk(chunk, docformat::JMarkdown2PDF) =
should_render(chunk) ? highlight_term(MIME("text/latex"), chunk.output, docformat.highlight_theme) : ""

19
src/writer/latex.jl Normal file
View File

@ -0,0 +1,19 @@
function write_doc(docformat::JMarkdown2PDF, doc, rendered, out_path)
cd_back = let d = pwd(); () -> cd(d); end
cd(doc.cwd)
try
tex_path = basename(out_path)
write(tex_path, rendered)
cmds = copy(docformat.latex_cmd)
push!(cmds, tex_path)
cmd = Cmd(cmds)
run(cmd); run(cmd) # XXX: is twice enough for every case ?
catch
@warn "Error converting document to pdf. Try running latex manually"
rethrow()
finally
cd_back()
end
return get_out_path(doc, out_path, "pdf")
end

76
src/writer/pandoc.jl Normal file
View File

@ -0,0 +1,76 @@
function write_doc(docformat::Pandoc2HTML, doc, rendered, out_path)
_, weave_source = splitdir(abspath(doc.source))
weave_version, weave_date = weave_info()
# Header is inserted from displayed plots
header_script = doc.header_script
self_contained = (header_script "") ? [] : "--self-contained"
if haskey(doc.header, "bibliography")
filt = "--filter"
citeproc = "pandoc-citeproc"
else
filt = []
citeproc = []
end
out_path = get_out_path(doc, out_path, "html")
cd_back = let d = pwd(); () -> cd(d); end
cd(dirname(out_path))
try
out = basename(out_path)
highlight_stylesheet = get_highlight_stylesheet(MIME("text/html"), docformat.highlight_theme)
cmd = `pandoc -f markdown+raw_html -s --mathjax=""
$filt $citeproc $(docformat.pandoc_options)
--template $(docformat.template_path)
-H $(docformat.stylesheet_path)
$(self_contained)
-V highlight_stylesheet=$(highlight_stylesheet)
-V weave_version=$(weave_version)
-V weave_date=$(weave_date)
-V weave_source=$(weave_source)
-V headerscript=$(header_script)
-o $(out)`
proc = open(cmd, "r+")
println(proc.in, rendered)
close(proc.in)
proc_output = read(proc.out, String)
catch
rethrow() # TODO: just show error content instead of rethrow the err
finally
cd_back()
end
return out_path
end
function write_doc(docformat::Pandoc2PDF, doc, rendered, out_path)
if haskey(doc.header, "bibliography")
filt = "--filter"
citeproc = "pandoc-citeproc"
else
filt = []
citeproc = []
end
out_path = get_out_path(doc, out_path, "pdf")
cd_back = let d = pwd(); () -> cd(d); end
cd(dirname(out_path))
try
out = basename(out_path)
cmd = `pandoc -f markdown+raw_tex -s --pdf-engine=xelatex --highlight-style=tango
$filt $citeproc $(docformat.pandoc_options)
--include-in-header=$(docformat.header_template)
-V fontsize=12pt -o $(out)`
proc = open(cmd, "r+")
println(proc.in, rendered)
close(proc.in)
proc_output = read(proc.out, String)
catch
rethrow()
finally
cd_back()
end
return out_path
end

11
src/writer/writer.jl Normal file
View File

@ -0,0 +1,11 @@
function write_doc(doc, rendered, out_path)
return write_doc(doc.format, doc, rendered, out_path)
end
function write_doc(::WeaveFormat, doc, rendered, out_path)
write(out_path, rendered)
return out_path
end
include("pandoc.jl")
include("latex.jl")