Merge pull request #330 from JunoLab/avi/refactor

refactor
pull/331/head
Shuhei Kadowaki 2020-05-16 01:25:25 +09:00 committed by GitHub
commit 95c3dac962
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 121 additions and 181 deletions

View File

@ -45,5 +45,5 @@ convert_doc("FIR_design.ipynb", "FIR_design.jmd")
```
```@docs
convert_doc(infile::AbstractString, outfile::AbstractString)
convert_doc
```

View File

@ -179,24 +179,25 @@ function weave(
open(io->write(io,formatted), outname, "w")
# Special for that need external programs
if doc.doctype == "pandoc2html"
doctype = doc.doctype
if doctype == "pandoc2html"
mdname = outname
outname = get_outname(out_path, doc, ext = "html")
pandoc2html(formatted, doc, outname, pandoc_options)
rm(mdname)
elseif doc.doctype == "pandoc2pdf"
elseif doctype == "pandoc2pdf"
mdname = outname
outname = get_outname(out_path, doc, ext = "pdf")
pandoc2pdf(formatted, doc, outname, pandoc_options)
rm(mdname)
elseif doc.doctype == "md2pdf"
elseif doctype == "md2pdf"
success = run_latex(doc, outname, latex_cmd)
success || return
outname = get_outname(out_path, doc, ext = "pdf")
end
doc.cwd == pwd() && (outname = basename(outname))
@info("Report weaved to $outname")
@info "Report weaved to $outname"
return abspath(outname)
end
@ -236,7 +237,7 @@ function notebook(
jupyter_path::AbstractString = "jupyter",
)
doc = WeaveDoc(source)
converted = convert_doc(doc, NotebookOutput())
converted = convert_to_notebook(doc)
doc.cwd = get_cwd(doc, out_path)
outfile = get_outname(out_path, doc, ext = "ipynb")
@ -245,7 +246,7 @@ function notebook(
end
@info "Running nbconvert"
return out = read(
return read(
`$jupyter_path nbconvert --ExecutePreprocessor.timeout=$timeout --to notebook --execute $outfile $nbconvert_options --output $outfile`,
String,
)
@ -272,11 +273,12 @@ function include_weave(
"\n",
)
include_string(m, code)
catch e
throw(e)
catch err
throw(err)
finally
cd(old_path)
end
return nothing
end
include_weave(source, informat = nothing) = include_weave(Main, source, informat)
@ -303,7 +305,7 @@ include("cache.jl")
include("formatters.jl")
include("format.jl")
include("pandoc.jl")
include("writers.jl")
include("converter.jl")
export weave,
list_out_formats,

View File

@ -1,58 +1,79 @@
import JSON
import Mustache
mutable struct NotebookOutput end
mutable struct MarkdownOutput end
mutable struct NowebOutput end
mutable struct ScriptOutput end
const output_formats = Dict{String,Any}(
"noweb" => NowebOutput(),
"notebook" => NotebookOutput(),
"markdown" => MarkdownOutput(),
"script" => ScriptOutput(),
)
"""Autodetect format for converter"""
function detect_outformat(outfile::String)
ext = lowercase(splitext(outfile)[2])
ext == ".jl" && return "script"
ext == ".jmd" && return "markdown"
ext == ".ipynb" && return "notebook"
return "noweb"
end
using JSON, Mustache
"""
convert_doc(infile::AbstractString, outfile::AbstractString; format::Union{Nothing,AbstractString} = nothing)
convert_doc(infile::AbstractString, outfile::AbstractString; outformat::Union{Nothing,AbstractString} = nothing)
Convert Weave documents between different formats
- `infile`: Path of the input document
- `outfile`: Path of the output document
- `format = nothing`: Output document format (optional). It will be detected automatically from the `outfile` extension. You can also specify either of `"script"`, `"markdown"`, `"notebook"`, or `"noweb"`
- `outformat = nothing`: Output document format (optional). By default (i.e. given `nothing`) Weave will try to automatically detect it from the `outfile`'s extension. You can also specify either of `"script"`, `"markdown"`, `"notebook"`, or `"noweb"`
"""
function convert_doc(
infile::AbstractString,
outfile::AbstractString;
format::Union{Nothing,AbstractString} = nothing,
outformat::Union{Nothing,AbstractString} = nothing,
)
doc = WeaveDoc(infile)
isnothing(format) && (format = detect_outformat(outfile))
if isnothing(outformat)
ext = lowercase(splitext(outfile)[2])
outformat =
ext == ".jl" ? "script" :
ext == ".jmd" ? "markdown" :
ext == ".ipynb" ? "notebook" :
"noweb" # fallback
end
converted = convert_doc(doc, output_formats[format])
converted = _convert_doc(doc, outformat)
open(outfile, "w") do f
write(f, converted)
end
return outfile
end
"""Convert Weave document to Jupyter notebook format"""
function convert_doc(doc::WeaveDoc, format::NotebookOutput)
function _convert_doc(doc, outformat)
outformat == "script" ? convert_to_script(doc) :
outformat == "markdown" ? convert_to_markdown(doc) :
outformat == "notebook" ? convert_to_notebook(doc) :
convert_to_noweb(doc)
end
function convert_to_script(doc)
output = ""
for chunk in doc.chunks
if typeof(chunk) == Weave.DocChunk
content = join([repr(c) for c in chunk.content], "")
output *= join(["#' " * s for s in split(content, "\n")], "\n")
else
output *= "\n#+ "
isempty(chunk.optionstring) || (output *= strip(chunk.optionstring))
output *= "\n\n" * lstrip(chunk.content)
output *= "\n"
end
end
return output
end
function convert_to_markdown(doc)
output = ""
for chunk in doc.chunks
if isa(chunk, DocChunk)
output *= join([repr(c) for c in chunk.content], "")
else
output *= "\n" * "```julia"
isempty(chunk.optionstring) || (output *= ";" * chunk.optionstring)
output *= "\n" * lstrip(chunk.content)
output *= "```\n"
end
end
return output
end
function convert_to_notebook(doc)
nb = Dict()
nb["nbformat"] = 4
nb["nbformat_minor"] = 2
@ -117,25 +138,7 @@ function convert_doc(doc::WeaveDoc, format::NotebookOutput)
return json_nb
end
"""Convert Weave document to Jupyter notebook format"""
function convert_doc(doc::WeaveDoc, format::MarkdownOutput)
output = ""
for chunk in doc.chunks
if isa(chunk, DocChunk)
output *= join([repr(c) for c in chunk.content], "")
else
output *= "\n" * "```julia"
isempty(chunk.optionstring) || (output *= ";" * chunk.optionstring)
output *= "\n" * lstrip(chunk.content)
output *= "```\n"
end
end
return output
end
"""Convert Weave document to noweb format"""
function convert_doc(doc::WeaveDoc, format::NowebOutput)
function convert_to_noweb(doc)
output = ""
for chunk in doc.chunks
if isa(chunk, DocChunk)
@ -152,28 +155,5 @@ function convert_doc(doc::WeaveDoc, format::NowebOutput)
return output
end
"""Convert Weave document to script format"""
function convert_doc(doc::WeaveDoc, format::ScriptOutput)
output = ""
for chunk in doc.chunks
if typeof(chunk) == Weave.DocChunk
content = join([repr(c) for c in chunk.content], "")
output *= join(["#' " * s for s in split(content, "\n")], "\n")
else
output *= "\n#+ "
isempty(chunk.optionstring) || (output *= strip(chunk.optionstring))
output *= "\n\n" * lstrip(chunk.content)
output *= "\n"
end
end
return output
end
function Base.repr(c::InlineText)
return c.content
end
function Base.repr(c::InlineCode)
return "`j $(c.content)`"
end
Base.repr(c::InlineText) = c.content
Base.repr(c::InlineCode) = "`j $(c.content)`"

View File

@ -1,5 +1,4 @@
using Markdown
import .WeaveMarkdown
using Markdown, .WeaveMarkdown
# Contains report global properties
mutable struct Report <: AbstractDisplay
@ -78,17 +77,11 @@ function Base.display(report::Report, data)
end
end
function Base.display(report::Report, m::MIME"image/png", data)
figname = add_figure(report, data, m, ".png")
end
Base.display(report::Report, m::MIME"image/png", data) = add_figure(report, data, m, ".png")
function Base.display(report::Report, m::MIME"image/svg+xml", data)
figname = add_figure(report, data, m, ".svg")
end
Base.display(report::Report, m::MIME"image/svg+xml", data) = add_figure(report, data, m, ".svg")
function Base.display(report::Report, m::MIME"application/pdf", data)
figname = add_figure(report, data, m, ".pdf")
end
Base.display(report::Report, m::MIME"application/pdf", data) = add_figure(report, data, m, ".pdf")
#Text is written to stdout, called from "term" mode chunks
function Base.display(report::Report, m::MIME"text/plain", data)

View File

@ -1,11 +1,9 @@
import Mustache, Highlights
import .WeaveMarkdown
using Dates
using Markdown
using Mustache, Highlights
using .WeaveMarkdown, Markdown, Dates
using REPL.REPLCompletions: latex_symbols
function format(doc::WeaveDoc)
formatted = AbstractString[]
formatted = String[]
docformat = doc.format
# Complete format dictionaries with defaults
@ -29,19 +27,12 @@ function format(doc::WeaveDoc)
formatted = join(formatted, "\n")
# Render using a template if needed
rendered = render_doc(formatted, doc, doc.format)
return rendered
return render_doc(formatted, doc)
end
"""
render_doc(formatted::AbstractString, format)
render_doc(formatted, doc) = render_doc(formatted, doc, doc.format)
Render formatted document to a template
"""
function render_doc(formatted, doc::WeaveDoc, format)
return formatted
end
render_doc(formatted, doc, format) = formatted
function highlight(
mime::MIME,
@ -56,7 +47,7 @@ function stylesheet(m::MIME, theme)
return sprint((io, x) -> Highlights.stylesheet(io, m, x), theme)
end
function render_doc(formatted, doc::WeaveDoc, format::JMarkdown2HTML)
function render_doc(formatted, doc, format::JMarkdown2HTML)
css = stylesheet(MIME("text/html"), doc.highlight_theme)
path, wsource = splitdir(abspath(doc.source))
# wversion = string(Pkg.installed("Weave"))
@ -94,7 +85,7 @@ function render_doc(formatted, doc::WeaveDoc, format::JMarkdown2HTML)
)
end
function render_doc(formatted, doc::WeaveDoc, format::JMarkdown2tex)
function render_doc(formatted, doc, format::JMarkdown2tex)
highlight = stylesheet(MIME("text/latex"), doc.highlight_theme)
path, wsource = splitdir(abspath(doc.source))
# wversion = string(Pkg.installed("Weave"))

View File

@ -43,21 +43,18 @@ function run_doc(
)
# cache :all, :user, :off, :refresh
doc.cwd = get_cwd(doc, out_path)
# doctype detection is unnecessary here, but existing unit test requires this.
isnothing(doctype) && (doctype = detect_doctype(doc.source))
doc.doctype = doctype
doc.format = formats[doctype]
if (haskey(doc.format.formatdict, :keep_unicode))
if haskey(doc.format.formatdict, :keep_unicode)
doc.format.formatdict[:keep_unicode] = latex_keep_unicode
end
doc.cwd = get_cwd(doc, out_path)
isdir(doc.cwd) || mkpath(doc.cwd)
if occursin("2pdf", doctype) && cache == :off
fig_path = mktempdir(abspath(doc.cwd))
elseif occursin("2html", doctype)
if (occursin("2pdf", doctype) && cache == :off) || occursin("2html", doctype)
fig_path = mktempdir(abspath(doc.cwd))
end
@ -70,14 +67,10 @@ function run_doc(
set_rc_params(doc, fig_path, fig_ext)
# New sandbox for each document with args exposed
isnothing(mod) && (mod::Module = sandbox::Module = Core.eval(Main, :(module $(gensym(:WeaveSandBox)) end)))
isnothing(mod) && (mod = sandbox = Core.eval(Main, :(module $(gensym(:WeaveSandBox)) end))::Module)
@eval mod WEAVE_ARGS = $args
if haskey(doc.format.formatdict, :mimetypes)
mimetypes = doc.format.formatdict[:mimetypes]
else
mimetypes = default_mime_types
end
mimetypes = get(doc.format.formatdict, :mimetypes, default_mime_types)
report = Report(doc.cwd, doc.basename, doc.format.formatdict, mimetypes, throw_errors)
pushdisplay(report)
@ -112,7 +105,7 @@ function run_doc(
cache !== :off && write_cache(doc, cache_path)
@isdefined(sandbox) && clear_module!(sandbox::Module)
@isdefined(sandbox) && clear_module!(sandbox)
catch err
rethrow(err)
finally
@ -139,26 +132,23 @@ function detect_doctype(pathname::AbstractString)
end
function run_chunk(chunk::CodeChunk, doc::WeaveDoc, report::Report, SandBox::Module)
@info("Weaving chunk $(chunk.number) from line $(chunk.start_line)")
result_chunks = eval_chunk(chunk, report, SandBox)
occursin("2html", report.formatdict[:doctype]) &&
(result_chunks = embed_figures(result_chunks, report.cwd))
return result_chunks
# TODO: integrate with Juno's progress metre
@info "Weaving chunk $(chunk.number) from line $(chunk.start_line)"
result = eval_chunk(chunk, report, SandBox)
occursin("2html", report.formatdict[:doctype]) && (embed_figures!(result, report.cwd))
return result
end
function embed_figures(chunk::CodeChunk, cwd)
chunk.figures = [img2base64(fig, cwd) for fig in chunk.figures]
return chunk
end
function embed_figures(result_chunks, cwd)
for i = 1:length(result_chunks)
figs = result_chunks[i].figures
if !isempty(figs)
result_chunks[i].figures = [img2base64(fig, cwd) for fig in figs]
end
function embed_figures!(chunk::CodeChunk, cwd)
for (i, fig) in enumerate(chunk.figures)
chunk.figures[i] = img2base64(fig, cwd)
end
end
function embed_figures!(chunks::Vector{CodeChunk}, cwd)
for chunk in chunks
embed_figures!(chunk, cwd)
end
return result_chunks
end
function img2base64(fig, cwd)
@ -193,8 +183,7 @@ function run_inline(inline::InlineCode, doc::WeaveDoc, report::Report, SandBox::
merge!(chunk.options, options)
chunks = eval_chunk(chunk, report, SandBox)
occursin("2html", report.formatdict[:doctype]) &&
(chunks = embed_figures(chunks, report.cwd))
occursin("2html", report.formatdict[:doctype]) && (embed_figures!(chunks, report.cwd))
output = chunks[1].output
endswith(output, "\n") && (output = output[1:end-1])
@ -270,17 +259,16 @@ function capture_output(expr, SandBox::Module, term, disp, lastline, throw_error
end
# Parse chunk input to array of expressions
function parse_input(input::AbstractString)
parsed = Tuple{AbstractString,Any}[]
input = lstrip(input)
n = sizeof(input)
function parse_input(s)
res = []
s = lstrip(s)
n = sizeof(s)
pos = 1 # The first character is extra line end
while pos n
oldpos = pos
code, pos = Meta.parse(input, pos)
push!(parsed, (input[oldpos:pos-1], code))
while (oldpos = pos) n
ex, pos = Meta.parse(s, pos)
push!(res, (s[oldpos:pos-1], ex))
end
parsed
return res
end
function eval_chunk(chunk::CodeChunk, report::Report, SandBox::Module)
@ -321,13 +309,9 @@ function eval_chunk(chunk::CodeChunk, report::Report, SandBox::Module)
# chunk.options[:fig] && (chunk.figures = copy(report.figures))
# end
chunks
return chunks
end
# function eval_chunk(chunk::DocChunk, report::Report, SandBox)
# chunk
# end
"""
clear_module!(mod::Module)

View File

@ -31,7 +31,7 @@ o = Weave.format_output(doc.chunks[4].content, doc.format)
@test o_check == o
doc.template = "templates/mini.tpl"
rendered = Weave.render_doc("Hello", doc, doc.format)
rendered = Weave.render_doc("Hello", doc)
@test rendered == "\nHello\n"
# Tex format
@ -48,7 +48,7 @@ o = Weave.format_output(doc.chunks[3].content, doc.format)
@test o_check == o
doc.template = "templates/mini.tpl"
rendered = Weave.render_doc("Hello", doc, doc.format)
rendered = Weave.render_doc("Hello", doc)
@test rendered == "\nHello\n"
# Test header parsing and stripping

View File

@ -1,6 +1,5 @@
using Weave
using Weave, Test
using Weave: run_doc
using Test
# TODO: add test for header processsing
@ -27,8 +26,8 @@ mock_doc(str, chunk_parser = Weave.parse_markdown) = Weave.WeaveDoc("dummy", chu
include("test_error_rendering.jl")
end
@testset "Conversions" begin
include("convert_test.jl")
@testset "convertions" begin
include("test_converter.jl")
end
@testset "Formatters" begin

View File

@ -1,5 +1,4 @@
using Weave
using Test
# TODO: refactor
function convert_test(outfile, infile="documents/chunk_options.noweb")
outfile = joinpath("documents/convert", outfile)
@ -16,17 +15,9 @@ convert_test("chunk_options.mdw")
convert_test("chunk_options_nb.mdw", "documents/chunk_options.ipynb")
# Separate test for notebook (output depends on julia version)
function contents(chunk::Weave.DocChunk)
return join([strip(c.content) for c in chunk.content], "")
end
function contents(chunk::Weave.CodeChunk)
return chunk.content
end
function contents(doc::Weave.WeaveDoc)
return join([contents(chunk) for chunk in doc.chunks], "")
end
contents(chunk::Weave.DocChunk) = join([strip(c.content) for c in chunk.content], "")
contents(chunk::Weave.CodeChunk) = chunk.content
contents(doc::Weave.WeaveDoc) = join([contents(chunk) for chunk in doc.chunks], "")
outfile = "documents/convert/chunk_options.ipynb"
infile = "documents/chunk_options.noweb"