Merge pull request #330 from JunoLab/avi/refactor

refactor

doc/src/notebooks.md

@@ -45,5 +45,5 @@ convert_doc("FIR_design.ipynb", "FIR_design.jmd")
 ```
 
 ```@docs
-convert_doc(infile::AbstractString, outfile::AbstractString)
+convert_doc
 ```

src/Weave.jl

@@ -179,24 +179,25 @@ function weave(
     open(io->write(io,formatted), outname, "w")
 
     # Special for that need external programs
-    if doc.doctype == "pandoc2html"
+    doctype = doc.doctype
+    if doctype == "pandoc2html"
         mdname = outname
         outname = get_outname(out_path, doc, ext = "html")
         pandoc2html(formatted, doc, outname, pandoc_options)
         rm(mdname)
-    elseif doc.doctype == "pandoc2pdf"
+    elseif doctype == "pandoc2pdf"
         mdname = outname
         outname = get_outname(out_path, doc, ext = "pdf")
         pandoc2pdf(formatted, doc, outname, pandoc_options)
         rm(mdname)
-    elseif doc.doctype == "md2pdf"
+    elseif doctype == "md2pdf"
         success = run_latex(doc, outname, latex_cmd)
         success || return
         outname = get_outname(out_path, doc, ext = "pdf")
     end
 
     doc.cwd == pwd() && (outname = basename(outname))
-    @info("Report weaved to $outname")
+    @info "Report weaved to $outname"
     return abspath(outname)
 end
 
@@ -236,7 +237,7 @@ function notebook(
     jupyter_path::AbstractString = "jupyter",
 )
     doc = WeaveDoc(source)
-    converted = convert_doc(doc, NotebookOutput())
+    converted = convert_to_notebook(doc)
     doc.cwd = get_cwd(doc, out_path)
     outfile = get_outname(out_path, doc, ext = "ipynb")
 
@@ -245,7 +246,7 @@ function notebook(
     end
 
     @info "Running nbconvert"
-    return out = read(
+    return read(
         `$jupyter_path nbconvert --ExecutePreprocessor.timeout=$timeout --to notebook --execute $outfile  $nbconvert_options --output $outfile`,
         String,
     )
@@ -272,11 +273,12 @@ function include_weave(
             "\n",
         )
         include_string(m, code)
-    catch e
+    catch err
-        throw(e)
+        throw(err)
     finally
         cd(old_path)
     end
+    return nothing
 end
 
 include_weave(source, informat = nothing) = include_weave(Main, source, informat)
@@ -303,7 +305,7 @@ include("cache.jl")
 include("formatters.jl")
 include("format.jl")
 include("pandoc.jl")
-include("writers.jl")
+include("converter.jl")
 
 export weave,
     list_out_formats,

src/converter.jl

@@ -1,58 +1,79 @@
-import JSON
+using JSON, Mustache
-import Mustache
-
-mutable struct NotebookOutput end
-
-mutable struct MarkdownOutput end
-
-mutable struct NowebOutput end
-
-mutable struct ScriptOutput end
-
-const output_formats = Dict{String,Any}(
-    "noweb" => NowebOutput(),
-    "notebook" => NotebookOutput(),
-    "markdown" => MarkdownOutput(),
-    "script" => ScriptOutput(),
-)
-
-"""Autodetect format for converter"""
-function detect_outformat(outfile::String)
-    ext = lowercase(splitext(outfile)[2])
-
-    ext == ".jl" && return "script"
-    ext == ".jmd" && return "markdown"
-    ext == ".ipynb" && return "notebook"
-    return "noweb"
-end
 
 """
-    convert_doc(infile::AbstractString, outfile::AbstractString; format::Union{Nothing,AbstractString} = nothing)
+    convert_doc(infile::AbstractString, outfile::AbstractString; outformat::Union{Nothing,AbstractString} = nothing)
 
 Convert Weave documents between different formats
 
 - `infile`: Path of the input document
 - `outfile`: Path of the output document
-- `format = nothing`: Output document format (optional). It will be detected automatically from the `outfile` extension. You can also specify either of `"script"`, `"markdown"`, `"notebook"`, or `"noweb"`
+- `outformat = nothing`: Output document format (optional). By default (i.e. given `nothing`) Weave will try to automatically detect it from the `outfile`'s extension. You can also specify either of `"script"`, `"markdown"`, `"notebook"`, or `"noweb"`
 """
 function convert_doc(
     infile::AbstractString,
     outfile::AbstractString;
-    format::Union{Nothing,AbstractString} = nothing,
+    outformat::Union{Nothing,AbstractString} = nothing,
 )
     doc = WeaveDoc(infile)
 
-    isnothing(format) && (format = detect_outformat(outfile))
+    if isnothing(outformat)
+        ext = lowercase(splitext(outfile)[2])
+        outformat =
+            ext == ".jl" ? "script" :
+            ext == ".jmd" ?  "markdown" :
+            ext == ".ipynb" ? "notebook" :
+            "noweb" # fallback
+    end
 
-    converted = convert_doc(doc, output_formats[format])
+    converted = _convert_doc(doc, outformat)
 
     open(outfile, "w") do f
         write(f, converted)
     end
+    return outfile
 end
 
-"""Convert Weave document to Jupyter notebook format"""
+function _convert_doc(doc, outformat)
-function convert_doc(doc::WeaveDoc, format::NotebookOutput)
+    outformat == "script" ? convert_to_script(doc) :
+    outformat == "markdown" ? convert_to_markdown(doc) :
+    outformat == "notebook" ? convert_to_notebook(doc) :
+    convert_to_noweb(doc)
+end
+
+function convert_to_script(doc)
+    output = ""
+    for chunk in doc.chunks
+        if typeof(chunk) == Weave.DocChunk
+            content = join([repr(c) for c in chunk.content], "")
+            output *= join(["#' " * s for s in split(content, "\n")], "\n")
+        else
+            output *= "\n#+ "
+            isempty(chunk.optionstring) || (output *= strip(chunk.optionstring))
+            output *= "\n\n" * lstrip(chunk.content)
+            output *= "\n"
+        end
+    end
+
+    return output
+end
+
+function convert_to_markdown(doc)
+    output = ""
+    for chunk in doc.chunks
+        if isa(chunk, DocChunk)
+            output *= join([repr(c) for c in chunk.content], "")
+        else
+            output *= "\n" * "```julia"
+            isempty(chunk.optionstring) || (output *= ";" * chunk.optionstring)
+            output *= "\n" * lstrip(chunk.content)
+            output *= "```\n"
+        end
+    end
+
+    return output
+end
+
+function convert_to_notebook(doc)
     nb = Dict()
     nb["nbformat"] = 4
     nb["nbformat_minor"] = 2
@@ -117,25 +138,7 @@ function convert_doc(doc::WeaveDoc, format::NotebookOutput)
     return json_nb
 end
 
-"""Convert Weave document to Jupyter notebook format"""
+function convert_to_noweb(doc)
-function convert_doc(doc::WeaveDoc, format::MarkdownOutput)
-    output = ""
-    for chunk in doc.chunks
-        if isa(chunk, DocChunk)
-            output *= join([repr(c) for c in chunk.content], "")
-        else
-            output *= "\n" * "```julia"
-            isempty(chunk.optionstring) || (output *= ";" * chunk.optionstring)
-            output *= "\n" * lstrip(chunk.content)
-            output *= "```\n"
-        end
-    end
-
-    return output
-end
-
-"""Convert Weave document to noweb format"""
-function convert_doc(doc::WeaveDoc, format::NowebOutput)
     output = ""
     for chunk in doc.chunks
         if isa(chunk, DocChunk)
@@ -152,28 +155,5 @@ function convert_doc(doc::WeaveDoc, format::NowebOutput)
     return output
 end
 
-"""Convert Weave document to script format"""
+Base.repr(c::InlineText) = c.content
-function convert_doc(doc::WeaveDoc, format::ScriptOutput)
+Base.repr(c::InlineCode) = "`j $(c.content)`"
-    output = ""
-    for chunk in doc.chunks
-        if typeof(chunk) == Weave.DocChunk
-            content = join([repr(c) for c in chunk.content], "")
-            output *= join(["#' " * s for s in split(content, "\n")], "\n")
-        else
-            output *= "\n#+ "
-            isempty(chunk.optionstring) || (output *= strip(chunk.optionstring))
-            output *= "\n\n" * lstrip(chunk.content)
-            output *= "\n"
-        end
-    end
-
-    return output
-end
-
-function Base.repr(c::InlineText)
-    return c.content
-end
-
-function Base.repr(c::InlineCode)
-    return "`j $(c.content)`"
-end

src/display_methods.jl

@@ -1,5 +1,4 @@
-using Markdown
+using Markdown, .WeaveMarkdown
-import .WeaveMarkdown
 
 # Contains report global properties
 mutable struct Report <: AbstractDisplay
@@ -78,17 +77,11 @@ function Base.display(report::Report, data)
     end
 end
 
-function Base.display(report::Report, m::MIME"image/png", data)
+Base.display(report::Report, m::MIME"image/png", data) = add_figure(report, data, m, ".png")
-    figname = add_figure(report, data, m, ".png")
-end
 
-function Base.display(report::Report, m::MIME"image/svg+xml", data)
+Base.display(report::Report, m::MIME"image/svg+xml", data) = add_figure(report, data, m, ".svg")
-    figname = add_figure(report, data, m, ".svg")
-end
 
-function Base.display(report::Report, m::MIME"application/pdf", data)
+Base.display(report::Report, m::MIME"application/pdf", data) = add_figure(report, data, m, ".pdf")
-    figname = add_figure(report, data, m, ".pdf")
-end
 
 #Text is written to stdout, called from "term" mode chunks
 function Base.display(report::Report, m::MIME"text/plain", data)

src/format.jl

@@ -1,11 +1,9 @@
-import Mustache, Highlights
+using Mustache, Highlights
-import .WeaveMarkdown
+using .WeaveMarkdown, Markdown, Dates
-using Dates
-using Markdown
 using REPL.REPLCompletions: latex_symbols
 
 function format(doc::WeaveDoc)
-    formatted = AbstractString[]
+    formatted = String[]
     docformat = doc.format
 
     # Complete format dictionaries with defaults
@@ -29,19 +27,12 @@ function format(doc::WeaveDoc)
 
     formatted = join(formatted, "\n")
     # Render using a template if needed
-    rendered = render_doc(formatted, doc, doc.format)
+    return render_doc(formatted, doc)
-
-    return rendered
 end
 
-"""
+render_doc(formatted, doc) = render_doc(formatted, doc, doc.format)
-  render_doc(formatted::AbstractString, format)
 
-Render formatted document to a template
+render_doc(formatted, doc, format) = formatted
-"""
-function render_doc(formatted, doc::WeaveDoc, format)
-    return formatted
-end
 
 function highlight(
     mime::MIME,
@@ -56,7 +47,7 @@ function stylesheet(m::MIME, theme)
     return sprint((io, x) -> Highlights.stylesheet(io, m, x), theme)
 end
 
-function render_doc(formatted, doc::WeaveDoc, format::JMarkdown2HTML)
+function render_doc(formatted, doc, format::JMarkdown2HTML)
     css = stylesheet(MIME("text/html"), doc.highlight_theme)
     path, wsource = splitdir(abspath(doc.source))
     # wversion = string(Pkg.installed("Weave"))
@@ -94,7 +85,7 @@ function render_doc(formatted, doc::WeaveDoc, format::JMarkdown2HTML)
     )
 end
 
-function render_doc(formatted, doc::WeaveDoc, format::JMarkdown2tex)
+function render_doc(formatted, doc, format::JMarkdown2tex)
     highlight = stylesheet(MIME("text/latex"), doc.highlight_theme)
     path, wsource = splitdir(abspath(doc.source))
     # wversion = string(Pkg.installed("Weave"))

src/run.jl

@@ -43,21 +43,18 @@ function run_doc(
 )
     # cache :all, :user, :off, :refresh
 
-    doc.cwd = get_cwd(doc, out_path)
     # doctype detection is unnecessary here, but existing unit test requires this.
     isnothing(doctype) && (doctype = detect_doctype(doc.source))
     doc.doctype = doctype
     doc.format = formats[doctype]
 
-    if (haskey(doc.format.formatdict, :keep_unicode))
+    if haskey(doc.format.formatdict, :keep_unicode)
         doc.format.formatdict[:keep_unicode] = latex_keep_unicode
     end
 
+    doc.cwd = get_cwd(doc, out_path)
     isdir(doc.cwd) || mkpath(doc.cwd)
-
+    if (occursin("2pdf", doctype) && cache == :off) || occursin("2html", doctype)
-    if occursin("2pdf", doctype) && cache == :off
-        fig_path = mktempdir(abspath(doc.cwd))
-    elseif occursin("2html", doctype)
         fig_path = mktempdir(abspath(doc.cwd))
     end
 
@@ -70,14 +67,10 @@ function run_doc(
     set_rc_params(doc, fig_path, fig_ext)
 
     # New sandbox for each document with args exposed
-    isnothing(mod) && (mod::Module = sandbox::Module = Core.eval(Main, :(module $(gensym(:WeaveSandBox)) end)))
+    isnothing(mod) && (mod = sandbox = Core.eval(Main, :(module $(gensym(:WeaveSandBox)) end))::Module)
     @eval mod WEAVE_ARGS = $args
 
-    if haskey(doc.format.formatdict, :mimetypes)
+    mimetypes = get(doc.format.formatdict, :mimetypes, default_mime_types)
-        mimetypes = doc.format.formatdict[:mimetypes]
-    else
-        mimetypes = default_mime_types
-    end
 
     report = Report(doc.cwd, doc.basename, doc.format.formatdict, mimetypes, throw_errors)
     pushdisplay(report)
@@ -112,7 +105,7 @@ function run_doc(
 
         cache !== :off && write_cache(doc, cache_path)
 
-        @isdefined(sandbox) && clear_module!(sandbox::Module)
+        @isdefined(sandbox) && clear_module!(sandbox)
     catch err
         rethrow(err)
     finally
@@ -139,26 +132,23 @@ function detect_doctype(pathname::AbstractString)
 end
 
 function run_chunk(chunk::CodeChunk, doc::WeaveDoc, report::Report, SandBox::Module)
-    @info("Weaving chunk $(chunk.number) from line $(chunk.start_line)")
+    # TODO: integrate with Juno's progress metre
-    result_chunks = eval_chunk(chunk, report, SandBox)
+    @info "Weaving chunk $(chunk.number) from line $(chunk.start_line)"
-    occursin("2html", report.formatdict[:doctype]) &&
+    result = eval_chunk(chunk, report, SandBox)
-        (result_chunks = embed_figures(result_chunks, report.cwd))
+    occursin("2html", report.formatdict[:doctype]) && (embed_figures!(result, report.cwd))
-    return result_chunks
+    return result
 end
 
-function embed_figures(chunk::CodeChunk, cwd)
+function embed_figures!(chunk::CodeChunk, cwd)
-    chunk.figures = [img2base64(fig, cwd) for fig in chunk.figures]
+    for (i, fig) in enumerate(chunk.figures)
-    return chunk
+        chunk.figures[i] = img2base64(fig, cwd)
-end
+    end
-
+end
-function embed_figures(result_chunks, cwd)
+
-    for i = 1:length(result_chunks)
+function embed_figures!(chunks::Vector{CodeChunk}, cwd)
-        figs = result_chunks[i].figures
+    for chunk in chunks
-        if !isempty(figs)
+        embed_figures!(chunk, cwd)
-            result_chunks[i].figures = [img2base64(fig, cwd) for fig in figs]
-        end
     end
-    return result_chunks
 end
 
 function img2base64(fig, cwd)
@@ -193,8 +183,7 @@ function run_inline(inline::InlineCode, doc::WeaveDoc, report::Report, SandBox::
     merge!(chunk.options, options)
 
     chunks = eval_chunk(chunk, report, SandBox)
-    occursin("2html", report.formatdict[:doctype]) &&
+    occursin("2html", report.formatdict[:doctype]) && (embed_figures!(chunks, report.cwd))
-        (chunks = embed_figures(chunks, report.cwd))
 
     output = chunks[1].output
     endswith(output, "\n") && (output = output[1:end-1])
@@ -270,17 +259,16 @@ function capture_output(expr, SandBox::Module, term, disp, lastline, throw_error
 end
 
 # Parse chunk input to array of expressions
-function parse_input(input::AbstractString)
+function parse_input(s)
-    parsed = Tuple{AbstractString,Any}[]
+    res = []
-    input = lstrip(input)
+    s = lstrip(s)
-    n = sizeof(input)
+    n = sizeof(s)
     pos = 1 # The first character is extra line end
-    while pos ≤ n
+    while (oldpos = pos) ≤ n
-        oldpos = pos
+        ex, pos = Meta.parse(s, pos)
-        code, pos = Meta.parse(input, pos)
+        push!(res, (s[oldpos:pos-1], ex))
-        push!(parsed, (input[oldpos:pos-1], code))
     end
-    parsed
+    return res
 end
 
 function eval_chunk(chunk::CodeChunk, report::Report, SandBox::Module)
@@ -321,13 +309,9 @@ function eval_chunk(chunk::CodeChunk, report::Report, SandBox::Module)
     #   chunk.options[:fig] && (chunk.figures = copy(report.figures))
     # end
 
-    chunks
+    return chunks
 end
 
-# function eval_chunk(chunk::DocChunk, report::Report, SandBox)
-#    chunk
-# end
-
 """
     clear_module!(mod::Module)
 

test/formatter_test.jl

@@ -31,7 +31,7 @@ o = Weave.format_output(doc.chunks[4].content, doc.format)
 @test o_check == o
 
 doc.template = "templates/mini.tpl"
-rendered = Weave.render_doc("Hello", doc, doc.format)
+rendered = Weave.render_doc("Hello", doc)
 @test rendered == "\nHello\n"
 
 # Tex format
@@ -48,7 +48,7 @@ o = Weave.format_output(doc.chunks[3].content, doc.format)
 @test o_check == o
 
 doc.template = "templates/mini.tpl"
-rendered = Weave.render_doc("Hello", doc, doc.format)
+rendered = Weave.render_doc("Hello", doc)
 @test rendered == "\nHello\n"
 
 # Test header parsing and stripping

test/runtests.jl

@@ -1,6 +1,5 @@
-using Weave
+using Weave, Test
 using Weave: run_doc
-using Test
 
 
 # TODO: add test for header processsing
@@ -27,8 +26,8 @@ mock_doc(str, chunk_parser = Weave.parse_markdown) = Weave.WeaveDoc("dummy", chu
         include("test_error_rendering.jl")
     end
 
-    @testset "Conversions" begin
+    @testset "convertions" begin
-        include("convert_test.jl")
+        include("test_converter.jl")
     end
 
     @testset "Formatters" begin

test/test_converter.jl

@@ -1,5 +1,4 @@
-using Weave
+# TODO: refactor
-using Test
 
 function convert_test(outfile, infile="documents/chunk_options.noweb")
   outfile = joinpath("documents/convert", outfile)
@@ -16,17 +15,9 @@ convert_test("chunk_options.mdw")
 convert_test("chunk_options_nb.mdw", "documents/chunk_options.ipynb")
 
 # Separate test for notebook (output depends on julia version)
-function contents(chunk::Weave.DocChunk)
+contents(chunk::Weave.DocChunk) = join([strip(c.content) for c in chunk.content], "")
-  return join([strip(c.content) for c in chunk.content], "")
+contents(chunk::Weave.CodeChunk) = chunk.content
-end
+contents(doc::Weave.WeaveDoc) = join([contents(chunk) for chunk in doc.chunks], "")
-
-function contents(chunk::Weave.CodeChunk)
-  return chunk.content
-end
-
-function contents(doc::Weave.WeaveDoc)
-  return join([contents(chunk) for chunk in doc.chunks], "")
-end
 
 outfile = "documents/convert/chunk_options.ipynb"
 infile = "documents/chunk_options.noweb"