Merge pull request #331 from JunoLab/avi/progres

integrate with Juno progress bar

.travis.yml

@@ -20,8 +20,7 @@ jobs:
       julia: 1
       os: linux
       script:
-        - julia --project=doc/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));
-                                               Pkg.instantiate()'
+        - julia --project=doc/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
         - julia --project=doc/ doc/make.jl
       after_success: skip
 

doc/src/header.md

@@ -5,7 +5,7 @@ A YAML header should be in the beginning of the input document delimited with `-
 
 
 !!! warning
-    YAML header configuration is only supported when `weave`ing [Julia markdown documents](@ref document-syntax).
+    YAML header configuration is only supported when `weave`ing [markdown or Noweb syntax documents](@ref document-syntax).
 
 
 ## Document Metadata
@@ -31,20 +31,20 @@ date: 16th May 2020
 
 ### Dynamic Metadata
 
-The metadata can be "dynamic"; if you have [inline code](@ref) within YAML header, they will be evaluated _after_ evaluating all the chunks and replaced with the results.
+The metadata can be given "dynamically"; if you have [inline code](@ref) within YAML header, they will be evaluated _after_ evaluating all the chunks and replaced with the results.
 
 The example document below will set `date` metadata dynamically.
 Note that `Date` is available since the chunk is evaluated first.
 ```md
-    ---
-    title : Header Example
-    author : Shuhei Kadowaki
-    date: `j Date(now())`
-    ---
+ ---
+ title : Header Example
+ author : Shuhei Kadowaki
+ date: `j Date(now())`
+ ---
 
-    ```julia; echo = false
-    using Datas
-    ```
+ ```julia; echo = false
+ using Datas
+ ```
 ```
 
 

doc/src/usage.md

@@ -75,7 +75,7 @@ return "noweb"
 
 ## Documentation Chunks
 
-In Markdown and Noweb input formats documentation chunks are the parts that aren't inside code delimiters. Documentation chunks can be written with several different markup languages.
+In markdown and Noweb input formats documentation chunks are the parts that aren't inside code delimiters. Documentation chunks can be written with several different markup languages.
 
 ## Code Chunks
 
@@ -148,7 +148,7 @@ Weave will remove the first empty space from each line of documentation.
 
 ## Configuration via YAML Header
 
-When `weave`ing markdown files, you use YAML header to provide additional metadata and configuration options.
+When `weave`ing markdown files, you can use YAML header to provide additional metadata and configuration options.
 See [Header Configuration](@ref) section for more details.
 
 

src/cache.jl

@@ -26,7 +26,7 @@ function restore_chunk(chunk::CodeChunk, cached)
 
     # Chunk types, don't match after loading. Fix by constructing chunks
     # from loaded content
-    new_chunks = Any[]
+    new_chunks = []
     for c in chunks
         newc = CodeChunk(c.content, c.number, c.start_line, c.optionstring, c.options)
         newc.result_no = c.result_no

src/reader/markdown.jl

@@ -1,12 +1,6 @@
-"""
-    parse_markdown(document_body, is_pandoc = false)::Vector{WeaveChunk}
-    parse_markdown(document_body, code_start, code_end)::Vector{WeaveChunk}
-
-Parses Weave markdown and returns [`WeaveChunk`](@ref)s.
-"""
-function parse_markdown end
-
-function parse_markdown(document_body, is_pandoc = false)::Vector{WeaveChunk}
+function parse_markdown(document_body; is_pandoc = false)
+    header_text, document_body, offset = separate_header_text(document_body)
+    header = parse_header(header_text)
     code_start, code_end = if is_pandoc
         r"^<<(?<options>.*?)>>=\s*$",
         r"^@\s*$"
@@ -14,10 +8,45 @@ function parse_markdown(document_body, is_pandoc = false)::Vector{WeaveChunk}
         r"^[`~]{3}(?:\{?)julia(?:;?)\s*(?<options>.*?)(\}|\s*)$",
         r"^[`~]{3}\s*$"
     end
-    return parse_markdown(document_body, code_start, code_end)
+    return header, parse_markdown_body(document_body, code_start, code_end, offset)
 end
 
-function parse_markdown(document_body, code_start, code_end)::Vector{WeaveChunk}
+# headers
+# -------
+
+const HEADER_REGEX = r"^---$(?<header>((?!---).)+)^---$"ms
+
+# TODO: non-Weave headers should keep live in a doc
+# separates header section from `text`
+function separate_header_text(text)
+    m = match(HEADER_REGEX, text)
+    isnothing(m) && return "", text, 0
+    header_text = m[:header]
+    offset = @static if VERSION ≥ v"1.4"
+        count("\n", header_text)
+    else
+        count(c->c==='\n', header_text)
+    end
+    return header_text, replace(text, HEADER_REGEX => ""; count = 1), offset
+end
+
+# HACK:
+# YAML.jl can't parse text including ``` characters, so first replace all the inline code
+# with these temporary code start/end string
+const HEADER_INLINE_START = "<weave_header_inline_start>"
+const   HEADER_INLINE_END = "<weave_header_inline_end>"
+
+function parse_header(header_text)
+    isempty(header_text) && return Dict()
+    pat = INLINE_REGEX => SubstitutionString("$(HEADER_INLINE_START)\\1$(HEADER_INLINE_END)")
+    header_text = replace(header_text, pat)
+    return YAML.load(header_text)
+end
+
+# body
+# ----
+
+function parse_markdown_body(document_body, code_start, code_end, offset)
     lines = split(document_body, '\n')
 
     state = "doc"
@@ -25,7 +54,7 @@ function parse_markdown(document_body, code_start, code_end)::Vector{WeaveChunk}
     docno = 1
     codeno = 1
     content = ""
-    start_line = 0
+    start_line = offset
 
     options = Dict()
     optionString = ""
@@ -57,7 +86,7 @@ function parse_markdown(document_body, code_start, code_end)::Vector{WeaveChunk}
             end
 
             content = ""
-            start_line = lineno
+            start_line = lineno + offset
 
             continue
         end
@@ -66,7 +95,7 @@ function parse_markdown(document_body, code_start, code_end)::Vector{WeaveChunk}
             chunk = CodeChunk(content, codeno, start_line, optionString, options)
 
             codeno += 1
-            start_line = lineno
+            start_line = lineno + offset
             content = ""
             state = "doc"
             push!(chunks, chunk)

src/reader/notebook.jl

@@ -1,9 +1,4 @@
-"""
-    parse_notebook(document_body)::Vector{WeaveChunk}
-
-Parses Jupyter notebook and returns [`WeaveChunk`](@ref)s.
-"""
-function parse_notebook(document_body)::Vector{WeaveChunk}
+function parse_notebook(document_body)
     nb = JSON.parse(document_body)
     chunks = WeaveChunk[]
     options = Dict{Symbol,Any}()
@@ -25,5 +20,5 @@ function parse_notebook(document_body)::Vector{WeaveChunk}
         end
     end
 
-    return chunks
+    return Dict(), chunks
 end

src/reader/reader.jl

@@ -55,11 +55,8 @@ end
 function parse_doc(document, informat)
     document = replace(document, "\r\n" => "\n") # normalize line ending
 
-    header_text, document = separate_header_text(document)
-
-    return parse_header(header_text),
-        informat == "markdown" ? parse_markdown(document) :
-        informat == "noweb" ? parse_markdown(document, true) :
+    return informat == "markdown" ? parse_markdown(document) :
+        informat == "noweb" ? parse_markdown(document; is_pandoc = true) :
         informat == "script" ? parse_script(document) :
         informat == "notebook" ? parse_notebook(document) :
         error("unsupported input format given: $informat")
@@ -128,32 +125,6 @@ end
 
 parse_inline(text) = Inline[InlineText(text, 1, length(text), 1)]
 
-# headers
-# -------
-
-const HEADER_REGEX = r"^---$(?<header>((?!---).)+)^---$"ms
-
-# TODO: non-Weave headers should keep live in a doc
-# separates header section from `text`
-function separate_header_text(text)
-    m = match(HEADER_REGEX, text)
-    isnothing(m) && return "", text
-    return m[:header], replace(text, HEADER_REGEX => ""; count = 1)
-end
-
-# HACK:
-# YAML.jl can't parse text including ``` characters, so first replace all the inline code
-# with these temporary code start/end string
-const HEADER_INLINE_START = "<weave_header_inline_start>"
-const   HEADER_INLINE_END = "<weave_header_inline_end>"
-
-function parse_header(header_text)
-    isempty(header_text) && return Dict()
-    pat = INLINE_REGEX => SubstitutionString("$(HEADER_INLINE_START)\\1$(HEADER_INLINE_END)")
-    header_text = replace(header_text, pat)
-    return YAML.load(header_text)
-end
-
 include("markdown.jl")
 include("script.jl")
 include("notebook.jl")

src/reader/script.jl

@@ -1,9 +1,4 @@
-"""
-    parse_script(document_body)::Vector{WeaveChunk}
-
-Parse Julia script and returns [`WeaveChunk`](@ref)s.
-"""
-function parse_script(document_body)::Vector{WeaveChunk}
+function parse_script(document_body)
     lines = split(document_body, "\n")
 
     doc_line = r"(^#'.*)|(^#%%.*)|(^# %%.*)"
@@ -20,8 +15,6 @@ function parse_script(document_body)::Vector{WeaveChunk}
     optionString = ""
     chunks = WeaveChunk[]
     state = "code"
-    lineno = 1
-    n_emptylines = 0
 
     for lineno = 1:length(lines)
         line = lines[lineno]
@@ -81,12 +74,6 @@ function parse_script(document_body)::Vector{WeaveChunk}
             docno += 1
         end
         read *= line * "\n"
-
-        if strip(line) == ""
-            n_emptylines += 1
-        else
-            n_emptylines = 0
-        end
     end
 
     # Handle the last chunk
@@ -98,5 +85,5 @@ function parse_script(document_body)::Vector{WeaveChunk}
         push!(chunks, chunk)
     end
 
-    return chunks
+    return Dict(), chunks
 end

src/run.jl

@@ -1,5 +1,8 @@
 using Base64
 
+
+const PROGRESS_ID = "weave_progress"
+
 """
     run_doc(doc::WeaveDoc; kwargs...)
 
@@ -83,20 +86,23 @@ function run_doc(
         end
 
         executed = []
+        n = length(filter(chunk->isa(chunk,CodeChunk), doc.chunks))
+        i = 0
         for chunk in doc.chunks
-            if isa(chunk, CodeChunk)
+            if chunk isa CodeChunk
                 options = merge(doc.chunk_defaults, chunk.options)
                 merge!(chunk.options, options)
+
+                @info "Weaving chunk $(chunk.number) from line $(chunk.start_line)" progress=(i)/n _id=PROGRESS_ID
+                i+=1
             end
 
-            restore = (cache === :user && typeof(chunk) == CodeChunk && chunk.options[:cache])
-
-            result_chunks = if cached != nothing && (cache === :all || restore)
+            restore = (cache === :user && chunk isa CodeChunk && chunk.options[:cache])
+            result_chunks = if cached ≠ nothing && (cache === :all || restore)
                 restore_chunk(chunk, cached)
             else
                 run_chunk(chunk, doc, report, mod)
             end
-
             executed = [executed; result_chunks]
         end
 
@@ -111,6 +117,7 @@ function run_doc(
     catch err
         rethrow(err)
     finally
+        @info "Weaved all chunks" progress=1 _id=PROGRESS_ID
         popdisplay(report) # ensure display pops out even if internal error occurs
     end
 
@@ -133,10 +140,8 @@ function detect_doctype(pathname::AbstractString)
     return "pandoc"
 end
 
-function run_chunk(chunk::CodeChunk, doc::WeaveDoc, report::Report, SandBox::Module)
-    # TODO: integrate with Juno's progress metre
-    @info "Weaving chunk $(chunk.number) from line $(chunk.start_line)"
-    result = eval_chunk(chunk, report, SandBox)
+function run_chunk(chunk::CodeChunk, doc, report, mod)
+    result = eval_chunk(chunk, report, mod)
     occursin("2html", report.formatdict[:doctype]) && (embed_figures!(result, report.cwd))
     return result
 end
@@ -169,8 +174,8 @@ function img2base64(fig, cwd)
     end
 end
 
-function run_chunk(chunk::DocChunk, doc::WeaveDoc, report::Report, SandBox::Module)
-    chunk.content = [run_inline(c, doc, report, SandBox) for c in chunk.content]
+function run_chunk(chunk::DocChunk, doc, report, mod)
+    chunk.content = [run_inline(c, doc, report, mod) for c in chunk.content]
     return chunk
 end
 

test/gadfly_formats.jl

@@ -4,9 +4,11 @@ using Gadfly, Cairo
 
 
 function test_gadfly(doctype, fig_ext)
-    out = weave(joinpath(@__DIR__ , "documents/gadfly_formats_test.jnw"),
-        out_path = joinpath(@__DIR__ , "documents/gadfly/"),
-        doctype = doctype, fig_ext = fig_ext)
+    out = weave(
+        joinpath(@__DIR__ , "documents/gadfly_formats_test.jnw"),
+        doctype = doctype,
+        fig_ext = fig_ext
+    )
     result = read(out, String)
     # cp(out, out*fig_ext*"."*doctype, force=true) # Used when adding new tests
     ref =  read(out*fig_ext*"."*doctype, String)

test/inline_test.jl

@@ -17,14 +17,20 @@ ms = collect(eachmatch(Weave.INLINE_REGEXES, doc))
 @test ms[2][1] == "code"
 @test ms[3][1] == "show(\"is\")"
 
-chunk = Weave.parse_markdown(doc)[1]
+let
+_, chunks = Weave.parse_markdown(doc)
+chunk = first(chunks)
 @test length(chunk.content) == 7
 @test chunk.content[2].content == ms[1][2]
 @test chunk.content[4].content == ms[2][1]
 @test chunk.content[6].content == ms[3][1]
+end
 
-chunknw = Weave.parse_markdown(doc, false)[1]
-@test all([chunknw.content[i].content == chunk.content[i].content for i in 1:7])
+let
+_, chunks = Weave.parse_markdown(doc)
+chunk = first(chunks)
+@test all([chunk.content[i].content == chunk.content[i].content for i in 1:7])
+end
 
 # Test with document