mirror of https://github.com/mpastell/Weave.jl
Implemented caching, added documentation and tests
parent
6635eb5c74
commit
d19b374fe9
|
@ -8,6 +8,7 @@ examples/*.html
|
||||||
examples/*.rst
|
examples/*.rst
|
||||||
examples/*.tex
|
examples/*.tex
|
||||||
test/documents/figures
|
test/documents/figures
|
||||||
|
test/documents/cache
|
||||||
test/documents/output/figures
|
test/documents/output/figures
|
||||||
test/documents/output/gadfly_formats_test.txt
|
test/documents/output/gadfly_formats_test.txt
|
||||||
test/documents/*.tex
|
test/documents/*.tex
|
||||||
|
|
1
NEWS.md
1
NEWS.md
|
@ -3,6 +3,7 @@
|
||||||
|
|
||||||
### Changes in master
|
### Changes in master
|
||||||
|
|
||||||
|
* Simple caching of code chunks
|
||||||
* Each document is executed in separate sandbox module instead of redefining the same one. Fixes warnings and occasional segfaults.
|
* Each document is executed in separate sandbox module instead of redefining the same one. Fixes warnings and occasional segfaults.
|
||||||
* New chunk option: `line_width`.
|
* New chunk option: `line_width`.
|
||||||
* Bug fix in wrapping output lines.
|
* Bug fix in wrapping output lines.
|
||||||
|
|
2
REQUIRE
2
REQUIRE
|
@ -2,4 +2,4 @@ julia 0.3
|
||||||
Compat
|
Compat
|
||||||
ArgParse
|
ArgParse
|
||||||
Docile
|
Docile
|
||||||
HDF5
|
JSON
|
||||||
|
|
|
@ -59,6 +59,7 @@ Weave currently supports the following chunk options with the following defaults
|
||||||
* `label`. Chunk label, will be used for figure labels in Latex as fig:label
|
* `label`. Chunk label, will be used for figure labels in Latex as fig:label
|
||||||
* `wrap = true`. Wrap long lines from output.
|
* `wrap = true`. Wrap long lines from output.
|
||||||
* `line_width = 75`. Line width for wrapped lines.
|
* `line_width = 75`. Line width for wrapped lines.
|
||||||
|
* `cache = false`. Cache results, depends on `cache` parameter on `weave` function.
|
||||||
|
|
||||||
**Options for figures**
|
**Options for figures**
|
||||||
|
|
||||||
|
|
|
@ -90,17 +90,20 @@ weave(source ; doctype = "pandoc", plotlib="Gadfly",
|
||||||
`"somepath"`: Path as a string e.g `"/home/mpastell/weaveout"`
|
`"somepath"`: Path as a string e.g `"/home/mpastell/weaveout"`
|
||||||
* `fig_path`: where figures will be generated, relative to out_path
|
* `fig_path`: where figures will be generated, relative to out_path
|
||||||
* `fig_ext`: Extension for saved figures e.g. `".pdf"`, `".png"`. Default setting depends on `doctype`.
|
* `fig_ext`: Extension for saved figures e.g. `".pdf"`, `".png"`. Default setting depends on `doctype`.
|
||||||
|
* `cache_path`: where of cached output will be saved.
|
||||||
|
* `cache`: controls caching of code: `:off` = no caching, `:all` = cache everything,
|
||||||
|
`:user` = cache based on chunk options, `:refresh`, run all code chunks and save new cache.
|
||||||
|
|
||||||
**Note:** Run Weave from terminal and not using IJulia, Juno or ESS, they tend to mess with capturing output.
|
**Note:** Run Weave from terminal and not using IJulia, Juno or ESS, they tend to mess with capturing output.
|
||||||
""" ->
|
""" ->
|
||||||
function weave(source ; doctype = "pandoc", plotlib="Gadfly",
|
function weave(source ; doctype = "pandoc", plotlib="Gadfly",
|
||||||
informat="noweb", out_path=:doc, fig_path = "figures", fig_ext = nothing,
|
informat="noweb", out_path=:doc, fig_path = "figures", fig_ext = nothing,
|
||||||
cache_path = "cache")
|
cache_path = "cache", cache=:off)
|
||||||
|
|
||||||
doc = read_doc(source, informat) #Reader toimii, muuten kesken...
|
doc = read_doc(source, informat) #Reader toimii, muuten kesken...
|
||||||
doc = run(doc, doctype = doctype, plotlib=plotlib,
|
doc = run(doc, doctype = doctype, plotlib=plotlib,
|
||||||
informat = informat, out_path=out_path,
|
informat = informat, out_path=out_path,
|
||||||
fig_path = fig_path, fig_ext = fig_ext, cache_path = cache_path)
|
fig_path = fig_path, fig_ext = fig_ext, cache_path = cache_path, cache=cache)
|
||||||
formatted = format(doc)
|
formatted = format(doc)
|
||||||
|
|
||||||
outname = "$(doc.cwd)/$(doc.basename).$(doc.format.formatdict[:extension])"
|
outname = "$(doc.cwd)/$(doc.basename).$(doc.format.formatdict[:extension])"
|
||||||
|
|
41
src/cache.jl
41
src/cache.jl
|
@ -1,25 +1,38 @@
|
||||||
import HDF5, JLD
|
import JSON
|
||||||
|
|
||||||
|
|
||||||
function write_cache(doc::WeaveDoc, cache_path)
|
function write_cache(doc::WeaveDoc, cache_path)
|
||||||
isdir(cache_path) || mkdir(cache_path)
|
cache_dir = "$(doc.cwd)/$cache_path"
|
||||||
name = "$cache_path/$(doc.basename).jld"
|
isdir(cache_dir) || mkpath(cache_dir)
|
||||||
JLD.save(name, "doc", doc)
|
name = "$cache_dir/$(doc.basename).json"
|
||||||
#open(name, "w") do io
|
open(name, "w") do io
|
||||||
# write(io, JSON.json(doc))
|
write(io, JSON.json(doc))
|
||||||
#end
|
end
|
||||||
return nothing
|
return nothing
|
||||||
end
|
end
|
||||||
|
|
||||||
function read_cache(doc::WeaveDoc, cache_path)
|
function read_cache(doc::WeaveDoc, cache_path)
|
||||||
name = "$cache_path/$(doc.basename).jld"
|
name = "$(doc.cwd)/$cache_path/$(doc.basename).json"
|
||||||
isfile(name) || return nothing
|
isfile(name) || return nothing
|
||||||
return JLD.load(name, "doc")
|
parsed = JSON.parsefile(name)
|
||||||
#parsed = JSON.parsefile(name)
|
|
||||||
#doc = WeaveDoc(parsed["source"], parsed["chunks"],
|
|
||||||
#parsed["cwd"], parsed["doctype"])
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
#read_cache returns a dictionary, parse to back to chunk
|
||||||
|
function restore_chunk(chunk::CodeChunk, cached, idx)
|
||||||
|
options = Dict{Symbol, Any}()
|
||||||
|
for (keys,vals) = cached["chunks"][idx]["options"]
|
||||||
|
options[symbol(keys)] = vals
|
||||||
|
end
|
||||||
|
haskey(options, :term_state) && (options[:term_state] = symbol(options[:term_state]))
|
||||||
|
|
||||||
|
chunk.options = options
|
||||||
|
chunk.content = cached["chunks"][idx]["content"]
|
||||||
|
chunk.output = cached["chunks"][idx]["output"]
|
||||||
|
chunk.figures = cached["chunks"][idx]["figures"]
|
||||||
|
|
||||||
#Todo caching of data, can get the contents of module using:
|
return chunk
|
||||||
#names(ReportSandBox, all=true)
|
end
|
||||||
|
|
||||||
|
function restore_chunk(chunk::DocChunk, cached, idx)
|
||||||
|
chunk
|
||||||
|
end
|
||||||
|
|
75
src/run.jl
75
src/run.jl
|
@ -1,10 +1,29 @@
|
||||||
|
@doc """
|
||||||
|
Run code chunks and capture output from parsed document.
|
||||||
|
|
||||||
|
```julia
|
||||||
|
|
||||||
function run(doc::WeaveDoc; doctype = "pandoc", plotlib="Gadfly", informat="noweb",
|
function run(doc::WeaveDoc; doctype = "pandoc", plotlib="Gadfly", informat="noweb",
|
||||||
|
out_path=:doc, fig_path = "figures", fig_ext = nothing,
|
||||||
|
cache_path = "cache", cache = :off)
|
||||||
|
```
|
||||||
|
|
||||||
|
* `doctype`: see `list_out_formats()`
|
||||||
|
* `plotlib`: `"PyPlot"`, `"Gadfly"`, or `"Winston"`
|
||||||
|
* `informat`: `"noweb"` of `"markdown"`
|
||||||
|
* `out_path`: Path where the output is generated. Can be: `:doc`: Path of the source document, `:pwd`: Julia working directory,
|
||||||
|
`"somepath"`: Path as a string e.g `"/home/mpastell/weaveout"`
|
||||||
|
* `fig_path`: where figures will be generated, relative to out_path
|
||||||
|
* `fig_ext`: Extension for saved figures e.g. `".pdf"`, `".png"`. Default setting depends on `doctype`.
|
||||||
|
* `cache_path`: where of cached output will be saved.
|
||||||
|
* `cache`: controls caching of code: `:off` = no caching, `:all` = cache everything,
|
||||||
|
`:user` = cache based on chunk options, `:refresh`, run all code chunks and save new cache.
|
||||||
|
|
||||||
|
**Note:** Run command from terminal and not using IJulia, Juno or ESS, they tend to mess with capturing output.
|
||||||
|
""" ->
|
||||||
|
function Base.run(doc::WeaveDoc; doctype = "pandoc", plotlib="Gadfly", informat="noweb",
|
||||||
out_path=:doc, fig_path = "figures", fig_ext = nothing,
|
out_path=:doc, fig_path = "figures", fig_ext = nothing,
|
||||||
cache_path = "cache", cache = :off)
|
cache_path = "cache", cache = :off)
|
||||||
#cache :all, :user, :off
|
#cache :all, :user, :off, :refresh
|
||||||
|
|
||||||
|
|
||||||
doc.cwd = get_cwd(doc, out_path)
|
doc.cwd = get_cwd(doc, out_path)
|
||||||
|
@ -23,7 +42,7 @@ function run(doc::WeaveDoc; doctype = "pandoc", plotlib="Gadfly", informat="nowe
|
||||||
report = Report(doc.cwd, doc.basename, doc.format.formatdict)
|
report = Report(doc.cwd, doc.basename, doc.format.formatdict)
|
||||||
pushdisplay(report)
|
pushdisplay(report)
|
||||||
|
|
||||||
if cache != :off
|
if cache != :off || cache != :refresh
|
||||||
cached = read_cache(doc, cache_path)
|
cached = read_cache(doc, cache_path)
|
||||||
cached == nothing && info("No cached results found, running code")
|
cached == nothing && info("No cached results found, running code")
|
||||||
else
|
else
|
||||||
|
@ -35,11 +54,7 @@ function run(doc::WeaveDoc; doctype = "pandoc", plotlib="Gadfly", informat="nowe
|
||||||
|
|
||||||
for i = 1:n
|
for i = 1:n
|
||||||
chunk = doc.chunks[i]
|
chunk = doc.chunks[i]
|
||||||
if cached != nothing && (cache == :all || (cache ==:user && chunk.options.cache))
|
result_chunk = run_chunk(chunk, report, SandBox, cached, cache, i)
|
||||||
result_chunk = cached.chunks[i]
|
|
||||||
else
|
|
||||||
result_chunk = eval_chunk(chunk, report, SandBox)
|
|
||||||
end
|
|
||||||
push!(executed, result_chunk)
|
push!(executed, result_chunk)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -55,6 +70,31 @@ function run(doc::WeaveDoc; doctype = "pandoc", plotlib="Gadfly", informat="nowe
|
||||||
return doc
|
return doc
|
||||||
end
|
end
|
||||||
|
|
||||||
|
function run_chunk(chunk::CodeChunk, report::Report, SandBox::Module, cached, cache, idx)
|
||||||
|
defaults = copy(rcParams[:chunk_defaults])
|
||||||
|
options = copy(chunk.options)
|
||||||
|
try
|
||||||
|
options = merge(rcParams[:chunk_defaults], options)
|
||||||
|
catch
|
||||||
|
options = rcParams[:chunk_defaults]
|
||||||
|
warn("Invalid format for chunk options line: $(chunk.start_line)")
|
||||||
|
end
|
||||||
|
|
||||||
|
merge!(chunk.options, options)
|
||||||
|
|
||||||
|
delete!(chunk.options, :options)
|
||||||
|
|
||||||
|
if cached != nothing && (cache == :all || (cache ==:user && chunk.options[:cache]))
|
||||||
|
result_chunk = restore_chunk(chunk, cached, idx)
|
||||||
|
else
|
||||||
|
result_chunk = eval_chunk(chunk, report, SandBox)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
function run_chunk(chunk::DocChunk, report::Report, SandBox::Module, cached, cache, idx)
|
||||||
|
return chunk
|
||||||
|
end
|
||||||
|
|
||||||
function run_block(code_str, report::Report, SandBox::Module)
|
function run_block(code_str, report::Report, SandBox::Module)
|
||||||
oldSTDOUT = STDOUT
|
oldSTDOUT = STDOUT
|
||||||
result = ""
|
result = ""
|
||||||
|
@ -113,18 +153,6 @@ end
|
||||||
|
|
||||||
function eval_chunk(chunk::CodeChunk, report::Report, SandBox::Module)
|
function eval_chunk(chunk::CodeChunk, report::Report, SandBox::Module)
|
||||||
info("Weaving chunk $(chunk.number) from line $(chunk.start_line)")
|
info("Weaving chunk $(chunk.number) from line $(chunk.start_line)")
|
||||||
defaults = copy(rcParams[:chunk_defaults])
|
|
||||||
options = copy(chunk.options)
|
|
||||||
try
|
|
||||||
options = merge(rcParams[:chunk_defaults], options)
|
|
||||||
catch
|
|
||||||
options = rcParams[:chunk_defaults]
|
|
||||||
warn("Invalid format for chunk options line: $(chunk.start_line)")
|
|
||||||
end
|
|
||||||
|
|
||||||
merge!(chunk.options, options)
|
|
||||||
#delete!(chunk.options, :options)
|
|
||||||
#@show chunk.options
|
|
||||||
|
|
||||||
if !chunk.options[:eval]
|
if !chunk.options[:eval]
|
||||||
chunk.output = ""
|
chunk.output = ""
|
||||||
|
@ -132,11 +160,13 @@ function eval_chunk(chunk::CodeChunk, report::Report, SandBox::Module)
|
||||||
return chunk
|
return chunk
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
report.fignum = 1
|
report.fignum = 1
|
||||||
report.cur_result = ""
|
report.cur_result = ""
|
||||||
report.figures = String[]
|
report.figures = String[]
|
||||||
report.cur_chunk = chunk
|
report.cur_chunk = chunk
|
||||||
report.term_state = :text
|
report.term_state = :text
|
||||||
|
|
||||||
if haskey(report.formatdict, :out_width) && chunk.options[:out_width] == nothing
|
if haskey(report.formatdict, :out_width) && chunk.options[:out_width] == nothing
|
||||||
chunk.options[:out_width] = report.formatdict[:out_width]
|
chunk.options[:out_width] = report.formatdict[:out_width]
|
||||||
end
|
end
|
||||||
|
@ -162,6 +192,7 @@ end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#Set all variables to nothing
|
#Set all variables to nothing
|
||||||
function clear_sandbox(SandBox::Module)
|
function clear_sandbox(SandBox::Module)
|
||||||
for name = names(SandBox, true)
|
for name = names(SandBox, true)
|
||||||
|
@ -174,7 +205,7 @@ end
|
||||||
|
|
||||||
function get_figname(report::Report, chunk; fignum = nothing)
|
function get_figname(report::Report, chunk; fignum = nothing)
|
||||||
figpath = joinpath(report.cwd, chunk.options[:fig_path])
|
figpath = joinpath(report.cwd, chunk.options[:fig_path])
|
||||||
isdir(figpath) || mkdir(figpath)
|
isdir(figpath) || mkpath(figpath)
|
||||||
ext = chunk.options[:fig_ext]
|
ext = chunk.options[:fig_ext]
|
||||||
fignum == nothing && (fignum = report.fignum)
|
fignum == nothing && (fignum = report.fignum)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,37 @@
|
||||||
|
using Weave
|
||||||
|
using Base.Test
|
||||||
|
|
||||||
|
#Test if running document with and without cache works
|
||||||
|
isdir("documents/cache") && rm("documents/cache", recursive = true)
|
||||||
|
weave("documents/chunk_options.noweb", plotlib=nothing, cache=:all)
|
||||||
|
ctime1 = ctime("documents/chunk_options.md")
|
||||||
|
result = readall(open("documents/chunk_options.md"))
|
||||||
|
weave("documents/chunk_options.noweb", plotlib=nothing, cache=:all)
|
||||||
|
ctime2 = ctime("documents/chunk_options.md")
|
||||||
|
cached_result = readall(open("documents/chunk_options.md"))
|
||||||
|
@test result == cached_result
|
||||||
|
@test ctime1 != ctime2
|
||||||
|
|
||||||
|
# cache = :user
|
||||||
|
isdir("documents/cache") && rm("documents/cache", recursive = true)
|
||||||
|
out = "documents/chunk_cache.md"
|
||||||
|
Weave.weave("documents/chunk_cache.noweb", plotlib=nothing, cache=:user);
|
||||||
|
result = readall(open(out))
|
||||||
|
ctime1 = ctime(out)
|
||||||
|
Weave.weave("documents/chunk_cache.noweb", plotlib=nothing, cache=:user);
|
||||||
|
cached_result = readall(open(out))
|
||||||
|
ctime2 = ctime(out)
|
||||||
|
@test result == cached_result
|
||||||
|
@test ctime1 != ctime2
|
||||||
|
|
||||||
|
using Gadfly
|
||||||
|
isdir("documents/cache") && rm("documents/cache", recursive = true)
|
||||||
|
#Caching with Gadfly
|
||||||
|
weave("documents/gadfly_formats_test.txt", doctype="tex", plotlib="gadfly", cache=:all)
|
||||||
|
result = readall(open("documents/gadfly_formats_test.tex"))
|
||||||
|
ctime1 = ctime("documents/gadfly_formats_test.tex")
|
||||||
|
weave("documents/gadfly_formats_test.txt", doctype="tex", plotlib="gadfly", cache=:all)
|
||||||
|
ctime2 = ctime("documents/gadfly_formats_test.tex")
|
||||||
|
cached_result = readall(open("documents/gadfly_formats_test.tex"))
|
||||||
|
@test result == cached_result
|
||||||
|
@test ctime1 != ctime2
|
|
@ -0,0 +1,35 @@
|
||||||
|
|
||||||
|
<<term=true>>=
|
||||||
|
y= [2, 5, 12]
|
||||||
|
@
|
||||||
|
|
||||||
|
|
||||||
|
<<term=true; cache=true>>=
|
||||||
|
y= [2, 5, 12]
|
||||||
|
@
|
||||||
|
|
||||||
|
|
||||||
|
<<>>=
|
||||||
|
x = [12, 10]
|
||||||
|
println(y)
|
||||||
|
println(x)
|
||||||
|
@
|
||||||
|
|
||||||
|
<<cache=true>>=
|
||||||
|
x = [12, 10]
|
||||||
|
println(y)
|
||||||
|
println(x)
|
||||||
|
@
|
||||||
|
|
||||||
|
<<>>=
|
||||||
|
y = 1:5
|
||||||
|
println(y)
|
||||||
|
@
|
||||||
|
|
||||||
|
|
||||||
|
<<cache=true>>=
|
||||||
|
y = 1:5
|
||||||
|
println(y)
|
||||||
|
@
|
||||||
|
|
||||||
|
Some text in the end
|
|
@ -9,7 +9,10 @@ using Base.Test
|
||||||
info("Test: Chunk options")
|
info("Test: Chunk options")
|
||||||
include("chunk_options.jl")
|
include("chunk_options.jl")
|
||||||
|
|
||||||
info("Testing: Weaving with Winston")
|
info("Test: Caching")
|
||||||
|
include("cache_test.jl")
|
||||||
|
|
||||||
|
info("Test: Weaving with Winston")
|
||||||
include("winston_formats.jl")
|
include("winston_formats.jl")
|
||||||
|
|
||||||
info("Test: Weaving with Gadfly")
|
info("Test: Weaving with Gadfly")
|
||||||
|
|
Loading…
Reference in New Issue