From ab09bddcd8a77969e4a8466a895fe5448b60ab3b Mon Sep 17 00:00:00 2001 From: schrimpf Date: Thu, 3 Jan 2019 05:40:32 -0800 Subject: [PATCH 1/7] change cache to work with figures (#158) * Changed read/write cache to use serialize/deserialize instead of JLD2. JLD2 fails to save strings longer than 2^16 characters, which can occur in the stringmime() of large figures. * Added test for cache with figures and html output. Updated REQUIRE & Project.toml to include Serialization instead of FileIO and JLD2 * Added "Import Pkg;" to expected output of errors, so that tests pass. --- Project.toml | 3 +-- REQUIRE | 3 +-- src/cache.jl | 17 +++++++++++------ src/run.jl | 2 +- test/errors_test.jl | 5 ++--- test/plotsjl_test.jl | 9 +++++++++ 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Project.toml b/Project.toml index 94389f7..bea5a43 100644 --- a/Project.toml +++ b/Project.toml @@ -9,8 +9,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" -JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Highlights = "eafb193a-b7ab-5a9e-9068-77385905fa72" Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70" diff --git a/REQUIRE b/REQUIRE index b786414..f046501 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,6 +1,5 @@ julia 0.7 -JLD2 -FileIO +Serialization JSON Highlights 0.3.0 Mustache diff --git a/src/cache.jl b/src/cache.jl index 173adca..0584681 100644 --- a/src/cache.jl +++ b/src/cache.jl @@ -1,16 +1,21 @@ -#FileIO and JLD2 are imported only if cache is used +#Serialization is imported only if cache is used function write_cache(doc::WeaveDoc, cache_path) cache_dir = joinpath(doc.cwd, cache_path) isdir(cache_dir) || mkpath(cache_dir) - Base.invokelatest(FileIO.save, joinpath(cache_dir, doc.basename * ".jld2"), Dict("doc" => doc)) + open(joinpath(cache_dir, doc.basename * ".cache"),"w") do io + Serialization.serialize(io, doc) + end return nothing end function read_cache(doc::WeaveDoc, cache_path) - name = joinpath(doc.cwd, cache_path, doc.basename * ".jld2") + name = joinpath(doc.cwd, cache_path, doc.basename * ".cache") isfile(name) || return nothing - return Base.invokelatest(FileIO.load, name, "doc") + open(name,"r") do io + doc = Serialization.deserialize(io) + end + return doc end function restore_chunk(chunk::CodeChunk, cached) @@ -35,7 +40,7 @@ end #Restore inline code function restore_chunk(chunk::DocChunk, cached::WeaveDoc) #Get chunk from cached doc - c_chunk = filter(x -> x.number == chunk.number && + c_chunk = filter(x -> x.number == chunk.number && isa(x, DocChunk), cached.chunks) isempty(c_chunk) && return chunk c_chunk = c_chunk[1] @@ -45,7 +50,7 @@ function restore_chunk(chunk::DocChunk, cached::WeaveDoc) isempty(c_inline) && return chunk #Restore cached results for Inline code - n = length(chunk.content) + n = length(chunk.content) for i in 1:n if isa(chunk.content[i], InlineCode) ci = filter(x -> x.number == chunk.content[i].number, c_inline) diff --git a/src/run.jl b/src/run.jl index d0b1c89..56e2a6e 100644 --- a/src/run.jl +++ b/src/run.jl @@ -41,7 +41,7 @@ function Base.run(doc::WeaveDoc; doctype = :auto, fig_path = mktempdir(abspath(doc.cwd)) end - cache == :off || @eval import FileIO, JLD2 + cache == :off || @eval import Serialization #This is needed for latex and should work on all output formats Sys.iswindows() && (fig_path = replace(fig_path, "\\" => "/")) diff --git a/test/errors_test.jl b/test/errors_test.jl index 2a284fa..f973da3 100644 --- a/test/errors_test.jl +++ b/test/errors_test.jl @@ -26,8 +26,7 @@ doc1 = Weave.run(doc, doctype = "pandoc") doc1.chunks[1].output -@test doc1.chunks[1].output == "Error: ArgumentError: Package NonExisting not found in current path:\n- Run `Pkg.add(\"NonExisting\")` to install the NonExisting package.\n\n" - +@test doc1.chunks[1].output == "Error: ArgumentError: Package NonExisting not found in current path:\n- Run `import Pkg; Pkg.add(\"NonExisting\")` to install the NonExisting package.\n\n" @test doc1.chunks[2].output == "Error: syntax: incomplete: premature end of input\n" @test doc1.chunks[3].output == "\njulia> plot(x)\nError: UndefVarError: plot not defined\n\njulia> y = 10\n10\n\njulia> print(y\nError: syntax: incomplete: premature end of input\n" @@ -35,7 +34,7 @@ doc1.chunks[1].output doc = Weave.WeaveDoc("dummy1.jmd", p1, Dict()) doc3 = Weave.run(doc, doctype = "md2html") -@test doc3.chunks[1].rich_output == "
\nERROR: ArgumentError: Package NonExisting not found in current path:\n- Run `Pkg.add("NonExisting")` to install the NonExisting package.\n\n
\n" +@test doc3.chunks[1].rich_output == "
\nERROR: ArgumentError: Package NonExisting not found in current path:\n- Run `import Pkg; Pkg.add("NonExisting")` to install the NonExisting package.\n\n
\n" @test doc3.chunks[2].rich_output == "
\nERROR: syntax: incomplete: premature end of input\n
\n" @test doc3.chunks[3].output == "\njulia> plot(x)\nError: UndefVarError: plot not defined\n\njulia> y = 10\n10\n\njulia> print(y\nError: syntax: incomplete: premature end of input\n" @test doc3.chunks[3].rich_output == "" diff --git a/test/plotsjl_test.jl b/test/plotsjl_test.jl index e395fbb..b3f26fb 100644 --- a/test/plotsjl_test.jl +++ b/test/plotsjl_test.jl @@ -11,3 +11,12 @@ end pljtest("plotsjl_test_gr.jmd", "plotsjl_test_gr.md", "pandoc") pljtest("plotsjl_test_gr.jmd", "plotsjl_test_gr.tex", "tex") + +# test cache with plots +isdir("documents/cache") && rm("documents/cache", recursive = true) +weave("documents/plotsjl_test_gr.jmd", cache=:all) +result = read("documents/plotsjl_test_gr.html", String) +rm("documents/plotsjl_test_gr.html") +weave("documents/plotsjl_test_gr.jmd", cache=:all) +cached_result = read("documents/plotsjl_test_gr.html", String) +@test result == cached_result From 921b9ec2ba13fc3ceec25c54cd22bfca660aa1a8 Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Thu, 3 Jan 2019 15:46:12 +0200 Subject: [PATCH 2/7] Remove 0.7 from tests --- .travis.yml | 5 ++--- appveyor.yml | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0fbf913..5b08870 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,11 @@ language: julia julia: - - 0.7 - 1.0 - - nightly + - 1.1 matrix: allow_failures: - - julia: nightly + - julia: 1.1 notifications: email: false script: diff --git a/appveyor.yml b/appveyor.yml index a930e94..abe8a15 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,5 @@ environment: matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7-latest-win64.exe" - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/1.0/julia-1.0-latest-win64.exe" branches: From d0c71ba5561437d8cc98c8355bd770a7afe30edb Mon Sep 17 00:00:00 2001 From: Dave Kleinschmidt Date: Mon, 28 Jan 2019 09:20:03 -0500 Subject: [PATCH 3/7] use Base.show (#165) * use Base.show * drop offending show method altogether --- src/Markdown2HTML.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Markdown2HTML.jl b/src/Markdown2HTML.jl index 8839bb7..89acd47 100644 --- a/src/Markdown2HTML.jl +++ b/src/Markdown2HTML.jl @@ -226,11 +226,4 @@ htmlinline(io::IO, x) = tohtml(io, x) html(md) = sprint(html, md) -function show(io::IO, ::MIME"text/html", md::MD) - withtag(io, :div, :class=>"markdown") do - html(io, md) - end -end - - end From 36339a07ff2afb55869f83af1c19574feb913f8f Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Tue, 29 Jan 2019 08:53:52 +0200 Subject: [PATCH 4/7] v0.7.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index bea5a43..6c3f1b7 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Weave" uuid = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9" -version="0.6.1" +version="0.7.0" [deps] Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" From 0a21a2ec90b125344a55f4ad032a3fb51d4cf432 Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Tue, 29 Jan 2019 09:01:11 +0200 Subject: [PATCH 5/7] v0.7.0 --- REQUIRE | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/REQUIRE b/REQUIRE index f046501..1e4c869 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,4 @@ -julia 0.7 -Serialization +julia 1.0 JSON Highlights 0.3.0 Mustache From 4bf8e0e209402ccb4a4500229db48d6d06654160 Mon Sep 17 00:00:00 2001 From: Dave Kleinschmidt Date: Fri, 1 Feb 2019 10:59:39 -0500 Subject: [PATCH 6/7] check fig_ext to choose preferred mimetype in display(::Report, d) --- src/display_methods.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/display_methods.jl b/src/display_methods.jl index debec51..75e9b97 100644 --- a/src/display_methods.jl +++ b/src/display_methods.jl @@ -31,9 +31,17 @@ const default_mime_types = ["image/svg+xml", "image/png", "text/html", "text/pla #From IJulia as a reminder #const supported_mime_types = [ "text/html", "text/latex", "image/svg+xml", "image/png", "image/jpeg", "text/plain", "text/markdown" ] +const mimetype_from_fig_ext = + Dict(".png" => "image/png", + ".jpg" => "image/jpeg", + ".jpeg" => "image/jpeg", + ".svg" => "image/svg+xml", + ".pdf" => "application/pdf") + function Base.display(report::Report, data) #Set preferred mimetypes for report based on format - for m in report.mimetypes + fig_ext = report.cur_chunk.options[:fig_ext] + for m in Iterators.flatten(((mimetype_from_fig_ext[fig_ext], ), report.mimetypes)) if showable(m, data) try if !istextmime(m) From 4a7d63bc7ea49dde1b2c95351cdb3a58bf28ad7a Mon Sep 17 00:00:00 2001 From: Matti Pastell Date: Fri, 8 Feb 2019 16:01:25 +0200 Subject: [PATCH 7/7] Clean up and fix svg embedding --- src/display_methods.jl | 4 ++-- src/run.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/display_methods.jl b/src/display_methods.jl index 75e9b97..a28becf 100644 --- a/src/display_methods.jl +++ b/src/display_methods.jl @@ -31,7 +31,7 @@ const default_mime_types = ["image/svg+xml", "image/png", "text/html", "text/pla #From IJulia as a reminder #const supported_mime_types = [ "text/html", "text/latex", "image/svg+xml", "image/png", "image/jpeg", "text/plain", "text/markdown" ] -const mimetype_from_fig_ext = +const mimetype_ext = Dict(".png" => "image/png", ".jpg" => "image/jpeg", ".jpeg" => "image/jpeg", @@ -41,7 +41,7 @@ const mimetype_from_fig_ext = function Base.display(report::Report, data) #Set preferred mimetypes for report based on format fig_ext = report.cur_chunk.options[:fig_ext] - for m in Iterators.flatten(((mimetype_from_fig_ext[fig_ext], ), report.mimetypes)) + for m in unique([mimetype_ext[fig_ext] ; report.mimetypes]) if showable(m, data) try if !istextmime(m) diff --git a/src/run.jl b/src/run.jl index 56e2a6e..ac7e53e 100644 --- a/src/run.jl +++ b/src/run.jl @@ -155,7 +155,7 @@ function img2base64(fig, cwd) if ext == ".png" return "data:image/png;base64," * stringmime(MIME("image/png"), raw) elseif ext == ".svg" - return "data:image/svg+xml;base64," * stringmime(MIME("image/svg+xml"), raw) + return "data:image/svg+xml;base64," * stringmime(MIME("image/svg"), raw) elseif ext == ".gif" return "data:image/gif;base64," * stringmime(MIME("image/gif"), raw) else