Merge pull request #349 from JunoLab/avi/fixunicode

fixes #348

src/reader/reader.jl

@@ -82,34 +82,23 @@ end
 const   INLINE_REGEX = r"`j\s+(.*?)`"
 const INLINE_REGEXES = r"`j\s+(.*?)`|^!\s(.*)$"m
 
-function parse_inlines(text)::Vector{Inline}
-    occursin(INLINE_REGEXES, text) || return parse_inline(text)
-
-    inline_chunks = eachmatch(INLINE_REGEXES, text)
+# handle code units correctly !
+function parse_inlines(str)
+    ret = Inline[]
     s = 1
-    e = 1
-    res = Inline[]
-    textno = 1
-    codeno = 1
-
-    for ic in inline_chunks
-        s = ic.offset
-        doc = InlineText(text[e:(s-1)], e, s - 1, textno)
-        textno += 1
-        push!(res, doc)
-        e = s + lastindex(ic.match)
-        ic.captures[1] !== nothing && (ctype = :inline)
-        ic.captures[2] !== nothing && (ctype = :line)
-        cap = filter(x -> x !== nothing, ic.captures)[1]
-        push!(res, InlineCode(cap, s, e, codeno, ctype))
-        codeno += 1
+    code_no = text_no = 0
+    for m in eachmatch(INLINE_REGEXES, str)
+        e = m.offset
+        push!(ret, InlineText((str[s:prevind(str,e)]), text_no += 1))
+        i = findfirst(!isnothing, m.captures)
+        push!(ret, InlineCode(m.captures[i], code_no += 1, isone(i) ? :inline : :line))
+        s = e + ncodeunits(m.match)
     end
-    push!(res, InlineText(text[e:end], e, length(text), textno))
-
-    return res
+    push!(ret, InlineText(str[s:end], text_no += 1))
+    return ret
 end
 
-parse_inline(text) = Inline[InlineText(text, 1, length(text), 1)]
+parse_inline(str) = Inline[InlineText(str, 1)]
 
 include("markdown.jl")
 include("script.jl")

src/run.jl

@@ -502,7 +502,7 @@ function _replace_header_inline!(doc, header, report, mod)
 end
 
 function run_inline_code(code, doc, report, mod)
-    inline = InlineCode(code, 1, 1, 1, :inline)
+    inline = InlineCode(code, 1, :inline)
     inline = run_inline(inline, doc, report, mod)
     return strip(inline.output, '"')
 end

src/types.jl

@@ -1,6 +1,7 @@
 # TODO: concreate typing
 
 abstract type WeaveChunk end
+abstract type Inline end
 
 mutable struct WeaveDoc
     source::AbstractString
@@ -50,34 +51,26 @@ mutable struct CodeChunk <: WeaveChunk
     end
 end
 
-abstract type Inline end
-
 mutable struct DocChunk <: WeaveChunk
     content::Vector{Inline}
     number::Int
     start_line::Int
 end
 
-mutable struct InlineText <: Inline
-    content::AbstractString
-    si::Int
-    ei::Int
+struct InlineText <: Inline
+    content::String
     number::Int
 end
 
 mutable struct InlineCode <: Inline
-    content::AbstractString
-    si::Int
-    ei::Int
+    content::String
     number::Int
     ctype::Symbol
-    output::AbstractString
-    rich_output::AbstractString
-    figures::Vector{AbstractString}
-    function InlineCode(content, si, ei, number, ctype)
-        new(content, si, ei, number, ctype, "", "", AbstractString[])
-    end
+    output::String
+    rich_output::String
+    figures::Vector{String}
 end
+InlineCode(content, number, ctype) = InlineCode(content, number, ctype, "", "", String[])
 
 struct TermResult end
 struct ScriptResult end

test/end2end.jl

@@ -0,0 +1,18 @@
+# NOTE
+# this file keeps old end2end tests, which are very fragile
+# - they are being gradually replaced with unit tests, that are much more maintainable and
+#   much more helpful for detecting bugs
+# - the purpose of this file is to temporarily keep the old end2end tests in a way that
+#   they're allowed to fail
+
+tpl = mt"""
+{{{ :body }}}
+"""
+
+out = weave(joinpath(@__DIR__, "documents", "markdown_beamer.jmd"), doctype="md2html", template=tpl)
+@test read(out, String) == read(out*".ref", String)
+rm(out)
+
+out = weave(joinpath(@__DIR__, "documents", "markdown_beamer.jmd"), doctype="md2tex", template=tpl)
+@test read(out, String) == read(out*".ref", String)
+rm(out)

test/inline_test.jl

@@ -1,47 +0,0 @@
-using Mustache
-
-# Test parsing
-
-doc = """
-
-! println("Something")
-
-Some markdown with inline stuff and `j code`
-
- ! Not julia code but `j show("is")`
-
-"""
-
-ms = collect(eachmatch(Weave.INLINE_REGEXES, doc))
-@test ms[1][2] == "println(\"Something\")"
-@test ms[2][1] == "code"
-@test ms[3][1] == "show(\"is\")"
-
-let
-_, chunks = Weave.parse_markdown(doc)
-chunk = first(chunks)
-@test length(chunk.content) == 7
-@test chunk.content[2].content == ms[1][2]
-@test chunk.content[4].content == ms[2][1]
-@test chunk.content[6].content == ms[3][1]
-end
-
-let
-_, chunks = Weave.parse_markdown(doc)
-chunk = first(chunks)
-@test all([chunk.content[i].content == chunk.content[i].content for i in 1:7])
-end
-
-# Test with document
-
-tpl = mt"""
-{{{ :body }}}
-"""
-
-out = weave(joinpath(@__DIR__, "documents", "markdown_beamer.jmd"), doctype="md2html", template=tpl)
-@test read(out, String) == read(out*".ref", String)
-rm(out)
-
-out = weave(joinpath(@__DIR__, "documents", "markdown_beamer.jmd"), doctype="md2tex", template=tpl)
-@test read(out, String) == read(out*".ref", String)
-rm(out)

test/runtests.jl

@@ -28,6 +28,10 @@ macro jmd_str(s) mock_doc(s) end
         include("test_header.jl")
     end
 
+    @testset "inline" begin
+        include("test_inline.jl")
+    end
+
     @testset "error rendering" begin
         include("test_error_rendering.jl")
     end
@@ -62,12 +66,16 @@ macro jmd_str(s) mock_doc(s) end
         include("gadfly_formats.jl")
     end
 
-    @testset "Inline code" begin
-        include("inline_test.jl")
-    end
-
     # @testset "Notebooks" begin
     #     @info("Testing Jupyter options")
     #     include("notebooks.jl")
     # end
+
+    try
+        @testset "end2end (maybe fail)" begin
+            include("end2end.jl")
+        end
+    catch err
+        @error err
+    end
 end

test/test_inline.jl

@@ -0,0 +1,99 @@
+# TODO: test evaluation
+
+using Weave.Mustache
+using Weave: parse_inlines, InlineText, InlineCode
+
+
+@testset "`parse_inlines` basic" begin
+
+@test filter(parse_inlines("text")) do chunk
+    chunk isa InlineCode
+end |> isempty
+
+@test filter(parse_inlines("text")) do chunk
+    chunk isa InlineText &&
+    chunk.content == "text"
+end |> length === 1
+
+@test filter(parse_inlines("`j code`")) do chunk
+    chunk isa InlineCode &&
+    chunk.ctype === :inline &&
+    chunk.content == "code"
+end |> length == 1
+
+@test filter(parse_inlines("! code")) do chunk
+    chunk isa InlineCode &&
+    chunk.ctype === :line &&
+    chunk.content == "code"
+end |> length == 1
+
+@test filter(parse_inlines("text ! maybe_intended_to_be_code")) do chunk # invalid inline chunk
+    chunk isa InlineText &&
+    chunk.content == "maybe_intended_to_be_code"
+end |> isempty
+
+end
+
+
+@testset "`parse_inlines` multiple lines" begin
+
+str = """
+- item1
+- `j code`
+- item2
+"""
+chunks = parse_inlines(str)
+
+let chunk = chunks[1]
+    @test chunk isa InlineText
+    @test occursin("- item1", chunk.content)
+end
+
+let chunk = chunks[2]
+    @test chunk isa InlineCode
+    @test occursin("code", chunk.content)
+end
+
+let chunk = chunks[3]
+    @test chunk isa InlineText
+    @test occursin("- item2", chunk.content)
+end
+
+end
+
+
+@testset "`parse_inlines` unicode handling" begin
+
+str = """
+- eng1 `j :eng1`
+- eng2`j :eng2`
+- 日本語1 `j :日本語1`
+- 日本語2`j :日本語2`
+"""
+chunks = parse_inlines(str)
+
+@test filter(chunks) do chunk
+    chunk isa InlineCode &&
+    chunk.number === 1 &&
+    chunk.content == ":eng1"
+end |> length === 1
+
+@test filter(chunks) do chunk
+    chunk isa InlineCode &&
+    chunk.number === 2 &&
+    chunk.content == ":eng2"
+end |> length === 1
+
+@test filter(chunks) do chunk
+    chunk isa InlineCode &&
+    chunk.number === 3 &&
+    chunk.content == ":日本語1"
+end |> length === 1
+
+@test filter(chunks) do chunk
+    chunk isa InlineCode &&
+    chunk.number === 4 &&
+    chunk.content == ":日本語2"
+end |> length === 1
+
+end