handle codeunits gracefully in inline chunks

src/reader/reader.jl

@@ -82,34 +82,23 @@ end
 const   INLINE_REGEX = r"`j\s+(.*?)`"
 const INLINE_REGEXES = r"`j\s+(.*?)`|^!\s(.*)$"m
 
-function parse_inlines(text)::Vector{Inline}
-    occursin(INLINE_REGEXES, text) || return parse_inline(text)
-
-    inline_chunks = eachmatch(INLINE_REGEXES, text)
+# handle code units correctly !
+function parse_inlines(str)
+    ret = Inline[]
     s = 1
-    e = 1
-    res = Inline[]
-    textno = 1
-    codeno = 1
-
-    for ic in inline_chunks
-        s = ic.offset
-        doc = InlineText(text[e:(s-1)], e, s - 1, textno)
-        textno += 1
-        push!(res, doc)
-        e = s + lastindex(ic.match)
-        ic.captures[1] !== nothing && (ctype = :inline)
-        ic.captures[2] !== nothing && (ctype = :line)
-        cap = filter(x -> x !== nothing, ic.captures)[1]
-        push!(res, InlineCode(cap, s, e, codeno, ctype))
-        codeno += 1
+    code_no = text_no = 0
+    for m in eachmatch(INLINE_REGEXES, str)
+        e = m.offset
+        push!(ret, InlineText((str[s:prevind(str,e)]), text_no += 1))
+        i = findfirst(!isnothing, m.captures)
+        push!(ret, InlineCode(m.captures[i], code_no += 1, isone(i) ? :inline : :line))
+        s = e + ncodeunits(m.match)
     end
-    push!(res, InlineText(text[e:end], e, length(text), textno))
-
-    return res
+    push!(ret, InlineText(str[s:end], text_no += 1))
+    return ret
 end
 
-parse_inline(text) = Inline[InlineText(text, 1, length(text), 1)]
+parse_inline(str) = Inline[InlineText(str, 1)]
 
 include("markdown.jl")
 include("script.jl")

src/run.jl

@@ -502,7 +502,7 @@ function _replace_header_inline!(doc, header, report, mod)
 end
 
 function run_inline_code(code, doc, report, mod)
-    inline = InlineCode(code, 1, 1, 1, :inline)
+    inline = InlineCode(code, 1, :inline)
     inline = run_inline(inline, doc, report, mod)
     return strip(inline.output, '"')
 end

src/types.jl

@@ -1,6 +1,7 @@
 # TODO: concreate typing
 
 abstract type WeaveChunk end
+abstract type Inline end
 
 mutable struct WeaveDoc
     source::AbstractString
@@ -50,34 +51,26 @@ mutable struct CodeChunk <: WeaveChunk
     end
 end
 
-abstract type Inline end
-
 mutable struct DocChunk <: WeaveChunk
     content::Vector{Inline}
     number::Int
     start_line::Int
 end
 
-mutable struct InlineText <: Inline
-    content::AbstractString
-    si::Int
-    ei::Int
+struct InlineText <: Inline
+    content::String
     number::Int
 end
 
 mutable struct InlineCode <: Inline
-    content::AbstractString
-    si::Int
-    ei::Int
+    content::String
     number::Int
     ctype::Symbol
-    output::AbstractString
-    rich_output::AbstractString
-    figures::Vector{AbstractString}
-    function InlineCode(content, si, ei, number, ctype)
-        new(content, si, ei, number, ctype, "", "", AbstractString[])
-    end
+    output::String
+    rich_output::String
+    figures::Vector{String}
 end
+InlineCode(content, number, ctype) = InlineCode(content, number, ctype, "", "", String[])
 
 struct TermResult end
 struct ScriptResult end