avoid inserting extraneous spaces
This commit is contained in:
parent
75d1b6d74c
commit
88c6e10e83
|
@ -12,7 +12,7 @@ Parser(tokens::T, substitutions, records, line) where T =
|
||||||
Parser{T}(tokens, substitutions, records, line)
|
Parser{T}(tokens, substitutions, records, line)
|
||||||
|
|
||||||
parse_text(text) = begin
|
parse_text(text) = begin
|
||||||
tokens = matchall(r"[^\s\n\"#{}@,=]+|\n|\"|#|{|}|@|,|=", text)
|
tokens = matchall(r"[^\s\"#{}@,=]+|\s+|\"|#|{|}|@|,|=", text)
|
||||||
Parser(tokens, Dict{String, String}(), Dict{String, String}(), Ref(1))
|
Parser(tokens, Dict{String, String}(), Dict{String, String}(), Ref(1))
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -23,18 +23,27 @@ next_token_default!(parser) =
|
||||||
one(parser)
|
one(parser)
|
||||||
else
|
else
|
||||||
result = shift!(parser.tokens)
|
result = shift!(parser.tokens)
|
||||||
if result == "\n"
|
parser.line.x = parser.line.x + count(x -> x == '\n', result)
|
||||||
parser.line.x = parser.line.x + 1
|
if all(isspace, result)
|
||||||
next_token_default!(parser)
|
eltype(parser)(" ")
|
||||||
|
else
|
||||||
|
result
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
next_token_with_space!(parser, eol = "additional tokens") = begin
|
||||||
|
result = next_token_default!(parser)
|
||||||
|
if result == ""
|
||||||
|
error("Expected $eol $(location(parser))")
|
||||||
else
|
else
|
||||||
result
|
result
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
next_token!(parser, eol = "additional tokens") = begin
|
next_token!(parser, eol = "additional tokens") = begin
|
||||||
result = next_token_default!(parser)
|
result = next_token_with_space!(parser, eol)
|
||||||
if result == ""
|
if all(isspace, result)
|
||||||
error("Expected $eol $(location(parser))")
|
next_token_with_space!(parser, eol)
|
||||||
else
|
else
|
||||||
result
|
result
|
||||||
end
|
end
|
||||||
|
@ -48,7 +57,7 @@ expect(parser, result, expectation) =
|
||||||
expect!(parser, expectation) = expect(parser, next_token!(parser, expectation), expectation)
|
expect!(parser, expectation) = expect(parser, next_token!(parser, expectation), expectation)
|
||||||
|
|
||||||
token_and_counter!(parser, bracket_counter = 1) = begin
|
token_and_counter!(parser, bracket_counter = 1) = begin
|
||||||
token = next_token!(parser, "}")
|
token = next_token_with_space!(parser, "}")
|
||||||
if token == "{"
|
if token == "{"
|
||||||
bracket_counter += 1
|
bracket_counter += 1
|
||||||
elseif token == "}"
|
elseif token == "}"
|
||||||
|
@ -60,10 +69,10 @@ end
|
||||||
value!(parser, values = eltype(parser)[]) = begin
|
value!(parser, values = eltype(parser)[]) = begin
|
||||||
token = next_token!(parser)
|
token = next_token!(parser)
|
||||||
if token == "\""
|
if token == "\""
|
||||||
token = next_token!(parser, "\"")
|
token = next_token_with_space!(parser, "\"")
|
||||||
while token != "\""
|
while token != "\""
|
||||||
push!(values, token)
|
push!(values, token)
|
||||||
token = next_token!(parser, "\"")
|
token = next_token_with_space!(parser, "\"")
|
||||||
end
|
end
|
||||||
elseif token == "{"
|
elseif token == "{"
|
||||||
token, counter = token_and_counter!(parser)
|
token, counter = token_and_counter!(parser)
|
||||||
|
@ -76,9 +85,10 @@ value!(parser, values = eltype(parser)[]) = begin
|
||||||
end
|
end
|
||||||
token = next_token!(parser, ", or }")
|
token = next_token!(parser, ", or }")
|
||||||
if token == "#"
|
if token == "#"
|
||||||
|
push!(values, " ")
|
||||||
value!(parser, values)
|
value!(parser, values)
|
||||||
else
|
else
|
||||||
token, join(values, " ")
|
token, join(values)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue