From 88c6e10e834fd419109e68c815dcd0ca69e7eb22 Mon Sep 17 00:00:00 2001
From: Brandon Taylor <brandon.taylor221@gmail.com>
Date: Wed, 2 Aug 2017 20:04:31 -0400
Subject: [PATCH] avoid inserting extraneous spaces

---
 src/parser.jl | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/src/parser.jl b/src/parser.jl
index ee0ec49..9523726 100644
--- a/src/parser.jl
+++ b/src/parser.jl
@@ -12,7 +12,7 @@ Parser(tokens::T, substitutions, records, line) where T =
     Parser{T}(tokens, substitutions, records, line)
 
 parse_text(text) = begin
-    tokens = matchall(r"[^\s\n\"#{}@,=]+|\n|\"|#|{|}|@|,|=", text)
+    tokens = matchall(r"[^\s\"#{}@,=]+|\s+|\"|#|{|}|@|,|=", text)
     Parser(tokens, Dict{String, String}(), Dict{String, String}(), Ref(1))
 end
 
@@ -23,15 +23,15 @@ next_token_default!(parser) =
         one(parser)
     else
         result = shift!(parser.tokens)
-        if result == "\n"
-            parser.line.x = parser.line.x + 1
-            next_token_default!(parser)
+        parser.line.x = parser.line.x + count(x -> x == '\n', result)
+        if all(isspace, result)
+            eltype(parser)(" ")
         else
             result
         end
     end
 
-next_token!(parser, eol = "additional tokens") = begin
+next_token_with_space!(parser, eol = "additional tokens") = begin
     result = next_token_default!(parser)
     if result == ""
         error("Expected $eol $(location(parser))")
@@ -40,6 +40,15 @@ next_token!(parser, eol = "additional tokens") = begin
     end
 end
 
+next_token!(parser, eol = "additional tokens") = begin
+    result = next_token_with_space!(parser, eol)
+    if all(isspace, result)
+        next_token_with_space!(parser, eol)
+    else
+        result
+    end
+end
+
 expect(parser, result, expectation) =
     if result != expectation
         error("Expected $expectation $(location(parser))")
@@ -48,7 +57,7 @@ expect(parser, result, expectation) =
 expect!(parser, expectation) = expect(parser, next_token!(parser, expectation), expectation)
 
 token_and_counter!(parser, bracket_counter = 1) = begin
-    token = next_token!(parser, "}")
+    token = next_token_with_space!(parser, "}")
     if token == "{"
         bracket_counter += 1
     elseif token == "}"
@@ -60,10 +69,10 @@ end
 value!(parser, values = eltype(parser)[]) = begin
     token = next_token!(parser)
     if token == "\""
-        token = next_token!(parser, "\"")
+        token = next_token_with_space!(parser, "\"")
         while token != "\""
             push!(values, token)
-            token = next_token!(parser, "\"")
+            token = next_token_with_space!(parser, "\"")
         end
     elseif token == "{"
         token, counter = token_and_counter!(parser)
@@ -76,9 +85,10 @@ value!(parser, values = eltype(parser)[]) = begin
     end
     token = next_token!(parser, ", or }")
     if token == "#"
+        push!(values, " ")
         value!(parser, values)
     else
-        token, join(values, " ")
+        token, join(values)
     end
 end
 
@@ -114,7 +124,7 @@ julia> preamble, result = parse_bibtex(""\"
             @comment blah blah
             @string{short = long}
             @a{b,
-              c = { {c} c},
+              c = {{c} c},
               d = "d d",
               e = f # short
             }
@@ -127,7 +137,7 @@ julia> result["b"]["__type__"]
 "a"
 
 julia> result["b"]["c"]
-"{ c } c"
+"{c} c"
 
 julia> result["b"]["d"]
 "d d"