diff --git a/src/StringDistances.jl b/src/StringDistances.jl index 47f187b..e02b340 100755 --- a/src/StringDistances.jl +++ b/src/StringDistances.jl @@ -5,9 +5,10 @@ using Distances include("utils.jl") include("edit.jl") include("qgram.jl") +include("modifiers.jl") include("normalize.jl") -const StringDistance = Union{Jaro, Levenshtein, DamerauLevenshtein, RatcliffObershelp, QGramDistance, Winkler, Partial, TokenSort, TokenSet, TokenMax, Normalize} +const StringDistance = Union{Jaro, Levenshtein, DamerauLevenshtein, RatcliffObershelp, QGramDistance, Winkler, Partial, TokenSort, TokenSet, TokenMax, Normalized} Distances.result_type(dist::StringDistance, s1, s2) = typeof(dist("", "")) """ diff --git a/src/edit.jl b/src/edit.jl index daa5f39..d54d247 100755 --- a/src/edit.jl +++ b/src/edit.jl @@ -49,6 +49,7 @@ function (dist::Jaro)(s1, s2) return 1.0 - (m / len1 + m / len2 + (m - t/2) / m) / 3.0 end +(dist::Jaro)(s1, s2, ::Nothing) = (dist::Jaro)(s1, s2) """ Levenshtein() @@ -228,4 +229,6 @@ function longest_common_pattern(s1, s2) end end return start1, start2, len -end \ No newline at end of file +end + +(dist::RatcliffObershelp)(s1, s2, ::Nothing) = dist(s1, s2) diff --git a/src/modifiers.jl b/src/modifiers.jl new file mode 100755 index 0000000..2711e66 --- /dev/null +++ b/src/modifiers.jl @@ -0,0 +1,121 @@ +""" + Partial(dist) + +Creates the `Partial{dist}` distance. + +`Partial{dist}` modifies the string distance `dist` to return the +minimum distance between the shorter string and substrings of the longer string + +### Examples +```julia-repl +julia> s1 = "New York Mets vs Atlanta Braves" +julia> s2 = "Atlanta Braves vs New York Mets" +julia> evaluate(Partial(RatcliffObershelp()), s1, s2) +0.5483870967741935 +``` +""" +struct Partial{S <: SemiMetric} <: SemiMetric + dist::S +end + +function (dist::Partial)(s1, s2, max_dist = nothing) + s1, s2 = reorder(s1, s2) + len1, len2 = length(s1), length(s2) + out = dist.dist(s1, s2, max_dist) + len1 == len2 && return out + len1 == 0 && return out + for x in qgrams(s2, len1) + curr = dist.dist(s1, x, max_dist) + out = min(out, curr) + max_dist !== nothing && (max_dist = min(out, max_dist)) + end + return out +end + +function (dist::Partial{RatcliffObershelp})(s1, s2, max_dist = nothing) + s1, s2 = reorder(s1, s2) + len1, len2 = length(s1), length(s2) + len1 == len2 && return dist.dist(s1, s2) + out = 1.0 + for r in matching_blocks(s1, s2) + # Make sure the substring of s2 has length len1 + s2_start = r[2] - r[1] + 1 + s2_end = s2_start + len1 - 1 + if s2_start <= 0 + s2_end += 1 - s2_start + s2_start += 1 - s2_start + elseif s2_end > len2 + s2_start += len2 - s2_end + s2_end += len2 - s2_end + end + curr = dist.dist(s1, _slice(s2, s2_start - 1, s2_end)) + out = min(out, curr) + end + return out +end + +""" + TokenSort(dist) + +Creates the `TokenSort{dist}` distance. + +`TokenSort{dist}` modifies the string distance `dist` to adjust for differences +in word orders by reording words alphabetically. + +### Examples +```julia-repl +julia> s1 = "New York Mets vs Atlanta Braves" +julia> s1 = "New York Mets vs Atlanta Braves" +julia> s2 = "Atlanta Braves vs New York Mets" +julia> evaluate(TokenSort(RatcliffObershelp()), s1, s2) +0.0 +``` +""" +struct TokenSort{S <: SemiMetric} <: SemiMetric + dist::S +end + +# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/ +function (dist::TokenSort)(s1::AbstractString, s2::AbstractString, max_dist = nothing) + s1 = join(sort!(split(s1)), " ") + s2 = join(sort!(split(s2)), " ") + out = dist.dist(s1, s2, max_dist) +end + +""" + TokenSet(dist) + +Creates the `TokenSet{dist}` distance. + +`TokenSet{dist}` modifies the string distance `dist` to adjust for differences +in word orders and word numbers by comparing the intersection of two strings with each string. + +### Examples +```julia-repl +julia> s1 = "New York Mets vs Atlanta" +julia> s2 = "Atlanta Braves vs New York Mets" +julia> evaluate(TokenSet(RatcliffObershelp()), s1, s2) +0.0 +``` +""" +struct TokenSet{S <: SemiMetric} <: SemiMetric + dist::S +end + +# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/ +function (dist::TokenSet)(s1::AbstractString, s2::AbstractString, max_dist = nothing) + v1 = unique!(sort!(split(s1))) + v2 = unique!(sort!(split(s2))) + v0 = intersect(v1, v2) + s0 = join(v0, " ") + s1 = join(v1, " ") + s2 = join(v2, " ") + isempty(s0) && return dist.dist(s1, s2, max_dist) + score_01 = dist.dist(s0, s1, max_dist) + max_dist !== nothing && (max_dist = min(max_dist, score_01)) + score_02 = dist.dist(s0, s2, max_dist) + max_dist !== nothing && (max_dist = min(max_dist, score_02)) + score_12 = dist.dist(s1, s2, max_dist) + min(score_01, score_02, score_12) +end + diff --git a/src/normalize.jl b/src/normalize.jl index 48cff75..f347f4f 100755 --- a/src/normalize.jl +++ b/src/normalize.jl @@ -1,16 +1,9 @@ -struct Normalize{S <: SemiMetric} <: SemiMetric + +struct Normalized{S <: SemiMetric} <: SemiMetric dist::S end -""" - normalize(dist::SemiMetric) - - Normalize a metric, so that `evaluate` always return a Float64 between 0 and 1 (or a `missing` if one element is missing) -""" -# also a normalized distance always accept a third argument, max_dist. - -normalize(dist::SemiMetric) = Normalize{typeof(dist)}(dist) - -function (dist::Normalize{<: Union{Levenshtein, DamerauLevenshtein}})(s1, s2, max_dist = 1.0) +# A normalized distance is between 0 and 1, and accept a third argument, max_dist. +function (dist::Normalized{<: Union{Levenshtein, DamerauLevenshtein}})(s1, s2, max_dist = 1.0) ((s1 === missing) | (s2 === missing)) && return missing s1, s2 = reorder(s1, s2) len1, len2 = length(s1), length(s2) @@ -20,7 +13,7 @@ function (dist::Normalize{<: Union{Levenshtein, DamerauLevenshtein}})(s1, s2, ma out > max_dist ? 1.0 : out end -function (dist::Normalize{<: QGramDistance})(s1, s2, max_dist = 1.0) +function (dist::Normalized{<: QGramDistance})(s1, s2, max_dist = 1.0) ((s1 === missing) | (s2 === missing)) && return missing # When string length < q for qgram distance, returns s1 == s2 s1, s2 = reorder(s1, s2) @@ -33,16 +26,26 @@ function (dist::Normalize{<: QGramDistance})(s1, s2, max_dist = 1.0) end end -function (dist::Normalize)(s1, s2, max_dist = 1.0) +function (dist::Normalized)(s1, s2, max_dist = 1.0) dist.dist(s1, s2) end +""" + normalize(dist::SemiMetric) + + Normalize a metric, so that `evaluate` always return a Float64 between 0 and 1 +""" +normalize(dist::SemiMetric) = Normalized(dist) +normalize(dist::Normalized) = dist + + + """ Winkler(dist; p::Real = 0.1, threshold::Real = 0.7, maxlength::Integer = 4) Creates the `Winkler{dist, p, threshold, maxlength}` distance. -`Winkler{dist, p, threshold, length)` modifies the string distance `dist` to decrease the +`Winkler{dist, p, threshold, length)` modifies the string distance `normalize(dist)` to decrease the distance between two strings, when their original distance is below some `threshold`. The boost is equal to `min(l, maxlength) * p * dist` where `l` denotes the length of their common prefix and `dist` denotes the original distance @@ -57,9 +60,10 @@ end function Winkler(dist::SemiMetric; p = 0.1, threshold = 0.7, maxlength = 4) p * maxlength <= 1 || throw("scaling factor times maxlength of common prefix must be lower than one") - Winkler{typeof(normalize(dist))}(normalize(dist), 0.1, 0.7, 4) + dist = normalize(dist) + Winkler{typeof(dist)}(dist, 0.1, 0.7, 4) end -normalize(dist::Winkler) = dist +isnormalized(dist::Winkler) = true function (dist::Winkler)(s1, s2, max_dist = 1.0) # cannot do max_dist because of boosting threshold @@ -72,144 +76,12 @@ function (dist::Winkler)(s1, s2, max_dist = 1.0) end -""" - Partial(dist) - -Creates the `Partial{dist}` distance. - -`Partial{dist}` modifies the string distance `dist` to return the -minimum distance between the shorter string and substrings of the longer string - -### Examples -```julia-repl -julia> s1 = "New York Mets vs Atlanta Braves" -julia> s2 = "Atlanta Braves vs New York Mets" -julia> evaluate(Partial(RatcliffObershelp()), s1, s2) -0.5483870967741935 -``` -""" -struct Partial{S <: SemiMetric} <: SemiMetric - dist::S - Partial{S}(dist::S) where {S <: SemiMetric} = new(dist) -end -Partial(dist::SemiMetric) = Partial{typeof(normalize(dist))}(normalize(dist)) -normalize(dist::Partial) = dist - -function (dist::Partial)(s1, s2, max_dist = 1.0) - s1, s2 = reorder(s1, s2) - len1, len2 = length(s1), length(s2) - len1 == len2 && return dist.dist(s1, s2, max_dist) - len1 == 0 && return 1.0 - out = 1.0 - for x in qgrams(s2, len1) - curr = dist.dist(s1, x, max_dist) - out = min(out, curr) - max_dist = min(out, max_dist) - end - return out -end - -function (dist::Partial{Normalize{RatcliffObershelp}})(s1, s2, max_dist = 1.0) - s1, s2 = reorder(s1, s2) - len1, len2 = length(s1), length(s2) - len1 == len2 && return dist.dist(s1, s2) - out = 1.0 - for r in matching_blocks(s1, s2) - # Make sure the substring of s2 has length len1 - s2_start = r[2] - r[1] + 1 - s2_end = s2_start + len1 - 1 - if s2_start <= 0 - s2_end += 1 - s2_start - s2_start += 1 - s2_start - elseif s2_end > len2 - s2_start += len2 - s2_end - s2_end += len2 - s2_end - end - curr = dist.dist(s1, _slice(s2, s2_start - 1, s2_end)) - out = min(out, curr) - end - return out -end - -""" - TokenSort(dist) - -Creates the `TokenSort{dist}` distance. - -`TokenSort{dist}` modifies the string distance `dist` to adjust for differences -in word orders by reording words alphabetically. - -### Examples -```julia-repl -julia> s1 = "New York Mets vs Atlanta Braves" -julia> s1 = "New York Mets vs Atlanta Braves" -julia> s2 = "Atlanta Braves vs New York Mets" -julia> evaluate(TokenSort(RatcliffObershelp()), s1, s2) -0.0 -``` -""" -struct TokenSort{S <: SemiMetric} <: SemiMetric - dist::S - TokenSort{S}(dist::S) where {S <: SemiMetric} = new(dist) -end -TokenSort(dist::SemiMetric) = TokenSort{typeof(normalize(dist))}(normalize(dist)) -normalize(dist::TokenSort) = dist - -# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/ -function (dist::TokenSort)(s1::AbstractString, s2::AbstractString, max_dist = 1.0) - s1 = join(sort!(split(s1)), " ") - s2 = join(sort!(split(s2)), " ") - dist.dist(s1, s2, max_dist) -end - - -""" - TokenSet(dist) - -Creates the `TokenSet{dist}` distance. - -`TokenSet{dist}` modifies the string distance `dist` to adjust for differences -in word orders and word numbers by comparing the intersection of two strings with each string. - -### Examples -```julia-repl -julia> s1 = "New York Mets vs Atlanta" -julia> s2 = "Atlanta Braves vs New York Mets" -julia> evaluate(TokenSet(RatcliffObershelp()), s1, s2) -0.0 -``` -""" -struct TokenSet{S <: SemiMetric} <: SemiMetric - dist::S - TokenSet{S}(dist::S) where {S <: SemiMetric} = new(dist) -end -TokenSet(dist::SemiMetric) = TokenSet{typeof(normalize(dist))}(normalize(dist)) -normalize(dist::TokenSet) = dist - -# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/ -function (dist::TokenSet)(s1::AbstractString, s2::AbstractString, max_dist = 1.0) - v1 = unique!(sort!(split(s1))) - v2 = unique!(sort!(split(s2))) - v0 = intersect(v1, v2) - s0 = join(v0, " ") - s1 = join(v1, " ") - s2 = join(v2, " ") - isempty(s0) && return dist.dist(s1, s2, max_dist) - score_01 = dist.dist(s0, s1, max_dist) - max_dist = min(max_dist, score_01) - score_02 = dist.dist(s0, s2, max_dist) - max_dist = min(max_dist, score_02) - score_12 = dist.dist(s1, s2, max_dist) - min(score_01, score_02, score_12) -end - - """ TokenMax(dist) Creates the `TokenMax{dist}` distance -`TokenMax{dist}` is the minimum of the base distance `dist`, +`TokenMax{dist}` is the minimum of the base distance `normalize(dist)`, its [`Partial`](@ref) modifier, its [`TokenSort`](@ref) modifier, and its [`TokenSet`](@ref) modifier, with penalty terms depending on string lengths. @@ -226,8 +98,11 @@ struct TokenMax{S <: SemiMetric} <: SemiMetric TokenMax{S}(dist::S) where {S <: SemiMetric} = new(dist) end -TokenMax(dist::SemiMetric) = TokenMax{typeof(normalize(dist))}(normalize(dist)) -normalize(dist::TokenMax) = dist +function TokenMax(dist::SemiMetric) + dist = normalize(dist) + TokenMax{typeof(dist)}(dist) +end +isnormalized(dist::TokenMax) = true function (dist::TokenMax)(s1::AbstractString, s2::AbstractString, max_dist = 1.0) s1, s2 = reorder(s1, s2) diff --git a/src/qgram.jl b/src/qgram.jl index b4c9d4d..bae2feb 100755 --- a/src/qgram.jl +++ b/src/qgram.jl @@ -107,6 +107,8 @@ function (dist::QGram)(s1, s2) n end +(dist::QGram)(s1, s2, ::Nothing) = dist(s1, s2) + """ Cosine(q::Int) @@ -134,6 +136,8 @@ function (dist::Cosine)(s1, s2) 1.0 - prodnorm / (sqrt(norm1) * sqrt(norm2)) end +(dist::Cosine)(s1, s2, ::Nothing) = dist(s1, s2) + """ Jaccard(q::Int) @@ -160,6 +164,8 @@ function (dist::Jaccard)(s1, s2) 1.0 - nintersect / (ndistinct1 + ndistinct2 - nintersect) end +(dist::Jaccard)(s1, s2, ::Nothing) = dist(s1, s2) + """ SorensenDice(q::Int) @@ -186,6 +192,8 @@ function (dist::SorensenDice)(s1, s2) 1.0 - 2.0 * nintersect / (ndistinct1 + ndistinct2) end +(dist::SorensenDice)(s1, s2, ::Nothing) = dist(s1, s2) + """ Overlap(q::Int) @@ -211,3 +219,5 @@ function (dist::Overlap)(s1, s2) end 1.0 - nintersect / min(ndistinct1, ndistinct2) end + +(dist::Overlap)(s1, s2, ::Nothing) = dist(s1, s2)