diff --git a/src/StringDistances.jl b/src/StringDistances.jl index 76fa7dd..724422f 100755 --- a/src/StringDistances.jl +++ b/src/StringDistances.jl @@ -4,6 +4,8 @@ using Distances import StatsAPI: pairwise, pairwise! abstract type StringSemiMetric <: SemiMetric end abstract type StringMetric <: Metric end +const StringDistance = Union{StringSemiMetric, StringMetric} +(dist::StringDistance)(s1, s2; max_dist = nothing) = dist(s1, s2) function Distances.result_type(dist::Union{StringSemiMetric, StringMetric}, s1::Type, s2::Type) T = typeof(dist("", "")) if (Missing <: s1) | (Missing <: s2) @@ -17,10 +19,8 @@ Distances.result_type(dist::Union{StringSemiMetric, StringMetric}, s1, s2) = res include("distances/utils.jl") include("distances/edit.jl") include("distances/qgram.jl") - - -include("normalize.jl") include("pairwise.jl") +include("normalize.jl") include("find.jl") include("fuzzywuzzy.jl") @@ -32,7 +32,8 @@ include("fuzzywuzzy.jl") ## ############################################################################## -export +export +StringDistance, StringSemiMetric, StringMetric, # edit distances diff --git a/src/distances/edit.jl b/src/distances/edit.jl index cf0ffb3..ca1d424 100755 --- a/src/distances/edit.jl +++ b/src/distances/edit.jl @@ -5,18 +5,15 @@ Creates the Hamming distance The Hamming distance is defined as the number of characters that do not match """ -struct Hamming{V <: Union{Int, Nothing}} <: StringMetric - max_dist::V -end -Hamming() = Hamming(nothing) +struct Hamming <: StringMetric end -function (dist::Hamming{T})(s1, s2) where {T} +function (dist::Hamming)(s1, s2; max_dist::Union{Integer, Nothing} = nothing) (s1 === missing) | (s2 === missing) && return missing out = abs(length(s2) - length(s1)) for (ch1, ch2) in zip(s1, s2) out += ch1 != ch2 - if T <: Int - out > dist.max_dist && return dist.max_dist + 1 + if max_dist !== nothing + out > max_dist && return Int(max_dist + 1) end end return out @@ -118,22 +115,20 @@ Creates the Levenshtein distance The Levenshtein distance is the minimum number of operations (consisting of insertions, deletions, substitutions of a single character) required to change one string into the other. """ -struct Levenshtein{V <: Union{Int, Nothing}} <: StringMetric - max_dist::V -end -Levenshtein() = Levenshtein(nothing) +struct Levenshtein <: StringMetric end + ## Source: http://blog.softwx.net/2014/12/optimizing-levenshtein-algorithm-in-c.html # Return max_dist + 1 if distance higher than max_dist # to differentiate distance equal to max_dist or not, which is important for find fctions. -function (dist::Levenshtein{T})(s1, s2) where {T} +function (dist::Levenshtein)(s1, s2; max_dist::Union{Integer, Nothing} = nothing) (s1 === missing) | (s2 === missing) && return missing len1, len2 = length(s1), length(s2) if len1 > len2 s1, s2 = s2, s1 len1, len2 = len2, len1 end - if T <: Int - len2 - len1 > dist.max_dist && return dist.max_dist + 1 + if max_dist !== nothing + len2 - len1 > max_dist && return Int(max_dist + 1) end # prefix common to both strings can be ignored k = common_prefix(s1, s2) @@ -144,7 +139,7 @@ function (dist::Levenshtein{T})(s1, s2) where {T} for (i1, ch1) in enumerate(s1) i1 > k || continue left = current = i1 - k - 1 - if T <: Int + if max_dist !== nothing value_lb = left - 1 end for (i2, ch2) in enumerate(s2) @@ -153,17 +148,17 @@ function (dist::Levenshtein{T})(s1, s2) where {T} if ch1 != ch2 current = min(current, above, left) + 1 end - if T <: Int + if max_dist !== nothing value_lb = min(value_lb, left) end @inbounds v[i2 - k] = current end - if T <: Int - value_lb > dist.max_dist && return dist.max_dist + 1 + if max_dist !== nothing + value_lb > max_dist && return Int(max_dist + 1) end end - if T <: Int - current > dist.max_dist && return dist.max_dist + 1 + if max_dist !== nothing + current > max_dist && return Int(max_dist + 1 ) end return current end @@ -183,22 +178,19 @@ end uses the optimal string alignment algorithm. In particular, the restricted distance does not satisfy the triangle inequality. """ -struct OptimalStringAlignement{V <: Union{Int, Nothing}} <: StringSemiMetric - max_dist::V -end -OptimalStringAlignement() = OptimalStringAlignement(nothing) +struct OptimalStringAlignement <: StringSemiMetric end ## http://blog.softwx.net/2015/01/optimizing-damerau-levenshtein_15.html # Return max_dist + 1 if distance higher than max_dist -function (dist::OptimalStringAlignement{T})(s1, s2) where {T} +function (dist::OptimalStringAlignement)(s1, s2; max_dist::Union{Integer, Nothing} = nothing) (s1 === missing) | (s2 === missing) && return missing len1, len2 = length(s1), length(s2) if len1 > len2 s1, s2 = s2, s1 len1, len2 = len2, len1 end - if T <: Int - len2 - len1 > dist.max_dist && return dist.max_dist + 1 + if max_dist !== nothing + len2 - len1 > max_dist && return Int(max_dist + 1) end k = common_prefix(s1, s2) k == len1 && return len2 - k @@ -206,28 +198,25 @@ function (dist::OptimalStringAlignement{T})(s1, s2) where {T} w = similar(v) prevch1, prevch2 = first(s1), first(s2) current = 0 - if T <: Int + if max_dist !== nothing i2_start = 0 - i2_end = dist.max_dist + i2_end = max_dist end for (i1, ch1) in enumerate(s1) i1 > k || (prevch1 = ch1 ; continue) left = i1 - k - 1 current = left + 1 nextTransCost = 0 - if T <: Int - i2_start += i1 - k - 1 + len2 - len1 > dist.max_dist + if max_dist !== nothing + i2_start += i1 - k - 1 + len2 - len1 > max_dist i2_end += i2_end < len2 end for (i2, ch2) in enumerate(s2) i2 > k || (prevch2 = ch2 ; continue) # no need to look beyond window of lower right diagonal - max distance cells - # lower right diag is i1 - (len2 - len1)) and the upper left diagonal + dist.max_dist cells (upper left is i1) - if T <: Int - if !(i2_start <= i2 - k - 1 < i2_end) - prevch2 = ch2 - continue - end + # lower right diag is i1 - (len2 - len1)) and the upper left diagonal + max_dist cells (upper left is i1) + if max_dist !== nothing + (i2_start <= i2 - k - 1 < i2_end) || (prevch2 = ch2 ; continue) end @inbounds above, current, left = current, left, v[i2 - k] @inbounds w[i2 - k], nextTransCost, thisTransCost = current, w[i2 - k], nextTransCost @@ -241,13 +230,13 @@ function (dist::OptimalStringAlignement{T})(s1, s2) where {T} @inbounds v[i2 - k] = current prevch2 = ch2 end - if T <: Int - v[i1 - k + len2 - len1] > dist.max_dist && return dist.max_dist + 1 + if max_dist !== nothing + v[i1 - k + len2 - len1] > max_dist && return Int(max_dist + 1) end prevch1 = ch1 end - if T <: Int - current > dist.max_dist && return dist.max_dist + 1 + if max_dist !== nothing + current > max_dist && return Int(max_dist + 1) end return current end diff --git a/src/find.jl b/src/find.jl index fd830f6..8fa4c9d 100644 --- a/src/find.jl +++ b/src/find.jl @@ -1,3 +1,19 @@ +""" + compare(s1, s2, dist) + +return a similarity score between 0 and 1 for the strings `s1` and +`s2` based on the distance `dist`. + +### Examples +```julia-repl +julia> compare("martha", "marhta", Levenshtein()) +0.6666666666666667 +``` +""" +function compare(s1, s2, dist::Union{StringSemiMetric, StringMetric}; min_score = 0.0) + 1 - Normalized(dist)(s1, s2; max_dist = 1 - min_score) +end + """ findnearest(s, itr, dist::Union{StringMetric, StringSemiMetric}) -> (x, index) @@ -18,7 +34,7 @@ julia> findnearest(s, iter, Levenshtein(); min_score = 0.9) (nothing, nothing) ``` """ -function findnearest(s, itr, dist::Union{StringSemiMetric, StringMetric}; min_score = 0.0) +function findnearest(s, itr, dist::StringDistance; min_score = 0.0) min_score_atomic = Threads.Atomic{Float64}(min_score) scores = [0.0 for _ in 1:Threads.nthreads()] is = [0 for _ in 1:Threads.nthreads()] @@ -37,15 +53,15 @@ function findnearest(s, itr, dist::Union{StringSemiMetric, StringMetric}; min_sc end _preprocess(dist::AbstractQGramDistance, ::Missing) = missing _preprocess(dist::AbstractQGramDistance, s) = QGramSortedVector(s, dist.q) -_preprocess(dist::Union{StringSemiMetric, StringMetric}, s) = s +_preprocess(dist::StringDistance, s) = s -function Base.findmax(s, itr, dist::Union{StringSemiMetric, StringMetric}; min_score = 0.0) +function Base.findmax(s, itr, dist::StringDistance; min_score = 0.0) @warn "findmax(s, itr, dist; min_score) is deprecated. Use findnearest(s, itr, dist; min_score)" findnearest(s, itr, dist; min_score = min_score) end """ - findall(s, itr , dist::Union{StringSemiMetric, StringMetric}; min_score = 0.8) + findall(s, itr , dist::StringDistance; min_score = 0.8) `findall` returns the vector of indices for elements of `itr` that have a similarity score higher or equal than `min_score` according to the distance `dist`. @@ -66,7 +82,7 @@ julia> findall(s, iter, Levenshtein(); min_score = 0.9) 0-element Array{Int64,1} ``` """ -function Base.findall(s, itr, dist::Union{StringSemiMetric, StringMetric}; min_score = 0.8) +function Base.findall(s, itr, dist::StringDistance; min_score = 0.8) out = [Int[] for _ in 1:Threads.nthreads()] s = _preprocess(dist, s) # need collect since @threads requires a length method diff --git a/src/fuzzywuzzy.jl b/src/fuzzywuzzy.jl index eb9c43f..dd80159 100755 --- a/src/fuzzywuzzy.jl +++ b/src/fuzzywuzzy.jl @@ -19,37 +19,37 @@ struct Partial{S <: Union{StringSemiMetric, StringMetric}} <: StringSemiMetric dist::S end -function (dist::Partial)(s1, s2) +function (dist::Partial)(s1, s2; max_dist = nothing) (s1 === missing) | (s2 === missing) && return missing s1, s2 = reorder(s1, s2) len1, len2 = length(s1), length(s2) - out = dist.dist(s1, s2) + out = dist.dist(s1, s2; max_dist = max_dist) + max_dist0 = (max_dist !== nothing) ? min(max_dist, out) : out ((len1 == 0) | (len1 == len2)) && return out for x in qgrams(s2, len1) - curr = dist.dist(s1, x) + curr = dist.dist(s1, x; max_dist = max_dist0) out = min(out, curr) + max_dist0 = min(max_dist0, curr) end return out end -function (dist::Partial{RatcliffObershelp})(s1, s2) +function (dist::Partial{T})(s1, s2; max_dist = nothing) where {T <: Union{RatcliffObershelp, Normalized{RatcliffObershelp}}} (s1 === missing) | (s2 === missing) && return missing + s1, s2 = reorder(s1, s2) len1, len2 = length(s1), length(s2) len1 == len2 && return dist.dist(s1, s2) out = 1.0 for r in matching_blocks(s1, s2, 1, 1, len1, len2) # Make sure the substring of s2 has length len1 s2_start = r[2] - r[1] + 1 - s2_end = s2_start + len1 - 1 if s2_start < 1 - s2_end += 1 - s2_start - s2_start += 1 - s2_start - elseif s2_end > len2 - s2_start += len2 - s2_end - s2_end += len2 - s2_end + s2_start = 1 + elseif s2_start + len1 - 1 > len2 + s2_start += len2 - (s2_start + len1 - 1) end - n_matched = length_matching_blocks(s1, s2, 1, s2_start, len1, s2_end) - curr = 1 - 2 * n_matched / (len1 + s2_end - s2_start + 1) + n_matched = length_matching_blocks(s1, s2, 1, s2_start, len1, s2_start + len1 - 1) + curr = 1 - 2 * n_matched / (len1 + len1) out = min(out, curr) end return out @@ -74,10 +74,6 @@ function matching_blocks!(x::Set{Tuple{Int, Int, Int}}, p::Vector{Int}, s1, s2, return x end -function normalize(dist::Partial; max_dist = 1.0) - Partial(normalize(dist.dist; max_dist = max_dist)) -end - """ TokenSort(dist) @@ -101,15 +97,10 @@ struct TokenSort{S <: Union{StringSemiMetric, StringMetric}} <: StringSemiMetric dist::S end -function (dist::TokenSort)(s1::Union{AbstractString, Missing}, s2::Union{AbstractString, Missing}) +function (dist::TokenSort)(s1::Union{AbstractString, Missing}, s2::Union{AbstractString, Missing}; max_dist = nothing) (s1 === missing) | (s2 === missing) && return missing - s1 = join(sort!(split(s1)), " ") - s2 = join(sort!(split(s2)), " ") - out = dist.dist(s1, s2) -end - -function normalize(dist::TokenSort; max_dist = 1.0) - TokenSort(normalize(dist.dist; max_dist = max_dist)) + f = s -> join(sort!(split(s)), " ") + dist.dist(f(s1), f(s2); max_dist = max_dist) end """ @@ -135,7 +126,7 @@ struct TokenSet{S <: Union{StringSemiMetric, StringMetric}} <: StringSemiMetric dist::S end -function (dist::TokenSet)(s1::Union{AbstractString, Missing}, s2::Union{AbstractString, Missing}) +function (dist::TokenSet)(s1::Union{AbstractString, Missing}, s2::Union{AbstractString, Missing}; max_dist = nothing) (s1 === missing) | (s2 === missing) && return missing v1 = unique!(sort!(split(s1))) v2 = unique!(sort!(split(s2))) @@ -143,18 +134,13 @@ function (dist::TokenSet)(s1::Union{AbstractString, Missing}, s2::Union{Abstract s0 = join(v0, " ") s1 = join(v1, " ") s2 = join(v2, " ") - isempty(s0) && return dist.dist(s1, s2) - score_01 = dist.dist(s0, s1) - score_02 = dist.dist(s0, s2) - score_12 = dist.dist(s1, s2) - min(score_01, score_02, score_12) + isempty(s0) && return dist.dist(s1, s2; max_dist = max_dist) + out_01 = dist.dist(s0, s1; max_dist = max_dist) + out_02 = dist.dist(s0, s2; max_dist = max_dist) + out_12 = dist.dist(s1, s2; max_dist = max_dist) + min(out_01, out_02, out_12) end -function normalize(dist::TokenSet; max_dist = 1.0) - TokenSet(normalize(dist.dist; max_dist = max_dist)) -end - - """ TokenMax(dist) @@ -173,44 +159,34 @@ julia> evaluate(TokenMax(RatcliffObershelp()), s1, s2) 0.05 ``` """ -struct TokenMax{S <: Union{StringSemiMetric, StringMetric}} <: StringSemiMetric +struct TokenMax{S <: Normalized} <: StringSemiMetric dist::S - max_dist::Float64 end -TokenMax(dist::SemiMetric; max_dist = 1.0) = TokenMax(dist, max_dist) +TokenMax(dist::Normalized) = TokenMax{typeof(dist)}(dist) +TokenMax(dist::Union{StringSemiMetric, StringMetric}) = TokenMax(Normalized(dist)) -function (dist::TokenMax)(s1::Union{AbstractString, Missing}, s2::Union{AbstractString, Missing}) +function (dist::TokenMax)(s1::Union{AbstractString, Missing}, s2::Union{AbstractString, Missing}; max_dist = 1.0) (s1 === missing) | (s2 === missing) && return missing s1, s2 = reorder(s1, s2) len1, len2 = length(s1), length(s2) - max_dist = dist.max_dist - dist0 = normalize(dist.dist; max_dist = max_dist) - score = dist0(s1, s2) - min_score = min(max_dist, score) - unbase_scale = 0.95 + dist0 = dist.dist + out = dist0(s1, s2; max_dist = max_dist) + max_dist = min(max_dist, out) + scale = 0.95 # if one string is much shorter than the other, use partial - if length(s2) >= 1.5 * length(s1) - partial_scale = length(s2) > (8 * length(s1)) ? 0.6 : 0.9 - dist0 = normalize(dist0, max_dist = 1 - (1 - max_dist) / partial_scale) - score_partial = 1 - partial_scale * (1 - Partial(dist0)(s1, s2)) - min_score = min(max_dist, score_partial) - dist0 = normalize(dist0, max_dist = 1 - (1 - max_dist) / (unbase_scale * partial_scale)) - score_sort = 1 - unbase_scale * partial_scale * (1 - TokenSort(Partial(dist0))(s1, s2)) - max_dist = min(max_dist, score_sort) - dist0 = normalize(dist0, max_dist = 1 - (1 - max_dist) / (unbase_scale * partial_scale)) - score_set = 1 - unbase_scale * partial_scale * (1 - TokenSet(Partial(dist0))(s1, s2)) - out = min(score, score_partial, score_sort, score_set) - else - dist0 = normalize(dist0, max_dist = 1 - (1 - max_dist) / unbase_scale) - score_sort = 1 - unbase_scale * (1 - TokenSort(dist0)(s1, s2)) - max_dist = min(max_dist, score_sort) - dist0 = normalize(dist0, max_dist = 1 - (1 - max_dist) / unbase_scale) - score_set = 1 - unbase_scale * (1 - TokenSet(dist0)(s1, s2)) - out = min(score, score_sort, score_set) + if len2 >= 1.5 * len1 + dist0 = Partial(dist0) + pscale = 0.9 + pout = 1 - pscale * (1 - dist0(s1, s2; max_dist = 1 - (1 - max_dist) / pscale)) + out = min(out, pout) + max_dist = min(max_dist, pout) + scale *= pscale end + out_sort = 1 - scale * (1 - TokenSort(dist0)(s1, s2; max_dist = 1 - (1 - max_dist) / scale)) + max_dist = min(max_dist, out_sort) + out_set = 1 - scale * (1 - TokenSet(dist0)(s1, s2; max_dist = 1 - (1 - max_dist) / scale)) + out = min(out, out_sort, out_set) out > max_dist ? 1.0 : out end -function normalize(dist::TokenMax; max_dist = 1.0) - TokenMax(dist.dist, max_dist) -end +Normalized(dist::TokenMax) = TokenMax(dist.dist) diff --git a/src/normalize.jl b/src/normalize.jl index b12ac39..77f1362 100755 --- a/src/normalize.jl +++ b/src/normalize.jl @@ -1,35 +1,52 @@ -struct Normalized{V <: Union{StringSemiMetric, StringMetric}} <: StringSemiMetric - dist::V - max_dist::Float64 -end +""" + Normalized(dist) -function (dist::Normalized{<: Union{Jaro, JaroWinkler, RatcliffObershelp}})(s1, s2) +Creates a normalized distance. The distance always return a Float64 between 0.0 and 1.0 (or a missing if one of the argument is missing) + +### Examples +```julia-repl +julia> s1 = "New York Mets vs Atlanta" +julia> s2 = "Atlanta Braves vs New York Mets" +julia> Levenshtein()(s1, s2) +25 +julia> StringDistances.Normalized(Levenshtein())(s1, s2) +0.8064 +``` +""" +struct Normalized{T <: Union{StringSemiMetric, StringMetric}} <: StringSemiMetric + dist::T +end +Normalized(dist::Normalized) = dist +function (dist::Normalized)(s1, s2; max_dist = 1.0) out = dist.dist(s1, s2) - out > dist.max_dist ? 1.0 : out + max_dist !== nothing && out > max_dist && return 1.0 + return out end -function (dist::Normalized{<:Union{Hamming, DamerauLevenshtein}})(s1, s2) +function (dist::Normalized{<:Union{Hamming, DamerauLevenshtein}})(s1, s2; max_dist = 1.0) (s1 === missing) | (s2 === missing) && return missing isempty(s1) && isempty(s2) && return 0.0 out = dist.dist(s1, s2) / length(s2) - out > dist.max_dist ? 1.0 : out + max_dist !== nothing && out > max_dist && return 1.0 + return out end -function (dist::Normalized{<:Union{Levenshtein{Nothing}, OptimalStringAlignement{Nothing}}})(s1, s2) +function (dist::Normalized{<:Union{Levenshtein, OptimalStringAlignement}})(s1, s2; max_dist = 1.0) (s1 === missing) | (s2 === missing) && return missing isempty(s1) && isempty(s2) && return 0.0 s1, s2 = reorder(s1, s2) len1, len2 = length(s1), length(s2) - if dist.dist isa Levenshtein - d = Levenshtein(ceil(Int, len2 * dist.max_dist))(s1, s2) + if max_dist === nothing || max_dist == 1.0 + d = dist.dist(s1, s2) else - d = OptimalStringAlignement(ceil(Int, len2 * dist.max_dist))(s1, s2) + d = dist.dist(s1, s2; max_dist = ceil(Int, len2 * max_dist)) end out = d / len2 - out > dist.max_dist ? 1.0 : out + max_dist !== nothing && out > max_dist && return 1.0 + return out end -function (dist::Normalized{<:AbstractQGramDistance})(s1, s2) +function (dist::Normalized{<:AbstractQGramDistance})(s1, s2; max_dist = 1.0) (s1 === missing) | (s2 === missing) && return missing # When string length < q for qgram distance, returns s1 == s2 s1, s2 = reorder(s1, s2) @@ -40,41 +57,6 @@ function (dist::Normalized{<:AbstractQGramDistance})(s1, s2) else out = dist.dist(s1, s2) end - out > dist.max_dist ? 1.0 : out + max_dist !== nothing && out > max_dist && return 1.0 + return out end - - -""" - normalize(dist) - -Creates a normalized distance. The distance always return a Float64 between 0.0 and 1.0 (or a missing if one of the argument is missing) - -### Examples -```julia-repl -julia> s1 = "New York Mets vs Atlanta" -julia> s2 = "Atlanta Braves vs New York Mets" -julia> Levenshtein()(s1, s2) -25 -julia> StringDistances.normalize(Levenshtein())(s1, s2) -0.8064 -``` -""" -normalize(dist::Union{StringSemiMetric, StringMetric}; max_dist = 1.0) = Normalized{typeof(dist)}(dist, max_dist) -normalize(dist::Normalized; max_dist = 1.0) = Normalized(dist.dist, max_dist) - - -""" - compare(s1, s2, dist) - -return a similarity score between 0 and 1 for the strings `s1` and -`s2` based on the distance `dist`. - -### Examples -```julia-repl -julia> compare("martha", "marhta", Levenshtein()) -0.6666666666666667 -``` -""" -function compare(s1, s2, dist::Union{StringSemiMetric, StringMetric}; min_score = 0.0) - 1 - normalize(dist, max_dist = 1 - min_score)(s1, s2) -end diff --git a/test/distances.jl b/test/distances.jl index a8a4f9e..57ce9b8 100644 --- a/test/distances.jl +++ b/test/distances.jl @@ -2,161 +2,162 @@ using StringDistances, Unicode, Test, Random @testset "Distances" begin @testset "Hamming" begin - @test evaluate(Hamming(), "martha", "marhta") ≈ 2 - @test evaluate(Hamming(), "es an ", " vs an") ≈ 6 - @test evaluate(Hamming(), [1, 2, 3], [1,2, 4]) ≈ 1 - @inferred evaluate(Hamming(), "", "") - @test ismissing(evaluate(Hamming(), "", missing)) + @test Hamming()("martha", "marhta") ≈ 2 + @test Hamming()("es an ", " vs an") ≈ 6 + @test Hamming()([1, 2, 3], [1,2, 4]) ≈ 1 + @inferred Hamming()("", "") + @test ismissing(Hamming()("", missing)) end @testset "Jaro" begin - @test evaluate(Jaro(), "martha", "marhta") ≈ 0.05555555555555547 - @test evaluate(Jaro(), "es an ", " vs an") ≈ 0.2777777777777777 - @test evaluate(Jaro(), " vs an", "es an ") ≈ 0.2777777777777777 - @test evaluate(Jaro(), [1, 2, 3], [1,2, 4]) ≈ 0.2222222222222222 - @test evaluate(Jaro(), graphemes("alborgów"), graphemes("amoniak")) == evaluate(Jaro(), "alborgów", "amoniak") + @test Jaro()("martha", "marhta") ≈ 0.05555555555555547 + @test Jaro()("es an ", " vs an") ≈ 0.2777777777777777 + @test Jaro()(" vs an", "es an ") ≈ 0.2777777777777777 + @test Jaro()([1, 2, 3], [1,2, 4]) ≈ 0.2222222222222222 + @test Jaro()(graphemes("alborgów"), graphemes("amoniak")) == Jaro()("alborgów", "amoniak") @test Jaro()(" vs an", "es an ") ≈ 0.2777777777777777 @test result_type(Jaro(), "hello", "world") == typeof(float(1)) - @inferred evaluate(Jaro(), "", "") - @test ismissing(evaluate(Jaro(), "", missing)) + @inferred Jaro()("", "") + @test ismissing(Jaro()("", missing)) end @testset "Levenshtein" begin - @test evaluate(Levenshtein(), "", "") == 0 - @test evaluate(Levenshtein(), "abc", "") == 3 - @test evaluate(Levenshtein(), "", "abc") == 3 - @test evaluate(Levenshtein(), "bc", "abc") == 1 - @test evaluate(Levenshtein(), "kitten", "sitting") == 3 - @test evaluate(Levenshtein(), "saturday", "sunday") == 3 - @test evaluate(Levenshtein(), "hi, my name is", "my name is") == 4 - @test evaluate(Levenshtein(), "a cat", "an act") == 3 - @test evaluate(Levenshtein(), "alborgów", "amoniak") == 6 - @test evaluate(Levenshtein(), [1, 2, 3], [1, 2, 4]) == 1 - @test evaluate(Levenshtein(), graphemes("alborgów"), graphemes("amoniak")) == evaluate(Levenshtein(), "alborgów", "amoniak") + @test Levenshtein()("", "") == 0 + @test Levenshtein()("abc", "") == 3 + @test Levenshtein()("", "abc") == 3 + @test Levenshtein()("bc", "abc") == 1 + @test Levenshtein()("kitten", "sitting") == 3 + @test Levenshtein()("saturday", "sunday") == 3 + @test Levenshtein()("hi, my name is", "my name is") == 4 + @test Levenshtein()("a cat", "an act") == 3 + @test Levenshtein()("alborgów", "amoniak") == 6 + @test Levenshtein()([1, 2, 3], [1, 2, 4]) == 1 + @test Levenshtein()(graphemes("alborgów"), graphemes("amoniak")) == Levenshtein()("alborgów", "amoniak") @test Levenshtein()("", "abc") == 3 @test result_type(Levenshtein(), "hello", "world") == Int - @inferred evaluate(Levenshtein(), "", "") - @test ismissing(evaluate(Levenshtein(), "", missing)) + @inferred Levenshtein()("", "") + @test ismissing(Levenshtein()("", missing)) end @testset "OptimalStringAlignement" begin - @test evaluate(OptimalStringAlignement(), "", "") == 0 - @test evaluate(OptimalStringAlignement(), "abc", "") == 3 - @test evaluate(OptimalStringAlignement(), "bc", "abc") == 1 - @test evaluate(OptimalStringAlignement(), "fuor", "four") == 1 - @test evaluate(OptimalStringAlignement(), "abcd", "acb") == 2 - @test evaluate(OptimalStringAlignement(), "cape sand recycling ", "edith ann graham") == 17 - @test evaluate(OptimalStringAlignement(), "jellyifhs", "jellyfish") == 2 - @test evaluate(OptimalStringAlignement(), "ifhs", "fish") == 2 - @test evaluate(OptimalStringAlignement(), "a cat", "an act") == 2 - @test evaluate(OptimalStringAlignement(), "a cat", "an abct") == 4 - @test evaluate(OptimalStringAlignement(), "a cat", "a tc") == 3 - @test OptimalStringAlignement(2)("abcdef", "abcxyf") == 2 + @test OptimalStringAlignement()("", "") == 0 + @test OptimalStringAlignement()("abc", "") == 3 + @test OptimalStringAlignement()("bc", "abc") == 1 + @test OptimalStringAlignement()("fuor", "four") == 1 + @test OptimalStringAlignement()("abcd", "acb") == 2 + @test OptimalStringAlignement()("cape sand recycling ", "edith ann graham") == 17 + @test OptimalStringAlignement()("jellyifhs", "jellyfish") == 2 + @test OptimalStringAlignement()("ifhs", "fish") == 2 + @test OptimalStringAlignement()("a cat", "an act") == 2 + @test OptimalStringAlignement()("a cat", "an abct") == 4 + @test OptimalStringAlignement()("a cat", "a tc") == 3 + @test OptimalStringAlignement()("abcdef", "abcxyf") == 2 + @test OptimalStringAlignement()("abcdef", "abcxyf"; max_dist = 2) == 2 - @test evaluate(OptimalStringAlignement(), [1, 2, 3], [1,2, 4]) == 1 - @test evaluate(OptimalStringAlignement(), graphemes("alborgów"), graphemes("amoniak")) == evaluate(OptimalStringAlignement(), "alborgów", "amoniak") + @test OptimalStringAlignement()([1, 2, 3], [1,2, 4]) == 1 + @test OptimalStringAlignement()(graphemes("alborgów"), graphemes("amoniak")) == OptimalStringAlignement()("alborgów", "amoniak") @test OptimalStringAlignement()("bc", "abc") == 1 @test result_type(OptimalStringAlignement(), "hello", "world") == Int - @inferred evaluate(OptimalStringAlignement(), "", "") - @test ismissing(evaluate(OptimalStringAlignement(), "", missing)) + @inferred OptimalStringAlignement()("", "") + @test ismissing(OptimalStringAlignement()("", missing)) end @testset "DamerauLevenshtein" begin - @test evaluate(DamerauLevenshtein(), "", "") == 0 - @test evaluate(DamerauLevenshtein(), "CA", "ABC") == 2 - @test evaluate(DamerauLevenshtein(), "ABCDEF", "ABDCEF") == 1 - @test evaluate(DamerauLevenshtein(), "ABCDEF", "BACDFE") == 2 - @test evaluate(DamerauLevenshtein(), "ABCDEF", "ABCDE") == 1 - @test evaluate(DamerauLevenshtein(), "a cat", "an act") == 2 - @test evaluate(DamerauLevenshtein(), "a cat", "an abct") == 3 - @test evaluate(DamerauLevenshtein(), "a cat", "a tc") == 2 + @test DamerauLevenshtein()("", "") == 0 + @test DamerauLevenshtein()("CA", "ABC") == 2 + @test DamerauLevenshtein()("ABCDEF", "ABDCEF") == 1 + @test DamerauLevenshtein()("ABCDEF", "BACDFE") == 2 + @test DamerauLevenshtein()("ABCDEF", "ABCDE") == 1 + @test DamerauLevenshtein()("a cat", "an act") == 2 + @test DamerauLevenshtein()("a cat", "an abct") == 3 + @test DamerauLevenshtein()("a cat", "a tc") == 2 @test result_type(DamerauLevenshtein(), "hello", "world") == Int - @inferred evaluate(DamerauLevenshtein(), "", "") - @test ismissing(evaluate(DamerauLevenshtein(), "", missing)) + @inferred DamerauLevenshtein()("", "") + @test ismissing(DamerauLevenshtein()("", missing)) end @testset "RatcliffObershelp" begin - @test evaluate(RatcliffObershelp(), "dixon", "dicksonx") ≈ 1 - 0.6153846153846154 - @test evaluate(RatcliffObershelp(), "alexandre", "aleksander") ≈ 1 - 0.7368421052631579 - @test evaluate(RatcliffObershelp(), "pennsylvania", "pencilvaneya") ≈ 1 - 0.6666666666666 - @test evaluate(RatcliffObershelp(), "", "pencilvaneya") ≈ 1.0 - @test evaluate(RatcliffObershelp(),"NEW YORK METS", "NEW YORK MEATS") ≈ 1 - 0.962962962963 - @test evaluate(RatcliffObershelp(), "Yankees", "New York Yankees") ≈ 0.3913043478260869 - @test evaluate(RatcliffObershelp(), "New York Mets", "New York Yankees") ≈ 0.24137931034482762 - @test evaluate(RatcliffObershelp(), [1, 2, 3], [1,2, 4]) ≈ 1/3 - @test evaluate(RatcliffObershelp(), graphemes("alborgów"), graphemes("amoniak")) == evaluate(RatcliffObershelp(), "alborgów", "amoniak") + @test RatcliffObershelp()("dixon", "dicksonx") ≈ 1 - 0.6153846153846154 + @test RatcliffObershelp()("alexandre", "aleksander") ≈ 1 - 0.7368421052631579 + @test RatcliffObershelp()("pennsylvania", "pencilvaneya") ≈ 1 - 0.6666666666666 + @test RatcliffObershelp()("", "pencilvaneya") ≈ 1.0 + @test RatcliffObershelp()("NEW YORK METS", "NEW YORK MEATS") ≈ 1 - 0.962962962963 + @test RatcliffObershelp()("Yankees", "New York Yankees") ≈ 0.3913043478260869 + @test RatcliffObershelp()("New York Mets", "New York Yankees") ≈ 0.24137931034482762 + @test RatcliffObershelp()([1, 2, 3], [1,2, 4]) ≈ 1/3 + @test RatcliffObershelp()(graphemes("alborgów"), graphemes("amoniak")) == RatcliffObershelp()("alborgów", "amoniak") @test RatcliffObershelp()("pennsylvania", "pencilvaneya") ≈ 1 - 0.6666666666666 @test result_type(RatcliffObershelp(), "hello", "world") == typeof(float(1)) - @inferred evaluate(RatcliffObershelp(), "", "") - @test ismissing(evaluate(RatcliffObershelp(), "", missing)) + @inferred RatcliffObershelp()("", "") + @test ismissing(RatcliffObershelp()("", missing)) end @testset "QGram" begin - @test evaluate(QGram(1), "abc", "abc") == 0 - @test evaluate(QGram(1), "", "abc") == 3 - @test evaluate(QGram(1), "abc", "cba") == 0 - @test evaluate(QGram(1), "abc", "ccc") == 4 - @test evaluate(QGram(4), "aü☃", "aüaüafs") == 4 - @test evaluate(QGram(2), SubString("aü☃", 1, 4), SubString("aüaüafs", 1, 4)) == 2 - @test evaluate(QGram(2), graphemes("alborgów"), graphemes("amoniak")) ≈ evaluate(QGram(2), "alborgów", "amoniak") + @test QGram(1)("abc", "abc") == 0 + @test QGram(1)("", "abc") == 3 + @test QGram(1)("abc", "cba") == 0 + @test QGram(1)("abc", "ccc") == 4 + @test QGram(4)("aü☃", "aüaüafs") == 4 + @test QGram(2)(SubString("aü☃", 1, 4), SubString("aüaüafs", 1, 4)) == 2 + @test QGram(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ QGram(2)("alborgów", "amoniak") @test QGram(1)("abc", "cba") == 0 @test result_type(QGram(1), "hello", "world") == Int - @test ismissing(evaluate(QGram(1), "", missing)) - @inferred evaluate(QGram(1), "", "") + @test ismissing(QGram(1)("", missing)) + @inferred QGram(1)("", "") end @testset "Cosine" begin - @test isnan(evaluate(Cosine(2), "", "abc")) - @test evaluate(Cosine(2), "abc", "ccc") ≈ 1 atol = 1e-4 - @test evaluate(Cosine(2), "leia", "leela") ≈ 0.7113249 atol = 1e-4 - @test evaluate(Cosine(2), [1, 2, 3], [1, 2, 4]) ≈ 0.5 - @test evaluate(Cosine(2), graphemes("alborgów"), graphemes("amoniak")) ≈ evaluate(Cosine(2), "alborgów", "amoniak") + @test isnan(Cosine(2)("", "abc")) + @test Cosine(2)("abc", "ccc") ≈ 1 atol = 1e-4 + @test Cosine(2)("leia", "leela") ≈ 0.7113249 atol = 1e-4 + @test Cosine(2)([1, 2, 3], [1, 2, 4]) ≈ 0.5 + @test Cosine(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ Cosine(2)("alborgów", "amoniak") @test Cosine(2)("leia", "leela") ≈ 0.7113249 atol = 1e-4 @test result_type(Cosine(2), "hello", "world") == typeof(float(1)) - @inferred evaluate(Cosine(2), "", "") - @test ismissing(evaluate(Cosine(2), "", missing)) + @inferred Cosine(2)("", "") + @test ismissing(Cosine(2)("", missing)) end @testset "Jaccard" begin - @test evaluate(Jaccard(1), "", "abc") ≈ 1.0 - @test evaluate(Jaccard(1), "abc", "ccc") ≈ 2/3 atol = 1e-4 - @test evaluate(Jaccard(2), "leia", "leela") ≈ 0.83333 atol = 1e-4 - @test evaluate(Jaccard(2), [1, 2, 3], [1, 2, 4]) ≈ 2/3 atol = 1e-4 - @test evaluate(Jaccard(2), graphemes("alborgów"), graphemes("amoniak")) ≈ evaluate(Jaccard(2), "alborgów", "amoniak") + @test Jaccard(1)("", "abc") ≈ 1.0 + @test Jaccard(1)("abc", "ccc") ≈ 2/3 atol = 1e-4 + @test Jaccard(2)("leia", "leela") ≈ 0.83333 atol = 1e-4 + @test Jaccard(2)([1, 2, 3], [1, 2, 4]) ≈ 2/3 atol = 1e-4 + @test Jaccard(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ Jaccard(2)("alborgów", "amoniak") @test Jaccard(2)("leia", "leela") ≈ 0.83333 atol = 1e-4 @test result_type(Jaccard(1), "hello", "world") == typeof(float(1)) - @inferred evaluate(Jaccard(1), "", "") - @test ismissing(evaluate(Jaccard(1), "", missing)) + @inferred Jaccard(1)("", "") + @test ismissing(Jaccard(1)("", missing)) end @testset "SorensenDice" begin - @test evaluate(SorensenDice(1), "night", "nacht") ≈ 0.4 atol = 1e-4 - @test evaluate(SorensenDice(2), "night", "nacht") ≈ 0.75 atol = 1e-4 - @test evaluate(SorensenDice(2), graphemes("alborgów"), graphemes("amoniak")) ≈ evaluate(SorensenDice(2), "alborgów", "amoniak") + @test SorensenDice(1)("night", "nacht") ≈ 0.4 atol = 1e-4 + @test SorensenDice(2)("night", "nacht") ≈ 0.75 atol = 1e-4 + @test SorensenDice(2)(graphemes("alborgów"), graphemes("amoniak")) ≈ SorensenDice(2)("alborgów", "amoniak") @test SorensenDice(2)("night", "nacht") ≈ 0.75 atol = 1e-4 @test result_type(SorensenDice(1), "hello", "world") == typeof(float(1)) - @inferred evaluate(SorensenDice(1), "", "") - @test ismissing(evaluate(SorensenDice(1), "", missing)) + @inferred SorensenDice(1)("", "") + @test ismissing(SorensenDice(1)("", missing)) end @testset "Overlap" begin - @test evaluate(Overlap(1), "night", "nacht") ≈ 0.4 atol = 1e-4 - @test evaluate(Overlap(1), "context", "contact") ≈ .2 atol = 1e-4 + @test Overlap(1)("night", "nacht") ≈ 0.4 atol = 1e-4 + @test Overlap(1)("context", "contact") ≈ .2 atol = 1e-4 @test Overlap(1)("context", "contact") ≈ .2 atol = 1e-4 @test result_type(Overlap(1), "hello", "world") == typeof(float(1)) - @inferred evaluate(Overlap(1), "", "") - @test ismissing(evaluate(Overlap(1), "", missing)) + @inferred Overlap(1)("", "") + @test ismissing(Overlap(1)("", missing)) end @testset "MorisitaOverlap" begin # overlap for 'n', 'h', and 't' and 5 q-grams per string: - @test evaluate(MorisitaOverlap(1), "night", "nacht") == 0.4 # 1.0-((2*3)/(5*5/5 + 5*5/5)) + @test MorisitaOverlap(1)("night", "nacht") == 0.4 # 1.0-((2*3)/(5*5/5 + 5*5/5)) # overlap for 'o', 'n', 2-overlap for 'c' and 't' and 7 unique q-grams in total so multiplicity vectors # ms1 = [1, 1, 1, 2, 1, 1, 0] # ms2 = [2, 1, 1, 2, 0, 0, 1] # sum(ms1 .* ms2) = 8, sum(ms1 .^ 2) = 9, sum(ms2 .^ 2) = 11, sum(ms1) = 7, sum(ms2) = 7 - @test evaluate(MorisitaOverlap(1), "context", "contact") ≈ .2 atol = 1e-4 # 1.0-((2*8)/(9*7/7 + 11*7/7)) = 16/20 + @test MorisitaOverlap(1)("context", "contact") ≈ .2 atol = 1e-4 # 1.0-((2*8)/(9*7/7 + 11*7/7)) = 16/20 @test MorisitaOverlap(1)("context", "contact") ≈ .2 atol = 1e-4 # Multiplicity vectors for 2-grams "co", "on", "nt", "te", "ex", "xt", "ta", "ac", "ct" @@ -166,17 +167,17 @@ using StringDistances, Unicode, Test, Random @test MorisitaOverlap(2)("context", "contact") == 0.5 # 1.0-((2*3)/(6*6/6 + 6*6/6)) @test result_type(MorisitaOverlap(1), "hello", "world") == typeof(float(1)) - @inferred evaluate(MorisitaOverlap(1), "", "") - @test ismissing(evaluate(MorisitaOverlap(1), "", missing)) + @inferred MorisitaOverlap(1)("", "") + @test ismissing(MorisitaOverlap(1)("", missing)) end @testset "NMD" begin # m(s1) = [1, 1, 1, 1, 1, 0, 0], m(s2) = [1, 0, 0, 1, 1, 1, 1] - @test evaluate(NMD(1), "night", "nacht") == 0.4 # (7-5)/5 + @test NMD(1)("night", "nacht") == 0.4 # (7-5)/5 # ms1 = [1, 1, 1, 2, 1, 1, 0] # ms2 = [2, 1, 1, 2, 0, 0, 1] - @test evaluate(NMD(1), "context", "contact") ≈ 0.2857 atol = 1e-4 # ((2+1+1+2+1+1+1)-7)/(7) + @test NMD(1)("context", "contact") ≈ 0.2857 atol = 1e-4 # ((2+1+1+2+1+1+1)-7)/(7) @test NMD(1)("context", "contact") ≈ 0.2857 atol = 1e-4 # ms1 = [1, 1, 1, 1, 1, 1, 0, 0, 0] @@ -184,8 +185,8 @@ using StringDistances, Unicode, Test, Random @test NMD(2)("context", "contact") == 0.5 # ((1+1+1+1+1+1+1+1+1)-6)/6 @test result_type(NMD(1), "hello", "world") == typeof(float(1)) - @inferred evaluate(NMD(1), "", "") - @test ismissing(evaluate(NMD(1), "", missing)) + @inferred NMD(1)("", "") + @test ismissing(NMD(1)("", missing)) end @testset "QGramDict and QGramSortedVector counts qgrams" begin @@ -236,37 +237,37 @@ using StringDistances, Unicode, Test, Random for _ in 1:100 qlen = rand(2:5) str1, str2 = partlyoverlappingstrings(6:100, Chars) - d = Jaccard(qlen) + dist = Jaccard(qlen) qd1 = QGramDict(str1, qlen) qd2 = QGramDict(str2, qlen) - @test evaluate(d, str1, str2) == evaluate(d, qd1, qd2) + @test dist(str1, str2) == dist(qd1, qd2) qd1b = QGramDict(graphemes(str1), qlen) qd2b = QGramDict(graphemes(str2), qlen) - @test evaluate(d, str1, str2) == evaluate(d, qd1b, qd2b) + @test dist(str1, str2) == dist(qd1b, qd2b) qc1 = QGramSortedVector(str1, qlen) qc2 = QGramSortedVector(str2, qlen) - @test evaluate(d, str1, str2) == evaluate(d, qc1, qc2) + @test dist(str1, str2) == dist(qc1, qc2) qc1b = QGramSortedVector(graphemes(str1), qlen) qc2b = QGramSortedVector(graphemes(str2), qlen) - @test evaluate(d, str1, str2) == evaluate(d, qc1b, qc2b) + @test dist(str1, str2) == dist(qc1b, qc2b) end end @testset "QGram distance on short strings" begin - @test isnan(evaluate(Overlap(2), "1", "2")) - @test isnan(evaluate(Jaccard(3), "s1", "s2")) - @test isnan(evaluate(Cosine(5), "s1", "s2")) + @test isnan(Overlap(2)( "1", "2")) + @test isnan(Jaccard(3)("s1", "s2")) + @test isnan(Cosine(5)( "s1", "s2")) - @test !isnan(evaluate(Overlap(2), "s1", "s2")) - @test !isnan(evaluate(Jaccard(3), "st1", "st2")) - @test !isnan(evaluate(Cosine(5), "stri1", "stri2")) + @test !isnan(Overlap(2)( "s1", "s2")) + @test !isnan(Jaccard(3)("st1", "st2")) + @test !isnan(Cosine(5)( "stri1", "stri2")) - @test !isnan(evaluate(Jaccard(3), "st1", "str2")) - @test !isnan(evaluate(Jaccard(3), "str1", "st2")) + @test !isnan(Jaccard(3)("st1", "str2")) + @test !isnan(Jaccard(3)("str1", "st2")) end @testset "Differential testing of String, QGramDict, and QGramSortedVector" begin @@ -279,13 +280,13 @@ using StringDistances, Unicode, Test, Random # QGramDict gets same result as for standard string qd1 = QGramDict(str1, qlen) qd2 = QGramDict(str2, qlen) - expected = evaluate(dist, str1, str2) - @test expected == evaluate(dist, qd1, qd2) + expected = dist(str1, str2) + @test expected == dist(qd1, qd2) # QGramSortedVector gets same result as for standard string qc1 = QGramSortedVector(str1, qlen) qc2 = QGramSortedVector(str2, qlen) - @test expected == evaluate(dist, qc1, qc2) + @test expected == dist(qc1, qc2) end end end @@ -319,33 +320,30 @@ using StringDistances, Unicode, Test, Random (Cosine(2), [0.6000000 0.7763932 0.6220355 0.0741799 NaN NaN 1.0000000 1.0000000 0.6348516 0.6619383 0.1679497 1.0000000 0.9407651 0.5000000 1.0000000 0.7113249])) # Test with R package StringDist for x in solutions - t, solution = x + dist, solution = x for i in eachindex(solution) - if isnan(evaluate(t, strings[i]...)) + if isnan(dist(strings[i]...)) @test isnan(solution[i]) else - @test evaluate(t, strings[i]...) ≈ solution[i] atol = 1e-4 + @test dist(strings[i]...) ≈ solution[i] atol = 1e-4 end end end # test RatcliffObershelp solution = [83, 73, 62, 93, 0, 100, 0, 33, 62, 71, 83, 27, 33, 78, 50, 67] for i in eachindex(strings) - @test round(Int, (1 - evaluate(RatcliffObershelp(), strings[i]...)) * 100) ≈ solution[i] atol = 1e-4 + @test round(Int, (1 - RatcliffObershelp()(strings[i]...)) * 100) ≈ solution[i] atol = 1e-4 end # test max_dist for i in eachindex(strings) d = Levenshtein()(strings[i]...) - @test Levenshtein(d)(strings[i]...) == d + @test Levenshtein()(strings[i]...; max_dist = d) == d d = OptimalStringAlignement()(strings[i]...) - @test OptimalStringAlignement(d)(strings[i]...) == d + @test OptimalStringAlignement()(strings[i]...; max_dist = d) == d end end -d = OptimalStringAlignement()("abcdef", "abcxyf") -@test OptimalStringAlignement(d)("abcdef", "abcxyf") == d - #= R test diff --git a/test/modifiers.jl b/test/modifiers.jl index 5c9846e..5223696 100644 --- a/test/modifiers.jl +++ b/test/modifiers.jl @@ -61,7 +61,7 @@ end compare("aüa", "aua", Levenshtein()) @test compare("ok", missing, Levenshtein()) === missing compare("aüa", "aua", OptimalStringAlignement()) - @test StringDistances.normalize(Partial(OptimalStringAlignement()))("ab", "cde") == 1.0 + @test StringDistances.Normalized(Partial(OptimalStringAlignement()))("ab", "cde") == 1.0 @test compare("ab", "de", Partial(OptimalStringAlignement())) == 0 # RatcliffObershelp