do not normalize Partial/TokenSet/TokenSort by default
matthieugomez
parent
09da838753
commit
8c2226bf4b
|
|
@ -5,9 +5,10 @@ using Distances
|
|||
include("utils.jl")
|
||||
include("edit.jl")
|
||||
include("qgram.jl")
|
||||
include("modifiers.jl")
|
||||
include("normalize.jl")
|
||||
|
||||
const StringDistance = Union{Jaro, Levenshtein, DamerauLevenshtein, RatcliffObershelp, QGramDistance, Winkler, Partial, TokenSort, TokenSet, TokenMax, Normalize}
|
||||
const StringDistance = Union{Jaro, Levenshtein, DamerauLevenshtein, RatcliffObershelp, QGramDistance, Winkler, Partial, TokenSort, TokenSet, TokenMax, Normalized}
|
||||
Distances.result_type(dist::StringDistance, s1, s2) = typeof(dist("", ""))
|
||||
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ function (dist::Jaro)(s1, s2)
|
|||
return 1.0 - (m / len1 + m / len2 + (m - t/2) / m) / 3.0
|
||||
end
|
||||
|
||||
(dist::Jaro)(s1, s2, ::Nothing) = (dist::Jaro)(s1, s2)
|
||||
"""
|
||||
Levenshtein()
|
||||
|
||||
|
|
@ -228,4 +229,6 @@ function longest_common_pattern(s1, s2)
|
|||
end
|
||||
end
|
||||
return start1, start2, len
|
||||
end
|
||||
end
|
||||
|
||||
(dist::RatcliffObershelp)(s1, s2, ::Nothing) = dist(s1, s2)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,121 @@
|
|||
"""
|
||||
Partial(dist)
|
||||
|
||||
Creates the `Partial{dist}` distance.
|
||||
|
||||
`Partial{dist}` modifies the string distance `dist` to return the
|
||||
minimum distance between the shorter string and substrings of the longer string
|
||||
|
||||
### Examples
|
||||
```julia-repl
|
||||
julia> s1 = "New York Mets vs Atlanta Braves"
|
||||
julia> s2 = "Atlanta Braves vs New York Mets"
|
||||
julia> evaluate(Partial(RatcliffObershelp()), s1, s2)
|
||||
0.5483870967741935
|
||||
```
|
||||
"""
|
||||
struct Partial{S <: SemiMetric} <: SemiMetric
|
||||
dist::S
|
||||
end
|
||||
|
||||
function (dist::Partial)(s1, s2, max_dist = nothing)
|
||||
s1, s2 = reorder(s1, s2)
|
||||
len1, len2 = length(s1), length(s2)
|
||||
out = dist.dist(s1, s2, max_dist)
|
||||
len1 == len2 && return out
|
||||
len1 == 0 && return out
|
||||
for x in qgrams(s2, len1)
|
||||
curr = dist.dist(s1, x, max_dist)
|
||||
out = min(out, curr)
|
||||
max_dist !== nothing && (max_dist = min(out, max_dist))
|
||||
end
|
||||
return out
|
||||
end
|
||||
|
||||
function (dist::Partial{RatcliffObershelp})(s1, s2, max_dist = nothing)
|
||||
s1, s2 = reorder(s1, s2)
|
||||
len1, len2 = length(s1), length(s2)
|
||||
len1 == len2 && return dist.dist(s1, s2)
|
||||
out = 1.0
|
||||
for r in matching_blocks(s1, s2)
|
||||
# Make sure the substring of s2 has length len1
|
||||
s2_start = r[2] - r[1] + 1
|
||||
s2_end = s2_start + len1 - 1
|
||||
if s2_start <= 0
|
||||
s2_end += 1 - s2_start
|
||||
s2_start += 1 - s2_start
|
||||
elseif s2_end > len2
|
||||
s2_start += len2 - s2_end
|
||||
s2_end += len2 - s2_end
|
||||
end
|
||||
curr = dist.dist(s1, _slice(s2, s2_start - 1, s2_end))
|
||||
out = min(out, curr)
|
||||
end
|
||||
return out
|
||||
end
|
||||
|
||||
"""
|
||||
TokenSort(dist)
|
||||
|
||||
Creates the `TokenSort{dist}` distance.
|
||||
|
||||
`TokenSort{dist}` modifies the string distance `dist` to adjust for differences
|
||||
in word orders by reording words alphabetically.
|
||||
|
||||
### Examples
|
||||
```julia-repl
|
||||
julia> s1 = "New York Mets vs Atlanta Braves"
|
||||
julia> s1 = "New York Mets vs Atlanta Braves"
|
||||
julia> s2 = "Atlanta Braves vs New York Mets"
|
||||
julia> evaluate(TokenSort(RatcliffObershelp()), s1, s2)
|
||||
0.0
|
||||
```
|
||||
"""
|
||||
struct TokenSort{S <: SemiMetric} <: SemiMetric
|
||||
dist::S
|
||||
end
|
||||
|
||||
# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
|
||||
function (dist::TokenSort)(s1::AbstractString, s2::AbstractString, max_dist = nothing)
|
||||
s1 = join(sort!(split(s1)), " ")
|
||||
s2 = join(sort!(split(s2)), " ")
|
||||
out = dist.dist(s1, s2, max_dist)
|
||||
end
|
||||
|
||||
"""
|
||||
TokenSet(dist)
|
||||
|
||||
Creates the `TokenSet{dist}` distance.
|
||||
|
||||
`TokenSet{dist}` modifies the string distance `dist` to adjust for differences
|
||||
in word orders and word numbers by comparing the intersection of two strings with each string.
|
||||
|
||||
### Examples
|
||||
```julia-repl
|
||||
julia> s1 = "New York Mets vs Atlanta"
|
||||
julia> s2 = "Atlanta Braves vs New York Mets"
|
||||
julia> evaluate(TokenSet(RatcliffObershelp()), s1, s2)
|
||||
0.0
|
||||
```
|
||||
"""
|
||||
struct TokenSet{S <: SemiMetric} <: SemiMetric
|
||||
dist::S
|
||||
end
|
||||
|
||||
# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
|
||||
function (dist::TokenSet)(s1::AbstractString, s2::AbstractString, max_dist = nothing)
|
||||
v1 = unique!(sort!(split(s1)))
|
||||
v2 = unique!(sort!(split(s2)))
|
||||
v0 = intersect(v1, v2)
|
||||
s0 = join(v0, " ")
|
||||
s1 = join(v1, " ")
|
||||
s2 = join(v2, " ")
|
||||
isempty(s0) && return dist.dist(s1, s2, max_dist)
|
||||
score_01 = dist.dist(s0, s1, max_dist)
|
||||
max_dist !== nothing && (max_dist = min(max_dist, score_01))
|
||||
score_02 = dist.dist(s0, s2, max_dist)
|
||||
max_dist !== nothing && (max_dist = min(max_dist, score_02))
|
||||
score_12 = dist.dist(s1, s2, max_dist)
|
||||
min(score_01, score_02, score_12)
|
||||
end
|
||||
|
||||
177
src/normalize.jl
177
src/normalize.jl
|
|
@ -1,16 +1,9 @@
|
|||
struct Normalize{S <: SemiMetric} <: SemiMetric
|
||||
|
||||
struct Normalized{S <: SemiMetric} <: SemiMetric
|
||||
dist::S
|
||||
end
|
||||
"""
|
||||
normalize(dist::SemiMetric)
|
||||
|
||||
Normalize a metric, so that `evaluate` always return a Float64 between 0 and 1 (or a `missing` if one element is missing)
|
||||
"""
|
||||
# also a normalized distance always accept a third argument, max_dist.
|
||||
|
||||
normalize(dist::SemiMetric) = Normalize{typeof(dist)}(dist)
|
||||
|
||||
function (dist::Normalize{<: Union{Levenshtein, DamerauLevenshtein}})(s1, s2, max_dist = 1.0)
|
||||
# A normalized distance is between 0 and 1, and accept a third argument, max_dist.
|
||||
function (dist::Normalized{<: Union{Levenshtein, DamerauLevenshtein}})(s1, s2, max_dist = 1.0)
|
||||
((s1 === missing) | (s2 === missing)) && return missing
|
||||
s1, s2 = reorder(s1, s2)
|
||||
len1, len2 = length(s1), length(s2)
|
||||
|
|
@ -20,7 +13,7 @@ function (dist::Normalize{<: Union{Levenshtein, DamerauLevenshtein}})(s1, s2, ma
|
|||
out > max_dist ? 1.0 : out
|
||||
end
|
||||
|
||||
function (dist::Normalize{<: QGramDistance})(s1, s2, max_dist = 1.0)
|
||||
function (dist::Normalized{<: QGramDistance})(s1, s2, max_dist = 1.0)
|
||||
((s1 === missing) | (s2 === missing)) && return missing
|
||||
# When string length < q for qgram distance, returns s1 == s2
|
||||
s1, s2 = reorder(s1, s2)
|
||||
|
|
@ -33,16 +26,26 @@ function (dist::Normalize{<: QGramDistance})(s1, s2, max_dist = 1.0)
|
|||
end
|
||||
end
|
||||
|
||||
function (dist::Normalize)(s1, s2, max_dist = 1.0)
|
||||
function (dist::Normalized)(s1, s2, max_dist = 1.0)
|
||||
dist.dist(s1, s2)
|
||||
end
|
||||
|
||||
"""
|
||||
normalize(dist::SemiMetric)
|
||||
|
||||
Normalize a metric, so that `evaluate` always return a Float64 between 0 and 1
|
||||
"""
|
||||
normalize(dist::SemiMetric) = Normalized(dist)
|
||||
normalize(dist::Normalized) = dist
|
||||
|
||||
|
||||
|
||||
"""
|
||||
Winkler(dist; p::Real = 0.1, threshold::Real = 0.7, maxlength::Integer = 4)
|
||||
|
||||
Creates the `Winkler{dist, p, threshold, maxlength}` distance.
|
||||
|
||||
`Winkler{dist, p, threshold, length)` modifies the string distance `dist` to decrease the
|
||||
`Winkler{dist, p, threshold, length)` modifies the string distance `normalize(dist)` to decrease the
|
||||
distance between two strings, when their original distance is below some `threshold`.
|
||||
The boost is equal to `min(l, maxlength) * p * dist` where `l` denotes the
|
||||
length of their common prefix and `dist` denotes the original distance
|
||||
|
|
@ -57,9 +60,10 @@ end
|
|||
|
||||
function Winkler(dist::SemiMetric; p = 0.1, threshold = 0.7, maxlength = 4)
|
||||
p * maxlength <= 1 || throw("scaling factor times maxlength of common prefix must be lower than one")
|
||||
Winkler{typeof(normalize(dist))}(normalize(dist), 0.1, 0.7, 4)
|
||||
dist = normalize(dist)
|
||||
Winkler{typeof(dist)}(dist, 0.1, 0.7, 4)
|
||||
end
|
||||
normalize(dist::Winkler) = dist
|
||||
isnormalized(dist::Winkler) = true
|
||||
|
||||
function (dist::Winkler)(s1, s2, max_dist = 1.0)
|
||||
# cannot do max_dist because of boosting threshold
|
||||
|
|
@ -72,144 +76,12 @@ function (dist::Winkler)(s1, s2, max_dist = 1.0)
|
|||
end
|
||||
|
||||
|
||||
"""
|
||||
Partial(dist)
|
||||
|
||||
Creates the `Partial{dist}` distance.
|
||||
|
||||
`Partial{dist}` modifies the string distance `dist` to return the
|
||||
minimum distance between the shorter string and substrings of the longer string
|
||||
|
||||
### Examples
|
||||
```julia-repl
|
||||
julia> s1 = "New York Mets vs Atlanta Braves"
|
||||
julia> s2 = "Atlanta Braves vs New York Mets"
|
||||
julia> evaluate(Partial(RatcliffObershelp()), s1, s2)
|
||||
0.5483870967741935
|
||||
```
|
||||
"""
|
||||
struct Partial{S <: SemiMetric} <: SemiMetric
|
||||
dist::S
|
||||
Partial{S}(dist::S) where {S <: SemiMetric} = new(dist)
|
||||
end
|
||||
Partial(dist::SemiMetric) = Partial{typeof(normalize(dist))}(normalize(dist))
|
||||
normalize(dist::Partial) = dist
|
||||
|
||||
function (dist::Partial)(s1, s2, max_dist = 1.0)
|
||||
s1, s2 = reorder(s1, s2)
|
||||
len1, len2 = length(s1), length(s2)
|
||||
len1 == len2 && return dist.dist(s1, s2, max_dist)
|
||||
len1 == 0 && return 1.0
|
||||
out = 1.0
|
||||
for x in qgrams(s2, len1)
|
||||
curr = dist.dist(s1, x, max_dist)
|
||||
out = min(out, curr)
|
||||
max_dist = min(out, max_dist)
|
||||
end
|
||||
return out
|
||||
end
|
||||
|
||||
function (dist::Partial{Normalize{RatcliffObershelp}})(s1, s2, max_dist = 1.0)
|
||||
s1, s2 = reorder(s1, s2)
|
||||
len1, len2 = length(s1), length(s2)
|
||||
len1 == len2 && return dist.dist(s1, s2)
|
||||
out = 1.0
|
||||
for r in matching_blocks(s1, s2)
|
||||
# Make sure the substring of s2 has length len1
|
||||
s2_start = r[2] - r[1] + 1
|
||||
s2_end = s2_start + len1 - 1
|
||||
if s2_start <= 0
|
||||
s2_end += 1 - s2_start
|
||||
s2_start += 1 - s2_start
|
||||
elseif s2_end > len2
|
||||
s2_start += len2 - s2_end
|
||||
s2_end += len2 - s2_end
|
||||
end
|
||||
curr = dist.dist(s1, _slice(s2, s2_start - 1, s2_end))
|
||||
out = min(out, curr)
|
||||
end
|
||||
return out
|
||||
end
|
||||
|
||||
"""
|
||||
TokenSort(dist)
|
||||
|
||||
Creates the `TokenSort{dist}` distance.
|
||||
|
||||
`TokenSort{dist}` modifies the string distance `dist` to adjust for differences
|
||||
in word orders by reording words alphabetically.
|
||||
|
||||
### Examples
|
||||
```julia-repl
|
||||
julia> s1 = "New York Mets vs Atlanta Braves"
|
||||
julia> s1 = "New York Mets vs Atlanta Braves"
|
||||
julia> s2 = "Atlanta Braves vs New York Mets"
|
||||
julia> evaluate(TokenSort(RatcliffObershelp()), s1, s2)
|
||||
0.0
|
||||
```
|
||||
"""
|
||||
struct TokenSort{S <: SemiMetric} <: SemiMetric
|
||||
dist::S
|
||||
TokenSort{S}(dist::S) where {S <: SemiMetric} = new(dist)
|
||||
end
|
||||
TokenSort(dist::SemiMetric) = TokenSort{typeof(normalize(dist))}(normalize(dist))
|
||||
normalize(dist::TokenSort) = dist
|
||||
|
||||
# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
|
||||
function (dist::TokenSort)(s1::AbstractString, s2::AbstractString, max_dist = 1.0)
|
||||
s1 = join(sort!(split(s1)), " ")
|
||||
s2 = join(sort!(split(s2)), " ")
|
||||
dist.dist(s1, s2, max_dist)
|
||||
end
|
||||
|
||||
|
||||
"""
|
||||
TokenSet(dist)
|
||||
|
||||
Creates the `TokenSet{dist}` distance.
|
||||
|
||||
`TokenSet{dist}` modifies the string distance `dist` to adjust for differences
|
||||
in word orders and word numbers by comparing the intersection of two strings with each string.
|
||||
|
||||
### Examples
|
||||
```julia-repl
|
||||
julia> s1 = "New York Mets vs Atlanta"
|
||||
julia> s2 = "Atlanta Braves vs New York Mets"
|
||||
julia> evaluate(TokenSet(RatcliffObershelp()), s1, s2)
|
||||
0.0
|
||||
```
|
||||
"""
|
||||
struct TokenSet{S <: SemiMetric} <: SemiMetric
|
||||
dist::S
|
||||
TokenSet{S}(dist::S) where {S <: SemiMetric} = new(dist)
|
||||
end
|
||||
TokenSet(dist::SemiMetric) = TokenSet{typeof(normalize(dist))}(normalize(dist))
|
||||
normalize(dist::TokenSet) = dist
|
||||
|
||||
# http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
|
||||
function (dist::TokenSet)(s1::AbstractString, s2::AbstractString, max_dist = 1.0)
|
||||
v1 = unique!(sort!(split(s1)))
|
||||
v2 = unique!(sort!(split(s2)))
|
||||
v0 = intersect(v1, v2)
|
||||
s0 = join(v0, " ")
|
||||
s1 = join(v1, " ")
|
||||
s2 = join(v2, " ")
|
||||
isempty(s0) && return dist.dist(s1, s2, max_dist)
|
||||
score_01 = dist.dist(s0, s1, max_dist)
|
||||
max_dist = min(max_dist, score_01)
|
||||
score_02 = dist.dist(s0, s2, max_dist)
|
||||
max_dist = min(max_dist, score_02)
|
||||
score_12 = dist.dist(s1, s2, max_dist)
|
||||
min(score_01, score_02, score_12)
|
||||
end
|
||||
|
||||
|
||||
"""
|
||||
TokenMax(dist)
|
||||
|
||||
Creates the `TokenMax{dist}` distance
|
||||
|
||||
`TokenMax{dist}` is the minimum of the base distance `dist`,
|
||||
`TokenMax{dist}` is the minimum of the base distance `normalize(dist)`,
|
||||
its [`Partial`](@ref) modifier, its [`TokenSort`](@ref) modifier, and its
|
||||
[`TokenSet`](@ref) modifier, with penalty terms depending on string lengths.
|
||||
|
||||
|
|
@ -226,8 +98,11 @@ struct TokenMax{S <: SemiMetric} <: SemiMetric
|
|||
TokenMax{S}(dist::S) where {S <: SemiMetric} = new(dist)
|
||||
end
|
||||
|
||||
TokenMax(dist::SemiMetric) = TokenMax{typeof(normalize(dist))}(normalize(dist))
|
||||
normalize(dist::TokenMax) = dist
|
||||
function TokenMax(dist::SemiMetric)
|
||||
dist = normalize(dist)
|
||||
TokenMax{typeof(dist)}(dist)
|
||||
end
|
||||
isnormalized(dist::TokenMax) = true
|
||||
|
||||
function (dist::TokenMax)(s1::AbstractString, s2::AbstractString, max_dist = 1.0)
|
||||
s1, s2 = reorder(s1, s2)
|
||||
|
|
|
|||
10
src/qgram.jl
10
src/qgram.jl
|
|
@ -107,6 +107,8 @@ function (dist::QGram)(s1, s2)
|
|||
n
|
||||
end
|
||||
|
||||
(dist::QGram)(s1, s2, ::Nothing) = dist(s1, s2)
|
||||
|
||||
"""
|
||||
Cosine(q::Int)
|
||||
|
||||
|
|
@ -134,6 +136,8 @@ function (dist::Cosine)(s1, s2)
|
|||
1.0 - prodnorm / (sqrt(norm1) * sqrt(norm2))
|
||||
end
|
||||
|
||||
(dist::Cosine)(s1, s2, ::Nothing) = dist(s1, s2)
|
||||
|
||||
"""
|
||||
Jaccard(q::Int)
|
||||
|
||||
|
|
@ -160,6 +164,8 @@ function (dist::Jaccard)(s1, s2)
|
|||
1.0 - nintersect / (ndistinct1 + ndistinct2 - nintersect)
|
||||
end
|
||||
|
||||
(dist::Jaccard)(s1, s2, ::Nothing) = dist(s1, s2)
|
||||
|
||||
"""
|
||||
SorensenDice(q::Int)
|
||||
|
||||
|
|
@ -186,6 +192,8 @@ function (dist::SorensenDice)(s1, s2)
|
|||
1.0 - 2.0 * nintersect / (ndistinct1 + ndistinct2)
|
||||
end
|
||||
|
||||
(dist::SorensenDice)(s1, s2, ::Nothing) = dist(s1, s2)
|
||||
|
||||
"""
|
||||
Overlap(q::Int)
|
||||
|
||||
|
|
@ -211,3 +219,5 @@ function (dist::Overlap)(s1, s2)
|
|||
end
|
||||
1.0 - nintersect / min(ndistinct1, ndistinct2)
|
||||
end
|
||||
|
||||
(dist::Overlap)(s1, s2, ::Nothing) = dist(s1, s2)
|
||||
|
|
|
|||
Loading…
Reference in New Issue