2017-08-05 20:45:19 +02:00
|
|
|
##############################################################################
|
|
|
|
##
|
|
|
|
## compare
|
2018-05-17 17:33:55 +02:00
|
|
|
## compare always return a value between 0 and 1.
|
2017-08-05 20:45:19 +02:00
|
|
|
##
|
|
|
|
##############################################################################
|
|
|
|
|
2019-08-15 17:07:12 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString, dist::PreMetric)
|
2017-08-05 20:45:19 +02:00
|
|
|
1.0 - evaluate(dist, s1, s2)
|
|
|
|
end
|
|
|
|
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString,
|
|
|
|
dist::Union{Hamming, Levenshtein, DamerauLevenshtein})
|
2017-08-05 20:45:19 +02:00
|
|
|
len = max(length(s1), length(s2))
|
2018-05-17 17:33:55 +02:00
|
|
|
len == 0 ? 1.0 : 1.0 - evaluate(dist, s1, s2) / len
|
2017-08-05 20:45:19 +02:00
|
|
|
end
|
2018-05-16 00:39:50 +02:00
|
|
|
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString,
|
|
|
|
dist::AbstractQGramDistance)
|
2018-05-17 17:33:55 +02:00
|
|
|
# When string length < q for qgram distance, returns s1 == s2
|
2019-08-17 19:12:55 +02:00
|
|
|
len1, len2 = length(s1), length(s2)
|
2019-08-17 17:56:54 +02:00
|
|
|
min(len1, len2) <= (dist.N - 1) && return convert(Float64, s1 == s2)
|
2018-05-17 17:33:55 +02:00
|
|
|
if typeof(dist) <: QGram
|
2019-08-17 17:56:54 +02:00
|
|
|
1 - evaluate(dist, s1, s2) / (len1 + len2 - 2 * dist.N + 2)
|
2018-05-17 17:33:55 +02:00
|
|
|
else
|
|
|
|
1 - evaluate(dist, s1, s2)
|
|
|
|
end
|
2018-05-16 00:39:50 +02:00
|
|
|
end
|
2018-05-17 17:38:55 +02:00
|
|
|
|
2019-08-17 18:26:24 +02:00
|
|
|
@deprecate compare(dist::PreMetric, s1::AbstractString, s2::AbstractString) compare(s1, s2, dist)
|
|
|
|
|
2018-05-17 17:38:55 +02:00
|
|
|
##############################################################################
|
|
|
|
##
|
|
|
|
## Winkler
|
|
|
|
##
|
|
|
|
##############################################################################
|
|
|
|
|
|
|
|
struct Winkler{T1 <: PreMetric, T2 <: Real, T3 <: Real} <: PreMetric
|
|
|
|
dist::T1
|
|
|
|
scaling_factor::T2 # scaling factor. Default to 0.1
|
|
|
|
boosting_limit::T3 # boost threshold. Default to 0.7
|
|
|
|
end
|
|
|
|
|
|
|
|
# restrict to distance between 0 and 1
|
|
|
|
Winkler(x) = Winkler(x, 0.1, 0.7)
|
|
|
|
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString, dist::Winkler)
|
|
|
|
score = compare(s1, s2, dist.dist)
|
2018-05-17 17:38:55 +02:00
|
|
|
l = common_prefix(s1, s2, 4)[1]
|
|
|
|
# common prefix adjustment
|
|
|
|
if score >= dist.boosting_limit
|
|
|
|
score += l * dist.scaling_factor * (1 - score)
|
|
|
|
end
|
|
|
|
return score
|
|
|
|
end
|
|
|
|
|
|
|
|
##############################################################################
|
|
|
|
##
|
|
|
|
## Partial
|
|
|
|
## http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
|
|
|
|
##
|
|
|
|
##############################################################################
|
|
|
|
struct Partial{T <: PreMetric} <: PreMetric
|
|
|
|
dist::T
|
|
|
|
end
|
|
|
|
|
|
|
|
# general
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString, dist::Partial)
|
2018-05-17 17:38:55 +02:00
|
|
|
s2, len2, s1, len1 = reorder(s1, s2)
|
2019-08-17 18:26:24 +02:00
|
|
|
len1 == len2 && return compare(s1, s2, dist.dist)
|
|
|
|
len1 == 0 && return compare("", "", dist.dist)
|
2018-07-04 20:02:50 +02:00
|
|
|
out = 0.0
|
2019-08-17 19:12:55 +02:00
|
|
|
for x in qgram_iterator(s2, len1)
|
|
|
|
curr = compare(s1, x, dist.dist)
|
2018-05-17 17:38:55 +02:00
|
|
|
out = max(out, curr)
|
|
|
|
end
|
|
|
|
return out
|
|
|
|
end
|
|
|
|
|
|
|
|
# Specialization for RatcliffObershelp distance
|
|
|
|
# Code follows https://github.com/seatgeek/fuzzywuzzy/blob/master/fuzzywuzzy/fuzz.py
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString, dist::Partial{RatcliffObershelp})
|
2018-05-17 17:38:55 +02:00
|
|
|
s2, len2, s1, len1 = reorder(s1, s2)
|
2019-08-17 18:26:24 +02:00
|
|
|
len1 == len2 && return compare(s1, s2, dist.dist)
|
2018-05-17 17:38:55 +02:00
|
|
|
out = 0.0
|
2019-08-17 19:12:55 +02:00
|
|
|
for r in matching_blocks(s1, s2)
|
2018-05-17 17:38:55 +02:00
|
|
|
# here I difffer from fuzz.py by making sure the substring of s2 has length len1
|
|
|
|
s2_start = r[2] - r[1] + 1
|
|
|
|
s2_end = s2_start + len1 - 1
|
|
|
|
if s2_start <= 0
|
|
|
|
s2_end += 1 - s2_start
|
|
|
|
s2_start += 1 - s2_start
|
|
|
|
elseif s2_end > len2
|
|
|
|
s2_start += len2 - s2_end
|
|
|
|
s2_end += len2 - s2_end
|
|
|
|
end
|
2018-07-04 18:07:26 +02:00
|
|
|
i2_start = nextind(s2, 0, s2_start)
|
2019-08-14 00:18:04 +02:00
|
|
|
i2_end = nextind(s2, 0, s2_end)
|
2019-08-17 18:26:24 +02:00
|
|
|
curr = compare(s1, SubString(s2, i2_start, i2_end), RatcliffObershelp())
|
2018-05-17 17:38:55 +02:00
|
|
|
out = max(out, curr)
|
|
|
|
end
|
|
|
|
return out
|
|
|
|
end
|
|
|
|
|
|
|
|
##############################################################################
|
|
|
|
##
|
|
|
|
## TokenSort
|
|
|
|
## http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
|
|
|
|
##
|
|
|
|
##############################################################################
|
|
|
|
struct TokenSort{T <: PreMetric} <: PreMetric
|
|
|
|
dist::T
|
|
|
|
end
|
|
|
|
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString, dist::TokenSort)
|
2018-07-04 20:02:50 +02:00
|
|
|
s1 = join(sort!(split(s1)), " ")
|
|
|
|
s2 = join(sort!(split(s2)), " ")
|
2019-08-17 18:26:24 +02:00
|
|
|
compare(s1, s2, dist.dist)
|
2018-05-17 17:38:55 +02:00
|
|
|
end
|
|
|
|
|
|
|
|
##############################################################################
|
|
|
|
##
|
|
|
|
## TokenSet
|
|
|
|
## http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
|
|
|
|
##
|
|
|
|
##############################################################################
|
|
|
|
struct TokenSet{T <: PreMetric} <: PreMetric
|
|
|
|
dist::T
|
|
|
|
end
|
|
|
|
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString, dist::TokenSet)
|
2019-08-17 21:46:22 +02:00
|
|
|
v0, v1, v2 = separate!(SortedSet(split(s1)), SortedSet(split(s2)))
|
2018-05-17 17:38:55 +02:00
|
|
|
s0 = join(v0, " ")
|
2019-08-17 21:46:22 +02:00
|
|
|
s1 = join(union(v0, v1), " ")
|
|
|
|
s2 = join(union(v0, v2), " ")
|
2019-08-17 19:12:55 +02:00
|
|
|
max(compare(s0, s1, dist.dist),
|
2019-08-17 21:46:22 +02:00
|
|
|
compare(s0, s2, dist.dist),
|
|
|
|
compare(s1, s2, dist.dist))
|
|
|
|
|
2018-05-17 17:38:55 +02:00
|
|
|
end
|
|
|
|
|
2019-08-17 21:46:22 +02:00
|
|
|
# separate 2 sets in intersection, setdiff1, setdiff2 (all sorted)
|
|
|
|
function separate!(v1::SortedSet, v2::SortedSet)
|
|
|
|
out = OrderedSet{eltype(v1)}()
|
|
|
|
for x in v1
|
|
|
|
if x in v2
|
|
|
|
pop!(v1, x)
|
|
|
|
pop!(v2, x)
|
2018-05-17 17:38:55 +02:00
|
|
|
push!(out, x)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
return out, v1, v2
|
|
|
|
end
|
|
|
|
|
|
|
|
##############################################################################
|
|
|
|
##
|
|
|
|
## TokenMax
|
|
|
|
##
|
|
|
|
##############################################################################
|
|
|
|
struct TokenMax{T <: PreMetric} <: PreMetric
|
|
|
|
dist::T
|
|
|
|
end
|
|
|
|
|
2019-08-17 18:26:24 +02:00
|
|
|
function compare(s1::AbstractString, s2::AbstractString, dist::TokenMax)
|
|
|
|
dist0 = compare(s1, s2, dist.dist)
|
2018-05-17 17:38:55 +02:00
|
|
|
s2, len2, s1, len1 = reorder(s1, s2)
|
|
|
|
unbase_scale = 0.95
|
|
|
|
# if one string is much much shorter than the other
|
|
|
|
if len2 >= 1.5 * len1
|
|
|
|
# if strings are of dissimilar length, use partials
|
2019-08-17 18:26:24 +02:00
|
|
|
partial = compare(s1, s2, Partial(dist.dist))
|
|
|
|
ptsor = compare(s1, s2, TokenSort(Partial(dist.dist)))
|
|
|
|
ptser = compare(s1, s2, TokenSet(Partial(dist.dist)))
|
2018-05-17 17:38:55 +02:00
|
|
|
partial_scale = len2 > (8 * len1) ? 0.6 : 0.9
|
|
|
|
return max(dist0,
|
|
|
|
partial * partial_scale,
|
|
|
|
ptsor * unbase_scale * partial_scale,
|
|
|
|
ptser * unbase_scale * partial_scale)
|
|
|
|
else
|
2019-08-17 18:26:24 +02:00
|
|
|
ptsor = compare(s1, s2, TokenSort(dist.dist))
|
|
|
|
ptser = compare(s1, s2, TokenSet(dist.dist))
|
2018-05-17 17:38:55 +02:00
|
|
|
return max(dist0,
|
|
|
|
ptsor * unbase_scale,
|
|
|
|
ptser * unbase_scale)
|
|
|
|
end
|
|
|
|
end
|