parent
4973c6d99c
commit
12866bc20e
@ -1 +1,7 @@ |
||||
julia 0.6 |
||||
HTTP |
||||
JSON |
||||
HttpCommon |
||||
URIParser |
||||
# BibTeX |
||||
# BibTeXFormat |
@ -0,0 +1,12 @@ |
||||
* List of bugs seen that must be fixed |
||||
** TODO Date year saved as 1996/09 |
||||
What to do? |
||||
- Should it understand automatically it is a month? |
||||
- Just filter it out? |
||||
** Similar ids |
||||
Those three are the same: |
||||
- srinivasan12t_haptics_1997 |
||||
- srinivasan_haptics_1997 |
||||
- srinivasanl’zl_haptics_1997 |
||||
|
||||
On which field should we check the distance? Which threshold? How to merge fields? |
@ -1,5 +1,20 @@ |
||||
__precompile__() |
||||
|
||||
module Scjrm |
||||
|
||||
# package code goes here |
||||
using BibTeX |
||||
|
||||
const hooks = Dict() |
||||
const libraries = Dict() |
||||
currentlibrary = "" |
||||
const bibliography = Bibliography("") |
||||
|
||||
include("default.jl") |
||||
include("server.jl") |
||||
include("macros.jl") |
||||
include("globals.jl") |
||||
include("configs.jl") |
||||
include("zotero.jl") |
||||
include("bibliography.jl") |
||||
|
||||
end # module |
||||
|
@ -0,0 +1,104 @@ |
||||
#= |
||||
Functions to manage the library |
||||
=# |
||||
using JSON, StringDistances |
||||
|
||||
""" |
||||
Add `entry` to `bibliography` on key `id` |
||||
|
||||
Defines `pre_add_to_bib` and `after_add_to_bib`. |
||||
""" |
||||
function addtobibliography!(bibliography::Bibliography, id::String, entry::Citation) |
||||
@hook :pre_add_to_bib |
||||
info("Here") |
||||
exists = id in keys(bibliography) |
||||
info("The id you are adding $(id) does " * (exists ? "" : "not ") * "exists") |
||||
if exists |
||||
oldentry = bibliography[id] |
||||
# TODO: run a Levenshtein distance on the two entry and decide |
||||
# what to do Options should be to merge/replace or keep both. |
||||
# In the latter the id must be modified (like id-$counter) |
||||
|
||||
old = lowercase(get(oldentry,"title","")) |
||||
new = lowercase(get(entry,"title","-")) |
||||
differencepercent = compare(TokenMax(RatcliffObershelp()), old, new) |
||||
if differencepercent > 0.8 |
||||
@hook :duplicated_paper |
||||
warn("The old and the new seems to be the same paper\n |
||||
(similarity is $(differencepercent))") |
||||
@show json(oldentry) |
||||
@show json(entry) |
||||
else |
||||
@hook :similar_paper |
||||
info("keeping both") |
||||
id = nextidnumber(bibliography,id) |
||||
end |
||||
end |
||||
bibliography[id] = entry |
||||
|
||||
@hook :after_add_to_bib |
||||
end |
||||
|
||||
""" Set `newbib` as a the bibliography, replacing `bibliography`. |
||||
""" |
||||
function setbibliography!(bibliography::Bibliography, newbib::Bibliography) |
||||
empty!(bibliography) |
||||
merge!(bibliography, newbib) |
||||
end |
||||
|
||||
import Base.pop! |
||||
function pop!(bibliography::Bibliography, key::String) |
||||
copy = Dict(bibliography) |
||||
p = Base.pop!(copy, key) # FIXME: remove this Base. |
||||
empty!(bibliography) |
||||
foreach(k -> bibliography[k] = copy[k], keys(copy)) |
||||
# Behaves like a real `pop`, returning the popped key value |
||||
p |
||||
end |
||||
|
||||
# FIXME: find files? |
||||
"""Add all entries in a bibtex string to the current bibliography""" |
||||
function addbibtex!(bibliography, bibstring) |
||||
bibs = Bibliography(bibstring) |
||||
foreach(bib -> addtobibliography!(bibliography, createcitekey(bibs[bib]), bibs[bib]), |
||||
keys(bibs)) |
||||
"ok" |
||||
end |
||||
|
||||
function splitauthors(bib::Citation) |
||||
authors = split(bib["author"], " and ") |
||||
map(a -> (split(strip(a, ['}', '{']), ", ")), |
||||
authors) |
||||
end |
||||
|
||||
"""Create the cite key given a bibtex object""" |
||||
function createcitekey(bib::Citation) |
||||
authors = splitauthors(bib) |
||||
firstauthor = length(authors) > 0 ? authors[1][1] : "Unnamed" |
||||
title = split(get(bib, "title", "Untitled"), " ")[1] |
||||
year = match(r"([0-9]{4})", get(bib, "year", get(bib, "date", "0000")))[1] |
||||
|
||||
string(firstauthor, "_", title, "_", year) |> lowercase |
||||
end |
||||
|
||||
"""Generates a new id, with the same key, but with a number at the end |
||||
(id-\$number) so that it is unique in the library. Fallback for |
||||
duplicates. """ |
||||
function nextidnumber(bibliography::Bibliography,id::String) |
||||
num = 0 |
||||
while true |
||||
nid = string(id, "-", num) |
||||
nid in keys(bibliography) || break |
||||
num += 1 |
||||
end |
||||
nid |
||||
end |
||||
|
||||
# """ |
||||
# Apply some cleaning to the bibtex citation, namely fixing the key id. |
||||
|
||||
# Useful when adding a bibtex directly to verify that it conforms to our "standard" |
||||
# """ |
||||
# function cleancitation(cit) |
||||
|
||||
# end |
@ -0,0 +1,5 @@ |
||||
"""This function creates the path of all the libraries in the "libraries" variable""" |
||||
function createlibraries(libraries::Dict) |
||||
foreach(library -> mkpath(expanduser(libraries[library]["path"])), |
||||
keys(libraries)) |
||||
end |
@ -0,0 +1,71 @@ |
||||
#= |
||||
|
||||
This file defines the default config parameters and provides a simple |
||||
way of crating a template config file. |
||||
|
||||
=# |
||||
|
||||
defaultconfig = """ |
||||
\"\"\"Custom functions to run in certain circumstances. |
||||
Read the documentation to know which hooks are available.\"\"\" |
||||
Scjrm.sethooks!(Dict{Symbol,Function}()) |
||||
""" |
||||
|
||||
# TODO: you can use merge to have customization + defaults |
||||
|
||||
eval(defaultconfig) |
||||
|
||||
""" |
||||
Writes the config to the file `name` |
||||
""" |
||||
function writeconfig(name::String) |
||||
open(name, "w") do f |
||||
write(f, defaultconfig) |
||||
end |
||||
end |
||||
|
||||
""" |
||||
Set the `hooks`. If you want to remove all the hooks and then set |
||||
the new one, you should call `empty!(Scjrm.hooks)` before. |
||||
""" |
||||
function sethooks!(h::Dict) |
||||
merge!(hooks, h) |
||||
end |
||||
|
||||
""" |
||||
Add an `hook` to `hooks`. |
||||
""" |
||||
addhook!(s::Symbol, f::Function) = hooks[:pre_receive] = f |
||||
|
||||
""" |
||||
Remove `hook` from `hooks` |
||||
""" |
||||
function rmhook!(s::Symbol) |
||||
# `hooks` is a `const`ant. So we: |
||||
# copy `hooks` |
||||
tmphooks = deepcopy(hooks) |
||||
# Remove `s` from the copy |
||||
pop!(tmphooks, s) |
||||
# Clear `hooks` |
||||
empty!(hooks) |
||||
# Merge the copy to the empty `hooks` |
||||
merge!(hooks, tmphooks) |
||||
end |
||||
|
||||
function setlibraries!(l::Dict) |
||||
merge!(libraries,l) |
||||
end |
||||
|
||||
"""Change the current library. |
||||
|
||||
If `existing` is false, don't verify the new library exists. |
||||
Returns the new library""" |
||||
function currentlibrary!(l::String; existing = true) |
||||
global currentlibrary, libraries |
||||
|
||||
if l in keys(libraries) |
||||
currentlibrary = l |
||||
end |
||||
|
||||
currentlibrary |
||||
end |
@ -0,0 +1,10 @@ |
||||
using HttpCommon |
||||
|
||||
function makeheaders() |
||||
headers = HttpCommon.headers() |
||||
headers["X-Zotero-Version"] = "5.0.25" |
||||
headers["X-Zotero-Connector-API-Version"] = "2" |
||||
return headers |
||||
end |
||||
|
||||
const headers = makeheaders() |
@ -0,0 +1,17 @@ |
||||
macro hook(hookname) |
||||
return quote |
||||
try |
||||
$hookname in keys(hooks) && hooks[$hookname]() |
||||
nothing |
||||
end |
||||
end |
||||
end |
||||
|
||||
macro hook(hookname, args) |
||||
return quote |
||||
try |
||||
$hookname in keys(hooks) && hooks[$hookname]($args...) |
||||
nothing |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,45 @@ |
||||
#= |
||||
This server is a port of |
||||
|
||||
https://raw.githubusercontent.com/zotero/zotero/master/chrome/content/zotero/xpcom/server_connector.js |
||||
=# |
||||
using Mux, JSON |
||||
using HttpCommon: parsequerystring |
||||
using URIParser: unescape_form |
||||
|
||||
out = nothing |
||||
|
||||
@app connector = ( |
||||
Mux.defaults, |
||||
page("/connector/ping", x -> setzoteroheaders(pong(x)...)), |
||||
page("/connector/getSelectedCollection", x -> setzoteroheaders(collection(x, libraries, currentlibrary)...)), |
||||
page("/connector/saveSnapshot", x -> setzoteroheaders(savesnapshot(x, libraries, currentlibrary)...)), |
||||
page("/connector/saveItems", x -> setzoteroheaders(saveitems(x, libraries, currentlibrary)...)), |
||||
page("ui", x -> """ |
||||
Insert the bibtex here</br> |
||||
<form action="/add" method="post"> |
||||
<textarea name="field" cols="80" rows="15"></textarea> |
||||
</br> |
||||
<input type="submit" value="Submit"> |
||||
</form> |
||||
"""), |
||||
page("add", x -> addbibtex!(bibliography, |
||||
(x[:data] |> String |> parsequerystring)["field"] |
||||
|> unescape_form)), |
||||
page("/cli/library/:library", x -> begin |
||||
currentlibrary!(String(x[:params][:library])) |
||||
# Prevent copying the old bibliography over! |
||||
empty!(bibliography) |
||||
end), |
||||
Mux.notfound()) |
||||
|
||||
""" |
||||
Starts the `zotero`-compatible connector. |
||||
|
||||
It's a web server running (by default) on port 23199 that listens for connection from the zotero web plugin. |
||||
""" |
||||
function serve(; host=IPv4(127,0,0,1), port=23119, args...) |
||||
Mux.serve(connector, host = host, port = port, args...) |
||||
end |
||||
|
||||
keep_running() = while !isinteractive(); sleep(100); end |
@ -0,0 +1,232 @@ |
||||
using JSON, BibTeX |
||||
# Downloading |
||||
using URIParser, HTTP |
||||
#= |
||||
Mapping between zotero and BibTeX. |
||||
=# |
||||
|
||||
# https://github.com/retorquere/zotero-better-bibtex/blob/master/translators/Better%20BibLaTeX.ts |
||||
|
||||
"Map zotero reference types to bibtex" |
||||
const zotero_type_map = Dict( |
||||
"artwork" => "artwork", |
||||
"audioRecording" => "audio", |
||||
"bill" => "legislation", |
||||
"blogPost" => "online", |
||||
"book" => "book", |
||||
"bookSection" => "incollection", |
||||
"case" => "jurisdiction", |
||||
"computerProgram" => "software", |
||||
"conferencePaper" => "inproceedings", |
||||
"dictionaryEntry" => "inreference", |
||||
"document" => "misc", |
||||
"email" => "letter", |
||||
"encyclopediaArticle" => "misc", # "inreference", |
||||
"film" => "movie", |
||||
"forumPost" => "online", |
||||
"hearing" => "jurisdiction", |
||||
"instantMessage" => "misc", |
||||
"interview" => "misc", |
||||
"journalArticle" => "article", |
||||
"letter" => "letter", |
||||
"magazineArticle" => "article", |
||||
"manuscript" => "unpublished", |
||||
"map" => "misc", |
||||
"newspaperArticle" => "article", |
||||
"patent" => "patent", |
||||
"podcast" => "audio", |
||||
"presentation" => "unpublished", |
||||
"radioBroadcast" => "audio", |
||||
"report" => "report", |
||||
"statute" => "legislation", |
||||
"thesis" => "thesis", |
||||
"tvBroadcast" => "video", |
||||
"videoRecording" => "video", |
||||
"webpage" => "online" |
||||
) |
||||
# , subtype: "magazine"}, |
||||
# , subtype: "newspaper"}, |
||||
|
||||
|
||||
"Map zotero fields to bibtex fields" |
||||
const zotero_field_map = Dict( |
||||
"title" => "title", |
||||
"pages" => "pages", |
||||
"DOI" => "DOI", |
||||
"ISSN" => "ISSN", |
||||
"publisher" => "publisher", |
||||
"publicationTitle" => "journal", |
||||
"date" => "year", |
||||
"volume" => "volume", |
||||
"issue" => "number" |
||||
# FIXME: ADD TAGS? |
||||
) |
||||
|
||||
"Parses the `creatorType` field and creates a string with authors" |
||||
function creatortoauthor(c::Dict) |
||||
ctype = c["creatorType"] |
||||
if ctype != "author" |
||||
error("Add this creator type($(ctype)) to the supported ones!") |
||||
end |
||||
"""{$(c["lastName"]), $(c["firstName"])}""" |
||||
end |
||||
|
||||
"Takes all zotero creators, converts and join them" |
||||
authors(c) = join(creatortoauthor.(c), " and ") |
||||
|
||||
#= |
||||
Functions used to interface with the zotero connector |
||||
=# |
||||
|
||||
"""Creates the response by using the response body `res`, the default |
||||
`headers` and the headers to add (`extraheaders`). |
||||
|
||||
Defines the hook `set_headers` (before setting them) and |
||||
`post_headers_merge` (before sending the response) |
||||
""" |
||||
function setzoteroheaders(res, extraheaders::Dict) |
||||
@hook :set_headers |
||||
newheaders = merge(headers, extraheaders) |
||||
@hook :post_headers_merge (newheaders,) |
||||
Dict(:headers => newheaders, :body => res) |
||||
end |
||||
|
||||
""" |
||||
The zotero connector sometimes pings the program to see if we are listening (e.g., before sending the data). This must support both GET and POST. |
||||
|
||||
Defines the hook `ping` |
||||
""" |
||||
function pong(app) |
||||
@hook :ping |
||||
if app[:method] == "POST" |
||||
headers = Dict("Content-Type" => "application/json") |
||||
# FIXME: get(libraries[currentlibrary],"autosnapshot", false) |
||||
response = json(Dict("prefs" => |
||||
Dict("automaticSnapshots" => |
||||
false |
||||
))) |
||||
else |
||||
headers = Dict("Content-Type" => "text/html") |
||||
response = """ |
||||
<!DOCTYPE html><html><head> |
||||
<title>Zotero Connector Server is Available</title></head> |
||||
<body>Zotero Connector Server is Available</body></html>""" |
||||
end |
||||
|
||||
response, headers |
||||
end |
||||
|
||||
""" |
||||
|
||||
Defines the hook `get_collection` |
||||
""" |
||||
function collection(any, libraries, currentlibrary) |
||||
@hook :get_collection |
||||
libraryName = "developing" |
||||
|
||||
json(Dict( |
||||
"libraryID" => 1, |
||||
"libraryName" => "default", |
||||
"libraryEditable" => !libraries[currentlibrary]["readonly"], |
||||
"editable" => true, # collection-level parameters |
||||
"id" => 1, # collection-level |
||||
"name" => currentlibrary |
||||
)), Dict("Content-Type" => "application/json") |
||||
end |
||||
|
||||
""" |
||||
Save page snapshot. |
||||
|
||||
Defines the hooks `pre_save_snapshot` and `post_save_snapshot`. |
||||
""" |
||||
function savesnapshot(req, libraries, currentlibrary) |
||||
@hook :pre_save_snapshot |
||||
if libraries[currentlibrary]["readonly"] |
||||
warn("Library is readonly") |
||||
return "no", Dict() |
||||
end |
||||
|
||||
parsed = JSON.parse(String(req[:data])) |
||||
if !parsed["skipSnapshot"] |
||||
info("Saving page snapshot") |
||||
open("./devel/snapshot.html", "w") do f |
||||
write(f, parsed["html"]) |
||||
end |
||||
else |
||||
info("NOT saving page snapshot") |
||||
end |
||||
@hook :post_save_snapshot |
||||
"savesnapshot", Dict() |
||||
end |
||||
|
||||
""" |
||||
Download (asynchronously) all the attachments. |
||||
|
||||
Defines `pre_get_attachment` and `post_get_attachment` |
||||
""" |
||||
function getattachment(id, attachment) |
||||
@hook :pre_get_attachment |
||||
title = attachment["title"] |
||||
url = attachment["url"] |
||||
|
||||
mime = nothing |
||||
if "mimeType" in keys(attachment) |
||||
mime = MIME(attachment["mimeType"]) |
||||
end |
||||
|
||||
name = basename(URI(url).path) |
||||
name == "" && (name = title * ".html") |
||||
path = expanduser(libraries[currentlibrary]["path"]) |
||||
mkpath("$path/$id") |
||||
file = "$path/$id/$name" |
||||
try |
||||
# FIXME: we should tell zotero-connector which is a success and which isn't |
||||
@spawn HTTP.open("GET", url) do resource |
||||
open(file, "w") do file |
||||
write(file, resource) |
||||
end |
||||
@hook :post_get_attachment |
||||
end |
||||
end |
||||
file |
||||
end |
||||
|
||||
function saveitems(req::Dict, libraries, currentlibrary) |
||||
@hook :pre_receive |
||||
if libraries[currentlibrary]["readonly"] |
||||
warn("Library is readonly") |
||||
return "no", Dict{String,String}("status"=> string(403)) |
||||
end |
||||
|
||||
parsed = JSON.parse(String(req[:data])) |
||||
|
||||
foreach(i -> addtobibliography!(bibliography, i...), |
||||
parseitem.(parsed["items"])) |
||||
|
||||
@hook :after_receive |
||||
|
||||
"ok", Dict{String,String}("status"=> string(201)) |
||||
end |
||||
|
||||
|
||||
function parseitem(item::Dict) |
||||
tp = zotero_type_map[item["itemType"]] |
||||
data = BibTeX.Citation{Symbol(tp)}() |
||||
data["author"] = authors(item["creators"]) |
||||
|
||||
for k in keys(item) |
||||
if k in keys(zotero_field_map) |
||||
data[zotero_field_map[k]] = item[k] |
||||
end |
||||
end |
||||
|
||||
id = createcitekey(data) |
||||
filenames = getattachment.(id, item["attachments"]) |
||||
data["file"] = join(filenames, "; ") |
||||
try |
||||
(id,data) |
||||
catch |
||||
warn("Type "* item["itemType"] * " does not exists!") |
||||
(nothing, nothing) |
||||
end |
||||
end |
@ -1,5 +1,9 @@ |
||||
using Scjrm |
||||
using Scjrm: createlibraries |
||||
using Base.Test |
||||
|
||||
# write your own tests here |
||||
@test 1 == 2 |
||||
testlibs = Dict("test" => Dict("path" => "/tmp/examplefolderscjrm")) |
||||
isfile("/tmp/examplefolderscjrm") && rm("/tmp/examplefolderscjrm") |
||||
|
||||
createlibraries(testlibs) |
||||
@test isdir("/tmp/examplefolderscjrm") |
||||
isdir("/tmp/examplefolderscjrm") && rm("/tmp/examplefolderscjrm") |
||||
|
Loading…
Reference in new issue