From 719c3056e5999b94a916156f7ffc7a7e9239b6fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Balzarotti?= Date: Mon, 27 May 2019 10:09:06 +0200 Subject: [PATCH] Rescale album art to improve speed --- JlSonic/api.jl | 69 +++++++++++++++++++++++++++++++++++-------- JlSonic/beet2xml.jl | 9 +++++- consistency.jl | 34 +++++++++++++++++++++ get-missing-albums.jl | 47 +++++++++++++++++++++++++++++ router.jl | 16 +++++++++- server.jl | 25 ++++++++++++++-- 6 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 consistency.jl create mode 100644 get-missing-albums.jl diff --git a/JlSonic/api.jl b/JlSonic/api.jl index 3100612..824603d 100644 --- a/JlSonic/api.jl +++ b/JlSonic/api.jl @@ -197,7 +197,7 @@ function getArtist(id::String) # Create the response (xdoc, xroot) = subsonic() artistXML = push!(xroot, artist) - for album in Beets.album(artist) + for album in sort(Beets.album(artist)) push!(artistXML, album) end return subsonic_return(xdoc) @@ -307,15 +307,18 @@ function makequery(q::String) return Regex(nq) end -function search3(req; dl = println) +function search3(req; dlalbum = println, dlall = println) query = HTTP.URIs.queryparams(req[:query]) q = get(query, "query", "") isempty(q) && return missing_parameter("query") - if req[:user][:user].upload && length(q) > 2 && q[end-1:end] == "!t" - @info "This is the special torrent mode!" - list = dl(string(strip(q[1:end-2]))) - list == nothing && return @subsonic(nothing) - + if req[:login][:user].upload && length(q) > 2 && q[end-1:end] == "!a" + @info "Downloading single album" + dlalbum(string(strip(q[1:end-2]))) + return @subsonic(nothing) + elseif req[:login][:user].upload && length(q) > 2 && q[end-1:end] == "!d" + @info "Downloading discography!" + dlall(string(strip(q[1:end-2]))) + return @subsonic(nothing) (xdoc, xroot) = subsonic() results = new_child(xroot, "searchResult3") # TODO: Push the results so that we can see what download just started @@ -501,27 +504,67 @@ function getUser(req) return subsonic_return(xroot) end +const cover_cache = Dict{String, Vector{UInt8}}() +const song_cache = Dict{Tuple{String,Int,String}, Vector{UInt8}}() + # Media retriveal "Returns a cover art image." function getCoverArt(req::Dict) + global cover_cache + query = HTTP.URIs.queryparams(req[:query]) + id = get(query, "id", "") isempty(id) && return missing_parameter("id") n = findfirst(a -> a.uuid == id, Beets.albums) - n === nothing && return not_found("id") + + (n === nothing || isempty(Beets.albums[n].cover)) && return not_found("id") + + if Beets.albums[n].cover in keys(cover_cache) + data = cover_cache[Beets.albums[n].cover] + else + io = IOBuffer() + p = run(pipeline(`convert $(Beets.albums[n].cover) -resize 200x200 png:-`, + stderr=devnull, stdout=io), wait = true) + data = take!(io) + cover_cache[Beets.albums[n].cover] = data + end + headers = Dict{String,String}() - return sendfile(Beets.albums[n].cover) + headers = Dict{String,String}() + suffix = "png" + mime = suffix in keys(Mux.mimetypes) ? Mux.mimetypes[suffix] : "application/octet-stream" + headers["Content-Type"] = mime + headers["Content-Length"] = string(length(data)) + return Dict(:body => data, + :headers => headers, + :file => join([split(basename(Beets.albums[n].cover), '.')[1:end-1], ".png"],"")) end function giveconverted(file, bitrate, format) - iodata = convert(file, bitrate = bitrate, format = format) + global song_cache + k = (file, bitrate, format) + if k in keys(song_cache) + @info "Using cached" + data = song_cache[k] + else + @info "Adding song to cache" + iodata = convert(file, bitrate = bitrate, format = format) + data = take!(iodata) + try + song_cache[k] = data + catch e + @warn e + end + end + headers = Dict{String,String}() suffix = format mime = suffix in keys(Mux.mimetypes) ? Mux.mimetypes[suffix] : "application/octet-stream" headers["Content-Type"] = mime - data = take!(iodata) headers["Content-Length"] = string(length(data)) + # headers["Transfer-Encoding"] = "chunked" return Dict(:body => data, :headers => headers, :file => join([split(basename(file), '.')[1:end-1], @@ -532,13 +575,15 @@ function convert(infile; bitrate = 64, format = "oga") global ffmpeg_threads io = IOBuffer() p = run(pipeline(`ffmpeg -i $infile -y -c:a libvorbis -b:a $(bitrate)k -threads $(ffmpeg_threads) -f $format pipe:1`, - stderr=devnull, stdout=io), wait = true) + stderr=devnull, stdout=io), wait = true) io end canstream(u::User) = u.stream "Streams a given media file." function stream(req::Dict) + @show req + query = HTTP.URIs.queryparams(req[:query]) canstream(req[:login][:user]) || return unuthorized() diff --git a/JlSonic/beet2xml.jl b/JlSonic/beet2xml.jl index 5daadce..0eaa347 100644 --- a/JlSonic/beet2xml.jl +++ b/JlSonic/beet2xml.jl @@ -32,7 +32,13 @@ function append!(root::XMLElement, p::Playlist) for song in p.songs entry = new_child(playlistXML, "entry") set_attributes(entry, props(song)) - set_attribute(entry, "artist", Beets.artist(song).name) + try + artist = Beets.artist(song) + n = artist == nothing ? artist.name : "" + set_attribute(entry, "artist", "") + catch e + @warn e + end end playlistXML end @@ -56,6 +62,7 @@ end import Base.sort sort(ss::Vector{Beets.Song}) = sort(ss, by = x -> x.track) +sort(a::Vector{Beets.Album}) = sort(a, by = a -> a.year) function append!(root::XMLElement, a::Beets.Album) albumXML = push!(root, a) diff --git a/consistency.jl b/consistency.jl new file mode 100644 index 0000000..068f6be --- /dev/null +++ b/consistency.jl @@ -0,0 +1,34 @@ +using DataFrames +import FileIO + +"Check that all songs have the correct format" +function format_check(s::Vector{Beets.Song}; format = "FLAC") + wrong = filter(s -> s.format != format, s) + paths = [x.path for x in wrong] .|> dirname |> unique + (success = length(wrong) == 0, howmany = length(wrong), broken = wrong, paths = paths) +end + +"Check that all songs exists" +function existing_check(s::Vector{Beets.Song}) + paths = getfield.(s, :path) + ok = isfile.(paths) + (success = all(ok), howmany = count(.!ok), broken = s[.!ok], paths = paths[.!ok]) +end + +"Check that all songs are under the right path" +function path_check(s::Vector{Beets.Song}; path = "/mnt/music/") + paths = getfield.(s, :path) + ok = startswith.(paths, path) + (success = all(ok), howmany = count(.!ok), broken = s[.!ok], paths = paths[.!ok]) +end + +brokenartists(r) = map(x -> Beets.artist(x) , r.broken) |> unique +brokenalbums(r) = map(x -> Beets.album(x) , r.broken) |> unique + +l() = map(a -> (artist = a.artist.name, title = a.title), brokenalbums(format_check(Beets.songs()))) +m() = map(a -> (artist = a.artist.name, title = a.title), brokenalbums(existing_check(Beets.songs()))) +n() = map(a -> (artist = a.artist.name, title = a.title), brokenalbums(path_check(Beets.songs()))) + +l() |> DataFrame |> d -> FileIO.save("format.csv", d) +m() |> DataFrame |> d -> FileIO.save("missing.csv", d) +n() |> DataFrame |> d -> FileIO.save("wrong_path.csv", d) diff --git a/get-missing-albums.jl b/get-missing-albums.jl new file mode 100644 index 0000000..3fd7e1c --- /dev/null +++ b/get-missing-albums.jl @@ -0,0 +1,47 @@ +artists = Beets.artists() +albums_id = getfield.(Beets.albums, :uuid) +albums_title = getfield.(Beets.albums, :title) .|> lowercase + +owned_ids = getfield.(artists, :uuid) + +counter = 0 +releases = map(id -> begin + global counter + sleep(0.1) + @show counter += 1 + MusicBrainz.releaselistbyid(id) + end, owned_ids) + + + +alldata = map((a, rs) -> [(artist = a, + album = r.title, uuid = r.id) + for r in rs], + artists, releases) |> Iterators.flatten |> collect + +todl = filter(x -> !(x.uuid in albums_id || + x.album in lowercase.(getfield.(Beets.album(x.artist), :title)) + ), alldata) + +found = [] +foreach(x -> begin + global me, found + sleep(0.1) + push!(found, + RuTrackers.search(me, string(x.artist.name, " ", x.album), + verbose = true)) + end, todl) + +# added = [] +for (n, r) in enumerate(found) + global me, rpc, added + @info "($(n)/$(length(found)))" + lossless = RuTrackers.islossless.(r) + discog = RuTrackers.isdiscography.(r) + m = findfirst(lossless .& .!discog) + m === nothing && continue + # Add it + TransmissionRPC.getauth(rpc) + TransmissionRPC.add(rpc, RuTrackers.download(me, r[m])) + # push!(added, found[n]) +end diff --git a/router.jl b/router.jl index 0b22422..2313e6d 100644 --- a/router.jl +++ b/router.jl @@ -14,6 +14,18 @@ function torrentdl(query::AbstractString) todl end +function albumdl(query::AbstractString) + global rpc, me + TransmissionRPC.getauth(rpc) + todl = RuTrackers.search(me, query) + @show todl + lossless = RuTrackers.islossless.(todl) + discog = RuTrackers.isdiscography.(todl) + m = findfirst(lossless .& .!discog) + m === nothing && return + TransmissionRPC.add(rpc, RuTrackers.download(me, todl[m])) +end + dispatch = stack( # Browsing restp("getMusicFolders", _ -> getMusicFolders()), @@ -27,7 +39,9 @@ dispatch = stack( # Album/song list restp("getRandomSongs", req -> getRandomSongs(req)), # Searching - restp("search3", req -> search3(req; dl = torrentdl)), + restp("search3", req -> search3(req; + dlalbum = albumdl, + dlall = torrentdl)), # Playlists restp("createPlaylist", req -> createPlaylist(req)), restp("getPlaylists", req -> getPlaylists(req)), diff --git a/server.jl b/server.jl index 278236a..81637be 100644 --- a/server.jl +++ b/server.jl @@ -15,7 +15,7 @@ me = RuTrackers.RuTracker(read("rutracker.json", String) |> JSON.parse) rpc = TransmissionRPC.Transmission(TransmissionRPC.Sockets.ip"192.168.1.3") -retry(Beets.update_albums, delays = Base.ExponentialBackOff(n=10, first_delay=5, max_delay = 100)); +Beets.update_albums() push!(LOAD_PATH, realpath("JlSonic")) using JlSonic @@ -25,9 +25,20 @@ JlSonic.loadusers() include("router.jl") include("login.jl") using Dates + +if isdefined(Main, :logfile) && isopen(logfile) + close(logfile) +end + +logfile = open("requests.log", "a") function logger(app, req) - println(string("[", Dates.now(), "] ", req[:method], ": ", req[:path][end])) -#, " - ", req[:headers]["User-Agent"])) + global logfile + if isopen(logfile) + logfile = open("requests.log", "a") + end + write(logfile, string(req, "\n")) + println(string("[", Dates.now(), "] ", req[:method], ": ", req[:path][end])) + #, " - ", req[:headers]["User-Agent"])) return app(req) end @@ -42,6 +53,7 @@ end defaults = stack(Mux.todict, basiccatch, Mux.splitquery, Mux.toresponse, Mux.assetserver, Mux.pkgfiles) @app sonic = ( Mux.defaults, + # logger, restp("ping", _ -> ping()), restp("getLicense", _ -> getLicense()), mux(logger, @@ -54,3 +66,10 @@ if !isdefined(Main, :started) serve(sonic) started = true end + +function nextbatch() + global rpc + ids = TransmissionRPC.getCompleteMusicIDs(rpc) + TransmissionRPC.moveMusicDone(rpc, ids) + TransmissionRPC.rm(rpc, ids) +end