Rescale album art to improve speed

JlSonic/api.jl

@@ -197,7 +197,7 @@ function getArtist(id::String)
     # Create the response
     (xdoc, xroot) = subsonic()
     artistXML = push!(xroot, artist)
-    for album in Beets.album(artist)
+    for album in sort(Beets.album(artist))
         push!(artistXML, album)
     end
     return subsonic_return(xdoc)
@@ -307,15 +307,18 @@ function makequery(q::String)
     return Regex(nq)
 end
 
-function search3(req; dl = println)
+function search3(req; dlalbum = println, dlall = println)
     query = HTTP.URIs.queryparams(req[:query])
     q = get(query, "query", "")
     isempty(q) && return missing_parameter("query")
-    if req[:user][:user].upload && length(q) > 2 && q[end-1:end] == "!t"
-        @info "This is the special torrent mode!"
-        list = dl(string(strip(q[1:end-2])))
-        list == nothing && return @subsonic(nothing)
-        
+    if req[:login][:user].upload && length(q) > 2 && q[end-1:end] == "!a"
+        @info "Downloading single album"
+        dlalbum(string(strip(q[1:end-2])))
+        return @subsonic(nothing)
+    elseif req[:login][:user].upload && length(q) > 2 && q[end-1:end] == "!d"
+        @info "Downloading discography!"
+        dlall(string(strip(q[1:end-2])))
+        return @subsonic(nothing)
         (xdoc, xroot) = subsonic()
         results = new_child(xroot, "searchResult3")
         # TODO: Push the results so that we can see what download just started
@@ -501,27 +504,67 @@ function getUser(req)
     return subsonic_return(xroot)
 end
 
+const cover_cache = Dict{String, Vector{UInt8}}()
+const song_cache = Dict{Tuple{String,Int,String}, Vector{UInt8}}()
+
 # Media retriveal
 "Returns a cover art image."
 function getCoverArt(req::Dict)
+    global cover_cache
+    
     query = HTTP.URIs.queryparams(req[:query])
+
     id = get(query, "id", "")
     isempty(id) && return missing_parameter("id")
     n = findfirst(a -> a.uuid == id, Beets.albums)
-    n === nothing && return not_found("id")
+
+    (n === nothing || isempty(Beets.albums[n].cover)) && return not_found("id")
+
+    if Beets.albums[n].cover in keys(cover_cache)
+        data = cover_cache[Beets.albums[n].cover]
+    else
+        io = IOBuffer()
+        p = run(pipeline(`convert $(Beets.albums[n].cover) -resize 200x200 png:-`,
+                         stderr=devnull, stdout=io), wait = true)
+        data = take!(io)
+        cover_cache[Beets.albums[n].cover] = data
+    end
+    
     headers = Dict{String,String}()
-    return sendfile(Beets.albums[n].cover)
+    headers = Dict{String,String}()
+    suffix = "png"
+    mime = suffix in keys(Mux.mimetypes) ? Mux.mimetypes[suffix] : "application/octet-stream"
+    headers["Content-Type"] = mime
+    headers["Content-Length"] = string(length(data))
+    return Dict(:body => data,
+                :headers => headers,
+                :file => join([split(basename(Beets.albums[n].cover), '.')[1:end-1], ".png"],""))
 end
 
 function giveconverted(file, bitrate, format)
-    iodata = convert(file, bitrate = bitrate, format = format)
+    global song_cache
+    k = (file, bitrate, format)
+    if k in keys(song_cache)
+        @info "Using cached"
+        data = song_cache[k]
+    else
+        @info "Adding song to cache"
+        iodata = convert(file, bitrate = bitrate, format = format)
+        data = take!(iodata)
+        try
+            song_cache[k] = data
+        catch e
+            @warn e
+        end
+    end
+
     headers = Dict{String,String}()
     suffix = format
     mime = suffix in keys(Mux.mimetypes) ? Mux.mimetypes[suffix] :
         "application/octet-stream"
     headers["Content-Type"] = mime
-    data = take!(iodata)
     headers["Content-Length"] = string(length(data))
+    # headers["Transfer-Encoding"] = "chunked"
     return Dict(:body => data,
                 :headers => headers,
                 :file => join([split(basename(file), '.')[1:end-1],
@@ -532,13 +575,15 @@ function convert(infile; bitrate = 64, format = "oga")
     global ffmpeg_threads
     io = IOBuffer()
     p = run(pipeline(`ffmpeg -i $infile -y -c:a libvorbis -b:a $(bitrate)k -threads $(ffmpeg_threads) -f $format pipe:1`,
-                 stderr=devnull, stdout=io), wait = true)
+                     stderr=devnull, stdout=io), wait = true)
     io
 end
 
 canstream(u::User) = u.stream
 "Streams a given media file."
 function stream(req::Dict)
+    @show req
+
     query = HTTP.URIs.queryparams(req[:query])
     canstream(req[:login][:user]) || return unuthorized()
 

JlSonic/beet2xml.jl

@@ -32,7 +32,13 @@ function append!(root::XMLElement, p::Playlist)
     for song in p.songs
         entry = new_child(playlistXML, "entry")
         set_attributes(entry, props(song))
-        set_attribute(entry, "artist", Beets.artist(song).name)
+        try
+            artist = Beets.artist(song)
+            n = artist == nothing ? artist.name : ""
+            set_attribute(entry, "artist", "")
+        catch e
+            @warn e
+        end
     end
     playlistXML
 end
@@ -56,6 +62,7 @@ end
 
 import Base.sort
 sort(ss::Vector{Beets.Song}) = sort(ss, by = x -> x.track)
+sort(a::Vector{Beets.Album}) = sort(a, by = a -> a.year)
 
 function append!(root::XMLElement, a::Beets.Album)
     albumXML = push!(root, a)

consistency.jl

@@ -0,0 +1,34 @@
+using DataFrames
+import FileIO
+
+"Check that all songs have the correct format"
+function format_check(s::Vector{Beets.Song}; format = "FLAC")
+   wrong = filter(s -> s.format != format, s)
+   paths = [x.path for x in wrong] .|> dirname |> unique
+   (success = length(wrong) == 0, howmany = length(wrong), broken = wrong, paths = paths)
+end
+
+"Check that all songs exists"
+function existing_check(s::Vector{Beets.Song})
+  paths = getfield.(s, :path)
+  ok = isfile.(paths)
+  (success = all(ok), howmany = count(.!ok), broken = s[.!ok], paths = paths[.!ok])
+end
+
+"Check that all songs are under the right path"
+function path_check(s::Vector{Beets.Song}; path = "/mnt/music/")
+  paths = getfield.(s, :path)
+  ok = startswith.(paths, path)
+  (success = all(ok), howmany = count(.!ok), broken = s[.!ok], paths = paths[.!ok])
+end
+
+brokenartists(r) = map(x -> Beets.artist(x) , r.broken) |> unique
+brokenalbums(r) = map(x -> Beets.album(x) , r.broken) |> unique
+
+l() = map(a -> (artist = a.artist.name, title = a.title), brokenalbums(format_check(Beets.songs())))
+m() = map(a -> (artist = a.artist.name, title = a.title), brokenalbums(existing_check(Beets.songs())))
+n() = map(a -> (artist = a.artist.name, title = a.title), brokenalbums(path_check(Beets.songs())))
+
+l() |> DataFrame |> d -> FileIO.save("format.csv", d)
+m() |> DataFrame |> d -> FileIO.save("missing.csv", d)
+n() |> DataFrame |> d -> FileIO.save("wrong_path.csv", d)

get-missing-albums.jl

@@ -0,0 +1,47 @@
+artists = Beets.artists()
+albums_id = getfield.(Beets.albums, :uuid)
+albums_title = getfield.(Beets.albums, :title) .|> lowercase
+
+owned_ids = getfield.(artists, :uuid)
+
+counter = 0
+releases = map(id -> begin
+               global counter
+               sleep(0.1)
+               @show counter += 1
+               MusicBrainz.releaselistbyid(id)
+               end, owned_ids)
+
+
+
+alldata = map((a, rs) -> [(artist = a,
+                           album = r.title, uuid = r.id)
+                          for r in rs],
+              artists, releases) |> Iterators.flatten |> collect
+
+todl = filter(x -> !(x.uuid in albums_id ||
+                     x.album in lowercase.(getfield.(Beets.album(x.artist), :title))
+                     ), alldata)
+
+found = []
+foreach(x -> begin
+        global me, found
+        sleep(0.1)
+        push!(found,
+              RuTrackers.search(me, string(x.artist.name, " ", x.album),
+                                verbose = true))
+        end, todl)
+
+# added = []
+for (n, r) in enumerate(found)
+    global me, rpc, added
+    @info "($(n)/$(length(found)))"
+    lossless = RuTrackers.islossless.(r)
+    discog = RuTrackers.isdiscography.(r)
+    m = findfirst(lossless .& .!discog)
+    m === nothing && continue
+    # Add it
+    TransmissionRPC.getauth(rpc)
+    TransmissionRPC.add(rpc, RuTrackers.download(me, r[m]))
+    # push!(added, found[n])
+end

router.jl

@@ -14,6 +14,18 @@ function torrentdl(query::AbstractString)
     todl
 end
 
+function albumdl(query::AbstractString)
+    global rpc, me
+    TransmissionRPC.getauth(rpc)
+    todl = RuTrackers.search(me, query)
+    @show todl
+    lossless = RuTrackers.islossless.(todl)
+    discog = RuTrackers.isdiscography.(todl)
+    m = findfirst(lossless .& .!discog)
+    m === nothing && return
+    TransmissionRPC.add(rpc, RuTrackers.download(me, todl[m]))
+end
+
 dispatch = stack(
     # Browsing
     restp("getMusicFolders", _ -> getMusicFolders()),
@@ -27,7 +39,9 @@ dispatch = stack(
     # Album/song list
     restp("getRandomSongs", req -> getRandomSongs(req)),
     # Searching
-    restp("search3", req -> search3(req; dl = torrentdl)),
+    restp("search3", req -> search3(req;
+                                    dlalbum = albumdl,
+                                    dlall = torrentdl)),
     # Playlists
     restp("createPlaylist", req -> createPlaylist(req)),
     restp("getPlaylists", req -> getPlaylists(req)),

server.jl

@@ -15,7 +15,7 @@ me = RuTrackers.RuTracker(read("rutracker.json", String) |> JSON.parse)
 rpc = TransmissionRPC.Transmission(TransmissionRPC.Sockets.ip"192.168.1.3")
 
 
-retry(Beets.update_albums, delays = Base.ExponentialBackOff(n=10, first_delay=5, max_delay = 100));
+Beets.update_albums()
 
 push!(LOAD_PATH, realpath("JlSonic"))
 using JlSonic
@@ -25,9 +25,20 @@ JlSonic.loadusers()
 include("router.jl")
 include("login.jl")
 using Dates
+
+if isdefined(Main, :logfile) && isopen(logfile)
+    close(logfile)    
+end
+
+logfile = open("requests.log", "a")
 function logger(app, req)
-   println(string("[", Dates.now(), "] ", req[:method], ": ", req[:path][end]))
-#, " - ", req[:headers]["User-Agent"]))
+    global logfile
+    if isopen(logfile)
+        logfile = open("requests.log", "a")
+    end
+    write(logfile, string(req, "\n"))
+    println(string("[", Dates.now(), "] ", req[:method], ": ", req[:path][end]))
+    #, " - ", req[:headers]["User-Agent"]))
    return app(req)
 end
 
@@ -42,6 +53,7 @@ end
 defaults = stack(Mux.todict, basiccatch, Mux.splitquery, Mux.toresponse, Mux.assetserver, Mux.pkgfiles)
 @app sonic = (
     Mux.defaults,
+    # logger,
     restp("ping", _ -> ping()),
     restp("getLicense", _ -> getLicense()),
     mux(logger, 
@@ -54,3 +66,10 @@ if !isdefined(Main, :started)
     serve(sonic)
     started = true
 end
+
+function nextbatch()
+    global rpc
+    ids = TransmissionRPC.getCompleteMusicIDs(rpc)
+    TransmissionRPC.moveMusicDone(rpc, ids)
+    TransmissionRPC.rm(rpc, ids)
+end