From d94bbd44cc43f0e1122fc52de06da1542b46a2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Balzarotti?= Date: Mon, 17 Sep 2018 12:59:48 +0200 Subject: [PATCH] MatrixChat: first version with basic functions working --- Manifest.toml | 158 +++++++++ Project.toml | 14 + src/MatrixChat.jl | 36 ++ src/repl.jl | 852 ++++++++++++++++++++++++++++++++++++++++++++++ src/rooms.jl | 66 ++++ src/sync.jl | 68 ++++ src/types.jl | 57 ++++ src/user.jl | 42 +++ src/users.jl | 40 +++ src/utils.jl | 91 +++++ 10 files changed, 1424 insertions(+) create mode 100644 Manifest.toml create mode 100644 Project.toml create mode 100644 src/MatrixChat.jl create mode 100644 src/repl.jl create mode 100644 src/rooms.jl create mode 100644 src/sync.jl create mode 100644 src/types.jl create mode 100644 src/user.jl create mode 100644 src/users.jl create mode 100644 src/utils.jl diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..15582a2 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,158 @@ +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[BinDeps]] +deps = ["Compat", "Libdl", "SHA", "URIParser"] +git-tree-sha1 = "12093ca6cdd0ee547c39b1870e0c9c3f154d9ca9" +uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee" +version = "0.8.10" + +[[BinaryProvider]] +deps = ["Libdl", "Pkg", "SHA", "Test"] +git-tree-sha1 = "b530fbeb6f41ab5a83fbe3db1fcbe879334bcd2d" +uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +version = "0.4.2" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "ae262fa91da6a74e8937add6b613f58cd56cdad4" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "1.1.0" + +[[Crayons]] +deps = ["Pkg", "Test"] +git-tree-sha1 = "3017c662a988bcb8a3f43306a793617c6524d476" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "1.0.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[Distributed]] +deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[HTTP]] +deps = ["Base64", "Dates", "Distributed", "IniFile", "MbedTLS", "Sockets", "Test"] +git-tree-sha1 = "8a0f75e8b09df01d9f1ba9ad3fbf8b4983595d20" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "0.6.14" + +[[IniFile]] +deps = ["Test"] +git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" +uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" +version = "0.5.0" + +[[InteractiveUtils]] +deps = ["LinearAlgebra", "Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[JSON]] +deps = ["Dates", "Distributed", "Mmap", "Pkg", "Sockets", "Test", "Unicode"] +git-tree-sha1 = "fec8e4d433072731466d37ed0061b3ba7f70eeb9" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.19.0" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS]] +deps = ["BinaryProvider", "Compat", "Libdl", "Pkg", "Sockets"] +git-tree-sha1 = "17d5a81dbb1e682d4ff707c01f0afe5948068fa6" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "0.6.0" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[Nettle]] +deps = ["BinDeps", "BinaryProvider", "Libdl", "Pkg", "Test"] +git-tree-sha1 = "f57e8e907faab4f55f9f164313a633509ac83e2c" +uuid = "49dea1ee-f6fa-5aa6-9a11-8816cee7d4b9" +version = "0.4.0" + +[[OhMyREPL]] +deps = ["Crayons", "InteractiveUtils", "Markdown", "Pkg", "Printf", "REPL", "Test", "Tokenize"] +git-tree-sha1 = "e00d5394d110afe279101ffe10cebd11eaedcb8a" +uuid = "5fb14364-9ced-5910-84b2-373655c76a03" +version = "0.3.0" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[Tokenize]] +deps = ["Printf", "Test"] +git-tree-sha1 = "4a9fefb5c5c831c6fc06fcc5d2ae7399918bb587" +uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" +version = "0.5.2" + +[[URIParser]] +deps = ["Test", "Unicode"] +git-tree-sha1 = "6ddf8244220dfda2f17539fa8c9de20d6c575b69" +uuid = "30578b45-9adc-5946-b283-645ec420af67" +version = "0.4.0" + +[[UUIDs]] +deps = ["Random"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..762ab32 --- /dev/null +++ b/Project.toml @@ -0,0 +1,14 @@ +name = "MatrixChat" +uuid = "2ff08b00-b8ed-11e8-2556-a18c63bdb31c" +authors = ["Nicolò Balzarotti "] +version = "0.1.0" + +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +Nettle = "49dea1ee-f6fa-5aa6-9a11-8816cee7d4b9" +OhMyREPL = "5fb14364-9ced-5910-84b2-373655c76a03" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +URIParser = "30578b45-9adc-5946-b283-645ec420af67" diff --git a/src/MatrixChat.jl b/src/MatrixChat.jl new file mode 100644 index 0000000..13ea00e --- /dev/null +++ b/src/MatrixChat.jl @@ -0,0 +1,36 @@ +module MatrixChat + +using OhMyREPL +using HTTP +using JSON +using Nettle +using Dates + +include("types.jl") +include("users.jl") +include("rooms.jl") +include("sync.jl") +include("utils.jl") +include("user.jl") +include("repl.jl") + +# types +export MatrixServer, MatrixUser, MatrixSync, MatrixMsg + +# functions +export get, post, put, upload, download, + # administration + new_user, + # rooms + listjoined, createroom, invite, getroomevent, getroomname, + joinroom, checkifjoin, history, + roomhistory, updateroomlastbatch, + send, setavatar, sync, sync!, + # users + getdisplayname, + # sync + background_sync, stop_sync!, set_hook!, remove_hook! + +export testrepl + +end # module diff --git a/src/repl.jl b/src/repl.jl new file mode 100644 index 0000000..e025bf2 --- /dev/null +++ b/src/repl.jl @@ -0,0 +1,852 @@ +# include("repl.jl") +import REPL +import REPL: LineEdit, REPLCompletions +using Markdown + +########### +# Options # +########### +struct OptionSpec + name::String + short_name::Union{Nothing,String} + api::Pair{Symbol, Any} + is_switch::Bool +end + +@enum(OptionClass, OPT_ARG, OPT_SWITCH) +const OptionDeclaration = Tuple{Union{String,Vector{String}}, # name + short_name? + OptionClass, # arg or switch + Pair{Symbol, Any} # api keywords + } + +function OptionSpec(x::OptionDeclaration)::OptionSpec + get_names(name::String) = (name, nothing) + function get_names(names::Vector{String}) + @assert length(names) == 2 + return (names[1], names[2]) + end + + is_switch = x[2] == OPT_SWITCH + api = x[3] + (name, short_name) = get_names(x[1]) + #TODO assert matching lex regex + if !is_switch + @assert api.second === nothing || hasmethod(api.second, Tuple{String}) + end + return OptionSpec(name, short_name, api, is_switch) +end + +function OptionSpecs(decs::Vector{OptionDeclaration})::Dict{String, OptionSpec} + specs = Dict() + for x in decs + opt_spec = OptionSpec(x) + @assert get(specs, opt_spec.name, nothing) === nothing # don't overwrite + specs[opt_spec.name] = opt_spec + if opt_spec.short_name !== nothing + @assert get(specs, opt_spec.short_name, nothing) === nothing # don't overwrite + specs[opt_spec.short_name] = opt_spec + end + end + return specs +end + +struct Option + val::String + argument::Union{Nothing,String} +end +Base.show(io::IO, opt::Option) = print(io, "--$(opt.val)", opt.argument == nothing ? "" : "=$(opt.argument)") + +function parse_option(word::AbstractString)::Option + m = match(r"^(?: -([a-z]) | --([a-z]{2,})(?:\s*=\s*(\S*))? )$"ix, word) + m == nothing && matrixerror("malformed option: ", repr(word)) + option_name = (m.captures[1] != nothing ? m.captures[1] : m.captures[2]) + option_arg = (m.captures[3] == nothing ? nothing : String(m.captures[3])) + return Option(option_name, option_arg) +end + +meta_option_declarations = OptionDeclaration[ + ("preview", OPT_SWITCH, :preview => true) +] +meta_option_specs = OptionSpecs(meta_option_declarations) + +################ +# Command Spec # +################ +@enum(CommandKind, CMD_HELP, CMD_MSG, CMD_DIALOG_LIST, CMD_HISTORY, CMD_LOAD_GROUP_PHOTO) +@enum(ArgClass, ARG_RAW, ARG_PKG, ARG_VERSION, ARG_REV, ARG_ALL) +struct ArgSpec + count::Pair + parser::Function + parser_keys::Vector{Pair{Symbol, Any}} +end +const CommandDeclaration = Tuple{CommandKind, + Vector{String}, # names + Union{Nothing,Function}, # handler + Tuple{Pair, # count + Function, # parser + Vector{Pair{Symbol, Any}}, # parser keys + }, # arguments + Vector{OptionDeclaration}, # options + String, #description + Union{Nothing}, #help + } +struct CommandSpec + kind::CommandKind + canonical_name::String + short_name::Union{Nothing,String} + handler::Union{Nothing,Function} + argument_spec::ArgSpec + option_specs::Dict{String, OptionSpec} + description::String + help::Union{Nothing} +end +command_specs = Dict{String,CommandSpec}() # TODO remove this ? + +function SuperSpecs(foo)::Dict{String,Dict{String,CommandSpec}} + super_specs = Dict() + for x in foo + sub_specs = CommandSpecs(x.second) + for name in x.first + @assert get(super_specs, name, nothing) === nothing + super_specs[name] = sub_specs + end + end + return super_specs +end + +# populate a dictionary: command_name -> command_spec +function CommandSpecs(declarations::Vector{CommandDeclaration})::Dict{String,CommandSpec} + specs = Dict() + for dec in declarations + names = dec[2] + spec = CommandSpec(dec[1], + names[1], + length(names) == 2 ? names[2] : nothing, + dec[3], + ArgSpec(dec[4]...), + OptionSpecs(dec[5]), + dec[6], + dec[end]) + for name in names + # TODO regex check name + @assert get(specs, name, nothing) === nothing + specs[name] = spec + end + end + return specs +end + +################### +# Package parsing # +################### +let uuid = raw"(?i)[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}(?-i)", + name = raw"(\w+)(?:\.jl)?" + global const name_re = Regex("^$name\$") + global const uuid_re = Regex("^$uuid\$") + global const name_uuid_re = Regex("^$name\\s*=\\s*($uuid)\$") +end + +# packages can be identified through: uuid, name, or name+uuid +# additionally valid for add/develop are: local path, url +function parse_package(word::AbstractString; add_or_develop=false)::PackageSpec + if add_or_develop && casesensitive_isdir(expanduser(word)) + return PackageSpec(Types.GitRepo(expanduser(word))) + elseif occursin(uuid_re, word) + return PackageSpec(UUID(word)) + elseif occursin(name_re, word) + return PackageSpec(String(match(name_re, word).captures[1])) + elseif occursin(name_uuid_re, word) + m = match(name_uuid_re, word) + return PackageSpec(String(m.captures[1]), UUID(m.captures[2])) + elseif add_or_develop + # Guess it is a url then + return PackageSpec(Types.GitRepo(word)) + else + matrixerror("`$word` cannot be parsed as a package") + end +end + +################ +# REPL parsing # +################ +mutable struct Statement + command::Union{Nothing,CommandSpec} + options::Vector{String} + arguments::Vector{String} + meta_options::Vector{String} + Statement() = new(nothing, [], [], []) +end + +struct QuotedWord + word::String + isquoted::Bool +end + +function unwrap_option(option::String) + if startswith(option, "--") + return length(option) == 2 ? "" : option[3:end] + elseif length(option) == 2 + return option[end] + end +end + +wrap_option(option::String) = + length(option) == 1 ? "-$option" : "--$option" + +function _statement(words) + is_option(word) = first(word) == '-' + + word = popfirst!(words) + # meta options + while is_option(word) + if isempty(words) + if unwrap_option(word) in keys(meta_option_specs) + return :cmd, "", nothing, true + else + return :meta, word, nothing, true + end + end + word = popfirst!(words) + end + # command + if word in keys(super_specs) # have a super command + super_name = word + super = super_specs[word] + if isempty(words) + return :sub, "", super_name, true + end + word = popfirst!(words) + command = get(super, word, nothing) + if command === nothing + if isempty(words) + return :sub, word, super_name, true + else + return nothing + end + end + elseif get(super_specs["matrix"], word, nothing) !== nothing # given a "matrix" command + command = get(super_specs["matrix"], word, nothing) + elseif isempty(words) # try to complete the super command + return :cmd, word, nothing, true + else + return nothing + end + if isempty(words) + return :arg, "", command, true + end + word = words[end] + manifest = any(x->x in ["--manifest", "-m"], filter(is_option, words)) + return is_option(word) ? + (:opt, word, command, true) : + (:arg, word, command, !manifest) +end + +function parse(cmd::String; for_completions=false) + # replace new lines with ; to support multiline commands + cmd = replace(replace(cmd, "\r\n" => "; "), "\n" => "; ") + # tokenize accoring to whitespace / quotes + # WIP + qwords = parse_quotes(cmd) + # tokenzie unquoted tokens according to matrix REPL syntax + words = lex(qwords) + # break up words according to ";"(doing this early makes subsequent processing easier) + word_groups = group_words(words) + # create statements + if for_completions + return _statement(word_groups[end]) + end + return map(Statement, word_groups) +end + +# vector of words -> structured statement +# minimal checking is done in this phase +function Statement(words)::Statement + is_option(word) = first(word) == '-' + statement = Statement() + + word = popfirst!(words) + # meta options + while is_option(word) + push!(statement.meta_options, word) + isempty(words) && matrixerror("no command specified") + word = popfirst!(words) + end + # command + # special handling for `preview`, just convert it to a meta option under the hood + if word == "preview" + if !("--preview" in statement.meta_options) + push!(statement.meta_options, "--preview") + end + isempty(words) && matrixerror("preview requires a command") + word = popfirst!(words) + end + if word in keys(super_specs) + super = super_specs[word] + isempty(words) && matrixerror("no subcommand specified") + word = popfirst!(words) + else + super = super_specs["matrix"] + end + command = get(super, word, nothing) + command !== nothing || matrixerror("expected command. instead got [$word]") + statement.command = command + # command arguments + for word in words + push!((is_option(word) ? statement.options : statement.arguments), word) + end + return statement +end + +# break up words according to `;`(doing this early makes subsequent processing easier) +# the final group does not require a trailing `;` +function group_words(words)::Vector{Vector{String}} + statements = Vector{String}[] + x = String[] + for word in words + if word == ";" + isempty(x) ? matrixerror("empty statement") : push!(statements, x) + x = String[] + else + push!(x, word) + end + end + isempty(x) || push!(statements, x) + return statements +end + +const lex_re = r"^[\?\./\+\-](?!\-) | ((git|ssh|http(s)?)|(git@[\w\-\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)? | [^@\#\s;]+\s*=\s*[^@\#\s;]+ | \#\s*[^@\#\s;]* | @\s*[^@\#\s;]* | [^@\#\s;]+|;"x + +function lex(qwords::Vector{QuotedWord})::Vector{String} + words = String[] + for qword in qwords + if qword.isquoted + push!(words, qword.word) + else + append!(words, map(m->m.match, eachmatch(lex_re, qword.word))) + end + end + return words +end + +function parse_quotes(cmd::String)::Vector{QuotedWord} + in_doublequote = false + in_singlequote = false + qwords = QuotedWord[] + token_in_progress = Char[] + + push_token!(is_quoted) = begin + push!(qwords, QuotedWord(String(token_in_progress), is_quoted)) + empty!(token_in_progress) + end + + for c in cmd + if c == '"' + if in_singlequote # raw char + push!(token_in_progress, c) + else # delimiter + in_doublequote ? push_token!(true) : push_token!(false) + in_doublequote = !in_doublequote + end + elseif c == '\'' + if in_doublequote # raw char + push!(token_in_progress, c) + else # delimiter + in_singlequote ? push_token!(true) : push_token!(false) + in_singlequote = !in_singlequote + end + else + push!(token_in_progress, c) + end + end + if (in_doublequote || in_singlequote) + matrixerror("unterminated quote") + else + push_token!(false) + end + # to avoid complexity in the main loop, empty tokens are allowed above and + # filtered out before returning + return filter(x->!isempty(x.word), qwords) +end + +############## +# MatrixCommand # +############## +function APIOptions(options::Vector{Option}, + specs::Dict{String, OptionSpec}, + )::Dict{Symbol, Any} + keyword_vec = map(options) do opt + spec = specs[opt.val] + # opt is switch + spec.is_switch && return spec.api + # no opt wrapper -> just use raw argument + spec.api.second === nothing && return spec.api.first => opt.argument + # given opt wrapper + return spec.api.first => spec.api.second(opt.argument) + end + return Dict(keyword_vec) +end + +# Only for PkgSpec +function word2token(word::AbstractString)::Token + if first(word) == '@' + return VersionRange(word[2:end]) + elseif first(word) == '#' + return Rev(word[2:end]) + else + return String(word) + end +end + +function parse_matrix(raw_args::Vector{String}; valid=[], add_or_dev=false) + + args::Vector{PkgToken} = map(word2token, raw_args) + enforce_argument_order(args) + # enforce spec + push!(valid, String) # always want at least PkgSpec identifiers + if !all(x->typeof(x) in valid, args) + matrixerror("invalid token") + end + # convert to final arguments + return package_args(args; add_or_dev=add_or_dev) +end + +function enforce_argument(raw_args::Vector{String}, spec::ArgSpec) + args = spec.parser(raw_args; spec.parser_keys...) + # enforce_argument_count(spec.count, args) + return args +end + +function enforce_option(option::String, specs::Dict{String,OptionSpec})::Option + opt = parse_option(option) + spec = get(specs, opt.val, nothing) + spec !== nothing || + matrixerror("option '$(opt.val)' is not a valid option") + if spec.is_switch + opt.argument === nothing || + matrixerror("option '$(opt.val)' does not take an argument, but '$(opt.argument)' given") + else # option takes an argument + opt.argument !== nothing || + matrixerror("option '$(opt.val)' expects an argument, but no argument given") + end + return opt +end + +function enforce_meta_options(options::Vector{String}, specs::Dict{String,OptionSpec})::Vector{Option} + meta_opt_names = keys(specs) + return map(options) do opt + tok = enforce_option(opt, specs) + tok.val in meta_opt_names || + matrixerror("option '$opt' is not a valid meta option.") + #TODO hint that maybe they intended to use it as a command option + return tok + end +end + +function enforce_opts(options::Vector{String}, specs::Dict{String,OptionSpec})::Vector{Option} + unique_keys = Symbol[] + get_key(opt::Option) = specs[opt.val].api.first + + # final parsing + toks = map(x->enforce_option(x,specs),options) + # checking + for opt in toks + # valid option + opt.val in keys(specs) || + matrixerror("option '$(opt.val)' is not supported") + # conflicting options + key = get_key(opt) + if key in unique_keys + conflicting = filter(opt->get_key(opt) == key, toks) + matrixerror("Conflicting options: $conflicting") + else + push!(unique_keys, key) + end + end + return toks +end + +struct MatrixCommand + spec::CommandSpec + arguments::Vector{String} +end + +Option(v::String) = Option(v, nothing) + +# this the entry point for the majority of input checks +function MatrixCommand(statement::Statement)::MatrixCommand + meta_opts = enforce_meta_options(statement.meta_options, + meta_option_specs) + + # return MatrixCommand(statement.command) + return MatrixCommand(statement.command, statement.arguments) +end + +############# +# Execution # +############# + +function do_cmd!(command::MatrixCommand, repl) + global user + if command.spec.kind == CMD_MSG + send(user, command.arguments[1], join(command.arguments[2:end], " ")) + elseif command.spec.kind == CMD_HISTORY + if length(command.arguments) == 0 + return + elseif length(command.arguments) == 1 + history_size = 10 + else + history_size = command.arguments + end + history(user, command.arguments[1], limit = history_size) + elseif command.spec.kind in [ CMD_DIALOG_LIST ] + global roomlist + map(println,roomlist["joined_rooms"]) + elseif command.spec.kind in [ CMD_LOAD_GROUP_PHOTO ] + + end +end + +function do_cmd(repl::REPL.AbstractREPL, input::String; do_rethrow=false) + try + statements = parse(input) + commands = map(MatrixCommand, statements) + for cmd in commands + do_cmd!(cmd, repl) + end + catch err + if do_rethrow + rethrow(err) + end + Base.display_error(repl.t.err_stream, err, Base.catch_backtrace()) + end +end + +function CommandSpec(command_name::String)::Union{Nothing,CommandSpec} + # maybe a "matrix" command + spec = get(super_specs["matrix"], command_name, nothing) + if spec !== nothing + return spec + end + # maybe a "compound command" + m = match(r"(\w+)-(\w+)", command_name) + m !== nothing || (return nothing) + super = get(super_specs, m.captures[1], nothing) + super !== nothing || (return nothing) + return get(super, m.captures[2], nothing) +end + +###################### +# REPL mode creation # +###################### + +# Provide a string macro matrix"cmd" that can be used in the same way +# as the REPLMode `matrix> cmd`. Useful for testing and in environments +# where we do not have a REPL, e.g. IJulia. +struct MiniREPL <: REPL.AbstractREPL + display::TextDisplay + t::REPL.Terminals.TTYTerminal +end +function MiniREPL() + MiniREPL(TextDisplay(stdout), REPL.Terminals.TTYTerminal(get(ENV, "TERM", Sys.iswindows() ? "" : "dumb"), stdin, stdout, stderr)) +end +REPL.REPLDisplay(repl::MiniREPL) = repl.display + + +const minirepl = Ref{MiniREPL}() + +__init__() = minirepl[] = MiniREPL() + +macro matrix_str(str::String) + :($(do_cmd)(minirepl[], $str; do_rethrow=true)) +end + +matrixstr(str::String) = do_cmd(minirepl[], str; do_rethrow=true) + +# handle completions +mutable struct CompletionCache + commands::Vector{String} + canonical_names::Vector{String} + meta_options::Vector{String} + options::Dict{CommandKind, Vector{String}} + subcommands::Dict{String, Vector{String}} + CompletionCache() = new([],[],[],Dict(),Dict()) +end + +completion_cache = CompletionCache() + +struct MatrixCompletionProvider <: LineEdit.CompletionProvider end + +function LineEdit.complete_line(c::MatrixCompletionProvider, s) + partial = REPL.beforecursor(s.input_buffer) + full = LineEdit.input_string(s) + ret, range, should_complete = completions(full, lastindex(partial)) + return ret, partial[range], should_complete +end + +function complete_local_path(s, i1, i2) + cmp = REPL.REPLCompletions.complete_path(s, i2) + [REPL.REPLCompletions.completion_text(p) for p in cmp[1]], cmp[2], !isempty(cmp[1]) +end + +function complete_installed_package(s, i1, i2, project_opt) + matrixs = project_opt ? API.__installed(PKGMODE_PROJECT) : API.__installed() + matrixs = sort!(collect(keys(filter((p) -> p[2] != nothing, matrixs)))) + cmp = filter(cmd -> startswith(cmd, s), matrixs) + return cmp, i1:i2, !isempty(cmp) +end + +function complete_remote_package(s, i1, i2) + cmp = String[] + julia_version = VERSION + for reg in Types.registries(;clone_default=false) + data = Types.read_registry(joinpath(reg, "Registry.toml")) + for (uuid, matrixinfo) in data["packages"] + name = matrixinfo["name"] + if startswith(name, s) + compat_data = Operations.load_package_data_raw( + VersionSpec, joinpath(reg, matrixinfo["path"], "Compat.toml")) + supported_julia_versions = VersionSpec(VersionRange[]) + for (ver_range, compats) in compat_data + for (compat, v) in compats + if compat == "julia" + union!(supported_julia_versions, VersionSpec(v)) + end + end + end + if VERSION in supported_julia_versions + push!(cmp, name) + end + end + end + end + return cmp, i1:i2, !isempty(cmp) +end + +function complete_argument(to_complete, i1, i2, lastcommand, project_opt + )::Tuple{Vector{String},UnitRange{Int},Bool} + # to_complete -> new letters typed + # lastcommand -> CMD_TYPE + # project_opt -> true + # i1 -> end command index + # i2 -> current index + if lastcommand in [ CMD_MSG, CMD_HISTORY ] + return complete_user_list(to_complete, i1, i2, project_opt) + elseif lastcommand in [ ] # file completition + end + return String[], 0:-1, false +end + +function complete_user_list(to_complete, i1, i2, _) + global roomlist + l = filter(x -> startswith(x, to_complete), roomlist["joined_rooms"]) + return l, i1:i2, true +end + +function completions(full, index)::Tuple{Vector{String},UnitRange{Int},Bool} + # Full holds the whole line, while index the current cursor position! + pre = full[1:index] + # Returns all commands if first position + if isempty(pre) + return completion_cache.commands, 0:-1, false + end + x = parse(pre; for_completions=true) + if x === nothing # failed parse (invalid command name) + return String[], 0:-1, false + end + (key::Symbol, to_complete::String, spec, proj::Bool) = x + last = split(pre, ' ', keepempty=true)[end] + offset = isempty(last) ? index+1 : last.offset+1 + if last != to_complete # require a space before completing next field + return String[], 0:-1, false + end + if key == :arg + return complete_argument(to_complete, offset, index, spec.kind, proj) + end + possible::Vector{String} = + key == :meta ? completion_cache.meta_options : + key == :cmd ? completion_cache.commands : + key == :sub ? completion_cache.subcommands[spec] : + key == :opt ? completion_cache.options[spec.kind] : + String[] + completions = filter(x->startswith(x,to_complete), possible) + return completions, offset:index, !isempty(completions) +end + +prev_project_file = nothing +prev_project_timestamp = nothing +prev_prefix = "" + +function promptf() + global prev_project_timestamp, prev_prefix, prev_project_file + project_file = try + Types.find_project_file() + catch + nothing + end + prefix = "" + if project_file !== nothing + if prev_project_file == project_file && prev_project_timestamp == mtime(project_file) + prefix = prev_prefix + else + project = try + Types.read_project(project_file) + catch + nothing + end + if project !== nothing + projname = get(project, "name", nothing) + if projname !== nothing + name = projname + else + name = basename(dirname(project_file)) + end + prefix = string("(", name, ") ") + prev_prefix = prefix + prev_project_timestamp = mtime(project_file) + prev_project_file = project_file + end + end + end + return prefix * "matrix> " +end + +# Set up the repl Pkg REPLMode +function create_mode(repl, main) + matrix_mode = LineEdit.Prompt(promptf; + prompt_prefix = Base.text_colors[:blue], + prompt_suffix = "", + complete = MatrixCompletionProvider(), + sticky = true) + + matrix_mode.repl = repl + hp = main.hist + hp.mode_mapping[:matrix] = matrix_mode + matrix_mode.hist = hp + + search_prompt, skeymap = LineEdit.setup_search_keymap(hp) + prefix_prompt, prefix_keymap = LineEdit.setup_prefix_keymap(hp, matrix_mode) + + matrix_mode.on_done = (s, buf, ok) -> begin + ok || return REPL.transition(s, :abort) + input = String(take!(buf)) + REPL.reset(repl) + do_cmd(repl, input) + REPL.prepare_next(repl) + REPL.reset_state(s) + s.current_mode.sticky || REPL.transition(s, main) + end + + mk = REPL.mode_keymap(main) + + repl_keymap = Dict() + b = Dict{Any,Any}[ + skeymap, repl_keymap, mk, prefix_keymap, LineEdit.history_keymap, + LineEdit.default_keymap, LineEdit.escape_defaults + ] + matrix_mode.keymap_dict = LineEdit.keymap(b) + return matrix_mode +end + +function repl() + repl = Base.active_repl + main_mode = repl.interface.modes[1] + matrix_mode = create_mode(repl, main_mode) + push!(repl.interface.modes, matrix_mode) + keymap = Dict{Any,Any}( + '/' => function (s,args...) + if isempty(s) || position(LineEdit.buffer(s)) == 0 + buf = copy(LineEdit.buffer(s)) + LineEdit.transition(s, matrix_mode) do + LineEdit.state(s, matrix_mode).input_buffer = buf + end + else + LineEdit.edit_insert(s, '/') + end + end + ) + main_mode.keymap_dict = LineEdit.keymap_merge(main_mode.keymap_dict, keymap) + return +end + +######## +# SPEC # +######## +command_declarations = [ +["matrix"] => CommandDeclaration[ +( CMD_MSG, + [ "msg", "send_message" ], + nothing, + (0=>Inf, identity, []), + [], + "send message to peer", + nothing + ), + ( CMD_HISTORY, + [ "history" ], + nothing, + (0=>Inf, identity, []), + [], + "show peer history", + nothing + ), + ( CMD_DIALOG_LIST, + [ "dialog_list" ], + nothing, + (0=>Inf, identity, []), + [], + "show available dialogs", + nothing + ), + ( CMD_LOAD_GROUP_PHOTO, + [ "load_chat_photo", "load_group_photo" ], + nothing, + (0=>Inf, identity, []), + [], + "Download a group's profile picture", + nothing + ) +], #matrix +] #command_declarations + +super_specs = SuperSpecs(command_declarations) +# cache things you need for completions +completion_cache.meta_options = sort(map(wrap_option, collect(keys(meta_option_specs)))) +completion_cache.commands = sort(append!(collect(keys(super_specs)), + collect(keys(super_specs["matrix"])))) +let names = String[] + for (super, specs) in pairs(super_specs) + super == "matrix" && continue # skip "matrix" + for spec in unique(values(specs)) + push!(names, join([super, spec.canonical_name], "-")) + end + end + for spec in unique(values(super_specs["matrix"])) + push!(names, spec.canonical_name) + end + completion_cache.canonical_names = names + sort!(completion_cache.canonical_names) +end +for (k, v) in pairs(super_specs) + completion_cache.subcommands[k] = sort(collect(keys(v))) + for spec in values(v) + completion_cache.options[spec.kind] = + sort(map(wrap_option, collect(keys(spec.option_specs)))) + end +end +# TODO remove this +command_specs = super_specs["matrix"] + +const help = md""" + +**Welcome to the Matrix REPL-mode**. To return to the `julia>` prompt, either press +backspace when the input line is empty or press Ctrl+C. + + +**Synopsis** + + matrix> cmd [opts] [args] + +Multiple commands can be given on the same line by interleaving a `;` between the commands. + +**Commands** +""" diff --git a/src/rooms.jl b/src/rooms.jl new file mode 100644 index 0000000..1337c83 --- /dev/null +++ b/src/rooms.jl @@ -0,0 +1,66 @@ +function listjoined(u::MatrixUser) + JSON.parse(String(get(u, "joined_rooms").body)) +end + +function createroom(u::MatrixUser, room::String) + raw_response = post(u, "createRoom", Dict("room_alias_name" => room)) + JSON.parse(String(raw_response.body)) +end + +function invite(u::MatrixUser, room::String, user::String) + post(u, string("rooms/",room,"/invite"), Dict("user_id" => user)) +end + +function send(u::MatrixUser, room_id::String, text::String) + res = post(u, string("rooms/", room_id, "/send/m.room.message"), + Dict("msgtype" => "m.text", "body" => text)) + JSON.parse(String(res.body)) +end + + +function getroomevent(u::MatrixUser, room_id::String, event::String) + get(u, string("rooms/", HTTP.escapeuri(room_id), "/state", + event == "" ? "" : string("/", event))) +end + +getroomevent(u::MatrixUser, room_id::String) = getroomevent(u, room_id, "") + +getroomname(u::MatrixUser, room_id::String) = getroomevent(u, room_id, "m.room.name") + +function roomhistory(u::MatrixUser, room_id::String; + from::Union{Nothing,String} = nothing, + direction::String = "f", limit = 15) + global rooms_last_batch + query = Dict("dir" => direction, + "limit" => string(limit), + "from" => from === nothing ? + rooms_last_batch[room_id] : from) + res = get(u, join(["rooms", room_id, "messages",], "/"), extraquery = query) + res = res.body |> String |> JSON.parse + rooms_last_batch[room_id] = res["end"] + res +end + +function checkifjoin(syncres) + requests = String[] + let joins = syncres["rooms"]["invite"] + # @show joins + for j in keys(joins) + # @show joins[j]["invite_state"]["events"] + push!(requests,j) + end + end + requests +end + +function joinroom(u::MatrixUser, room_id::String) + # @show join(["rooms", HTTP.escapeuri(room_id), "join"], "/") + post(u, join(["rooms", HTTP.escapeuri(room_id), "join"], "/"), Dict{String,String}()) +end + +function history(u::MatrixUser, room_id::String; limit = 15) + res = roomhistory(u, room_id, limit = limit) + map(x -> println(MatrixMsg(x["sender"], x["origin_server_ts"], + x["content"]["msgtype"], x["content"]["body"], + )), res["chunk"]) +end diff --git a/src/sync.jl b/src/sync.jl new file mode 100644 index 0000000..84dc708 --- /dev/null +++ b/src/sync.jl @@ -0,0 +1,68 @@ +function sync(u::MatrixUser; + timeout = 0, since::Union{String,Nothing} = nothing, full = false) + req = Dict("timeout" => string(timeout), + "full" => string(full)) + since === nothing ? nothing : (req["since"] = string(since)) + get(u, "sync"; extraquery = req) +end + +function sync!(s::MatrixSync) + # initial sync + res = s.last in [nothing, ""] ? + sync(s.user, timeout = s.timeout, full = true) : + # updates + sync(s.user, since = s.last, timeout = s.timeout, full = false) + res = JSON.parse(String(res.body)) + s.last = res["next_batch"] + s.sync = res + res +end + +sync_enabled = false +hooks = Dict{Symbol,Function}() +function background_sync(s::MatrixSync) + global sync_enabled, hooks + sync_enabled = true + @async while sync_enabled + res = sync!(s) + if :sync in keys(hooks) + # println("Calling hook") + hooks[:sync](res) + else + println("NOT Calling hook") + end + end +end +function stop_sync!() + global sync_enabled + oldstatus = sync_enabled + sync_enabled = false + sync_enabled +end + +function set_hook!(event::Symbol, f::Function) + global hooks + hooks[event] = f +end + +function remove_hook!(event::Symbol) + global hooks + if event in keys(hooks) + pop!(hooks, event) + return true + end + false +end + +rooms_last_batch = Dict() +function updateroomlastbatch(fullsync::Dict{String,Any}) + global rooms_last_batch + map(x -> + rooms_last_batch[x] = fullsync["rooms"]["join"][x]["timeline"]["prev_batch"], + collect(keys(fullsync["rooms"]["join"]))) + rooms_last_batch +end + +function updateroomlastbatch(user::MatrixUser) + updateroomlastbatch(sync(user, full = true).body |> String |> JSON.parse) +end diff --git a/src/types.jl b/src/types.jl new file mode 100644 index 0000000..43a592c --- /dev/null +++ b/src/types.jl @@ -0,0 +1,57 @@ +const API_PATH = "_matrix/client/r0" +struct MatrixServer + instance::String + secret::String + api::String +end + +struct MatrixUser + server::MatrixServer + userid::String + token::String + deviceid::String +end + +import Base.show +function Base.show(io::IO, n::MatrixServer) + print(io, string("(matrix) ", n.instance)) + nothing +end + +function Base.show(io::IO, u::MatrixUser) + print(io, string("(", u.deviceid, ") ", u.userid)) + nothing +end + +MatrixServer(s::String, secret::String) = MatrixServer(s, secret, API_PATH) +MatrixServer(s::String) = MatrixServer(s, "") + +mutable struct MatrixSync + user::MatrixUser + last::Union{Nothing,String} + timeout::Int + sync +end + +MatrixSync(u::MatrixUser) = MatrixSync(u, nothing, 30000, Dict()) + +struct MatrixMsg + sender::String + timestamp::Int + mtype::String + body::String +end + +using Dates +function Base.show(io::IO, m::MatrixMsg) + d = Dates.epochms2datetime(m.timestamp + + Dates.datetime2epochms(Dates.unix2datetime(0))) + dtfmt = if Dates.Hour(now()) - Dates.Hour(d) < Dates.Hour(24) + string(hour(d), ":", minute(d)) + else + string(day(d), " ", Dates.monthabbr(d)) + end + + print(io, string("[", dtfmt, "] ", m.sender, " »»» ", m.body)) + nothing +end diff --git a/src/user.jl b/src/user.jl new file mode 100644 index 0000000..5ae7cf2 --- /dev/null +++ b/src/user.jl @@ -0,0 +1,42 @@ +"""Logs into a matrix server. +`login(::MatrixServer, username::String, password::String)` + +the optional parameter `deviceid` is `JuliaMatrix` by default +""" +function login(s::MatrixServer, username::String, password::String; + deviceid::Union{Nothing,String} = nothing) + body = Dict( + "type" => "m.login.password", + "identifier" => Dict( + "type" => "m.id.user", + "user" => username + ), + "password" => password, + "device_id" => deviceid === nothing ? "JuliaMatrix" : deviceid + ) + try res = post(s, "login", body) + catch x + if isa(x,HTTP.ExceptionRequest.StatusError) && x.status == 403 + @error "Auth failed" + @show x + else + @warn "Matrix: Unknown error" + throw(x) + end + end + res +end + +function setavatar(u::MatrixUser, murl::String) + put(u, string("profile/", u.userid, "/avatar_url"), + Dict("avatar_url" => murl)) +end + +function setavatar(u::MatrixUser, data) + pic = upload(u, "image/png", data) + setavatar(u, pic["content_uri"]) +end + +# function setavatar(s::MatrixServer, u::MatrixUser, file::String) +# setavatar(s::MatrixServer, u::MatrixUser, read(file)) +# end diff --git a/src/users.jl b/src/users.jl new file mode 100644 index 0000000..951e68c --- /dev/null +++ b/src/users.jl @@ -0,0 +1,40 @@ +using Nettle + +function new_user(s::MatrixServer, + username::AbstractString, password::AbstractString; + admin::Bool = false) + path = "admin/register" + res = get(s, path) + nonce = JSON.parse(String(res.body))["nonce"] + h = HMACState("sha1", s.secret) + update!(h, nonce) + update!(h, "\x00") + update!(h, username) + update!(h, "\x00") + update!(h, password) + update!(h, "\x00") + update!(h, admin ? "admin" : "notadmin") + mac = hexdigest!(h) + + data = Dict("nonce" => nonce, + "username" => username, + "password" => password, + "mac" => mac, + "admin" => admin) + + raw_response = post(s, path, data) + + parsed = String(raw_response.body) + write("$(username).json", parsed) + MatrixUser(s, + parsed["user_id"], parsed["access_token"], + parsed["device_id"]) +end + +function getdisplayname(server::MatrixServer, userid::String) + # Errors: 404 not found + res = get(server, + string("profile/", userid, "/displayname") + ).body |> String |> JSON.parse + res["displayname"] +end diff --git a/src/utils.jl b/src/utils.jl new file mode 100644 index 0000000..1e229cb --- /dev/null +++ b/src/utils.jl @@ -0,0 +1,91 @@ +using URIParser + +import Base.get +function get(s::MatrixServer, path::String) + HTTP.get(join([s.instance, API_PATH, path], "/")) +end + +""" +Authenticated get request +""" +function get(u::MatrixUser, path::String; + extraquery = Dict()) + query = Dict( + "access_token" => u.token + ) + merge!(query, extraquery) + HTTP.get(join([u.server.instance, API_PATH, path], "/"), + query = query) +end + +function post(s::MatrixServer, path::String, data::Dict{String,Any}) + HTTP.request("POST", join([s.instance, API_PATH, path], "/"), + ["Content-Type" => "application/json"], + JSON.json(data)) +end + +""" +Authenticated post request +""" +function post(u::MatrixUser, path::String, + data; + mime = "application/json") + HTTP.request("POST", join([u.server.instance, "_matrix/media/v1/upload", path], "/"), + ["Content-Type" => mime], + data, + query = Dict( + "access_token" => u.token + )) +end + +function request(request::String, u::MatrixUser, path::String, + data::Dict{String,String}; + mime = "application/json") + HTTP.request(request, + join([u.server.instance, API_PATH, path], "/"), + ["Content-Type" => string(mime)], + JSON.json(data), + query = Dict( + "access_token" => u.token + )) +end + +function put(u::MatrixUser, path::String, + data::Dict{String,String}; + mime = "application/json") + request("PUT", u, path, data, mime = mime) +end + +function post(u::MatrixUser, path::String, + data::Dict{String,String}; + mime = "application/json") + HTTP.request("POST", + join([u.server.instance, API_PATH, path], "/"), + ["Content-Type" => string(mime)], + JSON.json(data), + query = Dict( + "access_token" => u.token + )) +end + +function upload(u::MatrixUser, mime, data::Any) + res = post(u, "media/upload", data, mime = mime) + JSON.parse(String(res.body)) +end + +# matrixurl +import Base.download +function download(u::MatrixUser, murl::String; download_path = "~/matrix/") + mkpath(expanduser(download_path)) + uri = URI(murl) + murl = string(uri.host, uri.path) + + query = Dict( + "access_token" => u.token + ) + res = HTTP.get(join([u.server.instance, "_matrix/media/r0/download", murl], "/"), query = query) + filetype = filter( x -> x[1] == "Content-Type", res.headers |> values)[1][2] + filepath = realpath(joinpath(expanduser(download_path), string(".", uri.path))) + filesize = write(filepath, res.body) + (filetype, filesize, filepath) +end