diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fa1eccb..9160ff1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,11 @@ +Changes in latest release + +BREAKING CHANGE: Olm now uses WebAssembly which means it needs +to load the wasm file asynchronously, and therefore needs to be +started up asynchronously. The imported module now has an init() +method which returns a promise. The library cannot be used until +this promise resolves. It will reject if the library fails to start. + Changes in `2.3.0 `_ This release includes the following changes since 2.2.2: diff --git a/Makefile b/Makefile index 901e78f..60024c5 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ DEBUG_TARGET := $(BUILD_DIR)/libolm_debug.so.$(VERSION) JS_TARGET := javascript/olm.js JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json +JS_EXTRA_EXPORTED_RUNTIME_METHODS := ALLOC_STACK +JS_EXTERNS := javascript/externs.js PUBLIC_HEADERS := include/olm/olm.h include/olm/outbound_group_session.h include/olm/inbound_group_session.h include/olm/pk.h @@ -39,11 +41,22 @@ FUZZER_BINARIES := $(addprefix $(BUILD_DIR)/,$(basename $(FUZZER_SOURCES))) FUZZER_DEBUG_BINARIES := $(patsubst $(BUILD_DIR)/fuzzers/fuzz_%,$(BUILD_DIR)/fuzzers/debug_%,$(FUZZER_BINARIES)) TEST_BINARIES := $(patsubst tests/%,$(BUILD_DIR)/tests/%,$(basename $(TEST_SOURCES))) JS_OBJECTS := $(addprefix $(BUILD_DIR)/javascript/,$(OBJECTS)) + +# pre & post are the js-pre/js-post options to emcc. +# They are injected inside the modularised code and +# processed by the optimiser. JS_PRE := $(wildcard javascript/*pre.js) JS_POST := javascript/olm_outbound_group_session.js \ javascript/olm_inbound_group_session.js \ javascript/olm_pk.js \ javascript/olm_post.js + +# The prefix & suffix are just added onto the start & end +# of what comes out emcc, so are outside of the modularised +# code and not seen by the opimiser. +JS_PREFIX := javascript/olm_prefix.js +JS_SUFFIX := javascript/olm_suffix.js + DOCS := tracing/README.html \ docs/megolm.html \ docs/olm.html \ @@ -60,11 +73,20 @@ CFLAGS += -Wall -Werror -std=c99 -fPIC CXXFLAGS += -Wall -Werror -std=c++11 -fPIC LDFLAGS += -Wall -Werror -EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 +EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 -s MODULARIZE=1 # NO_BROWSER is kept for compatibility with emscripten 1.35.24, but is no # longer needed. EMCCFLAGS += -s NO_BROWSER=1 +# Olm generally doesn't need a lot of memory to encrypt / decrypt its usual +# payloads (ie. Matrix messages), but we do need about 128K of heap to encrypt +# a 64K event (enough to store the ciphertext and the plaintext, bearing in +# mind that the plaintext can only be 48K because base64). We also have about +# 36K of statics. So let's have 256K of memory. +# (This can't be changed by the app with wasm since it's baked into the wasm). +EMCCFLAGS += -s TOTAL_STACK=65536 -s TOTAL_MEMORY=262144 + + EMCC.c = $(EMCC) $(CFLAGS) $(CPPFLAGS) -c EMCC.cc = $(EMCC) $(CXXFLAGS) $(CPPFLAGS) -c EMCC_LINK = $(EMCC) $(LDFLAGS) $(EMCCFLAGS) @@ -145,12 +167,19 @@ $(STATIC_RELEASE_TARGET): $(RELEASE_OBJECTS) js: $(JS_TARGET) .PHONY: js -$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) - $(EMCC_LINK) \ +# Note that the output file we give to emcc determines the name of the +# wasm file baked into the js, hence messing around outputting to olm.js +# and then renaming it. +$(JS_TARGET): $(JS_OBJECTS) $(JS_PRE) $(JS_POST) $(JS_EXPORTED_FUNCTIONS) $(JS_PREFIX) $(JS_SUFFIX) + EMCC_CLOSURE_ARGS="--externs $(JS_EXTERNS)" $(EMCC_LINK) \ $(foreach f,$(JS_PRE),--pre-js $(f)) \ $(foreach f,$(JS_POST),--post-js $(f)) \ -s "EXPORTED_FUNCTIONS=@$(JS_EXPORTED_FUNCTIONS)" \ + -s "EXTRA_EXPORTED_RUNTIME_METHODS=$(JS_EXTRA_EXPORTED_RUNTIME_METHODS)" \ $(JS_OBJECTS) -o $@ + mv $@ javascript/olmtmp.js + cat $(JS_PREFIX) javascript/olmtmp.js $(JS_SUFFIX) > $@ + rm javascript/olmtmp.js build_tests: $(TEST_BINARIES) diff --git a/javascript/.gitignore b/javascript/.gitignore index ec22345..3437f73 100644 --- a/javascript/.gitignore +++ b/javascript/.gitignore @@ -2,4 +2,5 @@ /node_modules /npm-debug.log /olm.js +/olm.wasm /reports diff --git a/javascript/externs.js b/javascript/externs.js new file mode 100644 index 0000000..752e937 --- /dev/null +++ b/javascript/externs.js @@ -0,0 +1,4 @@ +var OLM_OPTIONS; +var olm_exports; +var onInitSuccess; +var onInitFail; diff --git a/javascript/olm_inbound_group_session.js b/javascript/olm_inbound_group_session.js index 6bc745d..7d9e401 100644 --- a/javascript/olm_inbound_group_session.js +++ b/javascript/olm_inbound_group_session.js @@ -1,9 +1,3 @@ -/* The 'length' argument to Pointer_stringify doesn't work if the input includes - * characters >= 128; we therefore need to add a NULL character to all of our - * strings. This acts as a symbolic constant to help show what we're doing. - */ -var NULL_BYTE_PADDING_LENGTH = 1; - function InboundGroupSession() { var size = Module['_olm_inbound_group_session_size'](); this.buf = malloc(size); @@ -77,14 +71,14 @@ InboundGroupSession.prototype['decrypt'] = restore_stack(function( try { message_buffer = malloc(message.length); - Module['writeAsciiToMemory'](message, message_buffer, true); + writeAsciiToMemory(message, message_buffer, true); var max_plaintext_length = inbound_group_session_method( Module['_olm_group_decrypt_max_plaintext_length'] )(this.ptr, message_buffer, message.length); // caculating the length destroys the input buffer, so we need to re-copy it. - Module['writeAsciiToMemory'](message, message_buffer, true); + writeAsciiToMemory(message, message_buffer, true); plaintext_buffer = malloc(max_plaintext_length + NULL_BYTE_PADDING_LENGTH); var message_index = stack(4); @@ -100,14 +94,14 @@ InboundGroupSession.prototype['decrypt'] = restore_stack(function( // UTF8ToString requires a null-terminated argument, so add the // null terminator. - Module['setValue']( + setValue( plaintext_buffer+plaintext_length, 0, "i8" ); return { "plaintext": UTF8ToString(plaintext_buffer), - "message_index": Module['getValue'](message_index, "i32") + "message_index": getValue(message_index, "i32") } } finally { if (message_buffer !== undefined) { diff --git a/javascript/olm_outbound_group_session.js b/javascript/olm_outbound_group_session.js index 24ea644..e232883 100644 --- a/javascript/olm_outbound_group_session.js +++ b/javascript/olm_outbound_group_session.js @@ -1,10 +1,3 @@ -/* The 'length' argument to Pointer_stringify doesn't work if the input includes - * characters >= 128; we therefore need to add a NULL character to all of our - * strings. This acts as a symbolic constant to help show what we're doing. - */ -var NULL_BYTE_PADDING_LENGTH = 1; - - function OutboundGroupSession() { var size = Module['_olm_outbound_group_session_size'](); this.buf = malloc(size); @@ -66,7 +59,7 @@ OutboundGroupSession.prototype['create'] = restore_stack(function() { OutboundGroupSession.prototype['encrypt'] = function(plaintext) { var plaintext_buffer, message_buffer, plaintext_length; try { - plaintext_length = Module['lengthBytesUTF8'](plaintext); + plaintext_length = lengthBytesUTF8(plaintext); var message_length = outbound_group_session_method( Module['_olm_group_encrypt_message_length'] @@ -75,7 +68,7 @@ OutboundGroupSession.prototype['encrypt'] = function(plaintext) { // need to allow space for the terminator (which stringToUTF8 always // writes), hence + 1. plaintext_buffer = malloc(plaintext_length + 1); - Module['stringToUTF8'](plaintext, plaintext_buffer, plaintext_length + 1); + stringToUTF8(plaintext, plaintext_buffer, plaintext_length + 1); message_buffer = malloc(message_length + NULL_BYTE_PADDING_LENGTH); outbound_group_session_method(Module['_olm_group_encrypt'])( @@ -86,12 +79,12 @@ OutboundGroupSession.prototype['encrypt'] = function(plaintext) { // UTF8ToString requires a null-terminated argument, so add the // null terminator. - Module['setValue']( + setValue( message_buffer+message_length, 0, "i8" ); - return Module['UTF8ToString'](message_buffer); + return UTF8ToString(message_buffer); } finally { if (plaintext_buffer !== undefined) { // don't leave a copy of the plaintext in the heap. diff --git a/javascript/olm_pk.js b/javascript/olm_pk.js index 2542707..25e0fee 100644 --- a/javascript/olm_pk.js +++ b/javascript/olm_pk.js @@ -35,9 +35,9 @@ PkEncryption.prototype['encrypt'] = restore_stack(function( ) { var plaintext_buffer, ciphertext_buffer, plaintext_length; try { - plaintext_length = Module['lengthBytesUTF8'](plaintext) + plaintext_length = lengthBytesUTF8(plaintext) plaintext_buffer = malloc(plaintext_length + 1); - Module['stringToUTF8'](plaintext, plaintext_buffer, plaintext_length + 1); + stringToUTF8(plaintext, plaintext_buffer, plaintext_length + 1); var random_length = pk_encryption_method( Module['_olm_pk_encrypt_random_length'] )(); @@ -50,7 +50,7 @@ PkEncryption.prototype['encrypt'] = restore_stack(function( Module['_olm_pk_mac_length'] )(this.ptr); var mac_buffer = stack(mac_length + NULL_BYTE_PADDING_LENGTH); - Module['setValue']( + setValue( mac_buffer+mac_length, 0, "i8" ); @@ -58,7 +58,7 @@ PkEncryption.prototype['encrypt'] = restore_stack(function( Module['_olm_pk_key_length'] )(); var ephemeral_buffer = stack(ephemeral_length + NULL_BYTE_PADDING_LENGTH); - Module['setValue']( + setValue( ephemeral_buffer+ephemeral_length, 0, "i8" ); @@ -72,12 +72,12 @@ PkEncryption.prototype['encrypt'] = restore_stack(function( ); // UTF8ToString requires a null-terminated argument, so add the // null terminator. - Module['setValue']( + setValue( ciphertext_buffer+ciphertext_length, 0, "i8" ); return { - "ciphertext": Module['UTF8ToString'](ciphertext_buffer), + "ciphertext": UTF8ToString(ciphertext_buffer), "mac": Pointer_stringify(mac_buffer), "ephemeral": Pointer_stringify(ephemeral_buffer) }; @@ -169,9 +169,9 @@ PkDecryption.prototype['decrypt'] = restore_stack(function ( ) { var plaintext_buffer, ciphertext_buffer, plaintext_max_length; try { - ciphertext_length = Module['lengthBytesUTF8'](ciphertext) + var ciphertext_length = lengthBytesUTF8(ciphertext) ciphertext_buffer = malloc(ciphertext_length + 1); - Module['stringToUTF8'](ciphertext, ciphertext_buffer, ciphertext_length + 1); + stringToUTF8(ciphertext, ciphertext_buffer, ciphertext_length + 1); var ephemeralkey_array = array_from_string(ephemeral_key); var ephemeralkey_buffer = stack(ephemeralkey_array); var mac_array = array_from_string(mac); @@ -190,11 +190,11 @@ PkDecryption.prototype['decrypt'] = restore_stack(function ( ); // UTF8ToString requires a null-terminated argument, so add the // null terminator. - Module['setValue']( + setValue( plaintext_buffer+plaintext_length, 0, "i8" ); - return Module['UTF8ToString'](plaintext_buffer); + return UTF8ToString(plaintext_buffer); } finally { if (plaintext_buffer !== undefined) { // don't leave a copy of the plaintext in the heap. diff --git a/javascript/olm_post.js b/javascript/olm_post.js index 7a1d284..9e0294a 100644 --- a/javascript/olm_post.js +++ b/javascript/olm_post.js @@ -1,27 +1,17 @@ -var runtime = Module['Runtime']; var malloc = Module['_malloc']; var free = Module['_free']; -var Pointer_stringify = Module['Pointer_stringify']; -var OLM_ERROR = Module['_olm_error'](); - -/* The 'length' argument to Pointer_stringify doesn't work if the input - * includes characters >= 128, which makes Pointer_stringify unreliable. We - * could use it on strings which are known to be ascii, but that seems - * dangerous. Instead we add a NULL character to all of our strings and just - * use UTF8ToString. - */ -var NULL_BYTE_PADDING_LENGTH = 1; +var OLM_ERROR; /* allocate a number of bytes of storage on the stack. * * If size_or_array is a Number, allocates that number of zero-initialised bytes. */ function stack(size_or_array) { - return Module['allocate'](size_or_array, 'i8', Module['ALLOC_STACK']); + return allocate(size_or_array, 'i8', Module['ALLOC_STACK']); } function array_from_string(string) { - return Module['intArrayFromString'](string, true); + return intArrayFromString(string, true); } function random_stack(size) { @@ -33,11 +23,11 @@ function random_stack(size) { function restore_stack(wrapped) { return function() { - var sp = runtime.stackSave(); + var sp = stackSave(); try { return wrapped.apply(this, arguments); } finally { - runtime.stackRestore(sp); + stackRestore(sp); } } } @@ -315,7 +305,7 @@ Session.prototype['encrypt'] = restore_stack(function( Module['_olm_encrypt_message_type'] )(this.ptr); - plaintext_length = Module['lengthBytesUTF8'](plaintext); + plaintext_length = lengthBytesUTF8(plaintext); var message_length = session_method( Module['_olm_encrypt_message_length'] )(this.ptr, plaintext_length); @@ -325,7 +315,7 @@ Session.prototype['encrypt'] = restore_stack(function( // need to allow space for the terminator (which stringToUTF8 always // writes), hence + 1. plaintext_buffer = malloc(plaintext_length + 1); - Module['stringToUTF8'](plaintext, plaintext_buffer, plaintext_length + 1); + stringToUTF8(plaintext, plaintext_buffer, plaintext_length + 1); message_buffer = malloc(message_length + NULL_BYTE_PADDING_LENGTH); @@ -338,14 +328,14 @@ Session.prototype['encrypt'] = restore_stack(function( // UTF8ToString requires a null-terminated argument, so add the // null terminator. - Module['setValue']( + setValue( message_buffer+message_length, 0, "i8" ); return { "type": message_type, - "body": Module['UTF8ToString'](message_buffer), + "body": UTF8ToString(message_buffer), }; } finally { if (plaintext_buffer !== undefined) { @@ -366,14 +356,14 @@ Session.prototype['decrypt'] = restore_stack(function( try { message_buffer = malloc(message.length); - Module['writeAsciiToMemory'](message, message_buffer, true); + writeAsciiToMemory(message, message_buffer, true); max_plaintext_length = session_method( Module['_olm_decrypt_max_plaintext_length'] )(this.ptr, message_type, message_buffer, message.length); // caculating the length destroys the input buffer, so we need to re-copy it. - Module['writeAsciiToMemory'](message, message_buffer, true); + writeAsciiToMemory(message, message_buffer, true); plaintext_buffer = malloc(max_plaintext_length + NULL_BYTE_PADDING_LENGTH); @@ -385,7 +375,7 @@ Session.prototype['decrypt'] = restore_stack(function( // UTF8ToString requires a null-terminated argument, so add the // null terminator. - Module['setValue']( + setValue( plaintext_buffer+plaintext_length, 0, "i8" ); @@ -474,21 +464,11 @@ olm_exports["get_library_version"] = restore_stack(function() { ]; }); -})(); +Module['onRuntimeInitialized'] = function() { + OLM_ERROR = Module['_olm_error'](); + if (onInitSuccess) onInitSuccess(); +}; -// export the olm functions into the environment. -// -// make sure that we do this *after* populating olm_exports, so that we don't -// get a half-built window.Olm if there is an exception. - -if (typeof module !== 'undefined' && module.exports) { - // node / browserify - module.exports = olm_exports; -} - -if (typeof(window) !== 'undefined') { - // We've been imported directly into a browser. Define the global 'Olm' object. - // (we do this even if module.exports was defined, because it's useful to have - // Olm in the global scope for browserified and webpacked apps.) - window["Olm"] = olm_exports; -} +Module['onAbort'] = function(err) { + if (onInitFail) onInitFail(err); +}; diff --git a/javascript/olm_pre.js b/javascript/olm_pre.js index ae7aba5..4feff97 100644 --- a/javascript/olm_pre.js +++ b/javascript/olm_pre.js @@ -1,10 +1,7 @@ -var olm_exports = {}; var get_random_values; -var process; // Shadow the process object so that emscripten won't get - // confused by browserify if (typeof(window) !== 'undefined') { - // We've in a browser (directly, via browserify, or via webpack). + // We're in a browser (directly, via browserify, or via webpack). get_random_values = function(buf) { window.crypto.getRandomValues(buf); }; @@ -12,7 +9,9 @@ if (typeof(window) !== 'undefined') { // We're running in node. var nodeCrypto = require("crypto"); get_random_values = function(buf) { - var bytes = nodeCrypto.randomBytes(buf.length); + // [''] syntax needed here rather than '.' to prevent + // closure compiler from mangling the import(!) + var bytes = nodeCrypto['randomBytes'](buf.length); buf.set(bytes); }; process = global["process"]; @@ -20,14 +19,21 @@ if (typeof(window) !== 'undefined') { throw new Error("Cannot find global to attach library to"); } -(function() { - /* applications should define OLM_OPTIONS in the environment to override - * emscripten module settings */ - var Module = {}; - if (typeof(OLM_OPTIONS) !== 'undefined') { - for (var key in OLM_OPTIONS) { - if (OLM_OPTIONS.hasOwnProperty(key)) { - Module[key] = OLM_OPTIONS[key]; - } +/* applications should define OLM_OPTIONS in the environment to override + * emscripten module settings + */ +if (typeof(OLM_OPTIONS) !== 'undefined') { + for (var olm_option_key in OLM_OPTIONS) { + if (OLM_OPTIONS.hasOwnProperty(olm_option_key)) { + Module[olm_option_key] = OLM_OPTIONS[olm_option_key]; } } +} + +/* The 'length' argument to Pointer_stringify doesn't work if the input + * includes characters >= 128, which makes Pointer_stringify unreliable. We + * could use it on strings which are known to be ascii, but that seems + * dangerous. Instead we add a NULL character to all of our strings and just + * use UTF8ToString. + */ +var NULL_BYTE_PADDING_LENGTH = 1; diff --git a/javascript/olm_prefix.js b/javascript/olm_prefix.js new file mode 100644 index 0000000..b33dfe9 --- /dev/null +++ b/javascript/olm_prefix.js @@ -0,0 +1,3 @@ +var olm_exports = {}; +var onInitSuccess; +var onInitFail; diff --git a/javascript/olm_suffix.js b/javascript/olm_suffix.js new file mode 100644 index 0000000..7f19953 --- /dev/null +++ b/javascript/olm_suffix.js @@ -0,0 +1,30 @@ +var olmInitPromise; + +olm_exports['init'] = function(opts) { + if (olmInitPromise) return olmInitPromise; + + if (opts) OLM_OPTIONS = opts; + + olmInitPromise = new Promise(function(resolve, reject) { + onInitSuccess = function() { + resolve(); + }; + onInitFail = function(err) { + reject(err); + }; + Module(); + }); + return olmInitPromise; +}; + +if (typeof(window) !== 'undefined') { + // We've been imported directly into a browser. Define the global 'Olm' object. + // (we do this even if module.exports was defined, because it's useful to have + // Olm in the global scope for browserified and webpacked apps.) + window["Olm"] = olm_exports; +} + +// Emscripten sets the module exports to be its module +// with wrapped c functions. Clobber it with our higher +// level wrapper class. +module.exports = olm_exports; diff --git a/javascript/package.json b/javascript/package.json index 9cae60e..efe3705 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -5,6 +5,7 @@ "main": "olm.js", "files": [ "olm.js", + "olm.wasm", "README.md" ], "scripts": { diff --git a/javascript/test/megolm.spec.js b/javascript/test/megolm.spec.js index 8f9d24a..241d4bd 100644 --- a/javascript/test/megolm.spec.js +++ b/javascript/test/megolm.spec.js @@ -1,5 +1,6 @@ /* Copyright 2016 OpenMarket Ltd +Copyright 2018 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,9 +22,13 @@ var Olm = require('../olm'); describe("megolm", function() { var aliceSession, bobSession; - beforeEach(function() { - aliceSession = new Olm.OutboundGroupSession(); - bobSession = new Olm.InboundGroupSession(); + beforeEach(function(done) { + Olm.init().then(function() { + aliceSession = new Olm.OutboundGroupSession(); + bobSession = new Olm.InboundGroupSession(); + + done(); + }); }); afterEach(function() { diff --git a/javascript/test/olm.spec.js b/javascript/test/olm.spec.js index b7cc3ae..77dd712 100644 --- a/javascript/test/olm.spec.js +++ b/javascript/test/olm.spec.js @@ -1,5 +1,6 @@ /* Copyright 2016 OpenMarket Ltd +Copyright 2018 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -30,11 +31,18 @@ describe("olm", function() { var aliceAccount, bobAccount; var aliceSession, bobSession; - beforeEach(function() { - aliceAccount = new Olm.Account(); - bobAccount = new Olm.Account(); - aliceSession = new Olm.Session(); - bobSession = new Olm.Session(); + beforeEach(function(done) { + // This should really be in a beforeAll, but jasmine-node + // doesn't support that + debugger; + Olm.init().then(function() { + aliceAccount = new Olm.Account(); + bobAccount = new Olm.Account(); + aliceSession = new Olm.Session(); + bobSession = new Olm.Session(); + + done(); + }); }); afterEach(function() { diff --git a/javascript/test/pk.spec.js b/javascript/test/pk.spec.js index aec90ac..007882f 100644 --- a/javascript/test/pk.spec.js +++ b/javascript/test/pk.spec.js @@ -29,9 +29,13 @@ if (!Object.keys) { describe("pk", function() { var encryption, decryption; - beforeEach(function() { - encryption = new Olm.PkEncryption(); - decryption = new Olm.PkDecryption(); + beforeEach(function(done) { + Olm.init().then(function() { + encryption = new Olm.PkEncryption(); + decryption = new Olm.PkDecryption(); + + done(); + }); }); afterEach(function () {