release 3.2.16

Hello,

Setuptools 69.0.0 deprecated a bunch stuff leading to a nasty errors during install.

> File "/tmp/pip-build-env-w815o5v3/overlay/lib/python3.11/site-packages/setuptools/config/_apply_pyprojecttoml.py", line 183, in _license
>          _set_config(dist, "license", val["text"])
>                                       ~~~^^^^^^^^
> KeyError: 'text'
> [end of output]

__version__.py wasn't used anywhere except setup.py so removing and setting it all pyproject.toml is safe.

During this decided to move as much as I could out of setup.py, zip-safe has been obsolete for modern setuptools installation methods so dropped it.

From c0be008217350f03de7f856866a402d95b5db2a3 Mon Sep 17 00:00:00 2001
From: Alfred Wingate <parona@protonmail.com>
Date: Tue, 21 Nov 2023 15:13:35 +0200
Subject: [PATCH] Move metadata to project.toml

* Setuptools 69.0.0 deprecated a slew of old style configurations.

Signed-off-by: Alfred Wingate <parona@protonmail.com>

.gitignore

@@ -15,6 +15,7 @@
 .ccls-cache/
 /python/.eggs
 /python/install-temp
+/result
 
 # Xcode
 build/
@@ -36,4 +37,7 @@ xcuserdata/
 *.dSYM.zip
 *.dSYM
 Pods/
-*.xcworkspace
+*.xcworkspace
+
+# JetBrains tools
+.idea/

CHANGELOG.rst

@@ -1,3 +1,73 @@
+Changes in `3.2.16 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.16>`_
+===========================================================================
+
+This release includes the following changes since 3.2.15:
+
+* Fix and modernize the Python packaging (thanks to Alfred Wingate)
+
+Changes in `3.2.15 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.15>`_
+===========================================================================
+
+This release includes the following changes since 3.2.14:
+
+* Improvements to Python packaging
+  * No longer depend on ``future`` since Python 2 is no longer supported.
+  * Improve compatibility with tox 4.
+  * Add support for making standalone sdist.
+* Improvements to Nix flake (Thanks to Jon Ringer)
+  * Improve structure.
+  * Enable Darwin builds.
+* Typescript type fix.
+
+Changes in `3.2.14 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.14>`_
+===========================================================================
+
+This release includes the following changes since 3.2.13:
+
+* TypeScript type improvements.
+* Improvements to Python packaging
+* Documentation improvements.
+
+Changes in `3.2.13 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.13>`_
+===========================================================================
+
+This release includes the following changes since 3.2.12:
+
+* Fix compilation with newer versions of emscripten.
+  * The npm package is compiled with emscripten 3.1.17 to fix compatibility with
+    node 18.
+* Add py.typed to Python wheels.
+* Some documentation fixes and updates.
+* Improve the pkgconfig file.
+
+Changes in `3.2.12 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.12>`_
+===========================================================================
+
+This release includes the following changes since 3.2.11:
+
+* Expose olm_sas_calculate_mac_fixed_base64 in the bindings.
+* Allow memory to grow in wasm.  Thanks to benkuly for the suggestion.
+* Fix Python type hints.
+* Some Python build fixes.
+* Initial work on a Nix flake for building and testing.
+
+Changes in `3.2.11 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.11>`_
+===========================================================================
+
+This release includes the following changes since 3.2.10:
+
+* Fix building documentation.  Thanks to Jonas Smedegaard.  The documents
+  written in Markdown are now converted to HTML using Pandoc.
+* Add methods for getting unpublished fallback key in Objective-C binding.
+* Add public pickle/unpickle methods to Java binding.
+* Add wrapper for olm_session_describe to Java binding.  Thanks to Alex Baker.
+
+Changes in `3.2.10 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.10>`_
+===========================================================================
+
+This release includes no change since 3.2.9, but is created to be able to
+publish again the Android library on MavenCentral.
+
 Changes in `3.2.9 <https://gitlab.matrix.org/matrix-org/olm/tags/3.2.9>`_
 =========================================================================
 
@@ -107,11 +177,13 @@ Changes in `3.1.5 <https://gitlab.matrix.org/matrix-org/olm/tags/3.1.5>`_
 This release includes the following changes since 3.1.4:
 
 * Build improvements:
+
   * Fix CMake handling when installing in a non-standard location. Thanks to
     Alexey Rusakov.
   * Add support in the Makefile for creating a WASM-ready archive. Thanks to
     stoically.
   * Improve support for LLVM is Makefile. Thanks to caywin25 for reporting.
+
 * Add a TypeScript definition file.
 * Some documentation and example fixes.
 * Add list of bindings to the README.

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.4)
 
-project(olm VERSION 3.2.9 LANGUAGES CXX C)
+project(olm VERSION 3.2.16 LANGUAGES CXX C)
 
 option(OLM_TESTS "Build tests" ON)
 option(BUILD_SHARED_LIBS "Build as a shared library" ON)

Makefile

@@ -5,10 +5,9 @@ VERSION := $(MAJOR).$(MINOR).$(PATCH)
 PREFIX ?= /usr/local
 BUILD_DIR := build
 RELEASE_OPTIMIZE_FLAGS ?= -O3
-DEBUG_OPTIMIZE_FLAGS ?= -g -O0
+DEBUG_OPTIMIZE_FLAGS ?= -g -O0 -U_FORTIFY_SOURCE
 JS_OPTIMIZE_FLAGS ?= -O3
 FUZZER_OPTIMIZE_FLAGS ?= -O3
-CC = gcc
 EMCC = emcc
 EMAR = emar
 AR = ar
@@ -31,7 +30,7 @@ JS_ASMJS_TARGET := javascript/olm_legacy.js
 WASM_TARGET := $(BUILD_DIR)/wasm/libolm.a
 
 JS_EXPORTED_FUNCTIONS := javascript/exported_functions.json
-JS_EXPORTED_RUNTIME_METHODS := [ALLOC_STACK]
+JS_EXPORTED_RUNTIME_METHODS := [ALLOC_STACK,writeAsciiToMemory,intArrayFromString,UTF8ToString,stringToUTF8]
 JS_EXTERNS := javascript/externs.js
 
 PUBLIC_HEADERS := include/olm/olm.h include/olm/outbound_group_session.h include/olm/inbound_group_session.h include/olm/pk.h include/olm/sas.h include/olm/error.h include/olm/olm_export.h
@@ -94,7 +93,7 @@ LDFLAGS += -Wall -Werror
 CFLAGS_NATIVE = -fPIC
 CXXFLAGS_NATIVE = -fPIC
 
-EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 -s MODULARIZE=1
+EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0 -s MODULARIZE=1 -Wno-error=closure
 
 # Olm generally doesn't need a lot of memory to encrypt / decrypt its usual
 # payloads (ie. Matrix messages), but we do need about 128K of heap to encrypt
@@ -104,7 +103,7 @@ EMCCFLAGS = --closure 1 --memory-init-file 0 -s NO_FILESYSTEM=1 -s INVOKE_RUN=0
 # (This can't be changed by the app with wasm since it's baked into the wasm).
 # (emscripten also mandates at least 16MB of memory for asm.js now, so
 # we don't use this for the legacy build.)
-EMCCFLAGS_WASM += -s TOTAL_STACK=65536 -s TOTAL_MEMORY=262144
+EMCCFLAGS_WASM += -s TOTAL_STACK=65536 -s TOTAL_MEMORY=262144 -s ALLOW_MEMORY_GROWTH
 
 EMCCFLAGS_ASMJS += -s WASM=0
 
@@ -401,6 +400,9 @@ $(BUILD_DIR)/fuzzers/fuzz_%_msan: fuzzing/fuzzers/fuzz_%.cpp $(FUZZER_MSAN_OBJEC
 %.html: %.rst
 	rst2html $< $@
 
+%.html: %.md
+	pandoc --from markdown --to html5 --standalone --lua-filter gitlab-math.lua --katex -o $@ $<
+
 ### dependencies
 
 -include $(RELEASE_OBJECTS:.o=.d)

OLMKit.podspec

@@ -3,7 +3,7 @@ Pod::Spec.new do |s|
   # The libolm version
   MAJOR = 3
   MINOR = 2
-  PATCH = 9
+  PATCH = 16
 
   s.name         = "OLMKit"
   s.version      = "#{MAJOR}.#{MINOR}.#{PATCH}"

Package.swift

@@ -2,7 +2,7 @@
 
 import PackageDescription
 
-let major = 3, minor = 2, patch = 9
+let major = 3, minor = 2, patch = 16
 
 let package = Package(
     name: "Olm",

README.md

@@ -9,6 +9,69 @@ The specification of the Olm ratchet can be found in [docs/olm.md](docs/olm.md).
 This library also includes an implementation of the Megolm cryptographic
 ratchet, as specified in [docs/megolm.md](docs/megolm.md).
 
+## Installing
+
+### Linux and other Unix-like systems
+
+Your distribution may have pre-compiled packages available.  If not, or if you
+need a newer version, you will need to compile from source.  See the "Building"
+section below for more details.
+
+### macOS
+
+The easiest way to install on macOS is via Homebrew.  If you do not have
+Homebrew installed, follow the instructions at https://brew.sh/ to install it.
+
+You can then install libolm by running
+
+```bash
+brew install libolm
+```
+
+If you also need the Python packages, you can run
+
+```bash
+pip3 install python-olm --global-option="build_ext" --global-option="--include-dirs="`brew --prefix libolm`"/include" --global-option="--library-dirs="`brew --prefix libolm`"/lib"
+```
+
+Note that this will install an older version of the Python bindings, which may
+be missing some functions.  If you need the latest version, you will need to
+build from source.
+
+### Windows
+
+You will need to build from source.  See the "Building" section below for more
+details.
+
+### Bindings
+
+#### JavaScript
+
+You can use pre-built npm packages, available at
+<https://gitlab.matrix.org/matrix-org/olm/-/packages?type=npm>.
+
+#### Python
+
+A Python source package and pre-built packages for certain architectures from
+<https://pypi.org/project/python-olm/>.  If a pre-built package is not
+available for your architecture, you will need:
+
+- cmake (recommended) or GNU make
+- a C/C++ compiler
+
+to build the source package.
+
+You can then run `pip install python-olm`.
+
+Currently, we try to provide packages for all supported versions of Python on
+x86-64, i686, and aarch64, but we cannot guarantee that packages for all
+versions will be available on all architectures.
+
+#### Android
+
+Pre-built Android bindings are available at
+<https://gitlab.matrix.org/matrix-org/olm/-/packages?type=Maven>.
+
 ## Building
 
 To build olm as a shared library run:
@@ -41,22 +104,38 @@ target_link_libraries(my_exe Olm::Olm)
 
 ### Bindings
 
-To build the JavaScript bindings, install emscripten from https://emscripten.org/ and then run:
+#### JavaScript
+
+The recommended way to build the JavaScript bindings is using
+[Nix](https://nixos.org/).  With Nix, you can run
+
+```bash
+nix build .\#javascript
+```
+
+to build the bindings.
+
+If you do not have Nix you can, install emscripten from https://emscripten.org/
+and then run:
 
 ```bash
 make js
 ```
 
-Note that if you run emscripten in a docker container, you need to pass through
+Emscripten can also be run via Docker, in which case, you need to pass through
 the EMCC_CLOSURE_ARGS environment variable.
 
+#### Android
+
 To build the android project for Android bindings, run:
 
 ```bash
 cd android
-./gradlew clean assembleRelease
+./gradlew clean build
 ```
 
+#### Objective-C
+
 To build the Xcode workspace for Objective-C bindings, run:
 
 ```bash
@@ -65,7 +144,9 @@ pod install
 open OLMKit.xcworkspace
 ```
 
-To build the Python bindings, first build olm as a shared library as above, and
+#### Python
+
+To build the Python 3 bindings, first build olm as a library as above, and
 then run:
 
 ```bash
@@ -73,9 +154,6 @@ cd python
 make
 ```
 
-to make both the Python 2 and Python 3 bindings.  To make only one version, use
-``make olm-python2`` or ``make olm-python3`` instead of just ``make``.
-
 ### Using make instead of cmake
 
 **WARNING:** Using cmake is the preferred method for building the olm library;
@@ -119,6 +197,8 @@ repository, some bindings are (in alphabetical order):
 - [nim-olm](https://codeberg.org/BarrOff/nim-olm) (MIT) Nim bindings
 - [olm-sys](https://gitlab.gnome.org/BrainBlasted/olm-sys) (Apache-2.0) Rust
   bindings
+- [Trixnity](https://gitlab.com/trixnity/trixnity) (Apache-2.0) Kotlin SDK for
+  Matrix, including Olm bindings
 
 Note that bindings may have a different license from libolm, and are *not*
 endorsed by the Matrix.org Foundation C.I.C.
@@ -126,7 +206,7 @@ endorsed by the Matrix.org Foundation C.I.C.
 ## Release process
 
 First: bump version numbers in ``common.mk``, ``CMakeLists.txt``,
-``javascript/package.json``, ``python/olm/__version__.py``, ``OLMKit.podspec``,
+``javascript/package.json``, ``python/pyproject.toml``, ``OLMKit.podspec``,
 ``Package.swift``, and ``android/gradle.properties``.
 
 Also, ensure the changelog is up to date, and that everything is committed to
@@ -168,13 +248,13 @@ documentation contains instructions on how to set up twine (Python) and npm
 (JavaScript) to upload to the registry.
 
 To publish the Android library to MavenCentral (you will need some secrets), in the /android folder:
- - Run the command `./gradlew clean publish --no-daemon --no-parallel --stacktrace`.
+ - Run the command `./gradlew clean build publish --no-daemon --no-parallel --stacktrace`. The generated AAR must be approx 500 kb.
  - Connect to https://s01.oss.sonatype.org
  - Click on Staging Repositories and check the the files have been uploaded
  - Click on close
  - Wait (check Activity tab until step "Repository closed" is displayed)
  - Click on release. The staging repository will disappear
- - Check that the release is available in https://repo1.maven.org/maven2/org/matrix/android/olm/ (it can take a few minutes)
+ - Check that the release is available in https://repo1.maven.org/maven2/org/matrix/android/olm-sdk/ (it can take a few minutes)
 
 ## Design
 

Windows64.cmake

@@ -11,8 +11,8 @@
 SET(CMAKE_SYSTEM_NAME Windows)
 
 # which compilers to use for C and C++
-SET(CMAKE_C_COMPILER x86_64-w64-mingw32-gcc)
-SET(CMAKE_CXX_COMPILER x86_64-w64-mingw32-g++)
+SET(CMAKE_C_COMPILER x86_64-w64-mingw32-gcc-posix)
+SET(CMAKE_CXX_COMPILER x86_64-w64-mingw32-g++-posix)
 SET(CMAKE_RC_COMPILER x86_64-w64-mingw32-windres)
 
 # here is the target environment located

android/gradle.properties

@@ -26,7 +26,7 @@ org.gradle.configureondemand=false
 # Ref: https://github.com/vanniktech/gradle-maven-publish-plugin
 GROUP=org.matrix.android
 POM_ARTIFACT_ID=olm
-VERSION_NAME=3.2.9
+VERSION_NAME=3.2.16
 
 POM_PACKAGING=aar
 

android/olm-sdk/src/androidTest/java/org/matrix/olm/OlmSessionTest.java

@@ -1011,4 +1011,75 @@ public class OlmSessionTest {
         assertTrue(bobSession.isReleased());
     }
 
+    @Test
+    public void test07AliceBobSessionDescribe() {
+        // creates alice & bob accounts
+        OlmAccount aliceAccount = null;
+        OlmAccount bobAccount = null;
+        try {
+            aliceAccount = new OlmAccount();
+            bobAccount = new OlmAccount();
+        } catch (OlmException e) {
+            fail(e.getMessage());
+        }
+
+        // test accounts creation
+        assertTrue(0 != bobAccount.getOlmAccountId());
+        assertTrue(0 != aliceAccount.getOlmAccountId());
+
+        // CREATE ALICE SESSION
+
+        OlmSession aliceSession = null;
+        try {
+            aliceSession = new OlmSession();
+        } catch (OlmException e) {
+            fail("Exception Msg=" + e.getMessage());
+        }
+        assertTrue(0 != aliceSession.getOlmSessionId());
+
+        // CREATE ALICE SESSION
+        OlmSession bobSession = null;
+        try {
+            bobSession = new OlmSession();
+        } catch (OlmException e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+        assertTrue(0 != bobSession.getOlmSessionId());
+
+        String aliceSessionDescribe = null;
+        try {
+            aliceSessionDescribe = aliceSession.sessionDescribe();
+        } catch (Exception e) {
+            fail(e.getMessage());
+        }
+
+        assertNotNull(aliceSessionDescribe);
+
+        String bobSessionDescribe = null;
+        try {
+            bobSessionDescribe = bobSession.sessionDescribe();
+        } catch (Exception e) {
+            fail(e.getMessage());
+        }
+        assertNotNull(bobSessionDescribe);
+
+        // must be the same for both ends of the conversation
+        assertEquals(aliceSessionDescribe, bobSessionDescribe);
+
+        assertEquals(
+            "sender chain index: 0 receiver chain indices: skipped message keys:",
+            aliceSessionDescribe
+        );
+
+        aliceAccount.releaseAccount();
+        bobAccount.releaseAccount();
+        assertTrue(aliceAccount.isReleased());
+        assertTrue(bobAccount.isReleased());
+
+        bobSession.releaseSession();
+        aliceSession.releaseSession();
+        assertTrue(bobSession.isReleased());
+        assertTrue(aliceSession.isReleased());
+    }
 }

android/olm-sdk/src/main/java/org/matrix/olm/OlmException.java

@@ -60,6 +60,7 @@ public class OlmException extends IOException {
     public static final int EXCEPTION_CODE_SESSION_ENCRYPT_MESSAGE = 404;
     public static final int EXCEPTION_CODE_SESSION_DECRYPT_MESSAGE = 405;
     public static final int EXCEPTION_CODE_SESSION_SESSION_IDENTIFIER = 406;
+    public static final int EXCEPTION_CODE_SESSION_SESSION_DESCRIBE = 407;
 
     public static final int EXCEPTION_CODE_UTILITY_CREATION = 500;
     public static final int EXCEPTION_CODE_UTILITY_VERIFY_SIGNATURE = 501;

android/olm-sdk/src/main/java/org/matrix/olm/OlmInboundGroupSession.java

@@ -369,4 +369,29 @@ public class OlmInboundGroupSession extends CommonSerializeUtils implements Seri
      * @return the deserialized session
      **/
     private native long deserializeJni(byte[] aSerializedData, byte[] aKey);
+
+    /**
+     * Return a pickled inbound group session as a bytes buffer.<br>
+     * The session is serialized and encrypted with aKey.
+     * In case of failure, an error human readable
+     * description is provide in aErrorMsg.
+     * @param aKey encryption key
+     * @param aErrorMsg error message description
+     * @return the pickled inbound group session as bytes buffer
+     */
+    public byte[] pickle(byte[] aKey, StringBuffer aErrorMsg) {
+        return serialize(aKey, aErrorMsg);
+    }
+
+    /**
+     * Loads an inbound group session from a pickled bytes buffer.<br>
+     * See {@link #serialize(byte[], StringBuffer)}
+     * @param aSerializedData bytes buffer
+     * @param aKey key used to encrypted
+     * @exception Exception the exception
+     */
+    public void unpickle(byte[] aSerializedData, byte[] aKey) throws Exception {
+        deserialize(aSerializedData, aKey);
+    }
+
 }

android/olm-sdk/src/main/java/org/matrix/olm/OlmOutboundGroupSession.java

@@ -293,4 +293,28 @@ public class OlmOutboundGroupSession extends CommonSerializeUtils implements Ser
      **/
     private native long deserializeJni(byte[] aSerializedData, byte[] aKey);
 
+    /**
+     * Return a pickled outbound group session as a bytes buffer.<br>
+     * The session is serialized and encrypted with aKey.
+     * In case of failure, an error human readable
+     * description is provide in aErrorMsg.
+     * @param aKey encryption key
+     * @param aErrorMsg error message description
+     * @return the pickled outbound group session as bytes buffer
+     */
+    public byte[] pickle(byte[] aKey, StringBuffer aErrorMsg) {
+        return serialize(aKey, aErrorMsg);
+    }
+
+    /**
+     * Loads an outbound group session from a pickled bytes buffer.<br>
+     * See {@link #serialize(byte[], StringBuffer)}
+     * @param aSerializedData bytes buffer
+     * @param aKey key used to encrypted
+     * @exception Exception the exception
+     */
+    public void unpickle(byte[] aSerializedData, byte[] aKey) throws Exception {
+        deserialize(aSerializedData, aKey);
+    }
+
 }

android/olm-sdk/src/main/java/org/matrix/olm/OlmSAS.java

@@ -106,6 +106,16 @@ public class OlmSAS {
         return null;
     }
 
+    public String calculateMacFixedBase64(String message, String info) throws OlmException {
+        try {
+            byte[] bytes = calculateMacFixedBase64Jni(message.getBytes("UTF-8"), info.getBytes("UTF-8"));
+            if (bytes != null) return new String(bytes, "UTF-8");
+        } catch (UnsupportedEncodingException e) {
+            throw new OlmException(OlmException.EXCEPTION_CODE_SAS_ERROR, e.getMessage());
+        }
+        return null;
+    }
+
     public String calculateMacLongKdf(String message, String info) throws OlmException {
         try {
             byte[] bytes = calculateMacLongKdfJni(message.getBytes("UTF-8"), info.getBytes("UTF-8"));
@@ -140,6 +150,8 @@ public class OlmSAS {
 
     private native byte[] calculateMacJni(byte[] message, byte[] info);
 
+    private native byte[] calculateMacFixedBase64Jni(byte[] message, byte[] info);
+
     private native byte[] calculateMacLongKdfJni(byte[] message, byte[] info);
 
     /**

android/olm-sdk/src/main/java/org/matrix/olm/OlmSession.java

@@ -223,6 +223,23 @@ public class OlmSession extends CommonSerializeUtils implements Serializable {
      */
     private native byte[] getSessionIdentifierJni();
 
+    public String sessionDescribe() throws OlmException {
+        try {
+            byte[] buffer = olmSessionDescribeJni();
+
+            if (null != buffer) {
+                return new String(buffer, "UTF-8");
+            }
+        } catch (Exception e) {
+            Log.e(LOG_TAG, "## sessionDescribe(): " + e.getMessage());
+            throw new OlmException(OlmException.EXCEPTION_CODE_SESSION_SESSION_DESCRIBE, e.getMessage());
+        }
+
+        return null;
+    }
+
+    private native byte[] olmSessionDescribeJni();
+
     /**
      * Checks if the PRE_KEY({@link OlmMessage#MESSAGE_TYPE_PRE_KEY}) message is for this in-bound session.<br>
      * This API may be used to process a "m.room.encrypted" event when type = 1 (PRE_KEY).
@@ -448,5 +465,30 @@ public class OlmSession extends CommonSerializeUtils implements Serializable {
      * @return the deserialized session
      **/
     private native long deserializeJni(byte[] aSerializedData, byte[] aKey);
+
+    /**
+     * Return a pickled session as a bytes buffer.<br>
+     * The session is serialized and encrypted with aKey.
+     * In case of failure, an error human readable
+     * description is provide in aErrorMsg.
+     * @param aKey encryption key
+     * @param aErrorMsg error message description
+     * @return the pickled session as bytes buffer
+     */
+    public byte[] pickle(byte[] aKey, StringBuffer aErrorMsg) {
+        return serialize(aKey, aErrorMsg);
+    }
+
+    /**
+     * Loads a session from a pickled bytes buffer.<br>
+     * See {@link #serialize(byte[], StringBuffer)}
+     * @param aSerializedData bytes buffer
+     * @param aKey key used to encrypted
+     * @exception Exception the exception
+     */
+    public void unpickle(byte[] aSerializedData, byte[] aKey) throws Exception {
+        deserialize(aSerializedData, aKey);
+    }
+
 }
 

android/olm-sdk/src/main/jni/olm_sas.cpp

@@ -309,6 +309,86 @@ JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(calculateMacJni)(JNIEnv *env, jobject thiz
     return returnValue;
 }
 
+JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(calculateMacFixedBase64Jni)(JNIEnv *env, jobject thiz,jbyteArray messageBuffer,jbyteArray infoBuffer) {
+    LOGD("## calculateMacFixedBase64Jni(): IN");
+    const char* errorMessage = NULL;
+    jbyteArray returnValue = 0;
+    OlmSAS* sasPtr = getOlmSasInstanceId(env, thiz);
+
+    jbyte *messagePtr = NULL;
+    jboolean messageWasCopied = JNI_FALSE;
+
+    jbyte *infoPtr = NULL;
+    jboolean infoWasCopied = JNI_FALSE;
+
+    if (!sasPtr)
+    {
+        LOGE("## calculateMacFixedBase64Jni(): failure - invalid SAS ptr=NULL");
+        errorMessage = "invalid SAS ptr=NULL";
+    } else if(!messageBuffer) {
+        LOGE("## calculateMacFixedBase64Jni(): failure - invalid message");
+        errorMessage = "invalid info";
+    }
+    else if (!(messagePtr = env->GetByteArrayElements(messageBuffer, &messageWasCopied)))
+    {
+        LOGE(" ## calculateMacFixedBase64Jni(): failure - message JNI allocation OOM");
+        errorMessage = "message JNI allocation OOM";
+    }
+    else if (!(infoPtr = env->GetByteArrayElements(infoBuffer, &infoWasCopied)))
+    {
+        LOGE(" ## calculateMacFixedBase64Jni(): failure - info JNI allocation OOM");
+        errorMessage = "info JNI allocation OOM";
+    } else {
+
+        size_t infoLength = (size_t)env->GetArrayLength(infoBuffer);
+        size_t messageLength = (size_t)env->GetArrayLength(messageBuffer);
+        size_t macLength = olm_sas_mac_length(sasPtr);
+
+        void *macPtr = malloc(macLength*sizeof(uint8_t));
+
+        size_t result = olm_sas_calculate_mac_fixed_base64(sasPtr,messagePtr,messageLength,infoPtr,infoLength,macPtr,macLength);
+        if (result == olm_error())
+        {
+            errorMessage = (const char *)olm_sas_last_error(sasPtr);
+            LOGE("## calculateMacFixedBase64Jni(): failure - error calculating SAS mac Msg=%s", errorMessage);
+        }
+        else
+        {
+            returnValue = env->NewByteArray(macLength);
+            env->SetByteArrayRegion(returnValue, 0 , macLength, (jbyte*)macPtr);
+        }
+
+        if (macPtr) {
+            free(macPtr);
+        }
+    }
+
+    // free alloc
+    if (infoPtr)
+    {
+        if (infoWasCopied)
+        {
+            memset(infoPtr, 0, (size_t)env->GetArrayLength(infoBuffer));
+        }
+        env->ReleaseByteArrayElements(infoBuffer, infoPtr, JNI_ABORT);
+    }
+    if (messagePtr)
+    {
+        if (messageWasCopied)
+        {
+            memset(messagePtr, 0, (size_t)env->GetArrayLength(messageBuffer));
+        }
+        env->ReleaseByteArrayElements(messageBuffer, messagePtr, JNI_ABORT);
+    }
+
+    if (errorMessage)
+    {
+        env->ThrowNew(env->FindClass("java/lang/Exception"), errorMessage);
+    }
+
+    return returnValue;
+}
+
 JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(calculateMacLongKdfJni)(JNIEnv *env, jobject thiz,jbyteArray messageBuffer,jbyteArray infoBuffer) {
     LOGD("## calculateMacLongKdfJni(): IN");
     const char* errorMessage = NULL;
@@ -387,4 +467,4 @@ JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(calculateMacLongKdfJni)(JNIEnv *env, jobje
     }
 
     return returnValue;
-}
+}

android/olm-sdk/src/main/jni/olm_sas.h

@@ -32,6 +32,7 @@ JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(getPubKeyJni)(JNIEnv *env, jobject thiz);
 JNIEXPORT void OLM_SAS_FUNC_DEF(setTheirPubKey)(JNIEnv *env, jobject thiz,jbyteArray pubKey);
 JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(generateShortCodeJni)(JNIEnv *env, jobject thiz, jbyteArray infoStringBytes, jint byteNb);
 JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(calculateMacJni)(JNIEnv *env, jobject thiz, jbyteArray messageBuffer, jbyteArray infoBuffer);
+JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(calculateMacFixedBase64Jni)(JNIEnv *env, jobject thiz, jbyteArray messageBuffer, jbyteArray infoBuffer);
 JNIEXPORT jbyteArray OLM_SAS_FUNC_DEF(calculateMacLongKdfJni)(JNIEnv *env, jobject thiz, jbyteArray messageBuffer, jbyteArray infoBuffer);
 
 #ifdef __cplusplus

android/olm-sdk/src/main/jni/olm_session.cpp

@@ -798,6 +798,58 @@ JNIEXPORT jbyteArray OLM_SESSION_FUNC_DEF(getSessionIdentifierJni)(JNIEnv *env,
      return returnValue;
 }
 
+JNIEXPORT jbyteArray OLM_SESSION_FUNC_DEF(olmSessionDescribeJni(JNIEnv *env, jobject thiz))
+{
+    const char* errorMessage = NULL;
+    jbyteArray returnValue = 0;
+
+    LOGD("## olmSessionDescribeJni(): IN ");
+
+    OlmSession *sessionPtr = getSessionInstanceId(env, thiz);
+
+    if (!sessionPtr)
+    {
+        LOGE("## olmSessionDescribeJni(): failure - invalid Session ptr=NULL");
+        errorMessage = "invalid Session ptr=NULL";
+    }
+    else
+    {
+        int maxLength = 600;
+        char* describePtr = NULL;
+        describePtr = (char*) malloc(maxLength * sizeof *describePtr);
+        if (!describePtr)
+        {
+            LOGE("## olmSessionDescribeJni(): failure - describe allocation OOM");
+            errorMessage = "describe allocation OOM";
+        }
+        else
+        {
+            olm_session_describe(sessionPtr, describePtr, maxLength);
+            int length = strlen(describePtr);
+            if (length == 0)
+            {
+                LOGE("## olmSessionDescribeJni(): failure - get session describe");
+            }
+            else
+            {
+                LOGD("## olmSessionDescribeJni(): success - describe=%.*s", (char*)describePtr);
+
+                returnValue = env->NewByteArray(length);
+                env->SetByteArrayRegion(returnValue, 0, length, (jbyte*)describePtr);
+            }
+
+            free(describePtr);
+        }
+    }
+
+    if (errorMessage)
+    {
+        env->ThrowNew(env->FindClass("java/lang/Exception"), errorMessage);
+    }
+
+    return returnValue;
+}
+
 /**
  * Serialize and encrypt session instance.<br>
  * An exception is thrown if the operation fails.

android/olm-sdk/src/main/jni/olm_session.h

@@ -47,6 +47,7 @@ JNIEXPORT jbyteArray OLM_SESSION_FUNC_DEF(encryptMessageJni)(JNIEnv *env, jobjec
 JNIEXPORT jbyteArray OLM_SESSION_FUNC_DEF(decryptMessageJni)(JNIEnv *env, jobject thiz, jobject aEncryptedMsg);
 
 JNIEXPORT jbyteArray OLM_SESSION_FUNC_DEF(getSessionIdentifierJni)(JNIEnv *env, jobject thiz);
+JNIEXPORT jbyteArray OLM_SESSION_FUNC_DEF(olmSessionDescribeJni)(JNIEnv *env, jobject thiz);
 
 // serialization
 JNIEXPORT jbyteArray OLM_SESSION_FUNC_DEF(serializeJni)(JNIEnv *env, jobject thiz, jbyteArray aKey);

common.mk

@@ -1,4 +1,4 @@
 
 MAJOR := 3
 MINOR := 2
-PATCH := 9
+PATCH := 16

docs/megolm.md

@@ -109,7 +109,7 @@ discriminate between sessions.
 ### Sharing session data
 
 To allow other participants in the conversation to decrypt messages, the
-session data is formatted as described in [Session-sharing format](#Session-sharing-format). It is then
+session data is formatted as described in [Session-sharing format](#session-sharing-format). It is then
 shared with other participants in the conversation via a secure peer-to-peer
 channel (such as that provided by [Olm][]).
 
@@ -182,9 +182,13 @@ but the decision of which ratchet states to cache is left to the application.
 
 ## Data exchange formats
 
-### Session-sharing format
+### Session sharing format
 
-The Megolm key-sharing format is as follows:
+This format is used for the initial sharing of a Megolm session with other
+group participants who need to be able to read messages encrypted by this
+session.
+
+The session sharing format is as follows:
 
 ```
 +---+----+--------+--------+--------+--------+------+-----------+
@@ -202,6 +206,33 @@ part of the Ed25519 keypair $`K`$.
 The data is then signed using the Ed25519 keypair, and the 64-byte signature is
 appended.
 
+### Session export format
+
+Once the session is initially shared with the group participants, each
+participant needs to retain a copy of the session if they want to maintain
+their ability to decrypt messages encrypted with that session.
+
+For forward-secrecy purposes, a participant may choose to store a ratcheted
+version of the session. But since the ratchet index is covered by the
+signature, this would invalidate the signature. So we define a similar format,
+called the *session export format*, which is identical to the [session sharing
+format](#session-sharing-format) except for dropping the signature.
+
+The Megolm session export format is thus as follows:
+
+```
++---+----+--------+--------+--------+--------+------+
+| V | i  | R(i,0) | R(i,1) | R(i,2) | R(i,3) | Kpub |
++---+----+--------+--------+--------+--------+------+
+0   1    5        37       69      101      133    165   bytes
+```
+
+The version byte, ``V``, is ``"\x01"``.
+
+This is followed by the ratchet index, $`i`$, which is encoded as a
+big-endian 32-bit integer; the ratchet values $`R_{i,j}`$; and the public
+part of the Ed25519 keypair $`K`$.
+
 ### Message format
 
 Megolm messages consist of a one byte version, followed by a variable length

flake.lock

@@ -0,0 +1,60 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1664871473,
+        "narHash": "sha256-1LzbW6G6Uz8akWiOdlIi435GAm1ct5jF5tovw/9to0o=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "b7a6fde153d9470afdb6aa1da51c4117f03b84ed",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "npmlock2nix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1654775747,
+        "narHash": "sha256-9pXHDpIjmsK5390wmpGHu9aA4QOPpegPBvThHeBlef4=",
+        "owner": "nix-community",
+        "repo": "npmlock2nix",
+        "rev": "5c4f247688fc91d665df65f71c81e0726621aaa8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "npmlock2nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "npmlock2nix": "npmlock2nix"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,40 @@
+{
+  description = "An implementation of the Double Ratchet cryptographic ratchet";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  # We can't use the current stable release because of
+  # https://github.com/emscripten-core/emscripten/issues/16913
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.npmlock2nix = {
+    url = "github:nix-community/npmlock2nix";
+    flake = false;
+  };
+
+  outputs = { self, nixpkgs, flake-utils, npmlock2nix }:
+    let
+      localOverlay = import ./nix/overlay.nix;
+      pkgsForSystem = system: import nixpkgs {
+        inherit system;
+        overlays = [
+          (final: prev: {
+            npmlock2nix = final.callPackage npmlock2nix {};
+            node_modules = final.npmlock2nix.node_modules { src = ./javascript; };
+          })
+          localOverlay
+        ];
+      };
+    in (
+      # some systems cause issues, e.g. i686-linux is unsupported by gradle,
+      # which causes "nix flake check" to fail. Investigate more later, but for
+      # now, we will just allow x86_64-linux
+      flake-utils.lib.eachSystem [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" ] (system: rec {
+        legacyPackages = pkgsForSystem system;
+        checks = {
+          inherit (legacyPackages) olm-gcc-cmake olm-clang-cmake olm-gcc-make;
+        };
+        packages = {
+          javascript = legacyPackages.olm-javascript;
+        };
+      }
+    ));
+}

gitlab-math.lua

@@ -0,0 +1,17 @@
+function Math(el)
+    if el.mathtype == "InlineMath" then
+        if el.text:sub(1,1) == '`' and el.text:sub(#el.text) == '`' then
+            local text = el.text:sub(2,#el.text-1)
+            return pandoc.Math(el.mathtype, text)
+        else
+            local cont = pandoc.read(el.text)
+            return { pandoc.Str("$") } .. cont.blocks[1].content .. { pandoc.Str("$") }
+        end
+    end
+end
+
+function CodeBlock(el)
+    if el.classes[1] == "math" then
+        return pandoc.Para({ pandoc.Math("DisplayMath", el.text) })
+    end
+end

javascript/index.d.ts

@@ -18,28 +18,28 @@ export as namespace Olm;
 
 declare class Account {
     constructor();
-    free();
-    create();
+    free(): void;
+    create(): void;
     identity_keys(): string;
     sign(message: string | Uint8Array): string;
     one_time_keys(): string;
-    mark_keys_as_published();
+    mark_keys_as_published(): void;
     max_number_of_one_time_keys(): number;
-    generate_one_time_keys(number_of_keys: number);
-    remove_one_time_keys(session: Session);
-    generate_fallback_key();
+    generate_one_time_keys(number_of_keys: number): void;
+    remove_one_time_keys(session: Session): void;
+    generate_fallback_key(): void;
     fallback_key(): string;
     unpublished_fallback_key(): string;
     forget_old_fallback_key(): void;
     pickle(key: string | Uint8Array): string;
-    unpickle(key: string | Uint8Array, pickle: string);
+    unpickle(key: string | Uint8Array, pickle: string): void;
 }
 
 declare class Session {
     constructor();
     free(): void;
     pickle(key: string | Uint8Array): string;
-    unpickle(key: string | Uint8Array, pickle: string);
+    unpickle(key: string | Uint8Array, pickle: string): void;
     create_outbound(
         account: Account, their_identity_key: string, their_one_time_key: string,
     ): void;
@@ -51,7 +51,10 @@ declare class Session {
     has_received_message(): boolean;
     matches_inbound(one_time_key_message: string): boolean;
     matches_inbound_from(identity_key: string, one_time_key_message: string): boolean;
-    encrypt(plaintext: string): object;
+    encrypt(plaintext: string): {
+        type: 0 | 1; // 0: PreKey, 1: Message
+        body: string;
+    };
     decrypt(message_type: number, message: string): string;
     describe(): string;
 }
@@ -67,10 +70,13 @@ declare class InboundGroupSession {
     constructor();
     free(): void;
     pickle(key: string | Uint8Array): string;
-    unpickle(key: string | Uint8Array, pickle: string);
+    unpickle(key: string | Uint8Array, pickle: string): void;
     create(session_key: string): string;
     import_session(session_key: string): string;
-    decrypt(message: string): object;
+    decrypt(message: string): {
+        message_index: number;
+        plaintext: string;
+    };
     session_id(): string;
     first_known_index(): number;
     export_session(message_index: number): string;
@@ -80,7 +86,7 @@ declare class OutboundGroupSession {
     constructor();
     free(): void;
     pickle(key: string | Uint8Array): string;
-    unpickle(key: string | Uint8Array, pickle: string);
+    unpickle(key: string | Uint8Array, pickle: string): void;
     create(): void;
     encrypt(plaintext: string): string;
     session_id(): string;
@@ -92,7 +98,11 @@ declare class PkEncryption {
     constructor();
     free(): void;
     set_recipient_key(key: string): void;
-    encrypt(plaintext: string): object;
+    encrypt(plaintext: string): {
+        ciphertext: string;
+        mac: string;
+        ephemeral: string;
+    };
 }
 
 declare class PkDecryption {
@@ -121,6 +131,7 @@ declare class SAS {
     set_their_key(their_key: string): void;
     generate_bytes(info: string, length: number): Uint8Array;
     calculate_mac(input: string, info: string): string;
+    calculate_mac_fixed_base64(input: string, info: string): string;
     calculate_mac_long_kdf(input: string, info: string): string;
 }
 

javascript/olm_inbound_group_session.js

@@ -1,3 +1,4 @@
+/** @constructor */
 function InboundGroupSession() {
     var size = Module['_olm_inbound_group_session_size']();
     this.buf = malloc(size);

javascript/olm_outbound_group_session.js

@@ -1,3 +1,4 @@
+/** @constructor */
 function OutboundGroupSession() {
     var size = Module['_olm_outbound_group_session_size']();
     this.buf = malloc(size);

javascript/olm_pk.js

@@ -1,3 +1,4 @@
+/** @constructor */
 function PkEncryption() {
     var size = Module['_olm_pk_encryption_size']();
     this.buf = malloc(size);
@@ -98,6 +99,7 @@ PkEncryption.prototype['encrypt'] = restore_stack(function(
 });
 
 
+/** @constructor */
 function PkDecryption() {
     var size = Module['_olm_pk_decryption_size']();
     this.buf = malloc(size);
@@ -273,6 +275,7 @@ PkDecryption.prototype['decrypt'] = restore_stack(function (
 })
 
 
+/** @constructor */
 function PkSigning() {
     var size = Module['_olm_pk_signing_size']();
     this.buf = malloc(size);

javascript/olm_post.js

@@ -44,6 +44,7 @@ function bzero(ptr, n) {
     }
 }
 
+/** @constructor */
 function Account() {
     var size = Module['_olm_account_size']();
     this.buf = malloc(size);
@@ -244,6 +245,7 @@ Account.prototype['unpickle'] = restore_stack(function(key, pickle) {
     }
 });
 
+/** @constructor */
 function Session() {
     var size = Module['_olm_session_size']();
     this.buf = malloc(size);
@@ -530,6 +532,7 @@ Session.prototype['describe'] = restore_stack(function() {
     }
 });
 
+/** @constructor */
 function Utility() {
     var size = Module['_olm_utility_size']();
     this.buf = malloc(size);

javascript/olm_pre.js

@@ -14,7 +14,6 @@ if (typeof(window) !== 'undefined') {
         var bytes = nodeCrypto['randomBytes'](buf.length);
         buf.set(bytes);
     };
-    process = global["process"];
 } else {
     throw new Error("Cannot find global to attach library to");
 }

javascript/olm_sas.js

@@ -1,3 +1,4 @@
+/** @constructor */
 function SAS() {
     var size = Module['_olm_sas_size']();
     var random_length = Module['_olm_create_sas_random_length']();
@@ -82,6 +83,22 @@ SAS.prototype['calculate_mac'] = restore_stack(function(input, info) {
     return UTF8ToString(mac_buffer, mac_length);
 });
 
+SAS.prototype['calculate_mac_fixed_base64'] = restore_stack(function(input, info) {
+    var input_array = array_from_string(input);
+    var input_buffer = stack(input_array);
+    var info_array = array_from_string(info);
+    var info_buffer = stack(info_array);
+    var mac_length = sas_method(Module['_olm_sas_mac_length'])(this.ptr);
+    var mac_buffer = stack(mac_length + NULL_BYTE_PADDING_LENGTH);
+    sas_method(Module['_olm_sas_calculate_mac_fixed_base64'])(
+        this.ptr,
+        input_buffer, input_array.length,
+        info_buffer, info_array.length,
+        mac_buffer, mac_length
+    );
+    return UTF8ToString(mac_buffer, mac_length);
+});
+
 SAS.prototype['calculate_mac_long_kdf'] = restore_stack(function(input, info) {
     var input_array = array_from_string(input);
     var input_buffer = stack(input_array);

javascript/package-lock.json

@@ -0,0 +1,241 @@
+{
+  "name": "@matrix-org/olm",
+  "version": "3.2.11",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "@matrix-org/olm",
+      "version": "3.2.11",
+      "license": "Apache-2.0",
+      "devDependencies": {
+        "jasmine": "^3.0.0"
+      }
+    },
+    "node_modules/balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+      "dev": true
+    },
+    "node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
+    },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
+    },
+    "node_modules/glob": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
+      "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
+    },
+    "node_modules/jasmine": {
+      "version": "3.99.0",
+      "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.99.0.tgz",
+      "integrity": "sha512-YIThBuHzaIIcjxeuLmPD40SjxkEcc8i//sGMDKCgkRMVgIwRJf5qyExtlJpQeh7pkeoBSOe6lQEdg+/9uKg9mw==",
+      "dev": true,
+      "dependencies": {
+        "glob": "^7.1.6",
+        "jasmine-core": "~3.99.0"
+      },
+      "bin": {
+        "jasmine": "bin/jasmine.js"
+      }
+    },
+    "node_modules/jasmine-core": {
+      "version": "3.99.1",
+      "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.99.1.tgz",
+      "integrity": "sha512-Hu1dmuoGcZ7AfyynN3LsfruwMbxMALMka+YtZeGoLuDEySVmVAPaonkNoBRIw/ectu8b9tVQCJNgp4a4knp+tg==",
+      "dev": true
+    },
+    "node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
+    }
+  },
+  "dependencies": {
+    "balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+      "dev": true
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "requires": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
+    },
+    "glob": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
+      "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
+      "dev": true,
+      "requires": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
+      "requires": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
+    },
+    "jasmine": {
+      "version": "3.99.0",
+      "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.99.0.tgz",
+      "integrity": "sha512-YIThBuHzaIIcjxeuLmPD40SjxkEcc8i//sGMDKCgkRMVgIwRJf5qyExtlJpQeh7pkeoBSOe6lQEdg+/9uKg9mw==",
+      "dev": true,
+      "requires": {
+        "glob": "^7.1.6",
+        "jasmine-core": "~3.99.0"
+      }
+    },
+    "jasmine-core": {
+      "version": "3.99.1",
+      "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.99.1.tgz",
+      "integrity": "sha512-Hu1dmuoGcZ7AfyynN3LsfruwMbxMALMka+YtZeGoLuDEySVmVAPaonkNoBRIw/ectu8b9tVQCJNgp4a4knp+tg==",
+      "dev": true
+    },
+    "minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "requires": {
+        "brace-expansion": "^1.1.7"
+      }
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
+    }
+  }
+}

javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matrix-org/olm",
-  "version": "3.2.9",
+  "version": "3.2.16",
   "description": "An implementation of the Double Ratchet cryptographic ratchet",
   "main": "olm.js",
   "files": [
@@ -31,8 +31,5 @@
   "homepage": "https://gitlab.matrix.org/matrix-org/olm",
   "devDependencies": {
     "jasmine": "^3.0.0"
-  },
-  "publishConfig": {
-    "@matrix-org:registry":"https://gitlab.matrix.org/api/v4/projects/27/packages/npm/"
   }
 }

nix/overlay.nix

@@ -0,0 +1,76 @@
+final: prev: {
+  olm-gcc-cmake = prev.gccStdenv.mkDerivation {
+    name = "olm_gcc_cmake";
+
+    src = ./..;
+
+    nativeBuildInputs = [ prev.cmake ];
+    doCheck = true;
+    checkPhase = ''
+      (cd tests && ctest . -j $NIX_BUILD_CORES)
+    '';
+  };
+
+  olm-clang-cmake = prev.clangStdenv.mkDerivation {
+    name = "olm_clang_cmake";
+
+    src = ./..;
+
+    nativeBuildInputs = [ prev.cmake ];
+
+    doCheck = true;
+    checkPhase = ''
+      (cd tests && ctest . -j $NIX_BUILD_CORES)
+    '';
+  };
+
+  olm-gcc-make = prev.gccStdenv.mkDerivation {
+    name = "olm";
+
+    src = ./..;
+
+    doCheck = true;
+    makeFlags = [ "PREFIX=$out" ];
+  };
+
+  olm-javascript = final.buildEmscriptenPackage {
+    pname = "olm_javascript";
+    inherit (builtins.fromJSON (builtins.readFile ../javascript/package.json)) version;
+
+    src = ./..;
+
+    nativeBuildInputs = with prev; [ gnumake python3 nodejs ];
+
+    postPatch = ''
+      patchShebangs .
+    '';
+
+    configurePhase = false;
+
+    buildPhase = ''
+      export EM_CACHE=$TMPDIR
+      make javascript/exported_functions.json
+      make js
+    '';
+
+    installPhase = ''
+      mkdir -p $out/javascript
+      cd javascript
+      echo sha256: > checksums.txt
+      sha256sum olm.js olm_legacy.js olm.wasm >> checksums.txt
+      echo sha512: >> checksums.txt
+      sha512sum olm.js olm_legacy.js olm.wasm >> checksums.txt
+      cp package.json olm.js olm.wasm olm_legacy.js index.d.ts README.md checksums.txt $out/javascript
+      cd ..
+    '';
+
+    checkPhase = ''
+      cd javascript
+      export HOME=$TMPDIR
+      ln -s ${final.node_modules}/node_modules ./node_modules
+      npm test
+      cd ..
+    '';
+  };
+
+}

olm.pc.in

@@ -1,7 +1,7 @@
 prefix=@CMAKE_INSTALL_PREFIX@
 exec_prefix=${prefix}
-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
-libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
 
 Name: olm
 Description: implementation of the Double Ratchet cryptographic ratchet in C++

python/.gitlab-ci.yml

@@ -34,7 +34,7 @@ test:python:
   image: docker.io/python:$PYTHON_VERSIONS
   parallel:
     matrix:
-      - PYTHON_VERSIONS: [ "3.6", "3.7", "3.8", "3.9" ]
+      - PYTHON_VERSIONS: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
   script:
     - pip install tox
     - make headers

python/MANIFEST.in

@@ -1,5 +1,3 @@
-include include/olm/olm.h
-include include/olm/pk.h
-include include/olm/sas.h
+include include/olm/*.h
 include Makefile
 include olm_build.py

python/Makefile

@@ -20,6 +20,9 @@ include/olm/error.h: include/olm/olm.h ../include/olm/error.h
 
 headers: include/olm/olm.h include/olm/pk.h include/olm/sas.h include/olm/error.h
 
+olm-python3: headers
+	DEVELOP=$(DEVELOP) python3 setup.py build
+
 install: install-python3
 
 install-python3: olm-python3

python/README.md

@@ -15,6 +15,18 @@ found [here][6].
 
 The full API reference can be found [here][7].
 
+# Installation instructions
+
+To install from the source package, you will need:
+
+- cmake (recommended) or GNU make
+- a C/C++ compiler
+
+You can then run `pip install python-olm`.
+
+This should work in UNIX-like environments, including macOS, and may work in
+other environments too, but is known to not work yet in Windows.
+
 # Accounts
 
 Accounts create and hold the central identity of the Olm protocol, they consist of a fingerprint and identity
@@ -157,5 +169,5 @@ Pickling works the same way as for peer-to-peer Olm sessions.
 [3]: https://cffi.readthedocs.io/en/latest/
 [4]: https://git.matrix.org/git/olm/about/docs/olm.rst
 [5]: https://git.matrix.org/git/olm/about/docs/megolm.rst
-[6]: https://matrix.org/docs/guides/e2e_implementation.html
+[6]: https://matrix.org/docs/guides/end-to-end-encryption-implementation-guide
 [7]: https://poljar.github.io/python-olm/html/index.html

python/make_sdist.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+
+DIR=$(mktemp -d)
+SRC=$(pwd)
+
+echo "Making headers"
+make headers
+
+cd $DIR
+
+echo "Copying python module"
+cp -a $SRC/* .
+mkdir -p libolm
+echo "Cleaning sources"
+make clean > /dev/null
+cp -a $SRC/include .
+echo "Copying libolm sources"
+for src in cmake CMakeLists.txt common.mk include lib Makefile olm.pc.in src tests; do
+    cp -a $SRC/../$src libolm
+done
+find libolm -name \*~ -delete
+find libolm -name \#\*\# -delete
+
+echo "Building"
+patch -p1 < $SRC/packaging.diff
+python3 -m build -s
+
+echo "Copying result"
+mkdir -p $SRC/dist
+cp dist/* $SRC/dist
+
+echo "Cleaning up"
+cd $SRC
+rm -rf $DIR

python/olm/__version__.py

@@ -1,9 +0,0 @@
-__title__ = "python-olm"
-__description__ = ("python CFFI bindings for the olm "
-                   "cryptographic ratchet library")
-__url__ = "https://github.com/poljar/python-olm"
-__version__ = "3.2.9"
-__author__ = "Damir Jelić"
-__author_email__ = "poljar@termina.org.uk"
-__license__ = "Apache 2.0"
-__copyright__ = "Copyright 2018-2019 Damir Jelić"

python/olm/_finalize.py

@@ -23,7 +23,6 @@
 
 This is designed for avoiding __del__.
 """
-from __future__ import print_function
 
 import sys
 import traceback

python/olm/account.py

@@ -32,8 +32,6 @@ import json
 from builtins import bytes, super
 from typing import AnyStr, Dict, Optional, Type
 
-from future.utils import bytes_to_native_str
-
 # pylint: disable=no-name-in-module
 from _libolm import ffi, lib  # type: ignore
 
@@ -93,8 +91,7 @@ class Account(object):
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(
-            ffi.string((lib.olm_account_last_error(self._account))))
+        last_error = ffi.string((lib.olm_account_last_error(self._account))).decode()
 
         raise OlmAccountError(last_error)
 
@@ -209,7 +206,7 @@ class Account(object):
                 for i in range(0, len(bytes_message)):
                     bytes_message[i] = 0
 
-        return bytes_to_native_str(ffi.unpack(out_buffer, out_length))
+        return ffi.unpack(out_buffer, out_length).decode()
 
     @property
     def max_one_time_keys(self):

python/olm/group_session.py

@@ -28,8 +28,6 @@ Examples:
 from builtins import bytes, super
 from typing import AnyStr, Optional, Tuple, Type
 
-from future.utils import bytes_to_native_str
-
 # pylint: disable=no-name-in-module
 from _libolm import ffi, lib  # type: ignore
 
@@ -171,8 +169,9 @@ class InboundGroupSession(object):
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(ffi.string(
-            lib.olm_inbound_group_session_last_error(self._session)))
+        last_error = ffi.string(
+            lib.olm_inbound_group_session_last_error(self._session)
+        ).decode()
 
         raise OlmGroupSessionError(last_error)
 
@@ -252,7 +251,7 @@ class InboundGroupSession(object):
             id_length
         )
         self._check_error(ret)
-        return bytes_to_native_str(ffi.unpack(id_buffer, id_length))
+        return ffi.unpack(id_buffer, id_length).decode()
 
     @property
     def first_known_index(self):
@@ -290,7 +289,7 @@ class InboundGroupSession(object):
             message_index
         )
         self._check_error(ret)
-        export_str = bytes_to_native_str(ffi.unpack(export_buffer, export_length))
+        export_str = ffi.unpack(export_buffer, export_length).decode()
 
         # clear out copies of the key
         lib.memset(export_buffer, 0, export_length)
@@ -373,9 +372,9 @@ class OutboundGroupSession(object):
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(ffi.string(
+        last_error = ffi.string(
             lib.olm_outbound_group_session_last_error(self._session)
-        ))
+        ).decode()
 
         raise OlmGroupSessionError(last_error)
 
@@ -483,7 +482,7 @@ class OutboundGroupSession(object):
                 for i in range(0, len(byte_plaintext)):
                     byte_plaintext[i] = 0
 
-        return bytes_to_native_str(ffi.unpack(message_buffer, message_length))
+        return ffi.unpack(message_buffer, message_length).decode()
 
     @property
     def id(self):
@@ -499,7 +498,7 @@ class OutboundGroupSession(object):
         )
         self._check_error(ret)
 
-        return bytes_to_native_str(ffi.unpack(id_buffer, id_length))
+        return ffi.unpack(id_buffer, id_length).decode()
 
     @property
     def message_index(self):
@@ -529,4 +528,4 @@ class OutboundGroupSession(object):
         )
         self._check_error(ret)
 
-        return bytes_to_native_str(ffi.unpack(key_buffer, key_length))
+        return ffi.unpack(key_buffer, key_length).decode()

python/olm/pk.py

@@ -36,8 +36,6 @@ Examples:
 from builtins import super
 from typing import AnyStr, Type
 
-from future.utils import bytes_to_native_str
-
 from _libolm import ffi, lib  # type: ignore
 
 from ._compat import URANDOM, to_bytearray, to_unicode_str
@@ -116,8 +114,9 @@ class PkEncryption(object):
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(
-            ffi.string(lib.olm_pk_encryption_last_error(self._pk_encryption)))
+        last_error = ffi.string(
+            lib.olm_pk_encryption_last_error(self._pk_encryption)
+        ).decode()
 
         raise PkEncryptionError(last_error)
 
@@ -166,12 +165,9 @@ class PkEncryption(object):
                     byte_plaintext[i] = 0
 
         message = PkMessage(
-            bytes_to_native_str(
-                ffi.unpack(ephemeral_key, ephemeral_key_size)),
-            bytes_to_native_str(
-                ffi.unpack(mac, mac_length)),
-            bytes_to_native_str(
-                ffi.unpack(ciphertext, ciphertext_length))
+            ffi.unpack(ephemeral_key, ephemeral_key_size).decode(),
+            ffi.unpack(mac, mac_length).decode(),
+            ffi.unpack(ciphertext, ciphertext_length).decode(),
         )
         return message
 
@@ -217,18 +213,19 @@ class PkDecryption(object):
             random_buffer, random_length
         )
         self._check_error(ret)
-        self.public_key = bytes_to_native_str(ffi.unpack(
+        self.public_key: str = ffi.unpack(
             key_buffer,
             key_length
-        ))
+        ).decode()
 
     def _check_error(self, ret):
         # type: (int) -> None
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(
-            ffi.string(lib.olm_pk_decryption_last_error(self._pk_decryption)))
+        last_error = ffi.string(
+            lib.olm_pk_decryption_last_error(self._pk_decryption)
+        ).decode()
 
         raise PkDecryptionError(last_error)
 
@@ -267,7 +264,7 @@ class PkDecryption(object):
 
     @classmethod
     def from_pickle(cls, pickle, passphrase=""):
-        # types: (bytes, str) -> PkDecryption
+        # type: (bytes, str) -> PkDecryption
         """Restore a previously stored PkDecryption object.
 
         Creates a PkDecryption object from a pickled base64 string. Decrypts
@@ -306,15 +303,15 @@ class PkDecryption(object):
             for i in range(0, len(byte_key)):
                 byte_key[i] = 0
 
-        obj.public_key = bytes_to_native_str(ffi.unpack(
+        obj.public_key = ffi.unpack(
             pubkey_buffer,
             pubkey_length
-        ))
+        ).decode()
 
         return obj
 
     def decrypt(self, message, unicode_errors="replace"):
-        # type (PkMessage, str) -> str
+        # type: (PkMessage, str) -> str
         """Decrypt a previously encrypted Pk message.
 
         Returns the decrypted plaintext.
@@ -411,17 +408,14 @@ class PkSigning(object):
 
         self._check_error(ret)
 
-        self.public_key = bytes_to_native_str(
-            ffi.unpack(pubkey_buffer, pubkey_length)
-        )
+        self.public_key = ffi.unpack(pubkey_buffer, pubkey_length).decode()
 
     def _check_error(self, ret):
         # type: (int) -> None
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(
-            ffi.string(lib.olm_pk_signing_last_error(self._pk_signing)))
+        last_error = ffi.string(lib.olm_pk_signing_last_error(self._pk_signing)).decode()
 
         raise PkSigningError(last_error)
 
@@ -456,6 +450,4 @@ class PkSigning(object):
             signature_buffer, signature_length)
         self._check_error(ret)
 
-        return bytes_to_native_str(
-            ffi.unpack(signature_buffer, signature_length)
-        )
+        return ffi.unpack(signature_buffer, signature_length).decode()

python/olm/sas.py

@@ -34,8 +34,6 @@ from builtins import bytes
 from functools import wraps
 from typing import Optional
 
-from future.utils import bytes_to_native_str
-
 from _libolm import ffi, lib
 
 from ._compat import URANDOM, to_bytearray, to_bytes
@@ -92,8 +90,7 @@ class Sas(object):
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(
-            ffi.string((lib.olm_sas_last_error(self._sas))))
+        last_error = ffi.string((lib.olm_sas_last_error(self._sas))).decode()
 
         raise OlmSasError(last_error)
 
@@ -115,7 +112,7 @@ class Sas(object):
             lib.olm_sas_get_pubkey(self._sas, pubkey_buffer, pubkey_length)
         )
 
-        return bytes_to_native_str(ffi.unpack(pubkey_buffer, pubkey_length))
+        return ffi.unpack(pubkey_buffer, pubkey_length).decode()
 
     @property
     def other_key_set(self):
@@ -208,7 +205,41 @@ class Sas(object):
                 mac_length
             )
         )
-        return bytes_to_native_str(ffi.unpack(mac_buffer, mac_length))
+        return ffi.unpack(mac_buffer, mac_length).decode()
+
+    def calculate_mac_fixed_base64(self, message, extra_info):
+        # type: (str, str) -> str
+        """Generate a message authentication code based on the shared secret.
+
+        This function uses a fixed base64 encoding that is compatible with
+        other base64 implementations.
+
+        Args:
+            message (str): The message to produce the authentication code for.
+            extra_info (str): Extra information to mix in when generating the
+                MAC
+
+        Raises OlmSasError on failure.
+
+        """
+        byte_message = to_bytes(message)
+        byte_info = to_bytes(extra_info)
+
+        mac_length = lib.olm_sas_mac_length(self._sas)
+        mac_buffer = ffi.new("char[]", mac_length)
+
+        self._check_error(
+            lib.olm_sas_calculate_mac_fixed_base64(
+                self._sas,
+                ffi.from_buffer(byte_message),
+                len(byte_message),
+                ffi.from_buffer(byte_info),
+                len(byte_info),
+                mac_buffer,
+                mac_length
+            )
+        )
+        return ffi.unpack(mac_buffer, mac_length).decode()
 
     def calculate_mac_long_kdf(self, message, extra_info):
         # type: (str, str) -> str
@@ -242,4 +273,4 @@ class Sas(object):
                 mac_length
             )
         )
-        return bytes_to_native_str(ffi.unpack(mac_buffer, mac_length))
+        return ffi.unpack(mac_buffer, mac_length).decode()

python/olm/session.py

@@ -35,8 +35,6 @@ Examples:
 from builtins import bytes, super
 from typing import AnyStr, Optional, Type
 
-from future.utils import bytes_to_native_str
-
 # pylint: disable=no-name-in-module
 from _libolm import ffi, lib  # type: ignore
 
@@ -146,8 +144,7 @@ class Session(object):
         if ret != lib.olm_error():
             return
 
-        last_error = bytes_to_native_str(
-            ffi.string(lib.olm_session_last_error(self._session)))
+        last_error = ffi.string(lib.olm_session_last_error(self._session)).decode()
 
         raise OlmSessionError(last_error)
 
@@ -260,16 +257,16 @@ class Session(object):
 
         if message_type == lib.OLM_MESSAGE_TYPE_PRE_KEY:
             return OlmPreKeyMessage(
-                bytes_to_native_str(ffi.unpack(
+                ffi.unpack(
                     ciphertext_buffer,
                     ciphertext_length
-                )))
+                ).decode())
         elif message_type == lib.OLM_MESSAGE_TYPE_MESSAGE:
             return OlmMessage(
-                bytes_to_native_str(ffi.unpack(
+                ffi.unpack(
                     ciphertext_buffer,
                     ciphertext_length
-                )))
+                ).decode())
         else:  # pragma: no cover
             raise ValueError("Unknown message type")
 
@@ -340,7 +337,7 @@ class Session(object):
         self._check_error(
             lib.olm_session_id(self._session, id_buffer, id_length)
         )
-        return bytes_to_native_str(ffi.unpack(id_buffer, id_length))
+        return ffi.unpack(id_buffer, id_length).decode()
 
     def matches(self, message, identity_key=None):
         # type: (OlmPreKeyMessage, Optional[AnyStr]) -> bool
@@ -407,7 +404,7 @@ class Session(object):
         lib.olm_session_describe(
             self._session, describe_buffer, buffer_length
         )
-        return bytes_to_native_str(ffi.string(describe_buffer))
+        return ffi.string(describe_buffer).decode()
 
 
 class InboundSession(Session):

python/olm/utility.py

@@ -33,8 +33,6 @@ Examples:
 # pylint: disable=redefined-builtin,unused-import
 from typing import AnyStr, Type
 
-from future.utils import bytes_to_native_str
-
 # pylint: disable=no-name-in-module
 from _libolm import ffi, lib  # type: ignore
 
@@ -123,7 +121,7 @@ class _Utility(object):
 
         cls._check_error(ret, OlmHashError)
 
-        return bytes_to_native_str(ffi.unpack(hash, hash_length))
+        return ffi.unpack(hash, hash_length).decode()
 
 
 def ed25519_verify(key, message, signature):

python/olm_build.py

@@ -15,8 +15,6 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from __future__ import unicode_literals
-
 import os
 import subprocess
 
@@ -28,7 +26,6 @@ PATH = os.path.dirname(__file__)
 DEVELOP = os.environ.get("DEVELOP")
 
 compile_args = ["-I../include"]
-link_args = ["-L../build"]
 
 if DEVELOP and DEVELOP.lower() in ["yes", "true", "1"]:
     link_args.append('-Wl,-rpath=../build')
@@ -46,8 +43,10 @@ ffibuilder.set_source(
         #include <olm/sas.h>
     """,
     libraries=["olm"],
+    library_dirs=[os.path.join("..", "build")],
     extra_compile_args=compile_args,
-    extra_link_args=link_args)
+    source_extension=".cpp", # we need to link the C++ standard library, so use a C++ extension
+)
 
 with open(os.path.join(PATH, "include/olm/error.h")) as f:
     ffibuilder.cdef(f.read(), override=True)

python/packaging.diff

@@ -0,0 +1,56 @@
+--- a/MANIFEST.in
++++ b/MANIFEST.in
+@@ -1,3 +1,8 @@
+ include include/olm/*.h
+-include Makefile
+ include olm_build.py
++include libolm/*
++include libolm/cmake/*
++include libolm/include/olm/*
++recursive-include libolm/lib *
++include libolm/src/*
++recursive-include libolm/tests *
+--- a/olm_build.py
++++ b/olm_build.py
+@@ -25,12 +25,29 @@
+ 
+ DEVELOP = os.environ.get("DEVELOP")
+ 
+-compile_args = ["-I../include"]
++compile_args = ["-Ilibolm/include"]
+ 
+ if DEVELOP and DEVELOP.lower() in ["yes", "true", "1"]:
+     link_args.append('-Wl,-rpath=../build')
+ 
+-headers_build = subprocess.Popen("make headers", shell=True)
+-headers_build.wait()
++# Try to build with cmake first, fall back to GNU make
++try:
++    subprocess.run(
++        ["cmake", ".", "-Bbuild", "-DBUILD_SHARED_LIBS=NO"],
++        cwd="libolm", check=True,
++    )
++    subprocess.run(
++        ["cmake", "--build", "build"],
++        cwd="libolm", check=True,
++    )
++except FileNotFoundError:
++    try:
++        # try "gmake" first because some systems have a non-GNU make
++        # installed as "make"
++        subprocess.run(["gmake", "static"], cwd="libolm", check=True)
++    except FileNotFoundError:
++        # some systems have GNU make installed without the leading "g"
++        # so give that a try (though this may fail if it isn't GNU make)
++        subprocess.run(["make", "static"], cwd="libolm", check=True)
+ 
+ ffibuilder.set_source(
+@@ -43,7 +60,7 @@
+         #include <olm/sas.h>
+     """,
+     libraries=["olm"],
+-    library_dirs=[os.path.join("..", "build")],
++    library_dirs=[os.path.join("libolm", "build")],
+     extra_compile_args=compile_args,
+     source_extension=".cpp", # we need to link the C++ standard library, so use a C++ extension
+ )

python/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools", "cffi>=1.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "python-olm"
+version = "3.2.16"
+description = "python CFFI bindings for the olm cryptographic ratchet library"
+authors = [{name = "Damir Jelić", email = "poljar@termina.org.uk"}]
+license = {text = "Apache-2.0"}
+readme = "README.md"
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Communications",
+]
+dependencies = ["cffi>=1.0.0"]
+
+[project.urls]
+homepage = "https://gitlab.matrix.org/matrix-org/olm/-/tree/master/python"
+
+[tool.setuptools]
+packages = [ "olm" ]

python/requirements.txt

@@ -1,3 +1,2 @@
-future
 cffi
 typing

python/setup.cfg

@@ -3,6 +3,3 @@ testpaths = tests
 flake8-ignore =
     olm/*.py F401
     tests/*.py W503
-
-[coverage:run]
-omit=olm/__version__.py

python/setup.py

@@ -1,31 +1,6 @@
 # -*- coding: utf-8 -*-
-
-import os
-from codecs import open
-
 from setuptools import setup
 
-here = os.path.abspath(os.path.dirname(__file__))
-
-about = {}
-with open(os.path.join(here, "olm", "__version__.py"), "r", "utf-8") as f:
-    exec(f.read(), about)
-
 setup(
-    name=about["__title__"],
-    version=about["__version__"],
-    description=about["__description__"],
-    author=about["__author__"],
-    author_email=about["__author_email__"],
-    url=about["__url__"],
-    license=about["__license__"],
-    packages=["olm"],
-    setup_requires=["cffi>=1.0.0"],
-    cffi_modules=["olm_build.py:ffibuilder"],
-    install_requires=[
-        "cffi>=1.0.0",
-        "future",
-        "typing;python_version<'3.5'"
-    ],
-    zip_safe=False
+    cffi_modules=["olm_build.py:ffibuilder"]
 )

python/tests/utils_test.py

@@ -1,8 +1,6 @@
 import base64
 import hashlib
 
-from future.utils import bytes_to_native_str
-
 from olm import sha256
 from olm._compat import to_bytes
 
@@ -19,7 +17,7 @@ class TestClass(object):
             hashlib.sha256(to_bytes(input1)).digest()
         )
 
-        hashlib_hash = bytes_to_native_str(hashlib_hash[:-1])
+        hashlib_hash = hashlib_hash[:-1].decode()
 
         assert first_hash != second_hash
         assert hashlib_hash == first_hash

python/tox.ini

@@ -6,7 +6,7 @@ envlist = py27,py36,pypy,{py2,py3}-cov,coverage
 deps = -rrequirements.txt
        -rtest-requirements.txt
 
-passenv = TOXENV CI TRAVIS TRAVIS_*
+passenv = TOXENV,CI,TRAVIS,TRAVIS_*
 commands = pytest --benchmark-disable
 usedevelop = True
 

xcode/OLMKit/OLMAccount.h

@@ -35,14 +35,27 @@
 /** Public parts of the unpublished one time keys for the account */
 - (NSDictionary*) oneTimeKeys;
 
-/** Public part of the unpublished fallback key for the account */
+/**
+ * Deprecated use unPublishedFallbackKey
+ */
 - (NSDictionary*) fallbackKey;
 
+/**
+ Public part of the unpublished fallback key for the account, if present and unublished.
+ */
+- (NSDictionary*) unpublishedFallbackKey;
+
 - (BOOL) removeOneTimeKeysForSession:(OLMSession*)session;
 
 /** Marks the current set of one time keys as being published. */
 - (void) markOneTimeKeysAsPublished;
 
+/** Forget about the old fallback key.
+ * This should be called once you are reasonably certain that you will not
+ * receive any more messages that use the old fallback key
+ */
+- (void) forgetFallbackKey;
+
 /** The largest number of one time keys this account can store. */
 - (NSUInteger) maxOneTimeKeys;
 

xcode/OLMKit/OLMAccount.m

@@ -179,6 +179,33 @@
     return keyDictionary;
 }
 
+- (NSDictionary *) unpublishedFallbackKey {
+    size_t fallbackKeyLength = olm_account_unpublished_fallback_key_length(_account);
+    uint8_t *fallbackKeyBytes = malloc(fallbackKeyLength);
+    if (!fallbackKeyBytes) {
+        return nil;
+    }
+    
+    size_t result = olm_account_unpublished_fallback_key(_account, fallbackKeyBytes, fallbackKeyLength);
+    if (result == olm_error()) {
+        const char *error = olm_account_last_error(_account);
+        NSLog(@"error getting unpublished fallback key: %s", error);
+        free(fallbackKeyBytes);
+        return nil;
+    }
+    NSData *fallbackKeyData = [NSData dataWithBytesNoCopy:fallbackKeyBytes length:fallbackKeyLength freeWhenDone:YES];
+    NSError *error = nil;
+    NSDictionary *keyDictionary = [NSJSONSerialization JSONObjectWithData:fallbackKeyData options:0 error:&error];
+    if (error) {
+        NSLog(@"Could not decode JSON for unpublished fallback: %@", error.localizedDescription);
+    }
+    return keyDictionary;
+}
+
+- (void) forgetFallbackKey {
+    olm_account_forget_old_fallback_key(self.account);
+}
+
 - (void) generateFallbackKey {
     size_t randomLength = olm_account_generate_fallback_key_random_length(_account);
     NSMutableData *random = [OLMUtility randomBytesOfLength:randomLength];

xcode/OLMKit/OLMSAS.h

@@ -54,6 +54,17 @@ NS_ASSUME_NONNULL_BEGIN
  */
 - (NSString *)calculateMac:(NSString*)input info:(NSString*)info error:(NSError* _Nullable *)error;
 
+/**
+ Generate a message authentication code (MAC) based on the shared secret.
+ This version is compatible with other base64 implementations.
+
+ @param input the message to produce the authentication code for.
+ @param info extra information to mix in when generating the MAC, as per the Matrix spec.
+ @param error the error if any.
+ @return the MAC.
+ */
+- (NSString *)calculateMacFixedBase64:(NSString*)input info:(NSString*)info error:(NSError* _Nullable *)error;
+
 /**
  Generate a message authentication code (MAC) based on the shared secret.
  For compatibility with an old version of olm.js.

xcode/OLMKit/OLMSAS.m

@@ -137,6 +137,40 @@
     return mac;
 }
 
+- (NSString *)calculateMacFixedBase64:(NSString *)input info:(NSString *)info error:(NSError *__autoreleasing  _Nullable *)error {
+    NSMutableData *inputData = [input dataUsingEncoding:NSUTF8StringEncoding].mutableCopy;
+    NSData *infoData = [info dataUsingEncoding:NSUTF8StringEncoding];
+
+    size_t macLength = olm_sas_mac_length(olmSAS);
+    NSMutableData *macData = [NSMutableData dataWithLength:macLength];
+    if (!macData) {
+        return nil;
+    }
+
+    size_t result = olm_sas_calculate_mac_fixed_base64(olmSAS,
+                                                       inputData.mutableBytes, inputData.length,
+                                                       infoData.bytes, infoData.length,
+                                                       macData.mutableBytes, macLength);
+    if (result == olm_error()) {
+        const char *olm_error = olm_sas_last_error(olmSAS);
+        NSLog(@"[OLMSAS] calculateMac: olm_sas_calculate_mac error: %s", olm_error);
+
+        NSString *errorString = [NSString stringWithUTF8String:olm_error];
+        if (error && olm_error && errorString) {
+            *error = [NSError errorWithDomain:OLMErrorDomain
+                                         code:0
+                                     userInfo:@{
+                                                NSLocalizedDescriptionKey: errorString,
+                                                NSLocalizedFailureReasonErrorKey: [NSString stringWithFormat:@"olm_sas_calculate_mac error: %@", errorString]
+                                                }];
+        }
+        return nil;
+    }
+
+    NSString *mac = [[NSString alloc] initWithData:macData encoding:NSUTF8StringEncoding];
+    return mac;
+}
+
 - (NSString *)calculateMacLongKdf:(NSString *)input info:(NSString *)info error:(NSError *__autoreleasing  _Nullable *)error {
     NSMutableData *inputData = [input dataUsingEncoding:NSUTF8StringEncoding].mutableCopy;
     NSData *infoData = [info dataUsingEncoding:NSUTF8StringEncoding];

xcode/OLMKitTests/OLMKitTests.m

@@ -36,7 +36,27 @@ limitations under the License.
     OLMAccount *bob = [[OLMAccount alloc] initNewAccount];
     [bob generateFallbackKey];
     
-    [self _testAliceAndBob:bob withBobKeys:bob.fallbackKey];
+    [self _testAliceAndBob:bob withBobKeys:bob.unpublishedFallbackKey];
+
+}
+
+- (void)testMarkAsPublishedFallbackKey {
+    OLMAccount *bob = [[OLMAccount alloc] initNewAccount];
+    [bob generateFallbackKey];
+    
+    
+    NSDictionary *unpublished = bob.unpublishedFallbackKey;
+    __block NSString *bobKeyValue = ((NSDictionary *) unpublished[@"curve25519"]).allValues.lastObject;
+    
+    XCTAssertNotNil(bobKeyValue);
+    
+    [bob markOneTimeKeysAsPublished];
+    
+    NSDictionary *unpublishedAfter = bob.unpublishedFallbackKey;
+    
+    __block NSString *bobKeyValueAfter = ((NSDictionary *) unpublishedAfter[@"curve25519"]).allValues.lastObject;
+    
+    XCTAssertNil(bobKeyValueAfter);
 }
 
 - (void)_testAliceAndBob:(OLMAccount *)bob withBobKeys:(NSDictionary *)bobKeys {
@@ -89,7 +109,7 @@ limitations under the License.
     OLMAccount *bob = [[OLMAccount alloc] initNewAccount];
     [bob generateFallbackKey];
     
-    [self _testBackAndForthWithBob:bob andBobKeys:bob.fallbackKey];
+    [self _testBackAndForthWithBob:bob andBobKeys:bob.unpublishedFallbackKey];
 }
 
 - (void)_testBackAndForthWithBob:(OLMAccount *)bob andBobKeys:(NSDictionary *)bobKeys {
@@ -140,7 +160,7 @@ limitations under the License.
     [bob generateFallbackKey];
     NSDictionary *bobIdKeys = bob.identityKeys;
     NSDictionary *bobOneTimeKeys = bob.oneTimeKeys;
-    NSDictionary *bobFallbackKey = bob.fallbackKey;
+    NSDictionary *bobFallbackKey = bob.unpublishedFallbackKey;
     
     NSError *error;
     NSData *bobData = [NSKeyedArchiver archivedDataWithRootObject:bob requiringSecureCoding:NO error:&error];
@@ -151,7 +171,7 @@ limitations under the License.
     
     NSDictionary *bobIdKeys2 = bob2.identityKeys;
     NSDictionary *bobOneTimeKeys2 = bob2.oneTimeKeys;
-    NSDictionary *bobFallbackKey2 = bob2.fallbackKey;
+    NSDictionary *bobFallbackKey2 = bob2.unpublishedFallbackKey;
     
     XCTAssertEqualObjects(bobIdKeys, bobIdKeys2);
     XCTAssertEqualObjects(bobOneTimeKeys, bobOneTimeKeys2);
@@ -169,7 +189,7 @@ limitations under the License.
     OLMAccount *bob = [[OLMAccount alloc] initNewAccount];
     [bob generateFallbackKey];
     
-    [self _testSessionSerializationWithBob:bob bobKeys:bob.fallbackKey];
+    [self _testSessionSerializationWithBob:bob bobKeys:bob.unpublishedFallbackKey];
 }
 
 - (void)_testSessionSerializationWithBob:(OLMAccount *)bob bobKeys:(NSDictionary *)bobKeys {