matrix-js-sdk/src/rust-crypto/rust-crypto.ts

Matrix Client-Server SDK for Javascript

/*
Copyright 2022-2023 The Matrix.org Foundation C.I.C.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import anotherjson from "another-json";
import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";

import type { IMegolmSessionData } from "../@types/crypto.ts";
import { KnownMembership } from "../@types/membership.ts";
import { type IDeviceLists, type IToDeviceEvent, type ReceivedToDeviceMessage } from "../sync-accumulator.ts";
import type { ToDeviceBatch, ToDevicePayload } from "../models/ToDeviceMessage.ts";
import { type MatrixEvent, MatrixEventEvent } from "../models/event.ts";
import { type Room } from "../models/room.ts";
import { type RoomMember } from "../models/room-member.ts";
import {
    type BackupDecryptor,
    type CryptoBackend,
    DecryptionError,
    type EventDecryptionResult,
    type OnSyncCompletedData,
} from "../common-crypto/CryptoBackend.ts";
import { type Logger, LogSpan } from "../logger.ts";
import { type IHttpOpts, type MatrixHttpApi, Method } from "../http-api/index.ts";
import { RoomEncryptor } from "./RoomEncryptor.ts";
import { OutgoingRequestProcessor } from "./OutgoingRequestProcessor.ts";
import { KeyClaimManager } from "./KeyClaimManager.ts";
import { MapWithDefault } from "../utils.ts";
import {
    AllDevicesIsolationMode,
    type BackupTrustInfo,
    type BootstrapCrossSigningOpts,
    type CreateSecretStorageOpts,
    CrossSigningKey,
    type CrossSigningKeyInfo,
    type CrossSigningStatus,
    type CryptoApi,
    type CryptoCallbacks,
    CryptoEvent,
    type CryptoEventHandlerMap,
    DecryptionFailureCode,
    DecryptionKeyDoesNotMatchError,
    deriveRecoveryKeyFromPassphrase,
    type DeviceIsolationMode,
    DeviceIsolationModeKind,
    DeviceVerificationStatus,
    encodeRecoveryKey,
    type EventEncryptionInfo,
    EventShieldColour,
    EventShieldReason,
    type GeneratedSecretStorageKey,
    type ImportRoomKeysOpts,
    ImportRoomKeyStage,
    type KeyBackupCheck,
    type KeyBackupInfo,
    type KeyBackupRestoreOpts,
    type KeyBackupRestoreResult,
    type OwnDeviceKeys,
    type SecretStorageStatus,
    type StartDehydrationOpts,
    UserVerificationStatus,
    type VerificationRequest,
} from "../crypto-api/index.ts";
import { deviceKeysToDeviceMap, rustDeviceToJsDevice } from "./device-converter.ts";
import { type IDownloadKeyResult, type IQueryKeysRequest } from "../client.ts";
import { type Device, type DeviceMap } from "../models/device.ts";
import {
    SECRET_STORAGE_ALGORITHM_V1_AES,
    type SecretStorageKey,
    type ServerSideSecretStorage,
} from "../secret-storage.ts";
import { CrossSigningIdentity } from "./CrossSigningIdentity.ts";
import { secretStorageContainsCrossSigningKeys } from "./secret-storage.ts";
import { isVerificationEvent, RustVerificationRequest, verificationMethodIdentifierToMethod } from "./verification.ts";
import { EventType, MsgType } from "../@types/event.ts";
import { TypedEventEmitter } from "../models/typed-event-emitter.ts";
import { decryptionKeyMatchesKeyBackupInfo, RustBackupManager } from "./backup.ts";
import { TypedReEmitter } from "../ReEmitter.ts";
import { secureRandomString } from "../randomstring.ts";
import { ClientStoppedError } from "../errors.ts";
import { type ISignatures } from "../@types/signed.ts";
import { decodeBase64, encodeBase64 } from "../base64.ts";
import { OutgoingRequestsManager } from "./OutgoingRequestsManager.ts";
import { PerSessionKeyBackupDownloader } from "./PerSessionKeyBackupDownloader.ts";
import { DehydratedDeviceManager } from "./DehydratedDeviceManager.ts";
import { VerificationMethod } from "../types.ts";
import { keyFromAuthData } from "../common-crypto/key-passphrase.ts";
import { type UIAuthCallback } from "../interactive-auth.ts";
import { getHttpUriForMxc } from "../content-repo.ts";
import { type RoomState } from "../matrix.ts";

const ALL_VERIFICATION_METHODS = [
    VerificationMethod.Sas,
    VerificationMethod.ScanQrCode,
    VerificationMethod.ShowQrCode,
    VerificationMethod.Reciprocate,
];

interface ISignableObject {
    signatures?: ISignatures;
    unsigned?: object;
}

/** The maximum time, in milliseconds, since we accepted an invite, that we should accept a key bundle. */
export const MAX_INVITE_ACCEPTANCE_MS_FOR_KEY_BUNDLE = 24 * 60 * 60 * 1000; // 24 hours

/**
 * An implementation of {@link CryptoBackend} using the Rust matrix-sdk-crypto.
 *
 * @internal
 */
export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, CryptoEventHandlerMap> implements CryptoBackend {
    /**
     * The number of iterations to use when deriving a recovery key from a passphrase.
     */
    private readonly RECOVERY_KEY_DERIVATION_ITERATIONS = 500000;

    private _trustCrossSignedDevices = true;
    private deviceIsolationMode: DeviceIsolationMode = new AllDevicesIsolationMode(false);

    /** whether {@link stop} has been called */
    private stopped = false;

    /** mapping of roomId → encryptor class */
    private roomEncryptors: Record<string, RoomEncryptor> = {};

    private eventDecryptor: EventDecryptor;
    private keyClaimManager: KeyClaimManager;
    private outgoingRequestProcessor: OutgoingRequestProcessor;
    private crossSigningIdentity: CrossSigningIdentity;
    private readonly backupManager: RustBackupManager;
    private outgoingRequestsManager: OutgoingRequestsManager;
    private readonly perSessionBackupDownloader: PerSessionKeyBackupDownloader;
    private readonly dehydratedDeviceManager: DehydratedDeviceManager;
    private readonly reemitter = new TypedReEmitter<RustCryptoEvents, CryptoEventHandlerMap>(this);

    public constructor(
        private readonly logger: Logger,

        /** The `OlmMachine` from the underlying rust crypto sdk. */
        private readonly olmMachine: RustSdkCryptoJs.OlmMachine,

        /**
         * Low-level HTTP interface: used to make outgoing requests required by the rust SDK.
         *
         * We expect it to set the access token, etc.
         */
        private readonly http: MatrixHttpApi<IHttpOpts & { onlyData: true }>,

        /** The local user's User ID. */
        private readonly userId: string,

        /** The local user's Device ID. */
        _deviceId: string,

        /** Interface to server-side secret storage */
        private readonly secretStorage: ServerSideSecretStorage,

        /** Crypto callbacks provided by the application */
        private readonly cryptoCallbacks: CryptoCallbacks,

        /** Enable support for encrypted state events under MSC4362. */
        private readonly enableEncryptedStateEvents: boolean = false,
    ) {
        super();
        this.outgoingRequestProcessor = new OutgoingRequestProcessor(logger, olmMachine, http);
        this.outgoingRequestsManager = new OutgoingRequestsManager(
            this.logger,
            olmMachine,
            this.outgoingRequestProcessor,
        );

        this.keyClaimManager = new KeyClaimManager(olmMachine, this.outgoingRequestProcessor);

        this.backupManager = new RustBackupManager(logger, olmMachine, http, this.outgoingRequestProcessor);
        this.perSessionBackupDownloader = new PerSessionKeyBackupDownloader(
            this.logger,
            this.olmMachine,
            this.http,
            this.backupManager,
        );
        this.dehydratedDeviceManager = new DehydratedDeviceManager(
            this.logger,
            olmMachine,
            http,
            this.outgoingRequestProcessor,
            secretStorage,
        );
        this.eventDecryptor = new EventDecryptor(this.logger, olmMachine, this.perSessionBackupDownloader);

        // re-emit the events emitted by managers
        this.reemitter.reEmit(this.backupManager, [
            CryptoEvent.KeyBackupStatus,
            CryptoEvent.KeyBackupSessionsRemaining,
            CryptoEvent.KeyBackupFailed,
            CryptoEvent.KeyBackupDecryptionKeyCached,
        ]);
        this.reemitter.reEmit(this.dehydratedDeviceManager, [
            CryptoEvent.DehydratedDeviceCreated,
            CryptoEvent.DehydratedDeviceUploaded,
            CryptoEvent.RehydrationStarted,
            CryptoEvent.RehydrationProgress,
            CryptoEvent.RehydrationCompleted,
            CryptoEvent.RehydrationError,
            CryptoEvent.DehydrationKeyCached,
            CryptoEvent.DehydratedDeviceRotationError,
        ]);

        this.crossSigningIdentity = new CrossSigningIdentity(
            logger,
            olmMachine,
            this.outgoingRequestProcessor,
            secretStorage,
        );

        // Check and start in background the key backup connection
        this.checkKeyBackupAndEnable();
    }

    /**
     * Return the OlmMachine only if {@link RustCrypto#stop} has not been called.
     *
     * This allows us to better handle race conditions where the client is stopped before or during a crypto API call.
     *
     * @throws ClientStoppedError if {@link RustCrypto#stop} has been called.
     */
    private getOlmMachineOrThrow(): RustSdkCryptoJs.OlmMachine {
        if (this.stopped) {
            throw new ClientStoppedError();
        }
        return this.olmMachine;
    }

    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    //
    // CryptoBackend implementation
    //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////

    public set globalErrorOnUnknownDevices(_v: boolean) {
        // Not implemented for rust crypto.
    }

    public get globalErrorOnUnknownDevices(): boolean {
        // Not implemented for rust crypto.
        return false;
    }

    public stop(): void {
        // stop() may be called multiple times, but attempting to close() the OlmMachine twice
        // will cause an error.
        if (this.stopped) {
            return;
        }
        this.stopped = true;

        this.keyClaimManager.stop();
        this.backupManager.stop();
        this.outgoingRequestsManager.stop();
        this.perSessionBackupDownloader.stop();
        this.dehydratedDeviceManager.stop();

        // make sure we close() the OlmMachine; doing so means that all the Rust objects will be
        // cleaned up; in particular, the indexeddb connections will be closed, which means they
        // can then be deleted.
        this.olmMachine.close();
    }

    public async encryptEvent(event: MatrixEvent, _room: Room): Promise<void> {
        const roomId = event.getRoomId()!;
        const encryptor = this.roomEncryptors[roomId];

        if (!encryptor) {
            throw new Error(`Cannot encrypt event in unconfigured room ${roomId}`);
        }

        await encryptor.encryptEvent(event, this.globalBlacklistUnverifiedDevices, this.deviceIsolationMode);
    }

    public async decryptEvent(event: MatrixEvent): Promise<EventDecryptionResult> {
        const roomId = event.getRoomId();
        if (!roomId) {
            // presumably, a to-device message. These are normally decrypted in preprocessToDeviceMessages
            // so the fact it has come back here suggests that decryption failed.
            //
            // once we drop support for the libolm crypto implementation, we can stop passing to-device messages
            // through decryptEvent and hence get rid of this case.
            throw new Error("to-device event was not decrypted in preprocessToDeviceMessages");
        }
        return await this.eventDecryptor.attemptEventDecryption(event, this.deviceIsolationMode);
    }

    /**
     * Implementation of {@link CryptoBackend#getBackupDecryptor}.
     */
    public async getBackupDecryptor(backupInfo: KeyBackupInfo, privKey: Uint8Array): Promise<BackupDecryptor> {
        if (!(privKey instanceof Uint8Array)) {
            throw new Error(`getBackupDecryptor: expects Uint8Array`);
        }

        if (backupInfo.algorithm != "m.megolm_backup.v1.curve25519-aes-sha2") {
            throw new Error(`getBackupDecryptor: Unsupported algorithm ${backupInfo.algorithm}`);
        }

        const backupDecryptionKey = RustSdkCryptoJs.BackupDecryptionKey.fromBase64(encodeBase64(privKey));
        if (!decryptionKeyMatchesKeyBackupInfo(backupDecryptionKey, backupInfo)) {
            throw new Error(`getBackupDecryptor: key backup on server does not match the decryption key`);
        }

        return this.backupManager.createBackupDecryptor(backupDecryptionKey);
    }

    /**
     * Implementation of {@link CryptoBackend#importBackedUpRoomKeys}.
     */
    public async importBackedUpRoomKeys(
        keys: IMegolmSessionData[],
        backupVersion: string,
        opts?: ImportRoomKeysOpts,
    ): Promise<void> {
        return await this.backupManager.importBackedUpRoomKeys(keys, backupVersion, opts);
    }

    /**
     * Implementation of {@link CryptoBackend.maybeAcceptKeyBundle}.
     */
    public async maybeAcceptKeyBundle(roomId: string, inviter: string): Promise<boolean> {
        // TODO: retry this if it gets interrupted or it fails. (https://github.com/matrix-org/matrix-rust-sdk/issues/5112)
        // TODO: do this in the background.

        const logger = new LogSpan(this.logger, `maybeAcceptKeyBundle(${roomId}, ${inviter})`);

        // Make sure we have an up-to-date idea of the inviter's cross-signing keys, so that we can check if the
        // device that sent us the bundle data was correctly cross-signed.
        //
        // TODO: it would be nice to skip this step if we have an up-to-date copy of the inviter's cross-signing keys,
        //   but we don't have an easy way to check that. Possibly the rust side could trigger a key request and then
        //   block until it happens.
        logger.info(`Checking inviter cross-signing keys`);
        const request = this.olmMachine.queryKeysForUsers([new RustSdkCryptoJs.UserId(inviter)]);
        await this.outgoingRequestProcessor.makeOutgoingRequest(request);

        const bundleData = await this.olmMachine.getReceivedRoomKeyBundleData(
            new RustSdkCryptoJs.RoomId(roomId),
            new RustSdkCryptoJs.UserId(inviter),
        );
        if (!bundleData) {
            logger.info("No key bundle found for user");
            return false;
        }

        logger.info(`Fetching key bundle ${bundleData.url}`);
        const url = getHttpUriForMxc(
            this.http.opts.baseUrl,
            bundleData.url,
            undefined,
            undefined,
            undefined,
            /* allowDirectLinks */ false,
            /* allowRedirects */ true,
            /* useAuthentication */ true,
        );
        let encryptedBundle: Uint8Array;
        try {
            const bundleUrl = new URL(url);
            const encryptedBundleBlob = await this.http.authedRequest<Blob>(
                Method.Get,
                bundleUrl.pathname + bundleUrl.search,
                {},
                undefined,
                {
                    rawResponseBody: true,
                    prefix: "",
                },
            );
            logger.info(`Received blob of length ${encryptedBundleBlob.size}`);
            encryptedBundle = new Uint8Array(await encryptedBundleBlob.arrayBuffer());
        } catch (err) {
            logger.warn(`Error downloading encrypted bundle from ${url}:`, err);
            throw err;
        }

        try {
            await this.olmMachine.receiveRoomKeyBundle(bundleData, encryptedBundle);
        } catch (err) {
            logger.warn(`Error receiving encrypted bundle:`, err);

            throw err;
        } finally {
            // Even if we were unable to import the bundle, we still clear the flag that indicates that we
            // are waiting for the bundle to be received. The only reason this can happen is that the bundle was
            // malformed somehow, so we don't want to keep retrying it.
            await this.olmMachine.clearRoomPendingKeyBundle(new RustSdkCryptoJs.RoomId(roomId));
        }

        return true;
    }

    /**
     * Implementation of {@link CryptoBackend.markRoomAsPendingKeyBundle}.
     */
    public async markRoomAsPendingKeyBundle(roomId: string, inviter: string): Promise<void> {
        await this.olmMachine.storeRoomPendingKeyBundle(
            new RustSdkCryptoJs.RoomId(roomId),
            new RustSdkCryptoJs.UserId(inviter),
        );
    }

    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    //
    // CryptoApi implementation
    //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////

    public globalBlacklistUnverifiedDevices = false;

    /**
     * Implementation of {@link CryptoApi#getVersion}.
     */
    public getVersion(): string {
        const versions = RustSdkCryptoJs.getVersions();
        return `Rust SDK ${versions.matrix_sdk_crypto} (${versions.git_sha}), Vodozemac ${versions.vodozemac}`;
    }

    /**
     * Implementation of {@link CryptoApi#setDeviceIsolationMode}.
     */
    public setDeviceIsolationMode(isolationMode: DeviceIsolationMode): void {
        this.deviceIsolationMode = isolationMode;
    }

    /**
     * Implementation of {@link CryptoApi#isEncryptionEnabledInRoom}.
     */
    public async isEncryptionEnabledInRoom(roomId: string): Promise<boolean> {
        const roomSettings: RustSdkCryptoJs.RoomSettings | undefined = await this.olmMachine.getRoomSettings(
            new RustSdkCryptoJs.RoomId(roomId),
        );
        return Boolean(roomSettings?.algorithm);
    }

    /**
     * Implementation of {@link CryptoApi#isStateEncryptionEnabledInRoom}.
     */
    public async isStateEncryptionEnabledInRoom(roomId: string): Promise<boolean> {
        const roomSettings: RustSdkCryptoJs.RoomSettings | undefined = await this.olmMachine.getRoomSettings(
            new RustSdkCryptoJs.RoomId(roomId),
        );
        return Boolean(roomSettings?.encryptStateEvents);
    }

    /**
     * Implementation of {@link CryptoApi#getOwnDeviceKeys}.
     */
    public async getOwnDeviceKeys(): Promise<OwnDeviceKeys> {
        const keys = this.olmMachine.identityKeys;
        return { ed25519: keys.ed25519.toBase64(), curve25519: keys.curve25519.toBase64() };
    }

    public prepareToEncrypt(room: Room): void {
        const encryptor = this.roomEncryptors[room.roomId];

        if (encryptor) {
            encryptor.prepareForEncryption(this.globalBlacklistUnverifiedDevices, this.deviceIsolationMode);
        }
    }

    public forceDiscardSession(roomId: string): Promise<void> {
        return this.roomEncryptors[roomId]?.forceDiscardSession();
    }

    public async exportRoomKeys(): Promise<IMegolmSessionData[]> {
        const raw = await this.olmMachine.exportRoomKeys(() => true);
        return JSON.parse(raw);
    }

    public async exportRoomKeysAsJson(): Promise<string> {
        return await this.olmMachine.exportRoomKeys(() => true);
    }

    public async importRoomKeys(keys: IMegolmSessionData[], opts?: ImportRoomKeysOpts): Promise<void> {
        return await this.backupManager.importRoomKeys(keys, opts);
    }

    public async importRoomKeysAsJson(keys: string, opts?: ImportRoomKeysOpts): Promise<void> {
        return await this.backupManager.importRoomKeysAsJson(keys, opts);
    }

    /**
     * Implementation of {@link CryptoApi.userHasCrossSigningKeys}.
     */
    public async userHasCrossSigningKeys(userId = this.userId, downloadUncached = false): Promise<boolean> {
        // TODO: could probably do with a more efficient way of doing this than returning the whole set and searching
        const rustTrackedUsers: Set<RustSdkCryptoJs.UserId> = await this.olmMachine.trackedUsers();
        let rustTrackedUser: RustSdkCryptoJs.UserId | undefined;
        for (const u of rustTrackedUsers) {
            if (userId === u.toString()) {
                rustTrackedUser = u;
                break;
            }
        }

        if (rustTrackedUser !== undefined) {
            if (userId === this.userId) {
                /* make sure we have an *up-to-date* idea of the user's cross-signing keys. This is important, because if we
                 * return "false" here, we will end up generating new cross-signing keys and replacing the existing ones.
                 */
                const request = this.olmMachine.queryKeysForUsers(
                    // clone as rust layer will take ownership and it's reused later
                    [rustTrackedUser.clone()],
                );
                await this.outgoingRequestProcessor.makeOutgoingRequest(request);
            }
            const userIdentity = await this.olmMachine.getIdentity(rustTrackedUser);
            userIdentity?.free();
            return userIdentity !== undefined;
        } else if (downloadUncached) {
            // Download the cross signing keys and check if the master key is available
            const keyResult = await this.downloadDeviceList(new Set([userId]));
            const keys = keyResult.master_keys?.[userId];

            // No master key
            if (!keys) return false;

            // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
            // We assume only a single key, and we want the bare form without type
            // prefix, so we select the values.
            return Boolean(Object.values(keys.keys)[0]);
        } else {
            return false;
        }
    }

    /**
     * Get the device information for the given list of users.
     *
     * @param userIds - The users to fetch.
     * @param downloadUncached - If true, download the device list for users whose device list we are not
     *    currently tracking. Defaults to false, in which case such users will not appear at all in the result map.
     *
     * @returns A map `{@link DeviceMap}`.
     */
    public async getUserDeviceInfo(userIds: string[], downloadUncached = false): Promise<DeviceMap> {
        const deviceMapByUserId = new Map<string, Map<string, Device>>();
        const rustTrackedUsers: Set<RustSdkCryptoJs.UserId> = await this.getOlmMachineOrThrow().trackedUsers();

        // Convert RustSdkCryptoJs.UserId to a `Set<string>`
        const trackedUsers = new Set<string>();
        rustTrackedUsers.forEach((rustUserId) => trackedUsers.add(rustUserId.toString()));

        // Keep untracked user to download their keys after
        const untrackedUsers: Set<string> = new Set();

        for (const userId of userIds) {
            // if this is a tracked user, we can just fetch the device list from the rust-sdk
            // (NB: this is probably ok even if we race with a leave event such that we stop tracking the user's
            // devices: the rust-sdk will return the last-known device list, which will be good enough.)
            if (trackedUsers.has(userId)) {
                deviceMapByUserId.set(userId, await this.getUserDevices(userId));
            } else {
                untrackedUsers.add(userId);
            }
        }

        // for any users whose device lists we are not tracking, fall back to downloading the device list
        // over HTTP.
        if (downloadUncached && untrackedUsers.size >= 1) {
            const queryResult = await this.downloadDeviceList(untrackedUsers);
            Object.entries(queryResult.device_keys).forEach(([userId, deviceKeys]) =>
                deviceMapByUserId.set(userId, deviceKeysToDeviceMap(deviceKeys)),
            );
        }

        return deviceMapByUserId;
    }

    /**
     * Get the device list for the given user from the olm machine
     * @param userId - Rust SDK UserId
     */
    private async getUserDevices(userId: string): Promise<Map<string, Device>> {
        const rustUserId = new RustSdkCryptoJs.UserId(userId);

        // For reasons I don't really understand, the Javascript FinalizationRegistry doesn't seem to run the
        // registered callbacks when `userDevices` goes out of scope, nor when the individual devices in the array
        // returned by `userDevices.devices` do so.
        //
        // This is particularly problematic, because each of those structures holds a reference to the
        // VerificationMachine, which in turn holds a reference to the IndexeddbCryptoStore. Hence, we end up leaking
        // open connections to the crypto store, which means the store can't be deleted on logout.
        //
        // To fix this, we explicitly call `.free` on each of the objects, which tells the rust code to drop the
        // allocated memory and decrement the refcounts for the crypto store.

        // Wait for up to a second for any in-flight device list requests to complete.
        // The reason for this isn't so much to avoid races (some level of raciness is
        // inevitable for this method) but to make testing easier.
        const userDevices: RustSdkCryptoJs.UserDevices = await this.olmMachine.getUserDevices(rustUserId, 1);
        try {
            const deviceArray: RustSdkCryptoJs.Device[] = userDevices.devices();
            try {
                return new Map(
                    deviceArray.map((device) => [device.deviceId.toString(), rustDeviceToJsDevice(device, rustUserId)]),
                );
            } finally {
                deviceArray.forEach((d) => d.free());
            }
        } finally {
            userDevices.free();
        }
    }

    /**
     * Download the given user keys by calling `/keys/query` request
     * @param untrackedUsers - download keys of these users
     */
    private async downloadDeviceList(untrackedUsers: Set<string>): Promise<IDownloadKeyResult> {
        const queryBody: IQueryKeysRequest = { device_keys: {} };
        untrackedUsers.forEach((user) => (queryBody.device_keys[user] = []));

        return await this.http.authedRequest(Method.Post, "/_matrix/client/v3/keys/query", undefined, queryBody, {
            prefix: "",
        });
    }

    /**
     * Implementation of {@link CryptoApi#getTrustCrossSignedDevices}.
     */
    public getTrustCrossSignedDevices(): boolean {
        return this._trustCrossSignedDevices;
    }

    /**
     * Implementation of {@link CryptoApi#setTrustCrossSignedDevices}.
     */
    public setTrustCrossSignedDevices(val: boolean): void {
        this._trustCrossSignedDevices = val;
        // TODO: legacy crypto goes through the list of known devices and emits DeviceVerificationChanged
        //  events. Maybe we need to do the same?
    }

    /**
     * Mark the given device as locally verified.
     *
     * Implementation of {@link CryptoApi#setDeviceVerified}.
     */
    public async setDeviceVerified(userId: string, deviceId: string, verified = true): Promise<void> {
        const device: RustSdkCryptoJs.Device | undefined = await this.olmMachine.getDevice(
            new RustSdkCryptoJs.UserId(userId),
            new RustSdkCryptoJs.DeviceId(deviceId),
        );

        if (!device) {
            throw new Error(`Unknown device ${userId}|${deviceId}`);
        }
        try {
            await device.setLocalTrust(
                verified ? RustSdkCryptoJs.LocalTrust.Verified : RustSdkCryptoJs.LocalTrust.Unset,
            );
        } finally {
            device.free();
        }
    }

    /**
     * Blindly cross-sign one of our other devices.
     *
     * Implementation of {@link CryptoApi#crossSignDevice}.
     */
    public async crossSignDevice(deviceId: string): Promise<void> {
        const device: RustSdkCryptoJs.Device | undefined = await this.olmMachine.getDevice(
            new RustSdkCryptoJs.UserId(this.userId),
            new RustSdkCryptoJs.DeviceId(deviceId),
        );
        if (!device) {
            throw new Error(`Unknown device ${deviceId}`);
        }
        try {
            const outgoingRequest: RustSdkCryptoJs.SignatureUploadRequest = await device.verify();
            await this.outgoingRequestProcessor.makeOutgoingRequest(outgoingRequest);
        } finally {
            device.free();
        }
    }

    /**
     * Implementation of {@link CryptoApi#getDeviceVerificationStatus}.
     */
    public async getDeviceVerificationStatus(
        userId: string,
        deviceId: string,
    ): Promise<DeviceVerificationStatus | null> {
        const device: RustSdkCryptoJs.Device | undefined = await this.olmMachine.getDevice(
            new RustSdkCryptoJs.UserId(userId),
            new RustSdkCryptoJs.DeviceId(deviceId),
        );

        if (!device) return null;
        try {
            return new DeviceVerificationStatus({
                signedByOwner: device.isCrossSignedByOwner(),
                crossSigningVerified: device.isCrossSigningTrusted(),
                localVerified: device.isLocallyTrusted(),
                trustCrossSignedDevices: this._trustCrossSignedDevices,
            });
        } finally {
            device.free();
        }
    }

    /**
     * Implementation of {@link CryptoApi#getUserVerificationStatus}.
     */
    public async getUserVerificationStatus(userId: string): Promise<UserVerificationStatus> {
        const userIdentity: RustSdkCryptoJs.OtherUserIdentity | RustSdkCryptoJs.OwnUserIdentity | undefined =
            await this.getOlmMachineOrThrow().getIdentity(new RustSdkCryptoJs.UserId(userId));
        if (userIdentity === undefined) {
            return new UserVerificationStatus(false, false, false);
        }

        const verified = userIdentity.isVerified();
        const wasVerified = userIdentity.wasPreviouslyVerified();
        const needsUserApproval =
            userIdentity instanceof RustSdkCryptoJs.OtherUserIdentity
                ? userIdentity.identityNeedsUserApproval()
                : false;
        userIdentity.free();
        return new UserVerificationStatus(verified, wasVerified, false, needsUserApproval);
    }

    /**
     * Implementation of {@link CryptoApi#pinCurrentUserIdentity}.
     */
    public async pinCurrentUserIdentity(userId: string): Promise<void> {
        const userIdentity: RustSdkCryptoJs.OtherUserIdentity | RustSdkCryptoJs.OwnUserIdentity | undefined =
            await this.getOlmMachineOrThrow().getIdentity(new RustSdkCryptoJs.UserId(userId));

        if (userIdentity === undefined) {
            throw new Error("Cannot pin identity of unknown user");
        }

        if (userIdentity instanceof RustSdkCryptoJs.OwnUserIdentity) {
            throw new Error("Cannot pin identity of own user");
        }

        await userIdentity.pinCurrentMasterKey();
    }

    /**
     * Implementation of {@link CryptoApi#withdrawVerificationRequirement}.
     */
    public async withdrawVerificationRequirement(userId: string): Promise<void> {
        const userIdentity: RustSdkCryptoJs.OtherUserIdentity | RustSdkCryptoJs.OwnUserIdentity | undefined =
            await this.getOlmMachineOrThrow().getIdentity(new RustSdkCryptoJs.UserId(userId));

        if (userIdentity === undefined) {
            throw new Error("Cannot withdraw verification of unknown user");
        }

        await userIdentity.withdrawVerification();
    }

    /**
     * Implementation of {@link CryptoApi#isCrossSigningReady}
     */
    public async isCrossSigningReady(): Promise<boolean> {
        const { privateKeysInSecretStorage, privateKeysCachedLocally } = await this.getCrossSigningStatus();
        const hasKeysInCache =
            Boolean(privateKeysCachedLocally.masterKey) &&
            Boolean(privateKeysCachedLocally.selfSigningKey) &&
            Boolean(privateKeysCachedLocally.userSigningKey);

        const identity = await this.getOwnIdentity();

        // Cross-signing is ready if the public identity is trusted, and the private keys
        // are either cached, or accessible via secret-storage.
        return !!identity?.isVerified() && (hasKeysInCache || privateKeysInSecretStorage);
    }

    /**
     * Implementation of {@link CryptoApi#getCrossSigningKeyId}
     */
    public async getCrossSigningKeyId(type: CrossSigningKey = CrossSigningKey.Master): Promise<string | null> {
        const userIdentity = await this.getOwnIdentity();
        if (!userIdentity) {
            // The public keys are not available on this device
            return null;
        }

        try {
            const crossSigningStatus: RustSdkCryptoJs.CrossSigningStatus = await this.olmMachine.crossSigningStatus();

            const privateKeysOnDevice =
                crossSigningStatus.hasMaster && crossSigningStatus.hasUserSigning && crossSigningStatus.hasSelfSigning;

            if (!privateKeysOnDevice) {
                // The private keys are not available on this device
                return null;
            }

            if (!userIdentity.isVerified()) {
                // We have both public and private keys, but they don't match!
                return null;
            }

            let key: string;
            switch (type) {
                case CrossSigningKey.Master:
                    key = userIdentity.masterKey;
                    break;
                case CrossSigningKey.SelfSigning:
                    key = userIdentity.selfSigningKey;
                    break;
                case CrossSigningKey.UserSigning:
                    key = userIdentity.userSigningKey;
                    break;
                default:
                    // Unknown type
                    return null;
            }

            const parsedKey: CrossSigningKeyInfo = JSON.parse(key);
            // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
            // We assume only a single key, and we want the bare form without type
            // prefix, so we select the values.
            return Object.values(parsedKey.keys)[0];
        } finally {
            userIdentity.free();
        }
    }

    /**
     * Implementation of {@link CryptoApi#bootstrapCrossSigning}
     */
    public async bootstrapCrossSigning(opts: BootstrapCrossSigningOpts): Promise<void> {
        await this.crossSigningIdentity.bootstrapCrossSigning(opts);
    }

    /**
     * Implementation of {@link CryptoApi#isSecretStorageReady}
     */
    public async isSecretStorageReady(): Promise<boolean> {
        return (await this.getSecretStorageStatus()).ready;
    }

    /**
     * Implementation of {@link CryptoApi#getSecretStorageStatus}
     */
    public async getSecretStorageStatus(): Promise<SecretStorageStatus> {
        // make sure that the cross-signing keys are stored
        const secretsToCheck: SecretStorageKey[] = [
            "m.cross_signing.master",
            "m.cross_signing.user_signing",
            "m.cross_signing.self_signing",
        ];

        // If key backup is active, we also need to check that the backup decryption key is stored
        const keyBackupEnabled = (await this.backupManager.getActiveBackupVersion()) != null;
        if (keyBackupEnabled) {
            secretsToCheck.push("m.megolm_backup.v1");
        }

        const defaultKeyId = await this.secretStorage.getDefaultKeyId();

        const result: SecretStorageStatus = {
            // Assume we have all secrets until proven otherwise
            ready: true,
            defaultKeyId,
            secretStorageKeyValidityMap: {},
        };

        for (const secretName of secretsToCheck) {
            // Check which keys this particular secret is encrypted with
            const record = (await this.secretStorage.isStored(secretName)) || {};

            // If it's encrypted with the right key, it is valid
            const secretStored = !!defaultKeyId && defaultKeyId in record;
            result.secretStorageKeyValidityMap[secretName] = secretStored;
            result.ready = result.ready && secretStored;
        }

        return result;
    }

    /**
     * Implementation of {@link CryptoApi#bootstrapSecretStorage}
     */
    public async bootstrapSecretStorage({
        createSecretStorageKey,
        setupNewSecretStorage,
        setupNewKeyBackup,
    }: CreateSecretStorageOpts = {}): Promise<void> {
        // If an AES Key is already stored in the secret storage and setupNewSecretStorage is not set
        // we don't want to create a new key
        const isNewSecretStorageKeyNeeded = setupNewSecretStorage || !(await this.secretStorageHasAESKey());

        if (isNewSecretStorageKeyNeeded) {
            if (!createSecretStorageKey) {
                throw new Error("unable to create a new secret storage key, createSecretStorageKey is not set");
            }

            // Create a new storage key and add it to secret storage
            this.logger.info("bootstrapSecretStorage: creating new secret storage key");
            const recoveryKey = await createSecretStorageKey();
            if (!recoveryKey) {
                throw new Error("createSecretStorageKey() callback did not return a secret storage key");
            }
            await this.addSecretStorageKeyToSecretStorage(recoveryKey);
        }

        const crossSigningPrivateKeys: RustSdkCryptoJs.CrossSigningKeyExport | undefined =
            await this.olmMachine.exportCrossSigningKeys();
        const hasPrivateKeys =
            crossSigningPrivateKeys &&
            crossSigningPrivateKeys.masterKey !== undefined &&
            crossSigningPrivateKeys.self_signing_key !== undefined &&
            crossSigningPrivateKeys.userSigningKey !== undefined;

        // If we have cross-signing private keys cached, store them in secret
        // storage if they are not there already.
        if (
            hasPrivateKeys &&
            (isNewSecretStorageKeyNeeded || !(await secretStorageContainsCrossSigningKeys(this.secretStorage)))
        ) {
            this.logger.info("bootstrapSecretStorage: cross-signing keys not yet exported; doing so now.");

            await this.secretStorage.store("m.cross_signing.master", crossSigningPrivateKeys.masterKey);
            await this.secretStorage.store("m.cross_signing.user_signing", crossSigningPrivateKeys.userSigningKey);
            await this.secretStorage.store("m.cross_signing.self_signing", crossSigningPrivateKeys.self_signing_key);
        }

        // likewise with the key backup key: if we have one, store it in secret storage (if it's not already there)
        // also don't bother storing it if we're about to set up a new backup
        if (!setupNewKeyBackup) {
            await this.saveBackupKeyToStorage();
        } else {
            await this.resetKeyBackup();
        }
    }

    /**
     * If we have a backup key for the current, trusted backup in cache,
     * save it to secret storage.
     */
    private async saveBackupKeyToStorage(): Promise<void> {
        const keyBackupInfo = await this.backupManager.getServerBackupInfo();
        if (!keyBackupInfo || !keyBackupInfo.version) {
            this.logger.info("Not saving backup key to secret storage: no backup info");
            return;
        }

        const backupKeys: RustSdkCryptoJs.BackupKeys = await this.olmMachine.getBackupKeys();
        if (!backupKeys.decryptionKey) {
            this.logger.info("Not saving backup key to secret storage: no backup key");
            return;
        }

        if (!decryptionKeyMatchesKeyBackupInfo(backupKeys.decryptionKey, keyBackupInfo)) {
            this.logger.info("Not saving backup key to secret storage: decryption key does not match backup info");
            return;
        }

        const backupKeyBase64 = backupKeys.decryptionKey.toBase64();

        await this.secretStorage.store("m.megolm_backup.v1", backupKeyBase64);
    }

    /**
     * Add the secretStorage key to the secret storage
     * - The secret storage key must have the `keyInfo` field filled
     * - The secret storage key is set as the default key of the secret storage
     * - Call `cryptoCallbacks.cacheSecretStorageKey` when done
     *
     * @param secretStorageKey - The secret storage key to add in the secret storage.
     */
    private async addSecretStorageKeyToSecretStorage(secretStorageKey: GeneratedSecretStorageKey): Promise<void> {
        const secretStorageKeyObject = await this.secretStorage.addKey(SECRET_STORAGE_ALGORITHM_V1_AES, {
            passphrase: secretStorageKey.keyInfo?.passphrase,
            name: secretStorageKey.keyInfo?.name,
            key: secretStorageKey.privateKey,
        });

        await this.secretStorage.setDefaultKeyId(secretStorageKeyObject.keyId);

        this.cryptoCallbacks.cacheSecretStorageKey?.(
            secretStorageKeyObject.keyId,
            secretStorageKeyObject.keyInfo,
            secretStorageKey.privateKey,
        );
    }

    /**
     * Check if a secret storage AES Key is already added in secret storage
     *
     * @returns True if an AES key is in the secret storage
     */
    private async secretStorageHasAESKey(): Promise<boolean> {
        // See if we already have an AES secret-storage key.
        const secretStorageKeyTuple = await this.secretStorage.getKey();

        if (!secretStorageKeyTuple) return false;

        const [, keyInfo] = secretStorageKeyTuple;

        // Check if the key is an AES key
        return keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES;
    }

    /**
     * Implementation of {@link CryptoApi#getCrossSigningStatus}
     */
    public async getCrossSigningStatus(): Promise<CrossSigningStatus> {
        const userIdentity = await this.getOwnIdentity();

        const publicKeysOnDevice =
            Boolean(userIdentity?.masterKey) &&
            Boolean(userIdentity?.selfSigningKey) &&
            Boolean(userIdentity?.userSigningKey);
        userIdentity?.free();

        const privateKeysInSecretStorage = await secretStorageContainsCrossSigningKeys(this.secretStorage);
        const crossSigningStatus: RustSdkCryptoJs.CrossSigningStatus | null =
            await this.getOlmMachineOrThrow().crossSigningStatus();

        return {
            publicKeysOnDevice,
            privateKeysInSecretStorage,
            privateKeysCachedLocally: {
                masterKey: Boolean(crossSigningStatus?.hasMaster),
                userSigningKey: Boolean(crossSigningStatus?.hasUserSigning),
                selfSigningKey: Boolean(crossSigningStatus?.hasSelfSigning),
            },
        };
    }

    /**
     * Implementation of {@link CryptoApi#createRecoveryKeyFromPassphrase}
     */
    public async createRecoveryKeyFromPassphrase(password?: string): Promise<GeneratedSecretStorageKey> {
        if (password) {
            // Generate the key from the passphrase
            // first we generate a random salt
            const salt = secureRandomString(32);
            // then we derive the key from the passphrase
            const recoveryKey = await deriveRecoveryKeyFromPassphrase(
                password,
                salt,
                this.RECOVERY_KEY_DERIVATION_ITERATIONS,
            );
            return {
                keyInfo: {
                    passphrase: {
                        algorithm: "m.pbkdf2",
                        iterations: this.RECOVERY_KEY_DERIVATION_ITERATIONS,
                        salt,
                    },
                },
                privateKey: recoveryKey,
                encodedPrivateKey: encodeRecoveryKey(recoveryKey),
            };
        } else {
            // Using the navigator crypto API to generate the private key
            const key = new Uint8Array(32);
            globalThis.crypto.getRandomValues(key);
            return {
                privateKey: key,
                encodedPrivateKey: encodeRecoveryKey(key),
            };
        }
    }

    /**
     * Implementation of {@link CryptoApi#getEncryptionInfoForEvent}.
     */
    public async getEncryptionInfoForEvent(event: MatrixEvent): Promise<EventEncryptionInfo | null> {
        return this.eventDecryptor.getEncryptionInfoForEvent(event);
    }

    /**
     * Returns to-device verification requests that are already in progress for the given user id.
     *
     * Implementation of {@link CryptoApi#getVerificationRequestsToDeviceInProgress}
     *
     * @param userId - the ID of the user to query
     *
     * @returns the VerificationRequests that are in progress
     */
    public getVerificationRequestsToDeviceInProgress(userId: string): VerificationRequest[] {
        const requests: RustSdkCryptoJs.VerificationRequest[] = this.olmMachine.getVerificationRequests(
            new RustSdkCryptoJs.UserId(userId),
        );
        return requests
            .filter((request) => request.roomId === undefined && !request.isCancelled())
            .map((request) => this.makeVerificationRequest(request));
    }

    /**
     * Finds a DM verification request that is already in progress for the given room id
     *
     * Implementation of {@link CryptoApi#findVerificationRequestDMInProgress}
     *
     * @param roomId - the room to use for verification
     * @param userId - search the verification request for the given user
     *
     * @returns the VerificationRequest that is in progress, if any
     *
     */
    public findVerificationRequestDMInProgress(roomId: string, userId?: string): VerificationRequest | undefined {
        if (!userId) throw new Error("missing userId");

        const requests: RustSdkCryptoJs.VerificationRequest[] = this.olmMachine.getVerificationRequests(
            new RustSdkCryptoJs.UserId(userId),
        );

        // Search for the verification request for the given room id
        const request = requests.find((request) => request.roomId?.toString() === roomId && !request.isCancelled());

        if (request) {
            return this.makeVerificationRequest(request);
        }
    }

    /**
     * Implementation of {@link CryptoApi#requestVerificationDM}
     */
    public async requestVerificationDM(userId: string, roomId: string): Promise<VerificationRequest> {
        const userIdentity = (await this.olmMachine.getIdentity(new RustSdkCryptoJs.UserId(userId))) as
            | RustSdkCryptoJs.OtherUserIdentity
            | undefined;

        if (!userIdentity) throw new Error(`unknown userId ${userId}`);

        try {
            // Transform the verification methods into rust objects
            const methods = this._supportedVerificationMethods.map((method) =>
                verificationMethodIdentifierToMethod(method),
            );
            // Get the request content to send to the DM room
            const verCont: string = await userIdentity.verificationRequestContent(methods);

            // TODO: due to https://github.com/matrix-org/matrix-rust-sdk/issues/5643, we need to fix up the verification request content to include `msgtype`.
            const verContObj = JSON.parse(verCont);
            verContObj["msgtype"] = "m.key.verification.request";
            const verificationEventContent: string = JSON.stringify(verContObj);

            // Send the request content to send to the DM room
            const eventId = await this.sendVerificationRequestContent(roomId, verificationEventContent);

            // Get a verification request
            const request: RustSdkCryptoJs.VerificationRequest = await userIdentity.requestVerification(
                new RustSdkCryptoJs.RoomId(roomId),
                new RustSdkCryptoJs.EventId(eventId),
                methods,
            );
            return this.makeVerificationRequest(request);
        } finally {
            userIdentity.free();
        }
    }

    /**
     * Send the verification content to a room
     * See https://spec.matrix.org/v1.7/client-server-api/#put_matrixclientv3roomsroomidsendeventtypetxnid
     *
     * Prefer to use {@link OutgoingRequestProcessor.makeOutgoingRequest} when dealing with {@link RustSdkCryptoJs.RoomMessageRequest}
     *
     * @param roomId - the targeted room
     * @param verificationEventContent - the request body.
     *
     * @returns the event id
     */
    private async sendVerificationRequestContent(roomId: string, verificationEventContent: string): Promise<string> {
        const txId = secureRandomString(32);
        // Send the verification request content to the DM room
        const { event_id: eventId } = await this.http.authedRequest<{ event_id: string }>(
            Method.Put,
            `/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${encodeURIComponent(txId)}`,
            undefined,
            verificationEventContent,
            {
                prefix: "",
            },
        );

        return eventId;
    }

    /**
     * The verification methods we offer to the other side during an interactive verification.
     */
    private _supportedVerificationMethods: string[] = ALL_VERIFICATION_METHODS;

    /**
     * Set the verification methods we offer to the other side during an interactive verification.
     *
     * If `undefined`, we will offer all the methods supported by the Rust SDK.
     */
    public setSupportedVerificationMethods(methods: string[] | undefined): void {
        // by default, the Rust SDK does not offer `m.qr_code.scan.v1`, but we do want to offer that.
        this._supportedVerificationMethods = methods ?? ALL_VERIFICATION_METHODS;
    }

    /**
     * Send a verification request to our other devices.
     *
     * If a verification is already in flight, returns it. Otherwise, initiates a new one.