From a89e85ea79219118c83895627a7736b7dba5c314 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 15 Dec 2025 17:00:55 +1100 Subject: [PATCH 01/72] Add blinded community session encrypt functions --- include/session/session_protocol.h | 34 ++++ include/session/session_protocol.hpp | 19 +++ src/session_protocol.cpp | 201 ++++++++++++++++++---- tests/test_session_protocol.cpp | 241 ++++++++++++++++++++++----- 4 files changed, 417 insertions(+), 78 deletions(-) diff --git a/include/session/session_protocol.h b/include/session/session_protocol.h index 34efb16f..8e1061ac 100644 --- a/include/session/session_protocol.h +++ b/include/session/session_protocol.h @@ -139,8 +139,16 @@ typedef enum SESSION_PROTOCOL_PRO_FEATURES_FOR_MSG_STATUS { // See session::Pro typedef enum SESSION_PROTOCOL_DESTINATION_TYPE { // See session::DestinationType SESSION_PROTOCOL_DESTINATION_TYPE_SYNC_OR_1O1, SESSION_PROTOCOL_DESTINATION_TYPE_GROUP, + + // Old-style community messages that is content encrypted using the blinding protocol or + // plaintext content respectively (inbox vs non-inbox). SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX, SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY, + + // New-style community messages that are sent as envelopes, encrypted using the blinding + // protocol or plaintext content respectively (inbox vs non-inbox). + SESSION_PROTOCOL_DESTINATION_TYPE_ENVELOPE_COMMUNITY_INBOX, + SESSION_PROTOCOL_DESTINATION_TYPE_ENVELOPE_COMMUNITY, } SESSION_PROTOCOL_DESTINATION_TYPE; typedef struct session_protocol_destination session_protocol_destination; @@ -709,6 +717,32 @@ session_protocol_encoded_for_destination session_protocol_encode_for_destination LIBSESSION_EXPORT void session_protocol_encode_for_destination_free( session_protocol_encoded_for_destination* encrypt); +/// API: session_protocol/decode_for_community_inbox +/// +/// Given a blinded encrypted content or envelope payload extract the content and any associated pro +/// metadata if there was any in the message. +/// +/// This function is the same as calling `decrypt_from_blinded_recipient` to decrypt the payload and +/// then decode with `decode_for_community`. See those functions for more information. On failure +/// this function throws as per `decrypt_from_blinded_recipient` +LIBSESSION_EXPORT session_protocol_decoded_community_message +session_protocol_decode_for_community_inbox( + const unsigned char* ed25519_privkey, + size_t ed25519_privkey_len, + const unsigned char* community_pubkey, + size_t community_pubkey_len, + const unsigned char* sender_id, + size_t sender_id_len, + const unsigned char* recipient_id, + size_t recipient_id_len, + const unsigned char* ciphertext, + size_t ciphertext_len, + uint64_t unix_ts_ms, + OPTIONAL const void* pro_backend_pubkey, + OPTIONAL size_t pro_backend_pubkey_len, + OPTIONAL char* error, + size_t error_len); + /// API: session_protocol/session_protocol_decode_envelope /// /// Given an envelope payload (i.e.: protobuf encoded stream of `WebsocketRequestMessage` which diff --git a/include/session/session_protocol.hpp b/include/session/session_protocol.hpp index 2ef515d5..bd5dcc18 100644 --- a/include/session/session_protocol.hpp +++ b/include/session/session_protocol.hpp @@ -213,6 +213,8 @@ enum class DestinationType { Group = SESSION_PROTOCOL_DESTINATION_TYPE_GROUP, CommunityInbox = SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY_INBOX, Community = SESSION_PROTOCOL_DESTINATION_TYPE_COMMUNITY, + EnvelopeCommunityInbox = SESSION_PROTOCOL_DESTINATION_TYPE_ENVELOPE_COMMUNITY_INBOX, + EnvelopeCommunity = SESSION_PROTOCOL_DESTINATION_TYPE_ENVELOPE_COMMUNITY, }; struct Destination { @@ -606,6 +608,23 @@ DecodedEnvelope decode_envelope( std::span envelope_payload, const array_uc32& pro_backend_pubkey); +/// API: session_protocol/decode_for_community_inbox +/// +/// Given a blinded encrypted content or envelope payload extract the plaintext to the content and +/// any associated pro metadata if there was any in the message. +/// +/// This function is the same as calling `decrypt_from_blinded_recipient` to decrypt the payload and +/// then decode with `decode_for_community`. See those functions for more information. On failure +/// this function throws as per `decrypt_from_blinded_recipient` +DecodedCommunityMessage decode_for_community_inbox( + std::span ed25519_privkey, + std::span community_pubkey, + std::span sender_id, + std::span recipient_id, + std::span ciphertext, + std::chrono::sys_time unix_ts, + const array_uc32& pro_backend_pubkey); + /// API: session_protocol/decode_for_community /// /// Given an unencrypted content or envelope payload extract the plaintext to the content and any diff --git a/src/session_protocol.cpp b/src/session_protocol.cpp index aeabdee8..4d8bca23 100644 --- a/src/session_protocol.cpp +++ b/src/session_protocol.cpp @@ -445,6 +445,8 @@ static EncryptedForDestinationInternal encode_for_destination_internal( bool is_1o1 = dest_type == DestinationType::SyncOr1o1; bool is_community_inbox = dest_type == DestinationType::CommunityInbox; bool is_community = dest_type == DestinationType::Community; + bool is_community_envelope = dest_type == DestinationType::EnvelopeCommunityInbox || + dest_type == DestinationType::EnvelopeCommunityInbox; if (!is_community) { assert(ed25519_privkey.size() == crypto_sign_ed25519_SECRETKEYBYTES || ed25519_privkey.size() == crypto_sign_ed25519_SEEDBYTES); @@ -474,8 +476,12 @@ static EncryptedForDestinationInternal encode_for_destination_internal( EncryptedForDestinationInternal result = {}; switch (dest_type) { - case DestinationType::Group: /*FALLTHRU*/ + case DestinationType::EnvelopeCommunity: /*FALLTHRU*/ + case DestinationType::EnvelopeCommunityInbox: /*FALLTHRU*/ + case DestinationType::Group: /*FALLTHRU*/ case DestinationType::SyncOr1o1: { + assert(is_group || is_1o1 || is_community_envelope); + if (is_group && dest_group_ed25519_pubkey[0] != static_cast(SessionIDPrefix::group)) { // Legacy groups which have a 05 prefixed key @@ -484,14 +490,23 @@ static EncryptedForDestinationInternal encode_for_destination_internal( "no longer supported"}; } - // For Sync or 1o1 mesasges, we need to pad the contents to 160 bytes, see: - // https://github.com/session-foundation/session-desktop/blob/a04e62427034a6b6fee39dcff7dbabf0d0131b13/ts/session/crypto/BufferPadding.ts#L49 std::vector tmp_content_buffer; - if (is_1o1) { // Encrypt the padded output + if (is_1o1) { + // For Sync or 1o1 mesasges, we need to pad the contents to 160 bytes, see: + // https://github.com/session-foundation/session-desktop/blob/a04e62427034a6b6fee39dcff7dbabf0d0131b13/ts/session/crypto/BufferPadding.ts#L49 std::vector padded_payload = pad_message(content); + + // Encrypt the padded output tmp_content_buffer = encrypt_for_recipient_deterministic( ed25519_privkey, dest_recipient_pubkey, padded_payload); content = tmp_content_buffer; + } else if (dest_type == DestinationType::EnvelopeCommunityInbox) { + // For community messages we only pad the content if it's an inbox message (e.g. DMs + // require encryption in a community, so we pad to avoid leakage of metadata of the + // kind of message being sent being inferred from the size of the payload). + assert(is_community_envelope); + tmp_content_buffer = pad_message(content); + content = tmp_content_buffer; } // Create envelope @@ -499,9 +514,11 @@ static EncryptedForDestinationInternal encode_for_destination_internal( // https://github.com/session-foundation/session-ios/blob/82deef869d0f7389b799295817f42ad14f8a1316/SessionMessagingKit/Utilities/MessageWrapper.swift#L57 SessionProtos::Envelope envelope = {}; envelope.set_type( - is_1o1 ? SessionProtos::Envelope_Type_SESSION_MESSAGE - : SessionProtos::Envelope_Type_CLOSED_GROUP_MESSAGE); - envelope.set_sourcedevice(1); + (is_1o1 || is_community_envelope) + ? SessionProtos::Envelope_Type_SESSION_MESSAGE + : SessionProtos::Envelope_Type_CLOSED_GROUP_MESSAGE); + if (!is_community_envelope) + envelope.set_sourcedevice(1); envelope.set_timestamp(dest_sent_timestamp_ms.count()); envelope.set_content(content.data(), content.size()); @@ -540,7 +557,7 @@ static EncryptedForDestinationInternal encode_for_destination_internal( } else { result.ciphertext_cpp = std::move(ciphertext); } - } else { + } else if (is_1o1) { // 1o1, Wrap in websocket message WebSocketProtos::WebSocketMessage msg = {}; msg.set_type(WebSocketProtos::WebSocketMessage_Type::WebSocketMessage_Type_REQUEST); @@ -564,6 +581,35 @@ static EncryptedForDestinationInternal encode_for_destination_internal( result.ciphertext_cpp.data(), result.ciphertext_cpp.size()); } assert(serialized); + } else { + assert(is_community_envelope); + if (dest_type == DestinationType::EnvelopeCommunityInbox) { + std::string bytes = envelope.SerializeAsString(); + std::vector ciphertext = encrypt_for_blinded_recipient( + ed25519_privkey, + dest_community_inbox_server_pubkey, + dest_recipient_pubkey, // recipient blinded pubkey + to_span(bytes)); + + if (use_malloc == UseMalloc::Yes) { + result.ciphertext_c = + span_u8_copy_or_throw(ciphertext.data(), ciphertext.size()); + } else { + result.ciphertext_cpp = std::move(ciphertext); + } + } else { + [[maybe_unused]] bool serialized = false; + if (use_malloc == UseMalloc::Yes) { + result.ciphertext_c = span_u8_alloc_or_throw(envelope.ByteSizeLong()); + serialized = envelope.SerializeToArray( + result.ciphertext_c.data, result.ciphertext_c.size); + } else { + result.ciphertext_cpp.resize(envelope.ByteSizeLong()); + envelope.SerializeToArray( + result.ciphertext_cpp.data(), result.ciphertext_cpp.size()); + } + assert(serialized); + } } } break; @@ -573,19 +619,19 @@ static EncryptedForDestinationInternal encode_for_destination_internal( std::vector tmp_content_buffer; // Sign the message with the Session Pro key if given and then pad the message (both - // community message types require it) + // community message types require it for old-style content messages) // https://github.com/session-foundation/session-ios/blob/82deef869d0f7389b799295817f42ad14f8a1316/SessionMessagingKit/Sending%20%26%20Receiving/MessageSender.swift#L398 if (dest_pro_rotating_ed25519_privkey.size()) { // Key should be verified by the time we hit this branch assert(dest_pro_rotating_ed25519_privkey.size() == crypto_sign_ed25519_SECRETKEYBYTES); - // TODO: Sub-optimal, but we parse the content again to make sure it's valid. Sign - // the blob then, fill in the signature in-place as part of the transitioning of - // open groups messages to envelopes. As part of that, libsession is going to take - // responsibility of constructing community messages so that eventually all - // platforms switch over to envelopes and we can change the implementation across - // all platforms in one swoop and remove this. + // TODO: Sub-optimal, but we parse the content again to make sure it's valid. + // Sign the blob then, fill in the signature in-place as part of the + // transitioning of open groups messages to envelopes. As part of that, + // libsession is going to take responsibility of constructing community messages + // so that eventually all platforms switch over to envelopes and we can change + // the implementation across all platforms in one swoop and remove this. // // Parse the content blob SessionProtos::Content content_w_sig = {}; @@ -608,9 +654,9 @@ static EncryptedForDestinationInternal encode_for_destination_internal( dest_pro_rotating_ed25519_privkey.data()) == 0; assert(was_signed); - // Now assign the community specific pro signature field, reserialize it and we have - // to, yes, pad it again. This is all temporary wasted work whilst transitioning - // open groups. + // Now assign the community specific pro signature field, reserialize it and we + // have to, yes, pad it again. This is all temporary wasted work whilst + // transitioning open groups. content_w_sig.set_prosigforcommunitymessageonly(pro_sig.data(), pro_sig.size()); tmp_content_buffer.resize(content_w_sig.ByteSizeLong()); bool serialized = content_w_sig.SerializeToArray( @@ -624,10 +670,6 @@ static EncryptedForDestinationInternal encode_for_destination_internal( content = tmp_content_buffer; } - // TODO: We don't need to actually pad the community message since that's unencrypted, - // there's no need to make the message sizes uniform but we need it for backwards - // compat. We can remove this eventually, first step is to unify the clients. - if (is_community_inbox) { std::vector ciphertext = encrypt_for_blinded_recipient( ed25519_privkey, @@ -868,6 +910,8 @@ DecodedEnvelope decode_envelope( static_assert(sizeof(result.envelope.pro_sig) == crypto_sign_ed25519_BYTES); std::memcpy(result.envelope.pro_sig.data(), pro_sig.data(), pro_sig.size()); + // We only care about verifying the pro-signature if the content has pro data populated, + // otherwise we assume that it's a dummy signature for preventing metadata leakage. if (content.has_promessage()) { if (!content.sigtimestamp()) throw std::runtime_error{fmt::format( @@ -1007,9 +1051,10 @@ DecodedCommunityMessage decode_for_community( std::span unpadded_content = unpad_message(result.content_plaintext); SessionProtos::Content content = {}; if (!content.ParseFromArray(unpadded_content.data(), unpadded_content.size())) - throw std::runtime_error{ - "Decoding community message failed, could not interpret blob as content or " - "envelope"}; + throw std::runtime_error{fmt::format( + "Decoding community message failed, could not interpret blob as {}: {}b", + result.envelope ? "envelope" : "content", + unpadded_content.size())}; // Extract the pro signature from content if it was present if (content.has_prosigforcommunitymessageonly()) { @@ -1120,6 +1165,21 @@ DecodedCommunityMessage decode_for_community( return result; } +DecodedCommunityMessage decode_for_community_inbox( + std::span ed25519_privkey, + std::span community_pubkey, + std::span sender_id, + std::span recipient_id, + std::span ciphertext, + std::chrono::sys_time unix_ts, + const array_uc32& pro_backend_pubkey) { + auto [decrypted_blob, session_id] = decrypt_from_blinded_recipient( + ed25519_privkey, community_pubkey, sender_id, recipient_id, ciphertext); + DecodedCommunityMessage result = + decode_for_community(decrypted_blob, unix_ts, pro_backend_pubkey); + return result; +} + void make_blake2b32_hasher( crypto_generichash_blake2b_state* hasher, std::string_view personalization) { assert(personalization.data() == nullptr || @@ -1458,6 +1518,85 @@ LIBSESSION_C_API void session_protocol_encode_for_destination_free( } } +static session_protocol_decoded_community_message +session_protocol_decoded_community_message_from_cpp(const DecodedCommunityMessage& decoded) { + session_protocol_decoded_community_message result = {}; + result.has_envelope = decoded.envelope.has_value(); + if (result.has_envelope) + result.envelope = envelope_from_cpp(*decoded.envelope); + result.content_plaintext = session::span_u8_copy_or_throw( + decoded.content_plaintext.data(), decoded.content_plaintext.size()); + result.has_pro = decoded.pro.has_value(); + if (decoded.pro_sig) + std::memcpy(result.pro_sig.data, decoded.pro_sig->data(), decoded.pro_sig->max_size()); + if (decoded.pro) + result.pro = decoded_pro_from_cpp(*decoded.pro); + result.success = true; + return result; +} + +LIBSESSION_C_API session_protocol_decoded_community_message +session_protocol_decode_for_community_inbox( + const unsigned char* ed25519_privkey, + size_t ed25519_privkey_len, + const unsigned char* community_pubkey, + size_t community_pubkey_len, + const unsigned char* sender_id, + size_t sender_id_len, + const unsigned char* recipient_id, + size_t recipient_id_len, + const unsigned char* ciphertext, + size_t ciphertext_len, + uint64_t unix_ts_ms, + OPTIONAL const void* pro_backend_pubkey, + OPTIONAL size_t pro_backend_pubkey_len, + OPTIONAL char* error, + size_t error_len) { + std::span ed25519_privkey_span = {ed25519_privkey, ed25519_privkey_len}; + std::span community_pubkey_span = {community_pubkey, community_pubkey_len}; + std::span sender_id_span = {sender_id, sender_id_len}; + std::span recipient_id_span = {recipient_id, recipient_id_len}; + std::span ciphertext_span = {ciphertext, ciphertext_len}; + auto unix_ts = + std::chrono::sys_time(std::chrono::milliseconds(unix_ts_ms)); + + session_protocol_decoded_community_message result = {}; + array_uc32_from_ptr_result pro_backend_pubkey_cpp = + array_uc32_from_ptr(pro_backend_pubkey, pro_backend_pubkey_len); + if (!pro_backend_pubkey_cpp.success) { + result.error_len_incl_null_terminator = snprintf_clamped( + error, + error_len, + "Invalid pro_backend_pubkey: Key was " + "set but was not 32 bytes, was: %zu", + pro_backend_pubkey_len) + + 1; + return result; + } + + try { + DecodedCommunityMessage decoded = decode_for_community_inbox( + ed25519_privkey_span, + community_pubkey_span, + sender_id_span, + recipient_id_span, + ciphertext_span, + unix_ts, + pro_backend_pubkey_cpp.data); + result = session_protocol_decoded_community_message_from_cpp(decoded); + } catch (const std::exception& e) { + std::string error_cpp = e.what(); + result.error_len_incl_null_terminator = snprintf_clamped( + error, + error_len, + "%.*s", + static_cast(error_cpp.size()), + error_cpp.data()) + + 1; + } + return result; +} + LIBSESSION_C_API session_protocol_decoded_envelope session_protocol_decode_envelope( const session_protocol_decode_envelope_keys* keys, @@ -1597,17 +1736,7 @@ session_protocol_decoded_community_message session_protocol_decode_for_community try { DecodedCommunityMessage decoded = decode_for_community( content_or_envelope_payload_span, unix_ts, pro_backend_pubkey_cpp.data); - result.has_envelope = decoded.envelope.has_value(); - if (result.has_envelope) - result.envelope = envelope_from_cpp(*decoded.envelope); - result.content_plaintext = session::span_u8_copy_or_throw( - decoded.content_plaintext.data(), decoded.content_plaintext.size()); - result.has_pro = decoded.pro.has_value(); - if (decoded.pro_sig) - std::memcpy(result.pro_sig.data, decoded.pro_sig->data(), decoded.pro_sig->max_size()); - if (decoded.pro) - result.pro = decoded_pro_from_cpp(*decoded.pro); - result.success = true; + result = session_protocol_decoded_community_message_from_cpp(decoded); } catch (const std::exception& e) { std::string error_cpp = e.what(); result.success = false; diff --git a/tests/test_session_protocol.cpp b/tests/test_session_protocol.cpp index 8b154c5f..4e65f11d 100644 --- a/tests/test_session_protocol.cpp +++ b/tests/test_session_protocol.cpp @@ -109,7 +109,7 @@ static SerialisedProtobufContentWithProForTesting build_protobuf_content_with_se TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { // Do tests that require no setup - SECTION("Ensure get pro fetaures detects large message") { + SECTION("Ensure get pro features detects large message") { // Try a message below the size threshold { auto msg = std::string(SESSION_PROTOCOL_PRO_STANDARD_CHARACTER_LIMIT, 'a'); @@ -847,36 +847,37 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { REQUIRE(decoded.pro.status == SESSION_PROTOCOL_PRO_STATUS_VALID); } - SECTION("Encode/decode for community inbox (content message)") { - const auto community_seed = - "0123456789abcdef0123456789abcdeff00baadeadb33f000000000000000000"_hexbytes; - array_uc64 community_sk = {}; - array_uc32 community_pk = {}; - crypto_sign_ed25519_seed_keypair( - community_pk.data(), community_sk.data(), community_seed.data()); - - bytes32 session_blind15_sk0 = {}; - bytes33 session_blind15_pk0 = {}; - session_blind15_pk0.data[0] = 0x15; - session_blind15_key_pair( - keys.ed_sk0.data(), - community_pk.data(), - session_blind15_pk0.data + 1, - session_blind15_sk0.data); - - bytes32 session_blind15_sk1 = {}; - bytes33 session_blind15_pk1 = {}; - session_blind15_pk1.data[0] = 0x15; - session_blind15_key_pair( - keys.ed_sk1.data(), - community_pk.data(), - session_blind15_pk1.data + 1, - session_blind15_sk1.data); - - bytes33 recipient_pubkey = session_blind15_pk1; - bytes32 community_pubkey = {}; - std::memcpy(community_pubkey.data, community_pk.data(), community_pk.size()); + // NOTE: Setup some keys for community inbox testing + const auto community_seed = + "0123456789abcdef0123456789abcdeff00baadeadb33f000000000000000000"_hexbytes; + array_uc64 community_sk = {}; + array_uc32 community_pk = {}; + crypto_sign_ed25519_seed_keypair( + community_pk.data(), community_sk.data(), community_seed.data()); + + bytes32 session_blind15_sk0 = {}; + bytes33 session_blind15_pk0 = {}; + session_blind15_pk0.data[0] = 0x15; + session_blind15_key_pair( + keys.ed_sk0.data(), + community_pk.data(), + session_blind15_pk0.data + 1, + session_blind15_sk0.data); + + bytes32 session_blind15_sk1 = {}; + bytes33 session_blind15_pk1 = {}; + session_blind15_pk1.data[0] = 0x15; + session_blind15_key_pair( + keys.ed_sk1.data(), + community_pk.data(), + session_blind15_pk1.data + 1, + session_blind15_sk1.data); + + bytes33 recipient_pubkey = session_blind15_pk1; + bytes32 community_pubkey = {}; + std::memcpy(community_pubkey.data, community_pk.data(), community_pk.size()); + SECTION("Encode/decode for community inbox (content message)") { session_protocol_encoded_for_destination encoded = session_protocol_encode_for_community_inbox( protobuf_content.plaintext.data(), @@ -892,22 +893,178 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") { sizeof(error)); scope_exit encoded_free{[&]() { session_protocol_encode_for_destination_free(&encoded); }}; - auto [decrypted_cipher, sender_id] = session::decrypt_from_blinded_recipient( - keys.ed_sk1, - community_pk, - {session_blind15_pk0.data, sizeof(session_blind15_pk0.data)}, - {session_blind15_pk1.data, sizeof(session_blind15_pk1.data)}, - {encoded.ciphertext.data, encoded.ciphertext.size}); + session_protocol_decoded_community_message decoded = + session_protocol_decode_for_community_inbox( + keys.ed_sk1.data(), + keys.ed_sk1.size(), + community_pk.data(), + community_pk.size(), + /*sender*/ session_blind15_pk0.data, + sizeof(session_blind15_pk0.data), + /*recipient*/ session_blind15_pk1.data, + sizeof(session_blind15_pk1.data), + encoded.ciphertext.data, + encoded.ciphertext.size, + timestamp_ms.time_since_epoch().count(), + pro_backend_ed_pk.data(), + pro_backend_ed_pk.size(), + error, + sizeof(error)); + INFO(error); + REQUIRE(decoded.error_len_incl_null_terminator == 0); + REQUIRE(decoded.success); + scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; + REQUIRE(!decoded.has_pro); + } - session_protocol_decoded_community_message decoded = session_protocol_decode_for_community( - decrypted_cipher.data(), - decrypted_cipher.size(), - timestamp_ms.time_since_epoch().count(), - pro_backend_ed_pk.data(), - pro_backend_ed_pk.size(), + SECTION("Encode/decode for community inbox (content message+pro)") { + session_protocol_encoded_for_destination encoded = + session_protocol_encode_for_community_inbox( + protobuf_content.plaintext.data(), + protobuf_content.plaintext.size(), + keys.ed_sk0.data(), + keys.ed_sk0.size(), + timestamp_ms.time_since_epoch().count(), + &recipient_pubkey, + &community_pubkey, + user_pro_ed_sk.data(), + user_pro_ed_sk.size(), + error, + sizeof(error)); + scope_exit encoded_free{[&]() { session_protocol_encode_for_destination_free(&encoded); }}; + + session_protocol_decoded_community_message decoded = + session_protocol_decode_for_community_inbox( + keys.ed_sk1.data(), + keys.ed_sk1.size(), + community_pk.data(), + community_pk.size(), + /*sender*/ session_blind15_pk0.data, + sizeof(session_blind15_pk0.data), + /*recipient*/ session_blind15_pk1.data, + sizeof(session_blind15_pk1.data), + encoded.ciphertext.data, + encoded.ciphertext.size, + timestamp_ms.time_since_epoch().count(), + pro_backend_ed_pk.data(), + pro_backend_ed_pk.size(), + error, + sizeof(error)); + INFO(error); + REQUIRE(decoded.error_len_incl_null_terminator == 0); + REQUIRE(decoded.success); + scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; + REQUIRE(decoded.has_pro); + REQUIRE(decoded.pro.status == SESSION_PROTOCOL_PRO_STATUS_VALID); + } + + SECTION("Encode/decode for community inbox (envelope)") { + // TODO: For these tests we call directly into the encode for destination function. The + // public helper-functions that wrap over this lower level function construct content + // community messages. This is the default behaviour before we transition to envelopes for + // community messages. + // + // To test envelope construction for communities we bypass that function as we need to set + // the type to Envelope community inbox. + session_protocol_destination dest = {}; + dest.type = SESSION_PROTOCOL_DESTINATION_TYPE_ENVELOPE_COMMUNITY_INBOX; + dest.sent_timestamp_ms = timestamp_ms.time_since_epoch().count(); + dest.recipient_pubkey = session_blind15_pk1; + dest.community_inbox_server_pubkey = community_pubkey; + + // Build content without pro attached + std::string plaintext; + { + SessionProtos::Content content = {}; + content.set_sigtimestamp(timestamp_ms.time_since_epoch().count()); + + SessionProtos::DataMessage* data = content.mutable_datamessage(); + data->set_body(std::string(data_body)); + plaintext = content.SerializeAsString(); + REQUIRE(plaintext.size() > data_body.size()); + } + + memset(error, 0, sizeof(error)); + session_protocol_encoded_for_destination encoded = session_protocol_encode_for_destination( + plaintext.data(), + plaintext.size(), + keys.ed_sk0.data(), + keys.ed_sk0.size(), + &dest, error, sizeof(error)); + scope_exit encoded_free{[&]() { session_protocol_encode_for_destination_free(&encoded); }}; + REQUIRE(encoded.success); + + memset(error, 0, sizeof(error)); + session_protocol_decoded_community_message decoded = + session_protocol_decode_for_community_inbox( + keys.ed_sk1.data(), + keys.ed_sk1.size(), + community_pk.data(), + community_pk.size(), + /*sender*/ session_blind15_pk0.data, + sizeof(session_blind15_pk0.data), + /*recipient*/ session_blind15_pk1.data, + sizeof(session_blind15_pk1.data), + encoded.ciphertext.data, + encoded.ciphertext.size, + timestamp_ms.time_since_epoch().count(), + nullptr, + 0, + error, + sizeof(error)); + INFO("ERROR: " << error << ", cipher was: " << encoded.ciphertext.size << "b"); + REQUIRE(decoded.error_len_incl_null_terminator == 0); + REQUIRE(decoded.success); scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; REQUIRE(!decoded.has_pro); } + + SECTION("Encode/decode for community inbox (envelope+pro)") { + session_protocol_destination dest = {}; + dest.type = SESSION_PROTOCOL_DESTINATION_TYPE_ENVELOPE_COMMUNITY_INBOX; + dest.pro_rotating_ed25519_privkey = user_pro_ed_sk.data(); + dest.pro_rotating_ed25519_privkey_len = user_pro_ed_sk.size(); + dest.sent_timestamp_ms = timestamp_ms.time_since_epoch().count(); + dest.recipient_pubkey = session_blind15_pk1; + dest.community_inbox_server_pubkey = community_pubkey; + + memset(error, 0, sizeof(error)); + session_protocol_encoded_for_destination encoded = session_protocol_encode_for_destination( + protobuf_content.plaintext.data(), + protobuf_content.plaintext.size(), + keys.ed_sk0.data(), + keys.ed_sk0.size(), + &dest, + error, + sizeof(error)); + scope_exit encoded_free{[&]() { session_protocol_encode_for_destination_free(&encoded); }}; + REQUIRE(encoded.success); + + memset(error, 0, sizeof(error)); + session_protocol_decoded_community_message decoded = + session_protocol_decode_for_community_inbox( + keys.ed_sk1.data(), + keys.ed_sk1.size(), + community_pk.data(), + community_pk.size(), + /*sender*/ session_blind15_pk0.data, + sizeof(session_blind15_pk0.data), + /*recipient*/ session_blind15_pk1.data, + sizeof(session_blind15_pk1.data), + encoded.ciphertext.data, + encoded.ciphertext.size, + timestamp_ms.time_since_epoch().count(), + pro_backend_ed_pk.data(), + pro_backend_ed_pk.size(), + error, + sizeof(error)); + INFO("ERROR: " << error << ", cipher was: " << encoded.ciphertext.size << "b"); + REQUIRE(decoded.error_len_incl_null_terminator == 0); + REQUIRE(decoded.success); + scope_exit decoded_free{[&]() { session_protocol_decode_for_community_free(&decoded); }}; + REQUIRE(decoded.has_pro); + REQUIRE(decoded.pro.status == SESSION_PROTOCOL_PRO_STATUS_VALID); + } } From 44d5e338016765fee5dc91e28c70ba66f9bb720c Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 19 Mar 2025 13:29:00 +1100 Subject: [PATCH 02/72] Added SQLCipher dependency, started testing database access --- CMakeLists.txt | 3 + cmake/StaticBuild.cmake | 41 +++++++++ include/session/database/connection.hpp | 35 ++++++++ src/CMakeLists.txt | 13 +++ src/database/connection.cpp | 115 ++++++++++++++++++++++++ tests/CMakeLists.txt | 8 ++ tests/test_database.cpp | 22 +++++ 7 files changed, 237 insertions(+) create mode 100644 include/session/database/connection.hpp create mode 100644 src/database/connection.cpp create mode 100644 tests/test_database.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ee0e551..18f82f70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,9 @@ option(USE_LTO "Use Link-Time Optimization" ${use_lto_default}) # Provide this as an option for now because GMP and Desktop are sometimes unhappy with each other. option(ENABLE_ONIONREQ "Build with onion request functionality" ON) +# Provide this as an option for now so clients that don't use any database logic can exclude it. +option(ENABLE_DATABASE "Build with database functionality" ON) + if(USE_LTO) include(CheckIPOSupported) check_ipo_supported(RESULT IPO_ENABLED OUTPUT ipo_error) diff --git a/cmake/StaticBuild.cmake b/cmake/StaticBuild.cmake index 49863f8c..7e4ac1d7 100644 --- a/cmake/StaticBuild.cmake +++ b/cmake/StaticBuild.cmake @@ -5,6 +5,14 @@ set(LOCAL_MIRROR "" CACHE STRING "local mirror path/URL for lib downloads") +set(SQLCIPHER_VERSION v4.6.1 CACHE STRING "sqlcipher version") +set(SQLCIPHER_MIRROR ${LOCAL_MIRROR} https://github.com/sqlcipher/sqlcipher/archive/refs/tags/${SQLCIPHER_VERSION} + CACHE STRING "sqlcipher mirror(s)") +set(SQLCIPHER_SOURCE ${SQLCIPHER_VERSION}.tar.gz) +set(SQLCIPHER_HASH SHA512=023b2fc7248fe38b758ef93dd8436677ff0f5d08b1061e7eab0adb9e38ad92d523e0ab69016ee69bd35c1fd53c10f61e99b01f7a2987a1f1d492e1f7216a0a9c + CACHE STRING "sqlcipher source hash") + + include(ExternalProject) set(DEPS_DESTDIR ${CMAKE_BINARY_DIR}/static-deps) @@ -222,3 +230,36 @@ endif() if(MINGW) link_libraries(-Wl,-Bstatic -lpthread) endif() + +# SQLCipher configuration +if(ENABLE_DATABASE) + set(sqlcipher_extra_configure) + set(sqlcipher_extra_cflags) + set(sqlcipher_extra_ldflags) + + if(APPLE) + # On macOS, use CommonCrypto (installed by default). + set(sqlcipher_extra_configure "--with-crypto-lib=commoncrypto") + set(sqlcipher_extra_cflags " -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5 -DSQLCIPHER_CRYPTO_COMMONCRYPTO") + set(sqlcipher_extra_ldflags " -framework Security -framework Foundation -framework CoreFoundation") + else() + # On Linux, Windows, etc., use OpenSSL. + find_package(OpenSSL REQUIRED) + set(sqlcipher_extra_cflags " -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5 -DSQLCIPHER_CRYPTO_OPENSSL") + endif() + + build_external(sqlcipher + CONFIGURE_COMMAND ./configure ${build_host} --disable-shared --prefix=${DEPS_DESTDIR} --with-pic --enable-fts5 --enable-static ${sqlcipher_extra_configure} + "CC=${deps_cc}" "CXX=${deps_cxx}" "CFLAGS=${deps_CFLAGS}${apple_cflags_arch}${sqlcipher_extra_cflags}" "CXXFLAGS=${deps_CXXFLAGS}${apple_cxxflags_arch}" + "LDFLAGS=-L${DEPS_DESTDIR}/lib${apple_ldflags_arch}${sqlcipher_extra_ldflags}" ${cross_rc} CC_FOR_BUILD=cc CPP_FOR_BUILD=cpp + "--disable-tcl" "--disable-readline" + ) + add_static_target(sqlcipher::sqlcipher sqlcipher_external libsqlcipher.a) + + if(APPLE) + add_library(sqlcipher_frameworks INTERFACE) + target_link_libraries(sqlcipher_frameworks INTERFACE "-framework CoreFoundation" "-framework Security" "-framework Foundation") + add_dependencies(sqlcipher::sqlcipher sqlcipher_frameworks) + target_link_libraries(sqlcipher::sqlcipher INTERFACE sqlcipher_frameworks) + endif() +endif() \ No newline at end of file diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp new file mode 100644 index 00000000..6ae4b684 --- /dev/null +++ b/include/session/database/connection.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +struct sqlite3; +struct sqlite3_stmt; + +namespace session::database { + +class Connection { +private: + sqlite3* db_{nullptr}; + std::string key_; + +public: + Connection(const std::string& path, const std::string& key); + ~Connection(); + + // Prevent copying + Connection(const Connection&) = delete; + Connection& operator=(const Connection&) = delete; + + // Allow moving + Connection(Connection&& other) noexcept; + Connection& operator=(Connection&& other) noexcept; + + sqlite3* handle() { return db_; } + + void exec(const std::string& sql); + void query(const std::string& sql, std::function callback); +}; + +} // namespace session::database \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4354c189..ba1d4545 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -131,6 +131,19 @@ if(ENABLE_ONIONREQ) endif() endif() +if(ENABLE_DATABASE) + add_libsession_util_library(database + database/connection.cpp + ) + + target_link_libraries(database + PUBLIC + util + PRIVATE + sqlcipher::sqlcipher + ) +endif() + if(WARNINGS_AS_ERRORS AND NOT USE_LTO AND CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION MATCHES "^11\\.") # GCC 11 has an overzealous (and false) stringop-overread warning, but only when LTO is off. # Um, yeah. diff --git a/src/database/connection.cpp b/src/database/connection.cpp new file mode 100644 index 00000000..9ea37cd8 --- /dev/null +++ b/src/database/connection.cpp @@ -0,0 +1,115 @@ +#include "session/database/connection.hpp" +#include +#include +#include + +#define SQLITE_HAS_CODEC +#if(APPLE) +#define SQLCIPHER_CRYPTO_COMMONCRYPTO +#endif +#include + +namespace session::database { + +Connection::Connection(const std::string& path, const std::string& key) : key_(key) { + int rc = sqlite3_open(path.c_str(), &db_); + + if (rc != SQLITE_OK) { + std::string error = sqlite3_errmsg(db_); + sqlite3_close(db_); + db_ = nullptr; + throw std::runtime_error("Failed to open database: " + error); + } + + // Set encryption key + if (!key_.empty()) { + std::string formatted_key = "x'" + key_ + "'"; + rc = sqlite3_key(db_, formatted_key.c_str(), formatted_key.size()); + + if (rc != SQLITE_OK) { + std::string error = sqlite3_errmsg(db_); + sqlite3_close(db_); + db_ = nullptr; + throw std::runtime_error("Failed to set encryption key: " + error); + } + } + + // According to the SQLCipher docs iOS needs the 'cipher_plaintext_header_size' value set to at least + // 32 as iOS extends special privileges to the database and needs this header to be in plaintext + // to determine the file type + // + // For more info see: https://www.zetetic.net/sqlcipher/sqlcipher-api/#cipher_plaintext_header_size + rc = sqlite3_exec(db_, "PRAGMA cipher_plaintext_header_size = 32", nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) { + std::string error = sqlite3_errmsg(db_); + sqlite3_close(db_); + db_ = nullptr; + throw std::runtime_error("Failed to configure database: " + error); + } + + // Verify the key works by reading the sqlite_master table + rc = sqlite3_exec(db_, "SELECT count(*) FROM sqlite_master", nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) { + std::string error = sqlite3_errmsg(db_); + sqlite3_close(db_); + db_ = nullptr; + throw std::runtime_error("Failed to decrypt database: " + error); + } + + // oxen::log::debug("Database connection established successfully"); +} + +Connection::~Connection() { + if (db_) { + sqlite3_close(db_); + db_ = nullptr; + } +} + +Connection::Connection(Connection&& other) noexcept + : db_(other.db_), key_(std::move(other.key_)) { + other.db_ = nullptr; +} + +Connection& Connection::operator=(Connection&& other) noexcept { + if (this != &other) { + if (db_) sqlite3_close(db_); + db_ = other.db_; + key_ = std::move(other.key_); + other.db_ = nullptr; + } + return *this; +} + +void Connection::exec(const std::string& sql) { + char* error_msg = nullptr; + int rc = sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &error_msg); + + if (rc != SQLITE_OK) { + std::string error = error_msg ? error_msg : "unknown error"; + sqlite3_free(error_msg); + throw std::runtime_error("SQL execution failed: " + error); + } +} + +void Connection::query(const std::string& sql, std::function callback) { + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr); + + if (rc != SQLITE_OK) { + throw std::runtime_error("Failed to prepare statement: " + std::string(sqlite3_errmsg(db_))); + } + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + callback(stmt); + } + + if (rc != SQLITE_DONE) { + sqlite3_finalize(stmt); + throw std::runtime_error("Error executing query: " + std::string(sqlite3_errmsg(db_))); + } + + sqlite3_finalize(stmt); +} + +} // namespace session::database \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2e0558c0..87af4f6c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -42,6 +42,10 @@ if (ENABLE_ONIONREQ) list(APPEND LIB_SESSION_UTESTS_SOURCES test_onionreq.cpp) endif() +if (ENABLE_DATABASE) + list(APPEND LIB_SESSION_UTESTS_SOURCES test_database.cpp) +endif() + add_library(test_libs INTERFACE) target_link_libraries(test_libs INTERFACE @@ -56,6 +60,10 @@ else() target_compile_definitions(test_libs INTERFACE DISABLE_ONIONREQ) endif() +if (ENABLE_DATABASE) + target_link_libraries(test_libs INTERFACE libsession::database) +endif() + add_executable(testAll main.cpp ${LIB_SESSION_UTESTS_SOURCES}) target_link_libraries(testAll PRIVATE test_libs diff --git a/tests/test_database.cpp b/tests/test_database.cpp new file mode 100644 index 00000000..9662a3ff --- /dev/null +++ b/tests/test_database.cpp @@ -0,0 +1,22 @@ +#include + +#include +#include +#include +#include + +#include "session/database/connection.hpp" +#include "utils.hpp" + +const std::string test_db_path = ""; +const std::string test_db_key = ""; + +TEST_CASE("Database", "[database][open]") { + auto db = session::database::Connection(test_db_path, test_db_key); + + db.query("SELECT id, name FROM profile", [&](sqlite3_stmt* stmt) { + std::string id = reinterpret_cast(sqlite3_column_text(stmt, 0)); + std::string name = reinterpret_cast(sqlite3_column_text(stmt, 1)); + std::cout << "RAWR " + id + ", " + name << std::endl; + }); +} From e1b27e0739845c8c121adf82acb11a3d7923fbf5 Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 22 Oct 2025 17:36:40 +1100 Subject: [PATCH 03/72] Merge in wip-SQLite branch --- README.md | 3 +++ cmake/StaticBuild.cmake | 32 ++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ae491063..8e990bc8 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ ## Build ``` +# Pre-requisites +apt install cmake build-essential git tcl libssl-dev m4 + # Configure the build # # Options diff --git a/cmake/StaticBuild.cmake b/cmake/StaticBuild.cmake index 7e4ac1d7..7de4e795 100644 --- a/cmake/StaticBuild.cmake +++ b/cmake/StaticBuild.cmake @@ -234,32 +234,36 @@ endif() # SQLCipher configuration if(ENABLE_DATABASE) set(sqlcipher_extra_configure) - set(sqlcipher_extra_cflags) + set(sqlcipher_extra_cflags "-DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5") + set(sqlcipher_extra_cxxflags) set(sqlcipher_extra_ldflags) - if(APPLE) # On macOS, use CommonCrypto (installed by default). set(sqlcipher_extra_configure "--with-crypto-lib=commoncrypto") - set(sqlcipher_extra_cflags " -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5 -DSQLCIPHER_CRYPTO_COMMONCRYPTO") - set(sqlcipher_extra_ldflags " -framework Security -framework Foundation -framework CoreFoundation") + set(sqlcipher_extra_cflags "${sqlcipher_extra_cflags} ${apple_cflags_arch} -DSQLCIPHER_CRYPTO_COMMONCRYPTO") + set(sqlcipher_extra_cxxflags "${apple_cxxflags_arch}") + set(sqlcipher_extra_ldflags "${apple_ldflags_arch} -framework Security -framework Foundation -framework CoreFoundation") else() # On Linux, Windows, etc., use OpenSSL. find_package(OpenSSL REQUIRED) - set(sqlcipher_extra_cflags " -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5 -DSQLCIPHER_CRYPTO_OPENSSL") + set(sqlcipher_extra_cflags "-DSQLCIPHER_CRYPTO_OPENSSL") endif() build_external(sqlcipher - CONFIGURE_COMMAND ./configure ${build_host} --disable-shared --prefix=${DEPS_DESTDIR} --with-pic --enable-fts5 --enable-static ${sqlcipher_extra_configure} - "CC=${deps_cc}" "CXX=${deps_cxx}" "CFLAGS=${deps_CFLAGS}${apple_cflags_arch}${sqlcipher_extra_cflags}" "CXXFLAGS=${deps_CXXFLAGS}${apple_cxxflags_arch}" - "LDFLAGS=-L${DEPS_DESTDIR}/lib${apple_ldflags_arch}${sqlcipher_extra_ldflags}" ${cross_rc} CC_FOR_BUILD=cc CPP_FOR_BUILD=cpp - "--disable-tcl" "--disable-readline" + CONFIGURE_COMMAND ./configure ${build_host} --disable-shared --prefix=${DEPS_DESTDIR} + --with-pic --enable-fts5 --enable-static --disable-tcl --disable-readline ${sqlcipher_extra_configure} + "CC=${deps_cc}" + "CXX=${deps_cxx}" + "CFLAGS=${deps_CFLAGS} ${sqlcipher_extra_cflags} -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5" + "CXXFLAGS=${deps_CXXFLAGS} ${sqlcipher_extra_cxxflags}" + "LDFLAGS=-L${DEPS_DESTDIR}/lib ${sqlcipher_extra_ldflags}" + ${cross_rc} ) add_static_target(sqlcipher::sqlcipher sqlcipher_external libsqlcipher.a) if(APPLE) - add_library(sqlcipher_frameworks INTERFACE) - target_link_libraries(sqlcipher_frameworks INTERFACE "-framework CoreFoundation" "-framework Security" "-framework Foundation") - add_dependencies(sqlcipher::sqlcipher sqlcipher_frameworks) - target_link_libraries(sqlcipher::sqlcipher INTERFACE sqlcipher_frameworks) + target_link_libraries(sqlcipher::sqlcipher INTERFACE "-framework CoreFoundation" "-framework Security" "-framework Foundation") + else() + target_link_libraries(sqlcipher::sqlcipher INTERFACE OpenSSL::SSL) endif() -endif() \ No newline at end of file +endif() From cd32d82ce1e2e8bdbf08c267b52eadc0d37f0e2d Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 23 Oct 2025 11:03:25 +1100 Subject: [PATCH 04/72] Update pre-requisite dependencies line --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e990bc8..0c363c57 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ``` # Pre-requisites -apt install cmake build-essential git tcl libssl-dev m4 +apt install cmake build-essential git tcl libssl-dev m4 pkg-config # Configure the build # From 7717345df344994b24f7192797bad73b2a76b49b Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 23 Oct 2025 11:59:05 +1100 Subject: [PATCH 05/72] Remove defines from database.cpp, use target_compile_definitions --- cmake/StaticBuild.cmake | 47 ++++++++++++++++++++++++++----------- src/database/connection.cpp | 8 +------ 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/cmake/StaticBuild.cmake b/cmake/StaticBuild.cmake index 7de4e795..822a61b9 100644 --- a/cmake/StaticBuild.cmake +++ b/cmake/StaticBuild.cmake @@ -231,38 +231,57 @@ if(MINGW) link_libraries(-Wl,-Bstatic -lpthread) endif() -# SQLCipher configuration if(ENABLE_DATABASE) set(sqlcipher_extra_configure) - set(sqlcipher_extra_cflags "-DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5") + set(sqlcipher_extra_cflags) set(sqlcipher_extra_cxxflags) set(sqlcipher_extra_ldflags) + set(sqlcipher_compile_definitions) + list(APPEND sqlcipher_compile_definitions + SQLITE_HAS_CODEC + SQLITE_TEMP_STORE=3 + SQLITE_ENABLE_FTS5 + ) + if(APPLE) - # On macOS, use CommonCrypto (installed by default). - set(sqlcipher_extra_configure "--with-crypto-lib=commoncrypto") - set(sqlcipher_extra_cflags "${sqlcipher_extra_cflags} ${apple_cflags_arch} -DSQLCIPHER_CRYPTO_COMMONCRYPTO") - set(sqlcipher_extra_cxxflags "${apple_cxxflags_arch}") - set(sqlcipher_extra_ldflags "${apple_ldflags_arch} -framework Security -framework Foundation -framework CoreFoundation") + # On macOS, use CommonCrypto (comes with the OS) + list(APPEND sqlcipher_extra_configure --with-crypto-lib=commoncrypto) + list(APPEND sqlcipher_extra_cflags ${apple_cflags_arch}) + list(APPEND sqlcipher_extra_cxxflags ${apple_cxxflags_arch}) + list(APPEND sqlcipher_extra_ldflags ${apple_ldflags_arch} -framework Security -framework Foundation -framework CoreFoundation) + list(APPEND sqlcipher_compile_definitions SQLCIPHER_CRYPTO_COMMONCRYPTO) else() - # On Linux, Windows, etc., use OpenSSL. + # On Linux, Windows, etc., use OpenSSL find_package(OpenSSL REQUIRED) - set(sqlcipher_extra_cflags "-DSQLCIPHER_CRYPTO_OPENSSL") + list(APPEND sqlcipher_compile_definitions SQLCIPHER_CRYPTO_OPENSSL) endif() + set(sqlcipher_cflags "${deps_CFLAGS} ${sqlcipher_extra_cflags}") + set(sqlcipher_cxxflags "${deps_CXXFLAGS} ${sqlcipher_extra_cxxflags}") + set(sqlcipher_ldflags "-L${DEPS_DESTDIR}/lib ${sqlcipher_extra_ldflags}") + + # Append compile definitions static build of sqlcipher via CFLAGS + foreach(def ${sqlcipher_compile_definitions}) + set(sqlcipher_cflags "${sqlcipher_cflags} -D${def}") + endforeach() + + # Configure and build SQLCipher build_external(sqlcipher CONFIGURE_COMMAND ./configure ${build_host} --disable-shared --prefix=${DEPS_DESTDIR} --with-pic --enable-fts5 --enable-static --disable-tcl --disable-readline ${sqlcipher_extra_configure} "CC=${deps_cc}" "CXX=${deps_cxx}" - "CFLAGS=${deps_CFLAGS} ${sqlcipher_extra_cflags} -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -DSQLITE_ENABLE_FTS5" - "CXXFLAGS=${deps_CXXFLAGS} ${sqlcipher_extra_cxxflags}" - "LDFLAGS=-L${DEPS_DESTDIR}/lib ${sqlcipher_extra_ldflags}" + "CFLAGS=${sqlcipher_cflags}" + "CXXFLAGS=${sqlcipher_cxxflags}" + "LDFLAGS=${sqlcipher_ldflags}" ${cross_rc} ) - add_static_target(sqlcipher::sqlcipher sqlcipher_external libsqlcipher.a) + # Setup CMake target for the rest of libsession to use + add_static_target(sqlcipher::sqlcipher sqlcipher_external libsqlcipher.a) + target_compile_definitions(sqlcipher::sqlcipher INTERFACE ${sqlcipher_compile_definitions}) if(APPLE) - target_link_libraries(sqlcipher::sqlcipher INTERFACE "-framework CoreFoundation" "-framework Security" "-framework Foundation") + target_link_libraries(sqlcipher::sqlcipher INTERFACE ${sqlcipher_extra_ldflags}) else() target_link_libraries(sqlcipher::sqlcipher INTERFACE OpenSSL::SSL) endif() diff --git a/src/database/connection.cpp b/src/database/connection.cpp index 9ea37cd8..d81e3338 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -2,15 +2,9 @@ #include #include #include - -#define SQLITE_HAS_CODEC -#if(APPLE) -#define SQLCIPHER_CRYPTO_COMMONCRYPTO -#endif #include namespace session::database { - Connection::Connection(const std::string& path, const std::string& key) : key_(key) { int rc = sqlite3_open(path.c_str(), &db_); @@ -112,4 +106,4 @@ void Connection::query(const std::string& sql, std::function Date: Thu, 23 Oct 2025 16:18:13 +1100 Subject: [PATCH 06/72] Add pro revocation add/get/delete from SQL DB APIs --- include/session/database/connection.hpp | 118 +++++++-- include/session/types.h | 5 + src/database/connection.cpp | 306 ++++++++++++++++++------ tests/test_database.cpp | 85 ++++++- 4 files changed, 414 insertions(+), 100 deletions(-) diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index 6ae4b684..1c39d895 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -1,35 +1,117 @@ #pragma once -#include +#include + #include -#include +#include +#include +#include struct sqlite3; struct sqlite3_stmt; namespace session::database { -class Connection { -private: - sqlite3* db_{nullptr}; - std::string key_; +struct AddResult { + bool success; + int return_code; +}; + +struct DeleteResult { + bool success; + int return_code; + size_t count; // Number of rows that were deleted +}; + +struct Connection { + sqlite3* db_; -public: - Connection(const std::string& path, const std::string& key); + Connection(const std::string& path, const cleared_array<48>& raw_key); ~Connection(); - // Prevent copying + // Prevent copying and moving Connection(const Connection&) = delete; Connection& operator=(const Connection&) = delete; - - // Allow moving - Connection(Connection&& other) noexcept; - Connection& operator=(Connection&& other) noexcept; + Connection(Connection&& other) = delete; + Connection& operator=(Connection&& other) = delete; - sqlite3* handle() { return db_; } - + /// API: database/exec + /// + /// Prepares a statement and executes it. Throws if SQLite returned an error + /// + /// Inputs: + /// - `sql` -- SQL statement to prepare and execute void exec(const std::string& sql); - void query(const std::string& sql, std::function callback); -}; -} // namespace session::database \ No newline at end of file + /// API: database/query + /// + /// Prepares a statement, steps the statement, calls the provided `callback` and then finalizes + /// the statement once stepping no longer returns rows. Throws if SQLite returned an error + /// + /// Inputs: + /// - `sql` -- SQL statement to prepare + /// - `callback` -- User defined function to execute when a row is returned + void query(std::string_view sql, std::function callback); + + /// API: database/add_pro_revocations + /// + /// Add a list of Session Pro revocations into the database associated with this connection. + /// This function is transactional, on failure changes to the database are rolled back. + /// + /// Inputs: + /// - `revocations` -- The list of revocations to add + /// + /// Outputs: + /// - `success` -- True if the add was successful, false otherwise. + /// - `return_code` -- The SQLite3 error code that caused the error if `success` was false + AddResult add_pro_revocations( + std::span revocations) noexcept; + + /// API: database/delete_pro_revocations + /// + /// Delete a list of Session Pro revocations from the database associated with this connection. + /// Revocations are deleted by matching the item's `gen_index_hash`. This function is + /// transactional, on failure changes to the database are rolled back. + /// + /// Inputs: + /// - `revocations` -- The list of revocations to delete + /// + /// Outputs: + /// - `success` -- True if the delete was successful, false otherwise. + /// - `return_code` -- The SQLite3 error code that caused the error if `success` was false + /// - `count` -- Number of rows that were deleted in this operation + DeleteResult delete_pro_revocations( + std::span revocations) noexcept; + + /// API: database/get_pro_revocations_buffer + /// + /// Retrieve the Session Pro Backend revocation list given and output the rows into the given + /// `buf`. This function throws if SQLite returned an error + /// + /// Inputs: + /// - `buf` -- Buffer to write loaded revocations into. This can be nullptr in which case the + /// function returns the number of revocations currently in the DB. + /// - `buf_count` -- Size of the buffer and consequently the amount of revocations to load. This + /// can be 0 as well as setting `buf` to `nullptr` to make the function return the number of + /// revocations currently in the DB. + /// - `offset` -- Start retrieving revocation rows from this specified index of the list. Pass + /// in 0 to start from the beginning. + /// + /// Outputs: + /// - `size_t` -- Number of revocation items read from the database. If the buffer was + /// insufficient sized to receive the rows, the return value is always capped to the size of + /// the buffer. If `buf` and `buf_count` are nullptr or 0 respectively, then value returned + /// is the amount of revocation items in the DB at the time of execution. + size_t get_pro_revocations_buffer( + pro_backend::ProRevocationItem* buf, size_t buf_count, size_t offset); + + /// API: database/get_pro_revocations + /// + /// Retrieve the Session Pro Backend revocation list from the database. This function throws if + /// there was an allocation or SQLite returned an error + /// + /// Outputs: + /// - `std::vector` -- List of revocation items + std::vector get_pro_revocations(); +}; +} // namespace session::database diff --git a/include/session/types.h b/include/session/types.h index 5029cab3..500b2dff 100644 --- a/include/session/types.h +++ b/include/session/types.h @@ -44,6 +44,11 @@ struct bytes64 { uint8_t data[64]; }; +typedef struct bytes48 bytes48; +struct bytes48 { + uint8_t data[48]; +}; + /// Basic bump allocating arena typedef struct arena_t arena_t; struct arena_t { diff --git a/src/database/connection.cpp b/src/database/connection.cpp index d81e3338..63096436 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -1,56 +1,101 @@ #include "session/database/connection.hpp" -#include + #include -#include #include +#include +#include +#include +#include +#include + +auto logcat = oxen::log::Cat("database"); + +namespace { +void close_db_and_throw_error(sqlite3** db, int sql_result, std::string_view error_prefix) { + std::string msg = fmt::format("{}: {}", error_prefix, sqlite3_errstr(sql_result)); + sqlite3_close(*db); + *db = nullptr; + throw std::runtime_error(msg); +} + +int set_db_version_or_throw(sqlite3* db, uint8_t db_version) { + char sql[64]; + size_t sql_size = snprintf(sql, sizeof(sql), "PRAGMA user_version = %u", db_version); + assert(sql_size < sizeof(sql)); + int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr); + return result; +} +}; // namespace + namespace session::database { -Connection::Connection(const std::string& path, const std::string& key) : key_(key) { +Connection::Connection(const std::string& path, const cleared_array<48> &raw_key) { + cleared_array<48> ZERO_RAW_KEY = {}; + assert(memcmp(raw_key.data(), ZERO_RAW_KEY.data(), raw_key.size()) != 0 && "Raw key was not set"); + + // Open DB int rc = sqlite3_open(path.c_str(), &db_); - - if (rc != SQLITE_OK) { - std::string error = sqlite3_errmsg(db_); - sqlite3_close(db_); - db_ = nullptr; - throw std::runtime_error("Failed to open database: " + error); - } - - // Set encryption key - if (!key_.empty()) { - std::string formatted_key = "x'" + key_ + "'"; - rc = sqlite3_key(db_, formatted_key.c_str(), formatted_key.size()); - - if (rc != SQLITE_OK) { - std::string error = sqlite3_errmsg(db_); - sqlite3_close(db_); - db_ = nullptr; - throw std::runtime_error("Failed to set encryption key: " + error); - } - } + if (rc != SQLITE_OK) + close_db_and_throw_error(&db_, rc, "Failed to open database"); - // According to the SQLCipher docs iOS needs the 'cipher_plaintext_header_size' value set to at least - // 32 as iOS extends special privileges to the database and needs this header to be in plaintext - // to determine the file type + // According to the SQLCipher docs iOS needs the 'cipher_plaintext_header_size' value set to at + // least 32 as iOS extends special privileges to the database and needs this header to be in + // plaintext to determine the file type + // + // This keeps the first 32 bytes of the database unencrypted. iOS checks the headers of open + // file-descriptors as a heuristic to determine which processes can get elevated permissions or + // processes that can be culled. SQLite databases are one of those that get elevated. // - // For more info see: https://www.zetetic.net/sqlcipher/sqlcipher-api/#cipher_plaintext_header_size + // For more info, see: + // https://www.zetetic.net/sqlcipher/sqlcipher-api/#cipher_plaintext_header_size rc = sqlite3_exec(db_, "PRAGMA cipher_plaintext_header_size = 32", nullptr, nullptr, nullptr); - if (rc != SQLITE_OK) { - std::string error = sqlite3_errmsg(db_); - sqlite3_close(db_); - db_ = nullptr; - throw std::runtime_error("Failed to configure database: " + error); - } - + if (rc != SQLITE_OK) + close_db_and_throw_error(&db_, rc, "Failed to configure database"); + + // Set encryption key, this is the underlying function that "PRAGMA key = .." calls + std::string fmt_key = fmt::format("x'{}'", oxenc::to_hex(raw_key)); + rc = sqlite3_key(db_, fmt_key.c_str(), fmt_key.size()); + sodium_zero_buffer(fmt_key.data(), fmt_key.size()); + if (rc != SQLITE_OK) + close_db_and_throw_error(&db_, rc, "Failed to set encryption key"); + // Verify the key works by reading the sqlite_master table - rc = sqlite3_exec(db_, "SELECT count(*) FROM sqlite_master", nullptr, nullptr, nullptr); - if (rc != SQLITE_OK) { - std::string error = sqlite3_errmsg(db_); - sqlite3_close(db_); - db_ = nullptr; - throw std::runtime_error("Failed to decrypt database: " + error); + rc = sqlite3_exec(db_, "SELECT COUNT(*) FROM sqlite_master", nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) + close_db_and_throw_error(&db_, rc, "Failed to decrypt database with the given encryption key"); + oxen::log::debug(logcat, "Opened database {} successfully", path); + + // Query the DB version + uint8_t curr_db_version = 0; + query("PRAGMA user_version", + [&](sqlite3_stmt* stmt) { curr_db_version = sqlite3_column_int(stmt, 0); }); + + // Version migrations + const uint8_t TARGET_DB_VERSION = 1; + if (curr_db_version == 0) { + std::string_view bootstrap_sql = R"( +CREATE TABLE IF NOT EXISTS pro_revocations ( + gen_index_hash BLOB PRIMARY KEY NOT NULL, + expiry_unix_ts_ms INTEGER NOT NULL +))"; + // Create the initial DB tables + rc = sqlite3_exec(db_, bootstrap_sql.data(), nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) + close_db_and_throw_error(&db_, rc, "Failed to bootstrap tables"); + + // Teleport to the target version + rc = set_db_version_or_throw(db_, ++curr_db_version); + if (rc != SQLITE_OK) + close_db_and_throw_error( + &db_, rc, fmt::format("Failed to set DB version to {}", curr_db_version)); } - - // oxen::log::debug("Database connection established successfully"); + + // Requery the DB version and ensure all version migrations have occurred + [[maybe_unused]] uint8_t final_version_in_db = 0; + query("PRAGMA user_version", [&final_version_in_db](sqlite3_stmt* stmt) { + final_version_in_db = sqlite3_column_int(stmt, 0); + }); + assert(final_version_in_db == TARGET_DB_VERSION); } Connection::~Connection() { @@ -60,25 +105,9 @@ Connection::~Connection() { } } -Connection::Connection(Connection&& other) noexcept - : db_(other.db_), key_(std::move(other.key_)) { - other.db_ = nullptr; -} - -Connection& Connection::operator=(Connection&& other) noexcept { - if (this != &other) { - if (db_) sqlite3_close(db_); - db_ = other.db_; - key_ = std::move(other.key_); - other.db_ = nullptr; - } - return *this; -} - void Connection::exec(const std::string& sql) { char* error_msg = nullptr; int rc = sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &error_msg); - if (rc != SQLITE_OK) { std::string error = error_msg ? error_msg : "unknown error"; sqlite3_free(error_msg); @@ -86,24 +115,159 @@ void Connection::exec(const std::string& sql) { } } -void Connection::query(const std::string& sql, std::function callback) { +void Connection::query(std::string_view sql, std::function callback) { sqlite3_stmt* stmt = nullptr; - int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, nullptr); - + int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size(), &stmt, nullptr); if (rc != SQLITE_OK) { - throw std::runtime_error("Failed to prepare statement: " + std::string(sqlite3_errmsg(db_))); + throw std::runtime_error( + fmt::format("Failed to prepare statement: {}", sqlite3_errmsg(db_))); } - - while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) callback(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) + throw std::runtime_error(fmt::format("Error executing query: {}", sqlite3_errmsg(db_))); +} + +AddResult Connection::add_pro_revocations( + std::span revocations) noexcept { + + // The following consists of exception safe code so we do not need try catch and can trivially + // commit or rollback the at the end of the function. + exec("BEGIN DEFERRED TRANSACTION;"); + + sqlite3_stmt* stmt = nullptr; + std::string_view sql = R"( +INSERT INTO pro_revocations (gen_index_hash, expiry_unix_ts_ms) +VALUES (?, ?) +)"; + + int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); + for (size_t index = 0; (rc == SQLITE_OK || rc == SQLITE_DONE) && index < revocations.size(); + index++) { + const auto& it = revocations[index]; + int bind = 0; + int64_t expiry = static_cast(it.expiry_unix_ts.time_since_epoch().count()); + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_bind_blob( + stmt, + ++bind, + it.gen_index_hash.data(), + static_cast(it.gen_index_hash.size()), + nullptr) + : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_bind_int64(stmt, ++bind, expiry) : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_step(stmt) : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_reset(stmt) : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_clear_bindings(stmt) : rc; + } + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK || rc == SQLITE_DONE) + rc = finalize_rc; + + AddResult result = {}; + result.return_code = rc; + result.success = result.return_code == SQLITE_OK || result.return_code == SQLITE_DONE; + exec(result.success ? "COMMIT;" : "ROLLBACK;"); + return result; +} + +DeleteResult Connection::delete_pro_revocations( + std::span revocations) noexcept { + + // The following consists of exception safe code so we do not need try catch and can trivially + // commit or rollback the at the end of the function. + exec("BEGIN DEFERRED TRANSACTION;"); + + std::string_view sql = R"( +DELETE FROM pro_revocations +WHERE gen_index_hash = ? +)"; + + size_t row_count = 0; + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); + for (size_t index = 0; (rc == SQLITE_OK || rc == SQLITE_DONE) && index < revocations.size(); + index++) { + const auto& it = revocations[index]; + int64_t expiry = static_cast(it.expiry_unix_ts.time_since_epoch().count()); + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) + ? sqlite3_bind_blob( + stmt, + 1, + it.gen_index_hash.data(), + static_cast(it.gen_index_hash.size()), + nullptr) + : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_step(stmt) : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_reset(stmt) : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_clear_bindings(stmt) : rc; + row_count += sqlite3_changes(db_); } - - if (rc != SQLITE_DONE) { - sqlite3_finalize(stmt); - throw std::runtime_error("Error executing query: " + std::string(sqlite3_errmsg(db_))); + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK || rc == SQLITE_DONE) + rc = finalize_rc; + + DeleteResult result = {}; + result.return_code = rc; + result.success = result.return_code == SQLITE_OK || result.return_code == SQLITE_DONE; + if (result.success) { + exec("COMMIT;"); + result.count = row_count; + } else { + exec("ROLLBACK;"); + result.count = 0; } - - sqlite3_finalize(stmt); + return result; } -} // namespace session::database +size_t Connection::get_pro_revocations_buffer( + pro_backend::ProRevocationItem* buf, size_t buf_count, size_t offset) { + size_t result = 0; + query("SELECT COUNT(*) FROM pro_revocations", + [&](sqlite3_stmt* stmt) { result = sqlite3_column_int(stmt, 0); }); + + if (buf && buf_count) { + char sql[128]; + size_t sql_size = snprintf( + sql, + sizeof(sql), + R"( +SELECT gen_index_hash, expiry_unix_ts_ms +FROM pro_revocations +LIMIT %zu +OFFSET %zu +)", + buf_count, + offset); + assert(sql_size < sizeof(sql)); + + result = 0; + query(std::string_view(sql, sql_size), [&buf, buf_count, &result](sqlite3_stmt* stmt) { + pro_backend::ProRevocationItem& item = buf[result++]; + + // Copy out the gen index blob + const void* gen_index_blob = sqlite3_column_blob(stmt, 0); + int gen_index_hash_size = sqlite3_column_bytes(stmt, 0); + assert(gen_index_hash_size == 32); + std::memcpy( + item.gen_index_hash.data(), + gen_index_blob, + std::min(gen_index_hash_size, static_cast(item.gen_index_hash.size()))); + + // Copy out the expiry timestmap + auto expiry = std::chrono::milliseconds(sqlite3_column_int64(stmt, 1)); + item.expiry_unix_ts = std::chrono::sys_time(expiry); + }); + } + return result; +} + +std::vector Connection::get_pro_revocations() { + std::vector result; + size_t size_req = get_pro_revocations_buffer(nullptr, 0, 0); + result.resize(size_req); + size_t items_read = get_pro_revocations_buffer(result.data(), result.size(), 0); + assert(items_read == size_req); + return result; +} +} // namespace session::database diff --git a/tests/test_database.cpp b/tests/test_database.cpp index 9662a3ff..c44d6539 100644 --- a/tests/test_database.cpp +++ b/tests/test_database.cpp @@ -1,22 +1,85 @@ #include +#include +#include #include -#include -#include #include +#include #include "session/database/connection.hpp" +#include "session/pro_backend.hpp" #include "utils.hpp" -const std::string test_db_path = ""; -const std::string test_db_key = ""; - TEST_CASE("Database", "[database][open]") { - auto db = session::database::Connection(test_db_path, test_db_key); + session::cleared_array<48> raw_key = {}; + randombytes_buf(raw_key.data(), raw_key.size()); + auto db = session::database::Connection(":memory:", raw_key); +} + +TEST_CASE("Database", "[database][pro][revocations]") { + session::cleared_array<48> raw_key = {}; + randombytes_buf(raw_key.data(), raw_key.size()); + auto db = session::database::Connection(":memory:", raw_key); + + // Check that the DB has no revocations in it + size_t db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0); + REQUIRE(db_item_count == 0); + + // Create the revocations we will put into the DB + uint64_t unix_ts_ms = 1698765432ULL * 1000; // Arbitrary timestamp + auto unix_ts = + std::chrono::sys_time(std::chrono::milliseconds(unix_ts_ms)); + + session::pro_backend::ProRevocationItem src_items[] = { + { + .gen_index_hash = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + .expiry_unix_ts = unix_ts, + }, + { + .gen_index_hash = {0x33, 0xa1, 0xc4, 0x4e, 0x60, 0x94, 0x48, 0x8f, + 0x5c, 0xeb, 0xe2, 0x4b, 0xfc, 0xf9, 0x89, 0xda, + 0x07, 0xdd, 0xc4, 0x8d, 0xe2, 0xae, 0x86, 0x6c, + 0x8c, 0x78, 0xb9, 0x16, 0x60, 0xc8, 0x49, 0xf1}, + .expiry_unix_ts = unix_ts, + }, + }; + + // Add the items + session::database::AddResult add = db.add_pro_revocations(src_items); + INFO("Add failed: " << sqlite3_errstr(add.return_code)); + REQUIRE(add.success); + REQUIRE(add.return_code == SQLITE_OK); + + // Count the number of revocations in the DB (should be 2 as we've inserted them) + db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0); + REQUIRE(db_item_count == 2); + + // Check that the revocations was in the DB + std::vector db_items = db.get_pro_revocations(); + REQUIRE(src_items[0].gen_index_hash == db_items[0].gen_index_hash); + REQUIRE(src_items[0].expiry_unix_ts == db_items[0].expiry_unix_ts); + REQUIRE(src_items[1].gen_index_hash == db_items[1].gen_index_hash); + REQUIRE(src_items[1].expiry_unix_ts == db_items[1].expiry_unix_ts); + + // Delete the first item (src[0]) from the DB + session::pro_backend::ProRevocationItem delete_item = src_items[0]; + session::database::DeleteResult delete_result = + db.delete_pro_revocations(std::span(&delete_item, 1)); + INFO("Delete failed: " << sqlite3_errstr(delete_result.return_code)); + REQUIRE(delete_result.success); + REQUIRE(delete_result.return_code == SQLITE_OK); + REQUIRE(delete_result.count == 1); + + // Count the number of revocations in the DB (should be 1 as we've deleted one of them) + db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0); + REQUIRE(db_item_count == 1); - db.query("SELECT id, name FROM profile", [&](sqlite3_stmt* stmt) { - std::string id = reinterpret_cast(sqlite3_column_text(stmt, 0)); - std::string name = reinterpret_cast(sqlite3_column_text(stmt, 1)); - std::cout << "RAWR " + id + ", " + name << std::endl; - }); + // Verify that the DB now has just the item at src[1] + std::vector db_items_after_delete = db.get_pro_revocations(); + REQUIRE(db_items_after_delete.size() == 1); + REQUIRE(src_items[1].gen_index_hash == db_items_after_delete[0].gen_index_hash); + REQUIRE(src_items[1].expiry_unix_ts == db_items_after_delete[0].expiry_unix_ts); } From 696850f6670add5a272ff018c5de2c233de71969 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 27 Oct 2025 14:59:07 +1100 Subject: [PATCH 07/72] Make pro backend server tests off by default, specify URL to let it run --- tests/main.cpp | 8 +- tests/test_pro_backend.cpp | 818 +++++++++++++++++++------------------ 2 files changed, 422 insertions(+), 404 deletions(-) diff --git a/tests/main.cpp b/tests/main.cpp index 76c4cc49..d982b075 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -1,7 +1,7 @@ #include #include -std::string g_test_pro_backend_dev_server_url = "http://127.0.0.1:5000"; +std::string g_test_pro_backend_dev_server_url = ""; int main(int argc, char* argv[]) { Catch::Session session; @@ -20,7 +20,8 @@ int main(int argc, char* argv[]) { "enable oxen log tracing of test cases/sections") | Opt(g_test_pro_backend_dev_server_url, "url")["--pro-backend-dev-server-url"]( "URL to a SESH_PRO_BACKEND_DEV=1 enabled Session Pro Backend server. Only " - "used if compiled with -D TEST_PRO_BACKEND_WITH_DEV_SERVER=1 support"); + "used if compiled with -D TEST_PRO_BACKEND_WITH_DEV_SERVER=1 support. These " + "tests are skipped if not specified."); session.cli(cli); @@ -47,5 +48,8 @@ int main(int argc, char* argv[]) { oxen::log::Cat("testcase"), test_case_tracing ? oxen::log::Level::trace : oxen::log::Level::off); + if (g_test_pro_backend_dev_server_url.empty()) + fprintf(stdout, "Skipping --pro-backend-dev-server-url tests, no URL specified.\n"); + return session.run(); } diff --git a/tests/test_pro_backend.cpp b/tests/test_pro_backend.cpp index 4a4f7240..9ffc40da 100644 --- a/tests/test_pro_backend.cpp +++ b/tests/test_pro_backend.cpp @@ -889,433 +889,447 @@ std::string curl_do_basic_blocking_post_request( } TEST_CASE("Pro Backend Dev Server", "[pro_backend][dev_server]") { - // Setup: Generate keys and payment token hash - bytes32 master_pubkey = {}; - bytes64 master_privkey = {}; - crypto_sign_ed25519_keypair(master_pubkey.data, master_privkey.data); - - bytes32 rotating_pubkey = {}; - bytes64 rotating_privkey = {}; - crypto_sign_ed25519_keypair(rotating_pubkey.data, rotating_privkey.data); - - const auto DEV_BACKEND_PUBKEY = - "fc947730f49eb01427a66e050733294d9e520e545c7a27125a780634e0860a27"_hexbytes; + if (g_test_pro_backend_dev_server_url.size()) { + // Setup: Generate keys and payment token hash + bytes32 master_pubkey = {}; + bytes64 master_privkey = {}; + crypto_sign_ed25519_keypair(master_pubkey.data, master_privkey.data); + + bytes32 rotating_pubkey = {}; + bytes64 rotating_privkey = {}; + crypto_sign_ed25519_keypair(rotating_pubkey.data, rotating_privkey.data); + + const auto DEV_BACKEND_PUBKEY = + "fc947730f49eb01427a66e050733294d9e520e545c7a27125a780634e0860a27"_hexbytes; + + // Setup CURL + curl_global_init(CURL_GLOBAL_DEFAULT); + scope_exit curl_cleanup{[&]() { curl_global_cleanup(); }}; + + CURL* curl = curl_easy_init(); + REQUIRE(curl); + scope_exit curl_free{[&]() { curl_easy_cleanup(curl); }}; + + struct curl_slist* curl_headers = nullptr; + curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json"); + REQUIRE(curl_headers); + scope_exit curl_headers_free{[&]() { curl_slist_free_all(curl_headers); }}; + + // Add pro payment + session_protocol_pro_proof first_pro_proof = {}; + { + std::array fake_google_payment_token; + randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); + std::string fake_google_payment_token_hex = + "DEV." + oxenc::to_hex(fake_google_payment_token); + + std::array fake_google_order_id; + randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); + std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); + + session_pro_backend_add_pro_payment_user_transaction payment_tx = {}; + payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; + payment_tx.payment_id_count = fake_google_payment_token_hex.size(); + payment_tx.order_id_count = fake_google_order_id_hex.size(); + std::memcpy( + payment_tx.payment_id, + fake_google_payment_token_hex.data(), + payment_tx.payment_id_count); + std::memcpy( + payment_tx.order_id, + fake_google_order_id_hex.data(), + payment_tx.order_id_count); - // Setup CURL - curl_global_init(CURL_GLOBAL_DEFAULT); - scope_exit curl_cleanup{[&]() { curl_global_cleanup(); }}; + // Build request + session_pro_backend_master_rotating_signatures add_pro_sigs = + session_pro_backend_add_pro_payment_request_build_sigs( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + rotating_privkey.data, + sizeof(rotating_privkey.data), + payment_tx.provider, + reinterpret_cast(payment_tx.payment_id), + payment_tx.payment_id_count, + reinterpret_cast(payment_tx.order_id), + payment_tx.order_id_count); - CURL* curl = curl_easy_init(); - REQUIRE(curl); - scope_exit curl_free{[&]() { curl_easy_cleanup(curl); }}; + session_pro_backend_add_pro_payment_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.rotating_pkey = rotating_pubkey; + request.payment_tx = payment_tx; + request.master_sig = add_pro_sigs.master_sig; + request.rotating_sig = add_pro_sigs.rotating_sig; + + session_pro_backend_to_json request_json = + session_pro_backend_add_pro_payment_request_to_json(&request); + scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/add_pro_payment", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( + response_json.data(), response_json.size()); + scope_exit response_free{[&]() { + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); + }}; + + for (size_t index = 0; index < response.header.errors_count; index++) { + string8 error = response.header.errors[index]; + INFO("ERROR: " << error.data); + } - struct curl_slist* curl_headers = nullptr; - curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json"); - REQUIRE(curl_headers); - scope_exit curl_headers_free{[&]() { curl_slist_free_all(curl_headers); }}; + // Verify response + first_pro_proof = response.proof; + INFO("Signature: " << oxenc::to_hex( + first_pro_proof.sig.data, + std::end(first_pro_proof.sig.data)) + << ", backend pubkey: " << oxenc::to_hex(DEV_BACKEND_PUBKEY) + << ", response: " << response_json); + REQUIRE(session_protocol_pro_proof_verify_signature( + &first_pro_proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); + REQUIRE(std::memcmp( + response.proof.rotating_pubkey.data, + request.rotating_pkey.data, + sizeof(request.rotating_pkey.data)) == 0); + } - // Add pro payment - session_protocol_pro_proof first_pro_proof = {}; - { - std::array fake_google_payment_token; - randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); - std::string fake_google_payment_token_hex = - "DEV." + oxenc::to_hex(fake_google_payment_token); + // Authorise new key + { + uint64_t now_unix_ts_ms = time(nullptr) * 1000; + // Build request + session_pro_backend_master_rotating_signatures pro_sigs = + session_pro_backend_generate_pro_proof_request_build_sigs( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + rotating_privkey.data, + sizeof(rotating_privkey.data), + now_unix_ts_ms); - std::array fake_google_order_id; - randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); - std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); + session_pro_backend_generate_pro_proof_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.rotating_pkey = rotating_pubkey; + request.unix_ts_ms = now_unix_ts_ms; + request.master_sig = pro_sigs.master_sig; + request.rotating_sig = pro_sigs.rotating_sig; + + session_pro_backend_to_json request_json = + session_pro_backend_generate_pro_proof_request_to_json(&request); + scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do CURL request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/generate_pro_proof", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( + response_json.data(), response_json.size()); + scope_exit response_free{[&]() { + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); + }}; + + for (size_t index = 0; index < response.header.errors_count; index++) { + if (index == 0) + fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); + } + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - session_pro_backend_add_pro_payment_user_transaction payment_tx = {}; - payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; - payment_tx.payment_id_count = fake_google_payment_token_hex.size(); - payment_tx.order_id_count = fake_google_order_id_hex.size(); - std::memcpy( - payment_tx.payment_id, - fake_google_payment_token_hex.data(), - payment_tx.payment_id_count); - std::memcpy( - payment_tx.order_id, fake_google_order_id_hex.data(), payment_tx.order_id_count); + // Verify response + session_protocol_pro_proof proof = response.proof; + REQUIRE(session_protocol_pro_proof_verify_signature( + &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); + REQUIRE(std::memcmp( + response.proof.rotating_pubkey.data, + request.rotating_pkey.data, + sizeof(request.rotating_pkey.data)) == 0); - // Build request - session_pro_backend_master_rotating_signatures add_pro_sigs = - session_pro_backend_add_pro_payment_request_build_sigs( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - rotating_privkey.data, - sizeof(rotating_privkey.data), - payment_tx.provider, - reinterpret_cast(payment_tx.payment_id), - payment_tx.payment_id_count, - reinterpret_cast(payment_tx.order_id), - payment_tx.order_id_count); - - session_pro_backend_add_pro_payment_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.rotating_pkey = rotating_pubkey; - request.payment_tx = payment_tx; - request.master_sig = add_pro_sigs.master_sig; - request.rotating_sig = add_pro_sigs.rotating_sig; - - session_pro_backend_to_json request_json = - session_pro_backend_add_pro_payment_request_to_json(&request); - scope_exit request_json_free{[&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/add_pro_payment", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { + session_pro_backend_to_json_free(&request_json); session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - }}; - - for (size_t index = 0; index < response.header.errors_count; index++) { - string8 error = response.header.errors[index]; - INFO("ERROR: " << error.data); } - // Verify response - first_pro_proof = response.proof; - INFO("Signature: " << oxenc::to_hex( - first_pro_proof.sig.data, std::end(first_pro_proof.sig.data)) - << ", backend pubkey: " << oxenc::to_hex(DEV_BACKEND_PUBKEY) - << ", response: " << response_json); - REQUIRE(session_protocol_pro_proof_verify_signature( - &first_pro_proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); - REQUIRE(std::memcmp( - response.proof.rotating_pubkey.data, - request.rotating_pkey.data, - sizeof(request.rotating_pkey.data)) == 0); - } + // Get pro status + { + // Build request + session_pro_backend_get_pro_details_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.unix_ts_ms = time(nullptr) * 1000; + request.count = 10'000; - // Authorise new key - { - uint64_t now_unix_ts_ms = time(nullptr) * 1000; - // Build request - session_pro_backend_master_rotating_signatures pro_sigs = - session_pro_backend_generate_pro_proof_request_build_sigs( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - rotating_privkey.data, - sizeof(rotating_privkey.data), - now_unix_ts_ms); - - session_pro_backend_generate_pro_proof_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.rotating_pkey = rotating_pubkey; - request.unix_ts_ms = now_unix_ts_ms; - request.master_sig = pro_sigs.master_sig; - request.rotating_sig = pro_sigs.rotating_sig; - - session_pro_backend_to_json request_json = - session_pro_backend_generate_pro_proof_request_to_json(&request); - scope_exit request_json_free{[&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do CURL request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/generate_pro_proof", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - }}; + session_pro_backend_signature sig = + session_pro_backend_get_pro_details_request_build_sig( + request.version, + master_privkey.data, + sizeof(master_privkey.data), + request.unix_ts_ms, + request.count); + REQUIRE(sig.success); + request.master_sig = sig.sig; - for (size_t index = 0; index < response.header.errors_count; index++) { - if (index == 0) - fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); + // Do CURL request + session_pro_backend_to_json request_json = + session_pro_backend_get_pro_details_request_to_json(&request); + scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/get_pro_details", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_get_pro_details_response response = + session_pro_backend_get_pro_details_response_parse( + response_json.data(), response_json.size()); + scope_exit response_free{ + [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; + + // Verify the response + for (size_t index = 0; index < response.header.errors_count; index++) { + if (index == 0) + fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); + } + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); + REQUIRE(response.items_count > 0); } - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - - // Verify response - session_protocol_pro_proof proof = response.proof; - REQUIRE(session_protocol_pro_proof_verify_signature( - &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); - REQUIRE(std::memcmp( - response.proof.rotating_pubkey.data, - request.rotating_pkey.data, - sizeof(request.rotating_pkey.data)) == 0); - - session_pro_backend_to_json_free(&request_json); - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - } - // Get pro status - { - // Build request - session_pro_backend_get_pro_details_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.unix_ts_ms = time(nullptr) * 1000; - request.count = 10'000; - - session_pro_backend_signature sig = session_pro_backend_get_pro_details_request_build_sig( - request.version, - master_privkey.data, - sizeof(master_privkey.data), - request.unix_ts_ms, - request.count); - REQUIRE(sig.success); - request.master_sig = sig.sig; - - // Do CURL request - session_pro_backend_to_json request_json = - session_pro_backend_get_pro_details_request_to_json(&request); - scope_exit request_json_free{[&]() { session_pro_backend_to_json_free(&request_json); }}; - - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/get_pro_details", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_get_pro_details_response response = - session_pro_backend_get_pro_details_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{ - [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; - - // Verify the response - for (size_t index = 0; index < response.header.errors_count; index++) { - if (index == 0) - fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); - } - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); - REQUIRE(response.items_count > 0); - } + // Get pro status without history + { + // Build request + session_pro_backend_get_pro_details_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.unix_ts_ms = time(nullptr) * 1000; - // Get pro status without history - { - // Build request - session_pro_backend_get_pro_details_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.unix_ts_ms = time(nullptr) * 1000; - - session_pro_backend_signature sig = session_pro_backend_get_pro_details_request_build_sig( - request.version, - master_privkey.data, - sizeof(master_privkey.data), - request.unix_ts_ms, - request.count); - REQUIRE(sig.success); - request.master_sig = sig.sig; - - // Do CURL request - session_pro_backend_to_json request_json = - session_pro_backend_get_pro_details_request_to_json(&request); - scope_exit request_json_free{[&]() { session_pro_backend_to_json_free(&request_json); }}; - - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/get_pro_details", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_get_pro_details_response response = - session_pro_backend_get_pro_details_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{ - [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; - - for (size_t index = 0; index < response.header.errors_count; index++) { - if (index == 0) - fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); + session_pro_backend_signature sig = + session_pro_backend_get_pro_details_request_build_sig( + request.version, + master_privkey.data, + sizeof(master_privkey.data), + request.unix_ts_ms, + request.count); + REQUIRE(sig.success); + request.master_sig = sig.sig; + + // Do CURL request + session_pro_backend_to_json request_json = + session_pro_backend_get_pro_details_request_to_json(&request); + scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/get_pro_details", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_get_pro_details_response response = + session_pro_backend_get_pro_details_response_parse( + response_json.data(), response_json.size()); + scope_exit response_free{ + [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; + + for (size_t index = 0; index < response.header.errors_count; index++) { + if (index == 0) + fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); + } + + // Verify the response + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); + REQUIRE(response.items_count == 0); } - // Verify the response - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); - REQUIRE(response.items_count == 0); - } + // Add _another_ payment, same details + session_pro_backend_add_pro_payment_user_transaction another_payment_tx = {}; + { + std::array fake_google_payment_token; + randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); + std::string fake_google_payment_token_hex = + "DEV." + oxenc::to_hex(fake_google_payment_token); + + std::array fake_google_order_id; + randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); + std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); + + another_payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; + another_payment_tx.payment_id_count = fake_google_payment_token_hex.size(); + another_payment_tx.order_id_count = fake_google_order_id_hex.size(); + std::memcpy( + another_payment_tx.payment_id, + fake_google_payment_token_hex.data(), + another_payment_tx.payment_id_count); + std::memcpy( + another_payment_tx.order_id, + fake_google_order_id_hex.data(), + another_payment_tx.order_id_count); + + // Build request + session_pro_backend_master_rotating_signatures add_pro_sigs = + session_pro_backend_add_pro_payment_request_build_sigs( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + rotating_privkey.data, + sizeof(rotating_privkey.data), + another_payment_tx.provider, + reinterpret_cast(another_payment_tx.payment_id), + another_payment_tx.payment_id_count, + reinterpret_cast(another_payment_tx.order_id), + another_payment_tx.order_id_count); - // Add _another_ payment, same details - session_pro_backend_add_pro_payment_user_transaction another_payment_tx = {}; - { - std::array fake_google_payment_token; - randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); - std::string fake_google_payment_token_hex = - "DEV." + oxenc::to_hex(fake_google_payment_token); + session_pro_backend_add_pro_payment_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.rotating_pkey = rotating_pubkey; + request.payment_tx = another_payment_tx; + request.master_sig = add_pro_sigs.master_sig; + request.rotating_sig = add_pro_sigs.rotating_sig; + + session_pro_backend_to_json request_json = + session_pro_backend_add_pro_payment_request_to_json(&request); + scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/add_pro_payment", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( + response_json.data(), response_json.size()); + scope_exit response_free{[&]() { + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); + }}; + + // Verify response + session_protocol_pro_proof proof = response.proof; + REQUIRE(session_protocol_pro_proof_verify_signature( + &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); + REQUIRE(std::memcmp( + response.proof.rotating_pubkey.data, + request.rotating_pkey.data, + sizeof(request.rotating_pkey.data)) == 0); + } - std::array fake_google_order_id; - randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); - std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); + // Get revocation list + { + // Build request + session_pro_backend_get_pro_revocations_request request = {}; + request.version = 0; - another_payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; - another_payment_tx.payment_id_count = fake_google_payment_token_hex.size(); - another_payment_tx.order_id_count = fake_google_order_id_hex.size(); - std::memcpy( - another_payment_tx.payment_id, - fake_google_payment_token_hex.data(), - another_payment_tx.payment_id_count); - std::memcpy( - another_payment_tx.order_id, - fake_google_order_id_hex.data(), - another_payment_tx.order_id_count); - - // Build request - session_pro_backend_master_rotating_signatures add_pro_sigs = - session_pro_backend_add_pro_payment_request_build_sigs( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - rotating_privkey.data, - sizeof(rotating_privkey.data), - another_payment_tx.provider, - reinterpret_cast(another_payment_tx.payment_id), - another_payment_tx.payment_id_count, - reinterpret_cast(another_payment_tx.order_id), - another_payment_tx.order_id_count); - - session_pro_backend_add_pro_payment_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.rotating_pkey = rotating_pubkey; - request.payment_tx = another_payment_tx; - request.master_sig = add_pro_sigs.master_sig; - request.rotating_sig = add_pro_sigs.rotating_sig; - - session_pro_backend_to_json request_json = - session_pro_backend_add_pro_payment_request_to_json(&request); - scope_exit request_json_free{[&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/add_pro_payment", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - }}; - - // Verify response - session_protocol_pro_proof proof = response.proof; - REQUIRE(session_protocol_pro_proof_verify_signature( - &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); - REQUIRE(std::memcmp( - response.proof.rotating_pubkey.data, - request.rotating_pkey.data, - sizeof(request.rotating_pkey.data)) == 0); - } + session_pro_backend_to_json request_json = + session_pro_backend_get_pro_revocations_request_to_json(&request); + scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/get_pro_revocations", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_get_pro_revocations_response response = + session_pro_backend_get_pro_revocations_response_parse( + response_json.data(), response_json.size()); + scope_exit response_free{ + [&]() { session_pro_backend_get_pro_revocations_response_free(&response); }}; + + // Verify response + INFO("ERROR: JSON response: " << response_json.c_str()); + for (size_t index = 0; index < response.header.errors_count; index++) { + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); + } - // Get revocation list - { - // Build request - session_pro_backend_get_pro_revocations_request request = {}; - request.version = 0; - - session_pro_backend_to_json request_json = - session_pro_backend_get_pro_revocations_request_to_json(&request); - scope_exit request_json_free{[&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/get_pro_revocations", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_get_pro_revocations_response response = - session_pro_backend_get_pro_revocations_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{ - [&]() { session_pro_backend_get_pro_revocations_response_free(&response); }}; - - // Verify response - INFO("ERROR: JSON response: " << response_json.c_str()); - for (size_t index = 0; index < response.header.errors_count; index++) { - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); + // Verify the response + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.ticket == 0); + REQUIRE(response.items_count == 0); } - // Verify the response - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.ticket == 0); - REQUIRE(response.items_count == 0); - } + // Set payment refund requested + { + // Build request + uint64_t now_unix_ts_ms = time(nullptr) * 1000; + session_pro_backend_to_json request_json = + session_pro_backend_set_payment_refund_requested_request_build_to_json( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + /*unix_ts_ms*/ now_unix_ts_ms, + /*refund_requested_unix_ts_ms*/ now_unix_ts_ms, + another_payment_tx.provider, + reinterpret_cast(another_payment_tx.payment_id), + another_payment_tx.payment_id_count, + reinterpret_cast(another_payment_tx.order_id), + another_payment_tx.order_id_count); + + scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/set_payment_refund_requested", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_set_payment_refund_requested_response response = + session_pro_backend_set_payment_refund_requested_response_parse( + response_json.data(), response_json.size()); + scope_exit response_free{[&]() { + session_pro_backend_set_payment_refund_requested_response_free(&response); + }}; + + // Verify response + INFO("ERROR: JSON response: " << response_json.c_str()); + for (size_t index = 0; index < response.header.errors_count; index++) { + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); + } - // Set payment refund requested - { - // Build request - uint64_t now_unix_ts_ms = time(nullptr) * 1000; - session_pro_backend_to_json request_json = - session_pro_backend_set_payment_refund_requested_request_build_to_json( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - /*unix_ts_ms*/ now_unix_ts_ms, - /*refund_requested_unix_ts_ms*/ now_unix_ts_ms, - another_payment_tx.provider, - reinterpret_cast(another_payment_tx.payment_id), - another_payment_tx.payment_id_count, - reinterpret_cast(another_payment_tx.order_id), - another_payment_tx.order_id_count); - - scope_exit request_json_free{[&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/set_payment_refund_requested", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_set_payment_refund_requested_response response = - session_pro_backend_set_payment_refund_requested_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { - session_pro_backend_set_payment_refund_requested_response_free(&response); - }}; - - // Verify response - INFO("ERROR: JSON response: " << response_json.c_str()); - for (size_t index = 0; index < response.header.errors_count; index++) { - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); + // Verify the response + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.version == 0); + REQUIRE(response.updated); } - - // Verify the response - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.version == 0); - REQUIRE(response.updated); } } #endif From ff5c22999c487b055849463f9ab1ce5592ae377c Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 27 Oct 2025 17:06:21 +1100 Subject: [PATCH 08/72] Bundle the revocation ticket with the get/setter as they come together from the backend --- include/session/database/connection.hpp | 53 ++++---- src/database/connection.cpp | 158 ++++++++++++------------ tests/test_database.cpp | 49 +++++--- 3 files changed, 140 insertions(+), 120 deletions(-) diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index 1c39d895..b54840a4 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -12,15 +12,14 @@ struct sqlite3_stmt; namespace session::database { -struct AddResult { +struct SetResult { bool success; int return_code; }; -struct DeleteResult { - bool success; - int return_code; - size_t count; // Number of rows that were deleted +struct Runtime { + size_t id; + size_t pro_revocations_ticket; }; struct Connection { @@ -53,6 +52,17 @@ struct Connection { /// - `callback` -- User defined function to execute when a row is returned void query(std::string_view sql, std::function callback); + /// API: database/get_runtime + /// + /// Get the runtime row of the table which contains global metadata for the entire table. + /// There's only one runtime row per database. + /// + /// Outputs: + /// - `id` -- Row ID of the runtime row (essentially always 1 as there's only 1 runtime row) + /// - `pro_revocations_ticket` -- Current version of the pro revocations list that has been + /// synced from the Session Pro Backend. + Runtime get_runtime(); + /// API: database/add_pro_revocations /// /// Add a list of Session Pro revocations into the database associated with this connection. @@ -64,24 +74,8 @@ struct Connection { /// Outputs: /// - `success` -- True if the add was successful, false otherwise. /// - `return_code` -- The SQLite3 error code that caused the error if `success` was false - AddResult add_pro_revocations( - std::span revocations) noexcept; - - /// API: database/delete_pro_revocations - /// - /// Delete a list of Session Pro revocations from the database associated with this connection. - /// Revocations are deleted by matching the item's `gen_index_hash`. This function is - /// transactional, on failure changes to the database are rolled back. - /// - /// Inputs: - /// - `revocations` -- The list of revocations to delete - /// - /// Outputs: - /// - `success` -- True if the delete was successful, false otherwise. - /// - `return_code` -- The SQLite3 error code that caused the error if `success` was false - /// - `count` -- Number of rows that were deleted in this operation - DeleteResult delete_pro_revocations( - std::span revocations) noexcept; + SetResult set_pro_revocations( + uint32_t ticket, std::span revocations) noexcept; /// API: database/get_pro_revocations_buffer /// @@ -96,6 +90,8 @@ struct Connection { /// revocations currently in the DB. /// - `offset` -- Start retrieving revocation rows from this specified index of the list. Pass /// in 0 to start from the beginning. + /// - `ticket` -- Retrieve the current ticket for the revocation list which represents the + /// current version of the list that has been synced from the Session Pro Backend. /// /// Outputs: /// - `size_t` -- Number of revocation items read from the database. If the buffer was @@ -103,15 +99,22 @@ struct Connection { /// the buffer. If `buf` and `buf_count` are nullptr or 0 respectively, then value returned /// is the amount of revocation items in the DB at the time of execution. size_t get_pro_revocations_buffer( - pro_backend::ProRevocationItem* buf, size_t buf_count, size_t offset); + OPTIONAL pro_backend::ProRevocationItem* buf, + size_t buf_count, + size_t offset, + OPTIONAL uint32_t* ticket); /// API: database/get_pro_revocations /// /// Retrieve the Session Pro Backend revocation list from the database. This function throws if /// there was an allocation or SQLite returned an error /// + /// Inputs: + /// - `ticket` -- Retrieve the current ticket for the revocation list which represents the + /// current version of the list that has been synced from the Session Pro Backend. + /// /// Outputs: /// - `std::vector` -- List of revocation items - std::vector get_pro_revocations(); + std::vector get_pro_revocations(OPTIONAL uint32_t* ticket); }; } // namespace session::database diff --git a/src/database/connection.cpp b/src/database/connection.cpp index 63096436..d0f24b01 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -77,12 +77,24 @@ Connection::Connection(const std::string& path, const cleared_array<48> &raw_key CREATE TABLE IF NOT EXISTS pro_revocations ( gen_index_hash BLOB PRIMARY KEY NOT NULL, expiry_unix_ts_ms INTEGER NOT NULL -))"; +); + +CREATE TABLE IF NOT EXISTS runtime ( + id INTEGER PRIMARY KEY NOT NULL, + pro_revocations_ticket INTEGER NOT NULL +);)"; // Create the initial DB tables rc = sqlite3_exec(db_, bootstrap_sql.data(), nullptr, nullptr, nullptr); if (rc != SQLITE_OK) close_db_and_throw_error(&db_, rc, "Failed to bootstrap tables"); + // Seed the runtime table + std::string_view seed_runtime_sql = + R"(INSERT INTO runtime (pro_revocations_ticket) VALUES (0))"; + rc = sqlite3_exec(db_, seed_runtime_sql.data(), nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) + close_db_and_throw_error(&db_, rc, "Failed to seed the runtime table"); + // Teleport to the target version rc = set_db_version_or_throw(db_, ++curr_db_version); if (rc != SQLITE_OK) @@ -129,99 +141,84 @@ void Connection::query(std::string_view sql, std::function throw std::runtime_error(fmt::format("Error executing query: {}", sqlite3_errmsg(db_))); } -AddResult Connection::add_pro_revocations( - std::span revocations) noexcept { + +Runtime Connection::get_runtime() { + Runtime result = {}; + std::string_view sql = R"(SELECT id, pro_revocations_ticket FROM runtime LIMIT 1)"; + query(sql, [&result](sqlite3_stmt* stmt) { + result.id = sqlite3_column_int(stmt, 0); + result.pro_revocations_ticket = sqlite3_column_int(stmt, 1); + }); + return result; +} + +SetResult Connection::set_pro_revocations( + uint32_t ticket, std::span revocations) noexcept { // The following consists of exception safe code so we do not need try catch and can trivially // commit or rollback the at the end of the function. exec("BEGIN DEFERRED TRANSACTION;"); + exec("DELETE FROM pro_revocations"); // Clear the table - sqlite3_stmt* stmt = nullptr; - std::string_view sql = R"( + // Assign the pro-revocations + int rc = SQLITE_OK; + if (rc == SQLITE_OK || rc == SQLITE_DONE) { + sqlite3_stmt* stmt = nullptr; + std::string_view sql = R"( INSERT INTO pro_revocations (gen_index_hash, expiry_unix_ts_ms) VALUES (?, ?) )"; - int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); - for (size_t index = 0; (rc == SQLITE_OK || rc == SQLITE_DONE) && index < revocations.size(); - index++) { - const auto& it = revocations[index]; - int bind = 0; - int64_t expiry = static_cast(it.expiry_unix_ts.time_since_epoch().count()); - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_bind_blob( - stmt, - ++bind, - it.gen_index_hash.data(), - static_cast(it.gen_index_hash.size()), - nullptr) - : rc; - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_bind_int64(stmt, ++bind, expiry) : rc; - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_step(stmt) : rc; - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_reset(stmt) : rc; - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_clear_bindings(stmt) : rc; + rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); + for (size_t index = 0; (rc == SQLITE_OK || rc == SQLITE_DONE) && index < revocations.size(); + index++) { + const auto& it = revocations[index]; + int bind = 0; + int64_t expiry = static_cast(it.expiry_unix_ts.time_since_epoch().count()); + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) + ? sqlite3_bind_blob( + stmt, + ++bind, + it.gen_index_hash.data(), + static_cast(it.gen_index_hash.size()), + nullptr) + : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_bind_int64(stmt, ++bind, expiry) + : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_step(stmt) : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_reset(stmt) : rc; + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_clear_bindings(stmt) : rc; + } + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK || rc == SQLITE_DONE) + rc = finalize_rc; } - int finalize_rc = sqlite3_finalize(stmt); - if (rc == SQLITE_OK || rc == SQLITE_DONE) - rc = finalize_rc; - - AddResult result = {}; - result.return_code = rc; - result.success = result.return_code == SQLITE_OK || result.return_code == SQLITE_DONE; - exec(result.success ? "COMMIT;" : "ROLLBACK;"); - return result; -} -DeleteResult Connection::delete_pro_revocations( - std::span revocations) noexcept { + // Update the ticket + if (rc == SQLITE_OK || rc == SQLITE_DONE) { + sqlite3_stmt* stmt = nullptr; + std::string_view sql = R"(UPDATE runtime SET pro_revocations_ticket = ?)"; - // The following consists of exception safe code so we do not need try catch and can trivially - // commit or rollback the at the end of the function. - exec("BEGIN DEFERRED TRANSACTION;"); - - std::string_view sql = R"( -DELETE FROM pro_revocations -WHERE gen_index_hash = ? -)"; - - size_t row_count = 0; - sqlite3_stmt* stmt = nullptr; - int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); - for (size_t index = 0; (rc == SQLITE_OK || rc == SQLITE_DONE) && index < revocations.size(); - index++) { - const auto& it = revocations[index]; - int64_t expiry = static_cast(it.expiry_unix_ts.time_since_epoch().count()); - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) - ? sqlite3_bind_blob( - stmt, - 1, - it.gen_index_hash.data(), - static_cast(it.gen_index_hash.size()), - nullptr) - : rc; + int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); + rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_bind_int(stmt, 1, ticket) : rc; rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_step(stmt) : rc; - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_reset(stmt) : rc; - rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_clear_bindings(stmt) : rc; - row_count += sqlite3_changes(db_); + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK || rc == SQLITE_DONE) + rc = finalize_rc; } - int finalize_rc = sqlite3_finalize(stmt); - if (rc == SQLITE_OK || rc == SQLITE_DONE) - rc = finalize_rc; - DeleteResult result = {}; + SetResult result = {}; result.return_code = rc; result.success = result.return_code == SQLITE_OK || result.return_code == SQLITE_DONE; - if (result.success) { - exec("COMMIT;"); - result.count = row_count; - } else { - exec("ROLLBACK;"); - result.count = 0; - } + exec(result.success ? "COMMIT;" : "ROLLBACK;"); return result; } size_t Connection::get_pro_revocations_buffer( - pro_backend::ProRevocationItem* buf, size_t buf_count, size_t offset) { + pro_backend::ProRevocationItem* buf, size_t buf_count, size_t offset, uint32_t* ticket) { + + // Count the number of rows size_t result = 0; query("SELECT COUNT(*) FROM pro_revocations", [&](sqlite3_stmt* stmt) { result = sqlite3_column_int(stmt, 0); }); @@ -241,6 +238,7 @@ OFFSET %zu offset); assert(sql_size < sizeof(sql)); + // Retrieve the rows result = 0; query(std::string_view(sql, sql_size), [&buf, buf_count, &result](sqlite3_stmt* stmt) { pro_backend::ProRevocationItem& item = buf[result++]; @@ -258,15 +256,23 @@ OFFSET %zu auto expiry = std::chrono::milliseconds(sqlite3_column_int64(stmt, 1)); item.expiry_unix_ts = std::chrono::sys_time(expiry); }); + + } + + // Retrieve the ticket + if (ticket) { + query("SELECT pro_revocations_ticket FROM runtime LIMIT 1", [&ticket](sqlite3_stmt* stmt) { + *ticket = static_cast(sqlite3_column_int(stmt, 0)); + }); } return result; } -std::vector Connection::get_pro_revocations() { +std::vector Connection::get_pro_revocations(uint32_t *ticket) { std::vector result; - size_t size_req = get_pro_revocations_buffer(nullptr, 0, 0); + size_t size_req = get_pro_revocations_buffer(nullptr, 0, 0, ticket); result.resize(size_req); - size_t items_read = get_pro_revocations_buffer(result.data(), result.size(), 0); + size_t items_read = get_pro_revocations_buffer(result.data(), result.size(), 0, ticket); assert(items_read == size_req); return result; } diff --git a/tests/test_database.cpp b/tests/test_database.cpp index c44d6539..612aef00 100644 --- a/tests/test_database.cpp +++ b/tests/test_database.cpp @@ -3,12 +3,10 @@ #include #include -#include #include #include "session/database/connection.hpp" #include "session/pro_backend.hpp" -#include "utils.hpp" TEST_CASE("Database", "[database][open]") { session::cleared_array<48> raw_key = {}; @@ -21,8 +19,14 @@ TEST_CASE("Database", "[database][pro][revocations]") { randombytes_buf(raw_key.data(), raw_key.size()); auto db = session::database::Connection(":memory:", raw_key); + // Check runtime was seeded to ticket 0 + session::database::Runtime runtime = db.get_runtime(); + REQUIRE(runtime.id == 1); + REQUIRE(runtime.pro_revocations_ticket == 0); + // Check that the DB has no revocations in it - size_t db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0); + uint32_t ticket = 0; + size_t db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0, &ticket); REQUIRE(db_item_count == 0); // Create the revocations we will put into the DB @@ -47,39 +51,46 @@ TEST_CASE("Database", "[database][pro][revocations]") { }, }; - // Add the items - session::database::AddResult add = db.add_pro_revocations(src_items); - INFO("Add failed: " << sqlite3_errstr(add.return_code)); - REQUIRE(add.success); - REQUIRE(add.return_code == SQLITE_OK); + // Set the items + session::database::SetResult set_result = db.set_pro_revocations(1, src_items); + INFO("Set w/ 2 items failed: " << sqlite3_errstr(set_result.return_code)); + REQUIRE(set_result.success); + REQUIRE(set_result.return_code == SQLITE_OK); + + // Check runtime ticket was changed to 1 + runtime = db.get_runtime(); + REQUIRE(runtime.id == 1); + REQUIRE(runtime.pro_revocations_ticket == 1); // Count the number of revocations in the DB (should be 2 as we've inserted them) - db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0); + db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0, &ticket); + REQUIRE(ticket == runtime.pro_revocations_ticket); REQUIRE(db_item_count == 2); // Check that the revocations was in the DB - std::vector db_items = db.get_pro_revocations(); + std::vector db_items = db.get_pro_revocations(&ticket); + REQUIRE(ticket == 1); REQUIRE(src_items[0].gen_index_hash == db_items[0].gen_index_hash); REQUIRE(src_items[0].expiry_unix_ts == db_items[0].expiry_unix_ts); REQUIRE(src_items[1].gen_index_hash == db_items[1].gen_index_hash); REQUIRE(src_items[1].expiry_unix_ts == db_items[1].expiry_unix_ts); // Delete the first item (src[0]) from the DB - session::pro_backend::ProRevocationItem delete_item = src_items[0]; - session::database::DeleteResult delete_result = - db.delete_pro_revocations(std::span(&delete_item, 1)); - INFO("Delete failed: " << sqlite3_errstr(delete_result.return_code)); - REQUIRE(delete_result.success); - REQUIRE(delete_result.return_code == SQLITE_OK); - REQUIRE(delete_result.count == 1); + session::pro_backend::ProRevocationItem set_item = src_items[1]; + set_result = db.set_pro_revocations(2, std::span(&set_item, 1)); + INFO("Set w/ 1 item failed: " << sqlite3_errstr(set_result.return_code)); + REQUIRE(set_result.success); + REQUIRE(set_result.return_code == SQLITE_OK); // Count the number of revocations in the DB (should be 1 as we've deleted one of them) - db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0); + db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0, &ticket); REQUIRE(db_item_count == 1); + REQUIRE(ticket == 2); // Verify that the DB now has just the item at src[1] - std::vector db_items_after_delete = db.get_pro_revocations(); + std::vector db_items_after_delete = db.get_pro_revocations(&ticket); REQUIRE(db_items_after_delete.size() == 1); REQUIRE(src_items[1].gen_index_hash == db_items_after_delete[0].gen_index_hash); REQUIRE(src_items[1].expiry_unix_ts == db_items_after_delete[0].expiry_unix_ts); + REQUIRE(ticket == 2); } From d31ecb87badba9a66b6facad91b08867b3fac883 Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 29 Oct 2025 12:22:48 +1100 Subject: [PATCH 09/72] Introuce managed libsession context to maintain runtime state Pre-empting the idea that libsession should start managing state on behalf of the caller by first having a runtime cache of the Session Pro revocation list that backs up to the DB on update. --- include/session/core.hpp | 40 ++++++++++++++ include/session/database/connection.hpp | 41 ++++++++++---- src/CMakeLists.txt | 10 ++-- src/core.cpp | 60 ++++++++++++++++++++ src/database/connection.cpp | 73 +++++++++++++------------ tests/test_database.cpp | 14 +++-- 6 files changed, 180 insertions(+), 58 deletions(-) create mode 100644 include/session/core.hpp create mode 100644 src/core.cpp diff --git a/include/session/core.hpp b/include/session/core.hpp new file mode 100644 index 00000000..eb4fa6c0 --- /dev/null +++ b/include/session/core.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#if !defined(DISABLE_SQLCIPHER_DATABASE) +#include +#endif +#include +#include +#include + +namespace session { +namespace pro_backend { + struct ProRevocationItem; +} +}; // namespace session + +namespace session::core { +struct ProRevocationItemComparer { + bool operator()( + const pro_backend::ProRevocationItem& lhs, + const pro_backend::ProRevocationItem& rhs) const noexcept; +}; + +struct Core { + std::set revocations_; + int32_t revocations_ticket_; +#if !defined(DISABLE_SQLCIPHER_DATABASE) + session::database::Connection db_conn; +#endif + + bool pro_proof_is_revoked( + const array_uc32& gen_index_hash, + std::chrono::sys_time unix_ts) const; + + void pro_update_revocations( + int32_t revocations_ticket, + std::span revocations); +}; + +} // namespace session::core diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index b54840a4..57270906 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -14,25 +15,41 @@ namespace session::database { struct SetResult { bool success; - int return_code; + int sql_return_code; + /// SQL's string-ified `sql_return_code` pointing to memory in the data segment. Should not be + /// modified and is valid for program lifetime. + const char* sql_error; }; +/// The row from the runtime table which is the table housing global settings of the Session +/// database. There's only 1 row in the runtime table which gets extracted and filled out into this +/// struct. struct Runtime { - size_t id; - size_t pro_revocations_ticket; + int32_t id; + int32_t pro_revocations_ticket; }; -struct Connection { - sqlite3* db_; +struct sqlite3_deleter { + void operator()(sqlite3* db) const noexcept; +}; - Connection(const std::string& path, const cleared_array<48>& raw_key); - ~Connection(); +struct Connection { + std::unique_ptr db_; - // Prevent copying and moving - Connection(const Connection&) = delete; - Connection& operator=(const Connection&) = delete; - Connection(Connection&& other) = delete; - Connection& operator=(Connection&& other) = delete; + /// API: database/open + /// + /// Open a connection to the DB specified at `path`. If this connection previously has an open + /// DB that connection is gracefully closed before opening up the newly requested one. If this + /// function fails to open the DB, the previous DB connection is untouched. + /// + /// This function throws an error if the DB was not openable, if the `raw_key` was the incorrect + /// key to decrypt the DB or the contents of the DB were malformed. + /// + /// Inputs: + /// - `path` -- Path to the DB to open, this can be a URI or path on disk + /// - `raw_key` -- Encryption key to use to open the specified DB. If the DB does not exist then + /// the database will be created, encrypted with this key. + void open(const std::string& path, const cleared_array<48>& raw_key); /// API: database/exec /// diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ba1d4545..5f315241 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -46,6 +46,7 @@ add_libsession_util_library(crypto attachments.cpp blinding.cpp curve25519.cpp + core.cpp ed25519.cpp hash.cpp multi_encrypt.cpp @@ -136,12 +137,9 @@ if(ENABLE_DATABASE) database/connection.cpp ) - target_link_libraries(database - PUBLIC - util - PRIVATE - sqlcipher::sqlcipher - ) + target_link_libraries(database PUBLIC util PRIVATE sqlcipher::sqlcipher) +else() + target_compile_definitions(database INTERFACE DISABLE_SQLCIPHER_DATABASE) endif() if(WARNINGS_AS_ERRORS AND NOT USE_LTO AND CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION MATCHES "^11\\.") diff --git a/src/core.cpp b/src/core.cpp new file mode 100644 index 00000000..2dd3cd40 --- /dev/null +++ b/src/core.cpp @@ -0,0 +1,60 @@ +#include +#include + +auto logcat = oxen::log::Cat("core"); + +namespace session::core { +bool ProRevocationItemComparer::operator()( + const pro_backend::ProRevocationItem& lhs, + const pro_backend::ProRevocationItem& rhs) const noexcept { + bool result = lhs.gen_index_hash < rhs.gen_index_hash; + return result; +} + +bool Core::pro_proof_is_revoked( + const array_uc32& gen_index_hash, + std::chrono::sys_time unix_ts) const { + bool result = false; + pro_backend::ProRevocationItem item = {}; + item.gen_index_hash = gen_index_hash; + auto it = revocations_.find(item); + if (it != revocations_.end()) + result = unix_ts >= it->expiry_unix_ts; + return result; +} + +void Core::pro_update_revocations( + int32_t revocations_ticket, + std::span revocations) { + if (revocations_ticket_ == revocations_ticket) + return; + + // Currently we just dump the entire thing and re-write it, we don't expect this list to get big + revocations_.clear(); + revocations_.insert(revocations.begin(), revocations.end()); + revocations_ticket_ = revocations_ticket; + +#if !defined(DISABLED_SQLCIPHER_DATABASE) + session::database::SetResult set_result = + db_conn.set_pro_revocations(revocations_ticket, revocations); + + // There's not much we can do here for whatever reason it failed. The runtime cache is updated + // but not the DB. The DB is only for permanence of the list across restarts of libsession at + // which point, it will load from the DB, query the backend and notice the ticket is out of sync + // and try again. + if (!set_result.success) { + oxen::log::warning( + logcat, + "Failed to update SQL revocations from (items {}; ticket {}) -> (items {}; ticket " + "{}): ({}) {}", + revocations_.size(), + revocations_ticket_, + revocations.size(), + revocations_ticket, + set_result.sql_return_code, + set_result.sql_error); + } +#endif +} + +}; // namespace session::core diff --git a/src/database/connection.cpp b/src/database/connection.cpp index d0f24b01..1b343a22 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -12,10 +12,8 @@ auto logcat = oxen::log::Cat("database"); namespace { -void close_db_and_throw_error(sqlite3** db, int sql_result, std::string_view error_prefix) { +void throw_sql_error(int sql_result, std::string_view error_prefix) { std::string msg = fmt::format("{}: {}", error_prefix, sqlite3_errstr(sql_result)); - sqlite3_close(*db); - *db = nullptr; throw std::runtime_error(msg); } @@ -29,14 +27,23 @@ int set_db_version_or_throw(sqlite3* db, uint8_t db_version) { }; // namespace namespace session::database { -Connection::Connection(const std::string& path, const cleared_array<48> &raw_key) { + +void sqlite3_deleter::operator()(sqlite3* db) const noexcept { + sqlite3_close(db); +} + +void Connection::open(const std::string& path, const cleared_array<48> &raw_key) { cleared_array<48> ZERO_RAW_KEY = {}; assert(memcmp(raw_key.data(), ZERO_RAW_KEY.data(), raw_key.size()) != 0 && "Raw key was not set"); // Open DB - int rc = sqlite3_open(path.c_str(), &db_); + sqlite3* db_ptr = nullptr; + int rc = sqlite3_open(path.c_str(), &db_ptr); if (rc != SQLITE_OK) - close_db_and_throw_error(&db_, rc, "Failed to open database"); + throw_sql_error(rc, "Failed to open database"); + + // Replace the old connection w/ the new one (if there was one previously) + db_.reset(db_ptr); // According to the SQLCipher docs iOS needs the 'cipher_plaintext_header_size' value set to at // least 32 as iOS extends special privileges to the database and needs this header to be in @@ -48,21 +55,21 @@ Connection::Connection(const std::string& path, const cleared_array<48> &raw_key // // For more info, see: // https://www.zetetic.net/sqlcipher/sqlcipher-api/#cipher_plaintext_header_size - rc = sqlite3_exec(db_, "PRAGMA cipher_plaintext_header_size = 32", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db_.get(), "PRAGMA cipher_plaintext_header_size = 32", nullptr, nullptr, nullptr); if (rc != SQLITE_OK) - close_db_and_throw_error(&db_, rc, "Failed to configure database"); + throw_sql_error(rc, "Failed to configure database"); // Set encryption key, this is the underlying function that "PRAGMA key = .." calls std::string fmt_key = fmt::format("x'{}'", oxenc::to_hex(raw_key)); - rc = sqlite3_key(db_, fmt_key.c_str(), fmt_key.size()); + rc = sqlite3_key(db_.get(), fmt_key.c_str(), fmt_key.size()); sodium_zero_buffer(fmt_key.data(), fmt_key.size()); if (rc != SQLITE_OK) - close_db_and_throw_error(&db_, rc, "Failed to set encryption key"); + throw_sql_error(rc, "Failed to set encryption key"); // Verify the key works by reading the sqlite_master table - rc = sqlite3_exec(db_, "SELECT COUNT(*) FROM sqlite_master", nullptr, nullptr, nullptr); + rc = sqlite3_exec(db_.get(), "SELECT COUNT(*) FROM sqlite_master", nullptr, nullptr, nullptr); if (rc != SQLITE_OK) - close_db_and_throw_error(&db_, rc, "Failed to decrypt database with the given encryption key"); + throw_sql_error(rc, "Failed to decrypt database with the given encryption key"); oxen::log::debug(logcat, "Opened database {} successfully", path); // Query the DB version @@ -84,22 +91,21 @@ CREATE TABLE IF NOT EXISTS runtime ( pro_revocations_ticket INTEGER NOT NULL );)"; // Create the initial DB tables - rc = sqlite3_exec(db_, bootstrap_sql.data(), nullptr, nullptr, nullptr); + rc = sqlite3_exec(db_.get(), bootstrap_sql.data(), nullptr, nullptr, nullptr); if (rc != SQLITE_OK) - close_db_and_throw_error(&db_, rc, "Failed to bootstrap tables"); + throw_sql_error(rc, "Failed to bootstrap tables"); // Seed the runtime table std::string_view seed_runtime_sql = R"(INSERT INTO runtime (pro_revocations_ticket) VALUES (0))"; - rc = sqlite3_exec(db_, seed_runtime_sql.data(), nullptr, nullptr, nullptr); + rc = sqlite3_exec(db_.get(), seed_runtime_sql.data(), nullptr, nullptr, nullptr); if (rc != SQLITE_OK) - close_db_and_throw_error(&db_, rc, "Failed to seed the runtime table"); + throw_sql_error(rc, "Failed to seed the runtime table"); // Teleport to the target version - rc = set_db_version_or_throw(db_, ++curr_db_version); + rc = set_db_version_or_throw(db_.get(), ++curr_db_version); if (rc != SQLITE_OK) - close_db_and_throw_error( - &db_, rc, fmt::format("Failed to set DB version to {}", curr_db_version)); + throw_sql_error(rc, fmt::format("Failed to set DB version to {}", curr_db_version)); } // Requery the DB version and ensure all version migrations have occurred @@ -110,16 +116,9 @@ CREATE TABLE IF NOT EXISTS runtime ( assert(final_version_in_db == TARGET_DB_VERSION); } -Connection::~Connection() { - if (db_) { - sqlite3_close(db_); - db_ = nullptr; - } -} - void Connection::exec(const std::string& sql) { char* error_msg = nullptr; - int rc = sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &error_msg); + int rc = sqlite3_exec(db_.get(), sql.c_str(), nullptr, nullptr, &error_msg); if (rc != SQLITE_OK) { std::string error = error_msg ? error_msg : "unknown error"; sqlite3_free(error_msg); @@ -129,16 +128,17 @@ void Connection::exec(const std::string& sql) { void Connection::query(std::string_view sql, std::function callback) { sqlite3_stmt* stmt = nullptr; - int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size(), &stmt, nullptr); + int rc = sqlite3_prepare_v2(db_.get(), sql.data(), sql.size(), &stmt, nullptr); if (rc != SQLITE_OK) { throw std::runtime_error( - fmt::format("Failed to prepare statement: {}", sqlite3_errmsg(db_))); + fmt::format("Failed to prepare statement: {}", sqlite3_errmsg(db_.get()))); } while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) callback(stmt); sqlite3_finalize(stmt); if (rc != SQLITE_DONE) - throw std::runtime_error(fmt::format("Error executing query: {}", sqlite3_errmsg(db_))); + throw std::runtime_error( + fmt::format("Error executing query: {}", sqlite3_errmsg(db_.get()))); } @@ -169,7 +169,7 @@ INSERT INTO pro_revocations (gen_index_hash, expiry_unix_ts_ms) VALUES (?, ?) )"; - rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); + rc = sqlite3_prepare_v2(db_.get(), sql.data(), sql.size() + 1, &stmt, nullptr); for (size_t index = 0; (rc == SQLITE_OK || rc == SQLITE_DONE) && index < revocations.size(); index++) { const auto& it = revocations[index]; @@ -199,7 +199,7 @@ VALUES (?, ?) sqlite3_stmt* stmt = nullptr; std::string_view sql = R"(UPDATE runtime SET pro_revocations_ticket = ?)"; - int rc = sqlite3_prepare_v2(db_, sql.data(), sql.size() + 1, &stmt, nullptr); + int rc = sqlite3_prepare_v2(db_.get(), sql.data(), sql.size() + 1, &stmt, nullptr); rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_bind_int(stmt, 1, ticket) : rc; rc = (rc == SQLITE_OK || rc == SQLITE_DONE) ? sqlite3_step(stmt) : rc; @@ -209,14 +209,19 @@ VALUES (?, ?) } SetResult result = {}; - result.return_code = rc; - result.success = result.return_code == SQLITE_OK || result.return_code == SQLITE_DONE; + result.sql_return_code = rc; + result.sql_error = sqlite3_errstr(rc); + result.success = result.sql_return_code == SQLITE_OK || result.sql_return_code == SQLITE_DONE; exec(result.success ? "COMMIT;" : "ROLLBACK;"); return result; } size_t Connection::get_pro_revocations_buffer( pro_backend::ProRevocationItem* buf, size_t buf_count, size_t offset, uint32_t* ticket) { + // Note this operation is not atomic, the collecting of revocations and the querying of the + // ticket happens in 2 separate read steps. This is probably not an issue as I expect the + // getting of revocations to only happen on startup where it'll get cached into runtime memory. + // Startup and initialisation of the libsession core is single threaded. // Count the number of rows size_t result = 0; diff --git a/tests/test_database.cpp b/tests/test_database.cpp index 612aef00..a3beae5d 100644 --- a/tests/test_database.cpp +++ b/tests/test_database.cpp @@ -11,13 +11,15 @@ TEST_CASE("Database", "[database][open]") { session::cleared_array<48> raw_key = {}; randombytes_buf(raw_key.data(), raw_key.size()); - auto db = session::database::Connection(":memory:", raw_key); + session::database::Connection db = {}; + db.open(":memory:", raw_key); } TEST_CASE("Database", "[database][pro][revocations]") { session::cleared_array<48> raw_key = {}; randombytes_buf(raw_key.data(), raw_key.size()); - auto db = session::database::Connection(":memory:", raw_key); + session::database::Connection db = {}; + db.open(":memory:", raw_key); // Check runtime was seeded to ticket 0 session::database::Runtime runtime = db.get_runtime(); @@ -53,9 +55,9 @@ TEST_CASE("Database", "[database][pro][revocations]") { // Set the items session::database::SetResult set_result = db.set_pro_revocations(1, src_items); - INFO("Set w/ 2 items failed: " << sqlite3_errstr(set_result.return_code)); + INFO("Set w/ 2 items failed: " << sqlite3_errstr(set_result.sql_return_code)); REQUIRE(set_result.success); - REQUIRE(set_result.return_code == SQLITE_OK); + REQUIRE(set_result.sql_return_code == SQLITE_OK); // Check runtime ticket was changed to 1 runtime = db.get_runtime(); @@ -78,9 +80,9 @@ TEST_CASE("Database", "[database][pro][revocations]") { // Delete the first item (src[0]) from the DB session::pro_backend::ProRevocationItem set_item = src_items[1]; set_result = db.set_pro_revocations(2, std::span(&set_item, 1)); - INFO("Set w/ 1 item failed: " << sqlite3_errstr(set_result.return_code)); + INFO("Set w/ 1 item failed: " << sqlite3_errstr(set_result.sql_return_code)); REQUIRE(set_result.success); - REQUIRE(set_result.return_code == SQLITE_OK); + REQUIRE(set_result.sql_return_code == SQLITE_OK); // Count the number of revocations in the DB (should be 1 as we've deleted one of them) db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0, &ticket); From c8d2d784198ad9b5a0708bf2a5aab5ce1264920a Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 29 Oct 2025 16:09:45 +1100 Subject: [PATCH 10/72] Add C interface for database connections --- include/session/database/connection.h | 111 +++ include/session/database/connection.hpp | 11 +- include/session/pro_backend.hpp | 3 + include/session/util.hpp | 12 +- src/database/connection.cpp | 143 +++- src/pro_backend.cpp | 25 +- tests/test_database.cpp | 98 ++- tests/test_pro_backend.cpp | 856 ++++++++++++------------ tests/utils.hpp | 8 - 9 files changed, 780 insertions(+), 487 deletions(-) create mode 100644 include/session/database/connection.h diff --git a/include/session/database/connection.h b/include/session/database/connection.h new file mode 100644 index 00000000..dc29003b --- /dev/null +++ b/include/session/database/connection.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include + +#include "../export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct session_pro_backend_pro_revocation_item; + +typedef struct session_database_connection session_database_connection; +struct session_database_connection { + char opaque[16]; +}; + +typedef struct session_database_result session_database_result; +struct session_database_result { + bool success; + char error[256]; + size_t error_count; +}; + +typedef struct session_database_set_result session_database_set_result; +struct session_database_set_result { + session_database_result db; + int sql_return_code; + /// SQL's string-ified `sql_return_code` pointing to memory in the data segment. Should not be + /// modified and is valid for program lifetime. + const char* sql_error; +}; + +typedef struct session_database_get_pro_revocation_result session_database_get_pro_revocation_result; +struct session_database_get_pro_revocation_result { + session_database_result db; + size_t count; +}; + +/// API: session_database_connection_open +/// +/// Open a connection to the DB specified at `path`. If this connection previously has an open +/// DB that connection is gracefully closed before opening up the newly requested one. +/// +/// This function returns an if the DB was not openable, if the `raw_key` was the incorrect key to +/// decrypt the DB or the contents of the DB were malformed. +/// +/// Inputs: +/// - `conn` -- DB connection object that was zero-initialised or used previously +/// - `path` -- Path to the DB to open, this can be a URI or path on disk +/// - `raw_key` -- Encryption key to use to open the specified DB. If the DB does not exist then +/// the database will be created, encrypted with this key. +LIBSESSION_EXPORT session_database_result session_database_connection_open( + session_database_connection* conn, string8 path, span_u8 raw_key) NON_NULL_ARG(1); + +/// API: session_database_connection_close +/// +/// Close the DB connection which closes the underlying file descriptor referencing the database +/// +/// Inputs: +/// - `conn` -- DB connection object to close +LIBSESSION_EXPORT void session_database_connection_close(session_database_connection* conn) + NON_NULL_ARG(1); + +/// API: session_database_connection_set_pro_revocations +/// +/// Set the list of Session Pro revocations into the database associated with this connection +/// replacing the old revocations. This function is transactional, on failure changes to the +/// database are rolled back. +/// +/// Inputs: +/// - `ticket` -- Monotonic integer which is the version of the list, received by the Session +/// Pro Backend when syncing the revocation list. +/// - `revocations` -- The list of revocations to set +LIBSESSION_EXPORT session_database_set_result session_database_connection_set_pro_revocations( + session_database_connection* conn, + uint32_t ticket, + session_pro_backend_pro_revocation_item* revocations, + size_t revocations_len); + +/// API: database/get_pro_revocations_buffer +/// +/// Retrieve the Session Pro Backend revocation list given and output the rows into the given +/// `buf`. This function errors if SQLite returned an error +/// +/// Inputs: +/// - `buf` -- Buffer to write loaded revocations into. This can be nullptr in which case the +/// function returns the number of revocations currently in the DB. +/// - `buf_count` -- Size of the buffer and consequently the amount of revocations to load. This +/// can be 0 as well as setting `buf` to `nullptr` to make the function return the number of +/// revocations currently in the DB. +/// - `offset` -- Start retrieving revocation rows from this specified index of the list. Pass +/// in 0 to start from the beginning. +/// - `ticket` -- Retrieve the current ticket for the revocation list which represents the +/// current version of the list that has been synced from the Session Pro Backend. +/// +/// Outputs: +/// - `count` -- Number of revocation items read from the database. If the buffer was +/// insufficient sized to receive the rows, the return value is always capped to the size of +/// the buffer. If `buf` and `buf_count` are nullptr or 0 respectively, then value returned +/// is the amount of revocation items in the DB at the time of execution. +LIBSESSION_EXPORT session_database_get_pro_revocation_result session_database_connection_get_pro_revocations_buffer( + session_database_connection* conn, + OPTIONAL session_pro_backend_pro_revocation_item* buf, + size_t buf_count, + size_t offset, + OPTIONAL uint32_t* ticket); +#ifdef __cplusplus +} +#endif diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index 57270906..d9bb92d5 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -80,13 +80,16 @@ struct Connection { /// synced from the Session Pro Backend. Runtime get_runtime(); - /// API: database/add_pro_revocations + /// API: database/set_pro_revocations /// - /// Add a list of Session Pro revocations into the database associated with this connection. - /// This function is transactional, on failure changes to the database are rolled back. + /// Set the list of Session Pro revocations into the database associated with this connection + /// replacing the old revocations. This function is transactional, on failure changes to the + /// database are rolled back. /// /// Inputs: - /// - `revocations` -- The list of revocations to add + /// - `ticket` -- Monotonic integer which is the version of the list, received by the Session + /// Pro Backend when syncing the revocation list. + /// - `revocations` -- The list of revocations to set /// /// Outputs: /// - `success` -- True if the add was successful, false otherwise. diff --git a/include/session/pro_backend.hpp b/include/session/pro_backend.hpp index 6222d2a7..3cd4c6eb 100644 --- a/include/session/pro_backend.hpp +++ b/include/session/pro_backend.hpp @@ -668,4 +668,7 @@ struct SetPaymentRefundRequestedResponse : public ResponseHeader { /// `errors` static SetPaymentRefundRequestedResponse parse(std::string_view json); }; +session_pro_backend_pro_revocation_item revocation_c_from_cpp(ProRevocationItem const &src); + +ProRevocationItem revocation_cpp_from_c(session_pro_backend_pro_revocation_item const &src); } // namespace session::pro_backend diff --git a/include/session/util.hpp b/include/session/util.hpp index 8ee05286..28d82cd9 100644 --- a/include/session/util.hpp +++ b/include/session/util.hpp @@ -7,14 +7,13 @@ #include #include #include +#include #include #include #include #include #include -#include "types.hpp" - namespace session { using namespace oxenc; @@ -299,4 +298,13 @@ std::vector zstd_compress( /// then this returns nullopt if the decompressed size would exceed that limit. std::optional> zstd_decompress( std::span data, size_t max_size = 0); + +struct scope_exit { + explicit scope_exit(std::function func) : cleanup(func) {} + std::function cleanup; + ~scope_exit() { + if (cleanup) + cleanup(); + } +}; } // namespace session diff --git a/src/database/connection.cpp b/src/database/connection.cpp index 1b343a22..64b7b21c 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -3,14 +3,14 @@ #include #include -#include #include #include #include #include -auto logcat = oxen::log::Cat("database"); +#include "session/database/connection.h" +auto logcat = oxen::log::Cat("database"); namespace { void throw_sql_error(int sql_result, std::string_view error_prefix) { std::string msg = fmt::format("{}: {}", error_prefix, sqlite3_errstr(sql_result)); @@ -32,9 +32,10 @@ void sqlite3_deleter::operator()(sqlite3* db) const noexcept { sqlite3_close(db); } -void Connection::open(const std::string& path, const cleared_array<48> &raw_key) { +void Connection::open(const std::string& path, const cleared_array<48>& raw_key) { cleared_array<48> ZERO_RAW_KEY = {}; - assert(memcmp(raw_key.data(), ZERO_RAW_KEY.data(), raw_key.size()) != 0 && "Raw key was not set"); + assert(memcmp(raw_key.data(), ZERO_RAW_KEY.data(), raw_key.size()) != 0 && + "Raw key was not set"); // Open DB sqlite3* db_ptr = nullptr; @@ -55,13 +56,14 @@ void Connection::open(const std::string& path, const cleared_array<48> &raw_key) // // For more info, see: // https://www.zetetic.net/sqlcipher/sqlcipher-api/#cipher_plaintext_header_size - rc = sqlite3_exec(db_.get(), "PRAGMA cipher_plaintext_header_size = 32", nullptr, nullptr, nullptr); + rc = sqlite3_exec( + db_.get(), "PRAGMA cipher_plaintext_header_size = 32", nullptr, nullptr, nullptr); if (rc != SQLITE_OK) throw_sql_error(rc, "Failed to configure database"); // Set encryption key, this is the underlying function that "PRAGMA key = .." calls std::string fmt_key = fmt::format("x'{}'", oxenc::to_hex(raw_key)); - rc = sqlite3_key(db_.get(), fmt_key.c_str(), fmt_key.size()); + rc = sqlite3_key(db_.get(), fmt_key.c_str(), fmt_key.size()); sodium_zero_buffer(fmt_key.data(), fmt_key.size()); if (rc != SQLITE_OK) throw_sql_error(rc, "Failed to set encryption key"); @@ -141,7 +143,6 @@ void Connection::query(std::string_view sql, std::function fmt::format("Error executing query: {}", sqlite3_errmsg(db_.get()))); } - Runtime Connection::get_runtime() { Runtime result = {}; std::string_view sql = R"(SELECT id, pro_revocations_ticket FROM runtime LIMIT 1)"; @@ -158,7 +159,7 @@ SetResult Connection::set_pro_revocations( // The following consists of exception safe code so we do not need try catch and can trivially // commit or rollback the at the end of the function. exec("BEGIN DEFERRED TRANSACTION;"); - exec("DELETE FROM pro_revocations"); // Clear the table + exec("DELETE FROM pro_revocations"); // Clear the table // Assign the pro-revocations int rc = SQLITE_OK; @@ -261,7 +262,6 @@ OFFSET %zu auto expiry = std::chrono::milliseconds(sqlite3_column_int64(stmt, 1)); item.expiry_unix_ts = std::chrono::sys_time(expiry); }); - } // Retrieve the ticket @@ -273,7 +273,7 @@ OFFSET %zu return result; } -std::vector Connection::get_pro_revocations(uint32_t *ticket) { +std::vector Connection::get_pro_revocations(uint32_t* ticket) { std::vector result; size_t size_req = get_pro_revocations_buffer(nullptr, 0, 0, ticket); result.resize(size_req); @@ -282,3 +282,126 @@ std::vector Connection::get_pro_revocations(uint return result; } } // namespace session::database + +using namespace session::database; + +LIBSESSION_C_API session_database_result session_database_connection_open( + session_database_connection* conn, string8 path, span_u8 raw_key) { + session_database_result result = {}; + + static_assert( + sizeof(((session_database_connection*)0)->opaque) >= sizeof(Connection), + "C struct instantiates the C++ instance with an `opaque` buffer via placement new so " + "the capacity must be large enough to hold the `Connection` instance"); + + session_database_connection_close(conn); + Connection* conn_cpp = new (conn->opaque) Connection(); + + session::cleared_array<48> raw_key_cpp; + if (raw_key.size != raw_key_cpp.max_size()) { + result.error_count = snprintf_clamped( + result.error, + sizeof(result.error), + "Raw key must be 48 bytes, received %zu", + raw_key.size); + return result; + } + + // Must be string because we need to guarantee that `path` was null-terminated for the SQL API. + std::string path_cpp = std::string(path.data, path.data + path.size); + memcpy(raw_key_cpp.data(), raw_key.data, raw_key.size); + + try { + conn_cpp->open(path_cpp, raw_key_cpp); + result.success = true; + } catch (const std::exception& e) { + const std::string& error = e.what(); + result.error_count = snprintf_clamped( + result.error, + sizeof(result.error), + "%.*s", + static_cast(error.size()), + error.data()); + } + + return result; +} + +LIBSESSION_C_API void session_database_connection_close(session_database_connection* conn) { + auto* conn_cpp = reinterpret_cast(conn->opaque); + if (conn_cpp) { + conn_cpp->~Connection(); + memset(conn->opaque, 0, sizeof(conn->opaque)); + } +} + +LIBSESSION_C_API session_database_set_result session_database_connection_set_pro_revocations( + session_database_connection* conn, + uint32_t ticket, + session_pro_backend_pro_revocation_item* revocations, + size_t revocations_len) { + session_database_set_result result = {}; + auto* conn_cpp = reinterpret_cast(conn->opaque); + try { + // Convert revocations to CPP instance + std::vector revocations_cpp; + revocations_cpp.reserve(revocations_len); + for (size_t index = 0; index < revocations_len; index++) { + const session_pro_backend_pro_revocation_item& src = revocations[index]; + session::pro_backend::ProRevocationItem& dest = revocations_cpp.emplace_back(); + dest = session::pro_backend::revocation_cpp_from_c(src); + } + + // Do the operation + SetResult result_cpp = conn_cpp->set_pro_revocations(ticket, revocations_cpp); + result.db.success = result_cpp.success; + result.sql_return_code = result_cpp.sql_return_code; + result.sql_error = result_cpp.sql_error; + } catch (const std::exception& e) { + const std::string& error = e.what(); + result.db.error_count = snprintf_clamped( + result.db.error, + sizeof(result.db.error), + "%.*s", + static_cast(error.size()), + error.data()); + } + + return result; +} + +LIBSESSION_C_API session_database_get_pro_revocation_result session_database_connection_get_pro_revocations_buffer( + session_database_connection* conn, + OPTIONAL session_pro_backend_pro_revocation_item* buf, + size_t buf_count, + size_t offset, + OPTIONAL uint32_t* ticket) { + auto* conn_cpp = reinterpret_cast(conn->opaque); + session_database_get_pro_revocation_result result = {}; + try { + std::vector buf_cpp; + if (buf && buf_count) + buf_cpp.resize(buf_count); + + result.count = + conn_cpp->get_pro_revocations_buffer(buf_cpp.data(), buf_count, offset, ticket); + buf_cpp.resize(result.count); + + if (buf) { + for (size_t index = 0; index < result.count; index++) + buf[index] = session::pro_backend::revocation_c_from_cpp(buf_cpp[index]); + } + + result.db.success = true; + } catch (std::exception& e) { + const std::string& error = e.what(); + result.db.error_count = snprintf_clamped( + result.db.error, + sizeof(result.db.error), + "%.*s", + static_cast(error.size()), + error.data()); + } + + return result; +} diff --git a/src/pro_backend.cpp b/src/pro_backend.cpp index ae6714df..8c70d2d4 100644 --- a/src/pro_backend.cpp +++ b/src/pro_backend.cpp @@ -776,6 +776,25 @@ GetProDetailsResponse GetProDetailsResponse::parse(std::string_view json) { return result; } +session_pro_backend_pro_revocation_item revocation_c_from_cpp(ProRevocationItem const &src) +{ + session_pro_backend_pro_revocation_item result = {}; + std::memcpy(result.gen_index_hash.data, src.gen_index_hash.data(), src.gen_index_hash.size()); + result.expiry_unix_ts_ms = std::chrono::duration_cast( + src.expiry_unix_ts.time_since_epoch()) + .count(); + return result; +} + +ProRevocationItem revocation_cpp_from_c(session_pro_backend_pro_revocation_item const &src) +{ + pro_backend::ProRevocationItem result = {}; + memcpy(result.gen_index_hash.data(), src.gen_index_hash.data, sizeof(src.gen_index_hash.data)); + result.expiry_unix_ts = std::chrono::sys_time( + std::chrono::milliseconds(src.expiry_unix_ts_ms)); + return result; +} + array_uc64 SetPaymentRefundRequestedRequest::build_sig( uint8_t version, std::span master_privkey, @@ -1407,11 +1426,7 @@ session_pro_backend_get_pro_revocations_response_parse(const char* json, size_t for (size_t index = 0; index < result.items_count; ++index) { const ProRevocationItem& src = cpp.items[index]; - session_pro_backend_pro_revocation_item& dest = result.items[index]; - std::memcpy(dest.gen_index_hash.data, src.gen_index_hash.data(), src.gen_index_hash.size()); - dest.expiry_unix_ts_ms = std::chrono::duration_cast( - src.expiry_unix_ts.time_since_epoch()) - .count(); + result.items[index] = revocation_c_from_cpp(src); } return result; } diff --git a/tests/test_database.cpp b/tests/test_database.cpp index a3beae5d..68dacaca 100644 --- a/tests/test_database.cpp +++ b/tests/test_database.cpp @@ -5,94 +5,134 @@ #include #include +#include "session/database/connection.h" #include "session/database/connection.hpp" #include "session/pro_backend.hpp" TEST_CASE("Database", "[database][open]") { session::cleared_array<48> raw_key = {}; randombytes_buf(raw_key.data(), raw_key.size()); - session::database::Connection db = {}; - db.open(":memory:", raw_key); + span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; + + session_database_connection zero_db = {}; + session_database_connection db = {}; + session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); + REQUIRE(memcmp(db.opaque, zero_db.opaque, sizeof(db.opaque)) != 0); + + session_database_connection_close(&db); + REQUIRE(memcmp(db.opaque, zero_db.opaque, sizeof(db.opaque)) == 0); } TEST_CASE("Database", "[database][pro][revocations]") { session::cleared_array<48> raw_key = {}; randombytes_buf(raw_key.data(), raw_key.size()); - session::database::Connection db = {}; - db.open(":memory:", raw_key); + span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; + + session_database_connection db = {}; + session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); + + auto* db_cpp = reinterpret_cast(db.opaque); // Check runtime was seeded to ticket 0 - session::database::Runtime runtime = db.get_runtime(); + session::database::Runtime runtime = db_cpp->get_runtime(); REQUIRE(runtime.id == 1); REQUIRE(runtime.pro_revocations_ticket == 0); // Check that the DB has no revocations in it uint32_t ticket = 0; - size_t db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0, &ticket); - REQUIRE(db_item_count == 0); + session_database_get_pro_revocation_result get_result = + session_database_connection_get_pro_revocations_buffer(&db, nullptr, 0, 0, &ticket); + REQUIRE(get_result.db.success); + REQUIRE(get_result.count == 0); // Create the revocations we will put into the DB uint64_t unix_ts_ms = 1698765432ULL * 1000; // Arbitrary timestamp auto unix_ts = std::chrono::sys_time(std::chrono::milliseconds(unix_ts_ms)); - session::pro_backend::ProRevocationItem src_items[] = { + session_pro_backend_pro_revocation_item src_items[] = { { .gen_index_hash = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - .expiry_unix_ts = unix_ts, + .expiry_unix_ts_ms = static_cast(unix_ts.time_since_epoch().count()), }, { .gen_index_hash = {0x33, 0xa1, 0xc4, 0x4e, 0x60, 0x94, 0x48, 0x8f, 0x5c, 0xeb, 0xe2, 0x4b, 0xfc, 0xf9, 0x89, 0xda, 0x07, 0xdd, 0xc4, 0x8d, 0xe2, 0xae, 0x86, 0x6c, 0x8c, 0x78, 0xb9, 0x16, 0x60, 0xc8, 0x49, 0xf1}, - .expiry_unix_ts = unix_ts, + .expiry_unix_ts_ms = static_cast(unix_ts.time_since_epoch().count()), }, }; // Set the items - session::database::SetResult set_result = db.set_pro_revocations(1, src_items); + session_database_set_result set_result = session_database_connection_set_pro_revocations( + &db, 1, src_items, sizeof(src_items) / sizeof(src_items[0])); INFO("Set w/ 2 items failed: " << sqlite3_errstr(set_result.sql_return_code)); - REQUIRE(set_result.success); + REQUIRE(set_result.db.success); REQUIRE(set_result.sql_return_code == SQLITE_OK); // Check runtime ticket was changed to 1 - runtime = db.get_runtime(); + runtime = db_cpp->get_runtime(); REQUIRE(runtime.id == 1); REQUIRE(runtime.pro_revocations_ticket == 1); // Count the number of revocations in the DB (should be 2 as we've inserted them) - db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0, &ticket); + get_result = + session_database_connection_get_pro_revocations_buffer(&db, nullptr, 0, 0, &ticket); REQUIRE(ticket == runtime.pro_revocations_ticket); - REQUIRE(db_item_count == 2); + REQUIRE(get_result.db.success); + REQUIRE(get_result.count == 2); // Check that the revocations was in the DB - std::vector db_items = db.get_pro_revocations(&ticket); + std::vector db_items = + db_cpp->get_pro_revocations(&ticket); REQUIRE(ticket == 1); - REQUIRE(src_items[0].gen_index_hash == db_items[0].gen_index_hash); - REQUIRE(src_items[0].expiry_unix_ts == db_items[0].expiry_unix_ts); - REQUIRE(src_items[1].gen_index_hash == db_items[1].gen_index_hash); - REQUIRE(src_items[1].expiry_unix_ts == db_items[1].expiry_unix_ts); + REQUIRE(memcmp(src_items[0].gen_index_hash.data, + db_items[0].gen_index_hash.data(), + db_items[0].gen_index_hash.size()) == 0); + REQUIRE(src_items[0].expiry_unix_ts_ms == + db_items[0].expiry_unix_ts.time_since_epoch().count()); + + REQUIRE(memcmp(src_items[1].gen_index_hash.data, + db_items[1].gen_index_hash.data(), + db_items[1].gen_index_hash.size()) == 0); + REQUIRE(src_items[1].expiry_unix_ts_ms == + db_items[1].expiry_unix_ts.time_since_epoch().count()); // Delete the first item (src[0]) from the DB - session::pro_backend::ProRevocationItem set_item = src_items[1]; - set_result = db.set_pro_revocations(2, std::span(&set_item, 1)); + session_pro_backend_pro_revocation_item set_item = src_items[1]; + set_result = session_database_connection_set_pro_revocations(&db, 2, &set_item, 1); INFO("Set w/ 1 item failed: " << sqlite3_errstr(set_result.sql_return_code)); - REQUIRE(set_result.success); + REQUIRE(set_result.db.success); REQUIRE(set_result.sql_return_code == SQLITE_OK); // Count the number of revocations in the DB (should be 1 as we've deleted one of them) - db_item_count = db.get_pro_revocations_buffer(nullptr, 0, 0, &ticket); - REQUIRE(db_item_count == 1); + get_result = + session_database_connection_get_pro_revocations_buffer(&db, nullptr, 0, 0, &ticket); + REQUIRE(get_result.db.success); + REQUIRE(get_result.count == 1); REQUIRE(ticket == 2); // Verify that the DB now has just the item at src[1] - std::vector db_items_after_delete = db.get_pro_revocations(&ticket); - REQUIRE(db_items_after_delete.size() == 1); - REQUIRE(src_items[1].gen_index_hash == db_items_after_delete[0].gen_index_hash); - REQUIRE(src_items[1].expiry_unix_ts == db_items_after_delete[0].expiry_unix_ts); + session_pro_backend_pro_revocation_item db_items_after_delete[2]; + assert(sizeof(db_items_after_delete) / sizeof(db_items_after_delete[0]) >= get_result.count); + + session_database_get_pro_revocation_result db_items_after_delete_result = + session_database_connection_get_pro_revocations_buffer( + &db, + db_items_after_delete, + sizeof(db_items_after_delete) / sizeof(db_items_after_delete[0]), + 0, + &ticket); + + REQUIRE(db_items_after_delete_result.db.success); + REQUIRE(db_items_after_delete_result.count == 1); + REQUIRE(memcmp(src_items[1].gen_index_hash.data, + db_items_after_delete[0].gen_index_hash.data, + sizeof(db_items_after_delete[0].gen_index_hash.data)) == 0); + REQUIRE(src_items[1].expiry_unix_ts_ms == db_items_after_delete[0].expiry_unix_ts_ms); REQUIRE(ticket == 2); } diff --git a/tests/test_pro_backend.cpp b/tests/test_pro_backend.cpp index 9ffc40da..83d1f64e 100644 --- a/tests/test_pro_backend.cpp +++ b/tests/test_pro_backend.cpp @@ -229,7 +229,8 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { // Valid request auto result = session_pro_backend_add_pro_payment_request_to_json(&request); { - scope_exit result_free{[&]() { session_pro_backend_to_json_free(&result); }}; + session::scope_exit result_free{ + [&]() { session_pro_backend_to_json_free(&result); }}; REQUIRE(result.success); REQUIRE(result.json.data != nullptr); REQUIRE(result.json.size > 0); @@ -310,7 +311,8 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { // Valid request auto result = session_pro_backend_generate_pro_proof_request_to_json(&request); { - scope_exit result_free{[&]() { session_pro_backend_to_json_free(&result); }}; + session::scope_exit result_free{ + [&]() { session_pro_backend_to_json_free(&result); }}; REQUIRE(result.success); REQUIRE(result.json.data != nullptr); REQUIRE(result.json.size > 0); @@ -367,7 +369,8 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { // Valid request auto result = session_pro_backend_get_pro_revocations_request_to_json(&request); { - scope_exit result_free{[&]() { session_pro_backend_to_json_free(&result); }}; + session::scope_exit result_free{ + [&]() { session_pro_backend_to_json_free(&result); }}; REQUIRE(result.success); REQUIRE(result.json.data != nullptr); REQUIRE(result.json.size > 0); @@ -411,7 +414,8 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { // Valid request auto result = session_pro_backend_get_pro_details_request_to_json(&request); { - scope_exit result_free{[&]() { session_pro_backend_to_json_free(&result); }}; + session::scope_exit result_free{ + [&]() { session_pro_backend_to_json_free(&result); }}; REQUIRE(result.success); REQUIRE(result.json.data != nullptr); REQUIRE(result.json.size > 0); @@ -471,7 +475,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( json.data(), json.size()); { - scope_exit result_free{[&]() { + session::scope_exit result_free{[&]() { session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free( &result); }}; @@ -528,7 +532,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { result = session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( json.data(), json.size()); { - scope_exit result_free{[&]() { + session::scope_exit result_free{[&]() { session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free( &result); }}; @@ -571,7 +575,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { auto result = session_pro_backend_get_pro_revocations_response_parse( json.data(), json.size()); { - scope_exit result_free{ + session::scope_exit result_free{ [&]() { session_pro_backend_get_pro_revocations_response_free(&result); }}; for (size_t index = 0; index < result.header.errors_count; index++) INFO(result.header.errors[index].data); @@ -598,7 +602,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { { result = session_pro_backend_get_pro_revocations_response_parse( json.data(), json.size()); - scope_exit result_free{ + session::scope_exit result_free{ [&]() { session_pro_backend_get_pro_revocations_response_free(&result); }}; for (size_t index = 0; index < result.header.errors_count; index++) REQUIRE(result.header.status != SESSION_PRO_BACKEND_STATUS_SUCCESS); @@ -652,7 +656,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { auto result = session_pro_backend_get_pro_details_response_parse(json.data(), json.size()); { - scope_exit result_free{ + session::scope_exit result_free{ [&]() { session_pro_backend_get_pro_details_response_free(&result); }}; for (size_t index = 0; index < result.header.errors_count; index++) INFO(result.header.errors[index].data); @@ -702,7 +706,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { json = "{invalid}"; result = session_pro_backend_get_pro_details_response_parse(json.data(), json.size()); { - scope_exit result_free{ + session::scope_exit result_free{ [&]() { session_pro_backend_get_pro_details_response_free(&result); }}; REQUIRE(result.header.status != SESSION_PRO_BACKEND_STATUS_SUCCESS); REQUIRE(result.header.errors_count > 0); @@ -767,7 +771,8 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { auto result = session_pro_backend_set_payment_refund_requested_request_to_json(&request); { - scope_exit result_free{[&]() { session_pro_backend_to_json_free(&result); }}; + session::scope_exit result_free{ + [&]() { session_pro_backend_to_json_free(&result); }}; REQUIRE(result.success); REQUIRE(result.json.data != nullptr); REQUIRE(result.json.size > 0); @@ -814,7 +819,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { auto result = session_pro_backend_set_payment_refund_requested_response_parse( json.data(), json.size()); { - scope_exit result_free{[&]() { + session::scope_exit result_free{[&]() { session_pro_backend_set_payment_refund_requested_response_free(&result); }}; for (size_t index = 0; index < result.header.errors_count; index++) @@ -834,7 +839,7 @@ TEST_CASE("Pro Backend C API", "[pro_backend]") { { result = session_pro_backend_set_payment_refund_requested_response_parse( json.data(), json.size()); - scope_exit result_free{[&]() { + session::scope_exit result_free{[&]() { session_pro_backend_set_payment_refund_requested_response_free(&result); }}; for (size_t index = 0; index < result.header.errors_count; index++) @@ -889,447 +894,440 @@ std::string curl_do_basic_blocking_post_request( } TEST_CASE("Pro Backend Dev Server", "[pro_backend][dev_server]") { - if (g_test_pro_backend_dev_server_url.size()) { - // Setup: Generate keys and payment token hash - bytes32 master_pubkey = {}; - bytes64 master_privkey = {}; - crypto_sign_ed25519_keypair(master_pubkey.data, master_privkey.data); - - bytes32 rotating_pubkey = {}; - bytes64 rotating_privkey = {}; - crypto_sign_ed25519_keypair(rotating_pubkey.data, rotating_privkey.data); - - const auto DEV_BACKEND_PUBKEY = - "fc947730f49eb01427a66e050733294d9e520e545c7a27125a780634e0860a27"_hexbytes; - - // Setup CURL - curl_global_init(CURL_GLOBAL_DEFAULT); - scope_exit curl_cleanup{[&]() { curl_global_cleanup(); }}; - - CURL* curl = curl_easy_init(); - REQUIRE(curl); - scope_exit curl_free{[&]() { curl_easy_cleanup(curl); }}; - - struct curl_slist* curl_headers = nullptr; - curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json"); - REQUIRE(curl_headers); - scope_exit curl_headers_free{[&]() { curl_slist_free_all(curl_headers); }}; - - // Add pro payment - session_protocol_pro_proof first_pro_proof = {}; - { - std::array fake_google_payment_token; - randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); - std::string fake_google_payment_token_hex = - "DEV." + oxenc::to_hex(fake_google_payment_token); - - std::array fake_google_order_id; - randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); - std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); - - session_pro_backend_add_pro_payment_user_transaction payment_tx = {}; - payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; - payment_tx.payment_id_count = fake_google_payment_token_hex.size(); - payment_tx.order_id_count = fake_google_order_id_hex.size(); - std::memcpy( - payment_tx.payment_id, - fake_google_payment_token_hex.data(), - payment_tx.payment_id_count); - std::memcpy( - payment_tx.order_id, - fake_google_order_id_hex.data(), - payment_tx.order_id_count); + // Setup: Generate keys and payment token hash + bytes32 master_pubkey = {}; + bytes64 master_privkey = {}; + crypto_sign_ed25519_keypair(master_pubkey.data, master_privkey.data); - // Build request - session_pro_backend_master_rotating_signatures add_pro_sigs = - session_pro_backend_add_pro_payment_request_build_sigs( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - rotating_privkey.data, - sizeof(rotating_privkey.data), - payment_tx.provider, - reinterpret_cast(payment_tx.payment_id), - payment_tx.payment_id_count, - reinterpret_cast(payment_tx.order_id), - payment_tx.order_id_count); + bytes32 rotating_pubkey = {}; + bytes64 rotating_privkey = {}; + crypto_sign_ed25519_keypair(rotating_pubkey.data, rotating_privkey.data); - session_pro_backend_add_pro_payment_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.rotating_pkey = rotating_pubkey; - request.payment_tx = payment_tx; - request.master_sig = add_pro_sigs.master_sig; - request.rotating_sig = add_pro_sigs.rotating_sig; - - session_pro_backend_to_json request_json = - session_pro_backend_add_pro_payment_request_to_json(&request); - scope_exit request_json_free{ - [&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/add_pro_payment", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - }}; - - for (size_t index = 0; index < response.header.errors_count; index++) { - string8 error = response.header.errors[index]; - INFO("ERROR: " << error.data); - } + const auto DEV_BACKEND_PUBKEY = + "fc947730f49eb01427a66e050733294d9e520e545c7a27125a780634e0860a27"_hexbytes; - // Verify response - first_pro_proof = response.proof; - INFO("Signature: " << oxenc::to_hex( - first_pro_proof.sig.data, - std::end(first_pro_proof.sig.data)) - << ", backend pubkey: " << oxenc::to_hex(DEV_BACKEND_PUBKEY) - << ", response: " << response_json); - REQUIRE(session_protocol_pro_proof_verify_signature( - &first_pro_proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); - REQUIRE(std::memcmp( - response.proof.rotating_pubkey.data, - request.rotating_pkey.data, - sizeof(request.rotating_pkey.data)) == 0); - } + // Setup CURL + curl_global_init(CURL_GLOBAL_DEFAULT); + session::scope_exit curl_cleanup{[&]() { curl_global_cleanup(); }}; - // Authorise new key - { - uint64_t now_unix_ts_ms = time(nullptr) * 1000; - // Build request - session_pro_backend_master_rotating_signatures pro_sigs = - session_pro_backend_generate_pro_proof_request_build_sigs( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - rotating_privkey.data, - sizeof(rotating_privkey.data), - now_unix_ts_ms); + CURL* curl = curl_easy_init(); + REQUIRE(curl); + session::scope_exit curl_free{[&]() { curl_easy_cleanup(curl); }}; - session_pro_backend_generate_pro_proof_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.rotating_pkey = rotating_pubkey; - request.unix_ts_ms = now_unix_ts_ms; - request.master_sig = pro_sigs.master_sig; - request.rotating_sig = pro_sigs.rotating_sig; - - session_pro_backend_to_json request_json = - session_pro_backend_generate_pro_proof_request_to_json(&request); - scope_exit request_json_free{ - [&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do CURL request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/generate_pro_proof", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - }}; - - for (size_t index = 0; index < response.header.errors_count; index++) { - if (index == 0) - fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); - } - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + struct curl_slist* curl_headers = nullptr; + curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json"); + REQUIRE(curl_headers); + session::scope_exit curl_headers_free{[&]() { curl_slist_free_all(curl_headers); }}; - // Verify response - session_protocol_pro_proof proof = response.proof; - REQUIRE(session_protocol_pro_proof_verify_signature( - &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); - REQUIRE(std::memcmp( - response.proof.rotating_pubkey.data, - request.rotating_pkey.data, - sizeof(request.rotating_pkey.data)) == 0); + // Add pro payment + session_protocol_pro_proof first_pro_proof = {}; + { + std::array fake_google_payment_token; + randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); + std::string fake_google_payment_token_hex = + "DEV." + oxenc::to_hex(fake_google_payment_token); - session_pro_backend_to_json_free(&request_json); - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - } + std::array fake_google_order_id; + randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); + std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); - // Get pro status - { - // Build request - session_pro_backend_get_pro_details_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.unix_ts_ms = time(nullptr) * 1000; - request.count = 10'000; + session_pro_backend_add_pro_payment_user_transaction payment_tx = {}; + payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; + payment_tx.payment_id_count = fake_google_payment_token_hex.size(); + payment_tx.order_id_count = fake_google_order_id_hex.size(); + std::memcpy( + payment_tx.payment_id, + fake_google_payment_token_hex.data(), + payment_tx.payment_id_count); + std::memcpy( + payment_tx.order_id, fake_google_order_id_hex.data(), payment_tx.order_id_count); - session_pro_backend_signature sig = - session_pro_backend_get_pro_details_request_build_sig( - request.version, - master_privkey.data, - sizeof(master_privkey.data), - request.unix_ts_ms, - request.count); - REQUIRE(sig.success); - request.master_sig = sig.sig; + // Build request + session_pro_backend_master_rotating_signatures add_pro_sigs = + session_pro_backend_add_pro_payment_request_build_sigs( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + rotating_privkey.data, + sizeof(rotating_privkey.data), + payment_tx.provider, + reinterpret_cast(payment_tx.payment_id), + payment_tx.payment_id_count, + reinterpret_cast(payment_tx.order_id), + payment_tx.order_id_count); + + session_pro_backend_add_pro_payment_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.rotating_pkey = rotating_pubkey; + request.payment_tx = payment_tx; + request.master_sig = add_pro_sigs.master_sig; + request.rotating_sig = add_pro_sigs.rotating_sig; + + session_pro_backend_to_json request_json = + session_pro_backend_add_pro_payment_request_to_json(&request); + session::scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/add_pro_payment", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( + response_json.data(), response_json.size()); + session::scope_exit response_free{[&]() { + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); + }}; - // Do CURL request - session_pro_backend_to_json request_json = - session_pro_backend_get_pro_details_request_to_json(&request); - scope_exit request_json_free{ - [&]() { session_pro_backend_to_json_free(&request_json); }}; - - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/get_pro_details", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_get_pro_details_response response = - session_pro_backend_get_pro_details_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{ - [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; - - // Verify the response - for (size_t index = 0; index < response.header.errors_count; index++) { - if (index == 0) - fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); - } - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); - REQUIRE(response.items_count > 0); + for (size_t index = 0; index < response.header.errors_count; index++) { + string8 error = response.header.errors[index]; + INFO("ERROR: " << error.data); } - // Get pro status without history - { - // Build request - session_pro_backend_get_pro_details_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.unix_ts_ms = time(nullptr) * 1000; - - session_pro_backend_signature sig = - session_pro_backend_get_pro_details_request_build_sig( - request.version, - master_privkey.data, - sizeof(master_privkey.data), - request.unix_ts_ms, - request.count); - REQUIRE(sig.success); - request.master_sig = sig.sig; + // Verify response + first_pro_proof = response.proof; + INFO("Signature: " << oxenc::to_hex( + first_pro_proof.sig.data, std::end(first_pro_proof.sig.data)) + << ", backend pubkey: " << oxenc::to_hex(DEV_BACKEND_PUBKEY) + << ", response: " << response_json); + REQUIRE(session_protocol_pro_proof_verify_signature( + &first_pro_proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); + REQUIRE(std::memcmp( + response.proof.rotating_pubkey.data, + request.rotating_pkey.data, + sizeof(request.rotating_pkey.data)) == 0); + } - // Do CURL request - session_pro_backend_to_json request_json = - session_pro_backend_get_pro_details_request_to_json(&request); - scope_exit request_json_free{ - [&]() { session_pro_backend_to_json_free(&request_json); }}; - - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/get_pro_details", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_get_pro_details_response response = - session_pro_backend_get_pro_details_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{ - [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; - - for (size_t index = 0; index < response.header.errors_count; index++) { - if (index == 0) - fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); - } + // Authorise new key + { + uint64_t now_unix_ts_ms = time(nullptr) * 1000; + // Build request + session_pro_backend_master_rotating_signatures pro_sigs = + session_pro_backend_generate_pro_proof_request_build_sigs( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + rotating_privkey.data, + sizeof(rotating_privkey.data), + now_unix_ts_ms); + + session_pro_backend_generate_pro_proof_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.rotating_pkey = rotating_pubkey; + request.unix_ts_ms = now_unix_ts_ms; + request.master_sig = pro_sigs.master_sig; + request.rotating_sig = pro_sigs.rotating_sig; + + session_pro_backend_to_json request_json = + session_pro_backend_generate_pro_proof_request_to_json(&request); + session::scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do CURL request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/generate_pro_proof", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( + response_json.data(), response_json.size()); + session::scope_exit response_free{[&]() { + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); + }}; - // Verify the response - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); - REQUIRE(response.items_count == 0); + for (size_t index = 0; index < response.header.errors_count; index++) { + if (index == 0) + fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); } + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + + // Verify response + session_protocol_pro_proof proof = response.proof; + REQUIRE(session_protocol_pro_proof_verify_signature( + &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); + REQUIRE(std::memcmp( + response.proof.rotating_pubkey.data, + request.rotating_pkey.data, + sizeof(request.rotating_pkey.data)) == 0); + + session_pro_backend_to_json_free(&request_json); + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); + } - // Add _another_ payment, same details - session_pro_backend_add_pro_payment_user_transaction another_payment_tx = {}; - { - std::array fake_google_payment_token; - randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); - std::string fake_google_payment_token_hex = - "DEV." + oxenc::to_hex(fake_google_payment_token); - - std::array fake_google_order_id; - randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); - std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); - - another_payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; - another_payment_tx.payment_id_count = fake_google_payment_token_hex.size(); - another_payment_tx.order_id_count = fake_google_order_id_hex.size(); - std::memcpy( - another_payment_tx.payment_id, - fake_google_payment_token_hex.data(), - another_payment_tx.payment_id_count); - std::memcpy( - another_payment_tx.order_id, - fake_google_order_id_hex.data(), - another_payment_tx.order_id_count); - - // Build request - session_pro_backend_master_rotating_signatures add_pro_sigs = - session_pro_backend_add_pro_payment_request_build_sigs( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - rotating_privkey.data, - sizeof(rotating_privkey.data), - another_payment_tx.provider, - reinterpret_cast(another_payment_tx.payment_id), - another_payment_tx.payment_id_count, - reinterpret_cast(another_payment_tx.order_id), - another_payment_tx.order_id_count); + // Get pro status + { + // Build request + session_pro_backend_get_pro_details_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.unix_ts_ms = time(nullptr) * 1000; + request.count = 10'000; + + session_pro_backend_signature sig = session_pro_backend_get_pro_details_request_build_sig( + request.version, + master_privkey.data, + sizeof(master_privkey.data), + request.unix_ts_ms, + request.count); + REQUIRE(sig.success); + request.master_sig = sig.sig; + + // Do CURL request + session_pro_backend_to_json request_json = + session_pro_backend_get_pro_details_request_to_json(&request); + session::scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/get_pro_details", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_get_pro_details_response response = + session_pro_backend_get_pro_details_response_parse( + response_json.data(), response_json.size()); + session::scope_exit response_free{ + [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; + + // Verify the response + for (size_t index = 0; index < response.header.errors_count; index++) { + if (index == 0) + fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); + } + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); + REQUIRE(response.items_count > 0); + } - session_pro_backend_add_pro_payment_request request = {}; - request.version = 0; - request.master_pkey = master_pubkey; - request.rotating_pkey = rotating_pubkey; - request.payment_tx = another_payment_tx; - request.master_sig = add_pro_sigs.master_sig; - request.rotating_sig = add_pro_sigs.rotating_sig; - - session_pro_backend_to_json request_json = - session_pro_backend_add_pro_payment_request_to_json(&request); - scope_exit request_json_free{ - [&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/add_pro_payment", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { - session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); - }}; - - // Verify response - session_protocol_pro_proof proof = response.proof; - REQUIRE(session_protocol_pro_proof_verify_signature( - &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); - REQUIRE(std::memcmp( - response.proof.rotating_pubkey.data, - request.rotating_pkey.data, - sizeof(request.rotating_pkey.data)) == 0); + // Get pro status without history + { + // Build request + session_pro_backend_get_pro_details_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.unix_ts_ms = time(nullptr) * 1000; + + session_pro_backend_signature sig = session_pro_backend_get_pro_details_request_build_sig( + request.version, + master_privkey.data, + sizeof(master_privkey.data), + request.unix_ts_ms, + request.count); + REQUIRE(sig.success); + request.master_sig = sig.sig; + + // Do CURL request + session_pro_backend_to_json request_json = + session_pro_backend_get_pro_details_request_to_json(&request); + session::scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/get_pro_details", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_get_pro_details_response response = + session_pro_backend_get_pro_details_response_parse( + response_json.data(), response_json.size()); + session::scope_exit response_free{ + [&]() { session_pro_backend_get_pro_details_response_free(&response); }}; + + for (size_t index = 0; index < response.header.errors_count; index++) { + if (index == 0) + fprintf(stderr, "ERROR: JSON response: %s\n", response_json.c_str()); + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); } - // Get revocation list - { - // Build request - session_pro_backend_get_pro_revocations_request request = {}; - request.version = 0; + // Verify the response + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.status == SESSION_PRO_BACKEND_USER_PRO_STATUS_ACTIVE); + REQUIRE(response.items_count == 0); + } - session_pro_backend_to_json request_json = - session_pro_backend_get_pro_revocations_request_to_json(&request); - scope_exit request_json_free{ - [&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/get_pro_revocations", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_get_pro_revocations_response response = - session_pro_backend_get_pro_revocations_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{ - [&]() { session_pro_backend_get_pro_revocations_response_free(&response); }}; - - // Verify response - INFO("ERROR: JSON response: " << response_json.c_str()); - for (size_t index = 0; index < response.header.errors_count; index++) { - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); - } + // Add _another_ payment, same details + session_pro_backend_add_pro_payment_user_transaction another_payment_tx = {}; + { + std::array fake_google_payment_token; + randombytes_buf(fake_google_payment_token.data(), fake_google_payment_token.size()); + std::string fake_google_payment_token_hex = + "DEV." + oxenc::to_hex(fake_google_payment_token); - // Verify the response - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.ticket == 0); - REQUIRE(response.items_count == 0); + std::array fake_google_order_id; + randombytes_buf(fake_google_order_id.data(), fake_google_order_id.size()); + std::string fake_google_order_id_hex = "DEV." + oxenc::to_hex(fake_google_order_id); + + another_payment_tx.provider = SESSION_PRO_BACKEND_PAYMENT_PROVIDER_GOOGLE_PLAY_STORE; + another_payment_tx.payment_id_count = fake_google_payment_token_hex.size(); + another_payment_tx.order_id_count = fake_google_order_id_hex.size(); + std::memcpy( + another_payment_tx.payment_id, + fake_google_payment_token_hex.data(), + another_payment_tx.payment_id_count); + std::memcpy( + another_payment_tx.order_id, + fake_google_order_id_hex.data(), + another_payment_tx.order_id_count); + + // Build request + session_pro_backend_master_rotating_signatures add_pro_sigs = + session_pro_backend_add_pro_payment_request_build_sigs( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + rotating_privkey.data, + sizeof(rotating_privkey.data), + another_payment_tx.provider, + reinterpret_cast(another_payment_tx.payment_id), + another_payment_tx.payment_id_count, + reinterpret_cast(another_payment_tx.order_id), + another_payment_tx.order_id_count); + + session_pro_backend_add_pro_payment_request request = {}; + request.version = 0; + request.master_pkey = master_pubkey; + request.rotating_pkey = rotating_pubkey; + request.payment_tx = another_payment_tx; + request.master_sig = add_pro_sigs.master_sig; + request.rotating_sig = add_pro_sigs.rotating_sig; + + session_pro_backend_to_json request_json = + session_pro_backend_add_pro_payment_request_to_json(&request); + session::scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/add_pro_payment", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_add_pro_payment_or_generate_pro_proof_response response = + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_parse( + response_json.data(), response_json.size()); + session::scope_exit response_free{[&]() { + session_pro_backend_add_pro_payment_or_generate_pro_proof_response_free(&response); + }}; + + // Verify response + session_protocol_pro_proof proof = response.proof; + REQUIRE(session_protocol_pro_proof_verify_signature( + &proof, DEV_BACKEND_PUBKEY.data(), DEV_BACKEND_PUBKEY.size())); + REQUIRE(std::memcmp( + response.proof.rotating_pubkey.data, + request.rotating_pkey.data, + sizeof(request.rotating_pkey.data)) == 0); + } + + // Get revocation list + { + // Build request + session_pro_backend_get_pro_revocations_request request = {}; + request.version = 0; + + session_pro_backend_to_json request_json = + session_pro_backend_get_pro_revocations_request_to_json(&request); + session::scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/get_pro_revocations", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_get_pro_revocations_response response = + session_pro_backend_get_pro_revocations_response_parse( + response_json.data(), response_json.size()); + session::scope_exit response_free{ + [&]() { session_pro_backend_get_pro_revocations_response_free(&response); }}; + + // Verify response + INFO("ERROR: JSON response: " << response_json.c_str()); + for (size_t index = 0; index < response.header.errors_count; index++) { + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); } - // Set payment refund requested - { - // Build request - uint64_t now_unix_ts_ms = time(nullptr) * 1000; - session_pro_backend_to_json request_json = - session_pro_backend_set_payment_refund_requested_request_build_to_json( - /*version*/ 0, - master_privkey.data, - sizeof(master_privkey.data), - /*unix_ts_ms*/ now_unix_ts_ms, - /*refund_requested_unix_ts_ms*/ now_unix_ts_ms, - another_payment_tx.provider, - reinterpret_cast(another_payment_tx.payment_id), - another_payment_tx.payment_id_count, - reinterpret_cast(another_payment_tx.order_id), - another_payment_tx.order_id_count); - - scope_exit request_json_free{ - [&]() { session_pro_backend_to_json_free(&request_json); }}; - - // Do curl request - std::string response_json = curl_do_basic_blocking_post_request( - curl, - curl_headers, - g_test_pro_backend_dev_server_url + "/set_payment_refund_requested", - std::string_view(request_json.json.data, request_json.json.size)); - - // Parse response - session_pro_backend_set_payment_refund_requested_response response = - session_pro_backend_set_payment_refund_requested_response_parse( - response_json.data(), response_json.size()); - scope_exit response_free{[&]() { - session_pro_backend_set_payment_refund_requested_response_free(&response); - }}; - - // Verify response - INFO("ERROR: JSON response: " << response_json.c_str()); - for (size_t index = 0; index < response.header.errors_count; index++) { - string8 error = response.header.errors[index]; - fprintf(stderr, "ERROR: %s\n", error.data); - } + // Verify the response + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.ticket == 0); + REQUIRE(response.items_count == 0); + } - // Verify the response - REQUIRE(response.header.errors_count == 0); - REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); - REQUIRE(response.version == 0); - REQUIRE(response.updated); + // Set payment refund requested + { + // Build request + uint64_t now_unix_ts_ms = time(nullptr) * 1000; + session_pro_backend_to_json request_json = + session_pro_backend_set_payment_refund_requested_request_build_to_json( + /*version*/ 0, + master_privkey.data, + sizeof(master_privkey.data), + /*unix_ts_ms*/ now_unix_ts_ms, + /*refund_requested_unix_ts_ms*/ now_unix_ts_ms, + another_payment_tx.provider, + reinterpret_cast(another_payment_tx.payment_id), + another_payment_tx.payment_id_count, + reinterpret_cast(another_payment_tx.order_id), + another_payment_tx.order_id_count); + + session::scope_exit request_json_free{ + [&]() { session_pro_backend_to_json_free(&request_json); }}; + + // Do curl request + std::string response_json = curl_do_basic_blocking_post_request( + curl, + curl_headers, + g_test_pro_backend_dev_server_url + "/set_payment_refund_requested", + std::string_view(request_json.json.data, request_json.json.size)); + + // Parse response + session_pro_backend_set_payment_refund_requested_response response = + session_pro_backend_set_payment_refund_requested_response_parse( + response_json.data(), response_json.size()); + session::scope_exit response_free{[&]() { + session_pro_backend_set_payment_refund_requested_response_free(&response); + }}; + + // Verify response + INFO("ERROR: JSON response: " << response_json.c_str()); + for (size_t index = 0; index < response.header.errors_count; index++) { + string8 error = response.header.errors[index]; + fprintf(stderr, "ERROR: %s\n", error.data); } + + // Verify the response + REQUIRE(response.header.errors_count == 0); + REQUIRE(response.header.status == SESSION_PRO_BACKEND_STATUS_SUCCESS); + REQUIRE(response.version == 0); + REQUIRE(response.updated); } } #endif diff --git a/tests/utils.hpp b/tests/utils.hpp index 4cd8f0c7..2730f3be 100644 --- a/tests/utils.hpp +++ b/tests/utils.hpp @@ -191,11 +191,3 @@ static inline TestKeys get_deterministic_test_keys() { return result; } -struct scope_exit { - explicit scope_exit(std::function func) : cleanup(func) {} - std::function cleanup; - ~scope_exit() { - if (cleanup) - cleanup(); - } -}; From d65dcebbed9251661406960dd9f3a0c0a88abbaa Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 30 Oct 2025 16:21:00 +1100 Subject: [PATCH 11/72] Add C api for libsession core context --- include/session/core.h | 67 +++++++++++++ include/session/core.hpp | 33 +++++-- include/session/database/connection.h | 24 ++--- include/session/database/connection.hpp | 14 +-- include/session/types.h | 8 ++ include/session/util.hpp | 4 + src/core.cpp | 110 +++++++++++++++++---- src/database/connection.cpp | 53 +++++----- src/pro_backend.cpp | 6 +- src/util.cpp | 9 ++ tests/CMakeLists.txt | 5 +- tests/{test_database.cpp => test_core.cpp} | 51 +++++++--- tests/utils.hpp | 1 - 13 files changed, 282 insertions(+), 103 deletions(-) create mode 100644 include/session/core.h rename tests/{test_database.cpp => test_core.cpp} (80%) diff --git a/include/session/core.h b/include/session/core.h new file mode 100644 index 00000000..a70224fc --- /dev/null +++ b/include/session/core.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include + +#include "export.h" +#include "types.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef struct session_core_core session_core_core; +struct session_core_core { + uint64_t opaque[8]; +}; + +/// API: core/session_core_core_init +/// +/// Pass in a zero-initialised session core object to initialise core for usage in the core layer +/// This object should be considered unique, it should not be copied and it must only be initialised +/// and deinitialised once. +/// +/// After initialisation you must call `session_database_connection_open` if you wish to use +/// functions requiring the database +LIBSESSION_EXPORT void session_core_core_init(session_core_core* core); + +/// API: core/session_core_core_deinit +/// +/// Shutdown a initialised core object. This function does a no-op if a NULL pointer is passed in +LIBSESSION_EXPORT void session_core_core_deinit(session_core_core* core); + +/// API: core/session_core_core_db_conn +/// +/// Get a DB handle from the core object. This DB connection may be uninitialised if +/// `session_database_connection_open` has not been called on the returned database connection +/// before for this instance. +LIBSESSION_EXPORT session_database_connection session_core_core_db_conn(session_core_core* core) + NON_NULL_ARG(1); + +/// API: core/session_core_core_pro_proof_is_revoked +/// +/// Update the list of pro-revocations being managed by the core. This updates the in-memory +/// list as well as the copy stored in the database. If the `revocations_ticket` matches the +/// in-memory ticket, this is a no-op. +LIBSESSION_EXPORT bool session_core_core_pro_proof_is_revoked( + session_core_core* core, const bytes32* gen_index_hash, uint64_t unix_ts_ms) + NON_NULL_ARG(1, 2); + +/// API: core/session_core_core_pro_update_revocations +/// +/// Update the list of pro-revocations being managed by the core. This updates the in-memory list as +/// well as the copy stored in the database. If the `revocations_ticket` matches the in-memory +/// ticket, this is a no-op. +/// +/// If the handle returned by `session_core_core_db_conn` does not have an open connection then only +/// the in-memory revocation list is updated. +LIBSESSION_EXPORT session_c_result session_core_core_pro_update_revocations( + session_core_core* core, + uint32_t revocations_ticket, + session_pro_backend_pro_revocation_item* revocations, + size_t revocations_count) NON_NULL_ARG(1); + +#if defined(__cplusplus) +} +#endif diff --git a/include/session/core.hpp b/include/session/core.hpp index eb4fa6c0..72981cde 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -4,15 +4,13 @@ #if !defined(DISABLE_SQLCIPHER_DATABASE) #include #endif +#include #include #include -#include -namespace session { -namespace pro_backend { +namespace session { namespace pro_backend { struct ProRevocationItem; -} -}; // namespace session +}}; // namespace session::pro_backend namespace session::core { struct ProRevocationItemComparer { @@ -23,18 +21,37 @@ struct ProRevocationItemComparer { struct Core { std::set revocations_; - int32_t revocations_ticket_; + + uint32_t revocations_ticket_; + + /// After initialisation you must call `db_conn.open()` if you wish to use + /// functions requiring the database which are denoted by DISABLE_SQLCIPHER_DATABASE. #if !defined(DISABLE_SQLCIPHER_DATABASE) session::database::Connection db_conn; #endif + /// API: core/Core::pro_proof_is_revoked + /// + /// Check if the proof identified by its `gen_index_hash` is revoked with respect to the given + /// timestamp from the list of proofs stored in memory. If `gen_index_hash` does not exist this + /// function will always return `false`. + /// + /// Outputs: + /// - `bool` -- True if the proof was revoked, false otherwise. bool pro_proof_is_revoked( const array_uc32& gen_index_hash, std::chrono::sys_time unix_ts) const; + /// API: core/Core::pro_update_revocations + /// + /// Update the list of pro-revocations being managed by the core. This updates the in-memory + /// list as well as the copy stored in the database. If the `revocations_ticket` matches the + /// in-memory ticket, this is a no-op. + /// + /// If `db_conn` does not have an open connection then only the in-memory revocation list is + /// updated. void pro_update_revocations( - int32_t revocations_ticket, + uint32_t revocations_ticket, std::span revocations); }; - } // namespace session::core diff --git a/include/session/database/connection.h b/include/session/database/connection.h index dc29003b..89def5e7 100644 --- a/include/session/database/connection.h +++ b/include/session/database/connection.h @@ -13,28 +13,22 @@ struct session_pro_backend_pro_revocation_item; typedef struct session_database_connection session_database_connection; struct session_database_connection { - char opaque[16]; -}; - -typedef struct session_database_result session_database_result; -struct session_database_result { - bool success; - char error[256]; - size_t error_count; + uint64_t opaque[2]; }; typedef struct session_database_set_result session_database_set_result; struct session_database_set_result { - session_database_result db; + session_c_result db; int sql_return_code; /// SQL's string-ified `sql_return_code` pointing to memory in the data segment. Should not be /// modified and is valid for program lifetime. const char* sql_error; }; -typedef struct session_database_get_pro_revocation_result session_database_get_pro_revocation_result; +typedef struct session_database_get_pro_revocation_result + session_database_get_pro_revocation_result; struct session_database_get_pro_revocation_result { - session_database_result db; + session_c_result db; size_t count; }; @@ -51,7 +45,7 @@ struct session_database_get_pro_revocation_result { /// - `path` -- Path to the DB to open, this can be a URI or path on disk /// - `raw_key` -- Encryption key to use to open the specified DB. If the DB does not exist then /// the database will be created, encrypted with this key. -LIBSESSION_EXPORT session_database_result session_database_connection_open( +LIBSESSION_EXPORT session_c_result session_database_connection_open( session_database_connection* conn, string8 path, span_u8 raw_key) NON_NULL_ARG(1); /// API: session_database_connection_close @@ -60,8 +54,7 @@ LIBSESSION_EXPORT session_database_result session_database_connection_open( /// /// Inputs: /// - `conn` -- DB connection object to close -LIBSESSION_EXPORT void session_database_connection_close(session_database_connection* conn) - NON_NULL_ARG(1); +LIBSESSION_EXPORT void session_database_connection_close(session_database_connection* conn); /// API: session_database_connection_set_pro_revocations /// @@ -100,7 +93,8 @@ LIBSESSION_EXPORT session_database_set_result session_database_connection_set_pr /// insufficient sized to receive the rows, the return value is always capped to the size of /// the buffer. If `buf` and `buf_count` are nullptr or 0 respectively, then value returned /// is the amount of revocation items in the DB at the time of execution. -LIBSESSION_EXPORT session_database_get_pro_revocation_result session_database_connection_get_pro_revocations_buffer( +LIBSESSION_EXPORT session_database_get_pro_revocation_result +session_database_connection_get_pro_revocations_buffer( session_database_connection* conn, OPTIONAL session_pro_backend_pro_revocation_item* buf, size_t buf_count, diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index d9bb92d5..0e7852fb 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -1,9 +1,9 @@ #pragma once #include -#include #include +#include #include #include #include @@ -36,7 +36,7 @@ struct sqlite3_deleter { struct Connection { std::unique_ptr db_; - /// API: database/open + /// API: database/Connection::open /// /// Open a connection to the DB specified at `path`. If this connection previously has an open /// DB that connection is gracefully closed before opening up the newly requested one. If this @@ -51,7 +51,7 @@ struct Connection { /// the database will be created, encrypted with this key. void open(const std::string& path, const cleared_array<48>& raw_key); - /// API: database/exec + /// API: database/Connection::exec /// /// Prepares a statement and executes it. Throws if SQLite returned an error /// @@ -59,7 +59,7 @@ struct Connection { /// - `sql` -- SQL statement to prepare and execute void exec(const std::string& sql); - /// API: database/query + /// API: database/Connection::query /// /// Prepares a statement, steps the statement, calls the provided `callback` and then finalizes /// the statement once stepping no longer returns rows. Throws if SQLite returned an error @@ -69,7 +69,7 @@ struct Connection { /// - `callback` -- User defined function to execute when a row is returned void query(std::string_view sql, std::function callback); - /// API: database/get_runtime + /// API: database/Connection::get_runtime /// /// Get the runtime row of the table which contains global metadata for the entire table. /// There's only one runtime row per database. @@ -97,7 +97,7 @@ struct Connection { SetResult set_pro_revocations( uint32_t ticket, std::span revocations) noexcept; - /// API: database/get_pro_revocations_buffer + /// API: database/Connection::get_pro_revocations_buffer /// /// Retrieve the Session Pro Backend revocation list given and output the rows into the given /// `buf`. This function throws if SQLite returned an error @@ -124,7 +124,7 @@ struct Connection { size_t offset, OPTIONAL uint32_t* ticket); - /// API: database/get_pro_revocations + /// API: database/Connection::get_pro_revocations /// /// Retrieve the Session Pro Backend revocation list from the database. This function throws if /// there was an allocation or SQLite returned an error diff --git a/include/session/types.h b/include/session/types.h index 500b2dff..59042b98 100644 --- a/include/session/types.h +++ b/include/session/types.h @@ -27,6 +27,14 @@ struct string8 { size_t size; }; +/// Generic function result structure +typedef struct session_result session_result; +struct session_c_result { + bool success; + char error[256]; + size_t error_count; +}; + #define string8_literal(literal) {(char*)literal, sizeof(literal) - 1} typedef struct bytes32 bytes32; diff --git a/include/session/util.hpp b/include/session/util.hpp index 28d82cd9..08b6fdc1 100644 --- a/include/session/util.hpp +++ b/include/session/util.hpp @@ -307,4 +307,8 @@ struct scope_exit { cleanup(); } }; + +// Write the e.what() result into `result.error` +void write_exception_to_session_c_result(struct session_c_result* result, const std::string& what); + } // namespace session diff --git a/src/core.cpp b/src/core.cpp index 2dd3cd40..aee915ad 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -1,7 +1,11 @@ +#include +#include +#include + #include #include -auto logcat = oxen::log::Cat("core"); +static auto logcat = oxen::log::Cat("core"); namespace session::core { bool ProRevocationItemComparer::operator()( @@ -24,7 +28,7 @@ bool Core::pro_proof_is_revoked( } void Core::pro_update_revocations( - int32_t revocations_ticket, + uint32_t revocations_ticket, std::span revocations) { if (revocations_ticket_ == revocations_ticket) return; @@ -35,26 +39,90 @@ void Core::pro_update_revocations( revocations_ticket_ = revocations_ticket; #if !defined(DISABLED_SQLCIPHER_DATABASE) - session::database::SetResult set_result = - db_conn.set_pro_revocations(revocations_ticket, revocations); - - // There's not much we can do here for whatever reason it failed. The runtime cache is updated - // but not the DB. The DB is only for permanence of the list across restarts of libsession at - // which point, it will load from the DB, query the backend and notice the ticket is out of sync - // and try again. - if (!set_result.success) { - oxen::log::warning( - logcat, - "Failed to update SQL revocations from (items {}; ticket {}) -> (items {}; ticket " - "{}): ({}) {}", - revocations_.size(), - revocations_ticket_, - revocations.size(), - revocations_ticket, - set_result.sql_return_code, - set_result.sql_error); + if (db_conn.db_) { + session::database::SetResult set_result = + db_conn.set_pro_revocations(revocations_ticket, revocations); + + // There's not much we can do here for whatever reason it failed. The runtime cache is + // updated but not the DB. The DB is only for permanence of the list across restarts of + // libsession at which point, it will load from the DB, query the backend and notice the + // ticket is out of sync and try again. + if (!set_result.success) { + oxen::log::warning( + logcat, + "Failed to update SQL revocations from (items {}; ticket {}) -> (items {}; " + "ticket " + "{}): ({}) {}", + revocations_.size(), + revocations_ticket_, + revocations.size(), + revocations_ticket, + set_result.sql_return_code, + set_result.sql_error); + } } #endif } - }; // namespace session::core + +using namespace session::core; + +LIBSESSION_C_API void session_core_core_init(session_core_core* core) { + static_assert(sizeof(core->opaque) >= sizeof(Core)); + if (core) { + new (core->opaque) Core(); + } +} + +LIBSESSION_C_API void session_core_core_deinit(session_core_core* core) { + if (core) { + auto* core_cpp = reinterpret_cast(core->opaque); + if (core_cpp) { + core_cpp->~Core(); + memset(core->opaque, 0, sizeof(core->opaque)); + } + } +} + +LIBSESSION_C_API session_database_connection session_core_core_db_conn(session_core_core* core) { + auto* core_cpp = reinterpret_cast(core->opaque); + session_database_connection result = {}; + uintptr_t ptr_address = reinterpret_cast(&core_cpp->db_conn); + memcpy(result.opaque, &ptr_address, sizeof(ptr_address)); + return result; +} + +LIBSESSION_C_API bool session_core_core_pro_proof_is_revoked( + session_core_core* core, const bytes32* gen_index_hash, uint64_t unix_ts_ms) { + bool result = false; + auto* core_cpp = reinterpret_cast(core->opaque); + session::array_uc32 gen_index_hash_cpp = {}; + memcpy(gen_index_hash_cpp.data(), gen_index_hash->data, gen_index_hash_cpp.max_size()); + auto unix_ts = + std::chrono::sys_time(std::chrono::milliseconds(unix_ts_ms)); + result = core_cpp->pro_proof_is_revoked(gen_index_hash_cpp, unix_ts); + return result; +} + +LIBSESSION_C_API session_c_result session_core_core_pro_update_revocations( + session_core_core* core, + uint32_t revocations_ticket, + session_pro_backend_pro_revocation_item* revocations, + size_t revocations_count) { + session_c_result result = {}; + auto* core_cpp = reinterpret_cast(core->opaque); + try { + if (revocations_count && revocations) { + std::vector revocations_cpp; + revocations_cpp.resize(revocations_count); + for (size_t index = 0; index < revocations_count; index++) + revocations_cpp[index] = + session::pro_backend::revocation_cpp_from_c(revocations[index]); + core_cpp->pro_update_revocations(revocations_ticket, revocations_cpp); + } + result.success = true; + } catch (const std::exception& e) { + session::write_exception_to_session_c_result(&result, e.what()); + } + return result; +} diff --git a/src/database/connection.cpp b/src/database/connection.cpp index 64b7b21c..9922e4f8 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -10,7 +10,8 @@ #include "session/database/connection.h" -auto logcat = oxen::log::Cat("database"); +static auto logcat = oxen::log::Cat("database"); + namespace { void throw_sql_error(int sql_result, std::string_view error_prefix) { std::string msg = fmt::format("{}: {}", error_prefix, sqlite3_errstr(sql_result)); @@ -119,6 +120,7 @@ CREATE TABLE IF NOT EXISTS runtime ( } void Connection::exec(const std::string& sql) { + assert(db_); char* error_msg = nullptr; int rc = sqlite3_exec(db_.get(), sql.c_str(), nullptr, nullptr, &error_msg); if (rc != SQLITE_OK) { @@ -129,6 +131,7 @@ void Connection::exec(const std::string& sql) { } void Connection::query(std::string_view sql, std::function callback) { + assert(db_); sqlite3_stmt* stmt = nullptr; int rc = sqlite3_prepare_v2(db_.get(), sql.data(), sql.size(), &stmt, nullptr); if (rc != SQLITE_OK) { @@ -144,6 +147,7 @@ void Connection::query(std::string_view sql, std::function } Runtime Connection::get_runtime() { + assert(db_); Runtime result = {}; std::string_view sql = R"(SELECT id, pro_revocations_ticket FROM runtime LIMIT 1)"; query(sql, [&result](sqlite3_stmt* stmt) { @@ -155,7 +159,7 @@ Runtime Connection::get_runtime() { SetResult Connection::set_pro_revocations( uint32_t ticket, std::span revocations) noexcept { - + assert(db_); // The following consists of exception safe code so we do not need try catch and can trivially // commit or rollback the at the end of the function. exec("BEGIN DEFERRED TRANSACTION;"); @@ -219,6 +223,7 @@ VALUES (?, ?) size_t Connection::get_pro_revocations_buffer( pro_backend::ProRevocationItem* buf, size_t buf_count, size_t offset, uint32_t* ticket) { + assert(db_); // Note this operation is not atomic, the collecting of revocations and the querying of the // ticket happens in 2 separate read steps. This is probably not an issue as I expect the // getting of revocations to only happen on startup where it'll get cached into runtime memory. @@ -274,6 +279,7 @@ OFFSET %zu } std::vector Connection::get_pro_revocations(uint32_t* ticket) { + assert(db_); std::vector result; size_t size_req = get_pro_revocations_buffer(nullptr, 0, 0, ticket); result.resize(size_req); @@ -285,9 +291,9 @@ std::vector Connection::get_pro_revocations(uint using namespace session::database; -LIBSESSION_C_API session_database_result session_database_connection_open( - session_database_connection* conn, string8 path, span_u8 raw_key) { - session_database_result result = {}; +LIBSESSION_C_API session_c_result +session_database_connection_open(session_database_connection* conn, string8 path, span_u8 raw_key) { + session_c_result result = {}; static_assert( sizeof(((session_database_connection*)0)->opaque) >= sizeof(Connection), @@ -315,23 +321,19 @@ LIBSESSION_C_API session_database_result session_database_connection_open( conn_cpp->open(path_cpp, raw_key_cpp); result.success = true; } catch (const std::exception& e) { - const std::string& error = e.what(); - result.error_count = snprintf_clamped( - result.error, - sizeof(result.error), - "%.*s", - static_cast(error.size()), - error.data()); + session::write_exception_to_session_c_result(&result, e.what()); } return result; } LIBSESSION_C_API void session_database_connection_close(session_database_connection* conn) { - auto* conn_cpp = reinterpret_cast(conn->opaque); - if (conn_cpp) { - conn_cpp->~Connection(); - memset(conn->opaque, 0, sizeof(conn->opaque)); + if (conn) { + auto* conn_cpp = reinterpret_cast(conn->opaque); + if (conn_cpp) { + conn_cpp->~Connection(); + memset(conn->opaque, 0, sizeof(conn->opaque)); + } } } @@ -358,19 +360,14 @@ LIBSESSION_C_API session_database_set_result session_database_connection_set_pro result.sql_return_code = result_cpp.sql_return_code; result.sql_error = result_cpp.sql_error; } catch (const std::exception& e) { - const std::string& error = e.what(); - result.db.error_count = snprintf_clamped( - result.db.error, - sizeof(result.db.error), - "%.*s", - static_cast(error.size()), - error.data()); + session::write_exception_to_session_c_result(&result.db, e.what()); } return result; } -LIBSESSION_C_API session_database_get_pro_revocation_result session_database_connection_get_pro_revocations_buffer( +LIBSESSION_C_API session_database_get_pro_revocation_result +session_database_connection_get_pro_revocations_buffer( session_database_connection* conn, OPTIONAL session_pro_backend_pro_revocation_item* buf, size_t buf_count, @@ -394,13 +391,7 @@ LIBSESSION_C_API session_database_get_pro_revocation_result session_database_con result.db.success = true; } catch (std::exception& e) { - const std::string& error = e.what(); - result.db.error_count = snprintf_clamped( - result.db.error, - sizeof(result.db.error), - "%.*s", - static_cast(error.size()), - error.data()); + session::write_exception_to_session_c_result(&result.db, e.what()); } return result; diff --git a/src/pro_backend.cpp b/src/pro_backend.cpp index 8c70d2d4..9167a45a 100644 --- a/src/pro_backend.cpp +++ b/src/pro_backend.cpp @@ -776,8 +776,7 @@ GetProDetailsResponse GetProDetailsResponse::parse(std::string_view json) { return result; } -session_pro_backend_pro_revocation_item revocation_c_from_cpp(ProRevocationItem const &src) -{ +session_pro_backend_pro_revocation_item revocation_c_from_cpp(ProRevocationItem const& src) { session_pro_backend_pro_revocation_item result = {}; std::memcpy(result.gen_index_hash.data, src.gen_index_hash.data(), src.gen_index_hash.size()); result.expiry_unix_ts_ms = std::chrono::duration_cast( @@ -786,8 +785,7 @@ session_pro_backend_pro_revocation_item revocation_c_from_cpp(ProRevocationItem return result; } -ProRevocationItem revocation_cpp_from_c(session_pro_backend_pro_revocation_item const &src) -{ +ProRevocationItem revocation_cpp_from_c(session_pro_backend_pro_revocation_item const& src) { pro_backend::ProRevocationItem result = {}; memcpy(result.gen_index_hash.data(), src.gen_index_hash.data, sizeof(src.gen_index_hash.data)); result.expiry_unix_ts = std::chrono::sys_time( diff --git a/src/util.cpp b/src/util.cpp index a8b24360..8818788e 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -223,6 +224,14 @@ size_t utf16_count(std::span utf16_string) { return simdutf::count_utf16(utf16_string.data(), utf16_string.size()); } +void write_exception_to_session_c_result(struct session_c_result* result, const std::string& what) { + result->error_count = snprintf_clamped( + result->error, + sizeof(result->error), + "%.*s", + static_cast(what.size()), + what.data()); +} } // namespace session LIBSESSION_C_API size_t utf16_count_truncated_to_codepoints( diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 87af4f6c..9c4d9795 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -9,6 +9,7 @@ set(LIB_SESSION_UTESTS_SOURCES test_blinding.cpp test_bt_merge.cpp test_bugs.cpp + test_core.cpp test_compression.cpp test_config_userprofile.cpp test_config_user_groups.cpp @@ -42,10 +43,6 @@ if (ENABLE_ONIONREQ) list(APPEND LIB_SESSION_UTESTS_SOURCES test_onionreq.cpp) endif() -if (ENABLE_DATABASE) - list(APPEND LIB_SESSION_UTESTS_SOURCES test_database.cpp) -endif() - add_library(test_libs INTERFACE) target_link_libraries(test_libs INTERFACE diff --git a/tests/test_database.cpp b/tests/test_core.cpp similarity index 80% rename from tests/test_database.cpp rename to tests/test_core.cpp index 68dacaca..db779541 100644 --- a/tests/test_database.cpp +++ b/tests/test_core.cpp @@ -1,36 +1,62 @@ -#include -#include -#include +#include +#include #include +#include #include -#include "session/database/connection.h" -#include "session/database/connection.hpp" -#include "session/pro_backend.hpp" +#if !defined(DISABLE_SQLCIPHER_DATABASE) +#include +#include + +#include +#endif + +TEST_CASE("Core", "[core][database][open]") { + session_core_core core = {}; + session_core_core_init(&core); + auto on_exit = session::scope_exit([&]() { session_core_core_deinit(&core); }); -TEST_CASE("Database", "[database][open]") { + // Check that the core opaque handle is not zero + session_core_core zero_core = {}; + REQUIRE(memcmp(core.opaque, zero_core.opaque, sizeof(core.opaque)) != 0); + +#if !defined(DISABLE_SQLCIPHER_DATABASE) + // Setup the encryption key session::cleared_array<48> raw_key = {}; randombytes_buf(raw_key.data(), raw_key.size()); span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; + // Get the DB connection from core, and check that the handle is not zero + session_database_connection db = session_core_core_db_conn(&core); session_database_connection zero_db = {}; - session_database_connection db = {}; - session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); REQUIRE(memcmp(db.opaque, zero_db.opaque, sizeof(db.opaque)) != 0); + // Open a DB connection + session_c_result open_result = + session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); + REQUIRE(open_result.success); + + // Close the DB connection session_database_connection_close(&db); REQUIRE(memcmp(db.opaque, zero_db.opaque, sizeof(db.opaque)) == 0); +#endif } -TEST_CASE("Database", "[database][pro][revocations]") { +#if !defined(DISABLE_SQLCIPHER_DATABASE) +TEST_CASE("Core", "[core][database][pro][revocations]") { + session_core_core core = {}; + session_core_core_init(&core); + auto on_exit = session::scope_exit([&]() { session_core_core_deinit(&core); }); + + // Setup the encryption key session::cleared_array<48> raw_key = {}; randombytes_buf(raw_key.data(), raw_key.size()); span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; - session_database_connection db = {}; + // Open the DB + session_database_connection db = session_core_core_db_conn(&core); session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); - auto* db_cpp = reinterpret_cast(db.opaque); // Check runtime was seeded to ticket 0 @@ -136,3 +162,4 @@ TEST_CASE("Database", "[database][pro][revocations]") { REQUIRE(src_items[1].expiry_unix_ts_ms == db_items_after_delete[0].expiry_unix_ts_ms); REQUIRE(ticket == 2); } +#endif // !defined(DISABLE_SQLCIPHER_DATABASE) diff --git a/tests/utils.hpp b/tests/utils.hpp index 2730f3be..ce801905 100644 --- a/tests/utils.hpp +++ b/tests/utils.hpp @@ -190,4 +190,3 @@ static inline TestKeys get_deterministic_test_keys() { // clang-format on return result; } - From edbd92540025ea2ad757fc5d6ae7b01fe0f8901f Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 31 Oct 2025 15:01:17 +1100 Subject: [PATCH 12/72] Document the role of core.hpp and connection.hpp --- include/session/core.hpp | 24 +++++++++++++++++++++--- include/session/database/connection.h | 3 +++ include/session/database/connection.hpp | 5 +++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/include/session/core.hpp b/include/session/core.hpp index 72981cde..a77ab77f 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -8,9 +8,24 @@ #include #include -namespace session { namespace pro_backend { - struct ProRevocationItem; -}}; // namespace session::pro_backend +/// The fundamental library context that an application should instantiate at the start of their +/// libsession integrated application. Its goal is to maintain libsession data structures for +/// communicating on the protocol at runtime but also persist it to disk if/where necessary to +/// maintain state across application restarts. +/// +/// A typical application will instantiate the Core context, open a DB connection at the desired +/// path where libsession will persist state. Periodically the integrating application will invoke +/// the Core context to feed it data that it will managed. In future, the Core context will be +/// runnable in a background thread for it to maintain itself and automatically subscribe to the +/// Session Pro Backend, the swarms of the Session Account it manages to send and receive messages +/// in a way that abstracts that functionality from the implementing application. +/// +/// Currently the integrating application must update the Core context when it receives the +/// appropriate data from the network. + +namespace session::pro_backend { +struct ProRevocationItem; +}; // namespace session::pro_backend namespace session::core { struct ProRevocationItemComparer { @@ -20,8 +35,11 @@ struct ProRevocationItemComparer { }; struct Core { + /// List of Session Pro revocations that the core will reject proofs from std::set revocations_; + /// Version of the revocation list that is currently stored in this core context. It is received + /// from the Session Pro Backend when the revocation list is queried. uint32_t revocations_ticket_; /// After initialisation you must call `db_conn.open()` if you wish to use diff --git a/include/session/database/connection.h b/include/session/database/connection.h index 89def5e7..71c26b02 100644 --- a/include/session/database/connection.h +++ b/include/session/database/connection.h @@ -2,6 +2,7 @@ #include #include +#include #include "../export.h" @@ -40,6 +41,8 @@ struct session_database_get_pro_revocation_result { /// This function returns an if the DB was not openable, if the `raw_key` was the incorrect key to /// decrypt the DB or the contents of the DB were malformed. /// +/// If the DB has never been initialised before, the DB is initialised with the required schema. +/// /// Inputs: /// - `conn` -- DB connection object that was zero-initialised or used previously /// - `path` -- Path to the DB to open, this can be a URI or path on disk diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index 0e7852fb..0ccfe1d1 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -8,6 +8,9 @@ #include #include +/// Functions to interact with a SQLCipher database that maintains Libsession persistent state such +/// as the Session Pro revocation list. + struct sqlite3; struct sqlite3_stmt; @@ -45,6 +48,8 @@ struct Connection { /// This function throws an error if the DB was not openable, if the `raw_key` was the incorrect /// key to decrypt the DB or the contents of the DB were malformed. /// + /// If the DB has never been initialised before, the DB is initialised with the required schema. + /// /// Inputs: /// - `path` -- Path to the DB to open, this can be a URI or path on disk /// - `raw_key` -- Encryption key to use to open the specified DB. If the DB does not exist then From 2a502ce372b5bdd8b0df390f9c72df5ed2e062b6 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 3 Nov 2025 10:35:21 +1100 Subject: [PATCH 13/72] Add connection close to CPP db layer --- include/session/core.hpp | 28 ++++++++++++++++++++++--- include/session/database/connection.hpp | 7 +++++++ src/database/connection.cpp | 6 +++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/include/session/core.hpp b/include/session/core.hpp index a77ab77f..2c416d3d 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -15,13 +15,35 @@ /// /// A typical application will instantiate the Core context, open a DB connection at the desired /// path where libsession will persist state. Periodically the integrating application will invoke -/// the Core context to feed it data that it will managed. In future, the Core context will be +/// the Core context to feed it data that it will manage. In future, the Core context will be /// runnable in a background thread for it to maintain itself and automatically subscribe to the /// Session Pro Backend, the swarms of the Session Account it manages to send and receive messages -/// in a way that abstracts that functionality from the implementing application. +/// in a way that abstracts that book-keeping from the implementing application. /// /// Currently the integrating application must update the Core context when it receives the -/// appropriate data from the network. +/// appropriate data from the network and you can opt out of using a database by either, +/// +/// - Compiling without database support +/// - Not opening the database or ensuring the database is closed if it was open +/// +/// The typical intended flow for using the Core is as follows: +/// +/// ``` +/// #include +/// #include +/// +/// session::cleared_array<48> db_enc_key = {}; +/// randombytes_buf(db_enc_key.data(), db_enc_key.size()); +/// +/// try { +/// session::core::Core core = {}; +/// core.db_conn.open(":memory:", db_enc_key); +/// +/// // Then use the Core API ... +/// if (core.pro_proof_is_revoked(...)) { ... } +/// } catch (const std::exception& e) { +/// } +/// ``` namespace session::pro_backend { struct ProRevocationItem; diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index 0ccfe1d1..4ee21e5c 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -56,6 +56,13 @@ struct Connection { /// the database will be created, encrypted with this key. void open(const std::string& path, const cleared_array<48>& raw_key); + /// API: database/Connection::close + /// + /// Close the database connection if it is open, no-op if the database is not open. This does + /// not need to be called unless you explicitly want to close the connection. On connection + /// destruction, the database closes itself. + void close(); + /// API: database/Connection::exec /// /// Prepares a statement and executes it. Throws if SQLite returned an error diff --git a/src/database/connection.cpp b/src/database/connection.cpp index 9922e4f8..c7342239 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -119,6 +119,10 @@ CREATE TABLE IF NOT EXISTS runtime ( assert(final_version_in_db == TARGET_DB_VERSION); } +void Connection::close() { + db_.reset(); +} + void Connection::exec(const std::string& sql) { assert(db_); char* error_msg = nullptr; @@ -331,7 +335,7 @@ LIBSESSION_C_API void session_database_connection_close(session_database_connect if (conn) { auto* conn_cpp = reinterpret_cast(conn->opaque); if (conn_cpp) { - conn_cpp->~Connection(); + conn_cpp->close(); memset(conn->opaque, 0, sizeof(conn->opaque)); } } From 2a90ebd68637251dddd619a8dedbe0b74647615f Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 4 Dec 2025 16:01:06 +1100 Subject: [PATCH 14/72] Allow storing of private keys into the DB --- include/session/database/connection.h | 31 ++++++++ include/session/database/connection.hpp | 27 +++++++ src/database/connection.cpp | 95 +++++++++++++++++++++++++ tests/test_core.cpp | 44 ++++++++++++ 4 files changed, 197 insertions(+) diff --git a/include/session/database/connection.h b/include/session/database/connection.h index 71c26b02..080f7e06 100644 --- a/include/session/database/connection.h +++ b/include/session/database/connection.h @@ -33,6 +33,12 @@ struct session_database_get_pro_revocation_result { size_t count; }; +struct session_database_get_account { + bool found; + uint32_t db_id; + bytes64 long_term_privkey; +}; + /// API: session_database_connection_open /// /// Open a connection to the DB specified at `path`. If this connection previously has an open @@ -59,6 +65,31 @@ LIBSESSION_EXPORT session_c_result session_database_connection_open( /// - `conn` -- DB connection object to close LIBSESSION_EXPORT void session_database_connection_close(session_database_connection* conn); +/// API: session_database_connection::get_account +/// +/// Get the Session account secrets stored in this database. If no account was initialised yet +/// then the output object's found flag is set to false. +/// +/// Outputs: +/// - `found` -- True if there was an account secret in the DB, false otherwise +/// - `db_id` -- Primary key of the row that the secret was retrieved from. 0 if `found` is +/// false +/// - `long_term_privkey` -- Session account's long term 64 byte libsodium-style private key. +/// This key is all 0s if `found` was false. +LIBSESSION_EXPORT session_database_get_account +session_database_connection_get_account(session_database_connection* conn); + +/// API: session_database_connection_set_account +/// +/// Sets the long-term 64 byte libsodium-style private key as the Session account's secret +/// associated with this database. This overwrites any pre-existing key, if any. +/// +/// This function errors if the key is incorrectly sized or if the DB insertion failed. +LIBSESSION_EXPORT session_c_result session_database_connection_set_account( + session_database_connection* conn, + void const* long_term_privkey, + size_t long_term_privkey_size); + /// API: session_database_connection_set_pro_revocations /// /// Set the list of Session Pro revocations into the database associated with this connection diff --git a/include/session/database/connection.hpp b/include/session/database/connection.hpp index 4ee21e5c..b94b3fcd 100644 --- a/include/session/database/connection.hpp +++ b/include/session/database/connection.hpp @@ -24,6 +24,12 @@ struct SetResult { const char* sql_error; }; +struct GetAccount { + bool found; + uint32_t db_id; + uc64 long_term_privkey; +}; + /// The row from the runtime table which is the table housing global settings of the Session /// database. There's only 1 row in the runtime table which gets extracted and filled out into this /// struct. @@ -92,6 +98,27 @@ struct Connection { /// synced from the Session Pro Backend. Runtime get_runtime(); + /// API: database/Connection::get_account + /// + /// Get the Session account secrets stored in this database. If no account was initialised yet + /// then the output object's found flag is set to false. + /// + /// Outputs: + /// - `found` -- True if there was an account secret in the DB, false otherwise + /// - `db_id` -- Primary key of the row that the secret was retrieved from. 0 if `found` is + /// false + /// - `long_term_privkey` -- Session account's long term 64 byte libsodium-style private key. + /// This key is all 0s if `found` was false. + GetAccount get_account(); + + /// API: database/Connection::set_account + /// + /// Sets the long-term 64 byte libsodium-style private key as the Session account's secret + /// associated with this database. This overwrites any pre-existing key, if any. + /// + /// This function throws if the key is incorrectly sized or if the DB insertion failed. + void set_account(std::span long_term_privkey); + /// API: database/set_pro_revocations /// /// Set the list of Session Pro revocations into the database associated with this connection diff --git a/src/database/connection.cpp b/src/database/connection.cpp index c7342239..1ae58f46 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -1,5 +1,6 @@ #include "session/database/connection.hpp" +#include #include #include @@ -89,6 +90,12 @@ CREATE TABLE IF NOT EXISTS pro_revocations ( expiry_unix_ts_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS accounts ( + id INTEGER PRIMARY KEY NOT NULL, + long_term_privkey BLOB NOT NULL, -- 64 byte libsodium-style secret key + UNIQUE(long_term_privkey) +); + CREATE TABLE IF NOT EXISTS runtime ( id INTEGER PRIMARY KEY NOT NULL, pro_revocations_ticket INTEGER NOT NULL @@ -161,6 +168,57 @@ Runtime Connection::get_runtime() { return result; } +GetAccount Connection::get_account() +{ + GetAccount result = {}; + sqlite3_stmt* stmt = nullptr; + query("SELECT id, long_term_privkey FROM accounts ORDER BY id ASC LIMIT 1", + [&result](sqlite3_stmt* stmt) { + int db_id = sqlite3_column_int(stmt, 0); + const void* privkey = sqlite3_column_blob(stmt, 1); + int privkey_size = sqlite3_column_bytes(stmt, 1); + if (privkey_size != result.long_term_privkey.max_size()) { + throw std::runtime_error( + fmt::format( + "Privkey for account was not {}b was {}b", + result.long_term_privkey.max_size(), + privkey_size)); + } + result.found = true; + result.db_id = db_id; + std::memcpy(result.long_term_privkey.data(), privkey, privkey_size); + }); + return result; +} + +void Connection::set_account(std::span long_term_privkey) +{ + if (long_term_privkey.size() != crypto_sign_ed25519_SECRETKEYBYTES) { + throw std::invalid_argument( + fmt::format( + "Privkey must be {}b, was {}b, unable to set account keys", + crypto_sign_ed25519_SECRETKEYBYTES, + long_term_privkey.size())); + } + + sqlite3_stmt* stmt = nullptr; + std::string_view sql = R"( +INSERT OR REPLACE INTO accounts (id, long_term_privkey) +VALUES (1, ?); +)"; + + int rc = sqlite3_prepare_v2(db_.get(), sql.data(), sql.size() + 1, &stmt, nullptr); + if (rc == SQLITE_OK) + rc = sqlite3_bind_blob(stmt, 1, long_term_privkey.data(), long_term_privkey.size(), nullptr); + if (rc == SQLITE_OK) + rc = sqlite3_step(stmt); + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK) + rc = finalize_rc; + if (rc != SQLITE_DONE) + throw_sql_error(rc, "Failed to update account keys"); +} + SetResult Connection::set_pro_revocations( uint32_t ticket, std::span revocations) noexcept { assert(db_); @@ -341,6 +399,43 @@ LIBSESSION_C_API void session_database_connection_close(session_database_connect } } +LIBSESSION_C_API session_database_get_account +session_database_connection_get_account(session_database_connection* conn) +{ + session_database_get_account result = {}; + if (conn) { + auto* conn_cpp = reinterpret_cast(conn->opaque); + GetAccount cpp = conn_cpp->get_account(); + result.db_id = cpp.db_id; + result.found = cpp.found; + static_assert(sizeof(result.long_term_privkey.data) == cpp.long_term_privkey.max_size()); + std::memcpy( + result.long_term_privkey.data, + cpp.long_term_privkey.data(), + cpp.long_term_privkey.size()); + } + return result; +} + +LIBSESSION_C_API session_c_result session_database_connection_set_account( + session_database_connection* conn, + void const* long_term_privkey, + size_t long_term_privkey_size) { + session_c_result result = {}; + if (conn) { + auto* conn_cpp = reinterpret_cast(conn->opaque); + auto long_term_privkey_cpp = std::span{ + reinterpret_cast(long_term_privkey), long_term_privkey_size}; + try { + conn_cpp->set_account(long_term_privkey_cpp); + result.success = true; + } catch (const std::exception& e) { + session::write_exception_to_session_c_result(&result, e.what()); + } + } + return result; +} + LIBSESSION_C_API session_database_set_result session_database_connection_set_pro_revocations( session_database_connection* conn, uint32_t ticket, diff --git a/tests/test_core.cpp b/tests/test_core.cpp index db779541..e528e816 100644 --- a/tests/test_core.cpp +++ b/tests/test_core.cpp @@ -162,4 +162,48 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { REQUIRE(src_items[1].expiry_unix_ts_ms == db_items_after_delete[0].expiry_unix_ts_ms); REQUIRE(ticket == 2); } + +TEST_CASE("Core", "[core][database][pro][account]") { + session_core_core core = {}; + session_core_core_init(&core); + auto on_exit = session::scope_exit([&]() { session_core_core_deinit(&core); }); + + // Setup the encryption key + session::cleared_array<48> raw_key = {}; + randombytes_buf(raw_key.data(), raw_key.size()); + span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; + + // Open the DB + session_database_connection db = session_core_core_db_conn(&core); + session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); + + // Try get an account before we added one + bytes64 zero_long_term_key = {}; + session_database_get_account get = session_database_connection_get_account(&db); + REQUIRE_FALSE(get.found); + REQUIRE(get.db_id == 0); + REQUIRE(memcmp(get.long_term_privkey.data, + zero_long_term_key.data, + sizeof(zero_long_term_key.data)) == 0); + + // Set a 1 byte zero key for the account and check that it does not accept it + session_c_result c_result = + session_database_connection_set_account(&db, zero_long_term_key.data, 1); + REQUIRE(!c_result.success); + REQUIRE(c_result.error_count > 0); + + // Generate a key and set it + bytes64 long_term_key = {}; + randombytes_buf(long_term_key.data, sizeof(long_term_key.data)); + c_result = session_database_connection_set_account(&db, long_term_key.data, sizeof(long_term_key.data)); + INFO(c_result.error); + REQUIRE(c_result.success); + REQUIRE(c_result.error_count == 0); + + // Try retrieving the account again + session_database_get_account get_again = session_database_connection_get_account(&db); + REQUIRE(get_again.found); + REQUIRE(get_again.db_id == 1); + REQUIRE(memcmp(get_again.long_term_privkey.data, long_term_key.data, sizeof(long_term_key.data)) == 0); +} #endif // !defined(DISABLE_SQLCIPHER_DATABASE) From 9fb8926067058b461d7ba28cd109453973ce850d Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 11 Dec 2025 11:28:32 +1100 Subject: [PATCH 15/72] Separate pro backend dev server tests from the main test block --- README.md | 4 ++-- include/session/pro_backend.hpp | 4 ++-- src/database/connection.cpp | 32 ++++++++++++++------------------ tests/test_core.cpp | 7 +++++-- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0c363c57..08847397 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ apt install cmake build-essential git tcl libssl-dev m4 pkg-config # `find_package(CURL)` succeeds (e.g. a system installed libcurl) for this to compile # successfully. # -# By default, it contacts http://127.0.0.1:5000 but this URL can be changed using the CLI arg -# --pro-backend-dev-server-url="" when invoking the test suite. +# These tests do not run by default, they can be invoked by passing the dev server URL in the CLI +# arg --pro-backend-dev-server-url="" when invoking the test suite. # cmake -G Ninja -S . -B Build diff --git a/include/session/pro_backend.hpp b/include/session/pro_backend.hpp index 3cd4c6eb..07aeb7b8 100644 --- a/include/session/pro_backend.hpp +++ b/include/session/pro_backend.hpp @@ -668,7 +668,7 @@ struct SetPaymentRefundRequestedResponse : public ResponseHeader { /// `errors` static SetPaymentRefundRequestedResponse parse(std::string_view json); }; -session_pro_backend_pro_revocation_item revocation_c_from_cpp(ProRevocationItem const &src); +session_pro_backend_pro_revocation_item revocation_c_from_cpp(ProRevocationItem const& src); -ProRevocationItem revocation_cpp_from_c(session_pro_backend_pro_revocation_item const &src); +ProRevocationItem revocation_cpp_from_c(session_pro_backend_pro_revocation_item const& src); } // namespace session::pro_backend diff --git a/src/database/connection.cpp b/src/database/connection.cpp index 1ae58f46..d31fe6af 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -1,7 +1,7 @@ #include "session/database/connection.hpp" -#include #include +#include #include #include @@ -168,8 +168,7 @@ Runtime Connection::get_runtime() { return result; } -GetAccount Connection::get_account() -{ +GetAccount Connection::get_account() { GetAccount result = {}; sqlite3_stmt* stmt = nullptr; query("SELECT id, long_term_privkey FROM accounts ORDER BY id ASC LIMIT 1", @@ -178,11 +177,10 @@ GetAccount Connection::get_account() const void* privkey = sqlite3_column_blob(stmt, 1); int privkey_size = sqlite3_column_bytes(stmt, 1); if (privkey_size != result.long_term_privkey.max_size()) { - throw std::runtime_error( - fmt::format( - "Privkey for account was not {}b was {}b", - result.long_term_privkey.max_size(), - privkey_size)); + throw std::runtime_error(fmt::format( + "Privkey for account was not {}b was {}b", + result.long_term_privkey.max_size(), + privkey_size)); } result.found = true; result.db_id = db_id; @@ -191,14 +189,12 @@ GetAccount Connection::get_account() return result; } -void Connection::set_account(std::span long_term_privkey) -{ +void Connection::set_account(std::span long_term_privkey) { if (long_term_privkey.size() != crypto_sign_ed25519_SECRETKEYBYTES) { - throw std::invalid_argument( - fmt::format( - "Privkey must be {}b, was {}b, unable to set account keys", - crypto_sign_ed25519_SECRETKEYBYTES, - long_term_privkey.size())); + throw std::invalid_argument(fmt::format( + "Privkey must be {}b, was {}b, unable to set account keys", + crypto_sign_ed25519_SECRETKEYBYTES, + long_term_privkey.size())); } sqlite3_stmt* stmt = nullptr; @@ -209,7 +205,8 @@ VALUES (1, ?); int rc = sqlite3_prepare_v2(db_.get(), sql.data(), sql.size() + 1, &stmt, nullptr); if (rc == SQLITE_OK) - rc = sqlite3_bind_blob(stmt, 1, long_term_privkey.data(), long_term_privkey.size(), nullptr); + rc = sqlite3_bind_blob( + stmt, 1, long_term_privkey.data(), long_term_privkey.size(), nullptr); if (rc == SQLITE_OK) rc = sqlite3_step(stmt); int finalize_rc = sqlite3_finalize(stmt); @@ -400,8 +397,7 @@ LIBSESSION_C_API void session_database_connection_close(session_database_connect } LIBSESSION_C_API session_database_get_account -session_database_connection_get_account(session_database_connection* conn) -{ +session_database_connection_get_account(session_database_connection* conn) { session_database_get_account result = {}; if (conn) { auto* conn_cpp = reinterpret_cast(conn->opaque); diff --git a/tests/test_core.cpp b/tests/test_core.cpp index e528e816..798dbde7 100644 --- a/tests/test_core.cpp +++ b/tests/test_core.cpp @@ -195,7 +195,8 @@ TEST_CASE("Core", "[core][database][pro][account]") { // Generate a key and set it bytes64 long_term_key = {}; randombytes_buf(long_term_key.data, sizeof(long_term_key.data)); - c_result = session_database_connection_set_account(&db, long_term_key.data, sizeof(long_term_key.data)); + c_result = session_database_connection_set_account( + &db, long_term_key.data, sizeof(long_term_key.data)); INFO(c_result.error); REQUIRE(c_result.success); REQUIRE(c_result.error_count == 0); @@ -204,6 +205,8 @@ TEST_CASE("Core", "[core][database][pro][account]") { session_database_get_account get_again = session_database_connection_get_account(&db); REQUIRE(get_again.found); REQUIRE(get_again.db_id == 1); - REQUIRE(memcmp(get_again.long_term_privkey.data, long_term_key.data, sizeof(long_term_key.data)) == 0); + REQUIRE(memcmp(get_again.long_term_privkey.data, + long_term_key.data, + sizeof(long_term_key.data)) == 0); } #endif // !defined(DISABLE_SQLCIPHER_DATABASE) From 277d4d727a5ff138bfd926945fecc864cdc2858f Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 5 Jan 2026 11:03:23 +1100 Subject: [PATCH 16/72] Fix pro backend tests not being skipped if server URL is not specified --- tests/test_pro_backend.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_pro_backend.cpp b/tests/test_pro_backend.cpp index 83d1f64e..cf7a6275 100644 --- a/tests/test_pro_backend.cpp +++ b/tests/test_pro_backend.cpp @@ -894,6 +894,10 @@ std::string curl_do_basic_blocking_post_request( } TEST_CASE("Pro Backend Dev Server", "[pro_backend][dev_server]") { + if (g_test_pro_backend_dev_server_url.empty()) { + return; + } + // Setup: Generate keys and payment token hash bytes32 master_pubkey = {}; bytes64 master_privkey = {}; From 0404f958b6397ab862bc2df05303549eb50b3e58 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 5 Jan 2026 16:53:50 +1100 Subject: [PATCH 17/72] Add open DB to core API which restores state from DB --- include/session/core.h | 23 +++- include/session/core.hpp | 68 ++++++++---- include/session/database/connection.h | 2 +- src/core.cpp | 144 ++++++++++++++++++++------ src/database/connection.cpp | 14 +-- tests/test_core.cpp | 96 ++++++++--------- 6 files changed, 226 insertions(+), 121 deletions(-) diff --git a/include/session/core.h b/include/session/core.h index a70224fc..c4c545a9 100644 --- a/include/session/core.h +++ b/include/session/core.h @@ -33,12 +33,27 @@ LIBSESSION_EXPORT void session_core_core_deinit(session_core_core* core); /// API: core/session_core_core_db_conn /// -/// Get a DB handle from the core object. This DB connection may be uninitialised if -/// `session_database_connection_open` has not been called on the returned database connection -/// before for this instance. -LIBSESSION_EXPORT session_database_connection session_core_core_db_conn(session_core_core* core) +/// Get a DB handle from the core object if the DB has been opened before. If the DB has not been +/// opened, this function returns a nullptr. If libsession is built without DB support this will +/// also cause the function to return a nullptr. +/// +/// This pointer's lifetime is bound to the current instance of the DB associated with the Core. The +/// caller must take care not to deinitialise the connection independently from the Core as +/// ownership of the database is bound to `session_core_core_deinit`. +LIBSESSION_EXPORT session_database_connection *session_core_core_db_conn(session_core_core* core) NON_NULL_ARG(1); +/// API: core/session_core_core_open_db +/// +/// Create/open the SQLCipher DB at the specified `path`. Upon load the core in-memory runtime +/// state will be reset and populated with the contents of the DB. This closes the DB automatically +/// if the core previously opened it. +// +/// This function returns an error if there were issues opening the database. No-op if libsession +/// has been compiled without database support. +LIBSESSION_EXPORT session_c_result +session_core_core_open_db(session_core_core* core, string8 path, span_u8 raw_key); + /// API: core/session_core_core_pro_proof_is_revoked /// /// Update the list of pro-revocations being managed by the core. This updates the in-memory diff --git a/include/session/core.hpp b/include/session/core.hpp index 2c416d3d..e7459558 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -24,26 +24,38 @@ /// appropriate data from the network and you can opt out of using a database by either, /// /// - Compiling without database support -/// - Not opening the database or ensuring the database is closed if it was open +/// - Not opening the database and/or ensuring the database is on the Core object is is closed +/// during use. /// /// The typical intended flow for using the Core is as follows: -/// -/// ``` -/// #include -/// #include -/// -/// session::cleared_array<48> db_enc_key = {}; -/// randombytes_buf(db_enc_key.data(), db_enc_key.size()); -/// -/// try { -/// session::core::Core core = {}; -/// core.db_conn.open(":memory:", db_enc_key); -/// -/// // Then use the Core API ... -/// if (core.pro_proof_is_revoked(...)) { ... } -/// } catch (const std::exception& e) { -/// } -/// ``` +/* +``` + #include + #include + + session::cleared_array<48> db_enc_key = {}; + randombytes_buf(db_enc_key.data(), db_enc_key.size()); + + session::core::Core core = {}; + + // Optionally create/open the DB to persist state to. If this step is skipped the core will only + // maintain libsession state (like the user's long term seed or the pro revocation list) in + // runtime memory and will be lost on shutdown. Persisting user state is then left to the + // integrating application's discretion. + try { + core.db_conn.open(":memory:", db_enc_key); + } catch (const std::exception& e) { + // ... error handling + } + + // Then use the Core API ... + if (core.pro_proof_is_revoked(...)) { ... } + + // Update the revocation list stored in Core (if the DB was opened successfully, this will also + // persist the revocation list to the DB for example). + if (core.pro_update_revocations(...)) { ... } +``` +*/ namespace session::pro_backend { struct ProRevocationItem; @@ -64,12 +76,20 @@ struct Core { /// from the Session Pro Backend when the revocation list is queried. uint32_t revocations_ticket_; - /// After initialisation you must call `db_conn.open()` if you wish to use - /// functions requiring the database which are denoted by DISABLE_SQLCIPHER_DATABASE. #if !defined(DISABLE_SQLCIPHER_DATABASE) session::database::Connection db_conn; #endif + /// API: core/Core::open_db + /// + /// Create/open the SQLCipher DB at the specified `path`. Upon load the core in-memory runtime + /// state will be reset and populated with the contents of the DB. This closes the DB + /// automatically if the core previously opened it. + // + /// This function throws if there was an error opening the database. No-op if libsession has + /// been compiled without database support. + void open_db(const std::string& path, const cleared_array<48>& raw_key); + /// API: core/Core::pro_proof_is_revoked /// /// Check if the proof identified by its `gen_index_hash` is revoked with respect to the given @@ -90,6 +110,14 @@ struct Core { /// /// If `db_conn` does not have an open connection then only the in-memory revocation list is /// updated. + /// + /// Inputs: + /// - `revocations_ticket` -- Ticket that describes the version of the revocations. This value + /// comes alongside the revocation list when queried. If the ticket is the same as the ticket + /// stored in the core, this function no-ops (because the revocation list is the same in this + /// case). + /// - `revocations` -- List of Session Pro revocations to update in the core. Overwrites the + /// previous list stored in the core. void pro_update_revocations( uint32_t revocations_ticket, std::span revocations); diff --git a/include/session/database/connection.h b/include/session/database/connection.h index 080f7e06..4a458545 100644 --- a/include/session/database/connection.h +++ b/include/session/database/connection.h @@ -14,7 +14,7 @@ struct session_pro_backend_pro_revocation_item; typedef struct session_database_connection session_database_connection; struct session_database_connection { - uint64_t opaque[2]; + uint64_t opaque; }; typedef struct session_database_set_result session_database_set_result; diff --git a/src/core.cpp b/src/core.cpp index aee915ad..f69308a4 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -7,6 +7,53 @@ static auto logcat = oxen::log::Cat("core"); +namespace { +enum class SaveToDB { No, Yes }; +void pro_update_revocations_internal( + uint32_t& core_revocations_ticket, + std::set& + core_revocations, +#if !defined(DISABLED_SQLCIPHER_DATABASE) + session::database::Connection& core_db_conn, +#endif + uint32_t revocations_ticket, + std::span revocations, + [[maybe_unused]] SaveToDB save_to_db) { + if (core_revocations_ticket == revocations_ticket) + return; + +#if !defined(DISABLED_SQLCIPHER_DATABASE) + if (core_db_conn.db_ && save_to_db == SaveToDB::Yes) { + session::database::SetResult set_result = + core_db_conn.set_pro_revocations(revocations_ticket, revocations); + + // There's not much we can do here for whatever reason it failed. The runtime cache is + // updated but not the DB. The DB is only for permanence of the list across restarts of + // libsession at which point, it will load from the DB, query the backend and notice the + // ticket is out of sync and try again. + if (!set_result.success) { + oxen::log::warning( + logcat, + "Failed to update SQL revocations from (items {}; ticket {}) -> (items {}; " + "ticket " + "{}): ({}) {}", + core_revocations.size(), + core_revocations_ticket, + revocations.size(), + revocations_ticket, + set_result.sql_return_code, + set_result.sql_error); + } + } +#endif + + // Currently we just dump the entire thing and re-write it, we don't expect this list to get big + core_revocations.clear(); + core_revocations.insert(revocations.begin(), revocations.end()); + core_revocations_ticket = revocations_ticket; +} +}; + namespace session::core { bool ProRevocationItemComparer::operator()( const pro_backend::ProRevocationItem& lhs, @@ -15,6 +62,30 @@ bool ProRevocationItemComparer::operator()( return result; } +void Core::open_db( + [[maybe_unused]] const std::string& path, + [[maybe_unused]] const cleared_array<48>& raw_key) { +#if !defined(DISABLE_SQLCIPHER_DATABASE) + // NOTE: Zero initialise everything + *this = {}; + + // NOTE: Open the DB + db_conn.open(path, raw_key); + + // NOTE: Load in the pro-revocations from the DB + uint32_t pro_revocations_ticket = 0; + std::vector pro_revocations = + db_conn.get_pro_revocations(&pro_revocations_ticket); + pro_update_revocations_internal( + revocations_ticket_, + revocations_, + db_conn, + pro_revocations_ticket, + pro_revocations, + SaveToDB::No); +#endif +} + bool Core::pro_proof_is_revoked( const array_uc32& gen_index_hash, std::chrono::sys_time unix_ts) const { @@ -30,38 +101,15 @@ bool Core::pro_proof_is_revoked( void Core::pro_update_revocations( uint32_t revocations_ticket, std::span revocations) { - if (revocations_ticket_ == revocations_ticket) - return; - - // Currently we just dump the entire thing and re-write it, we don't expect this list to get big - revocations_.clear(); - revocations_.insert(revocations.begin(), revocations.end()); - revocations_ticket_ = revocations_ticket; - + pro_update_revocations_internal( + revocations_ticket_, + revocations_, #if !defined(DISABLED_SQLCIPHER_DATABASE) - if (db_conn.db_) { - session::database::SetResult set_result = - db_conn.set_pro_revocations(revocations_ticket, revocations); - - // There's not much we can do here for whatever reason it failed. The runtime cache is - // updated but not the DB. The DB is only for permanence of the list across restarts of - // libsession at which point, it will load from the DB, query the backend and notice the - // ticket is out of sync and try again. - if (!set_result.success) { - oxen::log::warning( - logcat, - "Failed to update SQL revocations from (items {}; ticket {}) -> (items {}; " - "ticket " - "{}): ({}) {}", - revocations_.size(), - revocations_ticket_, - revocations.size(), - revocations_ticket, - set_result.sql_return_code, - set_result.sql_error); - } - } + db_conn, #endif + revocations_ticket, + revocations, + SaveToDB::Yes); } }; // namespace session::core @@ -84,11 +132,39 @@ LIBSESSION_C_API void session_core_core_deinit(session_core_core* core) { } } -LIBSESSION_C_API session_database_connection session_core_core_db_conn(session_core_core* core) { +LIBSESSION_C_API session_database_connection *session_core_core_db_conn(session_core_core* core) { + session_database_connection *result = nullptr; auto* core_cpp = reinterpret_cast(core->opaque); - session_database_connection result = {}; - uintptr_t ptr_address = reinterpret_cast(&core_cpp->db_conn); - memcpy(result.opaque, &ptr_address, sizeof(ptr_address)); +#if !defined(DISABLED_SQLCIPHER_DATABASE) + if (core_cpp->db_conn.db_.get()) + result = reinterpret_cast(&core_cpp->db_conn); +#endif + return result; +} + +LIBSESSION_C_API session_c_result +session_core_core_open_db(session_core_core* core, string8 path, span_u8 raw_key) { + auto* core_cpp = reinterpret_cast(core->opaque); + session::cleared_array<48> raw_key_cpp; + + session_c_result result = {}; + if (raw_key.size != raw_key_cpp.max_size()) { + result.error_count = snprintf_clamped( + result.error, + sizeof(result.error), + "Raw key must be %zu bytes, unable to open DB. Received: %zu", + raw_key.size, + raw_key_cpp.max_size()); + return result; + } + + try { + std::string path_cpp = std::string(path.data, path.size); + core_cpp->open_db(path_cpp, raw_key_cpp); + result.success = true; + } catch (const std::exception& e) { + session::write_exception_to_session_c_result(&result, e.what()); + } return result; } diff --git a/src/database/connection.cpp b/src/database/connection.cpp index d31fe6af..66dd9be3 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -360,7 +360,7 @@ session_database_connection_open(session_database_connection* conn, string8 path "the capacity must be large enough to hold the `Connection` instance"); session_database_connection_close(conn); - Connection* conn_cpp = new (conn->opaque) Connection(); + Connection* conn_cpp = new (&conn->opaque) Connection(); session::cleared_array<48> raw_key_cpp; if (raw_key.size != raw_key_cpp.max_size()) { @@ -388,10 +388,10 @@ session_database_connection_open(session_database_connection* conn, string8 path LIBSESSION_C_API void session_database_connection_close(session_database_connection* conn) { if (conn) { - auto* conn_cpp = reinterpret_cast(conn->opaque); + auto* conn_cpp = reinterpret_cast(&conn->opaque); if (conn_cpp) { conn_cpp->close(); - memset(conn->opaque, 0, sizeof(conn->opaque)); + memset(&conn->opaque, 0, sizeof(conn->opaque)); } } } @@ -400,7 +400,7 @@ LIBSESSION_C_API session_database_get_account session_database_connection_get_account(session_database_connection* conn) { session_database_get_account result = {}; if (conn) { - auto* conn_cpp = reinterpret_cast(conn->opaque); + auto* conn_cpp = reinterpret_cast(&conn->opaque); GetAccount cpp = conn_cpp->get_account(); result.db_id = cpp.db_id; result.found = cpp.found; @@ -419,7 +419,7 @@ LIBSESSION_C_API session_c_result session_database_connection_set_account( size_t long_term_privkey_size) { session_c_result result = {}; if (conn) { - auto* conn_cpp = reinterpret_cast(conn->opaque); + auto* conn_cpp = reinterpret_cast(&conn->opaque); auto long_term_privkey_cpp = std::span{ reinterpret_cast(long_term_privkey), long_term_privkey_size}; try { @@ -438,7 +438,7 @@ LIBSESSION_C_API session_database_set_result session_database_connection_set_pro session_pro_backend_pro_revocation_item* revocations, size_t revocations_len) { session_database_set_result result = {}; - auto* conn_cpp = reinterpret_cast(conn->opaque); + auto* conn_cpp = reinterpret_cast(&conn->opaque); try { // Convert revocations to CPP instance std::vector revocations_cpp; @@ -468,7 +468,7 @@ session_database_connection_get_pro_revocations_buffer( size_t buf_count, size_t offset, OPTIONAL uint32_t* ticket) { - auto* conn_cpp = reinterpret_cast(conn->opaque); + auto* conn_cpp = reinterpret_cast(&conn->opaque); session_database_get_pro_revocation_result result = {}; try { std::vector buf_cpp; diff --git a/tests/test_core.cpp b/tests/test_core.cpp index 798dbde7..39ded9d4 100644 --- a/tests/test_core.cpp +++ b/tests/test_core.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -12,37 +13,6 @@ #include #endif -TEST_CASE("Core", "[core][database][open]") { - session_core_core core = {}; - session_core_core_init(&core); - auto on_exit = session::scope_exit([&]() { session_core_core_deinit(&core); }); - - // Check that the core opaque handle is not zero - session_core_core zero_core = {}; - REQUIRE(memcmp(core.opaque, zero_core.opaque, sizeof(core.opaque)) != 0); - -#if !defined(DISABLE_SQLCIPHER_DATABASE) - // Setup the encryption key - session::cleared_array<48> raw_key = {}; - randombytes_buf(raw_key.data(), raw_key.size()); - span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; - - // Get the DB connection from core, and check that the handle is not zero - session_database_connection db = session_core_core_db_conn(&core); - session_database_connection zero_db = {}; - REQUIRE(memcmp(db.opaque, zero_db.opaque, sizeof(db.opaque)) != 0); - - // Open a DB connection - session_c_result open_result = - session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); - REQUIRE(open_result.success); - - // Close the DB connection - session_database_connection_close(&db); - REQUIRE(memcmp(db.opaque, zero_db.opaque, sizeof(db.opaque)) == 0); -#endif -} - #if !defined(DISABLE_SQLCIPHER_DATABASE) TEST_CASE("Core", "[core][database][pro][revocations]") { session_core_core core = {}; @@ -55,9 +25,12 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; // Open the DB - session_database_connection db = session_core_core_db_conn(&core); - session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); - auto* db_cpp = reinterpret_cast(db.opaque); + string8 db_path = string8_literal("file::memory:?cache=shared"); + session_c_result open_db_result = session_core_core_open_db(&core, db_path, raw_key_span); + REQUIRE(open_db_result.success); + + session_database_connection *db = session_core_core_db_conn(&core); + auto* db_cpp = reinterpret_cast(&db->opaque); // Check runtime was seeded to ticket 0 session::database::Runtime runtime = db_cpp->get_runtime(); @@ -67,7 +40,7 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { // Check that the DB has no revocations in it uint32_t ticket = 0; session_database_get_pro_revocation_result get_result = - session_database_connection_get_pro_revocations_buffer(&db, nullptr, 0, 0, &ticket); + session_database_connection_get_pro_revocations_buffer(db, nullptr, 0, 0, &ticket); REQUIRE(get_result.db.success); REQUIRE(get_result.count == 0); @@ -93,12 +66,11 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { }, }; - // Set the items - session_database_set_result set_result = session_database_connection_set_pro_revocations( - &db, 1, src_items, sizeof(src_items) / sizeof(src_items[0])); - INFO("Set w/ 2 items failed: " << sqlite3_errstr(set_result.sql_return_code)); - REQUIRE(set_result.db.success); - REQUIRE(set_result.sql_return_code == SQLITE_OK); + // Set the items in Core (and consequently, the DB) + session_c_result set_result = session_core_core_pro_update_revocations( + &core, 1 /*ticket*/, src_items, sizeof(src_items) / sizeof(src_items[0])); + INFO("Set w/ 2 items failed: " << set_result.error); + REQUIRE(set_result.success); // Check runtime ticket was changed to 1 runtime = db_cpp->get_runtime(); @@ -107,7 +79,7 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { // Count the number of revocations in the DB (should be 2 as we've inserted them) get_result = - session_database_connection_get_pro_revocations_buffer(&db, nullptr, 0, 0, &ticket); + session_database_connection_get_pro_revocations_buffer(db, nullptr, 0, 0, &ticket); REQUIRE(ticket == runtime.pro_revocations_ticket); REQUIRE(get_result.db.success); REQUIRE(get_result.count == 2); @@ -128,16 +100,15 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { REQUIRE(src_items[1].expiry_unix_ts_ms == db_items[1].expiry_unix_ts.time_since_epoch().count()); - // Delete the first item (src[0]) from the DB + // Delete the first item (src[0]) from the Core (and consequently the DB) session_pro_backend_pro_revocation_item set_item = src_items[1]; - set_result = session_database_connection_set_pro_revocations(&db, 2, &set_item, 1); - INFO("Set w/ 1 item failed: " << sqlite3_errstr(set_result.sql_return_code)); - REQUIRE(set_result.db.success); - REQUIRE(set_result.sql_return_code == SQLITE_OK); + set_result = session_core_core_pro_update_revocations(&core, 2 /*ticket*/, &set_item, 1); + INFO("Set w/ 1 item failed: " << set_result.error); + REQUIRE(set_result.success); // Count the number of revocations in the DB (should be 1 as we've deleted one of them) get_result = - session_database_connection_get_pro_revocations_buffer(&db, nullptr, 0, 0, &ticket); + session_database_connection_get_pro_revocations_buffer(db, nullptr, 0, 0, &ticket); REQUIRE(get_result.db.success); REQUIRE(get_result.count == 1); REQUIRE(ticket == 2); @@ -148,7 +119,7 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { session_database_get_pro_revocation_result db_items_after_delete_result = session_database_connection_get_pro_revocations_buffer( - &db, + db, db_items_after_delete, sizeof(db_items_after_delete) / sizeof(db_items_after_delete[0]), 0, @@ -161,6 +132,18 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { sizeof(db_items_after_delete[0].gen_index_hash.data)) == 0); REQUIRE(src_items[1].expiry_unix_ts_ms == db_items_after_delete[0].expiry_unix_ts_ms); REQUIRE(ticket == 2); + + // Test a 2nd core, opening the DB that the 1st core created + session_core_core core_2nd = {}; + session_core_core_init(&core_2nd); + auto on_exit_2nd = session::scope_exit([&]() { session_core_core_deinit(&core_2nd); }); + session_c_result open_db_2nd = session_core_core_open_db(&core_2nd, db_path, raw_key_span); + REQUIRE(open_db_2nd.success); + + // Verify that the 2nd core loaded into memory the same contents as the 1st core + auto *core_1st_cpp = reinterpret_cast(core.opaque); + auto *core_2nd_cpp = reinterpret_cast(core_2nd.opaque); + REQUIRE(core_1st_cpp->revocations_ticket_ == core_2nd_cpp->revocations_ticket_); } TEST_CASE("Core", "[core][database][pro][account]") { @@ -174,12 +157,15 @@ TEST_CASE("Core", "[core][database][pro][account]") { span_u8 raw_key_span = {raw_key.data(), raw_key.size()}; // Open the DB - session_database_connection db = session_core_core_db_conn(&core); - session_database_connection_open(&db, string8_literal(":memory:"), raw_key_span); + string8 db_path = string8_literal(":memory:"); + session_c_result open_db_result = session_core_core_open_db(&core, db_path, raw_key_span); + REQUIRE(open_db_result.success); + + session_database_connection *db = session_core_core_db_conn(&core); // Try get an account before we added one bytes64 zero_long_term_key = {}; - session_database_get_account get = session_database_connection_get_account(&db); + session_database_get_account get = session_database_connection_get_account(db); REQUIRE_FALSE(get.found); REQUIRE(get.db_id == 0); REQUIRE(memcmp(get.long_term_privkey.data, @@ -188,7 +174,7 @@ TEST_CASE("Core", "[core][database][pro][account]") { // Set a 1 byte zero key for the account and check that it does not accept it session_c_result c_result = - session_database_connection_set_account(&db, zero_long_term_key.data, 1); + session_database_connection_set_account(db, zero_long_term_key.data, 1); REQUIRE(!c_result.success); REQUIRE(c_result.error_count > 0); @@ -196,13 +182,13 @@ TEST_CASE("Core", "[core][database][pro][account]") { bytes64 long_term_key = {}; randombytes_buf(long_term_key.data, sizeof(long_term_key.data)); c_result = session_database_connection_set_account( - &db, long_term_key.data, sizeof(long_term_key.data)); + db, long_term_key.data, sizeof(long_term_key.data)); INFO(c_result.error); REQUIRE(c_result.success); REQUIRE(c_result.error_count == 0); // Try retrieving the account again - session_database_get_account get_again = session_database_connection_get_account(&db); + session_database_get_account get_again = session_database_connection_get_account(db); REQUIRE(get_again.found); REQUIRE(get_again.db_id == 1); REQUIRE(memcmp(get_again.long_term_privkey.data, From 78ac664e56837531c3274f9b453fa6ce4cd94808 Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 8 Jan 2026 11:12:18 +1100 Subject: [PATCH 18/72] Update readme build instructions --- README.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 08847397..c4ac5c48 100644 --- a/README.md +++ b/README.md @@ -4,35 +4,39 @@ ``` # Pre-requisites -apt install cmake build-essential git tcl libssl-dev m4 pkg-config +apt install cmake build-essential git libssl-dev m4 pkg-config ninja-build # Configure the build # # Options -# Enable APIs for creating onion-requests with: +# - Enable APIs for creating onion-requests with (default: ON) # -# -D ENABLE_ONIONERQ +# -D ENABLE_ONIONREQ=ON # -# Enable testing of a Session Pro Backend by defining on the configure line: +# - Enable SQLCipher database support (default: ON) # -# -D TEST_PRO_BACKEND_WITH_DEV_SERVER=1 +# -D ENABLE_DATABASE=ON # -# These tests require the Session Pro Backend running in development mode (SESH_PRO_BACKEND_DEV=1) -# to be running and tests the request and response flow of registering, updating and revoking -# Session Pro from the development backend. You must also have a libcurl available such that -# `find_package(CURL)` succeeds (e.g. a system installed libcurl) for this to compile -# successfully. +# - Enable testing of a Session Pro Backend by defining on the configure line (default: OFF) # -# These tests do not run by default, they can be invoked by passing the dev server URL in the CLI -# arg --pro-backend-dev-server-url="" when invoking the test suite. +# -D TEST_PRO_BACKEND_WITH_DEV_SERVER=ON +# +# These tests require the Session Pro Backend running in development mode +# (SESH_PRO_BACKEND_DEV=1) to be running and tests the request and response flow of registering, +# updating and revoking Session Pro from the development backend. You must also have a libcurl +# available such that `find_package(CURL)` succeeds (e.g. a system installed libcurl) for this +# to compile successfully. +# +# These tests do not run by default, they can be invoked by passing the dev server URL in the +# CLI arg --pro-backend-dev-server-url="" when invoking the test suite. # cmake -G Ninja -S . -B Build -# Regenerate protobuf files -cmake --build Build --target regen-protobuf --parallel --verbose - # Build cmake --build Build --parallel --verbose + +# Regenerate protobuf files +cmake --build Build --target regen-protobuf --parallel --verbose ``` ## Docs From 7003da7e02a5aacb6c47db9ac3e562b6dce1d20e Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 09:45:56 +1100 Subject: [PATCH 19/72] Switch to system_or_submodule sqlcipher 4.12 from vendored 4.6.1 Alongside upgrading from 4.6.1 to 4.12 there are a few new changes to some of the conventions followed by SQLCipher which are summarised as follows: - SQLCipher now generates a libsqlite3.a instead of libsqlcipher.a - The include directory mirrors sqlite3 layout so #include turns into #include - tcl is no longer required as a dependency which was previously used to create the amalgamated SQLite source file (they now have an internal tool jimsh0 to create this if tcl's missing). - Some SQLCipher ./configure options are updated - --with-pic does not exist anymore - --enable-fts5 was updated to --fts5 - --with-tempstore=3 (or =always) was added - -DSQLITE_EXTRA_INIT=sqlcipher_extra_init and -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown are new defines that must be set --- .gitmodules | 3 + CMakeLists.txt | 4 - cmake/StaticBuild.cmake | 288 ------------------------ external/CMakeLists.txt | 4 + external/sqlcipher | 1 + external/sqlcipher-cmake/CMakeLists.txt | 59 +++++ src/CMakeLists.txt | 5 +- src/database/connection.cpp | 2 +- tests/test_core.cpp | 3 +- 9 files changed, 70 insertions(+), 299 deletions(-) delete mode 100644 cmake/StaticBuild.cmake create mode 160000 external/sqlcipher create mode 100644 external/sqlcipher-cmake/CMakeLists.txt diff --git a/.gitmodules b/.gitmodules index 5461d327..dce6425b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,3 +25,6 @@ [submodule "external/simdutf"] path = external/simdutf url = https://github.com/simdutf/simdutf.git +[submodule "external/sqlcipher"] + path = external/sqlcipher + url = https://github.com/sqlcipher/sqlcipher diff --git a/CMakeLists.txt b/CMakeLists.txt index 18f82f70..df3efe57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,10 +139,6 @@ endif() add_subdirectory(src) add_subdirectory(proto) -if (BUILD_STATIC_DEPS) - include(StaticBuild) -endif() - if(STATIC_BUNDLE) include(combine_archives) diff --git a/cmake/StaticBuild.cmake b/cmake/StaticBuild.cmake deleted file mode 100644 index 822a61b9..00000000 --- a/cmake/StaticBuild.cmake +++ /dev/null @@ -1,288 +0,0 @@ -# cmake bits to do a full static build, downloading and building all dependencies. - -# Most of these are CACHE STRINGs so that you can override them using -DWHATEVER during cmake -# invocation to override. - -set(LOCAL_MIRROR "" CACHE STRING "local mirror path/URL for lib downloads") - -set(SQLCIPHER_VERSION v4.6.1 CACHE STRING "sqlcipher version") -set(SQLCIPHER_MIRROR ${LOCAL_MIRROR} https://github.com/sqlcipher/sqlcipher/archive/refs/tags/${SQLCIPHER_VERSION} - CACHE STRING "sqlcipher mirror(s)") -set(SQLCIPHER_SOURCE ${SQLCIPHER_VERSION}.tar.gz) -set(SQLCIPHER_HASH SHA512=023b2fc7248fe38b758ef93dd8436677ff0f5d08b1061e7eab0adb9e38ad92d523e0ab69016ee69bd35c1fd53c10f61e99b01f7a2987a1f1d492e1f7216a0a9c - CACHE STRING "sqlcipher source hash") - - -include(ExternalProject) - -set(DEPS_DESTDIR ${CMAKE_BINARY_DIR}/static-deps) -set(DEPS_SOURCEDIR ${CMAKE_BINARY_DIR}/static-deps-sources) - -file(MAKE_DIRECTORY ${DEPS_DESTDIR}/include) - -add_library(libsession-external-libs INTERFACE IMPORTED GLOBAL) -target_include_directories(libsession-external-libs SYSTEM BEFORE INTERFACE ${DEPS_DESTDIR}/include) - -set(deps_cc "${CMAKE_C_COMPILER}") -set(deps_cxx "${CMAKE_CXX_COMPILER}") - - -function(expand_urls output source_file) - set(expanded) - foreach(mirror ${ARGN}) - list(APPEND expanded "${mirror}/${source_file}") - endforeach() - set(${output} "${expanded}" PARENT_SCOPE) -endfunction() - -function(add_static_target target ext_target libname) - add_library(${target} STATIC IMPORTED GLOBAL) - add_dependencies(${target} ${ext_target}) - target_link_libraries(${target} INTERFACE libsession-external-libs) - set_target_properties(${target} PROPERTIES - IMPORTED_LOCATION ${DEPS_DESTDIR}/lib/${libname} - ) - if(ARGN) - target_link_libraries(${target} INTERFACE ${ARGN}) - endif() - libsession_static_bundle(${target}) -endfunction() - - - -set(cross_host "") -set(cross_rc "") -if(CMAKE_CROSSCOMPILING) - if(APPLE AND NOT ARCH_TRIPLET AND APPLE_TARGET_TRIPLE) - set(ARCH_TRIPLET "${APPLE_TARGET_TRIPLE}") - endif() - set(cross_host "--host=${ARCH_TRIPLET}") - if (ARCH_TRIPLET MATCHES mingw AND CMAKE_RC_COMPILER) - set(cross_rc "WINDRES=${CMAKE_RC_COMPILER}") - endif() -endif() -if(ANDROID) - set(android_toolchain_suffix linux-android) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) - set(cross_host "--host=x86_64-linux-android") - set(android_compiler_prefix x86_64) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix x86_64) - set(android_toolchain_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) - set(cross_host "--host=i686-linux-android") - set(android_compiler_prefix i686) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix i686) - set(android_toolchain_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) - set(cross_host "--host=armv7a-linux-androideabi") - set(android_compiler_prefix armv7a) - set(android_compiler_suffix linux-androideabi${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix arm) - set(android_toolchain_suffix linux-androideabi) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) - set(cross_host "--host=aarch64-linux-android") - set(android_compiler_prefix aarch64) - set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) - set(android_toolchain_prefix aarch64) - set(android_toolchain_suffix linux-android) - else() - message(FATAL_ERROR "unknown android arch: ${CMAKE_ANDROID_ARCH_ABI}") - endif() - set(deps_cc "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_compiler_suffix}-clang") - set(deps_cxx "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_compiler_suffix}-clang++") - set(deps_ld "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_toolchain_suffix}-ld") - set(deps_ranlib "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_toolchain_prefix}-${android_toolchain_suffix}-ranlib") - set(deps_ar "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_toolchain_prefix}-${android_toolchain_suffix}-ar") -endif() - -set(deps_CFLAGS "-O2") -set(deps_CXXFLAGS "-O2") - -if(CMAKE_C_COMPILER_LAUNCHER) - set(deps_cc "${CMAKE_C_COMPILER_LAUNCHER} ${deps_cc}") -endif() -if(CMAKE_CXX_COMPILER_LAUNCHER) - set(deps_cxx "${CMAKE_CXX_COMPILER_LAUNCHER} ${deps_cxx}") -endif() - -if(WITH_LTO) - set(deps_CFLAGS "${deps_CFLAGS} -flto") -endif() - -if(APPLE AND CMAKE_OSX_DEPLOYMENT_TARGET) - if(SDK_NAME) - set(deps_CFLAGS "${deps_CFLAGS} -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - set(deps_CXXFLAGS "${deps_CXXFLAGS} -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - else() - set(deps_CFLAGS "${deps_CFLAGS} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - set(deps_CXXFLAGS "${deps_CXXFLAGS} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - endif() -endif() - -if(_winver) - set(deps_CFLAGS "${deps_CFLAGS} -D_WIN32_WINNT=${_winver}") - set(deps_CXXFLAGS "${deps_CXXFLAGS} -D_WIN32_WINNT=${_winver}") -endif() - - -if("${CMAKE_GENERATOR}" STREQUAL "Unix Makefiles") - set(_make $(MAKE)) -else() - set(_make make) -endif() - - -# Builds a target; takes the target name (e.g. "readline") and builds it in an external project with -# target name suffixed with `_external`. Its upper-case value is used to get the download details -# (from the variables set above). The following options are supported and passed through to -# ExternalProject_Add if specified. If omitted, these defaults are used: -set(build_def_DEPENDS "") -set(build_def_PATCH_COMMAND "") -set(build_def_CONFIGURE_COMMAND ./configure ${cross_host} --disable-shared --prefix=${DEPS_DESTDIR} --with-pic - "CC=${deps_cc}" "CXX=${deps_cxx}" "CFLAGS=${deps_CFLAGS}" "CXXFLAGS=${deps_CXXFLAGS}" ${cross_rc}) -set(build_def_CONFIGURE_EXTRA "") -set(build_def_BUILD_COMMAND ${_make}) -set(build_def_INSTALL_COMMAND ${_make} install) -set(build_def_BUILD_BYPRODUCTS ${DEPS_DESTDIR}/lib/lib___TARGET___.a ${DEPS_DESTDIR}/include/___TARGET___.h) - -function(build_external target) - set(options DEPENDS PATCH_COMMAND CONFIGURE_COMMAND CONFIGURE_EXTRA BUILD_COMMAND INSTALL_COMMAND BUILD_BYPRODUCTS) - cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "${options}") - foreach(o ${options}) - if(NOT DEFINED arg_${o}) - set(arg_${o} ${build_def_${o}}) - endif() - endforeach() - string(REPLACE ___TARGET___ ${target} arg_BUILD_BYPRODUCTS "${arg_BUILD_BYPRODUCTS}") - - string(TOUPPER "${target}" prefix) - expand_urls(urls ${${prefix}_SOURCE} ${${prefix}_MIRROR}) - set(extract_ts) - if(NOT CMAKE_VERSION VERSION_LESS 3.24) - set(extract_ts DOWNLOAD_EXTRACT_TIMESTAMP ON) - endif() - ExternalProject_Add("${target}_external" - DEPENDS ${arg_DEPENDS} - BUILD_IN_SOURCE ON - PREFIX ${DEPS_SOURCEDIR} - URL ${urls} - URL_HASH ${${prefix}_HASH} - DOWNLOAD_NO_PROGRESS ON - PATCH_COMMAND ${arg_PATCH_COMMAND} - CONFIGURE_COMMAND ${arg_CONFIGURE_COMMAND} ${arg_CONFIGURE_EXTRA} - BUILD_COMMAND ${arg_BUILD_COMMAND} - INSTALL_COMMAND ${arg_INSTALL_COMMAND} - BUILD_BYPRODUCTS ${arg_BUILD_BYPRODUCTS} - EXCLUDE_FROM_ALL ON - ${extract_ts} - ) -endfunction() - - -set(apple_cflags_arch) -set(apple_cxxflags_arch) -set(apple_ldflags_arch) -set(gmp_build_host "${cross_host}") -if(APPLE AND CMAKE_CROSSCOMPILING) - if(gmp_build_host MATCHES "^(.*-.*-)ios([0-9.]+)(-.*)?$") - set(gmp_build_host "${CMAKE_MATCH_1}darwin${CMAKE_MATCH_2}${CMAKE_MATCH_3}") - endif() - if(gmp_build_host MATCHES "^(.*-.*-.*)-simulator$") - set(gmp_build_host "${CMAKE_MATCH_1}") - endif() - - set(apple_arch) - if(ARCH_TRIPLET MATCHES "^(arm|aarch)64.*") - set(apple_arch "arm64") - elseif(ARCH_TRIPLET MATCHES "^x86_64.*") - set(apple_arch "x86_64") - else() - message(FATAL_ERROR "Don't know how to specify -arch for GMP for ${ARCH_TRIPLET} (${APPLE_TARGET_TRIPLE})") - endif() - - set(apple_cflags_arch " -arch ${apple_arch}") - set(apple_cxxflags_arch " -arch ${apple_arch}") - if(CMAKE_OSX_DEPLOYMENT_TARGET) - if (SDK_NAME) - set(apple_ldflags_arch " -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - elseif(CMAKE_OSX_DEPLOYMENT_TARGET) - set(apple_ldflags_arch " -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - endif() - endif() - set(apple_ldflags_arch "${apple_ldflags_arch} -arch ${apple_arch}") - - if(CMAKE_OSX_SYSROOT) - foreach(f c cxx ld) - set(apple_${f}flags_arch "${apple_${f}flags_arch} -isysroot ${CMAKE_OSX_SYSROOT}") - endforeach() - endif() -elseif(gmp_build_host STREQUAL "") - set(gmp_build_host "--build=${CMAKE_LIBRARY_ARCHITECTURE}") -endif() - -link_libraries(-static-libstdc++) -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - link_libraries(-static-libgcc) -endif() -if(MINGW) - link_libraries(-Wl,-Bstatic -lpthread) -endif() - -if(ENABLE_DATABASE) - set(sqlcipher_extra_configure) - set(sqlcipher_extra_cflags) - set(sqlcipher_extra_cxxflags) - set(sqlcipher_extra_ldflags) - set(sqlcipher_compile_definitions) - list(APPEND sqlcipher_compile_definitions - SQLITE_HAS_CODEC - SQLITE_TEMP_STORE=3 - SQLITE_ENABLE_FTS5 - ) - - if(APPLE) - # On macOS, use CommonCrypto (comes with the OS) - list(APPEND sqlcipher_extra_configure --with-crypto-lib=commoncrypto) - list(APPEND sqlcipher_extra_cflags ${apple_cflags_arch}) - list(APPEND sqlcipher_extra_cxxflags ${apple_cxxflags_arch}) - list(APPEND sqlcipher_extra_ldflags ${apple_ldflags_arch} -framework Security -framework Foundation -framework CoreFoundation) - list(APPEND sqlcipher_compile_definitions SQLCIPHER_CRYPTO_COMMONCRYPTO) - else() - # On Linux, Windows, etc., use OpenSSL - find_package(OpenSSL REQUIRED) - list(APPEND sqlcipher_compile_definitions SQLCIPHER_CRYPTO_OPENSSL) - endif() - - set(sqlcipher_cflags "${deps_CFLAGS} ${sqlcipher_extra_cflags}") - set(sqlcipher_cxxflags "${deps_CXXFLAGS} ${sqlcipher_extra_cxxflags}") - set(sqlcipher_ldflags "-L${DEPS_DESTDIR}/lib ${sqlcipher_extra_ldflags}") - - # Append compile definitions static build of sqlcipher via CFLAGS - foreach(def ${sqlcipher_compile_definitions}) - set(sqlcipher_cflags "${sqlcipher_cflags} -D${def}") - endforeach() - - # Configure and build SQLCipher - build_external(sqlcipher - CONFIGURE_COMMAND ./configure ${build_host} --disable-shared --prefix=${DEPS_DESTDIR} - --with-pic --enable-fts5 --enable-static --disable-tcl --disable-readline ${sqlcipher_extra_configure} - "CC=${deps_cc}" - "CXX=${deps_cxx}" - "CFLAGS=${sqlcipher_cflags}" - "CXXFLAGS=${sqlcipher_cxxflags}" - "LDFLAGS=${sqlcipher_ldflags}" - ${cross_rc} - ) - - # Setup CMake target for the rest of libsession to use - add_static_target(sqlcipher::sqlcipher sqlcipher_external libsqlcipher.a) - target_compile_definitions(sqlcipher::sqlcipher INTERFACE ${sqlcipher_compile_definitions}) - if(APPLE) - target_link_libraries(sqlcipher::sqlcipher INTERFACE ${sqlcipher_extra_ldflags}) - else() - target_link_libraries(sqlcipher::sqlcipher INTERFACE OpenSSL::SSL) - endif() -endif() diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 25f42b8e..327dd41f 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -205,3 +205,7 @@ function(simdutf_subdir) endfunction() simdutf_subdir() libsession_static_bundle(simdutf) + +if(ENABLE_DATABASE) + libsession_system_or_submodule(SQLCIPHER sqlcipher sqlcipher>=4.12.0 sqlcipher-cmake) +endif() diff --git a/external/sqlcipher b/external/sqlcipher new file mode 160000 index 00000000..ab223bd8 --- /dev/null +++ b/external/sqlcipher @@ -0,0 +1 @@ +Subproject commit ab223bd801ec225d1497a077da08777d21d1266d diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt new file mode 100644 index 00000000..880875ae --- /dev/null +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required(VERSION 3.14) + +project(sqlcipher-cmake LANGUAGES C) +include(ExternalProject) + +set(SQLCIPHER_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/install) +set(SQLCIPHER_LIBDIR ${SQLCIPHER_PREFIX}/lib) +set(SQLCIPHER_INCLUDEDIR ${SQLCIPHER_PREFIX}/include) +file(MAKE_DIRECTORY ${SQLCIPHER_INCLUDEDIR}) + +set(sqlcipher_cflags "${sqlcipher_cflags} -O2") +if(USE_LTO) + string(APPEND sqlcipher_cflags "${sqlcipher_cflags} -flto") +endif() + +set(cross_host "") +if(CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_PROCESSOR) + set(cross_host "--host=${CMAKE_SYSTEM_PROCESSOR}") +endif() + +set(sqlcipher_ldflags "") +set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown") +set(sqlcipher_link_libs "") + +if(APPLE) + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_COMMONCRYPTO") + list(APPEND sqlcipher_link_libs -framework Security -framework Foundation -framework CoreFoundation) + set(sqlcipher_ldflags "${sqlcipher_ldflags} ${sqlcipher_link_libs}") +else() + find_package(OpenSSL REQUIRED) + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") + list(APPEND sqlcipher_link_libs OpenSSL::SSL OpenSSL::Crypto) + + # NOTE: For example libraries are "//lssl.so;//lcrypto.so" the semicolon confuses + # the linker so we remove the delimiter to make it compliant. + string(REPLACE ";" " " OPENSSL_LIBRARIES_STR "${OPENSSL_LIBRARIES}") + set(sqlcipher_ldflags "${sqlcipher_ldflags} ${OPENSSL_LIBRARIES_STR}") +endif() + +ExternalProject_Add(sqlcipher_external + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher + CONFIGURE_COMMAND + ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared + --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always + --prefix=${SQLCIPHER_PREFIX} CC=${CMAKE_C_COMPILER} CFLAGS=${sqlcipher_cflags} + LDFLAGS=${sqlcipher_ldflags} + BUILD_COMMAND make lib + BUILD_BYPRODUCTS + ${SQLCIPHER_LIBDIR}/libsqlite3.a +) + +add_library(sqlcipher::sqlcipher STATIC IMPORTED GLOBAL) +add_dependencies(sqlcipher::sqlcipher sqlcipher_external) +set_target_properties(sqlcipher::sqlcipher PROPERTIES + IMPORTED_LOCATION ${SQLCIPHER_LIBDIR}/libsqlite3.a + INTERFACE_INCLUDE_DIRECTORIES ${SQLCIPHER_INCLUDEDIR} +) +target_compile_definitions(sqlcipher::sqlcipher INTERFACE SQLITE_HAS_CODEC) +target_link_libraries(sqlcipher::sqlcipher INTERFACE ${sqlcipher_link_libs}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5f315241..90ac4d7c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -133,10 +133,7 @@ if(ENABLE_ONIONREQ) endif() if(ENABLE_DATABASE) - add_libsession_util_library(database - database/connection.cpp - ) - + add_libsession_util_library(database database/connection.cpp) target_link_libraries(database PUBLIC util PRIVATE sqlcipher::sqlcipher) else() target_compile_definitions(database INTERFACE DISABLE_SQLCIPHER_DATABASE) diff --git a/src/database/connection.cpp b/src/database/connection.cpp index 66dd9be3..d174f99e 100644 --- a/src/database/connection.cpp +++ b/src/database/connection.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/tests/test_core.cpp b/tests/test_core.cpp index 39ded9d4..b98a1d76 100644 --- a/tests/test_core.cpp +++ b/tests/test_core.cpp @@ -8,8 +8,7 @@ #if !defined(DISABLE_SQLCIPHER_DATABASE) #include -#include - +#include #include #endif From 483259c34cb95fcab3ac6dff51cd2815a2e96e07 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 10:48:18 +1100 Subject: [PATCH 20/72] Add locking to Core APIs and update usage guide --- include/session/core.h | 2 +- include/session/core.hpp | 51 ++++++++++++++++++++++++++-------------- src/core.cpp | 19 +++++++++------ 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/include/session/core.h b/include/session/core.h index c4c545a9..26e0df53 100644 --- a/include/session/core.h +++ b/include/session/core.h @@ -13,7 +13,7 @@ extern "C" { typedef struct session_core_core session_core_core; struct session_core_core { - uint64_t opaque[8]; + uint64_t opaque[16]; }; /// API: core/session_core_core_init diff --git a/include/session/core.hpp b/include/session/core.hpp index e7459558..614d615a 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -7,6 +7,7 @@ #include #include #include +#include /// The fundamental library context that an application should instantiate at the start of their /// libsession integrated application. Its goal is to maintain libsession data structures for @@ -33,27 +34,35 @@ #include #include - session::cleared_array<48> db_enc_key = {}; - randombytes_buf(db_enc_key.data(), db_enc_key.size()); + int main() { + session::core::Core core = {}; - session::core::Core core = {}; + // Optionally create/open the DB to persist state to. If this step is skipped the core will only + // maintain libsession state (like the user's long term seed or the pro revocation list) in + // runtime memory and will be lost on shutdown. Persisting user state is then left to the + // integrating application's discretion. + try { + // Generate the encryption key for the DB (if you had a pre-existing DB this is where you + // would load the key to pass in). + session::cleared_array<48> db_enc_key = {}; + randombytes_buf(db_enc_key.data(), db_enc_key.size()); - // Optionally create/open the DB to persist state to. If this step is skipped the core will only - // maintain libsession state (like the user's long term seed or the pro revocation list) in - // runtime memory and will be lost on shutdown. Persisting user state is then left to the - // integrating application's discretion. - try { - core.db_conn.open(":memory:", db_enc_key); - } catch (const std::exception& e) { - // ... error handling - } + core.open_db(":memory:", db_enc_key); + } catch (const std::exception& e) { + // ... error handling + } - // Then use the Core API ... - if (core.pro_proof_is_revoked(...)) { ... } + // Update the revocation list stored in Core (if the DB was opened successfully, this will also + // persist the revocation list to the DB for example). + // + // In a production application you would sleep on an event loop responsible for dispatching and + // receiving the revocation list queries and call this function to update the revocation list + // that is cached and the DB + if (core.pro_update_revocations(...)) { ... } - // Update the revocation list stored in Core (if the DB was opened successfully, this will also - // persist the revocation list to the DB for example). - if (core.pro_update_revocations(...)) { ... } + // Interfacing code calls this API to check if the specific proof in question is revoked or not + if (core.pro_proof_is_revoked(...)) { ... } + } ``` */ @@ -76,8 +85,14 @@ struct Core { /// from the Session Pro Backend when the revocation list is queried. uint32_t revocations_ticket_; + /// This class is intended to be use on an network event loop alongside the application which + /// calls into functions that lookup the cache. When the event loop updates the data stored in + /// the in-memory cache and database it requires an exclusive lock. When the application queries + /// the in-memory caches and database, concurrent reads are accepted if there are ongoing writes + mutable std::shared_mutex shared_mutex_; + #if !defined(DISABLE_SQLCIPHER_DATABASE) - session::database::Connection db_conn; + session::database::Connection db_conn_; #endif /// API: core/Core::open_db diff --git a/src/core.cpp b/src/core.cpp index f69308a4..c57998db 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -66,20 +66,22 @@ void Core::open_db( [[maybe_unused]] const std::string& path, [[maybe_unused]] const cleared_array<48>& raw_key) { #if !defined(DISABLE_SQLCIPHER_DATABASE) + std::lock_guard lock{shared_mutex_}; // NOTE: Zero initialise everything - *this = {}; + revocations_.clear(); + revocations_ticket_ = 0; // NOTE: Open the DB - db_conn.open(path, raw_key); + db_conn_.open(path, raw_key); // NOTE: Load in the pro-revocations from the DB uint32_t pro_revocations_ticket = 0; std::vector pro_revocations = - db_conn.get_pro_revocations(&pro_revocations_ticket); + db_conn_.get_pro_revocations(&pro_revocations_ticket); pro_update_revocations_internal( revocations_ticket_, revocations_, - db_conn, + db_conn_, pro_revocations_ticket, pro_revocations, SaveToDB::No); @@ -92,6 +94,8 @@ bool Core::pro_proof_is_revoked( bool result = false; pro_backend::ProRevocationItem item = {}; item.gen_index_hash = gen_index_hash; + + std::shared_lock lock{shared_mutex_}; auto it = revocations_.find(item); if (it != revocations_.end()) result = unix_ts >= it->expiry_unix_ts; @@ -101,11 +105,12 @@ bool Core::pro_proof_is_revoked( void Core::pro_update_revocations( uint32_t revocations_ticket, std::span revocations) { + std::lock_guard lock{shared_mutex_}; pro_update_revocations_internal( revocations_ticket_, revocations_, #if !defined(DISABLED_SQLCIPHER_DATABASE) - db_conn, + db_conn_, #endif revocations_ticket, revocations, @@ -136,8 +141,8 @@ LIBSESSION_C_API session_database_connection *session_core_core_db_conn(session_ session_database_connection *result = nullptr; auto* core_cpp = reinterpret_cast(core->opaque); #if !defined(DISABLED_SQLCIPHER_DATABASE) - if (core_cpp->db_conn.db_.get()) - result = reinterpret_cast(&core_cpp->db_conn); + if (core_cpp->db_conn_.db_.get()) + result = reinterpret_cast(&core_cpp->db_conn_); #endif return result; } From 8d0bdb3c73cb1266d0aaead80301250af666c23a Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 10:52:27 +1100 Subject: [PATCH 21/72] Linting --- include/session/core.h | 2 +- include/session/core.hpp | 2 ++ src/core.cpp | 6 +++--- tests/test_core.cpp | 17 ++++++++--------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/session/core.h b/include/session/core.h index 26e0df53..d3c7c691 100644 --- a/include/session/core.h +++ b/include/session/core.h @@ -40,7 +40,7 @@ LIBSESSION_EXPORT void session_core_core_deinit(session_core_core* core); /// This pointer's lifetime is bound to the current instance of the DB associated with the Core. The /// caller must take care not to deinitialise the connection independently from the Core as /// ownership of the database is bound to `session_core_core_deinit`. -LIBSESSION_EXPORT session_database_connection *session_core_core_db_conn(session_core_core* core) +LIBSESSION_EXPORT session_database_connection* session_core_core_db_conn(session_core_core* core) NON_NULL_ARG(1); /// API: core/session_core_core_open_db diff --git a/include/session/core.hpp b/include/session/core.hpp index 614d615a..45281cf4 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -62,6 +62,8 @@ // Interfacing code calls this API to check if the specific proof in question is revoked or not if (core.pro_proof_is_revoked(...)) { ... } + + core.deinit(); } ``` */ diff --git a/src/core.cpp b/src/core.cpp index c57998db..9eb299e1 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -52,7 +52,7 @@ void pro_update_revocations_internal( core_revocations.insert(revocations.begin(), revocations.end()); core_revocations_ticket = revocations_ticket; } -}; +}; // namespace namespace session::core { bool ProRevocationItemComparer::operator()( @@ -137,8 +137,8 @@ LIBSESSION_C_API void session_core_core_deinit(session_core_core* core) { } } -LIBSESSION_C_API session_database_connection *session_core_core_db_conn(session_core_core* core) { - session_database_connection *result = nullptr; +LIBSESSION_C_API session_database_connection* session_core_core_db_conn(session_core_core* core) { + session_database_connection* result = nullptr; auto* core_cpp = reinterpret_cast(core->opaque); #if !defined(DISABLED_SQLCIPHER_DATABASE) if (core_cpp->db_conn_.db_.get()) diff --git a/tests/test_core.cpp b/tests/test_core.cpp index b98a1d76..2cc17e43 100644 --- a/tests/test_core.cpp +++ b/tests/test_core.cpp @@ -1,14 +1,15 @@ #include -#include #include #include +#include #include #include #if !defined(DISABLE_SQLCIPHER_DATABASE) #include #include + #include #endif @@ -28,7 +29,7 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { session_c_result open_db_result = session_core_core_open_db(&core, db_path, raw_key_span); REQUIRE(open_db_result.success); - session_database_connection *db = session_core_core_db_conn(&core); + session_database_connection* db = session_core_core_db_conn(&core); auto* db_cpp = reinterpret_cast(&db->opaque); // Check runtime was seeded to ticket 0 @@ -77,8 +78,7 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { REQUIRE(runtime.pro_revocations_ticket == 1); // Count the number of revocations in the DB (should be 2 as we've inserted them) - get_result = - session_database_connection_get_pro_revocations_buffer(db, nullptr, 0, 0, &ticket); + get_result = session_database_connection_get_pro_revocations_buffer(db, nullptr, 0, 0, &ticket); REQUIRE(ticket == runtime.pro_revocations_ticket); REQUIRE(get_result.db.success); REQUIRE(get_result.count == 2); @@ -106,8 +106,7 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { REQUIRE(set_result.success); // Count the number of revocations in the DB (should be 1 as we've deleted one of them) - get_result = - session_database_connection_get_pro_revocations_buffer(db, nullptr, 0, 0, &ticket); + get_result = session_database_connection_get_pro_revocations_buffer(db, nullptr, 0, 0, &ticket); REQUIRE(get_result.db.success); REQUIRE(get_result.count == 1); REQUIRE(ticket == 2); @@ -140,8 +139,8 @@ TEST_CASE("Core", "[core][database][pro][revocations]") { REQUIRE(open_db_2nd.success); // Verify that the 2nd core loaded into memory the same contents as the 1st core - auto *core_1st_cpp = reinterpret_cast(core.opaque); - auto *core_2nd_cpp = reinterpret_cast(core_2nd.opaque); + auto* core_1st_cpp = reinterpret_cast(core.opaque); + auto* core_2nd_cpp = reinterpret_cast(core_2nd.opaque); REQUIRE(core_1st_cpp->revocations_ticket_ == core_2nd_cpp->revocations_ticket_); } @@ -160,7 +159,7 @@ TEST_CASE("Core", "[core][database][pro][account]") { session_c_result open_db_result = session_core_core_open_db(&core, db_path, raw_key_span); REQUIRE(open_db_result.success); - session_database_connection *db = session_core_core_db_conn(&core); + session_database_connection* db = session_core_core_db_conn(&core); // Try get an account before we added one bytes64 zero_long_term_key = {}; From aa0cd00981001c4b9528a482168728b63fdb38c6 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 11:07:08 +1100 Subject: [PATCH 22/72] Move types.cpp into the util layer where it belongs --- include/session/core.hpp | 2 -- src/CMakeLists.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/session/core.hpp b/include/session/core.hpp index 45281cf4..614d615a 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -62,8 +62,6 @@ // Interfacing code calls this API to check if the specific proof in question is revoked or not if (core.pro_proof_is_revoked(...)) { ... } - - core.deinit(); } ``` */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 90ac4d7c..d8d7a9c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -40,6 +40,7 @@ add_libsession_util_library(util file.cpp logging.cpp util.cpp + types.cpp ) add_libsession_util_library(crypto @@ -56,7 +57,6 @@ add_libsession_util_library(crypto sodium_array.cpp xed25519.cpp pro_backend.cpp - types.cpp ) add_libsession_util_library(config From bd0d95ed55fe13c8c6d8d0ae7853f458b41fd827 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 11:50:56 +1100 Subject: [PATCH 23/72] Bump Core placement new buffer to atleast 184b for debian sid --- include/session/core.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/session/core.h b/include/session/core.h index d3c7c691..b164cc25 100644 --- a/include/session/core.h +++ b/include/session/core.h @@ -13,7 +13,8 @@ extern "C" { typedef struct session_core_core session_core_core; struct session_core_core { - uint64_t opaque[16]; + /// ~184 bytes on debian sid libstdc++ 3.4.33 + uint64_t opaque[24]; }; /// API: core/session_core_core_init From bc5f73c3e898de91d519aca610b2a188be362aa4 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 12:14:17 +1100 Subject: [PATCH 24/72] Copy the frameworks to link to verbatim to avoid fixing CMake crap --- external/sqlcipher-cmake/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 880875ae..f7a4b7a5 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -24,8 +24,8 @@ set(sqlcipher_link_libs "") if(APPLE) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_COMMONCRYPTO") - list(APPEND sqlcipher_link_libs -framework Security -framework Foundation -framework CoreFoundation) - set(sqlcipher_ldflags "${sqlcipher_ldflags} ${sqlcipher_link_libs}") + list(APPEND sqlcipher_link_libs -framework Security -framework Foundation -framework CoreFoundation) + set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() find_package(OpenSSL REQUIRED) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") From 378c5146b203d0a7387df5f86360331fb20b545c Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 12:23:59 +1100 Subject: [PATCH 25/72] SQLCipher v4.12.0 checks for SQLCIPHER_CRYPTO_CC instead of SQLCIPHER_CRYPTO_COMMONCRYPTO --- external/sqlcipher-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index f7a4b7a5..5a21c2ea 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -23,7 +23,7 @@ set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT set(sqlcipher_link_libs "") if(APPLE) - set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_COMMONCRYPTO") + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") list(APPEND sqlcipher_link_libs -framework Security -framework Foundation -framework CoreFoundation) set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() From 453aca5fcdfa64fef8b05fabf41c553a9900815a Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 12:35:49 +1100 Subject: [PATCH 26/72] Bump core opaque placement new buffer to 208b for apple clang --- include/session/core.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/session/core.h b/include/session/core.h index b164cc25..017b4bf1 100644 --- a/include/session/core.h +++ b/include/session/core.h @@ -14,7 +14,8 @@ extern "C" { typedef struct session_core_core session_core_core; struct session_core_core { /// ~184 bytes on debian sid libstdc++ 3.4.33 - uint64_t opaque[24]; + /// ~208 bytes on macos intel clang 16.0.0.16000026 + uint64_t opaque[26]; }; /// API: core/session_core_core_init From c80e18c4f655bd91672282c381c23612e84aebc7 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 15:14:02 +1100 Subject: [PATCH 27/72] Introduce libsession-services high-level APIs that use primitives provided by libsession Initially these changes were stored as a separate database library and it was tempting to put everything into the crypto library but from the architecture aspect this didn't seem right. The crypto library contains all the low-level primitives to talk on the Session Protocol. These new changes include a database and a manager for libsession state which exclusively uses the primitives and tooling provided by libsession that merging it into the low-level cryptography didn't seem right. In essence this is more the service layer that provides, _additional_ services for applications to use to book-keep the libsession state such as - persistent storage, i.e. the database - caching of the runtime state of libsession i.e. the core --- include/session/core.hpp | 4 ++-- src/CMakeLists.txt | 30 ++++++++++++++++++------------ src/core.cpp | 10 +++++----- tests/CMakeLists.txt | 5 +---- tests/test_core.cpp | 14 ++++---------- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/include/session/core.hpp b/include/session/core.hpp index 614d615a..4857aa91 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -1,7 +1,7 @@ #pragma once #include -#if !defined(DISABLE_SQLCIPHER_DATABASE) +#if !defined(DISABLE_SQLCIPHER) #include #endif #include @@ -91,7 +91,7 @@ struct Core { /// the in-memory caches and database, concurrent reads are accepted if there are ongoing writes mutable std::shared_mutex shared_mutex_; -#if !defined(DISABLE_SQLCIPHER_DATABASE) +#if !defined(DISABLE_SQLCIPHER) session::database::Connection db_conn_; #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d8d7a9c9..30532713 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,8 +34,6 @@ macro(add_libsession_util_library name) list(APPEND export_targets ${name}) endmacro() - - add_libsession_util_library(util file.cpp logging.cpp @@ -47,7 +45,6 @@ add_libsession_util_library(crypto attachments.cpp blinding.cpp curve25519.cpp - core.cpp ed25519.cpp hash.cpp multi_encrypt.cpp @@ -80,8 +77,6 @@ add_libsession_util_library(config fields.cpp ) - - target_link_libraries(util PUBLIC common @@ -99,6 +94,24 @@ target_link_libraries(crypto libsession::protos ) +# NOTE: High level services/layers that use the primitives (cryptography, utils...) provided +# by libsession +add_libsession_util_library(services core.cpp) +target_link_libraries( + services + PUBLIC + crypto + PRIVATE + libsodium::sodium-internal +) + +if(ENABLE_DATABASE) + target_sources(services PUBLIC database/connection.cpp) + target_link_libraries(services PUBLIC sqlcipher::sqlcipher) +else() + target_compile_definitions(services PUBLIC DISABLE_SQLCIPHER) +endif() + target_link_libraries(config PUBLIC crypto @@ -132,13 +145,6 @@ if(ENABLE_ONIONREQ) endif() endif() -if(ENABLE_DATABASE) - add_libsession_util_library(database database/connection.cpp) - target_link_libraries(database PUBLIC util PRIVATE sqlcipher::sqlcipher) -else() - target_compile_definitions(database INTERFACE DISABLE_SQLCIPHER_DATABASE) -endif() - if(WARNINGS_AS_ERRORS AND NOT USE_LTO AND CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION MATCHES "^11\\.") # GCC 11 has an overzealous (and false) stringop-overread warning, but only when LTO is off. # Um, yeah. diff --git a/src/core.cpp b/src/core.cpp index 9eb299e1..5a15a576 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -13,7 +13,7 @@ void pro_update_revocations_internal( uint32_t& core_revocations_ticket, std::set& core_revocations, -#if !defined(DISABLED_SQLCIPHER_DATABASE) +#if !defined(DISABLE_SQLCIPHER) session::database::Connection& core_db_conn, #endif uint32_t revocations_ticket, @@ -22,7 +22,7 @@ void pro_update_revocations_internal( if (core_revocations_ticket == revocations_ticket) return; -#if !defined(DISABLED_SQLCIPHER_DATABASE) +#if !defined(DISABLE_SQLCIPHER) if (core_db_conn.db_ && save_to_db == SaveToDB::Yes) { session::database::SetResult set_result = core_db_conn.set_pro_revocations(revocations_ticket, revocations); @@ -65,7 +65,7 @@ bool ProRevocationItemComparer::operator()( void Core::open_db( [[maybe_unused]] const std::string& path, [[maybe_unused]] const cleared_array<48>& raw_key) { -#if !defined(DISABLE_SQLCIPHER_DATABASE) +#if !defined(DISABLE_SQLCIPHER) std::lock_guard lock{shared_mutex_}; // NOTE: Zero initialise everything revocations_.clear(); @@ -109,7 +109,7 @@ void Core::pro_update_revocations( pro_update_revocations_internal( revocations_ticket_, revocations_, -#if !defined(DISABLED_SQLCIPHER_DATABASE) +#if !defined(DISABLE_SQLCIPHER) db_conn_, #endif revocations_ticket, @@ -140,7 +140,7 @@ LIBSESSION_C_API void session_core_core_deinit(session_core_core* core) { LIBSESSION_C_API session_database_connection* session_core_core_db_conn(session_core_core* core) { session_database_connection* result = nullptr; auto* core_cpp = reinterpret_cast(core->opaque); -#if !defined(DISABLED_SQLCIPHER_DATABASE) +#if !defined(DISABLE_SQLCIPHER) if (core_cpp->db_conn_.db_.get()) result = reinterpret_cast(&core_cpp->db_conn_); #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9c4d9795..1f6e7094 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,6 +46,7 @@ endif() add_library(test_libs INTERFACE) target_link_libraries(test_libs INTERFACE + libsession::services libsession::config libsodium::sodium-internal nlohmann_json::nlohmann_json @@ -57,10 +58,6 @@ else() target_compile_definitions(test_libs INTERFACE DISABLE_ONIONREQ) endif() -if (ENABLE_DATABASE) - target_link_libraries(test_libs INTERFACE libsession::database) -endif() - add_executable(testAll main.cpp ${LIB_SESSION_UTESTS_SOURCES}) target_link_libraries(testAll PRIVATE test_libs diff --git a/tests/test_core.cpp b/tests/test_core.cpp index 2cc17e43..4559746f 100644 --- a/tests/test_core.cpp +++ b/tests/test_core.cpp @@ -1,19 +1,13 @@ +#if !defined(DISABLE_SQLCIPHER) #include -#include +#include #include #include +#include #include #include -#if !defined(DISABLE_SQLCIPHER_DATABASE) -#include -#include - -#include -#endif - -#if !defined(DISABLE_SQLCIPHER_DATABASE) TEST_CASE("Core", "[core][database][pro][revocations]") { session_core_core core = {}; session_core_core_init(&core); @@ -193,4 +187,4 @@ TEST_CASE("Core", "[core][database][pro][account]") { long_term_key.data, sizeof(long_term_key.data)) == 0); } -#endif // !defined(DISABLE_SQLCIPHER_DATABASE) +#endif // !defined(DISABLE_SQLCIPHER) From 70aea95659397e94439af65a65cff1887c08d257 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 15:27:58 +1100 Subject: [PATCH 28/72] Explicitly delimit the sqlcipher_link_libs for Apple to try and stop it from doing -lFramework --- external/sqlcipher-cmake/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 5a21c2ea..6ad43000 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -24,8 +24,8 @@ set(sqlcipher_link_libs "") if(APPLE) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") - list(APPEND sqlcipher_link_libs -framework Security -framework Foundation -framework CoreFoundation) - set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") + list(APPEND sqlcipher_link_libs "-framework Security" "-framework Foundation" "-framework CoreFoundation") + set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() find_package(OpenSSL REQUIRED) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") From 6dd40d6e6d6ef648e2c09fcca1f60e5691febac1 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 9 Jan 2026 16:35:51 +1100 Subject: [PATCH 29/72] Fix build when database is disabled --- include/session/core.hpp | 1 + src/core.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/session/core.hpp b/include/session/core.hpp index 4857aa91..4231ff21 100644 --- a/include/session/core.hpp +++ b/include/session/core.hpp @@ -5,6 +5,7 @@ #include #endif #include +#include #include #include #include diff --git a/src/core.cpp b/src/core.cpp index 5a15a576..3c6f64d7 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -1,9 +1,9 @@ #include #include -#include #include #include +#include static auto logcat = oxen::log::Cat("core"); From 0f461913b88f922a9ca44f6045d62fc475065f76 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 10:19:50 +1100 Subject: [PATCH 30/72] Remove unused static/cross-compiling vars in external/CMakeLists.txt --- external/CMakeLists.txt | 60 ----------------------------------------- 1 file changed, 60 deletions(-) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 327dd41f..94445aef 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -62,44 +62,6 @@ macro(libsession_system_or_submodule BIGNAME smallname pkgconf subdir) endif() endmacro() - -set(deps_cc "${CMAKE_C_COMPILER}") -set(cross_host "") -set(cross_rc "") -if(CMAKE_CROSSCOMPILING) - if(APPLE_TARGET_TRIPLE) - set(cross_host "--host=${APPLE_TARGET_TRIPLE}") - elseif(ANDROID) - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) - set(cross_host "--host=x86_64-linux-android") - set(android_compiler_prefix x86_64) - set(android_compiler_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) - set(cross_host "--host=i686-linux-android") - set(android_compiler_prefix i686) - set(android_compiler_suffix linux-android) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) - set(cross_host "--host=armv7a-linux-androideabi") - set(android_compiler_prefix armv7a) - set(android_compiler_suffix linux-androideabi) - elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) - set(cross_host "--host=aarch64-linux-android") - set(android_compiler_prefix aarch64) - set(android_compiler_suffix linux-android) - else() - message(FATAL_ERROR "unknown android arch: ${CMAKE_ANDROID_ARCH_ABI}") - endif() - - string(REPLACE "android-" "" android_platform_num "${ANDROID_PLATFORM}") - set(deps_cc "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_compiler_suffix}${android_platform_num}-clang") - else() - set(cross_host "--host=${ARCH_TRIPLET}") - if (ARCH_TRIPLET MATCHES mingw AND CMAKE_RC_COMPILER) - set(cross_rc "WINDRES=${CMAKE_RC_COMPILER}") - endif() - endif() -endif() - set(LIBQUIC_BUILD_TESTS OFF CACHE BOOL "") if(ENABLE_ONIONREQ) libsession_system_or_submodule(OXENQUIC quic liboxenquic>=1.3.0 oxen-libquic) @@ -126,25 +88,6 @@ if(APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL AppleClang AND NOT CMAKE_CXX_COMPILE oxen_logging_add_source_dir("${CMAKE_CURRENT_SOURCE_DIR}/oxen-libquic/external/oxen-logging/include/oxen") endif() -if(CMAKE_C_COMPILER_LAUNCHER) - set(deps_cc "${CMAKE_C_COMPILER_LAUNCHER} ${deps_cc}") -endif() -set(deps_CFLAGS "-O2") - -if(IPO_ENABLED) - set(deps_CFLAGS "${deps_CFLAGS} -flto") -endif() - -if(APPLE) - foreach(lang C CXX) - string(APPEND deps_${lang}FLAGS " ${CMAKE_${lang}_SYSROOT_FLAG} ${CMAKE_OSX_SYSROOT} ${CMAKE_${lang}_OSX_DEPLOYMENT_TARGET_FLAG}${CMAKE_OSX_DEPLOYMENT_TARGET}") - foreach(arch ${CMAKE_OSX_ARCHITECTURES}) - string(APPEND deps_${lang}FLAGS " -arch ${arch}") - endforeach() - endforeach() -endif() - - function(libsodium_internal_subdir) set(BUILD_SHARED_LIBS OFF) add_subdirectory(libsodium-internal) @@ -152,7 +95,6 @@ endfunction() libsodium_internal_subdir() libsession_static_bundle(libsodium::sodium-internal) - set(protobuf_VERBOSE ON CACHE BOOL "" FORCE) set(protobuf_INSTALL ON CACHE BOOL "" FORCE) set(protobuf_WITH_ZLIB OFF CACHE BOOL "" FORCE) @@ -168,7 +110,6 @@ if(TARGET PkgConfig::PROTOBUF_LITE AND NOT TARGET protobuf::libprotobuf-lite) add_library(protobuf::libprotobuf-lite ALIAS PkgConfig::PROTOBUF_LITE) endif() - set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "") set(ZSTD_BUILD_TESTS OFF CACHE BOOL "") set(ZSTD_BUILD_CONTRIB OFF CACHE BOOL "") @@ -189,7 +130,6 @@ export( ) libsession_static_bundle(libzstd_static) - set(JSON_BuildTests OFF CACHE INTERNAL "") set(JSON_Install ON CACHE INTERNAL "") # Required to export targets that we use libsession_system_or_submodule(NLOHMANN nlohmann_json nlohmann_json>=3.7.0 nlohmann-json) From d8cd25c08a9aeb14af695172929a8484fb7c2eb8 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 10:20:07 +1100 Subject: [PATCH 31/72] Pass cross-compiling host args to SQLCipher build This matches how SQLite is built in oxen-core statically. It checks for the cross-compiling flag and sets the host accordingly for the architecture. --- external/sqlcipher-cmake/CMakeLists.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 6ad43000..532f27d6 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -14,8 +14,18 @@ if(USE_LTO) endif() set(cross_host "") -if(CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_PROCESSOR) - set(cross_host "--host=${CMAKE_SYSTEM_PROCESSOR}") +if(CMAKE_CROSSCOMPILING) + if (APPLE) + if(sane_cross_host MATCHES "^(.*-)ios([0-9.]+)(-.*)?$") + set(cross_host "--host=${CMAKE_MATCH_1}darwin${CMAKE_MATCH_2}${CMAKE_MATCH_3}") + endif() + if(sane_cross_host MATCHES "^(.*)-simulator$") + set(cross_host "--host=${CMAKE_MATCH_1}") + endif() + endif() + if(CMAKE_SYSTEM_PROCESSOR) + set(cross_host "--host=${CMAKE_SYSTEM_PROCESSOR}") + endif() endif() set(sqlcipher_ldflags "") From e1f3c4bb42ae4ba843b09def3202b581f1ec24e0 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 10:55:25 +1100 Subject: [PATCH 32/72] Add static OpenSSL build capabilities to link to SQLCipher --- README.md | 7 + external/CMakeLists.txt | 3 + external/openssl-cmake/CMakeLists.txt | 217 ++++++++++++++++++++++++++ 3 files changed, 227 insertions(+) create mode 100644 external/openssl-cmake/CMakeLists.txt diff --git a/README.md b/README.md index c4ac5c48..c4e09cb2 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,13 @@ apt install cmake build-essential git libssl-dev m4 pkg-config ninja-build # Configure the build # # Options +# - Build libsession with its dependencies statically linked into the library (default: ON) +# +# -D BUILD_STATIC_DEPS=ON +# +# This currently influences top-level dependencies and forces OpenSSL on Linux/Windows to be +# statically linked into SQLCipher for database support. +# # - Enable APIs for creating onion-requests with (default: ON) # # -D ENABLE_ONIONREQ=ON diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 94445aef..b5906fc3 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,3 +1,5 @@ +option(BUILD_STATIC_DEPS "Link top-level libsession dependencies statically into the generated library" ON) + option(SUBMODULE_CHECK "Enables checking that vendored library submodules are up to date" ON) if(SUBMODULE_CHECK) find_package(Git) @@ -147,5 +149,6 @@ simdutf_subdir() libsession_static_bundle(simdutf) if(ENABLE_DATABASE) + libsession_system_or_submodule(OPENSSL openssl openssl>=3.5.4 openssl-cmake) libsession_system_or_submodule(SQLCIPHER sqlcipher sqlcipher>=4.12.0 sqlcipher-cmake) endif() diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt new file mode 100644 index 00000000..24d2d8e7 --- /dev/null +++ b/external/openssl-cmake/CMakeLists.txt @@ -0,0 +1,217 @@ +set(OPENSSL_VERSION 3.5.4 CACHE STRING "openssl version") +set(OPENSSL_MIRROR ${LOCAL_MIRROR} https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION} CACHE STRING "openssl download mirror(s)") +set(OPENSSL_SOURCE openssl-${OPENSSL_VERSION}.tar.gz) +set(OPENSSL_HASH SHA256=967311f84955316969bdb1d8d4b983718ef42338639c621ec4c34fddef355e99 + CACHE STRING "openssl source hash") + +set(DEPS_DESTDIR ${CMAKE_BINARY_DIR}/static-deps) +set(DEPS_SOURCEDIR ${CMAKE_BINARY_DIR}/static-deps-sources) +include_directories(BEFORE SYSTEM ${DEPS_DESTDIR}/include) +file(MAKE_DIRECTORY ${DEPS_DESTDIR}/include) + +set(deps_cc "${CMAKE_C_COMPILER}") +set(deps_cxx "${CMAKE_CXX_COMPILER}") +if (ANDROID) + if(NOT ANDROID_TOOLCHAIN_NAME) + message(FATAL_ERROR "ANDROID_TOOLCHAIN_NAME not set; did you run with the proper android toolchain options?") + endif() + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) + set(android_clang x86_64-linux-android${ANDROID_PLATFORM_LEVEL}-clang) + elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) + set(android_clang i686-linux-android${ANDROID_PLATFORM_LEVEL}-clang) + elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) + set(android_clang armv7a-linux-androideabi${ANDROID_PLATFORM_LEVEL}-clang) + elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) + set(android_clang aarch64-linux-android${ANDROID_PLATFORM_LEVEL}-clang) + else() + message(FATAL_ERROR "Don't know how to build for android arch abi ${CMAKE_ANDROID_ARCH_ABI}") + endif() + set(deps_cc "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_clang}") + set(deps_cxx "${deps_cc}++") +endif() + +if(CMAKE_C_COMPILER_LAUNCHER) + set(deps_cc "${CMAKE_C_COMPILER_LAUNCHER} ${deps_cc}") +endif() +if(CMAKE_CXX_COMPILER_LAUNCHER) + set(deps_cxx "${CMAKE_CXX_COMPILER_LAUNCHER} ${deps_cxx}") +endif() + +function(expand_urls output source_file) + set(expanded) + foreach(mirror ${ARGN}) + list(APPEND expanded "${mirror}/${source_file}") + endforeach() + set(${output} "${expanded}" PARENT_SCOPE) +endfunction() + +function(add_static_target target ext_target libname) + add_library(${target} STATIC IMPORTED GLOBAL) + add_dependencies(${target} ${ext_target}) + set_target_properties(${target} PROPERTIES + IMPORTED_LOCATION ${DEPS_DESTDIR}/lib/${libname} + ) + if(ARGN) + target_link_libraries(${target} INTERFACE ${ARGN}) + endif() +endfunction() + +if(USE_LTO) + set(flto "-flto") +else() + set(flto "") +endif() + +set(cross_host "") +set(cross_extra "") +if (ANDROID) + set(cross_host "--host=${CMAKE_LIBRARY_ARCHITECTURE}") + set(cross_extra "LD=${ANDROID_TOOLCHAIN_ROOT}/bin/${CMAKE_LIBRARY_ARCHITECTURE}-ld" "RANLIB=${CMAKE_RANLIB}" "AR=${CMAKE_AR}") +elseif(CMAKE_CROSSCOMPILING) + if(APPLE) + set(ARCH_TRIPLET "${APPLE_TARGET_TRIPLE}") + endif() + set(cross_host "--host=${ARCH_TRIPLET}") + if (ARCH_TRIPLET MATCHES mingw AND CMAKE_RC_COMPILER) + set(cross_extra "WINDRES=${CMAKE_RC_COMPILER}") + endif() +endif() + +set(apple_cflags_arch) +set(apple_cxxflags_arch) +set(apple_ldflags_arch) +set(sane_cross_host "${cross_host}") +if(APPLE AND CMAKE_CROSSCOMPILING) + if(sane_cross_host MATCHES "^(.*-)ios([0-9.]+)(-.*)?$") + set(sane_cross_host "${CMAKE_MATCH_1}darwin${CMAKE_MATCH_2}${CMAKE_MATCH_3}") + endif() + if(sane_cross_host MATCHES "^(.*)-simulator$") + set(sane_cross_host "${CMAKE_MATCH_1}") + endif() + + set(apple_arch) + if(ARCH_TRIPLET MATCHES "^(arm|aarch)64.*") + set(apple_arch "arm64") + elseif(ARCH_TRIPLET MATCHES "^x86_64.*") + set(apple_arch "x86_64") + else() + message(FATAL_ERROR "Don't know how to specify -arch for GMP for ${ARCH_TRIPLET} (${APPLE_TARGET_TRIPLE})") + endif() + + set(apple_cflags_arch " -arch ${apple_arch}") + set(apple_cxxflags_arch " -arch ${apple_arch}") + if(CMAKE_OSX_DEPLOYMENT_TARGET) + if (SDK_NAME) + set(apple_ldflags_arch " -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + elseif(CMAKE_OSX_DEPLOYMENT_TARGET) + set(apple_ldflags_arch " -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() + endif() + set(apple_ldflags_arch "${apple_ldflags_arch} -arch ${apple_arch}") + + if(CMAKE_OSX_SYSROOT) + foreach(f c cxx ld) + set(apple_${f}flags_arch "${apple_${f}flags_arch} -isysroot ${CMAKE_OSX_SYSROOT}") + endforeach() + endif() +elseif(cross_host STREQUAL "" AND CMAKE_LIBRARY_ARCHITECTURE) + set(build_host "--build=${CMAKE_LIBRARY_ARCHITECTURE}") +endif() + +set(deps_CFLAGS "-O2 ${flto}") +set(deps_CXXFLAGS "-O2 ${flto}") +set(deps_noarch_CFLAGS "${deps_CFLAGS}") +set(deps_noarch_CXXFLAGS "${deps_CXXFLAGS}") + +if(APPLE) + foreach(lang C CXX) + string(APPEND deps_${lang}FLAGS " ${CMAKE_${lang}_SYSROOT_FLAG} ${CMAKE_OSX_SYSROOT} ${CMAKE_${lang}_OSX_DEPLOYMENT_TARGET_FLAG}${CMAKE_OSX_DEPLOYMENT_TARGET}") + + set(deps_noarch_${lang}FLAGS "${deps_${lang}FLAGS}") + + foreach(arch ${CMAKE_OSX_ARCHITECTURES}) + string(APPEND deps_${lang}FLAGS " -arch ${arch}") + endforeach() + endforeach() +endif() + +# Builds a target; takes the target name (e.g. "readline") and builds it in an external project with +# target name suffixed with `_external`. Its upper-case value is used to get the download details +# (from the variables set above). The following options are supported and passed through to +# ExternalProject_Add if specified. If omitted, these defaults are used: +set(build_def_DEPENDS "") +set(build_def_PATCH_COMMAND "") +set(build_def_CONFIGURE_COMMAND ./configure ${sane_cross_host} --disable-shared --prefix=${DEPS_DESTDIR} --with-pic + "CC=${deps_cc}" "CXX=${deps_cxx}" "CFLAGS=${deps_CFLAGS}" "CXXFLAGS=${deps_CXXFLAGS}" ${cross_extra}) +set(build_def_BUILD_COMMAND make) +set(build_def_INSTALL_COMMAND make install) +set(build_def_BUILD_BYPRODUCTS ${DEPS_DESTDIR}/lib/lib___TARGET___.a ${DEPS_DESTDIR}/include/___TARGET___.h) +set(build_dep_TARGET_SUFFIX "") + +function(build_external target) + set(options TARGET_SUFFIX DEPENDS PATCH_COMMAND CONFIGURE_COMMAND BUILD_COMMAND INSTALL_COMMAND BUILD_BYPRODUCTS) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "${options}") + foreach(o ${options}) + if(NOT DEFINED arg_${o}) + set(arg_${o} ${build_def_${o}}) + endif() + endforeach() + string(REPLACE ___TARGET___ ${target} arg_BUILD_BYPRODUCTS "${arg_BUILD_BYPRODUCTS}") + + set(externalproject_extra) + if(NOT CMAKE_VERSION VERSION_LESS 3.24) + # Default in cmake 3.24+ is to not extract timestamps for ExternalProject, which breaks pretty + # much every autotools package (which thinks it must reconfigure) because timestamps got + # updated). + list(APPEND externalproject_extra DOWNLOAD_EXTRACT_TIMESTAMP ON) + endif() + + string(TOUPPER "${target}" prefix) + expand_urls(urls ${${prefix}_SOURCE} ${${prefix}_MIRROR}) + ExternalProject_Add("${target}${arg_TARGET_SUFFIX}_external" + DEPENDS ${arg_DEPENDS} + BUILD_IN_SOURCE ON + PREFIX ${DEPS_SOURCEDIR} + URL ${urls} + URL_HASH ${${prefix}_HASH} + DOWNLOAD_NO_PROGRESS ON + PATCH_COMMAND ${arg_PATCH_COMMAND} + CONFIGURE_COMMAND ${arg_CONFIGURE_COMMAND} + BUILD_COMMAND ${arg_BUILD_COMMAND} + INSTALL_COMMAND ${arg_INSTALL_COMMAND} + BUILD_BYPRODUCTS ${arg_BUILD_BYPRODUCTS} + ${externalproject_extra} + ) +endfunction() + +if(NOT APPLE AND NOT WIN32) + set(openssl_system_env "") + set(openssl_cc "${deps_cc}") + if(CMAKE_CROSSCOMPILING) + if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) + set(openssl_system_env SYSTEM=MINGW64 RC=${CMAKE_RC_COMPILER}) + elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) + set(openssl_system_env SYSTEM=MINGW64 RC=${CMAKE_RC_COMPILER}) + elseif(ANDROID) + set(openssl_system_env SYSTEM=Linux MACHINE=${openssl_machine} ${cross_extra}) + set(openssl_extra_opts no-asm) + endif() + endif() + build_external(openssl + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env CC=${openssl_cc} ${openssl_system_env} ./config + --prefix=${DEPS_DESTDIR} --libdir=lib ${openssl_extra_opts} + no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost + no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 + no-static-engine no-tests no-weak-ssl-ciphers no-zlib no-zlib-dynamic "CFLAGS=${deps_CFLAGS}" + INSTALL_COMMAND make install_sw + BUILD_BYPRODUCTS + ${DEPS_DESTDIR}/lib/libssl.a ${DEPS_DESTDIR}/lib/libcrypto.a + ${DEPS_DESTDIR}/include/openssl/ssl.h ${DEPS_DESTDIR}/include/openssl/crypto.h + ) + add_static_target(OpenSSL::SSL openssl_external libssl.a) + add_static_target(OpenSSL::Crypto openssl_external libcrypto.a) + target_link_libraries(OpenSSL::SSL INTERFACE OpenSSL::Crypto) + set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) + set(OPENSSL_ROOT_DIR ${DEPS_DESTDIR} CACHE PATH "" FORCE) +endif() + From 43632de2d33a600f67a04cab3f52e83351d18f04 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 11:26:40 +1100 Subject: [PATCH 33/72] Require specific OpenSSL version in SQLCipher --- external/openssl-cmake/CMakeLists.txt | 2 ++ external/sqlcipher-cmake/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 24d2d8e7..c4d2065e 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -1,3 +1,5 @@ +set(LOCAL_MIRROR "" CACHE STRING "local mirror path/URL for lib downloads") + set(OPENSSL_VERSION 3.5.4 CACHE STRING "openssl version") set(OPENSSL_MIRROR ${LOCAL_MIRROR} https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION} CACHE STRING "openssl download mirror(s)") set(OPENSSL_SOURCE openssl-${OPENSSL_VERSION}.tar.gz) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 532f27d6..918d71ff 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -37,7 +37,7 @@ if(APPLE) list(APPEND sqlcipher_link_libs "-framework Security" "-framework Foundation" "-framework CoreFoundation") set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() - find_package(OpenSSL REQUIRED) + find_package(OpenSSL 3.5.4 REQUIRED) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") list(APPEND sqlcipher_link_libs OpenSSL::SSL OpenSSL::Crypto) From 3c75fea912eab5192b0bb66c5451d7b016b820c4 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 11:33:50 +1100 Subject: [PATCH 34/72] Do not require OpenSSL immediately for SQLCipher On CI machine where the version does not suffice it will try find the package, notice it's not available and fail immediately instead of allowing the static OpenSSL build to kick in --- external/sqlcipher-cmake/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 918d71ff..3577e775 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -37,7 +37,6 @@ if(APPLE) list(APPEND sqlcipher_link_libs "-framework Security" "-framework Foundation" "-framework CoreFoundation") set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() - find_package(OpenSSL 3.5.4 REQUIRED) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") list(APPEND sqlcipher_link_libs OpenSSL::SSL OpenSSL::Crypto) From 90ed01ef5e77902930aa9cff322c2586fd2601bb Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 11:58:05 +1100 Subject: [PATCH 35/72] Explicitly define that SQLCipher depends on OpenSSL for non-Apple builds --- external/sqlcipher-cmake/CMakeLists.txt | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 3577e775..27896ac3 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -8,6 +8,13 @@ set(SQLCIPHER_LIBDIR ${SQLCIPHER_PREFIX}/lib) set(SQLCIPHER_INCLUDEDIR ${SQLCIPHER_PREFIX}/include) file(MAKE_DIRECTORY ${SQLCIPHER_INCLUDEDIR}) +# NOTE: Detect if openssl is being built locally for libsession, if so this build step depends on +# that first before proceeding +set(sqlcipher_depends) +if (NOT APPLE AND TARGET openssl_external) + set(sqlcipher_depends openssl_external) +endif() + set(sqlcipher_cflags "${sqlcipher_cflags} -O2") if(USE_LTO) string(APPEND sqlcipher_cflags "${sqlcipher_cflags} -flto") @@ -30,16 +37,12 @@ endif() set(sqlcipher_ldflags "") set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown") -set(sqlcipher_link_libs "") if(APPLE) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") - list(APPEND sqlcipher_link_libs "-framework Security" "-framework Foundation" "-framework CoreFoundation") set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") - list(APPEND sqlcipher_link_libs OpenSSL::SSL OpenSSL::Crypto) - # NOTE: For example libraries are "//lssl.so;//lcrypto.so" the semicolon confuses # the linker so we remove the delimiter to make it compliant. string(REPLACE ";" " " OPENSSL_LIBRARIES_STR "${OPENSSL_LIBRARIES}") @@ -48,6 +51,7 @@ endif() ExternalProject_Add(sqlcipher_external SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher + DEPENDS ${sqlcipher_depends} CONFIGURE_COMMAND ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always @@ -66,3 +70,9 @@ set_target_properties(sqlcipher::sqlcipher PROPERTIES ) target_compile_definitions(sqlcipher::sqlcipher INTERFACE SQLITE_HAS_CODEC) target_link_libraries(sqlcipher::sqlcipher INTERFACE ${sqlcipher_link_libs}) + +if(APPLE) + target_link_libraries(sqlcipher::sqlcipher "-framework Security" "-framework Foundation" "-framework CoreFoundation") +else() + target_link_libraries(sqlcipher::sqlcipher INTERFACE OpenSSL::SSL OpenSSL::Crypto) +endif() From e6c962a9568b548add4b1eef13d10fb926a67a1d Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 12:06:10 +1100 Subject: [PATCH 36/72] We lookup libraries via pkgconfig which defines openssl::openssl for us --- external/openssl-cmake/CMakeLists.txt | 7 +++++++ external/sqlcipher-cmake/CMakeLists.txt | 4 +--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index c4d2065e..962a98f2 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -215,5 +215,12 @@ if(NOT APPLE AND NOT WIN32) target_link_libraries(OpenSSL::SSL INTERFACE OpenSSL::Crypto) set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) set(OPENSSL_ROOT_DIR ${DEPS_DESTDIR} CACHE PATH "" FORCE) + + # NOTE: Create target so then libsession_system_or_submodule exports the + # :: target so that we can universally use that target for both PkgConfig + # and non-pkconfig versions of OpenSSL. + add_library(openssl INTERFACE) + target_link_libraries(openssl INTERFACE OpenSSL::SSL OpenSSL::Crypto) + target_include_directories(openssl INTERFACE ${OPENSSL_INCLUDE_DIRS}) endif() diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 27896ac3..4ccc8d36 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -69,10 +69,8 @@ set_target_properties(sqlcipher::sqlcipher PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${SQLCIPHER_INCLUDEDIR} ) target_compile_definitions(sqlcipher::sqlcipher INTERFACE SQLITE_HAS_CODEC) -target_link_libraries(sqlcipher::sqlcipher INTERFACE ${sqlcipher_link_libs}) - if(APPLE) target_link_libraries(sqlcipher::sqlcipher "-framework Security" "-framework Foundation" "-framework CoreFoundation") else() - target_link_libraries(sqlcipher::sqlcipher INTERFACE OpenSSL::SSL OpenSSL::Crypto) + target_link_libraries(sqlcipher::sqlcipher INTERFACE openssl::openssl) endif() From 3bb71259632965a68c942d5e5def4f54bea4d163 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 13:43:25 +1100 Subject: [PATCH 37/72] SQLCiper+OpenSSL requires -lm --- external/sqlcipher-cmake/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 4ccc8d36..6bd64b22 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -44,9 +44,10 @@ if(APPLE) else() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") # NOTE: For example libraries are "//lssl.so;//lcrypto.so" the semicolon confuses - # the linker so we remove the delimiter to make it compliant. + # the linker so we remove the delimiter to make it compliant. Also note that -lm (math) is + # needed if we choose OpenSSL as our encryption library for SQLCipher. string(REPLACE ";" " " OPENSSL_LIBRARIES_STR "${OPENSSL_LIBRARIES}") - set(sqlcipher_ldflags "${sqlcipher_ldflags} ${OPENSSL_LIBRARIES_STR}") + set(sqlcipher_ldflags "${sqlcipher_ldflags} ${OPENSSL_LIBRARIES_STR} -lm") endif() ExternalProject_Add(sqlcipher_external From 09949e4b2ccfc1152fec23004cc18282b2943ace Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 14:39:38 +1100 Subject: [PATCH 38/72] Export OPENSSL_LIBRARIES to set SQLCipher's LDFLAGS in pkg/non-pkg config builds --- external/openssl-cmake/CMakeLists.txt | 1 + external/sqlcipher | 2 +- external/sqlcipher-cmake/CMakeLists.txt | 15 +++++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 962a98f2..3abe9196 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -215,6 +215,7 @@ if(NOT APPLE AND NOT WIN32) target_link_libraries(OpenSSL::SSL INTERFACE OpenSSL::Crypto) set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) set(OPENSSL_ROOT_DIR ${DEPS_DESTDIR} CACHE PATH "" FORCE) + set(OPENSSL_LIBRARIES "${DEPS_DESTDIR}/lib/libssl.a;${DEPS_DESTDIR}/lib/libcrypto.a" CACHE PATH "" FORCE) # NOTE: Create target so then libsession_system_or_submodule exports the # :: target so that we can universally use that target for both PkgConfig diff --git a/external/sqlcipher b/external/sqlcipher index ab223bd8..d41a25f4 160000 --- a/external/sqlcipher +++ b/external/sqlcipher @@ -1 +1 @@ -Subproject commit ab223bd801ec225d1497a077da08777d21d1266d +Subproject commit d41a25f448ba08ce24c0a599cf322046bdaa135a diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 6bd64b22..3c448d83 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -39,25 +39,28 @@ set(sqlcipher_ldflags "") set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown") if(APPLE) - set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") - set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") + set(sqlcipher_libs "${sqlcipher_libs} -framework Security -framework Foundation -framework CoreFoundation") else() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") # NOTE: For example libraries are "//lssl.so;//lcrypto.so" the semicolon confuses # the linker so we remove the delimiter to make it compliant. Also note that -lm (math) is # needed if we choose OpenSSL as our encryption library for SQLCipher. - string(REPLACE ";" " " OPENSSL_LIBRARIES_STR "${OPENSSL_LIBRARIES}") - set(sqlcipher_ldflags "${sqlcipher_ldflags} ${OPENSSL_LIBRARIES_STR} -lm") + string(REPLACE ";" " -l" OPENSSL_LIBRARIES_STR "-l${OPENSSL_LIBRARIES}") + set(sqlcipher_libs "${sqlcipher_libs} -lm ${OPENSSL_LIBRARIES_STR}") endif() ExternalProject_Add(sqlcipher_external SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher DEPENDS ${sqlcipher_depends} + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_OUTPUT_ON_FAILURE ON CONFIGURE_COMMAND ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always --prefix=${SQLCIPHER_PREFIX} CC=${CMAKE_C_COMPILER} CFLAGS=${sqlcipher_cflags} - LDFLAGS=${sqlcipher_ldflags} + LDFLAGS=${sqlcipher_libs} BUILD_COMMAND make lib BUILD_BYPRODUCTS ${SQLCIPHER_LIBDIR}/libsqlite3.a @@ -71,7 +74,7 @@ set_target_properties(sqlcipher::sqlcipher PROPERTIES ) target_compile_definitions(sqlcipher::sqlcipher INTERFACE SQLITE_HAS_CODEC) if(APPLE) - target_link_libraries(sqlcipher::sqlcipher "-framework Security" "-framework Foundation" "-framework CoreFoundation") + target_link_libraries(sqlcipher::sqlcipher INTERFACE "-framework Security" "-framework Foundation" "-framework CoreFoundation") else() target_link_libraries(sqlcipher::sqlcipher INTERFACE openssl::openssl) endif() From c201f21e8e9330e1fd0ad2403be911fc6a5737a7 Mon Sep 17 00:00:00 2001 From: doylet Date: Mon, 12 Jan 2026 17:37:41 +1100 Subject: [PATCH 39/72] Try removing the X_VERSION variable incase pkgconfig found a match but it was incompatible --- external/CMakeLists.txt | 1 + external/sqlcipher-cmake/CMakeLists.txt | 15 +++++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index b5906fc3..b0a2b18b 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -54,6 +54,7 @@ macro(libsession_system_or_submodule BIGNAME smallname pkgconf subdir) message(STATUS "Found system ${smallname} ${${BIGNAME}_VERSION}") else() message(STATUS "using ${smallname} submodule") + unset(${BIGNAME}_VERSION) add_subdirectory(${subdir}) endif() if(TARGET ${smallname} AND NOT TARGET ${smallname}::${smallname}) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 3c448d83..676ebc10 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -39,15 +39,15 @@ set(sqlcipher_ldflags "") set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown") if(APPLE) - set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") - set(sqlcipher_libs "${sqlcipher_libs} -framework Security -framework Foundation -framework CoreFoundation") + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") + set(sqlcipher_ldflags "${sqlcipher_libs} -framework Security -framework Foundation -framework CoreFoundation") else() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") # NOTE: For example libraries are "//lssl.so;//lcrypto.so" the semicolon confuses # the linker so we remove the delimiter to make it compliant. Also note that -lm (math) is # needed if we choose OpenSSL as our encryption library for SQLCipher. string(REPLACE ";" " -l" OPENSSL_LIBRARIES_STR "-l${OPENSSL_LIBRARIES}") - set(sqlcipher_libs "${sqlcipher_libs} -lm ${OPENSSL_LIBRARIES_STR}") + set(sqlcipher_ldflags "${sqlcipher_libs} -lm ${OPENSSL_LIBRARIES_STR}") endif() ExternalProject_Add(sqlcipher_external @@ -56,11 +56,10 @@ ExternalProject_Add(sqlcipher_external LOG_CONFIGURE ON LOG_BUILD ON LOG_OUTPUT_ON_FAILURE ON - CONFIGURE_COMMAND - ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared - --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always - --prefix=${SQLCIPHER_PREFIX} CC=${CMAKE_C_COMPILER} CFLAGS=${sqlcipher_cflags} - LDFLAGS=${sqlcipher_libs} + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env LDFLAGS=${sqlcipher_ldflags} + ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared + --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always + --prefix=${SQLCIPHER_PREFIX} CC=${CMAKE_C_COMPILER} CFLAGS=${sqlcipher_cflags} BUILD_COMMAND make lib BUILD_BYPRODUCTS ${SQLCIPHER_LIBDIR}/libsqlite3.a From 84bf5974b9b3c2ce13d7d5f99c6df12ac35b7678 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 11:14:23 +1100 Subject: [PATCH 40/72] Try work-around OPENSSL_VERSION getting blanked out on CI machines --- external/CMakeLists.txt | 1 - external/openssl-cmake/CMakeLists.txt | 12 +++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index b0a2b18b..b5906fc3 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -54,7 +54,6 @@ macro(libsession_system_or_submodule BIGNAME smallname pkgconf subdir) message(STATUS "Found system ${smallname} ${${BIGNAME}_VERSION}") else() message(STATUS "using ${smallname} submodule") - unset(${BIGNAME}_VERSION) add_subdirectory(${subdir}) endif() if(TARGET ${smallname} AND NOT TARGET ${smallname}::${smallname}) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 3abe9196..2a48fbc7 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -1,6 +1,16 @@ set(LOCAL_MIRROR "" CACHE STRING "local mirror path/URL for lib downloads") -set(OPENSSL_VERSION 3.5.4 CACHE STRING "openssl version") +set(desired_version 3.5.4) +set(OPENSSL_VERSION ${desired_version} CACHE STRING "openssl version") + +# NOTE: Workaround issue on CI where OpenSSL version seems to get blanked out on +# Debian 12 for some reason. Those machines have 3.0.18 found via pkgconfig +# which is insufficient and I suspect something there is blanking out the +# version causing this to fail that I can't figure out how to fix. +if(NOT OPENSSL_VERSION OR OPENSSL_VERSION STREQUAL "") + set(OPENSSL_VERSION "${desired_version}" CACHE STRING "openssl version" FORCE) +endif() + set(OPENSSL_MIRROR ${LOCAL_MIRROR} https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION} CACHE STRING "openssl download mirror(s)") set(OPENSSL_SOURCE openssl-${OPENSSL_VERSION}.tar.gz) set(OPENSSL_HASH SHA256=967311f84955316969bdb1d8d4b983718ef42338639c621ec4c34fddef355e99 From 35c851e7b05e724474849e517c8ef0328a3bdf25 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 11:30:21 +1100 Subject: [PATCH 41/72] Set explicit link path when using submodule OpenSSL Otherwise specifically on our Debian 12 target, `${OPENSSL_ROOT_DIR}/lib` resolves to /lib/ however, checking with `ldconfig -v -N` reveals that this library is actually in `/lib/x86_64-linux-gnu` breaking the entire build (because OPENSSL_ROOT_DIR is not returning valid values for a system provided OpenSSL vs a submodule provided OpenSSL, but, only on some specific platforms. We work around this by only adding the `-L` argument when we know we are using OpenSSL from the submodule and/or otherwise link to OpenSSL without it when using the system provided one. --- external/openssl-cmake/CMakeLists.txt | 1 - external/sqlcipher-cmake/CMakeLists.txt | 15 ++++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 2a48fbc7..094c7998 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -225,7 +225,6 @@ if(NOT APPLE AND NOT WIN32) target_link_libraries(OpenSSL::SSL INTERFACE OpenSSL::Crypto) set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) set(OPENSSL_ROOT_DIR ${DEPS_DESTDIR} CACHE PATH "" FORCE) - set(OPENSSL_LIBRARIES "${DEPS_DESTDIR}/lib/libssl.a;${DEPS_DESTDIR}/lib/libcrypto.a" CACHE PATH "" FORCE) # NOTE: Create target so then libsession_system_or_submodule exports the # :: target so that we can universally use that target for both PkgConfig diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 676ebc10..de9c8625 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -40,14 +40,18 @@ set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT if(APPLE) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") - set(sqlcipher_ldflags "${sqlcipher_libs} -framework Security -framework Foundation -framework CoreFoundation") + set(sqlcipher_ldflags "-framework Security -framework Foundation -framework CoreFoundation") else() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") - # NOTE: For example libraries are "//lssl.so;//lcrypto.so" the semicolon confuses + # NOTE: For example libraries are "/libssl.so;//libcrypto.so" the semicolon confuses # the linker so we remove the delimiter to make it compliant. Also note that -lm (math) is # needed if we choose OpenSSL as our encryption library for SQLCipher. - string(REPLACE ";" " -l" OPENSSL_LIBRARIES_STR "-l${OPENSSL_LIBRARIES}") - set(sqlcipher_ldflags "${sqlcipher_libs} -lm ${OPENSSL_LIBRARIES_STR}") + string(REPLACE ";" " -l" OPENSSL_LIBRARIES_STR "-lcrypto -lssl}") + if (TARGET openssl_external) + set(sqlcipher_ldflags "-lm -L${OPENSSL_ROOT_DIR}/lib -lcrypto -lssl") + else() + set(sqlcipher_ldflags "-lm -lcrypto -lssl") + endif() endif() ExternalProject_Add(sqlcipher_external @@ -56,10 +60,11 @@ ExternalProject_Add(sqlcipher_external LOG_CONFIGURE ON LOG_BUILD ON LOG_OUTPUT_ON_FAILURE ON - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env LDFLAGS=${sqlcipher_ldflags} + CONFIGURE_COMMAND ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always --prefix=${SQLCIPHER_PREFIX} CC=${CMAKE_C_COMPILER} CFLAGS=${sqlcipher_cflags} + LDFLAGS=${sqlcipher_ldflags} BUILD_COMMAND make lib BUILD_BYPRODUCTS ${SQLCIPHER_LIBDIR}/libsqlite3.a From 8c1a448041b1e27e8e0de21085b8f82496858de7 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 14:08:40 +1100 Subject: [PATCH 42/72] Use updated cross compiling cmake impl for SQLCipher --- external/sqlcipher-cmake/CMakeLists.txt | 64 ++++++++++++++++++++----- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index de9c8625..6e8bf716 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -21,19 +21,57 @@ if(USE_LTO) endif() set(cross_host "") +set(cross_rc "") if(CMAKE_CROSSCOMPILING) - if (APPLE) - if(sane_cross_host MATCHES "^(.*-)ios([0-9.]+)(-.*)?$") - set(cross_host "--host=${CMAKE_MATCH_1}darwin${CMAKE_MATCH_2}${CMAKE_MATCH_3}") - endif() - if(sane_cross_host MATCHES "^(.*)-simulator$") - set(cross_host "--host=${CMAKE_MATCH_1}") - endif() - endif() - if(CMAKE_SYSTEM_PROCESSOR) - set(cross_host "--host=${CMAKE_SYSTEM_PROCESSOR}") - endif() + if(APPLE AND NOT ARCH_TRIPLET AND APPLE_TARGET_TRIPLE) + set(ARCH_TRIPLET "${APPLE_TARGET_TRIPLE}") + endif() + set(cross_host "--host=${ARCH_TRIPLET}") + if (ARCH_TRIPLET MATCHES mingw AND CMAKE_RC_COMPILER) + set(cross_rc "WINDRES=${CMAKE_RC_COMPILER}") + endif() +endif() + +set(deps_cc ${CMAKE_C_COMPILER}) +if(CMAKE_C_COMPILER_LAUNCHER) + set(deps_cc "${CMAKE_C_COMPILER_LAUNCHER} ${deps_cc}") endif() +if(ANDROID) + set(android_toolchain_suffix linux-android) + set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) + set(cross_host "--host=x86_64-linux-android") + set(android_compiler_prefix x86_64) + set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) + set(android_toolchain_prefix x86_64) + set(android_toolchain_suffix linux-android) + elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) + set(cross_host "--host=i686-linux-android") + set(android_compiler_prefix i686) + set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) + set(android_toolchain_prefix i686) + set(android_toolchain_suffix linux-android) + elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) + set(cross_host "--host=armv7a-linux-androideabi") + set(android_compiler_prefix armv7a) + set(android_compiler_suffix linux-androideabi${ANDROID_PLATFORM_LEVEL}) + set(android_toolchain_prefix arm) + set(android_toolchain_suffix linux-androideabi) + elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) + set(cross_host "--host=aarch64-linux-android") + set(android_compiler_prefix aarch64) + set(android_compiler_suffix linux-android${ANDROID_PLATFORM_LEVEL}) + set(android_toolchain_prefix aarch64) + set(android_toolchain_suffix linux-android) + else() + message(FATAL_ERROR "unknown android arch: ${CMAKE_ANDROID_ARCH_ABI}") + endif() + set(deps_cc "${ANDROID_TOOLCHAIN_ROOT}/bin/${android_compiler_prefix}-${android_compiler_suffix}-clang") +endif() + +set(deps_CFLAGS "-O2") +set(deps_CXXFLAGS "-O2") + set(sqlcipher_ldflags "") set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown") @@ -63,8 +101,8 @@ ExternalProject_Add(sqlcipher_external CONFIGURE_COMMAND ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always - --prefix=${SQLCIPHER_PREFIX} CC=${CMAKE_C_COMPILER} CFLAGS=${sqlcipher_cflags} - LDFLAGS=${sqlcipher_ldflags} + --prefix=${SQLCIPHER_PREFIX} CC=${deps_cc} CFLAGS=${sqlcipher_cflags} + LDFLAGS=${sqlcipher_ldflags} ${cross_rc} BUILD_COMMAND make lib BUILD_BYPRODUCTS ${SQLCIPHER_LIBDIR}/libsqlite3.a From 27903851d28b5467a924d651a5cbe1e8780a6a8e Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 15:14:14 +1100 Subject: [PATCH 43/72] Augment SQLCipher build w/ OpenSSL include dir, fixes static android target --- external/openssl-cmake/CMakeLists.txt | 6 +++++- external/sqlcipher-cmake/CMakeLists.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 094c7998..a51ec7fe 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -223,9 +223,13 @@ if(NOT APPLE AND NOT WIN32) add_static_target(OpenSSL::SSL openssl_external libssl.a) add_static_target(OpenSSL::Crypto openssl_external libcrypto.a) target_link_libraries(OpenSSL::SSL INTERFACE OpenSSL::Crypto) - set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) set(OPENSSL_ROOT_DIR ${DEPS_DESTDIR} CACHE PATH "" FORCE) + # NOTE: FindOpenSSL defines OPENSSL_INCLUDE_DIR, we use pkg_check_modules which defines + # OPENSSL_INCLUDE_DIRS, we define both for maximum compatibility... + set(OPENSSL_INCLUDE_DIRS ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) + set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) + # NOTE: Create target so then libsession_system_or_submodule exports the # :: target so that we can universally use that target for both PkgConfig # and non-pkconfig versions of OpenSSL. diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 6e8bf716..a92dc2b6 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -80,7 +80,7 @@ if(APPLE) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") set(sqlcipher_ldflags "-framework Security -framework Foundation -framework CoreFoundation") else() - set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL") + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS}") # NOTE: For example libraries are "/libssl.so;//libcrypto.so" the semicolon confuses # the linker so we remove the delimiter to make it compliant. Also note that -lm (math) is # needed if we choose OpenSSL as our encryption library for SQLCipher. From c12cf96d8b554fa7b23decb5693779b081a78b5f Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 15:30:31 +1100 Subject: [PATCH 44/72] Add Android's liblog required for SQLite/cipher --- external/sqlcipher-cmake/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index a92dc2b6..a73f0c25 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -92,6 +92,11 @@ else() endif() endif() +if (ANDROID) + # NOTE: SQLite uses __android_log_print and __android_log_write which is in their liblog library + set(sqlcipher_ldflags "${sqlcipher_ldflags} -llog") +endif() + ExternalProject_Add(sqlcipher_external SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher DEPENDS ${sqlcipher_depends} From 8ce54e164afc455a38f8f38b9d0f343c49fb692c Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 16:12:48 +1100 Subject: [PATCH 45/72] Specify NDK sysroot for OpenSSL --- external/openssl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index a51ec7fe..6780d63d 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -206,7 +206,7 @@ if(NOT APPLE AND NOT WIN32) set(openssl_system_env SYSTEM=MINGW64 RC=${CMAKE_RC_COMPILER}) elseif(ANDROID) set(openssl_system_env SYSTEM=Linux MACHINE=${openssl_machine} ${cross_extra}) - set(openssl_extra_opts no-asm) + set(openssl_extra_opts no-asm --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot) endif() endif() build_external(openssl From 83b508a742a6af40a68050a19ea5c44ffae6a322 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 16:18:53 +1100 Subject: [PATCH 46/72] TEST: Try removing WIN32 guard for OpenSSL for the static windows x64 xbuild --- external/openssl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 6780d63d..b9c52afc 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -196,7 +196,7 @@ function(build_external target) ) endfunction() -if(NOT APPLE AND NOT WIN32) +if(NOT APPLE) set(openssl_system_env "") set(openssl_cc "${deps_cc}") if(CMAKE_CROSSCOMPILING) From 1753038d36fc167fc4de1e5ec5948d4635f2c8fd Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 16:34:19 +1100 Subject: [PATCH 47/72] Specify NDK sysroot before config options for OpenSSL --- external/openssl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index b9c52afc..762ef490 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -206,7 +206,7 @@ if(NOT APPLE) set(openssl_system_env SYSTEM=MINGW64 RC=${CMAKE_RC_COMPILER}) elseif(ANDROID) set(openssl_system_env SYSTEM=Linux MACHINE=${openssl_machine} ${cross_extra}) - set(openssl_extra_opts no-asm --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot) + set(openssl_extra_opts --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot no-asm) endif() endif() build_external(openssl From 7a1496656065a317bdf15d194f68be11f63b7a58 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 16:57:33 +1100 Subject: [PATCH 48/72] Use updated method to pass toolchain for MINGW re: NOTES-WINDOWS.md for OpenSSL --- external/openssl-cmake/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 762ef490..0dbfd7c4 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -201,9 +201,9 @@ if(NOT APPLE) set(openssl_cc "${deps_cc}") if(CMAKE_CROSSCOMPILING) if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) - set(openssl_system_env SYSTEM=MINGW64 RC=${CMAKE_RC_COMPILER}) + set(openssl_extra_opts mingw64 --cross-compile-prefix=x86_64-w64-mingw32-) elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) - set(openssl_system_env SYSTEM=MINGW64 RC=${CMAKE_RC_COMPILER}) + set(openssl_extra_opts mingw --cross-compile-prefix=i686-w64-mingw32-) elseif(ANDROID) set(openssl_system_env SYSTEM=Linux MACHINE=${openssl_machine} ${cross_extra}) set(openssl_extra_opts --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot no-asm) From 5108ca889ff8cfd07b80ed14f70a81c0763f7aec Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 17:08:56 +1100 Subject: [PATCH 49/72] Switch to the modern Configure tool for OpenSSL --- external/openssl-cmake/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 0dbfd7c4..b4da1038 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -197,7 +197,6 @@ function(build_external target) endfunction() if(NOT APPLE) - set(openssl_system_env "") set(openssl_cc "${deps_cc}") if(CMAKE_CROSSCOMPILING) if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) @@ -205,16 +204,17 @@ if(NOT APPLE) elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) set(openssl_extra_opts mingw --cross-compile-prefix=i686-w64-mingw32-) elseif(ANDROID) - set(openssl_system_env SYSTEM=Linux MACHINE=${openssl_machine} ${cross_extra}) - set(openssl_extra_opts --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot no-asm) + set(openssl_extra_opts ${CMAKE_ANDROID_ARCH_ABI} no-asm) endif() endif() build_external(openssl - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env CC=${openssl_cc} ${openssl_system_env} ./config - --prefix=${DEPS_DESTDIR} --libdir=lib ${openssl_extra_opts} - no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost - no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 - no-static-engine no-tests no-weak-ssl-ciphers no-zlib no-zlib-dynamic "CFLAGS=${deps_CFLAGS}" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env + CC=${openssl_cc} + CFLAGS=${deps_CFLAGS} + ./Configure --prefix=${DEPS_DESTDIR} --libdir=lib ${openssl_extra_opts} + no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost + no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 + no-static-engine no-tests no-weak-ssl-ciphers no-zlib no-zlib-dynamic INSTALL_COMMAND make install_sw BUILD_BYPRODUCTS ${DEPS_DESTDIR}/lib/libssl.a ${DEPS_DESTDIR}/lib/libcrypto.a From 64a9894442dce808581790e7ec784ecabf9db6bf Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 17:52:16 +1100 Subject: [PATCH 50/72] Specify correct android target for OpenSSL Configure --- external/openssl-cmake/CMakeLists.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index b4da1038..ed726f15 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -28,12 +28,16 @@ if (ANDROID) message(FATAL_ERROR "ANDROID_TOOLCHAIN_NAME not set; did you run with the proper android toolchain options?") endif() if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) + set(openssl_android_arch android-x86_64) set(android_clang x86_64-linux-android${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) + set(openssl_android_arch android-x86) set(android_clang i686-linux-android${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) + set(openssl_android_arch android-armeabi) set(android_clang armv7a-linux-androideabi${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) + set(openssl_android_arch android-arm64) set(android_clang aarch64-linux-android${ANDROID_PLATFORM_LEVEL}-clang) else() message(FATAL_ERROR "Don't know how to build for android arch abi ${CMAKE_ANDROID_ARCH_ABI}") @@ -197,20 +201,21 @@ function(build_external target) endfunction() if(NOT APPLE) - set(openssl_cc "${deps_cc}") if(CMAKE_CROSSCOMPILING) if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) set(openssl_extra_opts mingw64 --cross-compile-prefix=x86_64-w64-mingw32-) elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) set(openssl_extra_opts mingw --cross-compile-prefix=i686-w64-mingw32-) elseif(ANDROID) - set(openssl_extra_opts ${CMAKE_ANDROID_ARCH_ABI} no-asm) + set(openssl_extra_opts ${openssl_android_arch} no-asm) + set(openssl_extra_env ANDROID_NDK_ROOT=${CMAKE_ANDROID_NDK} PATH="${ANDROID_TOOLCHAIN_ROOT}/bin:$ENV{PATH}") endif() endif() build_external(openssl - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env - CC=${openssl_cc} + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env + CC=${deps_cc} CFLAGS=${deps_CFLAGS} + ${openssl_extra_env} ./Configure --prefix=${DEPS_DESTDIR} --libdir=lib ${openssl_extra_opts} no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 From 09086ba53a7015dc699f08d54cad2ae02d730efb Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 18:19:33 +1100 Subject: [PATCH 51/72] Switch to arm target for armeabi --- external/openssl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index ed726f15..1207de71 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -34,7 +34,7 @@ if (ANDROID) set(openssl_android_arch android-x86) set(android_clang i686-linux-android${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) - set(openssl_android_arch android-armeabi) + set(openssl_android_arch android-arm) set(android_clang armv7a-linux-androideabi${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) set(openssl_android_arch android-arm64) From de9f44bbe8b65376401c0f39c9e628ee9a01472c Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 18:35:15 +1100 Subject: [PATCH 52/72] TEMP: Test arm64 for 32 bit OpenSSL hack --- external/openssl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 1207de71..b5f08db3 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -34,7 +34,7 @@ if (ANDROID) set(openssl_android_arch android-x86) set(android_clang i686-linux-android${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) - set(openssl_android_arch android-arm) + set(openssl_android_arch android-arm64) set(android_clang armv7a-linux-androideabi${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) set(openssl_android_arch android-arm64) From f2945f67b3240cc8473eb470c90cec23a3f3eb33 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 18:50:19 +1100 Subject: [PATCH 53/72] Cross-compile-prefix for mingw breaks ccache hijacking --- external/openssl-cmake/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index b5f08db3..76111905 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -204,8 +204,10 @@ if(NOT APPLE) if(CMAKE_CROSSCOMPILING) if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) set(openssl_extra_opts mingw64 --cross-compile-prefix=x86_64-w64-mingw32-) + set(deps_cc "") elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) set(openssl_extra_opts mingw --cross-compile-prefix=i686-w64-mingw32-) + set(deps_cc "") elseif(ANDROID) set(openssl_extra_opts ${openssl_android_arch} no-asm) set(openssl_extra_env ANDROID_NDK_ROOT=${CMAKE_ANDROID_NDK} PATH="${ANDROID_TOOLCHAIN_ROOT}/bin:$ENV{PATH}") From 4490bfbed67966f35c0f1e5da29594ec11acc652 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 19:04:15 +1100 Subject: [PATCH 54/72] Ensure ws2_32 crypt32 is available for mingw --- external/sqlcipher-cmake/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index a73f0c25..32f0e3bf 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -124,4 +124,7 @@ if(APPLE) target_link_libraries(sqlcipher::sqlcipher INTERFACE "-framework Security" "-framework Foundation" "-framework CoreFoundation") else() target_link_libraries(sqlcipher::sqlcipher INTERFACE openssl::openssl) + if (WIN32 OR MINGW) + target_link_libraries(sqlcipher::sqlcipher INTERFACE ws2_32 crypt32) + endif() endif() From 718d0a1dac5b26c54ee99b8bce740be2f727968b Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 19:22:15 +1100 Subject: [PATCH 55/72] Specify the exact library to build for sqlcipher --- external/sqlcipher-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 32f0e3bf..7de6d031 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -108,7 +108,7 @@ ExternalProject_Add(sqlcipher_external --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always --prefix=${SQLCIPHER_PREFIX} CC=${deps_cc} CFLAGS=${sqlcipher_cflags} LDFLAGS=${sqlcipher_ldflags} ${cross_rc} - BUILD_COMMAND make lib + BUILD_COMMAND make libsqlite3.a BUILD_BYPRODUCTS ${SQLCIPHER_LIBDIR}/libsqlite3.a ) From 15e824df2ac02a6c2a0123a93f258797ee9ab8b1 Mon Sep 17 00:00:00 2001 From: doylet Date: Tue, 13 Jan 2026 19:30:56 +1100 Subject: [PATCH 56/72] Add work-around for sqlite3.exe being built on windows --- external/sqlcipher-cmake/CMakeLists.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 7de6d031..2017fce0 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -90,11 +90,17 @@ else() else() set(sqlcipher_ldflags "-lm -lcrypto -lssl") endif() -endif() -if (ANDROID) - # NOTE: SQLite uses __android_log_print and __android_log_write which is in their liblog library - set(sqlcipher_ldflags "${sqlcipher_ldflags} -llog") + if(WIN32) + # TODO: I'm unable to stop SQLCipher from building sqlite3.exe in the mingw cross-compile + # so we need to augment that executable with some Window's libraries to get us through... + set(sqlcipher_ldflags "${sqlcipher_ldflags} -lws2_32 -lcrypt32") + endif() + + if(ANDROID) + # NOTE: SQLite uses __android_log_print and __android_log_write which is in their liblog library + set(sqlcipher_ldflags "${sqlcipher_ldflags} -llog") + endif() endif() ExternalProject_Add(sqlcipher_external From 96ed6a9cfceda1508c31ae20ee3643963b3da321 Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 10:07:50 +1100 Subject: [PATCH 57/72] Try getting ccache working w/ mingw openssl build again, try fix android builds Whilst we can use the new android targets for the ./Configure tool that openssl has, this has issues in that it doesn't support the clang NDK toolchain, it still defaults to gcc and there doesn't seem to be a way to override this without setting CC/AR/CXX/RANLIB e.t.c. Oxen-core can statically build openssl already without issues so we're going to revert back to what it's doing and KISS. --- external/openssl-cmake/CMakeLists.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 76111905..1dd8799a 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -203,14 +203,12 @@ endfunction() if(NOT APPLE) if(CMAKE_CROSSCOMPILING) if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) - set(openssl_extra_opts mingw64 --cross-compile-prefix=x86_64-w64-mingw32-) - set(deps_cc "") + set(openssl_extra_opts mingw64) elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) - set(openssl_extra_opts mingw --cross-compile-prefix=i686-w64-mingw32-) - set(deps_cc "") + set(openssl_extra_opts mingw) elseif(ANDROID) - set(openssl_extra_opts ${openssl_android_arch} no-asm) - set(openssl_extra_env ANDROID_NDK_ROOT=${CMAKE_ANDROID_NDK} PATH="${ANDROID_TOOLCHAIN_ROOT}/bin:$ENV{PATH}") + set(openssl_extra_opts no-asm) + set(openssl_extra_env SYSTEM=Linux ${cross_extra}) endif() endif() build_external(openssl From 9d6a28a54bb55b7d13dc1ac70403943a0cef3824 Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 10:25:52 +1100 Subject: [PATCH 58/72] Ensure WINDRES is set for mingw OpenSSL cross compile --- external/openssl-cmake/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 1dd8799a..bfeee291 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -208,7 +208,7 @@ if(NOT APPLE) set(openssl_extra_opts mingw) elseif(ANDROID) set(openssl_extra_opts no-asm) - set(openssl_extra_env SYSTEM=Linux ${cross_extra}) + set(openssl_extra_env SYSTEM=Linux) endif() endif() build_external(openssl @@ -216,6 +216,7 @@ if(NOT APPLE) CC=${deps_cc} CFLAGS=${deps_CFLAGS} ${openssl_extra_env} + ${cross_extra} ./Configure --prefix=${DEPS_DESTDIR} --libdir=lib ${openssl_extra_opts} no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 From d419c7a84fc9ccf5a5260bf3d1a2bc6bf6982431 Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 10:47:12 +1100 Subject: [PATCH 59/72] OpenSSL uses RC= for Win32 resource compiler, set the NDK sysroot for OpenSSL --- external/openssl-cmake/CMakeLists.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index bfeee291..14e17741 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -28,16 +28,12 @@ if (ANDROID) message(FATAL_ERROR "ANDROID_TOOLCHAIN_NAME not set; did you run with the proper android toolchain options?") endif() if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) - set(openssl_android_arch android-x86_64) set(android_clang x86_64-linux-android${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES x86) - set(openssl_android_arch android-x86) set(android_clang i686-linux-android${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES armeabi-v7a) - set(openssl_android_arch android-arm64) set(android_clang armv7a-linux-androideabi${ANDROID_PLATFORM_LEVEL}-clang) elseif(CMAKE_ANDROID_ARCH_ABI MATCHES arm64-v8a) - set(openssl_android_arch android-arm64) set(android_clang aarch64-linux-android${ANDROID_PLATFORM_LEVEL}-clang) else() message(FATAL_ERROR "Don't know how to build for android arch abi ${CMAKE_ANDROID_ARCH_ABI}") @@ -204,11 +200,14 @@ if(NOT APPLE) if(CMAKE_CROSSCOMPILING) if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) set(openssl_extra_opts mingw64) + set(openssl_extra_env RC=${CMAKE_RC_COMPILER}) elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) set(openssl_extra_opts mingw) + set(openssl_extra_env RC=${CMAKE_RC_COMPILER}) elseif(ANDROID) set(openssl_extra_opts no-asm) set(openssl_extra_env SYSTEM=Linux) + set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${CMAKE_ANDROID_NDK}") endif() endif() build_external(openssl From bc7279d5ddbab4399ad9b2492267151b2cb8c63f Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 10:59:20 +1100 Subject: [PATCH 60/72] Specify the correct sysroot for the android NDK toolchain in OpenSSL --- external/openssl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 14e17741..63adc37a 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -207,7 +207,7 @@ if(NOT APPLE) elseif(ANDROID) set(openssl_extra_opts no-asm) set(openssl_extra_env SYSTEM=Linux) - set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${CMAKE_ANDROID_NDK}") + set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot") endif() endif() build_external(openssl From 5871b2c04ab5b0f965262417b38ec1d4877fa86f Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 14:53:57 +1100 Subject: [PATCH 61/72] Help OpenSSL android static vbuild find asm/types.h --- external/openssl-cmake/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 63adc37a..df5f9aef 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -208,6 +208,13 @@ if(NOT APPLE) set(openssl_extra_opts no-asm) set(openssl_extra_env SYSTEM=Linux) set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot") + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64 OR CMAKE_ANDROID_ARCH_ABI MATCHES x86) + # NOTE: Sysroot isn't sufficient to find the asm/ folder sitting in the host-tagged folder + # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/linux/types.h:9:10: fatal error: 'asm/types.h' file not found + # At + # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/x86_64-linux-android/asm/ + set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/includes/x86_64-linux-android") + endif() endif() endif() build_external(openssl From 5c7caacb87f472bcfd36c92344050f73f643e789 Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 15:16:00 +1100 Subject: [PATCH 62/72] TEMP: Apple triple check --- external/openssl-cmake/CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index df5f9aef..e479af06 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -76,13 +76,16 @@ endif() set(cross_host "") set(cross_extra "") +if(APPLE) + set(ARCH_TRIPLET "${APPLE_TARGET_TRIPLE}") + if(NOT ARCH_TRIPLET OR ARCH_TRIPLET STREQUAL "") + message(FATAL_ERROR "Unknown or undefined Apple target triple") + endif() +endif() if (ANDROID) set(cross_host "--host=${CMAKE_LIBRARY_ARCHITECTURE}") set(cross_extra "LD=${ANDROID_TOOLCHAIN_ROOT}/bin/${CMAKE_LIBRARY_ARCHITECTURE}-ld" "RANLIB=${CMAKE_RANLIB}" "AR=${CMAKE_AR}") elseif(CMAKE_CROSSCOMPILING) - if(APPLE) - set(ARCH_TRIPLET "${APPLE_TARGET_TRIPLE}") - endif() set(cross_host "--host=${ARCH_TRIPLET}") if (ARCH_TRIPLET MATCHES mingw AND CMAKE_RC_COMPILER) set(cross_extra "WINDRES=${CMAKE_RC_COMPILER}") From 6d46c406a06e9232ac97c8c147e4e668a3ecd1ec Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 15:30:07 +1100 Subject: [PATCH 63/72] Apple does not need to build OpenSSL so skip it entirely --- external/openssl-cmake/CMakeLists.txt | 166 +++++++++----------------- 1 file changed, 55 insertions(+), 111 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index e479af06..4d7219d8 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -1,3 +1,9 @@ +# NOTE: Apple does not build OpenSSL which we use for SQLCipher as SQLCipher compiles natively +# against Apple's Common Crypto +if (APPLE) + return() +endif() + set(LOCAL_MIRROR "" CACHE STRING "local mirror path/URL for lib downloads") set(desired_version 3.5.4) @@ -76,12 +82,6 @@ endif() set(cross_host "") set(cross_extra "") -if(APPLE) - set(ARCH_TRIPLET "${APPLE_TARGET_TRIPLE}") - if(NOT ARCH_TRIPLET OR ARCH_TRIPLET STREQUAL "") - message(FATAL_ERROR "Unknown or undefined Apple target triple") - endif() -endif() if (ANDROID) set(cross_host "--host=${CMAKE_LIBRARY_ARCHITECTURE}") set(cross_extra "LD=${ANDROID_TOOLCHAIN_ROOT}/bin/${CMAKE_LIBRARY_ARCHITECTURE}-ld" "RANLIB=${CMAKE_RANLIB}" "AR=${CMAKE_AR}") @@ -92,64 +92,11 @@ elseif(CMAKE_CROSSCOMPILING) endif() endif() -set(apple_cflags_arch) -set(apple_cxxflags_arch) -set(apple_ldflags_arch) -set(sane_cross_host "${cross_host}") -if(APPLE AND CMAKE_CROSSCOMPILING) - if(sane_cross_host MATCHES "^(.*-)ios([0-9.]+)(-.*)?$") - set(sane_cross_host "${CMAKE_MATCH_1}darwin${CMAKE_MATCH_2}${CMAKE_MATCH_3}") - endif() - if(sane_cross_host MATCHES "^(.*)-simulator$") - set(sane_cross_host "${CMAKE_MATCH_1}") - endif() - - set(apple_arch) - if(ARCH_TRIPLET MATCHES "^(arm|aarch)64.*") - set(apple_arch "arm64") - elseif(ARCH_TRIPLET MATCHES "^x86_64.*") - set(apple_arch "x86_64") - else() - message(FATAL_ERROR "Don't know how to specify -arch for GMP for ${ARCH_TRIPLET} (${APPLE_TARGET_TRIPLE})") - endif() - - set(apple_cflags_arch " -arch ${apple_arch}") - set(apple_cxxflags_arch " -arch ${apple_arch}") - if(CMAKE_OSX_DEPLOYMENT_TARGET) - if (SDK_NAME) - set(apple_ldflags_arch " -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - elseif(CMAKE_OSX_DEPLOYMENT_TARGET) - set(apple_ldflags_arch " -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") - endif() - endif() - set(apple_ldflags_arch "${apple_ldflags_arch} -arch ${apple_arch}") - - if(CMAKE_OSX_SYSROOT) - foreach(f c cxx ld) - set(apple_${f}flags_arch "${apple_${f}flags_arch} -isysroot ${CMAKE_OSX_SYSROOT}") - endforeach() - endif() -elseif(cross_host STREQUAL "" AND CMAKE_LIBRARY_ARCHITECTURE) - set(build_host "--build=${CMAKE_LIBRARY_ARCHITECTURE}") -endif() - set(deps_CFLAGS "-O2 ${flto}") set(deps_CXXFLAGS "-O2 ${flto}") set(deps_noarch_CFLAGS "${deps_CFLAGS}") set(deps_noarch_CXXFLAGS "${deps_CXXFLAGS}") -if(APPLE) - foreach(lang C CXX) - string(APPEND deps_${lang}FLAGS " ${CMAKE_${lang}_SYSROOT_FLAG} ${CMAKE_OSX_SYSROOT} ${CMAKE_${lang}_OSX_DEPLOYMENT_TARGET_FLAG}${CMAKE_OSX_DEPLOYMENT_TARGET}") - - set(deps_noarch_${lang}FLAGS "${deps_${lang}FLAGS}") - - foreach(arch ${CMAKE_OSX_ARCHITECTURES}) - string(APPEND deps_${lang}FLAGS " -arch ${arch}") - endforeach() - endforeach() -endif() - # Builds a target; takes the target name (e.g. "readline") and builds it in an external project with # target name suffixed with `_external`. Its upper-case value is used to get the download details # (from the variables set above). The following options are supported and passed through to @@ -199,57 +146,54 @@ function(build_external target) ) endfunction() -if(NOT APPLE) - if(CMAKE_CROSSCOMPILING) - if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) - set(openssl_extra_opts mingw64) - set(openssl_extra_env RC=${CMAKE_RC_COMPILER}) - elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) - set(openssl_extra_opts mingw) - set(openssl_extra_env RC=${CMAKE_RC_COMPILER}) - elseif(ANDROID) - set(openssl_extra_opts no-asm) - set(openssl_extra_env SYSTEM=Linux) - set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot") - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64 OR CMAKE_ANDROID_ARCH_ABI MATCHES x86) - # NOTE: Sysroot isn't sufficient to find the asm/ folder sitting in the host-tagged folder - # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/linux/types.h:9:10: fatal error: 'asm/types.h' file not found - # At - # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/x86_64-linux-android/asm/ - set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/includes/x86_64-linux-android") - endif() - endif() +if(CMAKE_CROSSCOMPILING) + if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32) + set(openssl_extra_opts mingw64) + set(openssl_extra_env RC=${CMAKE_RC_COMPILER}) + elseif(ARCH_TRIPLET STREQUAL i686-w64-mingw32) + set(openssl_extra_opts mingw) + set(openssl_extra_env RC=${CMAKE_RC_COMPILER}) + elseif(ANDROID) + set(openssl_extra_opts no-asm) + set(openssl_extra_env SYSTEM=Linux) + set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot") + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64 OR CMAKE_ANDROID_ARCH_ABI MATCHES x86) + # NOTE: Sysroot isn't sufficient to find the asm/ folder sitting in the host-tagged folder + # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/linux/types.h:9:10: fatal error: 'asm/types.h' file not found + # At + # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/x86_64-linux-android/asm/ + set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/include/x86_64-linux-android") endif() - build_external(openssl - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env - CC=${deps_cc} - CFLAGS=${deps_CFLAGS} - ${openssl_extra_env} - ${cross_extra} - ./Configure --prefix=${DEPS_DESTDIR} --libdir=lib ${openssl_extra_opts} - no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost - no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 - no-static-engine no-tests no-weak-ssl-ciphers no-zlib no-zlib-dynamic - INSTALL_COMMAND make install_sw - BUILD_BYPRODUCTS - ${DEPS_DESTDIR}/lib/libssl.a ${DEPS_DESTDIR}/lib/libcrypto.a - ${DEPS_DESTDIR}/include/openssl/ssl.h ${DEPS_DESTDIR}/include/openssl/crypto.h - ) - add_static_target(OpenSSL::SSL openssl_external libssl.a) - add_static_target(OpenSSL::Crypto openssl_external libcrypto.a) - target_link_libraries(OpenSSL::SSL INTERFACE OpenSSL::Crypto) - set(OPENSSL_ROOT_DIR ${DEPS_DESTDIR} CACHE PATH "" FORCE) - - # NOTE: FindOpenSSL defines OPENSSL_INCLUDE_DIR, we use pkg_check_modules which defines - # OPENSSL_INCLUDE_DIRS, we define both for maximum compatibility... - set(OPENSSL_INCLUDE_DIRS ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) - set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) - - # NOTE: Create target so then libsession_system_or_submodule exports the - # :: target so that we can universally use that target for both PkgConfig - # and non-pkconfig versions of OpenSSL. - add_library(openssl INTERFACE) - target_link_libraries(openssl INTERFACE OpenSSL::SSL OpenSSL::Crypto) - target_include_directories(openssl INTERFACE ${OPENSSL_INCLUDE_DIRS}) + endif() endif() - +build_external(openssl + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env + CC=${deps_cc} + CFLAGS=${deps_CFLAGS} + ${openssl_extra_env} + ${cross_extra} + ./Configure --prefix=${DEPS_DESTDIR} --libdir=lib ${openssl_extra_opts} + no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost + no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 + no-static-engine no-tests no-weak-ssl-ciphers no-zlib no-zlib-dynamic + INSTALL_COMMAND make install_sw + BUILD_BYPRODUCTS + ${DEPS_DESTDIR}/lib/libssl.a ${DEPS_DESTDIR}/lib/libcrypto.a + ${DEPS_DESTDIR}/include/openssl/ssl.h ${DEPS_DESTDIR}/include/openssl/crypto.h +) +add_static_target(OpenSSL::SSL openssl_external libssl.a) +add_static_target(OpenSSL::Crypto openssl_external libcrypto.a) +target_link_libraries(OpenSSL::SSL INTERFACE OpenSSL::Crypto) +set(OPENSSL_ROOT_DIR ${DEPS_DESTDIR} CACHE PATH "" FORCE) + +# NOTE: FindOpenSSL defines OPENSSL_INCLUDE_DIR, we use pkg_check_modules which defines +# OPENSSL_INCLUDE_DIRS, we define both for maximum compatibility... +set(OPENSSL_INCLUDE_DIRS ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) +set(OPENSSL_INCLUDE_DIR ${DEPS_DESTDIR}/include CACHE PATH "" FORCE) + +# NOTE: Create target so then libsession_system_or_submodule exports the +# :: target so that we can universally use that target for both PkgConfig +# and non-pkconfig versions of OpenSSL. +add_library(openssl INTERFACE) +target_link_libraries(openssl INTERFACE OpenSSL::SSL OpenSSL::Crypto) +target_include_directories(openssl INTERFACE ${OPENSSL_INCLUDE_DIRS}) From 8673e75bc22e65d7d8a181e250e31e3db7336457 Mon Sep 17 00:00:00 2001 From: doylet Date: Wed, 14 Jan 2026 17:21:52 +1100 Subject: [PATCH 64/72] Use apple arch flags from oxen-libquic staticbuild --- external/sqlcipher-cmake/CMakeLists.txt | 49 ++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 2017fce0..152ee961 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -72,13 +72,52 @@ endif() set(deps_CFLAGS "-O2") set(deps_CXXFLAGS "-O2") +set(apple_cflags_arch) +set(apple_cxxflags_arch) +set(apple_ldflags_arch) +if(APPLE AND CMAKE_CROSSCOMPILING) + if(cross_host MATCHES "^(.*-.*-)ios([0-9.]+)(-.*)?$") + set(cross_host "${CMAKE_MATCH_1}darwin${CMAKE_MATCH_2}${CMAKE_MATCH_3}") + endif() + if(cross_host MATCHES "^(.*-.*-.*)-simulator$") + set(cross_host "${CMAKE_MATCH_1}") + endif() + + set(apple_arch) + if(ARCH_TRIPLET MATCHES "^(arm|aarch)64.*") + set(apple_arch "arm64") + elseif(ARCH_TRIPLET MATCHES "^x86_64.*") + set(apple_arch "x86_64") + else() + message(FATAL_ERROR "Don't know how to specify -arch for GMP for ${ARCH_TRIPLET} (${APPLE_TARGET_TRIPLE})") + endif() + + set(apple_cflags_arch " -arch ${apple_arch}") + set(apple_cxxflags_arch " -arch ${apple_arch}") + if(CMAKE_OSX_DEPLOYMENT_TARGET) + if (SDK_NAME) + set(apple_ldflags_arch " -m${SDK_NAME}-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + elseif(CMAKE_OSX_DEPLOYMENT_TARGET) + set(apple_ldflags_arch " -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() + endif() + set(apple_ldflags_arch "${apple_ldflags_arch} -arch ${apple_arch}") + + if(CMAKE_OSX_SYSROOT) + foreach(f c cxx ld) + set(apple_${f}flags_arch "${apple_${f}flags_arch} -isysroot ${CMAKE_OSX_SYSROOT}") + endforeach() + endif() +elseif(cross_host STREQUAL "" AND CMAKE_LIBRARY_ARCHITECTURE) + set(cross_host "--build=${CMAKE_LIBRARY_ARCHITECTURE}") +endif() set(sqlcipher_ldflags "") set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown") if(APPLE) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") - set(sqlcipher_ldflags "-framework Security -framework Foundation -framework CoreFoundation") + set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS}") # NOTE: For example libraries are "/libssl.so;//libcrypto.so" the semicolon confuses @@ -86,9 +125,9 @@ else() # needed if we choose OpenSSL as our encryption library for SQLCipher. string(REPLACE ";" " -l" OPENSSL_LIBRARIES_STR "-lcrypto -lssl}") if (TARGET openssl_external) - set(sqlcipher_ldflags "-lm -L${OPENSSL_ROOT_DIR}/lib -lcrypto -lssl") + set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm -L${OPENSSL_ROOT_DIR}/lib -lcrypto -lssl") else() - set(sqlcipher_ldflags "-lm -lcrypto -lssl") + set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm -lcrypto -lssl") endif() if(WIN32) @@ -112,8 +151,8 @@ ExternalProject_Add(sqlcipher_external CONFIGURE_COMMAND ${CMAKE_CURRENT_LIST_DIR}/../sqlcipher/configure ${cross_host} --disable-shared --enable-static --disable-tcl --disable-readline --fts5 --with-tempstore=always - --prefix=${SQLCIPHER_PREFIX} CC=${deps_cc} CFLAGS=${sqlcipher_cflags} - LDFLAGS=${sqlcipher_ldflags} ${cross_rc} + --prefix=${SQLCIPHER_PREFIX} CC=${deps_cc} CFLAGS=${apple_cflags_arch}${sqlcipher_cflags} + LDFLAGS=${apple_ldflags_arch}${sqlcipher_ldflags} ${cross_rc} BUILD_COMMAND make libsqlite3.a BUILD_BYPRODUCTS ${SQLCIPHER_LIBDIR}/libsqlite3.a From c4a9f444f3fcef2dced43130f2eb09fb22de2933 Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 15 Jan 2026 11:49:35 +1100 Subject: [PATCH 65/72] Explicitly install just the SQLCipher headers and library This prevents the install target building the shell which does not work on some targets like iOS because the shell uses `system()` which is disallowed in the simulator for instance. --- external/sqlcipher-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 152ee961..5193dc8c 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -154,6 +154,7 @@ ExternalProject_Add(sqlcipher_external --prefix=${SQLCIPHER_PREFIX} CC=${deps_cc} CFLAGS=${apple_cflags_arch}${sqlcipher_cflags} LDFLAGS=${apple_ldflags_arch}${sqlcipher_ldflags} ${cross_rc} BUILD_COMMAND make libsqlite3.a + INSTALL_COMMAND make install-lib install-headers BUILD_BYPRODUCTS ${SQLCIPHER_LIBDIR}/libsqlite3.a ) From 36962393387119a033fd26fad74b3aa41f79dd42 Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 15 Jan 2026 13:45:02 +1100 Subject: [PATCH 66/72] TEMP: Try not matching against android x86 --- external/openssl-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 4d7219d8..7432913d 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -157,7 +157,7 @@ if(CMAKE_CROSSCOMPILING) set(openssl_extra_opts no-asm) set(openssl_extra_env SYSTEM=Linux) set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot") - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64 OR CMAKE_ANDROID_ARCH_ABI MATCHES x86) + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) # NOTE: Sysroot isn't sufficient to find the asm/ folder sitting in the host-tagged folder # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/linux/types.h:9:10: fatal error: 'asm/types.h' file not found # At From e07b4e4927382f1545251e943f4cb5304ab2b50a Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 15 Jan 2026 14:11:35 +1100 Subject: [PATCH 67/72] OpenSSL build only static libs and install development items _only_ --- external/openssl-cmake/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 7432913d..82956abd 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -157,7 +157,7 @@ if(CMAKE_CROSSCOMPILING) set(openssl_extra_opts no-asm) set(openssl_extra_env SYSTEM=Linux) set(deps_CFLAGS "${deps_CFLAGS} --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot") - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64 OR CMAKE_ANDROID_ARCH_ABI MATCHES x86) # NOTE: Sysroot isn't sufficient to find the asm/ folder sitting in the host-tagged folder # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/linux/types.h:9:10: fatal error: 'asm/types.h' file not found # At @@ -176,7 +176,8 @@ build_external(openssl no-shared no-capieng no-dso no-dtls1 no-ec_nistp_64_gcc_128 no-gost no-heartbeats no-md2 no-rc5 no-rdrand no-rfc3779 no-sctp no-ssl-trace no-ssl2 no-ssl3 no-static-engine no-tests no-weak-ssl-ciphers no-zlib no-zlib-dynamic - INSTALL_COMMAND make install_sw + BUILD_COMMAND make build_libs + INSTALL_COMMAND make install_dev BUILD_BYPRODUCTS ${DEPS_DESTDIR}/lib/libssl.a ${DEPS_DESTDIR}/lib/libcrypto.a ${DEPS_DESTDIR}/include/openssl/ssl.h ${DEPS_DESTDIR}/include/openssl/crypto.h From 35651fd79c40346af8ac6a3255cda7ce83cbb35f Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 15 Jan 2026 15:06:35 +1100 Subject: [PATCH 68/72] Apparently Android NDK does not have libmath, it's included in GLIBC itself --- external/sqlcipher-cmake/CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 5193dc8c..1677088b 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -119,17 +119,6 @@ if(APPLE) set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_CC") set(sqlcipher_ldflags "${sqlcipher_ldflags} -framework Security -framework Foundation -framework CoreFoundation") else() - set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS}") - # NOTE: For example libraries are "/libssl.so;//libcrypto.so" the semicolon confuses - # the linker so we remove the delimiter to make it compliant. Also note that -lm (math) is - # needed if we choose OpenSSL as our encryption library for SQLCipher. - string(REPLACE ";" " -l" OPENSSL_LIBRARIES_STR "-lcrypto -lssl}") - if (TARGET openssl_external) - set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm -L${OPENSSL_ROOT_DIR}/lib -lcrypto -lssl") - else() - set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm -lcrypto -lssl") - endif() - if(WIN32) # TODO: I'm unable to stop SQLCipher from building sqlite3.exe in the mingw cross-compile # so we need to augment that executable with some Window's libraries to get us through... @@ -139,7 +128,18 @@ else() if(ANDROID) # NOTE: SQLite uses __android_log_print and __android_log_write which is in their liblog library set(sqlcipher_ldflags "${sqlcipher_ldflags} -llog") + else() + # NOTE: Apparently Android doesn't ship libmath, it's included in glibc itself + set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm") endif() + + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS}") + if (TARGET openssl_external) + set(sqlcipher_ldflags "${sqlcipher_ldflags} -L${OPENSSL_ROOT_DIR}/lib -lcrypto -lssl") + else() + set(sqlcipher_ldflags "${sqlcipher_ldflags} -lcrypto -lssl") + endif() + endif() ExternalProject_Add(sqlcipher_external From 909e95588e753c89281f5348f559da71db5c3c03 Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 15 Jan 2026 15:17:43 +1100 Subject: [PATCH 69/72] Test removing -lm entirely from SQLCipher --- external/sqlcipher-cmake/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 1677088b..0d0f06a1 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -128,9 +128,6 @@ else() if(ANDROID) # NOTE: SQLite uses __android_log_print and __android_log_write which is in their liblog library set(sqlcipher_ldflags "${sqlcipher_ldflags} -llog") - else() - # NOTE: Apparently Android doesn't ship libmath, it's included in glibc itself - set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm") endif() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS}") From 0ddff803b11a4c90bc1d3bb42df1a383ec250256 Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 15 Jan 2026 16:07:05 +1100 Subject: [PATCH 70/72] Assist SQLCipher in finding the liblog directory --- external/openssl-cmake/CMakeLists.txt | 6 +++++- external/sqlcipher-cmake/CMakeLists.txt | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 82956abd..44791f79 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -162,7 +162,11 @@ if(CMAKE_CROSSCOMPILING) # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/linux/types.h:9:10: fatal error: 'asm/types.h' file not found # At # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/x86_64-linux-android/asm/ - set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/include/x86_64-linux-android") + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) + set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/include/x86_64-linux-android") + elseif() + set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/include/i686-linux-android") + endif() endif() endif() endif() diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 0d0f06a1..5d7d75fe 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -126,6 +126,12 @@ else() endif() if(ANDROID) + # NOTE: x86 build seems to have trouble finding -llog, help it find it by specifying the + # library path + if(CMAKE_ANDROID_ARCH_ABI MATCHES x86) + set(sqlcipher_ldflags "${sqlcipher_ldflags} -L${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/lib/i686-linux-android/${ANDROID_PLATFORM_LEVEL} -lm") + endif() + # NOTE: SQLite uses __android_log_print and __android_log_write which is in their liblog library set(sqlcipher_ldflags "${sqlcipher_ldflags} -llog") endif() From 76452c2d6255e2aff62f8732ba5432b618f2e9a6 Mon Sep 17 00:00:00 2001 From: doylet Date: Thu, 15 Jan 2026 16:56:07 +1100 Subject: [PATCH 71/72] All Android targets need help finding the liblog folder for SQLCipher --- external/openssl-cmake/CMakeLists.txt | 2 +- external/sqlcipher-cmake/CMakeLists.txt | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/external/openssl-cmake/CMakeLists.txt b/external/openssl-cmake/CMakeLists.txt index 44791f79..95f12070 100644 --- a/external/openssl-cmake/CMakeLists.txt +++ b/external/openssl-cmake/CMakeLists.txt @@ -164,7 +164,7 @@ if(CMAKE_CROSSCOMPILING) # /usr/lib/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/x86_64-linux-android/asm/ if(CMAKE_ANDROID_ARCH_ABI MATCHES x86_64) set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/include/x86_64-linux-android") - elseif() + else() set(deps_CFLAGS "${deps_CFLAGS} -I${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/include/i686-linux-android") endif() endif() diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index 5d7d75fe..a2539ef2 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -126,14 +126,12 @@ else() endif() if(ANDROID) - # NOTE: x86 build seems to have trouble finding -llog, help it find it by specifying the - # library path - if(CMAKE_ANDROID_ARCH_ABI MATCHES x86) - set(sqlcipher_ldflags "${sqlcipher_ldflags} -L${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/lib/i686-linux-android/${ANDROID_PLATFORM_LEVEL} -lm") - endif() + # NOTE: SQLCipher seems to have trouble finding liblog and libm which is in this folder here + # in the sysroot + set(sqlcipher_ldflags "${sqlcipher_ldflags} -L${ANDROID_TOOLCHAIN_ROOT}/sysroot/usr/lib/${android_toolchain_prefix}-${android_toolchain_suffix}/${ANDROID_PLATFORM_LEVEL}") # NOTE: SQLite uses __android_log_print and __android_log_write which is in their liblog library - set(sqlcipher_ldflags "${sqlcipher_ldflags} -llog") + set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm -llog") endif() set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS}") From 1258890c96cb24db5676203031a215f429f1f275 Mon Sep 17 00:00:00 2001 From: doylet Date: Fri, 16 Jan 2026 10:09:20 +1100 Subject: [PATCH 72/72] Add Android NDK sysroot to sqlcipher build --- external/sqlcipher-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/sqlcipher-cmake/CMakeLists.txt b/external/sqlcipher-cmake/CMakeLists.txt index a2539ef2..1df18d8e 100644 --- a/external/sqlcipher-cmake/CMakeLists.txt +++ b/external/sqlcipher-cmake/CMakeLists.txt @@ -134,7 +134,7 @@ else() set(sqlcipher_ldflags "${sqlcipher_ldflags} -lm -llog") endif() - set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS}") + set(sqlcipher_cflags "${sqlcipher_cflags} -DSQLCIPHER_CRYPTO_OPENSSL -I${OPENSSL_INCLUDE_DIRS} --sysroot=${ANDROID_TOOLCHAIN_ROOT}/sysroot") if (TARGET openssl_external) set(sqlcipher_ldflags "${sqlcipher_ldflags} -L${OPENSSL_ROOT_DIR}/lib -lcrypto -lssl") else()