feat: receive pre-messages and adapt download on demand

deltachat-ffi/deltachat.h

@@ -488,11 +488,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    0=use IMAP IDLE if the server supports it.
  *                    This is a developer option used for testing polling used as an IDLE fallback.
  * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
- *                    For larger messages, only the header is downloaded and a placeholder is shown.
+ *                    For messages with large attachments, two messages are sent:
+ *                    a Pre-Message containing metadata and a Post-Message containing the attachment.
+ *                    Pre-Messages are always downloaded and show a placeholder message.
  *                    These messages can be downloaded fully using dc_download_full_msg() later.
- *                    The limit is compared against raw message sizes, including headers.
- *                    The actually used limit may be corrected
- *                    to not mess up with non-delivery-reports or read-receipts.
+ *                    Post-Messages are automatically downloaded if they are smaller than the download_limit.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
  * - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
@@ -4310,7 +4310,8 @@ char*             dc_msg_get_webxdc_info      (const dc_msg_t* msg);
 
 /**
  * Get the size of the file. Returns the size of the file associated with a
- * message, if applicable.
+ * message, if applicable. 
+ * If message is a pre-message, then this returns size of the to be downloaded file.
  *
  * Typically, this is used to show the size of document files, e.g. a PDF.
  *

deltachat-jsonrpc/src/api/types/message.rs

@@ -92,6 +92,9 @@ pub struct MessageObject {
 
     file: Option<String>,
     file_mime: Option<String>,
+
+    /// The size of the file in bytes, if applicable.
+    /// If message is a pre-message, then this is the size of the to be downloaded file.
     file_bytes: u64,
     file_name: Option<String>,
 

src/chat.rs

@@ -12,6 +12,7 @@ use std::time::Duration;
 use anyhow::{Context as _, Result, anyhow, bail, ensure};
 use chrono::TimeZone;
 use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line};
+use humansize::{BINARY, format_size};
 use mail_builder::mime::MimePart;
 use serde::{Deserialize, Serialize};
 use strum_macros::EnumIter;
@@ -2730,10 +2731,10 @@ async fn prepare_send_msg(
     Ok(row_ids)
 }
 
-/// Renders the message or Full-Message and Pre-Message.
+/// Renders the Message or splits it into Post-Message and Pre-Message.
 ///
-/// Pre-Message is a small message with metadata which announces a larger Full-Message.
-/// Full messages are not downloaded in the background.
+/// Pre-Message is a small message with metadata which announces a larger Post-Message.
+/// Post-Messages are not downloaded in the background.
 ///
 /// If pre-message is not nessesary this returns a normal message instead.
 async fn render_mime_message_and_pre_message(
@@ -2750,9 +2751,14 @@ async fn render_mime_message_and_pre_message(
             > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
 
     if needs_pre_message {
-        let mut mimefactory_full_msg = mimefactory.clone();
-        mimefactory_full_msg.set_as_full_message();
-        let rendered_msg = mimefactory_full_msg.render(context).await?;
+        info!(
+            context,
+            "Message is large and will be split into a pre- and a post-message.",
+        );
+
+        let mut mimefactory_post_msg = mimefactory.clone();
+        mimefactory_post_msg.set_as_post_message();
+        let rendered_msg = mimefactory_post_msg.render(context).await?;
 
         let mut mimefactory_pre_msg = mimefactory;
         mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg);
@@ -2856,6 +2862,21 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             }
         }?;
 
+    if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) {
+        info!(
+            context,
+            "Message Sizes: Pre-Message {}; Post-Message: {}",
+            format_size(pre_msg.message.len(), BINARY),
+            format_size(post_msg.message.len(), BINARY)
+        );
+    } else {
+        info!(
+            context,
+            "Message will be sent as normal message (no pre- and post message). Size: {}",
+            format_size(rendered_msg.message.len(), BINARY)
+        );
+    }
+
     if needs_encryption && !rendered_msg.is_encrypted {
         /* unrecoverable */
         message::set_msg_failed(
@@ -4319,6 +4340,14 @@ pub async fn forward_msgs_2ctx(
             msg.viewtype = Viewtype::Text;
         }
 
+        if msg.download_state != DownloadState::Done {
+            // we don't use Message.get_text() here,
+            // because it may change in future,
+            // when UI shows this info itself,
+            // then the additional_text will not be added in get_text anymore.
+            msg.text += &msg.additional_text;
+        }
+
         let param = &mut param;
         msg.param.steal(param, Param::File);
         msg.param.steal(param, Param::Filename);
@@ -4329,6 +4358,7 @@ pub async fn forward_msgs_2ctx(
         msg.param.steal(param, Param::ProtectQuote);
         msg.param.steal(param, Param::Quote);
         msg.param.steal(param, Param::Summary1);
+
         msg.in_reply_to = None;
 
         // do not leak data as group names; a default subject is generated by mimefactory
@@ -4395,31 +4425,42 @@ pub(crate) async fn save_copy_in_self_talk(
     msg.param.remove(Param::WebxdcDocumentTimestamp);
     msg.param.remove(Param::WebxdcSummary);
     msg.param.remove(Param::WebxdcSummaryTimestamp);
+    msg.param.remove(Param::PostMessageFileBytes);
+    msg.param.remove(Param::PostMessageViewtype);
+
+    if msg.download_state != DownloadState::Done {
+        // we don't use Message.get_text() here,
+        // because it may change in future,
+        // when UI shows this info itself,
+        // then the additional_text will not be added in get_text anymore.
+        msg.text += &msg.additional_text;
+    }
 
     if !msg.original_msg_id.is_unset() {
         bail!("message already saved.");
     }
 
-    let copy_fields = "from_id, to_id, timestamp_rcvd, type, txt,
+    let copy_fields = "from_id, to_id, timestamp_rcvd, type,
                        mime_modified, mime_headers, mime_compressed, mime_in_reply_to, subject, msgrmsg";
     let row_id = context
         .sql
         .insert(
             &format!(
                 "INSERT INTO msgs ({copy_fields},
                                    timestamp_sent,
-                                   chat_id, rfc724_mid, state, timestamp, param, starred)
+                                   txt, chat_id, rfc724_mid, state, timestamp, param, starred)
                  SELECT            {copy_fields},
                                    -- Outgoing messages on originating device
                                    -- have timestamp_sent == 0.
                                    -- We copy sort timestamp instead
                                    -- so UIs display the same timestamp
                                    -- for saved and original message.
                                    IIF(timestamp_sent == 0, timestamp, timestamp_sent),
-                                   ?, ?, ?, ?, ?, ?
+                                   ?, ?, ?, ?, ?, ?, ?
                  FROM msgs WHERE id=?;"
             ),
             (
+                msg.text,
                 dest_chat_id,
                 dest_rfc724_mid,
                 if msg.from_id == ContactId::SELF {

src/context.rs

@@ -622,6 +622,12 @@ impl Context {
             {
                 warn!(self, "Failed to update quota: {err:#}.");
             }
+
+            // OPTIONAL TODO: if time left start downloading messages
+            // while (msg = download_when_normal_starts) {
+            //  if not time_left {break;}
+            //  connection.download_message(msg) }
+            // }
         }
 
         info!(
@@ -1091,13 +1097,6 @@ impl Context {
                 .await?
                 .unwrap_or_default(),
         );
-        res.insert(
-            "fail_on_receiving_full_msg",
-            self.sql
-                .get_raw_config("fail_on_receiving_full_msg")
-                .await?
-                .unwrap_or_default(),
-        );
         res.insert(
             "std_header_protection_composing",
             self.sql