diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index b540cc5339..5379ec6782 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -488,11 +488,11 @@ char* dc_get_blobdir (const dc_context_t* context); * 0=use IMAP IDLE if the server supports it. * This is a developer option used for testing polling used as an IDLE fallback. * - `download_limit` = Messages up to this number of bytes are downloaded automatically. - * For larger messages, only the header is downloaded and a placeholder is shown. + * For messages with large attachments, two messages are sent: + * a Pre-Message containing metadata and a Post-Message containing the attachment. + * Pre-Messages are always downloaded and show a placeholder message. * These messages can be downloaded fully using dc_download_full_msg() later. - * The limit is compared against raw message sizes, including headers. - * The actually used limit may be corrected - * to not mess up with non-delivery-reports or read-receipts. + * Post-Messages are automatically downloaded if they are smaller than the download_limit. * 0=no limit (default). * Changes affect future messages only. * - `protect_autocrypt` = Enable Header Protection for Autocrypt header. @@ -4310,7 +4310,8 @@ char* dc_msg_get_webxdc_info (const dc_msg_t* msg); /** * Get the size of the file. Returns the size of the file associated with a - * message, if applicable. + * message, if applicable. + * If message is a pre-message, then this returns size of the to be downloaded file. * * Typically, this is used to show the size of document files, e.g. a PDF. * @@ -7263,22 +7264,9 @@ void dc_event_unref(dc_event_t* event); /// `%1$s` will be replaced by the percentage used #define DC_STR_QUOTA_EXCEEDING_MSG_BODY 98 -/// "%1$s message" -/// -/// Used as the message body when a message -/// was not yet downloaded completely -/// (dc_msg_get_download_state() is e.g. @ref DC_DOWNLOAD_AVAILABLE). -/// -/// `%1$s` will be replaced by human-readable size (e.g. "1.2 MiB"). +/// @deprecated Deprecated 2025-11-12, this string is no longer needed. #define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY 99 -/// "Download maximum available until %1$s" -/// -/// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY. -/// -/// `%1$s` will be replaced by human-readable date and time. -#define DC_STR_DOWNLOAD_AVAILABILITY 100 - /// "Multi Device Synchronization" /// /// Used in subjects of outgoing sync messages. diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index 567e3ffdc9..27ba8dca0e 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -92,6 +92,9 @@ pub struct MessageObject { file: Option, file_mime: Option, + + /// The size of the file in bytes, if applicable. + /// If message is a pre-message, then this is the size of the to be downloaded file. file_bytes: u64, file_name: Option, diff --git a/deltachat-rpc-client/tests/test_chatlist_events.py b/deltachat-rpc-client/tests/test_chatlist_events.py index 05bec0795f..f13769e071 100644 --- a/deltachat-rpc-client/tests/test_chatlist_events.py +++ b/deltachat-rpc-client/tests/test_chatlist_events.py @@ -1,7 +1,5 @@ from __future__ import annotations -import base64 -import os from typing import TYPE_CHECKING from deltachat_rpc_client import Account, EventType, const @@ -129,7 +127,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None: msg.get_snapshot().chat.accept() bob.get_chat_by_id(chat_id).send_message( "Hello World, this message is bigger than 5 bytes", - html=base64.b64encode(os.urandom(300000)).decode("utf-8"), + file="../test-data/image/screenshot.jpg", ) message = alice.wait_for_incoming_msg() diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index b6337e01c2..8d80ab12a3 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -5,13 +5,12 @@ import os import socket import subprocess -import time from unittest.mock import MagicMock import pytest -from deltachat_rpc_client import Contact, EventType, Message, events -from deltachat_rpc_client.const import DownloadState, MessageState +from deltachat_rpc_client import EventType, events +from deltachat_rpc_client.const import MessageState, DownloadState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError @@ -333,7 +332,7 @@ def test_receive_imf_failure(acfactory) -> None: alice_contact_bob = alice.create_contact(bob, "Bob") alice_chat_bob = alice_contact_bob.create_chat() - bob.set_config("fail_on_receiving_full_msg", "1") + bob.set_config("simulate_receive_imf_error", "1") alice_chat_bob.send_text("Hello!") event = bob.wait_for_event(EventType.MSGS_CHANGED) assert event.chat_id == bob.get_device_chat().id @@ -343,18 +342,17 @@ def test_receive_imf_failure(acfactory) -> None: version = bob.get_info()["deltachat_core_version"] assert ( snapshot.text == "❌ Failed to receive a message:" - " Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`." + " Condition failed: `!context.get_config_bool(Config::SimulateReceiveImfError).await?`." f" Core version {version}." " Please report this bug to delta@merlinux.eu or https://support.delta.chat/." ) # The failed message doesn't break the IMAP loop. - bob.set_config("fail_on_receiving_full_msg", "0") + bob.set_config("simulate_receive_imf_error", "0") alice_chat_bob.send_text("Hello again!") message = bob.wait_for_incoming_msg() snapshot = message.get_snapshot() assert snapshot.text == "Hello again!" - assert snapshot.download_state == DownloadState.DONE assert snapshot.error is None @@ -687,94 +685,6 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None: assert snapshot.show_padlock -def test_reaction_to_partially_fetched_msg(acfactory, tmp_path): - """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded - messages are received out of order". - - If the Inbox contains X small messages followed by Y large messages followed by Z small - messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages. - - This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced - with online test as follows: - - Bob enables download limit and goes offline. - - Alice sends a large message to Bob and reacts to this message with a thumbs-up. - - Bob goes online - - Bob first processes a reaction message and throws it away because there is no corresponding - message, then processes a partially downloaded message. - - As a result, Bob does not see a reaction - """ - download_limit = 300000 - ac1, ac2 = acfactory.get_online_accounts(2) - ac1_addr = ac1.get_config("addr") - chat = ac1.create_chat(ac2) - ac2.set_config("download_limit", str(download_limit)) - ac2.stop_io() - - logging.info("sending small+large messages from ac1 to ac2") - msgs = [] - msgs.append(chat.send_text("hi")) - path = tmp_path / "large" - path.write_bytes(os.urandom(download_limit + 1)) - msgs.append(chat.send_file(str(path))) - for m in msgs: - m.wait_until_delivered() - - logging.info("sending a reaction to the large message from ac1 to ac2") - # TODO: Find the reason of an occasional message reordering on the server (so that the reaction - # has a lower UID than the previous message). W/a is to sleep for some time to let the reaction - # have a later INTERNALDATE. - time.sleep(1.1) - react_str = "\N{THUMBS UP SIGN}" - msgs.append(msgs[-1].send_reaction(react_str)) - msgs[-1].wait_until_delivered() - - ac2.start_io() - - logging.info("wait for ac2 to receive a reaction") - msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id) - assert msg2.get_sender_contact().get_snapshot().address == ac1_addr - assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE - reactions = msg2.get_reactions() - contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact] - assert len(contacts) == 1 - assert contacts[0].get_snapshot().address == ac1_addr - assert list(reactions.reactions_by_contact.values())[0] == [react_str] - - -@pytest.mark.parametrize("n_accounts", [3, 2]) -def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts): - download_limit = 300000 - - alice, *others = acfactory.get_online_accounts(n_accounts) - bob = others[0] - - alice_group = alice.create_group("test group") - for account in others: - chat = account.create_chat(alice) - chat.send_text("Hello Alice!") - assert alice.wait_for_incoming_msg().get_snapshot().text == "Hello Alice!" - - contact = alice.create_contact(account) - alice_group.add_contact(contact) - - bob.set_config("download_limit", str(download_limit)) - - alice_group.send_text("hi") - snapshot = bob.wait_for_incoming_msg().get_snapshot() - assert snapshot.text == "hi" - bob_group = snapshot.chat - - path = tmp_path / "large" - path.write_bytes(os.urandom(download_limit + 1)) - - for i in range(10): - logging.info("Sending message %s", i) - alice_group.send_file(str(path)) - snapshot = bob.wait_for_incoming_msg().get_snapshot() - assert snapshot.download_state == DownloadState.AVAILABLE - assert snapshot.chat == bob_group - - def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages @@ -1152,3 +1062,21 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log): assert chat.num_contacts() == 2 assert msg.get_snapshot().chat.num_contacts() == 2 + + +def test_large_message(acfactory) -> None: + """ + Test sending large message without download limit set, + so it is sent with pre-message but downloaded without user interaction. + """ + alice, bob = acfactory.get_online_accounts(2) + + alice_chat_bob = alice.create_chat(bob) + alice_chat_bob.send_message( + "Hello World, this message is bigger than 5 bytes", + file="../test-data/image/screenshot.jpg", + ) + + msg = bob.wait_for_incoming_msg() + snapshot = msg.get_snapshot() + assert snapshot.text == "Hello World, this message is bigger than 5 bytes" diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 8eda148ddc..0ed3a75d68 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1,7 +1,6 @@ import os import queue import sys -import base64 from datetime import datetime, timezone import pytest @@ -221,38 +220,6 @@ def test_webxdc_huge_update(acfactory, data, lp): assert update["payload"] == payload -def test_webxdc_download_on_demand(acfactory, data, lp): - ac1, ac2 = acfactory.get_online_accounts(2) - acfactory.introduce_each_other([ac1, ac2]) - chat = acfactory.get_accepted_chat(ac1, ac2) - - msg1 = Message.new_empty(ac1, "webxdc") - msg1.set_text("message1") - msg1.set_file(data.get_path("webxdc/minimal.xdc")) - msg1 = chat.send_msg(msg1) - assert msg1.is_webxdc() - assert msg1.filename - - msg2 = ac2._evtracker.wait_next_incoming_message() - assert msg2.is_webxdc() - - lp.sec("ac2 sets download limit") - ac2.set_config("download_limit", "100") - assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data") - ac2_update = ac2._evtracker.wait_next_incoming_message() - assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE - assert not msg2.get_status_updates() - - ac2_update.download_full() - ac2._evtracker.get_matching("DC_EVENT_WEBXDC_STATUS_UPDATE") - assert msg2.get_status_updates() - - # Get a event notifying that the message disappeared from the chat. - msgs_changed_event = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert msgs_changed_event.data1 == msg2.chat.id - assert msgs_changed_event.data2 == 0 - - def test_enable_mvbox_move(acfactory, lp): (ac1,) = acfactory.get_online_accounts(1) diff --git a/src/calls/calls_tests.rs b/src/calls/calls_tests.rs index 3f983d8438..23b3947a2e 100644 --- a/src/calls/calls_tests.rs +++ b/src/calls/calls_tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::chat::forward_msgs; use crate::config::Config; use crate::constants::DC_CHAT_ID_TRASH; -use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; +use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, TestContextManager}; struct CallSetup { @@ -610,65 +610,3 @@ async fn test_end_text_call() -> Result<()> { Ok(()) } - -/// Tests that partially downloaded "call ended" -/// messages are not processed. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_no_partial_calls() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - - let seen = false; - - // The messages in the test - // have no `Date` on purpose, - // so they are treated as new. - let received_call = receive_imf( - alice, - b"From: bob@example.net\n\ - To: alice@example.org\n\ - Message-ID: \n\ - Chat-Version: 1.0\n\ - Chat-Content: call\n\ - Chat-Webrtc-Room: YWFhYWFhYWFhCg==\n\ - \n\ - Hello, this is a call\n", - seen, - ) - .await? - .unwrap(); - assert_eq!(received_call.msg_ids.len(), 1); - let call_msg = Message::load_from_db(alice, received_call.msg_ids[0]) - .await - .unwrap(); - assert_eq!(call_msg.viewtype, Viewtype::Call); - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting); - - let imf_raw = b"From: bob@example.net\n\ - To: alice@example.org\n\ - Message-ID: \n\ - In-Reply-To: \n\ - Chat-Version: 1.0\n\ - Chat-Content: call-ended\n\ - \n\ - Call ended\n"; - receive_imf_from_inbox( - alice, - "second@example.net", - imf_raw, - seen, - Some(imf_raw.len().try_into().unwrap()), - ) - .await?; - - // The call is still not ended. - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting); - - // Fully downloading the message ends the call. - receive_imf_from_inbox(alice, "second@example.net", imf_raw, seen, None) - .await - .context("Failed to fully download end call message")?; - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Missed); - - Ok(()) -} diff --git a/src/chat.rs b/src/chat.rs index 533cf8c760..1973eb0075 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -12,6 +12,7 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail, ensure}; use chrono::TimeZone; use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line}; +use humansize::{BINARY, format_size}; use mail_builder::mime::MimePart; use serde::{Deserialize, Serialize}; use strum_macros::EnumIter; @@ -27,7 +28,9 @@ use crate::constants::{ use crate::contact::{self, Contact, ContactId, Origin}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc; -use crate::download::DownloadState; +use crate::download::{ + DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, PRE_MSG_SIZE_WARNING_THRESHOLD, +}; use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers}; use crate::events::EventType; use crate::key::self_fingerprint; @@ -35,7 +38,7 @@ use crate::location; use crate::log::{LogExt, warn}; use crate::logged_debug_assert; use crate::message::{self, Message, MessageState, MsgId, Viewtype}; -use crate::mimefactory::MimeFactory; +use crate::mimefactory::{MimeFactory, RenderedEmail}; use crate::mimeparser::SystemMessage; use crate::param::{Param, Params}; use crate::receive_imf::ReceivedMsg; @@ -2732,6 +2735,57 @@ async fn prepare_send_msg( Ok(row_ids) } +/// Renders the Message or splits it into Post-Message and Pre-Message. +/// +/// Pre-Message is a small message with metadata which announces a larger Post-Message. +/// Post-Messages are not downloaded in the background. +/// +/// If pre-message is not nessesary this returns a normal message instead. +async fn render_mime_message_and_pre_message( + context: &Context, + msg: &mut Message, + mimefactory: MimeFactory, +) -> Result<(RenderedEmail, Option)> { + let needs_pre_message = msg.viewtype.has_file() + && mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages + && msg + .get_filebytes(context) + .await? + .context("filebytes not available, even though message has attachment")? + > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; + + if needs_pre_message { + info!( + context, + "Message is large and will be split into a pre- and a post-message.", + ); + + let mut mimefactory_post_msg = mimefactory.clone(); + mimefactory_post_msg.set_as_post_message(); + let rendered_msg = mimefactory_post_msg.render(context).await?; + + let mut mimefactory_pre_msg = mimefactory; + mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); + let rendered_pre_msg = mimefactory_pre_msg + .render(context) + .await + .context("pre-message failed to render")?; + + if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD { + warn!( + context, + "Pre-message for message (MsgId={}) is larger than expected: {}.", + msg.id, + rendered_pre_msg.message.len() + ); + } + + Ok((rendered_msg, Some(rendered_pre_msg))) + } else { + Ok((mimefactory.render(context).await?, None)) + } +} + /// Constructs jobs for sending a message and inserts them into the appropriate table. /// /// Updates the message `GuaranteeE2ee` parameter and persists it @@ -2803,13 +2857,29 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - return Ok(Vec::new()); } - let rendered_msg = match mimefactory.render(context).await { - Ok(res) => Ok(res), - Err(err) => { - message::set_msg_failed(context, msg, &err.to_string()).await?; - Err(err) - } - }?; + let (rendered_msg, rendered_pre_msg) = + match render_mime_message_and_pre_message(context, msg, mimefactory).await { + Ok(res) => Ok(res), + Err(err) => { + message::set_msg_failed(context, msg, &err.to_string()).await?; + Err(err) + } + }?; + + if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) { + info!( + context, + "Message Sizes: Pre-Message {}; Post-Message: {}", + format_size(pre_msg.message.len(), BINARY), + format_size(post_msg.message.len(), BINARY) + ); + } else { + info!( + context, + "Message will be sent as normal message (no pre- and post message). Size: {}", + format_size(rendered_msg.message.len(), BINARY) + ); + } if needs_encryption && !rendered_msg.is_encrypted { /* unrecoverable */ @@ -2868,12 +2938,26 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - } else { for recipients_chunk in recipients.chunks(chunk_size) { let recipients_chunk = recipients_chunk.join(" "); + // send pre-message before actual message + if let Some(pre_msg) = &rendered_pre_msg { + let row_id = t.execute( + "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) + VALUES (?1, ?2, ?3, ?4)", + ( + &pre_msg.rfc724_mid, + &recipients_chunk, + &pre_msg.message, + msg.id, + ), + )?; + row_ids.push(row_id.try_into()?); + } let row_id = t.execute( - "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \ + "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) VALUES (?1, ?2, ?3, ?4)", ( &rendered_msg.rfc724_mid, - recipients_chunk, + &recipients_chunk, &rendered_msg.message, msg.id, ), @@ -4260,6 +4344,14 @@ pub async fn forward_msgs_2ctx( msg.viewtype = Viewtype::Text; } + if msg.download_state != DownloadState::Done { + // we don't use Message.get_text() here, + // because it may change in future, + // when UI shows this info itself, + // then the additional_text will not be added in get_text anymore. + msg.text += &msg.additional_text; + } + let param = &mut param; msg.param.steal(param, Param::File); msg.param.steal(param, Param::Filename); @@ -4336,12 +4428,22 @@ pub(crate) async fn save_copy_in_self_talk( msg.param.remove(Param::WebxdcDocumentTimestamp); msg.param.remove(Param::WebxdcSummary); msg.param.remove(Param::WebxdcSummaryTimestamp); + msg.param.remove(Param::PostMessageFileBytes); + msg.param.remove(Param::PostMessageViewtype); + + if msg.download_state != DownloadState::Done { + // we don't use Message.get_text() here, + // because it may change in future, + // when UI shows this info itself, + // then the additional_text will not be added in get_text anymore. + msg.text += &msg.additional_text; + } if !msg.original_msg_id.is_unset() { bail!("message already saved."); } - let copy_fields = "from_id, to_id, timestamp_rcvd, type, txt, + let copy_fields = "from_id, to_id, timestamp_rcvd, type, mime_modified, mime_headers, mime_compressed, mime_in_reply_to, subject, msgrmsg"; let row_id = context .sql @@ -4349,7 +4451,7 @@ pub(crate) async fn save_copy_in_self_talk( &format!( "INSERT INTO msgs ({copy_fields}, timestamp_sent, - chat_id, rfc724_mid, state, timestamp, param, starred) + txt, chat_id, rfc724_mid, state, timestamp, param, starred) SELECT {copy_fields}, -- Outgoing messages on originating device -- have timestamp_sent == 0. @@ -4357,10 +4459,11 @@ pub(crate) async fn save_copy_in_self_talk( -- so UIs display the same timestamp -- for saved and original message. IIF(timestamp_sent == 0, timestamp, timestamp_sent), - ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ? FROM msgs WHERE id=?;" ), ( + msg.text, dest_chat_id, dest_rfc724_mid, if msg.from_id == ContactId::SELF { diff --git a/src/chat/chat_tests.rs b/src/chat/chat_tests.rs index a93ab2179a..1e8cff82fb 100644 --- a/src/chat/chat_tests.rs +++ b/src/chat/chat_tests.rs @@ -3116,7 +3116,7 @@ async fn test_broadcast_channel_protected_listid() -> Result<()> { .await? .grpid; - let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes(), None).await?; + let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes()).await?; assert_eq!( parsed.get_mailinglist_header().unwrap(), format!("My Channel <{}>", alice_list_id) @@ -3311,7 +3311,7 @@ async fn test_leave_broadcast_multidevice() -> Result<()> { remove_contact_from_chat(bob0, bob_chat_id, ContactId::SELF).await?; let leave_msg = bob0.pop_sent_msg().await; - let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes(), None).await?; + let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes()).await?; assert_eq!(parsed.parts[0].msg, "I left the group."); let rcvd = bob1.recv_msg(&leave_msg).await; diff --git a/src/config.rs b/src/config.rs index 62feb6bca1..9d07990481 100644 --- a/src/config.rs +++ b/src/config.rs @@ -438,14 +438,14 @@ pub enum Config { /// using this still run unmodified code. TestHooks, - /// Return an error from `receive_imf_inner()` for a fully downloaded message. For tests. - FailOnReceivingFullMsg, - /// Enable composing emails with Header Protection as defined in /// "Header Protection for Cryptographically /// Protected Email". #[strum(props(default = "1"))] StdHeaderProtectionComposing, + + /// Return an error from `receive_imf_inner()`. For tests. + SimulateReceiveImfError, } impl Config { diff --git a/src/context.rs b/src/context.rs index afff22e5ab..5e2be572a0 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1083,13 +1083,6 @@ impl Context { .await? .unwrap_or_default(), ); - res.insert( - "fail_on_receiving_full_msg", - self.sql - .get_raw_config("fail_on_receiving_full_msg") - .await? - .unwrap_or_default(), - ); res.insert( "std_header_protection_composing", self.sql diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 4a20c3af37..c4ffc1648a 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -297,6 +297,7 @@ async fn test_get_info_completeness() { "encrypted_device_token", "stats_last_update", "stats_last_old_contact_id", + "simulate_receive_imf_error", // only used in tests ]; let t = TestContext::new().await; let info = t.get_info().await.unwrap(); diff --git a/src/download.rs b/src/download.rs index cddd0fff4c..09e8d8869d 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,27 +1,18 @@ //! # Download large messages manually. -use std::cmp::max; use std::collections::BTreeMap; use anyhow::{Result, anyhow, bail, ensure}; use deltachat_derive::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use crate::config::Config; use crate::context::Context; use crate::imap::session::Session; -use crate::message::{Message, MsgId, Viewtype}; -use crate::mimeparser::{MimeMessage, Part}; -use crate::tools::time; -use crate::{EventType, chatlist_events, stock_str}; +use crate::log::warn; +use crate::message::{self, Message, MsgId, rfc724_mid_exists}; +use crate::{EventType, chatlist_events}; -/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`. -/// -/// For better UX, some messages as add-member, non-delivery-reports (NDN) or read-receipts (MDN) -/// should always be downloaded completely to handle them correctly, -/// also in larger groups and if group and contact avatar are attached. -/// Most of these cases are caught by `MIN_DOWNLOAD_LIMIT`. -pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840; +pub(crate) mod pre_msg_metadata; /// If a message is downloaded only partially /// and `delete_server_after` is set to small timeouts (eg. "at once"), @@ -29,6 +20,15 @@ pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840; /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case. pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60; +/// From this point onward outgoing messages are considered large +/// and get a Pre-Message, which announces the Post-Message. +// this is only about sending so we can modify it any time. +// current value is a bit less than the minimum auto download setting from the UIs (which is 160 KiB) +pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; + +/// Max size for pre messages. A warning is emitted when this is exceeded. +pub(crate) const PRE_MSG_SIZE_WARNING_THRESHOLD: usize = 150_000; + /// Download state of the message. #[derive( Debug, @@ -64,20 +64,8 @@ pub enum DownloadState { InProgress = 1000, } -impl Context { - // Returns validated download limit or `None` for "no limit". - pub(crate) async fn download_limit(&self) -> Result> { - let download_limit = self.get_config_int(Config::DownloadLimit).await?; - if download_limit <= 0 { - Ok(None) - } else { - Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32))) - } - } -} - impl MsgId { - /// Schedules full message download for partially downloaded message. + /// Schedules Post-Message download for partially downloaded message. pub async fn download_full(self, context: &Context) -> Result<()> { let msg = Message::load_from_db(context, self).await?; match msg.download_state() { @@ -86,11 +74,22 @@ impl MsgId { } DownloadState::InProgress => return Err(anyhow!("Download already in progress.")), DownloadState::Available | DownloadState::Failure => { + if msg.rfc724_mid().is_empty() { + return Err(anyhow!("Download not possible, message has no rfc724_mid")); + } self.update_download_state(context, DownloadState::InProgress) .await?; + info!( + context, + "Requesting full download of {:?}.", + msg.rfc724_mid() + ); context .sql - .execute("INSERT INTO download (msg_id) VALUES (?)", (self,)) + .execute( + "INSERT INTO download (rfc724_mid, msg_id) VALUES (?,?)", + (msg.rfc724_mid(), msg.id), + ) .await?; context.scheduler.interrupt_inbox().await; } @@ -139,20 +138,9 @@ impl Message { /// Most messages are downloaded automatically on fetch instead. pub(crate) async fn download_msg( context: &Context, - msg_id: MsgId, + rfc724_mid: String, session: &mut Session, ) -> Result<()> { - let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else { - // If partially downloaded message was already deleted - // we do not know its Message-ID anymore - // so cannot download it. - // - // Probably the message expired due to `delete_device_after` - // setting or was otherwise removed from the device, - // so we don't want it to reappear anyway. - return Ok(()); - }; - let transport_id = session.transport_id(); let row = context .sql @@ -161,7 +149,7 @@ pub(crate) async fn download_msg( WHERE rfc724_mid=? AND transport_id=? AND target!=''", - (&msg.rfc724_mid, transport_id), + (&rfc724_mid, transport_id), |row| { let server_uid: u32 = row.get(0)?; let server_folder: String = row.get(1)?; @@ -172,11 +160,13 @@ pub(crate) async fn download_msg( let Some((server_uid, server_folder)) = row else { // No IMAP record found, we don't know the UID and folder. - return Err(anyhow!("Call download_full() again to try over.")); + return Err(anyhow!( + "IMAP location for {rfc724_mid:?} post-message is unknown" + )); }; session - .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone()) + .fetch_single_msg(context, &server_folder, server_uid, rfc724_mid) .await?; Ok(()) } @@ -209,7 +199,7 @@ impl Session { let mut uid_message_ids: BTreeMap = BTreeMap::new(); uid_message_ids.insert(uid, rfc724_mid); let (sender, receiver) = async_channel::unbounded(); - self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, sender) + self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender) .await?; if receiver.recv().await.is_err() { bail!("Failed to fetch UID {uid}"); @@ -218,41 +208,143 @@ impl Session { } } -impl MimeMessage { - /// Creates a placeholder part and add that to `parts`. - /// - /// To create the placeholder, only the outermost header can be used, - /// the mime-structure itself is not available. - /// - /// The placeholder part currently contains a text with size and availability of the message. - pub(crate) async fn create_stub_from_partial_download( - &mut self, - context: &Context, - org_bytes: u32, - ) -> Result<()> { - let mut text = format!( - "[{}]", - stock_str::partial_download_msg_body(context, org_bytes).await - ); - if let Some(delete_server_after) = context.get_config_delete_server_after().await? { - let until = stock_str::download_availability( - context, - time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER), - ) - .await; - text += format!(" [{until}]").as_str(); - }; +async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { + if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? { + // Update download state to failure + // so it can be retried. + // + // On success update_download_state() is not needed + // as receive_imf() already + // set the state and emitted the event. + msg_id + .update_download_state(context, DownloadState::Failure) + .await?; + } + Ok(()) +} - info!(context, "Partial download: {}", text); +async fn available_post_msgs_contains_rfc724_mid( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(context + .sql + .query_get_value::( + "SELECT rfc724_mid FROM available_post_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await? + .is_some()) +} - self.do_add_single_part(Part { - typ: Viewtype::Text, - msg: text, - ..Default::default() - }); +async fn remove_from_available_post_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute( + "DELETE FROM available_post_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await?; + Ok(()) +} - Ok(()) +async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) + .await?; + Ok(()) +} + +// this is a dedicated method because it is used in multiple places. +pub(crate) async fn premessage_is_downloaded_for( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(message::rfc724_mid_exists(context, rfc724_mid) + .await? + .is_some()) +} + +pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM download", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + + for rfc724_mid in &rfc724_mids { + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err + ); + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // This is probably a classical email that vanished before we could download it + warn!( + context, + "{rfc724_mid} is probably a classical email that vanished before we could download it" + ); + remove_from_download_table(context, rfc724_mid).await?; + } else if available_post_msgs_contains_rfc724_mid(context, rfc724_mid).await? { + warn!( + context, + "{rfc724_mid} is in available_post_msgs table but we failed to fetch it, + so set the message to DownloadState::Failure - probably it was deleted on the server in the meantime" + ); + set_msg_state_to_failed(context, rfc724_mid).await?; + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } else { + // leave the message in DownloadState::InProgress; + // it will be downloaded once it arrives. + } + } } + + Ok(()) +} + +/// Download known post messages without pre_message +/// in order to guard against lost pre-messages: +pub(crate) async fn download_known_post_messages_without_pre_message( + context: &Context, + session: &mut Session, +) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM available_post_msgs", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + for rfc724_mid in &rfc724_mids { + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // Download the Post-Message unconditionally, + // because the Pre-Message got lost. + // The message may be in the wrong order, + // but at least we have it at all. + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "download_known_post_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.", + err + ); + } + } + } + Ok(()) } #[cfg(test)] @@ -260,11 +352,8 @@ mod tests { use num_traits::FromPrimitive; use super::*; - use crate::chat::{get_chat_msgs, send_msg}; - use crate::ephemeral::Timer; - use crate::message::delete_msgs; - use crate::receive_imf::receive_imf_from_inbox; - use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager}; + use crate::chat::send_msg; + use crate::test_utils::TestContext; #[test] fn test_downloadstate_values() { @@ -282,29 +371,6 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_download_limit() -> Result<()> { - let t = TestContext::new_alice().await; - - assert_eq!(t.download_limit().await?, None); - - t.set_config(Config::DownloadLimit, Some("200000")).await?; - assert_eq!(t.download_limit().await?, Some(200000)); - - t.set_config(Config::DownloadLimit, Some("20000")).await?; - assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - - t.set_config(Config::DownloadLimit, None).await?; - assert_eq!(t.download_limit().await?, None); - - for val in &["0", "-1", "-100", "", "foo"] { - t.set_config(Config::DownloadLimit, Some(val)).await?; - assert_eq!(t.download_limit().await?, None); - } - - Ok(()) - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_update_download_state() -> Result<()> { let t = TestContext::new_alice().await; @@ -336,230 +402,4 @@ mod tests { Ok(()) } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_receive_imf() -> Result<()> { - let t = TestContext::new_alice().await; - - let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ - From: bob@example.com\n\ - To: alice@example.org\n\ - Subject: foo\n\ - Message-ID: \n\ - Chat-Version: 1.0\n\ - Date: Sun, 22 Mar 2020 22:37:57 +0000\ - Content-Type: text/plain"; - - receive_imf_from_inbox( - &t, - "Mr.12345678901@example.com", - header.as_bytes(), - false, - Some(100000), - ) - .await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Available); - assert_eq!(msg.get_subject(), "foo"); - assert!( - msg.get_text() - .contains(&stock_str::partial_download_msg_body(&t, 100000).await) - ); - - receive_imf_from_inbox( - &t, - "Mr.12345678901@example.com", - format!("{header}\n\n100k text...").as_bytes(), - false, - None, - ) - .await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Done); - assert_eq!(msg.get_subject(), "foo"); - assert_eq!(msg.get_text(), "100k text..."); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_and_ephemeral() -> Result<()> { - let t = TestContext::new_alice().await; - let chat_id = t - .create_chat_with_contact("bob", "bob@example.org") - .await - .id; - chat_id - .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 }) - .await?; - - // download message from bob partially, this must not change the ephemeral timer - receive_imf_from_inbox( - &t, - "first@example.org", - b"From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain", - false, - Some(100000), - ) - .await?; - assert_eq!( - chat_id.get_ephemeral_timer(&t).await?, - Timer::Enabled { duration: 60 } - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_status_update_expands_to_nothing() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - let chat_id = alice.create_chat(&bob).await.id; - - let file = alice.get_blobdir().join("minimal.xdc"); - tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?; - let mut instance = Message::new(Viewtype::File); - instance.set_file_and_deduplicate(&alice, &file, None, None)?; - let _sent1 = alice.send_msg(chat_id, &mut instance).await; - - alice - .send_webxdc_status_update(instance.id, r#"{"payload":7}"#) - .await?; - alice.flush_status_updates().await?; - let sent2 = alice.pop_sent_msg().await; - let sent2_rfc724_mid = sent2.load_from_db().await.rfc724_mid; - - // not downloading the status update results in an placeholder - receive_imf_from_inbox( - &bob, - &sent2_rfc724_mid, - sent2.payload().as_bytes(), - false, - Some(sent2.payload().len() as u32), - ) - .await?; - let msg = bob.get_last_msg().await; - let chat_id = msg.chat_id; - assert_eq!( - get_chat_msgs(&bob, chat_id).await?.len(), - E2EE_INFO_MSGS + 1 - ); - assert_eq!(msg.download_state(), DownloadState::Available); - - // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat - // (usually status updates are too small for not being downloaded directly) - receive_imf_from_inbox( - &bob, - &sent2_rfc724_mid, - sent2.payload().as_bytes(), - false, - None, - ) - .await?; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), E2EE_INFO_MSGS); - assert!( - Message::load_from_db_optional(&bob, msg.id) - .await? - .is_none() - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_mdn_expands_to_nothing() -> Result<()> { - let bob = TestContext::new_bob().await; - let raw = b"Subject: Message opened\n\ - Date: Mon, 10 Jan 2020 00:00:00 +0000\n\ - Chat-Version: 1.0\n\ - Message-ID: \n\ - To: Alice \n\ - From: Bob \n\ - Content-Type: multipart/report; report-type=disposition-notification;\n\t\ - boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\ - Content-Type: text/plain; charset=utf-8\n\ - \n\ - bla\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\ - Content-Type: message/disposition-notification\n\ - \n\ - Reporting-UA: Delta Chat 1.88.0\n\ - Original-Recipient: rfc822;bob@example.org\n\ - Final-Recipient: rfc822;bob@example.org\n\ - Original-Message-ID: \n\ - Disposition: manual-action/MDN-sent-automatically; displayed\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ - "; - - // not downloading the mdn results in an placeholder - receive_imf_from_inbox(&bob, "bar@example.org", raw, false, Some(raw.len() as u32)).await?; - let msg = bob.get_last_msg().await; - let chat_id = msg.chat_id; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 1); - assert_eq!(msg.download_state(), DownloadState::Available); - - // downloading the mdn afterwards expands to nothing and deletes the placeholder directly - // (usually mdn are too small for not being downloaded directly) - receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None).await?; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0); - assert!( - Message::load_from_db_optional(&bob, msg.id) - .await? - .is_none() - ); - - Ok(()) - } - - /// Tests that fully downloading the message - /// works even if the Message-ID already exists - /// in the database assigned to the trash chat. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_trashed() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - - let imf_raw = b"From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain"; - - // Download message from Bob partially. - let partial_received_msg = - receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, Some(100000)) - .await? - .unwrap(); - assert_eq!(partial_received_msg.msg_ids.len(), 1); - - // Delete the received message. - // Not it is still in the database, - // but in the trash chat. - delete_msgs(alice, &[partial_received_msg.msg_ids[0]]).await?; - - // Fully download message after deletion. - let full_received_msg = - receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, None).await?; - - // The message does not reappear. - // However, `receive_imf` should not fail. - assert!(full_received_msg.is_none()); - - Ok(()) - } } diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs new file mode 100644 index 0000000000..cc19b9e1f7 --- /dev/null +++ b/src/download/pre_msg_metadata.rs @@ -0,0 +1,248 @@ +use anyhow::{Context as _, Result}; +use num_traits::ToPrimitive; +use serde::{Deserialize, Serialize}; + +use crate::context::Context; +use crate::log::warn; +use crate::message::Message; +use crate::message::Viewtype; +use crate::param::{Param, Params}; + +/// Metadata contained in Pre-Message that describes the Post-Message. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PreMsgMetadata { + /// size of the attachment in bytes + pub(crate) size: u64, + /// Real viewtype of message + pub(crate) viewtype: Viewtype, + /// the original file name + pub(crate) filename: String, + /// Dimensions: width and height of image or video + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) dimensions: Option<(i32, i32)>, + /// Duration of audio file or video in milliseconds + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) duration: Option, +} + +impl PreMsgMetadata { + // Returns PreMsgMetadata for messages with files and None for messages without file attachment + pub(crate) async fn from_msg(context: &Context, message: &Message) -> Result> { + if !message.viewtype.has_file() { + return Ok(None); + } + + let size = message + .get_filebytes(context) + .await? + .context("Unexpected: file has no size")?; + let filename = message + .param + .get(Param::Filename) + .unwrap_or_default() + .to_owned(); + let dimensions = { + match ( + message.param.get_int(Param::Width), + message.param.get_int(Param::Height), + ) { + (None, None) => None, + (Some(width), Some(height)) => Some((width, height)), + _ => { + warn!(context, "Message misses either width or height."); + None + } + } + }; + let duration = message.param.get_int(Param::Duration); + + Ok(Some(Self { + size, + filename, + viewtype: message.viewtype, + dimensions, + duration, + })) + } + + pub(crate) fn to_header_value(&self) -> Result { + Ok(serde_json::to_string(&self)?) + } + + pub(crate) fn try_from_header_value(value: &str) -> Result { + Ok(serde_json::from_str(value)?) + } +} + +impl Params { + /// Applies data from pre_msg_metadata to Params + pub(crate) fn apply_from_pre_msg_metadata( + &mut self, + pre_msg_metadata: &PreMsgMetadata, + ) -> &mut Self { + self.set(Param::PostMessageFileBytes, pre_msg_metadata.size); + if !pre_msg_metadata.filename.is_empty() { + self.set(Param::Filename, &pre_msg_metadata.filename); + } + self.set_i64( + Param::PostMessageViewtype, + pre_msg_metadata.viewtype.to_i64().unwrap_or_default(), + ); + if let Some((width, height)) = pre_msg_metadata.dimensions { + self.set(Param::Width, width); + self.set(Param::Height, height); + } + if let Some(duration) = pre_msg_metadata.duration { + self.set(Param::Duration, duration); + } + + self + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use pretty_assertions::assert_eq; + + use crate::{ + message::{Message, Viewtype}, + test_utils::{TestContextManager, create_test_image}, + }; + + use super::PreMsgMetadata; + + /// Build from message with file attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_file_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let mut file_msg = Message::new(Viewtype::File); + file_msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &file_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + }) + ); + Ok(()) + } + + /// Build from message with image attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_image_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let mut image_msg = Message::new(Viewtype::Image); + + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + image_msg.set_file_from_bytes(alice, "vacation.png", &test_img, None)?; + // this is usually done while sending, + // but we don't send it here, so we need to call it ourself + image_msg.try_calc_and_set_dimensions(alice).await?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &image_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1816098, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((width as i32, height as i32)), + duration: None, + }) + ); + + Ok(()) + } + + /// Test that serialisation results in expected format + #[test] + fn test_serialize_to_header() -> Result<()> { + assert_eq!( + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + .to_header_value()?, + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\"}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + .to_header_value()?, + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + .to_header_value()?, + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + ); + + Ok(()) + } + + /// Test that deserialisation from expected format works + /// This test will become important for compatibility between versions in the future + #[test] + fn test_deserialize_from_header() -> Result<()> { + assert_eq!( + serde_json::from_str::( + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}" + )?, + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + )?, + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + )?, + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + ); + + Ok(()) + } +} diff --git a/src/headerdef.rs b/src/headerdef.rs index c57f05033f..4baee1c57c 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -102,6 +102,21 @@ pub enum HeaderDef { /// used to encrypt and decrypt messages. /// This secret is sent to a new member in the member-addition message. ChatBroadcastSecret, + /// A message with a large attachment is split into two MIME messages: + /// A pre-message, which contains everything but the attachment, + /// and a Post-Message. + /// The Pre-Message gets a `Chat-Post-Message-Id` header + /// referencing the Post-Message's rfc724_mid. + ChatPostMessageId, + + /// Announce Post-Message metadata in a Pre-Message. + /// contains serialized PreMsgMetadata struct + ChatPostMessageMetadata, + + /// This message is preceded by a Pre-Message + /// and thus this message can be skipped while fetching messages. + /// This is a cleartext / unproteced header. + ChatIsPostMessage, /// [Autocrypt](https://autocrypt.org/) header. Autocrypt, @@ -147,6 +162,9 @@ pub enum HeaderDef { impl HeaderDef { /// Returns the corresponding header string. + /// + /// Format is lower-kebab-case for easy comparisons. + /// This method is used in message receiving and testing. pub fn get_headername(&self) -> &'static str { self.into() } diff --git a/src/imap.rs b/src/imap.rs index 5bac1e201b..25da91f33b 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -67,7 +67,6 @@ const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\ X-MICROSOFT-ORIGINAL-MESSAGE-ID\ )])"; const BODY_FULL: &str = "(FLAGS BODY.PEEK[])"; -const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])"; #[derive(Debug)] pub(crate) struct Imap { @@ -615,12 +614,23 @@ impl Imap { .context("prefetch")?; let read_cnt = msgs.len(); - let download_limit = context.download_limit().await?; - let mut uids_fetch = Vec::<(u32, bool /* partially? */)>::with_capacity(msgs.len() + 1); + let mut uids_fetch: Vec = Vec::new(); + let mut available_post_msgs: Vec = Vec::new(); + let mut download_later: Vec = Vec::new(); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; + let download_limit = { + let download_limit: Option = + context.get_config_parsed(Config::DownloadLimit).await?; + if download_limit == Some(0) { + None + } else { + download_limit + } + }; + // Store the info about IMAP messages in the database. for (uid, ref fetch_response) in msgs { let headers = match get_fetch_headers(fetch_response) { @@ -632,6 +642,9 @@ impl Imap { }; let message_id = prefetch_get_message_id(&headers); + let size = fetch_response + .size + .context("imap fetch response does not contain size")?; // Determine the target folder where the message should be moved to. // @@ -706,14 +719,24 @@ impl Imap { ) .await.context("prefetch_should_download")? { - match download_limit { - Some(download_limit) => uids_fetch.push(( - uid, - fetch_response.size.unwrap_or_default() > download_limit, - )), - None => uids_fetch.push((uid, false)), - } - uid_message_ids.insert(uid, message_id); + if headers + .get_header_value(HeaderDef::ChatIsPostMessage) + .is_some() + { + info!(context, "{message_id:?} is a post-message."); + available_post_msgs.push(message_id.clone()); + + // whether it fits download size limit + if download_limit.is_none_or(|download_limit| size < download_limit) { + download_later.push(message_id.clone()); + } + largest_uid_skipped = Some(uid); + } else { + info!(context, "{message_id:?} is not a post-message."); + + uids_fetch.push(uid); + uid_message_ids.insert(uid, message_id); + }; } else { largest_uid_skipped = Some(uid); } @@ -747,29 +770,10 @@ impl Imap { }; let actually_download_messages_future = async { - let sender = sender; - let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1)); - let mut fetch_partially = false; - uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1)); - for (uid, fp) in uids_fetch { - if fp != fetch_partially { - session - .fetch_many_msgs( - context, - folder, - uids_fetch_in_batch.split_off(0), - &uid_message_ids, - fetch_partially, - sender.clone(), - ) - .await - .context("fetch_many_msgs")?; - fetch_partially = fp; - } - uids_fetch_in_batch.push(uid); - } - - anyhow::Ok(()) + session + .fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender) + .await + .context("fetch_many_msgs") }; let (largest_uid_fetched, fetch_res) = @@ -804,6 +808,30 @@ impl Imap { chat::mark_old_messages_as_noticed(context, received_msgs).await?; + if fetch_res.is_ok() { + info!( + context, + "available_post_msgs: {}, download_when_normal_starts: {}", + available_post_msgs.len(), + download_later.len() + ); + for rfc724_mid in available_post_msgs { + context + .sql + .insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,)) + .await?; + } + for rfc724_mid in download_later { + context + .sql + .insert( + "INSERT INTO download (rfc724_mid, msg_id) VALUES (?,0)", + (rfc724_mid,), + ) + .await?; + } + } + // Now fail if fetching failed, so we will // establish a new session if this one is broken. fetch_res?; @@ -1375,7 +1403,6 @@ impl Session { folder: &str, request_uids: Vec, uid_message_ids: &BTreeMap, - fetch_partially: bool, received_msgs_channel: Sender<(u32, Option)>, ) -> Result<()> { if request_uids.is_empty() { @@ -1383,25 +1410,10 @@ impl Session { } for (request_uids, set) in build_sequence_sets(&request_uids)? { - info!( - context, - "Starting a {} FETCH of message set \"{}\".", - if fetch_partially { "partial" } else { "full" }, - set - ); - let mut fetch_responses = self - .uid_fetch( - &set, - if fetch_partially { - BODY_PARTIAL - } else { - BODY_FULL - }, - ) - .await - .with_context(|| { - format!("fetching messages {} from folder \"{}\"", &set, folder) - })?; + info!(context, "Starting a full FETCH of message set \"{}\".", set); + let mut fetch_responses = self.uid_fetch(&set, BODY_FULL).await.with_context(|| { + format!("fetching messages {} from folder \"{}\"", &set, folder) + })?; // Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here // when we want to process other messages first. @@ -1458,11 +1470,7 @@ impl Session { count += 1; let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted); - let (body, partial) = if fetch_partially { - (fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ... - } else { - (fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header() - }; + let body = fetch_response.body(); if is_deleted { info!(context, "Not processing deleted msg {}.", request_uid); @@ -1496,7 +1504,7 @@ impl Session { context, "Passing message UID {} to receive_imf().", request_uid ); - let res = receive_imf_inner(context, rfc724_mid, body, is_seen, partial).await; + let res = receive_imf_inner(context, rfc724_mid, body, is_seen).await; let received_msg = match res { Err(err) => { warn!(context, "receive_imf error: {err:#}."); diff --git a/src/imap/session.rs b/src/imap/session.rs index 0da1d7936f..1d8c2d7110 100644 --- a/src/imap/session.rs +++ b/src/imap/session.rs @@ -17,6 +17,7 @@ use crate::tools; /// - Chat-Version to check if a message is a chat message /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message, /// not necessarily sent by Delta Chat. +/// - Chat-Is-Post-Message to skip it in background fetch or when it is too large const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\ MESSAGE-ID \ DATE \ @@ -24,6 +25,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE FROM \ IN-REPLY-TO REFERENCES \ CHAT-VERSION \ + CHAT-IS-POST-MESSAGE \ AUTO-SUBMITTED \ AUTOCRYPT-SETUP-MESSAGE\ )])"; diff --git a/src/internals_for_benches.rs b/src/internals_for_benches.rs index 1fd80c5516..61fc9d7fca 100644 --- a/src/internals_for_benches.rs +++ b/src/internals_for_benches.rs @@ -21,7 +21,7 @@ pub async fn store_self_keypair(context: &Context, keypair: &KeyPair) -> Result< } pub async fn parse_and_get_text(context: &Context, imf_raw: &[u8]) -> Result { - let mime_parser = MimeMessage::from_bytes(context, imf_raw, None).await?; + let mime_parser = MimeMessage::from_bytes(context, imf_raw).await?; Ok(mime_parser.parts.into_iter().next().unwrap().msg) } diff --git a/src/message.rs b/src/message.rs index 129e037e63..f531edd856 100644 --- a/src/message.rs +++ b/src/message.rs @@ -8,6 +8,9 @@ use std::str; use anyhow::{Context as _, Result, ensure, format_err}; use deltachat_contact_tools::{VcardContact, parse_vcard}; use deltachat_derive::{FromSql, ToSql}; +use humansize::BINARY; +use humansize::format_size; +use num_traits::FromPrimitive; use serde::{Deserialize, Serialize}; use tokio::{fs, io}; @@ -430,6 +433,10 @@ pub struct Message { pub(crate) ephemeral_timer: EphemeralTimer, pub(crate) ephemeral_timestamp: i64, pub(crate) text: String, + /// Text that is added to the end of Message.text + /// + /// Currently used for adding the download information on pre-messages + pub(crate) additional_text: String, /// Message subject. /// @@ -488,7 +495,7 @@ impl Message { !id.is_special(), "Can not load special message ID {id} from DB" ); - let msg = context + let mut msg = context .sql .query_row_optional( concat!( @@ -570,6 +577,7 @@ impl Message { original_msg_id: row.get("original_msg_id")?, mime_modified: row.get("mime_modified")?, text, + additional_text: String::new(), subject: row.get("subject")?, param: row.get::<_, String>("param")?.parse().unwrap_or_default(), hidden: row.get("hidden")?, @@ -584,9 +592,48 @@ impl Message { .await .with_context(|| format!("failed to load message {id} from the database"))?; + if let Some(msg) = &mut msg { + msg.additional_text = + Self::get_additional_text(context, msg.download_state, &msg.param).await?; + } + Ok(msg) } + /// Returns additional text which is appended to the message's text field + /// when it is loaded from the database. + /// Currently this is used to add infomation to pre-messages of what the download will be and how large it is + async fn get_additional_text( + context: &Context, + download_state: DownloadState, + param: &Params, + ) -> Result { + if download_state != DownloadState::Done { + let file_size = param + .get(Param::PostMessageFileBytes) + .and_then(|s| s.parse().ok()) + .map(|file_size: usize| format_size(file_size, BINARY)) + .unwrap_or("?".to_owned()); + let viewtype = param + .get_i64(Param::PostMessageViewtype) + .and_then(Viewtype::from_i64) + .unwrap_or(Viewtype::Unknown); + let file_name = param + .get(Param::Filename) + .map(sanitize_filename) + .unwrap_or("?".to_owned()); + + return match viewtype { + Viewtype::File => Ok(format!(" [{file_name} - {file_size}]")), + _ => { + let translated_viewtype = viewtype.to_locale_string(context).await; + Ok(format!(" [{translated_viewtype} - {file_size}]")) + } + }; + } + Ok(String::new()) + } + /// Returns the MIME type of an attached file if it exists. /// /// If the MIME type is not known, the function guesses the MIME type @@ -769,7 +816,7 @@ impl Message { /// Returns the text of the message. pub fn get_text(&self) -> String { - self.text.clone() + self.text.clone() + &self.additional_text } /// Returns message subject. @@ -791,7 +838,17 @@ impl Message { } /// Returns the size of the file in bytes, if applicable. + /// If message is a pre-message, then this returns size of the to be downloaded file. pub async fn get_filebytes(&self, context: &Context) -> Result> { + // if download state is not downloaded then return value from from params metadata + if self.download_state != DownloadState::Done + && let Some(file_size) = self + .param + .get(Param::PostMessageFileBytes) + .and_then(|s| s.parse().ok()) + { + return Ok(Some(file_size)); + } if let Some(path) = self.param.get_file_path(context)? { Ok(Some(get_filebytes(context, &path).await.with_context( || format!("failed to get {} size in bytes", path.display()), @@ -801,6 +858,21 @@ impl Message { } } + /// If message is a Pre-Message, + /// then this returns the viewtype it will have when it is downloaded. + #[cfg(test)] + pub(crate) fn get_post_message_viewtype(&self) -> Option { + if self.download_state != DownloadState::Done + && let Some(viewtype) = self + .param + .get_i64(Param::PostMessageViewtype) + .and_then(Viewtype::from_i64) + { + return Some(viewtype); + } + None + } + /// Returns width of associated image or video file. pub fn get_width(&self) -> i32 { self.param.get_int(Param::Width).unwrap_or_default() @@ -1681,9 +1753,17 @@ pub async fn delete_msgs_ex( let update_db = |trans: &mut rusqlite::Transaction| { trans.execute( "UPDATE imap SET target=? WHERE rfc724_mid=?", - (target, msg.rfc724_mid), + (target, &msg.rfc724_mid), )?; trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?; + trans.execute( + "DELETE FROM download WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; + trans.execute( + "DELETE FROM available_post_msgs WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; Ok(()) }; if let Err(e) = context.sql.transaction(update_db).await { @@ -1751,7 +1831,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> "SELECT m.chat_id AS chat_id, m.state AS state, - m.download_state as download_state, m.ephemeral_timer AS ephemeral_timer, m.param AS param, m.from_id AS from_id, @@ -1764,7 +1843,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> |row| { let chat_id: ChatId = row.get("chat_id")?; let state: MessageState = row.get("state")?; - let download_state: DownloadState = row.get("download_state")?; let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default(); let from_id: ContactId = row.get("from_id")?; let rfc724_mid: String = row.get("rfc724_mid")?; @@ -1776,7 +1854,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> id, chat_id, state, - download_state, param, from_id, rfc724_mid, @@ -1809,7 +1886,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> id, curr_chat_id, curr_state, - curr_download_state, curr_param, curr_from_id, curr_rfc724_mid, @@ -1819,14 +1895,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> _curr_ephemeral_timer, ) in msgs { - if curr_download_state != DownloadState::Done { - if curr_state == MessageState::InFresh { - // Don't mark partially downloaded messages as seen or send a read receipt since - // they are not really seen by the user. - update_msg_state(context, id, MessageState::InNoticed).await?; - updated_chat_ids.insert(curr_chat_id); - } - } else if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed { + if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed { update_msg_state(context, id, MessageState::InSeen).await?; info!(context, "Seen message {}.", id); diff --git a/src/message/message_tests.rs b/src/message/message_tests.rs index be3ed01d9a..d43d21220e 100644 --- a/src/message/message_tests.rs +++ b/src/message/message_tests.rs @@ -326,112 +326,6 @@ async fn test_markseen_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_markseen_not_downloaded_msg() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let bob = &tcm.bob().await; - let bob_chat_id = bob.create_chat(alice).await.id; - alice.create_chat(bob).await; // Make sure the chat is accepted. - - tcm.section("Bob sends a large message to Alice"); - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; - - tcm.section("Alice receives a large message from Bob"); - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert!(!msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert_eq!(msg.state, MessageState::InFresh); - markseen_msgs(alice, vec![msg.id]).await?; - // A not downloaded message can be seen only if it's seen on another device. - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - // Marking the message as seen again is a no op. - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - - msg.id - .update_download_state(alice, DownloadState::InProgress) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - msg.id - .update_download_state(alice, DownloadState::Failure) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - msg.id - .update_download_state(alice, DownloadState::Undecipherable) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - - assert!( - !alice - .sql - .exists("SELECT COUNT(*) FROM smtp_mdns", ()) - .await? - ); - - alice.set_config(Config::DownloadLimit, None).await?; - // Let's assume that Alice and Bob resolved the problem with encryption. - let old_msg = msg; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.chat_id, old_msg.chat_id); - assert_eq!(msg.download_state, DownloadState::Done); - assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert!(msg.get_showpadlock()); - // The message state mustn't be downgraded to `InFresh`. - assert_eq!(msg.state, MessageState::InNoticed); - markseen_msgs(alice, vec![msg.id]).await?; - let msg = Message::load_from_db(alice, msg.id).await?; - assert_eq!(msg.state, MessageState::InSeen); - assert_eq!( - alice - .sql - .count("SELECT COUNT(*) FROM smtp_mdns", ()) - .await?, - 1 - ); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let bob = &tcm.bob().await; - let bob_chat_id = tcm.send_recv_accept(alice, bob, "hi").await.chat_id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(msg.state, MessageState::InFresh); - - alice.set_config(Config::DownloadLimit, None).await?; - let seen = true; - let rcvd_msg = receive_imf(alice, sent_msg.payload().as_bytes(), seen) - .await - .unwrap() - .unwrap(); - assert_eq!(rcvd_msg.chat_id, msg.chat_id); - let msg = Message::load_from_db(alice, *rcvd_msg.msg_ids.last().unwrap()) - .await - .unwrap(); - assert_eq!(msg.download_state, DownloadState::Done); - assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert!(msg.get_showpadlock()); - assert_eq!(msg.state, MessageState::InSeen); - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_state() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 732fff545a..049114847a 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -21,6 +21,7 @@ use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG}; use crate::constants::{Chattype, DC_FROM_HANDSHAKE}; use crate::contact::{Contact, ContactId, Origin}; use crate::context::Context; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::e2ee::EncryptHelper; use crate::ensure_and_debug_assert; use crate::ephemeral::Timer as EphemeralTimer; @@ -59,6 +60,15 @@ pub enum Loaded { }, } +#[derive(Debug, Clone, PartialEq)] +pub enum PreMessageMode { + /// adds the Chat-Is-Post-Message header in unprotected part + PostMessage, + /// adds the Chat-Post-Message-ID header to protected part + /// also adds metadata and explicitly excludes attachment + PreMessage { post_msg_rfc724_mid: String }, +} + /// Helper to construct mime messages. #[derive(Debug, Clone)] pub struct MimeFactory { @@ -146,6 +156,9 @@ pub struct MimeFactory { /// This field is used to sustain the topic id of webxdcs needed for peer channels. webxdc_topic: Option, + + /// This field is used when this is either a pre-message or a Post-Message. + pre_message_mode: Option, } /// Result of rendering a message, ready to be submitted to a send job. @@ -500,6 +513,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar, webxdc_topic, + pre_message_mode: None, }; Ok(factory) } @@ -548,6 +562,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar: false, webxdc_topic: None, + pre_message_mode: None, }; Ok(res) @@ -779,7 +794,10 @@ impl MimeFactory { headers.push(("Date", mail_builder::headers::raw::Raw::new(date).into())); let rfc724_mid = match &self.loaded { - Loaded::Message { msg, .. } => msg.rfc724_mid.clone(), + Loaded::Message { msg, .. } => match &self.pre_message_mode { + Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(), + _ => msg.rfc724_mid.clone(), + }, Loaded::Mdn { .. } => create_outgoing_rfc724_mid(), }; headers.push(( @@ -893,7 +911,7 @@ impl MimeFactory { )); } - let is_encrypted = self.encryption_pubkeys.is_some(); + let is_encrypted = self.will_be_encrypted(); // Add ephemeral timer for non-MDN messages. // For MDNs it does not matter because they are not visible @@ -978,6 +996,22 @@ impl MimeFactory { "MIME-Version", mail_builder::headers::raw::Raw::new("1.0").into(), )); + + if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + unprotected_headers.push(( + "Chat-Is-Post-Message", + mail_builder::headers::raw::Raw::new("1").into(), + )); + } else if let Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid, + }) = self.pre_message_mode.clone() + { + protected_headers.push(( + "Chat-Post-Message-ID", + mail_builder::headers::message_id::MessageId::new(post_msg_rfc724_mid).into(), + )); + } + for header @ (original_header_name, _header_value) in &headers { let header_name = original_header_name.to_lowercase(); if header_name == "message-id" { @@ -1119,6 +1153,10 @@ impl MimeFactory { for (addr, key) in &encryption_pubkeys { let fingerprint = key.dc_fingerprint().hex(); let cmd = msg.param.get_cmd(); + if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + continue; + } + let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup || cmd == SystemMessage::SecurejoinMessage || multiple_recipients && { @@ -1831,19 +1869,23 @@ impl MimeFactory { let footer = if is_reaction { "" } else { &self.selfstatus }; - let message_text = format!( - "{}{}{}{}{}{}", - fwdhint.unwrap_or_default(), - quoted_text.unwrap_or_default(), - escape_message_footer_marks(final_text), - if !final_text.is_empty() && !footer.is_empty() { - "\r\n\r\n" - } else { - "" - }, - if !footer.is_empty() { "-- \r\n" } else { "" }, - footer - ); + let message_text = if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + "".to_string() + } else { + format!( + "{}{}{}{}{}{}", + fwdhint.unwrap_or_default(), + quoted_text.unwrap_or_default(), + escape_message_footer_marks(final_text), + if !final_text.is_empty() && !footer.is_empty() { + "\r\n\r\n" + } else { + "" + }, + if !footer.is_empty() { "-- \r\n" } else { "" }, + footer + ) + }; let mut main_part = MimePart::new("text/plain", message_text); if is_reaction { @@ -1875,8 +1917,19 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { - let file_part = build_body_file(context, &msg).await?; - parts.push(file_part); + if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else { + bail!("Failed to generate metadata for pre-message") + }; + + headers.push(( + HeaderDef::ChatPostMessageMetadata.into(), + mail_builder::headers::raw::Raw::new(metadata.to_header_value()?).into(), + )); + } else { + let file_part = build_body_file(context, &msg).await?; + parts.push(file_part); + } } if let Some(msg_kml_part) = self.get_message_kml_part() { @@ -1921,6 +1974,8 @@ impl MimeFactory { } } + self.attach_selfavatar = + self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::PostMessage); if self.attach_selfavatar { match context.get_config(Config::Selfavatar).await? { Some(path) => match build_avatar_file(context, &path).await { @@ -1990,6 +2045,20 @@ impl MimeFactory { Ok(message) } + + pub fn will_be_encrypted(&self) -> bool { + self.encryption_pubkeys.is_some() + } + + pub fn set_as_post_message(&mut self) { + self.pre_message_mode = Some(PreMessageMode::PostMessage); + } + + pub fn set_as_pre_message_for(&mut self, post_message: &RenderedEmail) { + self.pre_message_mode = Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid: post_message.rfc724_mid.clone(), + }); + } } fn hidden_recipients() -> Address<'static> { diff --git a/src/mimefactory/mimefactory_tests.rs b/src/mimefactory/mimefactory_tests.rs index d29fe54645..bd79d92c5f 100644 --- a/src/mimefactory/mimefactory_tests.rs +++ b/src/mimefactory/mimefactory_tests.rs @@ -559,7 +559,7 @@ async fn test_render_reply() { "1.0" ); - let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes(), None) + let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes()) .await .unwrap(); } @@ -757,7 +757,7 @@ async fn test_protected_headers_directive() -> Result<()> { assert!(msg.get_showpadlock()); assert!(sent.payload.contains("\r\nSubject: [...]\r\n")); - let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes(), None).await?; + let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes()).await?; let mut payload = str::from_utf8(&mime.decoded_data)?.splitn(2, "\r\n\r\n"); let part = payload.next().unwrap(); assert_eq!( @@ -781,7 +781,7 @@ async fn test_hp_outer_headers() -> Result<()> { .await?; chat::send_text_msg(t, chat_id, "hi!".to_string()).await?; let sent_msg = t.pop_sent_msg().await; - let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes(), None).await?; + let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes()).await?; assert_eq!(msg.header_exists(HeaderDef::HpOuter), std_hp_composing); for hdr in ["Date", "From", "Message-ID"] { assert_eq!( @@ -811,7 +811,7 @@ async fn test_dont_remove_self() -> Result<()> { .await; println!("{}", sent.payload); - let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes(), None) + let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes()) .await .unwrap(); assert!(!mime_message.header_exists(HeaderDef::ChatGroupPastMembers)); diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 69bb198554..1c7080944f 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -23,6 +23,7 @@ use crate::contact::ContactId; use crate::context::Context; use crate::decrypt::{try_decrypt, validate_detached_signature}; use crate::dehtml::dehtml; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring}; @@ -147,6 +148,23 @@ pub(crate) struct MimeMessage { /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized /// clocks, but not too much. pub(crate) timestamp_sent: i64, + + pub(crate) pre_message: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum PreMessageMode { + /// This is Post-Message + /// it replaces it's Pre-Message attachment if it exists already, + /// and if the Pre-Message does not exist it is treated as normal message + PostMessage, + /// This is a Pre-Message, + /// it adds a message preview for a Post-Message + /// and it is ignored if the Post-Message was downloaded already + PreMessage { + post_msg_rfc724_mid: String, + metadata: Option, + }, } #[derive(Debug, PartialEq)] @@ -240,12 +258,9 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup"; impl MimeMessage { /// Parse a mime message. /// - /// If `partial` is set, it contains the full message size in bytes. - pub(crate) async fn from_bytes( - context: &Context, - body: &[u8], - partial: Option, - ) -> Result { + /// This method has some side-effects, + /// such as saving blobs and saving found public keys to the database. + pub(crate) async fn from_bytes(context: &Context, body: &[u8]) -> Result { let mail = mailparse::parse_mail(body)?; let timestamp_rcvd = smeared_time(context); @@ -302,7 +317,7 @@ impl MimeMessage { ); (part, part.ctype.mimetype.parse::()?) } else { - // If it's a partially fetched message, there are no subparts. + // Not a valid signed message, handle it as plaintext. (&mail, mimetype) } } else { @@ -352,6 +367,16 @@ impl MimeMessage { let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into()); + let mut pre_message = if mail + .headers + .get_header_value(HeaderDef::ChatIsPostMessage) + .is_some() + { + Some(PreMessageMode::PostMessage) + } else { + None + }; + let mail_raw; // Memory location for a possible decrypted message. let decrypted_msg; // Decrypted signed OpenPGP message. let secrets: Vec = context @@ -580,6 +605,36 @@ impl MimeMessage { signatures.clear(); } + if let (Ok(mail), true) = (mail, is_encrypted) + && let Some(post_msg_rfc724_mid) = + mail.headers.get_header_value(HeaderDef::ChatPostMessageId) + { + let post_msg_rfc724_mid = parse_message_id(&post_msg_rfc724_mid)?; + let metadata = if let Some(value) = mail + .headers + .get_header_value(HeaderDef::ChatPostMessageMetadata) + { + match PreMsgMetadata::try_from_header_value(&value) { + Ok(metadata) => Some(metadata), + Err(error) => { + error!( + context, + "failed to parse metadata header in pre-message: {error:#?}" + ); + None + } + } + } else { + warn!(context, "expected pre-message to have metadata header"); + None + }; + + pre_message = Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid, + metadata, + }); + } + let mut parser = MimeMessage { parts: Vec::new(), headers, @@ -615,33 +670,27 @@ impl MimeMessage { is_bot: None, timestamp_rcvd, timestamp_sent, + pre_message, }; - match partial { - Some(org_bytes) => { - parser - .create_stub_from_partial_download(context, org_bytes) - .await?; + match mail { + Ok(mail) => { + parser.parse_mime_recursive(context, mail, false).await?; + } + Err(err) => { + let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]"; + + let part = Part { + typ: Viewtype::Text, + msg_raw: Some(txt.to_string()), + msg: txt.to_string(), + // Don't change the error prefix for now, + // receive_imf.rs:lookup_chat_by_reply() checks it. + error: Some(format!("Decrypting failed: {err:#}")), + ..Default::default() + }; + parser.do_add_single_part(part); } - None => match mail { - Ok(mail) => { - parser.parse_mime_recursive(context, mail, false).await?; - } - Err(err) => { - let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]"; - - let part = Part { - typ: Viewtype::Text, - msg_raw: Some(txt.to_string()), - msg: txt.to_string(), - // Don't change the error prefix for now, - // receive_imf.rs:lookup_chat_by_reply() checks it. - error: Some(format!("Decrypting failed: {err:#}")), - ..Default::default() - }; - parser.do_add_single_part(part); - } - }, }; let is_location_only = parser.location_kml.is_some() && parser.parts.is_empty(); diff --git a/src/mimeparser/mimeparser_tests.rs b/src/mimeparser/mimeparser_tests.rs index 6d198075b9..42edbba7a8 100644 --- a/src/mimeparser/mimeparser_tests.rs +++ b/src/mimeparser/mimeparser_tests.rs @@ -25,58 +25,54 @@ impl AvatarAction { async fn test_mimeparser_fromheader() { let ctx = TestContext::new_alice().await; - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Goetz C".to_string())); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Goetz C".to_string())); - let mimemsg = - MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C \n\nhi", None) - .await - .unwrap(); + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C \n\nhi") + .await + .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Götz C".to_string())); // although RFC 2047 says, encoded-words shall not appear inside quoted-string, // this combination is used in the wild eg. by MailMate - let mimemsg = MimeMessage::from_bytes( - &ctx, - b"From: \"=?utf-8?q?G=C3=B6tz?= C\" \n\nhi", - None, - ) - .await - .unwrap(); + let mimemsg = + MimeMessage::from_bytes(&ctx, b"From: \"=?utf-8?q?G=C3=B6tz?= C\" \n\nhi") + .await + .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Götz C".to_string())); @@ -86,7 +82,7 @@ async fn test_mimeparser_fromheader() { async fn test_mimeparser_crash() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/issue_523.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -98,7 +94,7 @@ async fn test_mimeparser_crash() { async fn test_get_rfc724_mid_exists() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/mail_with_message_id.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -112,7 +108,7 @@ async fn test_get_rfc724_mid_exists() { async fn test_get_rfc724_mid_not_exists() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/issue_523.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(mimeparser.get_rfc724_mid(), None); @@ -324,7 +320,7 @@ async fn test_mailparse_0_16_0_panic() { // There should be an error, but no panic. assert!( - MimeMessage::from_bytes(&context.ctx, &raw[..], None) + MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .is_err() ); @@ -341,7 +337,7 @@ async fn test_parse_first_addr() { test1\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await; assert!(mimeparser.is_err()); } @@ -356,7 +352,7 @@ async fn test_get_parent_timestamp() { \n\ Some reply\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -402,7 +398,7 @@ async fn test_mimeparser_with_context() { --==break==--\n\ \n"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -438,26 +434,26 @@ async fn test_mimeparser_with_avatars() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/mail_attach_txt.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.user_avatar, None); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_avatar.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert!(mimeparser.user_avatar.unwrap().is_change()); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_avatar_deleted.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert_eq!(mimeparser.user_avatar, Some(AvatarAction::Delete)); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert!(mimeparser.user_avatar.unwrap().is_change()); @@ -467,9 +463,7 @@ async fn test_mimeparser_with_avatars() { let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml"); let raw = String::from_utf8_lossy(raw).to_string(); let raw = raw.replace("Chat-User-Avatar:", "Xhat-Xser-Xvatar:"); - let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes(), None) - .await - .unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes()).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Image); assert_eq!(mimeparser.user_avatar, None); @@ -485,7 +479,7 @@ async fn test_mimeparser_with_videochat() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/videochat_invitation.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert_eq!(mimeparser.parts[0].param.get(Param::WebrtcRoom), None); @@ -528,7 +522,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\ --==break==--\n\ ;"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -578,7 +572,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -659,7 +653,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ --outer--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -706,7 +700,7 @@ Additional-Message-IDs: \n\ --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -753,7 +747,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== ------=_Part_25_46172632.1581201680436-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -797,7 +791,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== ------=_Part_25_46172632.1581201680436-- "#; - let message = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let message = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(message.parts.len(), 1); assert_eq!(message.parts[0].typ, Viewtype::File); @@ -839,7 +833,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu ----11019878869865180-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("example".to_string())); @@ -903,7 +897,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu --------------779C1631600DF3DB8C02E53A--"#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Test subject".to_string())); @@ -966,7 +960,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu ------=_NextPart_000_0003_01D622B3.CA753E60-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1064,7 +1058,7 @@ From: alice Reply "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1096,7 +1090,7 @@ From: alice > Just a quote. "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1130,7 +1124,7 @@ On 2020-10-25, Bob wrote: > A quote. "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Re: top posting".to_string())); @@ -1148,7 +1142,7 @@ On 2020-10-25, Bob wrote: async fn test_attachment_quote() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/quote_attach.eml"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -1166,7 +1160,7 @@ async fn test_attachment_quote() { async fn test_quote_div() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/gmx-quote.eml"); - let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap(); assert_eq!(mimeparser.parts[0].msg, "YIPPEEEEEE\n\nMulti-line"); assert_eq!(mimeparser.parts[0].param.get(Param::Quote).unwrap(), "Now?"); } @@ -1176,7 +1170,7 @@ async fn test_allinkl_blockquote() { // all-inkl.com puts quotes into `
`. let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/allinkl-quote.eml"); - let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap(); assert!(mimeparser.parts[0].msg.starts_with("It's 1.0.")); assert_eq!( mimeparser.parts[0].param.get(Param::Quote).unwrap(), @@ -1217,7 +1211,7 @@ async fn test_add_subj_to_multimedia_msg() { async fn test_mime_modified_plain() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_plain_unspecified.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(!mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1229,7 +1223,7 @@ async fn test_mime_modified_plain() { async fn test_mime_modified_alt_plain_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_plain_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1241,7 +1235,7 @@ async fn test_mime_modified_alt_plain_html() { async fn test_mime_modified_alt_plain() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_plain.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(!mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1256,7 +1250,7 @@ async fn test_mime_modified_alt_plain() { async fn test_mime_modified_alt_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1268,7 +1262,7 @@ async fn test_mime_modified_alt_html() { async fn test_mime_modified_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1288,7 +1282,7 @@ async fn test_mime_modified_large_plain() -> Result<()> { assert!(long_txt.len() > DC_DESIRED_TEXT_LEN); { - let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?; + let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; assert!(mimemsg.is_mime_modified); assert!( mimemsg.parts[0].msg.matches("just repeated").count() @@ -1321,7 +1315,7 @@ async fn test_mime_modified_large_plain() -> Result<()> { t.set_config(Config::Bot, Some("1")).await?; { - let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?; + let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; assert!(!mimemsg.is_mime_modified); assert_eq!( format!("{}\n", mimemsg.parts[0].msg), @@ -1368,7 +1362,7 @@ async fn test_x_microsoft_original_message_id() { MIME-Version: 1.0\n\ \n\ Does it work with outlook now?\n\ - ", None) + ") .await .unwrap(); assert_eq!( @@ -1418,7 +1412,7 @@ async fn test_extra_imf_headers() -> Result<()> { "Message-ID:", "Chat-Forty-Two: 42\r\nForty-Two: 42\r\nMessage-ID:", ); - let msg = MimeMessage::from_bytes(t, payload.as_bytes(), None).await?; + let msg = MimeMessage::from_bytes(t, payload.as_bytes()).await?; assert!(msg.headers.contains_key("chat-version")); assert!(!msg.headers.contains_key("chat-forty-two")); assert_ne!(msg.headers.contains_key("forty-two"), std_hp_composing); @@ -1582,7 +1576,7 @@ async fn test_ms_exchange_mdn() -> Result<()> { // 1. Test mimeparser directly let mdn = include_bytes!("../../test-data/message/ms_exchange_report_disposition_notification.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn, None).await?; + let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn).await?; assert_eq!(mimeparser.mdn_reports.len(), 1); assert_eq!( mimeparser.mdn_reports[0].original_message_id.as_deref(), @@ -1608,7 +1602,6 @@ async fn test_receive_eml() -> Result<()> { let mime_message = MimeMessage::from_bytes( &alice, include_bytes!("../../test-data/message/attached-eml.eml"), - None, ) .await?; @@ -1651,7 +1644,6 @@ Content-Disposition: reaction\n\ \n\ \u{1F44D}" .as_bytes(), - None, ) .await?; @@ -1673,7 +1665,7 @@ async fn test_jpeg_as_application_octet_stream() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/jpeg-as-application-octet-stream.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 1); @@ -1691,7 +1683,7 @@ async fn test_schleuder() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/schleuder.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 2); @@ -1711,7 +1703,7 @@ async fn test_tlsrpt() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/tlsrpt.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 1); @@ -1744,7 +1736,6 @@ async fn test_time_in_future() -> Result<()> { Content-Type: text/plain; charset=utf-8\n\ \n\ Hi", - None, ) .await?; @@ -1806,7 +1797,7 @@ Content-Type: text/plain; charset=utf-8 /help "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Some subject".to_string())); @@ -1847,7 +1838,7 @@ async fn test_take_last_header() { Hello\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1900,9 +1891,7 @@ It DOES end with a linebreak.\r \r This is the epilogue. It is also to be ignored."; - let mimeparser = MimeMessage::from_bytes(&context, &raw[..], None) - .await - .unwrap(); + let mimeparser = MimeMessage::from_bytes(&context, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 2); @@ -1948,7 +1937,7 @@ Message with a correct Message-ID hidden header --luTiGu6GBoVLCvTkzVtmZmwsmhkNMw-- "#; - let message = MimeMessage::from_bytes(t, &raw[..], None).await.unwrap(); + let message = MimeMessage::from_bytes(t, &raw[..]).await.unwrap(); assert_eq!(message.get_rfc724_mid().unwrap(), "foo@example.org"); } @@ -2126,9 +2115,7 @@ Third alternative. --boundary-- "#; - let message = MimeMessage::from_bytes(context, &raw[..], None) - .await - .unwrap(); + let message = MimeMessage::from_bytes(context, &raw[..]).await.unwrap(); assert_eq!(message.parts.len(), 1); assert_eq!(message.parts[0].typ, Viewtype::Text); assert_eq!(message.parts[0].msg, "Third alternative."); diff --git a/src/param.rs b/src/param.rs index 0640b551ef..01f23c7271 100644 --- a/src/param.rs +++ b/src/param.rs @@ -251,6 +251,13 @@ pub enum Param { /// For info messages: Contact ID in added or removed to a group. ContactAddedRemoved = b'5', + + /// For (pre-)Message: ViewType of the Post-Message, + /// because pre message is always `Viewtype::Text`. + PostMessageViewtype = b'8', + + /// For (pre-)Message: File byte size of Post-Message attachment + PostMessageFileBytes = b'9', } /// An object for handling key=value parameter lists. @@ -441,6 +448,15 @@ impl Params { } self } + + /// Merge in parameters from other Params struct, + /// overwriting the keys that are in both + /// with the values from the new Params struct. + pub fn merge_in_from_params(&mut self, new_params: Self) -> &mut Self { + let mut new_params = new_params; + self.inner.append(&mut new_params.inner); + self + } } #[cfg(test)] @@ -503,4 +519,18 @@ mod tests { assert_eq!(p.get(Param::Height), Some("14")); Ok(()) } + + #[test] + fn test_merge() -> Result<()> { + let mut p = Params::from_str("w=12\na=5\nh=14")?; + let p2 = Params::from_str("L=1\nh=17")?; + assert_eq!(p.len(), 3); + p.merge_in_from_params(p2); + assert_eq!(p.len(), 4); + assert_eq!(p.get(Param::Width), Some("12")); + assert_eq!(p.get(Param::Height), Some("17")); + assert_eq!(p.get(Param::Forwarded), Some("5")); + assert_eq!(p.get(Param::IsEdited), Some("1")); + Ok(()) + } } diff --git a/src/reaction.rs b/src/reaction.rs index 0a00f7aeb1..82a75ed546 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -392,9 +392,8 @@ mod tests { use crate::chatlist::Chatlist; use crate::config::Config; use crate::contact::{Contact, Origin}; - use crate::download::DownloadState; use crate::message::{MessageState, Viewtype, delete_msgs}; - use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; + use crate::receive_imf::receive_imf; use crate::sql::housekeeping; use crate::test_utils::E2EE_INFO_MSGS; use crate::test_utils::TestContext; @@ -924,73 +923,6 @@ Content-Disposition: reaction\n\ Ok(()) } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_and_reaction() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - - alice - .create_chat_with_contact("Bob", "bob@example.net") - .await; - - let msg_header = "From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain"; - let msg_full = format!("{msg_header}\n\n100k text..."); - - // Alice downloads message from Bob partially. - let alice_received_message = receive_imf_from_inbox( - &alice, - "first@example.org", - msg_header.as_bytes(), - false, - Some(100000), - ) - .await? - .unwrap(); - let alice_msg_id = *alice_received_message.msg_ids.first().unwrap(); - - // Bob downloads own message on the other device. - let bob_received_message = receive_imf(&bob, msg_full.as_bytes(), false) - .await? - .unwrap(); - let bob_msg_id = *bob_received_message.msg_ids.first().unwrap(); - - // Bob reacts to own message. - send_reaction(&bob, bob_msg_id, "👍").await.unwrap(); - let bob_reaction_msg = bob.pop_sent_msg().await; - - // Alice receives a reaction. - alice.recv_msg_hidden(&bob_reaction_msg).await; - - let reactions = get_msg_reactions(&alice, alice_msg_id).await?; - assert_eq!(reactions.to_string(), "👍1"); - let msg = Message::load_from_db(&alice, alice_msg_id).await?; - assert_eq!(msg.download_state(), DownloadState::Available); - - // Alice downloads full message. - receive_imf_from_inbox( - &alice, - "first@example.org", - msg_full.as_bytes(), - false, - None, - ) - .await?; - - // Check that reaction is still on the message after full download. - let msg = Message::load_from_db(&alice, alice_msg_id).await?; - assert_eq!(msg.download_state(), DownloadState::Done); - let reactions = get_msg_reactions(&alice, alice_msg_id).await?; - assert_eq!(reactions.to_string(), "👍1"); - - Ok(()) - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_send_reaction_multidevice() -> Result<()> { let mut tcm = TestContextManager::new(); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 0013443fc3..f6257ca113 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -20,16 +20,14 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc_inner; -use crate::download::DownloadState; +use crate::download::{DownloadState, premessage_is_downloaded_for}; use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed}; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::imap::{GENERATED_PREFIX, markseen_on_imap_table}; use crate::key::{DcKey, Fingerprint}; use crate::key::{self_fingerprint, self_fingerprint_opt}; -use crate::log::LogExt; -use crate::log::warn; -use crate::logged_debug_assert; +use crate::log::{LogExt as _, warn}; use crate::message::{ self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists, }; @@ -47,6 +45,7 @@ use crate::tools::{ self, buf_compress, normalize_text, remove_subject_prefix, validate_broadcast_secret, }; use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location}; +use crate::{logged_debug_assert, mimeparser}; /// This is the struct that is returned after receiving one email (aka MIME message). /// @@ -157,24 +156,7 @@ pub async fn receive_imf( let mail = mailparse::parse_mail(imf_raw).context("can't parse mail")?; let rfc724_mid = crate::imap::prefetch_get_message_id(&mail.headers) .unwrap_or_else(crate::imap::create_message_id); - if let Some(download_limit) = context.download_limit().await? { - let download_limit: usize = download_limit.try_into()?; - if imf_raw.len() > download_limit { - let head = std::str::from_utf8(imf_raw)? - .split("\r\n\r\n") - .next() - .context("No empty line in the message")?; - return receive_imf_from_inbox( - context, - &rfc724_mid, - head.as_bytes(), - seen, - Some(imf_raw.len().try_into()?), - ) - .await; - } - } - receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None).await + receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen).await } /// Emulates reception of a message from "INBOX". @@ -186,9 +168,8 @@ pub(crate) async fn receive_imf_from_inbox( rfc724_mid: &str, imf_raw: &[u8], seen: bool, - is_partial_download: Option, ) -> Result> { - receive_imf_inner(context, rfc724_mid, imf_raw, seen, is_partial_download).await + receive_imf_inner(context, rfc724_mid, imf_raw, seen).await } /// Inserts a tombstone into `msgs` table @@ -211,7 +192,6 @@ async fn get_to_and_past_contact_ids( context: &Context, mime_parser: &MimeMessage, chat_assignment: &ChatAssignment, - is_partial_download: Option, parent_message: &Option, incoming_origin: Origin, ) -> Result<(Vec>, Vec>)> { @@ -254,7 +234,7 @@ async fn get_to_and_past_contact_ids( ChatAssignment::ExistingChat { chat_id, .. } => Some(*chat_id), ChatAssignment::MailingListOrBroadcast => None, ChatAssignment::OneOneChat => { - if is_partial_download.is_none() && !mime_parser.incoming { + if !mime_parser.incoming { parent_message.as_ref().map(|m| m.chat_id) } else { None @@ -484,15 +464,17 @@ async fn get_to_and_past_contact_ids( /// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))`. /// If the message is so wrong that we didn't even create a database entry, /// returns `Ok(None)`. -/// -/// If `is_partial_download` is set, it contains the full message size in bytes. pub(crate) async fn receive_imf_inner( context: &Context, rfc724_mid: &str, imf_raw: &[u8], seen: bool, - is_partial_download: Option, ) -> Result> { + ensure!( + !context + .get_config_bool(Config::SimulateReceiveImfError) + .await? + ); if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { info!( context, @@ -500,16 +482,8 @@ pub(crate) async fn receive_imf_inner( String::from_utf8_lossy(imf_raw), ); } - if is_partial_download.is_none() { - ensure!( - !context - .get_config_bool(Config::FailOnReceivingFullMsg) - .await? - ); - } - let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw, is_partial_download).await - { + let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw).await { Err(err) => { warn!(context, "receive_imf: can't parse MIME: {err:#}."); if rfc724_mid.starts_with(GENERATED_PREFIX) { @@ -542,7 +516,15 @@ pub(crate) async fn receive_imf_inner( // check, if the mail is already in our database. // make sure, this check is done eg. before securejoin-processing. let (replace_msg_id, replace_chat_id); - if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + if mime_parser.pre_message == Some(mimeparser::PreMessageMode::PostMessage) { + // Post-Message just replace the attachment and mofified Params, not the whole message + // This is done in the `handle_post_message` method. + replace_msg_id = None; + replace_chat_id = None; + } else if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + // This code handles the download of old partial download stub messages + // It will be removed after a transitioning period, + // after we have released a few versions with pre-messages replace_msg_id = Some(old_msg_id); replace_chat_id = if let Some(msg) = Message::load_from_db_optional(context, old_msg_id) .await? @@ -615,11 +597,7 @@ pub(crate) async fn receive_imf_inner( &mime_parser.from, fingerprint, prevent_rename, - is_partial_download.is_some() - && mime_parser - .get_header(HeaderDef::ContentType) - .unwrap_or_default() - .starts_with("multipart/encrypted"), + false, ) .await? { @@ -651,22 +629,14 @@ pub(crate) async fn receive_imf_inner( .await? .filter(|p| Some(p.id) != replace_msg_id); - let chat_assignment = decide_chat_assignment( - context, - &mime_parser, - &parent_message, - rfc724_mid, - from_id, - &is_partial_download, - ) - .await?; + let chat_assignment = + decide_chat_assignment(context, &mime_parser, &parent_message, rfc724_mid, from_id).await?; info!(context, "Chat assignment is {chat_assignment:?}."); let (to_ids, past_ids) = get_to_and_past_contact_ids( context, &mime_parser, &chat_assignment, - is_partial_download, &parent_message, incoming_origin, ) @@ -773,7 +743,6 @@ pub(crate) async fn receive_imf_inner( to_id, allow_creation, &mut mime_parser, - is_partial_download, parent_message, ) .await?; @@ -789,7 +758,6 @@ pub(crate) async fn receive_imf_inner( rfc724_mid_orig, from_id, seen, - is_partial_download, replace_msg_id, prevent_rename, chat_id, @@ -957,9 +925,7 @@ pub(crate) async fn receive_imf_inner( let delete_server_after = context.get_config_delete_server_after().await?; if !received_msg.msg_ids.is_empty() { - let target = if received_msg.needs_delete_job - || (delete_server_after == Some(0) && is_partial_download.is_none()) - { + let target = if received_msg.needs_delete_job || delete_server_after == Some(0) { Some(context.get_delete_msgs_target().await?) } else { None @@ -988,7 +954,7 @@ pub(crate) async fn receive_imf_inner( } } - if is_partial_download.is_none() && mime_parser.is_call() { + if mime_parser.is_call() { context .handle_call_msg(insert_msg_id, &mime_parser, from_id) .await?; @@ -1037,7 +1003,7 @@ pub(crate) async fn receive_imf_inner( /// * `find_key_contact_by_addr`: if true, we only know the e-mail address /// of the contact, but not the fingerprint, /// yet want to assign the message to some key-contact. -/// This can happen during prefetch or when the message is partially downloaded. +/// This can happen during prefetch. /// If we get it wrong, the message will be placed into the correct /// chat after downloading. /// @@ -1131,7 +1097,6 @@ async fn decide_chat_assignment( parent_message: &Option, rfc724_mid: &str, from_id: ContactId, - is_partial_download: &Option, ) -> Result { let should_trash = if !mime_parser.mdn_reports.is_empty() { info!(context, "Message is an MDN (TRASH)."); @@ -1147,9 +1112,39 @@ async fn decide_chat_assignment( { info!(context, "Chat edit/delete/iroh/sync message (TRASH)."); true - } else if is_partial_download.is_none() - && (mime_parser.is_system_message == SystemMessage::CallAccepted - || mime_parser.is_system_message == SystemMessage::CallEnded) + } else if let Some(pre_message) = &mime_parser.pre_message { + use crate::mimeparser::PreMessageMode::*; + match pre_message { + PostMessage => { + // if pre message exist, then trash after replacing, otherwise treat as normal message + let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?; + info!( + context, + "Message is a Post-Message ({}).", + if pre_message_exists { + "pre-message exists already, so trash after replacing attachment" + } else { + "no pre-message -> Keep" + } + ); + pre_message_exists + } + PreMessage { + post_msg_rfc724_mid, + .. + } => { + // if post message already exists, then trash/ignore + let post_msg_exists = + premessage_is_downloaded_for(context, post_msg_rfc724_mid).await?; + info!( + context, + "Message is a Pre-Message (post_msg_exists:{post_msg_exists})." + ); + post_msg_exists + } + } + } else if mime_parser.is_system_message == SystemMessage::CallAccepted + || mime_parser.is_system_message == SystemMessage::CallEnded { info!(context, "Call state changed (TRASH)."); true @@ -1250,7 +1245,7 @@ async fn decide_chat_assignment( } } else if let Some(parent) = &parent_message { if let Some((chat_id, chat_id_blocked)) = - lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await? + lookup_chat_by_reply(context, mime_parser, parent).await? { // Try to assign to a chat based on In-Reply-To/References. ChatAssignment::ExistingChat { @@ -1272,7 +1267,7 @@ async fn decide_chat_assignment( } } else if let Some(parent) = &parent_message { if let Some((chat_id, chat_id_blocked)) = - lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await? + lookup_chat_by_reply(context, mime_parser, parent).await? { // Try to assign to a chat based on In-Reply-To/References. ChatAssignment::ExistingChat { @@ -1314,7 +1309,6 @@ async fn do_chat_assignment( to_id: ContactId, allow_creation: bool, mime_parser: &mut MimeMessage, - is_partial_download: Option, parent_message: Option, ) -> Result<(ChatId, Blocked, bool)> { let is_bot = context.get_config_bool(Config::Bot).await?; @@ -1365,7 +1359,6 @@ async fn do_chat_assignment( && let Some((new_chat_id, new_chat_id_blocked)) = create_group( context, mime_parser, - is_partial_download.is_some(), create_blocked, from_id, to_ids, @@ -1414,7 +1407,6 @@ async fn do_chat_assignment( to_ids, allow_creation || test_normal_chat.is_some(), create_blocked, - is_partial_download.is_some(), ) .await? { @@ -1496,7 +1488,6 @@ async fn do_chat_assignment( && let Some((new_chat_id, new_chat_id_blocked)) = create_group( context, mime_parser, - is_partial_download.is_some(), Blocked::Not, from_id, to_ids, @@ -1560,7 +1551,6 @@ async fn do_chat_assignment( to_ids, allow_creation, Blocked::Not, - is_partial_download.is_some(), ) .await? { @@ -1641,7 +1631,6 @@ async fn add_parts( rfc724_mid: &str, from_id: ContactId, seen: bool, - is_partial_download: Option, mut replace_msg_id: Option, prevent_rename: bool, mut chat_id: ChatId, @@ -1713,10 +1702,9 @@ async fn add_parts( .get_rfc724_mid() .unwrap_or(rfc724_mid.to_string()); - // Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded. - let mut ephemeral_timer = if is_partial_download.is_some() { - chat_id.get_ephemeral_timer(context).await? - } else if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) { + // Extract ephemeral timer from the message + let mut ephemeral_timer = if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) + { match value.parse::() { Ok(timer) => timer, Err(err) => { @@ -1919,7 +1907,6 @@ async fn add_parts( let chat_id = if better_msg .as_ref() .is_some_and(|better_msg| better_msg.is_empty()) - && is_partial_download.is_none() { DC_CHAT_ID_TRASH } else { @@ -1968,10 +1955,10 @@ async fn add_parts( } handle_edit_delete(context, mime_parser, from_id).await?; + handle_post_message(context, mime_parser, from_id).await?; - if is_partial_download.is_none() - && (mime_parser.is_system_message == SystemMessage::CallAccepted - || mime_parser.is_system_message == SystemMessage::CallEnded) + if mime_parser.is_system_message == SystemMessage::CallAccepted + || mime_parser.is_system_message == SystemMessage::CallEnded { if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) { if let Some(call) = @@ -2052,6 +2039,14 @@ async fn add_parts( } }; + if let Some(mimeparser::PreMessageMode::PreMessage { + metadata: Some(metadata), + .. + }) = &mime_parser.pre_message + { + param.apply_from_pre_msg_metadata(metadata); + }; + // If you change which information is skipped if the message is trashed, // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified); @@ -2095,14 +2090,20 @@ RETURNING id "#)?; let row_id: MsgId = stmt.query_row(params![ replace_msg_id, - rfc724_mid_orig, + if let Some(mimeparser::PreMessageMode::PreMessage {post_msg_rfc724_mid, .. }) = &mime_parser.pre_message { + post_msg_rfc724_mid + } else { rfc724_mid_orig }, if trash { DC_CHAT_ID_TRASH } else { chat_id }, if trash { ContactId::UNDEFINED } else { from_id }, if trash { ContactId::UNDEFINED } else { to_id }, sort_timestamp, if trash { 0 } else { mime_parser.timestamp_sent }, if trash { 0 } else { mime_parser.timestamp_rcvd }, - if trash { Viewtype::Unknown } else { typ }, + if trash { + Viewtype::Unknown + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + Viewtype::Text + } else { typ }, if trash { MessageState::Undefined } else { state }, if trash { MessengerMessage::No } else { is_dc_message }, if trash || hidden { "" } else { msg }, @@ -2114,7 +2115,11 @@ RETURNING id param.to_string() }, !trash && hidden, - if trash { 0 } else { part.bytes as isize }, + if trash { + 0 + } else { + part.bytes as isize + }, if save_mime_modified && !(trash || hidden) { mime_headers.clone() } else { @@ -2128,10 +2133,10 @@ RETURNING id if trash { 0 } else { ephemeral_timestamp }, if trash { DownloadState::Done - } else if is_partial_download.is_some() { - DownloadState::Available } else if mime_parser.decrypting_failed { DownloadState::Undecipherable + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + DownloadState::Available } else { DownloadState::Done }, @@ -2324,6 +2329,82 @@ async fn handle_edit_delete( Ok(()) } +async fn handle_post_message( + context: &Context, + mime_parser: &MimeMessage, + from_id: ContactId, +) -> Result<()> { + if let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message { + // if Pre-Message exist, replace attachment + // only replacing attachment ensures that doesn't overwrite the text if it was edited before. + let rfc724_mid = mime_parser + .get_rfc724_mid() + .context("expected Post-Message to have a message id")?; + + let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else { + warn!( + context, + "Download Post-Message: Database entry does not exist." + ); + return Ok(()); + }; + let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else { + // else: message is processed like a normal message + warn!( + context, + "Download Post-Message: pre message was not downloaded, yet so treat as normal message" + ); + return Ok(()); + }; + + if original_msg.from_id != from_id { + warn!(context, "Download Post-Message: Bad sender."); + return Ok(()); + } + if let Some(part) = mime_parser.parts.first() { + if !part.typ.has_file() { + warn!( + context, + "Download Post-Message: First mime part's message-viewtype has no file" + ); + return Ok(()); + } + + let edit_msg_showpadlock = part + .param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or_default(); + + if edit_msg_showpadlock || !original_msg.get_showpadlock() { + let mut new_params = original_msg.param.clone(); + new_params + .merge_in_from_params(part.param.clone()) + .remove(Param::PostMessageFileBytes) + .remove(Param::PostMessageViewtype); + context + .sql + .execute( + "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?", + ( + new_params.to_string(), + part.typ, + part.bytes as isize, + part.error.as_deref().unwrap_or_default(), + DownloadState::Done as u32, + original_msg.id, + ), + ) + .await?; + context.emit_msgs_changed(original_msg.chat_id, original_msg.id); + } else { + warn!(context, "Download Post-Message: Not encrypted."); + } + } + } + + Ok(()) +} + async fn tweak_sort_timestamp( context: &Context, mime_parser: &mut MimeMessage, @@ -2413,7 +2494,6 @@ async fn lookup_chat_by_reply( context: &Context, mime_parser: &MimeMessage, parent: &Message, - is_partial_download: &Option, ) -> Result> { // If the message is encrypted and has group ID, // lookup by reply should never be needed @@ -2445,10 +2525,7 @@ async fn lookup_chat_by_reply( } // Do not assign unencrypted messages to encrypted chats. - if is_partial_download.is_none() - && parent_chat.is_encrypted(context).await? - && !mime_parser.was_encrypted() - { + if parent_chat.is_encrypted(context).await? && !mime_parser.was_encrypted() { return Ok(None); } @@ -2465,18 +2542,7 @@ async fn lookup_or_create_adhoc_group( to_ids: &[Option], allow_creation: bool, create_blocked: Blocked, - is_partial_download: bool, ) -> Result> { - // Partial download may be an encrypted message with protected Subject header. We do not want to - // create a group with "..." or "Encrypted message" as a subject. The same is for undecipherable - // messages. Instead, assign the message to 1:1 chat with the sender. - if is_partial_download { - info!( - context, - "Ad-hoc group cannot be created from partial download." - ); - return Ok(None); - } if mime_parser.decrypting_failed { warn!( context, @@ -2612,11 +2678,9 @@ async fn is_probably_private_reply( /// than two members, a new ad hoc group is created. /// /// On success the function returns the created (chat_id, chat_blocked) tuple. -#[expect(clippy::too_many_arguments)] async fn create_group( context: &Context, mime_parser: &mut MimeMessage, - is_partial_download: bool, create_blocked: Blocked, from_id: ContactId, to_ids: &[Option], @@ -2698,7 +2762,7 @@ async fn create_group( if let Some(chat_id) = chat_id { Ok(Some((chat_id, chat_id_blocked))) - } else if is_partial_download || mime_parser.decrypting_failed { + } else if mime_parser.decrypting_failed { // It is possible that the message was sent to a valid, // yet unknown group, which was rejected because // Chat-Group-Name, which is in the encrypted part, was diff --git a/src/receive_imf/receive_imf_tests.rs b/src/receive_imf/receive_imf_tests.rs index 40a6eddd05..6eafde62b3 100644 --- a/src/receive_imf/receive_imf_tests.rs +++ b/src/receive_imf/receive_imf_tests.rs @@ -10,7 +10,6 @@ use crate::chat::{ use crate::chatlist::Chatlist; use crate::constants::DC_GCL_FOR_FORWARDING; use crate::contact; -use crate::download::MIN_DOWNLOAD_LIMIT; use crate::imap::prefetch_should_download; use crate::imex::{ImexMode, imex}; use crate::securejoin::get_securejoin_qr; @@ -19,8 +18,6 @@ use crate::test_utils::{ }; use crate::tools::{SystemTime, time}; -use rand::distr::SampleString; - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_outgoing() -> Result<()> { let context = TestContext::new_alice().await; @@ -28,7 +25,7 @@ async fn test_outgoing() -> Result<()> { From: alice@example.org\n\ \n\ hello"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await?; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await?; assert_eq!(mimeparser.incoming, false); Ok(()) } @@ -43,7 +40,7 @@ async fn test_bad_from() { References: \n\ \n\ hello\x00"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await; assert!(mimeparser.is_err()); } @@ -2842,7 +2839,7 @@ References: Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no Message with references."#; - let mime_parser = MimeMessage::from_bytes(&t, &mime[..], None).await?; + let mime_parser = MimeMessage::from_bytes(&t, &mime[..]).await?; let parent = get_parent_message(&t, &mime_parser).await?.unwrap(); assert_eq!(parent.id, first.id); @@ -4417,37 +4414,6 @@ async fn test_adhoc_grp_name_no_prefix() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_download_later() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - - let bob = tcm.bob().await; - let bob_chat = bob.create_chat(&alice).await; - - // Generate a random string so OpenPGP does not compress it. - let text = - rand::distr::Alphanumeric.sample_string(&mut rand::rng(), MIN_DOWNLOAD_LIMIT as usize); - - let sent_msg = bob.send_text(bob_chat.id, &text).await; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(msg.state, MessageState::InFresh); - - let hi_msg = tcm.send_recv(&bob, &alice, "hi").await; - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); - assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); - - Ok(()) -} - /// Malice can pretend they have the same address as Alice and sends a message encrypted to Alice's /// key but signed with another one. Alice must detect that this message is wrongly signed and not /// treat it as Autocrypt-encrypted. @@ -4482,162 +4448,6 @@ async fn test_outgoing_msg_forgery() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_create_group_with_big_msg() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - let bob = tcm.bob().await; - let ba_contact = bob.add_or_lookup_contact_id(&alice).await; - let ab_chat_id = alice.create_chat(&bob).await.id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - - let bob_grp_id = create_group(&bob, "Group").await?; - add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; - assert!(msg.get_showpadlock()); - - alice.set_config(Config::DownloadLimit, Some("1")).await?; - assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - let alice_chat = Chat::load_from_db(&alice, msg.chat_id).await?; - // Incomplete message is assigned to 1:1 chat. - assert_eq!(alice_chat.typ, Chattype::Single); - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.viewtype, Viewtype::Image); - assert_ne!(msg.chat_id, alice_chat.id); - let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; - assert_eq!(alice_grp.typ, Chattype::Group); - assert_eq!(alice_grp.name, "Group"); - assert_eq!( - chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), - 2 - ); - - // Now Bob can send encrypted messages to Alice. - - let bob_grp_id = create_group(&bob, "Group1").await?; - add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; - assert!(msg.get_showpadlock()); - - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - // Until fully downloaded, an encrypted message must sit in the 1:1 chat. - assert_eq!(msg.chat_id, ab_chat_id); - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.viewtype, Viewtype::Image); - assert_ne!(msg.chat_id, ab_chat_id); - let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; - assert_eq!(alice_grp.typ, Chattype::Group); - assert_eq!(alice_grp.name, "Group1"); - assert_eq!( - chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), - 2 - ); - - // The big message must go away from the 1:1 chat. - let msgs = chat::get_chat_msgs(&alice, ab_chat_id).await?; - assert_eq!(msgs.len(), E2EE_INFO_MSGS); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_group_consistency() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - let bob = tcm.bob().await; - let fiona = tcm.fiona().await; - let bob_id = alice.add_or_lookup_contact_id(&bob).await; - let alice_chat_id = create_group(&alice, "foos").await?; - add_contact_to_chat(&alice, alice_chat_id, bob_id).await?; - - send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?; - let add = alice.pop_sent_msg().await; - bob.recv_msg(&add).await; - let bob_chat_id = bob.get_last_msg().await.chat_id; - let contacts = get_chat_contacts(&bob, bob_chat_id).await?; - assert_eq!(contacts.len(), 2); - - // Bob receives partial message. - let msg_id = receive_imf_from_inbox( - &bob, - "first@example.org", - b"From: Alice \n\ -To: , \n\ -Chat-Version: 1.0\n\ -Subject: subject\n\ -Message-ID: \n\ -Date: Sun, 14 Nov 2021 00:10:00 +0000\ -Content-Type: text/plain -Chat-Group-Member-Added: charlie@example.com", - false, - Some(100000), - ) - .await? - .context("no received message")?; - - let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?; - - // Partial download does not change the member list. - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts); - - // Alice sends normal message to bob, adding fiona. - add_contact_to_chat( - &alice, - alice_chat_id, - alice.add_or_lookup_contact_id(&fiona).await, - ) - .await?; - - bob.recv_msg(&alice.pop_sent_msg().await).await; - - let contacts = get_chat_contacts(&bob, bob_chat_id).await?; - assert_eq!(contacts.len(), 3); - - // Bob fully receives the partial message. - let msg_id = receive_imf_from_inbox( - &bob, - "first@example.org", - b"From: Alice \n\ -To: Bob \n\ -Chat-Version: 1.0\n\ -Subject: subject\n\ -Message-ID: \n\ -Date: Sun, 14 Nov 2021 00:10:00 +0000\ -Content-Type: text/plain -Chat-Group-Member-Added: charlie@example.com", - false, - None, - ) - .await? - .context("no received message")?; - - let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?; - - // After full download, the old message should not change group state. - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_protected_group_add_remove_member_missing_key() -> Result<()> { let mut tcm = TestContextManager::new(); @@ -4876,48 +4686,6 @@ async fn test_references() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_prefer_references_to_downloaded_msgs() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - bob.set_config(Config::DownloadLimit, Some("1")).await?; - let fiona = &tcm.fiona().await; - let alice_bob_id = tcm.send_recv(bob, alice, "hi").await.from_id; - let alice_fiona_id = tcm.send_recv(fiona, alice, "hi").await.from_id; - let alice_chat_id = create_group(alice, "Group").await?; - add_contact_to_chat(alice, alice_chat_id, alice_bob_id).await?; - // W/o fiona the test doesn't work -- the last message is assigned to the 1:1 chat due to - // `is_probably_private_reply()`. - add_contact_to_chat(alice, alice_chat_id, alice_fiona_id).await?; - let sent = alice.send_text(alice_chat_id, "Hi").await; - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Done); - let bob_chat_id = received.chat_id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.gif"); - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "file", file_bytes, None)?; - let mut sent = alice.send_msg(alice_chat_id, &mut msg).await; - sent.payload = sent - .payload - .replace("References:", "X-Microsoft-Original-References:") - .replace("In-Reply-To:", "X-Microsoft-Original-In-Reply-To:"); - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Available); - assert_ne!(received.chat_id, bob_chat_id); - assert_eq!(received.chat_id, bob.get_chat(alice).await.id); - - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "file", file_bytes, None)?; - let sent = alice.send_msg(alice_chat_id, &mut msg).await; - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Available); - assert_eq!(received.chat_id, bob_chat_id); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_list_from() -> Result<()> { let t = &TestContext::new_alice().await; @@ -5395,41 +5163,6 @@ async fn test_outgoing_plaintext_two_member_group() -> Result<()> { Ok(()) } -/// Tests that large messages are assigned -/// to non-key-contacts if the type is not `multipart/encrypted`. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_download_key_contact_lookup() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - - // Create two chats with Alice, both with key-contact and email address contact. - let encrypted_chat = bob.create_chat(alice).await; - let unencrypted_chat = bob.create_email_chat(alice).await; - - let seen = false; - let is_partial_download = Some(9999); - let received = receive_imf_from_inbox( - bob, - "3333@example.org", - b"From: alice@example.org\n\ - To: bob@example.net\n\ - Message-ID: <3333@example.org>\n\ - Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ - \n\ - hello\n", - seen, - is_partial_download, - ) - .await? - .unwrap(); - - assert_ne!(received.chat_id, encrypted_chat.id); - assert_eq!(received.chat_id, unencrypted_chat.id); - - Ok(()) -} - /// Tests that outgoing unencrypted message /// is assigned to a chat with email-contact. /// diff --git a/src/scheduler.rs b/src/scheduler.rs index 7dbd625761..0f33301891 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -14,13 +14,12 @@ pub(crate) use self::connectivity::ConnectivityStore; use crate::config::{self, Config}; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; -use crate::download::{DownloadState, download_msg}; +use crate::download::{download_known_post_messages_without_pre_message, download_msgs}; use crate::ephemeral::{self, delete_expired_imap_messages}; use crate::events::EventType; use crate::imap::{FolderMeaning, Imap, session::Session}; use crate::location; use crate::log::{LogExt, warn}; -use crate::message::MsgId; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; use crate::stats::maybe_send_stats; @@ -351,38 +350,6 @@ pub(crate) struct Scheduler { recently_seen_loop: RecentlySeenLoop, } -async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { - let msg_ids = context - .sql - .query_map_vec("SELECT msg_id FROM download", (), |row| { - let msg_id: MsgId = row.get(0)?; - Ok(msg_id) - }) - .await?; - - for msg_id in msg_ids { - if let Err(err) = download_msg(context, msg_id, session).await { - warn!(context, "Failed to download message {msg_id}: {:#}.", err); - - // Update download state to failure - // so it can be retried. - // - // On success update_download_state() is not needed - // as receive_imf() already - // set the state and emitted the event. - msg_id - .update_download_state(context, DownloadState::Failure) - .await?; - } - context - .sql - .execute("DELETE FROM download WHERE msg_id=?", (msg_id,)) - .await?; - } - - Ok(()) -} - async fn inbox_loop( ctx: Context, started: oneshot::Sender<()>, @@ -534,9 +501,6 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } } - download_msgs(ctx, &mut session) - .await - .context("Failed to download messages")?; session .update_metadata(ctx) .await @@ -619,6 +583,11 @@ async fn fetch_idle( delete_expired_imap_messages(ctx) .await .context("delete_expired_imap_messages")?; + + //------- + // TODO: verify that this is the correct position for this call + // in order to guard against lost pre-messages: + download_known_post_messages_without_pre_message(ctx, &mut session).await?; } else if folder_config == Config::ConfiguredInboxFolder { session.last_full_folder_scan.lock().await.take(); } @@ -663,6 +632,10 @@ async fn fetch_idle( .log_err(ctx) .ok(); + download_msgs(ctx, &mut session) + .await + .context("Failed to download messages")?; + connection.connectivity.set_idle(ctx); ctx.emit_event(EventType::ImapInboxIdle); @@ -704,6 +677,7 @@ async fn fetch_idle( Ok(session) } +/// The simplified IMAP IDLE loop to watch non primary folders (non-inbox folders) async fn simple_imap_loop( ctx: Context, started: oneshot::Sender<()>, diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 80ac2c564a..bfec5b69bc 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1466,6 +1466,30 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT; .await?; } + inc_and_check(&mut migration_version, 144)?; + if dbversion < migration_version { + // `msg_id` in `download` table is not needed anymore, + // but we still keep it so that it's possible to import a backup into an older DC version, + // because we don't always release at the same time on all platforms. + sql.execute_migration( + "CREATE TABLE download_new ( + rfc724_mid TEXT NOT NULL DEFAULT '', + msg_id INTEGER NOT NULL DEFAULT 0 + ) STRICT; + INSERT OR IGNORE INTO download_new (rfc724_mid, msg_id) + SELECT m.rfc724_mid, d.msg_id FROM download d + JOIN msgs m ON d.msg_id = m.id + WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != ''; + DROP TABLE download; + ALTER TABLE download_new RENAME TO download; + CREATE TABLE available_post_msgs ( + rfc724_mid TEXT NOT NULL + );", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? diff --git a/src/stock_str.rs b/src/stock_str.rs index 291141f0d9..68c65c46f0 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -4,7 +4,6 @@ use std::collections::HashMap; use std::sync::Arc; use anyhow::{Result, bail}; -use humansize::{BINARY, format_size}; use strum::EnumProperty as EnumPropertyTrait; use strum_macros::EnumProperty; use tokio::sync::RwLock; @@ -17,7 +16,6 @@ use crate::contact::{Contact, ContactId}; use crate::context::Context; use crate::message::{Message, Viewtype}; use crate::param::Param; -use crate::tools::timestamp_to_str; /// Storage for string translations. #[derive(Debug, Clone)] @@ -167,12 +165,6 @@ pub enum StockMessage { ))] QuotaExceedingMsgBody = 98, - #[strum(props(fallback = "%1$s message"))] - PartialDownloadMsgBody = 99, - - #[strum(props(fallback = "Download maximum available until %1$s"))] - DownloadAvailability = 100, - #[strum(props(fallback = "Multi Device Synchronization"))] SyncMsgSubject = 101, @@ -1119,21 +1111,6 @@ pub(crate) async fn quota_exceeding(context: &Context, highest_usage: u64) -> St .replace("%%", "%") } -/// Stock string: `%1$s message` with placeholder replaced by human-readable size. -pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) -> String { - let size = &format_size(org_bytes, BINARY); - translated(context, StockMessage::PartialDownloadMsgBody) - .await - .replace1(size) -} - -/// Stock string: `Download maximum available until %1$s`. -pub(crate) async fn download_availability(context: &Context, timestamp: i64) -> String { - translated(context, StockMessage::DownloadAvailability) - .await - .replace1(×tamp_to_str(timestamp)) -} - /// Stock string: `Incoming Messages`. pub(crate) async fn incoming_messages(context: &Context) -> String { translated(context, StockMessage::IncomingMessages).await @@ -1262,6 +1239,24 @@ pub(crate) async fn chat_unencrypted_explanation(context: &Context) -> String { translated(context, StockMessage::ChatUnencryptedExplanation).await } +impl Viewtype { + /// returns Localized name for message viewtype + pub async fn to_locale_string(&self, context: &Context) -> String { + match self { + Viewtype::Image => image(context).await, + Viewtype::Gif => gif(context).await, + Viewtype::Sticker => sticker(context).await, + Viewtype::Audio => audio(context).await, + Viewtype::Voice => voice_message(context).await, + Viewtype::Video => video(context).await, + Viewtype::File => file(context).await, + Viewtype::Webxdc => "Mini App".to_owned(), + Viewtype::Vcard => "👤".to_string(), + Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(), + } + } +} + impl Context { /// Set the stock string for the [StockMessage]. /// diff --git a/src/stock_str/stock_str_tests.rs b/src/stock_str/stock_str_tests.rs index 8af98be53f..37c03efabf 100644 --- a/src/stock_str/stock_str_tests.rs +++ b/src/stock_str/stock_str_tests.rs @@ -118,14 +118,6 @@ async fn test_quota_exceeding_stock_str() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_download_msg_body() -> Result<()> { - let t = TestContext::new().await; - let str = partial_download_msg_body(&t, 1024 * 1024).await; - assert_eq!(str, "1 MiB message"); - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_update_device_chats() { let t = TestContext::new_alice().await; diff --git a/src/summary.rs b/src/summary.rs index 90ddd1e35e..f47d100bb9 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -262,7 +262,7 @@ impl Message { } }; - let text = self.text.clone(); + let text = self.text.clone() + &self.additional_text; let summary = if let Some(type_file) = type_file { if append_text && !text.is_empty() { diff --git a/src/test_utils.rs b/src/test_utils.rs index 8f837ebf11..0503e2e274 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -10,6 +10,7 @@ use std::path::Path; use std::sync::{Arc, LazyLock}; use std::time::{Duration, Instant}; +use anyhow::Result; use async_channel::{self as channel, Receiver, Sender}; use chat::ChatItem; use deltachat_contact_tools::{ContactAddress, EmailAddress}; @@ -711,6 +712,32 @@ impl TestContext { }) } + pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec> { + self.ctx + .sql + .query_map_vec( + "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?", + (msg_id,), + |row| { + let _id: MsgId = row.get(0)?; + let msg_id: MsgId = row.get(1)?; + let mime: String = row.get(2)?; + let recipients: String = row.get(3)?; + Ok((msg_id, mime, recipients)) + }, + ) + .await + .unwrap() + .into_iter() + .map(|(msg_id, mime, recipients)| SentMessage { + payload: mime, + sender_msg_id: msg_id, + sender_context: &self.ctx, + recipients, + }) + .collect() + } + /// Retrieves a sent sync message from the db. /// /// This retrieves and removes a sync message which has been scheduled to send from the jobs @@ -759,7 +786,7 @@ impl TestContext { /// unlikely to be affected as the message would be processed again in exactly the /// same way. pub(crate) async fn parse_msg(&self, msg: &SentMessage<'_>) -> MimeMessage { - MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes(), None) + MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes()) .await .unwrap() } @@ -1702,6 +1729,21 @@ Until the false-positive is fixed: } } +/// Method to create a test image file +pub(crate) fn create_test_image(width: u32, height: u32) -> Result> { + use image::{ImageBuffer, Rgb, RgbImage}; + use std::io::Cursor; + + let mut img: RgbImage = ImageBuffer::new(width, height); + // fill with some pattern so it stays large after compression + for (x, y, pixel) in img.enumerate_pixels_mut() { + *pixel = Rgb([(x % 255) as u8, (x + y % 255) as u8, (y % 255) as u8]); + } + let mut bytes: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)?; + Ok(bytes) +} + mod tests { use super::*; diff --git a/src/tests.rs b/src/tests.rs index 6e642dce74..b7ae08fbad 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,3 +1,4 @@ mod account_events; mod aeap; +mod pre_messages; mod verified_chats; diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs new file mode 100644 index 0000000000..a0387ddee4 --- /dev/null +++ b/src/tests/pre_messages.rs @@ -0,0 +1,6 @@ +mod additional_text; +mod forward_and_save; +mod legacy; +mod receiving; +mod sending; +mod util; diff --git a/src/tests/pre_messages/additional_text.rs b/src/tests/pre_messages/additional_text.rs new file mode 100644 index 0000000000..b894d34e98 --- /dev/null +++ b/src/tests/pre_messages/additional_text.rs @@ -0,0 +1,40 @@ +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::message::Viewtype; +use crate::test_utils::TestContextManager; +use crate::tests::pre_messages::util::{ + send_large_file_message, send_large_image_message, send_large_webxdc_message, +}; + +/// Test the addition of the download info to message text +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_additional_text_on_different_viewtypes() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let a_group_id = alice.create_group_with_members("test group", &[bob]).await; + + tcm.section("Test metadata preview text for File"); + let (pre_message, _, _) = + send_large_file_message(alice, a_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_text(), "test [test.bin - 976.56 KiB]".to_owned()); + + tcm.section("Test metadata preview text for webxdc app"); + let (pre_message, _, _) = send_large_webxdc_message(alice, a_group_id).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Webxdc)); + assert_eq!(msg.get_text(), "test [Mini App - 976.68 KiB]".to_owned()); + + tcm.section("Test metadata preview text for Image"); + + let (pre_message, _, _) = send_large_image_message(alice, a_group_id).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_text(), "test [Image - 146.12 KiB]".to_owned()); + + Ok(()) +} diff --git a/src/tests/pre_messages/forward_and_save.rs b/src/tests/pre_messages/forward_and_save.rs new file mode 100644 index 0000000000..a3ca16ed8a --- /dev/null +++ b/src/tests/pre_messages/forward_and_save.rs @@ -0,0 +1,122 @@ +//! Tests about forwarding and saving Pre-Messages +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::chat::{self}; +use crate::chat::{forward_msgs, save_msgs}; +use crate::chatlist::get_last_message_for_chat; +use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD}; +use crate::message::{Message, Viewtype}; +use crate::test_utils::TestContextManager; + +/// Test that forwarding Pre-Message should forward additional text to not be empty +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_forwarding_pre_message_empty_text() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let pre_message = { + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 2); + smtp_rows.first().expect("Pre-Message exists").to_owned() + }; + + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state, DownloadState::Available); + bob_msg.chat_id.accept(bob).await?; + tcm.section("forward pre message and check it on bobs side"); + forward_msgs(bob, &[bob_msg.id], bob_msg.chat_id).await?; + let forwarded_msg_id = get_last_message_for_chat(bob, bob_msg.chat_id) + .await? + .unwrap(); + let forwarded_msg = Message::load_from_db(bob, forwarded_msg_id).await?; + assert_eq!(forwarded_msg.is_forwarded(), true); + assert_eq!(forwarded_msg.download_state(), DownloadState::Done); + assert_eq!( + forwarded_msg + .param + .exists(crate::param::Param::PostMessageFileBytes), + false, + "PostMessageFileBytes not set" + ); + assert_eq!( + forwarded_msg + .param + .exists(crate::param::Param::PostMessageViewtype), + false, + "PostMessageViewtype not set" + ); + assert_eq!( + forwarded_msg.get_text(), + " [test.bin - 976.56 KiB]".to_owned() + ); + assert_eq!(forwarded_msg.get_viewtype(), Viewtype::Text); + assert!(forwarded_msg.additional_text.is_empty()); + tcm.section("check it on alices side"); + let sent_forward_msg = bob.pop_sent_msg().await; + let alice_forwarded_msg = alice.recv_msg(&sent_forward_msg).await; + assert!(alice_forwarded_msg.additional_text.is_empty()); + assert_eq!(alice_forwarded_msg.is_forwarded(), true); + assert_eq!(alice_forwarded_msg.download_state(), DownloadState::Done); + assert_eq!( + alice_forwarded_msg + .param + .exists(crate::param::Param::PostMessageFileBytes), + false, + "PostMessageFileBytes not set" + ); + assert_eq!( + alice_forwarded_msg + .param + .exists(crate::param::Param::PostMessageViewtype), + false, + "PostMessageViewtype not set" + ); + assert_eq!( + alice_forwarded_msg.get_text(), + " [test.bin - 976.56 KiB]".to_owned() + ); + + Ok(()) +} + +/// Test that forwarding Pre-Message should forward additional text to not be empty +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_saving_pre_message_empty_text() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let pre_message = { + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 2); + smtp_rows.first().expect("Pre-Message exists").to_owned() + }; + + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state, DownloadState::Available); + bob_msg.chat_id.accept(bob).await?; + tcm.section("save pre message and check it"); + save_msgs(bob, &[bob_msg.id]).await?; + let saved_msg_id = get_last_message_for_chat(bob, bob.get_self_chat().await.id) + .await? + .unwrap(); + let saved_msg = Message::load_from_db(bob, saved_msg_id).await?; + assert!(saved_msg.additional_text.is_empty()); + assert!(saved_msg.get_original_msg_id(bob).await?.is_some()); + assert_eq!(saved_msg.download_state(), DownloadState::Done); + assert_eq!(saved_msg.get_text(), " [test.bin - 976.56 KiB]".to_owned()); + + Ok(()) +} diff --git a/src/tests/pre_messages/legacy.rs b/src/tests/pre_messages/legacy.rs new file mode 100644 index 0000000000..f8f086d1bd --- /dev/null +++ b/src/tests/pre_messages/legacy.rs @@ -0,0 +1,61 @@ +//! Test that downloading old stub messages still works +use anyhow::Result; + +use crate::download::DownloadState; +use crate::receive_imf::receive_imf_from_inbox; +use crate::test_utils::TestContext; + +// The code for downloading stub messages stays +// during the transition perios to pre-messages +// so people can still download their files shortly after they updated. +// After there are a few release with pre-message rolled out, +// we will remove the ability to download stub messages and replace the following test +// so it checks that it doesn't crash or that the messages are replaced by sth. +// like "download failed/expired, please ask sender to send it again" +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_download_stub_message() -> Result<()> { + let t = TestContext::new_alice().await; + + let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: bob@example.com\n\ + To: alice@example.org\n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\ + Content-Type: text/plain"; + + t.sql + .execute( + r#"INSERT INTO chats VALUES( + 11001,100,'bob@example.com',0,'',2,'', + replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0,''); + "#, + (), + ) + .await?; + t.sql.execute(r#"INSERT INTO msgs VALUES( + 11001,'Mr.12345678901@example.com','',0, + 11001,11001,1,1763151754,10,10,1,0, + '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'', + '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0); + "#, ()).await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.get_subject(), "foo"); + assert!(msg.get_text().contains("[97.66 KiB message]")); + + receive_imf_from_inbox( + &t, + "Mr.12345678901@example.com", + format!("{header}\n\n100k text...").as_bytes(), + false, + ) + .await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.get_subject(), "foo"); + assert_eq!(msg.get_text(), "100k text..."); + + Ok(()) +} diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs new file mode 100644 index 0000000000..fd9c376c7a --- /dev/null +++ b/src/tests/pre_messages/receiving.rs @@ -0,0 +1,522 @@ +//! Tests about receiving Pre-Messages and Post-Message +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::EventType; +use crate::chat; +use crate::contact; +use crate::download::{ + DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, pre_msg_metadata::PreMsgMetadata, +}; +use crate::message::{Message, MessageState, Viewtype, delete_msgs, markseen_msgs}; +use crate::mimeparser::MimeMessage; +use crate::param::Param; +use crate::reaction::{get_msg_reactions, send_reaction}; +use crate::test_utils::TestContextManager; +use crate::tests::pre_messages::util::{ + send_large_file_message, send_large_image_message, send_large_webxdc_message, +}; +use crate::webxdc::StatusUpdateSerial; + +/// Test that mimeparser can correctly detect and parse pre-messages and Post-Messages +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_mimeparser_pre_message_and_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let parsed_pre_message = MimeMessage::from_bytes(bob, pre_message.payload.as_bytes()).await?; + let parsed_post_message = MimeMessage::from_bytes(bob, post_message.payload.as_bytes()).await?; + + assert_eq!( + parsed_post_message.pre_message, + Some(crate::mimeparser::PreMessageMode::PostMessage) + ); + + assert_eq!( + parsed_pre_message.pre_message, + Some(crate::mimeparser::PreMessageMode::PreMessage { + post_msg_rfc724_mid: parsed_post_message.get_rfc724_mid().unwrap(), + metadata: Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None + }) + }) + ); + + Ok(()) +} + +/// Test receiving pre-messages and creation of the placeholder message with the metadata +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, _post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_filebytes(bob).await?, Some(1_000_000)); + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::File)); + assert_eq!(msg.get_filename(), Some("test.bin".to_owned())); + + Ok(()) +} + +/// Test receiving the Post-Message after receiving the pre-message +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_and_dl_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert!(msg.param.exists(Param::PostMessageViewtype)); + assert!(msg.param.exists(Param::PostMessageFileBytes)); + assert_eq!(msg.text, "test".to_owned()); + let _ = bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.param.exists(Param::PostMessageViewtype), false); + assert_eq!(msg.param.exists(Param::PostMessageFileBytes), false); + assert_eq!(msg.text, "test".to_owned()); + Ok(()) +} + +/// Test out of order receiving. Post-Message is received & downloaded before pre-message. +/// In that case pre-message shall be trashed. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_out_of_order_receiving() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&post_message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + let _ = bob.recv_msg_trash(&pre_message).await; + Ok(()) +} + +/// Test receiving the Post-Message after receiving an edit after receiving the pre-message +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_then_edit_and_then_dl_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + chat::send_edit_request(alice, alice_msg_id, "new_text".to_owned()).await?; + let edit_request = alice.pop_sent_msg().await; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "test".to_owned()); + let _ = bob.recv_msg_trash(&edit_request).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "new_text".to_owned()); + let _ = bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "new_text".to_owned()); + Ok(()) +} + +/// Process normal message with file attachment (neither post nor pre message) +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_normal_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes( + alice, + "test.bin", + &vec![0u8; (PRE_MSG_ATTACHMENT_SIZE_THRESHOLD - 10_000) as usize], + None, + )?; + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 1); + let message = smtp_rows.first().expect("message exists"); + + let msg = bob.recv_msg(message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "test".to_owned()); + Ok(()) +} + +/// Test receiving pre-messages and creation of the placeholder message with the metadata +/// for image attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_image() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, _post_message, _alice_msg_id) = + send_large_image_message(alice, alice_group_id).await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Image)); + // recoded image dimensions + assert_eq!(msg.get_filebytes(bob).await?, Some(149632)); + assert_eq!(msg.get_height(), 1280); + assert_eq!(msg.get_width(), 720); + + Ok(()) +} + +/// Test receiving reaction on pre-message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_reaction_on_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + // Bob receives pre-message + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state(), DownloadState::Available); + + // Alice sends reaction to her own message + send_reaction(alice, alice_msg_id, "👍").await?; + + // Bob receives the reaction + bob.recv_msg_hidden(&alice.pop_sent_msg().await).await; + + // Test if Bob sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + // Bob downloads Post-Message + bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, bob_msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + + // Test if Bob still sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + Ok(()) +} + +/// Tests that fully downloading the message +/// works but does not reappear when it was already deleted +/// (as in the Message-ID already exists in the database +/// and is assigned to the trash chat). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_full_download_after_trashed() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_group_id = bob.create_group_with_members("test group", &[alice]).await; + + let (pre_message, post_message, _bob_msg_id) = + send_large_file_message(bob, bob_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + // Download message from Bob partially. + let alice_msg = alice.recv_msg(&pre_message).await; + + // Delete the received message. + // Note that it remains in the database in the trash chat. + delete_msgs(alice, &[alice_msg.id]).await?; + + // Fully download message after deletion. + alice.recv_msg_trash(&post_message).await; + + // The message does not reappear. + let msg = Message::load_from_db_optional(bob, alice_msg.id).await?; + assert!(msg.is_none()); + + Ok(()) +} + +/// Test that webxdc updates are received for pre-messages +/// and available when the Post-Message is downloaded +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + // Alice sends a larger instance and an update + let (pre_message, post_message, alice_sent_instance_msg_id) = + send_large_webxdc_message(alice, alice_group_id).await?; + alice + .send_webxdc_status_update( + alice_sent_instance_msg_id, + r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, + ) + .await?; + alice.flush_status_updates().await?; + let webxdc_update = alice.pop_sent_msg().await; + + // Bob does not download instance but already receives update + let bob_instance = bob.recv_msg(&pre_message).await; + assert_eq!(bob_instance.download_state, DownloadState::Available); + bob.recv_msg_trash(&webxdc_update).await; + + // Bob downloads instance, updates should be assigned correctly + bob.recv_msg_trash(&post_message).await; + + let bob_instance = bob.get_last_msg().await; + assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); + assert_eq!(bob_instance.download_state, DownloadState::Done); + assert_eq!( + bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial::new(0)) + .await?, + r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# + ); + let info = bob_instance.get_webxdc_info(bob).await?; + assert_eq!(info.document, "doc"); + assert_eq!(info.summary, "sum"); + + Ok(()) +} + +/// Test mark seen pre-message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_markseen_pre_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Bob sends a large message to Alice"); + let (pre_message, post_message, _bob_msg_id) = + send_large_file_message(bob, bob_chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Alice receives a pre-message message from Bob"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!(msg.state, MessageState::InFresh); + + tcm.section("Alice marks the pre-message as read and sends a MDN"); + markseen_msgs(alice, vec![msg.id]).await?; + assert_eq!(msg.id.get_state(alice).await?, MessageState::InSeen); + assert_eq!( + alice + .sql + .count("SELECT COUNT(*) FROM smtp_mdns", ()) + .await?, + 1 + ); + + tcm.section("Alice downloads message"); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!( + msg.state, + MessageState::InSeen, + "The message state mustn't be downgraded to `InFresh`" + ); + + Ok(()) +} + +/// Test that pre-message can start a chat +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_pre_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (pre_message, _post_message, _alice_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) +} + +/// Test that Post-Message can start a chat +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_post_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (_pre_message, post_message, _bob_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&post_message).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) +} + +/// Test that message ordering is still correct after downloading +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_download_later_keeps_message_order() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, post_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Bob sends hi to Alice"); + let hi_msg = tcm.send_recv(bob, alice, "hi").await; + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + + tcm.section("Alice downloads Post-Message"); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); + + Ok(()) +} + +/// Test that ChatlistItemChanged event is emitted when downloading Post-Message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_chatlist_event_on_post_msg_download() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, post_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Alice downloads Post-Message and waits for ChatlistItemChanged event "); + alice.evtracker.clear_events(); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + alice + .evtracker + .get_matching(|e| { + e == &EventType::ChatlistItemChanged { + chat_id: Some(msg.chat_id), + } + }) + .await; + + Ok(()) +} diff --git a/src/tests/pre_messages/sending.rs b/src/tests/pre_messages/sending.rs new file mode 100644 index 0000000000..1d04a4a4d8 --- /dev/null +++ b/src/tests/pre_messages/sending.rs @@ -0,0 +1,337 @@ +//! Tests about sending pre-messages +//! - When to send a pre-message and post-message instead of a normal message +//! - Test that sent pre- and post-message contain the right Headers +//! and that they are send in the correct order (pre-message is sent first.) +use anyhow::Result; +use mailparse::MailHeaderMap; +use tokio::fs; + +use crate::chat::{self, create_group, send_msg}; +use crate::config::Config; +use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; +use crate::headerdef::{HeaderDef, HeaderDefMap}; +use crate::message::{Message, Viewtype}; +use crate::test_utils::{self, TestContext, TestContextManager}; +/// Tests that Pre-Message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` +/// Also test that Pre-Message is sent first, before the Post-Message +/// And that Autocrypt-gossip and selfavatar never go into Post-Messages +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_sending_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // Pre-Message and Post-Message should be present + // and test that correct headers are present on both messages + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?; + let post_message = smtp_rows.get(1).expect("second element exists"); + let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?; + + assert!( + pre_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + post_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_some() + ); + + assert_eq!( + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + Some(format!("<{}>", msg.rfc724_mid)), + "Post-Message should have the rfc message id of the database message" + ); + + assert_ne!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + "message ids of Pre-Message and Post-Message should be different" + ); + + let decrypted_post_message = bob.parse_msg(post_message).await; + assert_eq!(decrypted_post_message.decrypting_failed, false); + assert_eq!( + decrypted_post_message.header_exists(HeaderDef::ChatPostMessageId), + false + ); + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert_eq!( + decrypted_pre_message + .get_header(HeaderDef::ChatPostMessageId) + .map(String::from), + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId) + ); + assert!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::ChatPostMessageId) + .is_none(), + "no Chat-Post-Message-ID header in unprotected headers of Pre-Message" + ); + + Ok(()) +} + +/// Tests that Pre-Message has autocrypt gossip headers and self avatar +/// and Post-Message doesn't have these headers +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + // simulate conditions for sending self avatar + let avatar_src = alice.get_blobdir().join("avatar.png"); + fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?; + alice + .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap())) + .await?; + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let post_message = smtp_rows.get(1).expect("second element exists"); + let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?; + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert!( + decrypted_pre_message + .get_header(HeaderDef::ChatPostMessageId) + .is_some(), + "tested message is not a pre-message, sending order may be broken" + ); + assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0); + assert_ne!(decrypted_pre_message.user_avatar, None); + + let decrypted_post_message = bob.parse_msg(post_message).await; + assert!( + post_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_some(), + "tested message is not a Post-Message, sending order may be broken" + ); + assert_eq!(decrypted_post_message.gossiped_keys.len(), 0); + assert_eq!(decrypted_post_message.user_avatar, None); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_unecrypted_gets_no_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let chat = alice + .create_chat_with_contact("example", "email@example.org") + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1); + let message_bytes = smtp_rows + .first() + .expect("first element exists") + .payload + .as_bytes(); + let message = mailparse::parse_mail(message_bytes)?; + assert!( + message + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none(), + ); + Ok(()) +} + +/// Tests that no pre message is sent for normal message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_not_sending_pre_message_no_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none(), + "no 'Chat-Is-Post-Message'-header should be present" + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + + // test that pre message is not send for large large text + let mut msg = Message::new(Viewtype::Text); + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + msg.set_text(long_text); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + Ok(()) +} + +/// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 100_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is smaller than limit + assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is Post-Message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + + Ok(()) +} + +/// Tests that pre message is not send for large webxdc updates +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_render_webxdc_status_update_object_range() -> Result<()> { + let t = TestContext::new_alice().await; + let chat_id = create_group(&t, "a chat").await?; + + let instance = { + let mut instance = Message::new(Viewtype::File); + instance.set_file_from_bytes( + &t, + "minimal.xdc", + include_bytes!("../../../test-data/webxdc/minimal.xdc"), + None, + )?; + let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?; + assert_eq!(instance.viewtype, Viewtype::Webxdc); + Message::load_from_db(&t, instance_msg_id).await + } + .unwrap(); + + t.pop_sent_msg().await; + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0); + + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}")) + .await?; + t.flush_status_updates().await?; + + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1); + Ok(()) +} diff --git a/src/tests/pre_messages/util.rs b/src/tests/pre_messages/util.rs new file mode 100644 index 0000000000..0d100fed05 --- /dev/null +++ b/src/tests/pre_messages/util.rs @@ -0,0 +1,65 @@ +use anyhow::Result; +use async_zip::tokio::write::ZipFileWriter; +use async_zip::{Compression, ZipEntryBuilder}; +use futures::io::Cursor as FuturesCursor; +use pretty_assertions::assert_eq; +use tokio_util::compat::FuturesAsyncWriteCompatExt; + +use crate::chat::{self, ChatId}; +use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; +use crate::message::{Message, MsgId, Viewtype}; +use crate::test_utils::{SentMessage, TestContext, create_test_image}; + +pub async fn send_large_file_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, + view_type: Viewtype, + content: &[u8], +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let mut msg = Message::new(view_type); + let file_name = if view_type == Viewtype::Webxdc { + "test.xdc" + } else { + "test.bin" + }; + msg.set_file_from_bytes(sender, file_name, content, None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(sender).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(sender, target_chat, &mut msg).await?; + let smtp_rows = sender.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("Pre-Message exists"); + let post_message = smtp_rows.get(1).expect("Post-Message exists"); + Ok((pre_message.to_owned(), post_message.to_owned(), msg_id)) +} + +pub async fn send_large_webxdc_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let futures_cursor = FuturesCursor::new(Vec::new()); + let mut buffer = futures_cursor.compat_write(); + let mut writer = ZipFileWriter::with_tokio(&mut buffer); + writer + .write_entry_whole( + ZipEntryBuilder::new("index.html".into(), Compression::Stored), + &[0u8; 1_000_000], + ) + .await?; + writer.close().await?; + let big_webxdc_app = buffer.into_inner().into_inner(); + send_large_file_message(sender, target_chat, Viewtype::Webxdc, &big_webxdc_app).await +} + +pub async fn send_large_image_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + send_large_file_message(sender, target_chat, Viewtype::Image, &test_img).await +} diff --git a/src/webxdc/webxdc_tests.rs b/src/webxdc/webxdc_tests.rs index 3a03d606f7..792e5d92c3 100644 --- a/src/webxdc/webxdc_tests.rs +++ b/src/webxdc/webxdc_tests.rs @@ -10,9 +10,8 @@ use crate::chat::{ }; use crate::chatlist::Chatlist; use crate::config::Config; -use crate::download::DownloadState; use crate::ephemeral; -use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; +use crate::receive_imf::receive_imf; use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager}; use crate::tools::{self, SystemTime}; use crate::{message, sql}; @@ -329,69 +328,6 @@ async fn test_webxdc_contact_request() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> { - // Alice sends a larger instance and an update - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - let chat = alice.create_chat(&bob).await; - bob.set_config(Config::DownloadLimit, Some("40000")).await?; - let mut alice_instance = create_webxdc_instance( - &alice, - "chess.xdc", - include_bytes!("../../test-data/webxdc/chess.xdc"), - )?; - let sent1 = alice.send_msg(chat.id, &mut alice_instance).await; - let alice_instance = sent1.load_from_db().await; - alice - .send_webxdc_status_update( - alice_instance.id, - r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, - ) - .await?; - alice.flush_status_updates().await?; - let sent2 = alice.pop_sent_msg().await; - - // Bob does not download instance but already receives update - receive_imf_from_inbox( - &bob, - &alice_instance.rfc724_mid, - sent1.payload().as_bytes(), - false, - Some(70790), - ) - .await?; - let bob_instance = bob.get_last_msg().await; - bob_instance.chat_id.accept(&bob).await?; - bob.recv_msg_trash(&sent2).await; - assert_eq!(bob_instance.download_state, DownloadState::Available); - - // Bob downloads instance, updates should be assigned correctly - let received_msg = receive_imf_from_inbox( - &bob, - &alice_instance.rfc724_mid, - sent1.payload().as_bytes(), - false, - None, - ) - .await? - .unwrap(); - assert_eq!(*received_msg.msg_ids.first().unwrap(), bob_instance.id); - let bob_instance = bob.get_last_msg().await; - assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); - assert_eq!(bob_instance.download_state, DownloadState::Done); - assert_eq!( - bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0)) - .await?, - r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# - ); - let info = bob_instance.get_webxdc_info(&bob).await?; - assert_eq!(info.document, "doc"); - assert_eq!(info.summary, "sum"); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_delete_webxdc_instance() -> Result<()> { let t = TestContext::new_alice().await;