email date format flexibility (#4072)

we are seeing some .eml files come through the VLM partitioner. Which
then downgrades to hi-res i believe.

For some reason they have a date format that is not standard email
format. But it is still legitimate.

This uses a more robust date package to parse the date. This package is
already installed.

---------

Co-authored-by: ryannikolaidis <[email protected]>
Co-authored-by: potter-potter <[email protected]>

Author: potter-potter

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.18.13
+
+### Enhancements
+
+### Features
+
+### Fixes
+
+- **Parse a wider variety of date formats in email headers** The `partition_email` function is now more robust to non-standard date formats, including ISO-8601 dates with "Z" suffixes. This prevents `ValueError` exceptions when partitioning emails with these date formats. 
+
 ## 0.18.12
 
 ### Enhancements

example-docs/eml/test-invalid-date.eml

@@ -0,0 +1,6 @@
+Date: INVALID-DATE-FORMAT
+From: [email protected]
+To: [email protected]
+Subject: Test invalid date format
+
+This is a test-email with an invalid date format.

example-docs/eml/test-iso-8601-date.eml

@@ -0,0 +1,6 @@
+Date: 2025-07-29T12:42:06.000Z
+From: [email protected]
+To: [email protected]
+Subject: Test a Z-suffix date
+
+This is a test-email.

example-docs/eml/test-rfc2822-date.eml

@@ -0,0 +1,6 @@
+Date: Tue, 29 Jul 2025 12:42:06 +0000
+From: [email protected]
+To: [email protected]
+Subject: Test a standard RFC-2822 date
+
+This is a test-email.

scripts/docker-smoke-test.sh

@@ -3,6 +3,7 @@
 # Start the containerized repository and run ingest tests
 
 # shellcheck disable=SC2317  # Shellcheck complains that trap functions are unreachable...
+# shellcheck disable=SC2329  # Functions are invoked indirectly
 
 set -eux -o pipefail
 

test_unstructured/partition/test_email.py

@@ -521,10 +521,29 @@ def it_uses_the_metadata_last_modified_arg_value_when_one_was_provided(self):
         ctx = EmailPartitioningContext(metadata_last_modified=metadata_last_modified)
         assert ctx.metadata_last_modified == metadata_last_modified
 
-    def and_it_uses_the_msg_Date_header_date_when_metadata_last_modified_was_not_provided(self):
+    def and_it_uses_the_msg_Date_header_date_when_metadata_last_modified_was_not_provided(
+        self,
+    ):
         ctx = EmailPartitioningContext(example_doc_path("eml/simple-rfc-822.eml"))
         assert ctx.metadata_last_modified == "2024-10-01T17:34:56+00:00"
 
+    @pytest.mark.parametrize(
+        ("date_format", "expected_date"),
+        [
+            ("test-iso-8601-date.eml", "2025-07-29T12:42:06+00:00"),
+            ("test-rfc2822-date.eml", "2025-07-29T12:42:06+00:00"),
+        ],
+    )
+    def and_it_correctly_parses_various_date_formats_like_the_ones_that_occur_in_the_wild(
+        self, date_format: str, expected_date: str
+    ):
+        ctx = EmailPartitioningContext(example_doc_path(f"eml/{date_format}"))
+        assert ctx.metadata_last_modified == expected_date
+
+    def and_it_returns_none_when_date_header_is_invalid(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/test-invalid-date.eml"))
+        assert ctx._sent_date is None
+
     def and_it_falls_back_to_filesystem_last_modified_when_no_Date_header_is_present(
         self, get_last_modified_date_: Mock
     ):

test_unstructured_ingest/src/azure.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# shellcheck disable=SC2329  # Functions are invoked indirectly
+
 set -e
 
 SRC_PATH=$(dirname "$(realpath "$0")")

test_unstructured_ingest/src/google-drive.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# shellcheck disable=SC2329  # Functions are invoked indirectly
+
 set -e
 
 SRC_PATH=$(dirname "$(realpath "$0")")

test_unstructured_ingest/src/kafka-local.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# shellcheck disable=SC2329  # Functions are invoked indirectly
+
 set -e
 
 SRC_PATH=$(dirname "$(realpath "$0")")

test_unstructured_ingest/src/local-single-file-basic-chunking.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# shellcheck disable=SC2329  # Functions are invoked indirectly
+
 set -e
 
 SRC_PATH=$(dirname "$(realpath "$0")")