From 7bc5723657574f906a6928a74723178427a744c6 Mon Sep 17 00:00:00 2001 From: Griffin Date: Wed, 28 Oct 2026 18:51:49 -0700 Subject: [PATCH 1/5] gh-158780: Preserve defects in unstructured email headers --- Lib/email/headerregistry.py | 1 + Lib/test/test_email/test_headerregistry.py | 39 +++++++++++++++++++ ...-10-28-18-44-59.gh-issue-158780.J9qR4s.rst | 2 + 3 files changed, 42 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-10-28-18-44-59.gh-issue-158780.J9qR4s.rst diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 48cd85a65ba9f61..329a0cb3c45f83b 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -264,6 +264,7 @@ class UnstructuredHeader: def parse(cls, value, kwds): kwds['parse_tree'] = cls.value_parser(value) kwds['decoded'] = str(kwds['parse_tree']) + kwds['defects'].extend(kwds['parse_tree'].all_defects) class UniqueUnstructuredHeader(UnstructuredHeader): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index c118fd5e20d048f..ee2d1b181817468 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -129,6 +129,45 @@ def test_defects_is_tuple(self): @parameterize class TestUnstructuredHeader(TestHeaderBase): + def defects_as_value(self, source, decoded, defects): + for name in ('Subject', 'X-Test'): + with self.subTest(name=name): + h = self.make_header(name, source) + self.assertEqual(h, decoded) + self.assertDefectsEqual(h.defects, defects) + + defects_params = { + 'valid_encoded_word': ( + '=?utf-8?q?caf=C3=A9?=', + 'café', + []), + + 'invalid_base64_padding': ( + '=?utf-8?b?YQ?=', + 'a', + [errors.InvalidBase64PaddingDefect]), + + 'invalid_base64_characters': ( + '=?utf-8?b?Y!Q==?=', + 'a', + [errors.InvalidBase64CharactersDefect]), + + 'undecodable_bytes': ( + '=?utf-8?q?=FF?=', + '\ufffd', + [errors.UndecodableBytesDefect] * 2), + + 'unknown_charset': ( + '=?invalid-charset?q?word?=', + 'word', + [errors.CharsetError]), + + 'missing_whitespace': ( + 'prefix=?utf-8?q?word?=', + 'prefixword', + [errors.InvalidHeaderDefect]), + } + def string_as_value(self, source, decoded, diff --git a/Misc/NEWS.d/next/Library/2026-10-28-18-44-59.gh-issue-158780.J9qR4s.rst b/Misc/NEWS.d/next/Library/2026-10-28-18-44-59.gh-issue-158780.J9qR4s.rst new file mode 100644 index 000000000000000..db8fed83a3a3d16 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-28-18-44-59.gh-issue-158780.J9qR4s.rst @@ -0,0 +1,2 @@ +Fix :class:`email.headerregistry.UnstructuredHeader` to include detected +encoded-word and text defects in its public ``defects`` attribute. From fa54a1acd760a6c659b7b1d6295dca1af519d9fa Mon Sep 17 00:00:00 2001 From: Griffin Date: Wed, 28 Oct 2026 19:39:09 -0700 Subject: [PATCH 2/5] gh-158780: Cover text and combined header defects --- Lib/test/test_email/test_headerregistry.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index ee2d1b181817468..69a9b7ba0c91ec7 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -162,6 +162,16 @@ def defects_as_value(self, source, decoded, defects): 'word', [errors.CharsetError]), + 'non_printable_text': ( + 'text\x00value', + 'text\x00value', + [errors.NonPrintableDefect]), + + 'combined_defects': ( + 'prefix=?utf-8?b?YQ?=', + 'prefixa', + [errors.InvalidHeaderDefect, errors.InvalidBase64PaddingDefect]), + 'missing_whitespace': ( 'prefix=?utf-8?q?word?=', 'prefixword', From 69efd73a1dd4c0112e7882dde1b8d4a00e190d43 Mon Sep 17 00:00:00 2001 From: Griffin Date: Wed, 28 Oct 2026 19:44:48 -0700 Subject: [PATCH 3/5] gh-158780: Reuse unstructured header test cases --- Lib/test/test_email/test_headerregistry.py | 43 ++++++++++------------ 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 69a9b7ba0c91ec7..f18bed1007b7f5b 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -129,14 +129,19 @@ def test_defects_is_tuple(self): @parameterize class TestUnstructuredHeader(TestHeaderBase): - def defects_as_value(self, source, decoded, defects): + def string_as_value(self, source, decoded, defects=(), folded=None): + folded = source if folded is None else folded for name in ('Subject', 'X-Test'): with self.subTest(name=name): h = self.make_header(name, source) self.assertEqual(h, decoded) self.assertDefectsEqual(h.defects, defects) + prefix = name + ':' + (' ' if source else '') + self.assertEqual(h.fold(policy=policy.default), prefix + folded + '\n') + + string_params = { + 'empty': ('', ''), - defects_params = { 'valid_encoded_word': ( '=?utf-8?q?caf=C3=A9?=', 'café', @@ -145,22 +150,26 @@ def defects_as_value(self, source, decoded, defects): 'invalid_base64_padding': ( '=?utf-8?b?YQ?=', 'a', - [errors.InvalidBase64PaddingDefect]), + [errors.InvalidBase64PaddingDefect], + 'a'), 'invalid_base64_characters': ( '=?utf-8?b?Y!Q==?=', 'a', - [errors.InvalidBase64CharactersDefect]), + [errors.InvalidBase64CharactersDefect], + 'a'), 'undecodable_bytes': ( '=?utf-8?q?=FF?=', '\ufffd', - [errors.UndecodableBytesDefect] * 2), + [errors.UndecodableBytesDefect] * 2, + '=?unknown-8bit?q?=FF?='), 'unknown_charset': ( '=?invalid-charset?q?word?=', 'word', - [errors.CharsetError]), + [errors.CharsetError], + 'word'), 'non_printable_text': ( 'text\x00value', @@ -170,28 +179,14 @@ def defects_as_value(self, source, decoded, defects): 'combined_defects': ( 'prefix=?utf-8?b?YQ?=', 'prefixa', - [errors.InvalidHeaderDefect, errors.InvalidBase64PaddingDefect]), + [errors.InvalidHeaderDefect, errors.InvalidBase64PaddingDefect], + 'prefixa'), 'missing_whitespace': ( 'prefix=?utf-8?q?word?=', 'prefixword', - [errors.InvalidHeaderDefect]), - } - - def string_as_value(self, - source, - decoded, - *args): - l = len(args) - defects = args[0] if l>0 else [] - header = 'Subject:' + (' ' if source else '') - folded = header + (args[1] if l>1 else source) + '\n' - h = self.make_header('Subject', source) - self.assertEqual(h, decoded) - self.assertDefectsEqual(h.defects, defects) - self.assertEqual(h.fold(policy=policy.default), folded) - - string_params = { + [errors.InvalidHeaderDefect], + 'prefixword'), 'rfc2047_simple_quopri': ( '=?utf-8?q?this_is_a_test?=', From 8d10a015e302fe5d953a59a2d581bf49bfe3aa7e Mon Sep 17 00:00:00 2001 From: Griffin Date: Wed, 28 Oct 2026 19:51:42 -0700 Subject: [PATCH 4/5] gh-158780: Collect header defects without repeated copying --- Lib/email/_header_value_parser.py | 5 ++++- .../test_email/test__header_value_parser.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index f396f4f9b32a9a8..f6ff3264c2cb0aa 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -152,7 +152,10 @@ def value(self): @property def all_defects(self): - return sum((x.all_defects for x in self), self.defects) + if not self: + return self.defects + return self.defects + [ + defect for token in self for defect in token.all_defects] def startswith_fws(self): return self[0].startswith_fws() diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 26e9249ba9e948b..0ee1408e91a6579 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -7,6 +7,23 @@ class TestTokens(TestEmailBase): + # TokenList + + def test_all_defects_empty(self): + token = parser.TokenList() + self.assertIs(token.all_defects, token.defects) + token.defects.append(errors.InvalidHeaderDefect()) + self.assertIs(token.all_defects, token.defects) + + def test_all_defects(self): + leaf = parser.ValueTerminal('text\x00', 'utext') + leaf.defects.append(errors.NonPrintableDefect('\x00')) + token = parser.TokenList([parser.TokenList([leaf]), leaf]) + token.defects.append(errors.InvalidHeaderDefect()) + self.assertEqual(token.all_defects, token.defects + leaf.defects * 2) + self.assertEqual(len(token.defects), 1) + self.assertEqual(len(leaf.defects), 1) + # EWWhiteSpaceTerminal def test_EWWhiteSpaceTerminal(self): From 712db7d7cc8820acdbf2ead9c44cbbc2f9ae9beb Mon Sep 17 00:00:00 2001 From: Griffin Date: Wed, 28 Oct 2026 19:55:01 -0700 Subject: [PATCH 5/5] gh-158780: Match surrounding header test layout --- Lib/test/test_email/test_headerregistry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index f18bed1007b7f5b..b0808b576535eeb 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -137,9 +137,11 @@ def string_as_value(self, source, decoded, defects=(), folded=None): self.assertEqual(h, decoded) self.assertDefectsEqual(h.defects, defects) prefix = name + ':' + (' ' if source else '') - self.assertEqual(h.fold(policy=policy.default), prefix + folded + '\n') + self.assertEqual(h.fold(policy=policy.default), + prefix + folded + '\n') string_params = { + 'empty': ('', ''), 'valid_encoded_word': (