From 8cedff501ae918a76c833e94c334ff8a4b2b5d47 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 23 Dec 2025 06:46:03 +0900 Subject: [PATCH 1/2] GH-48623: [CI][Archery][Dev] Add missing headers to email reports --- dev/archery/archery/ci/cli.py | 12 ++++++---- dev/archery/archery/crossbow/cli.py | 23 ++++++++++++------- dev/archery/archery/crossbow/reports.py | 6 +++-- .../crossbow/tests/fixtures/email-report.txt | 4 ++++ .../archery/crossbow/tests/test_reports.py | 7 ++++-- .../templates/email_nightly_report.txt.j2 | 10 ++++---- .../templates/email_token_expiration.txt.j2 | 6 ++++- .../templates/email_workflow_report.txt.j2 | 10 ++++---- 8 files changed, 53 insertions(+), 25 deletions(-) diff --git a/dev/archery/archery/ci/cli.py b/dev/archery/archery/ci/cli.py index bf7b68d5327..5d34fd582ca 100644 --- a/dev/archery/archery/ci/cli.py +++ b/dev/archery/archery/ci/cli.py @@ -16,6 +16,7 @@ # under the License. import click +import email.utils from .core import Workflow from ..crossbow.reports import ChatReport, EmailReport, ReportUtils @@ -105,12 +106,15 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email, """ output = obj['output'] + workflow = Workflow(workflow_id, repository, + ignore_job=ignore, gh_token=obj['github_token']) email_report = EmailReport( - report=Workflow(workflow_id, repository, - ignore_job=ignore, gh_token=obj['github_token']), - sender_name=sender_name, + date=email.utils.formatdate(workflow.datetime), + message_id=email.utils.make_msgid(), + recipient_email=recipient_email, + report=workflow, sender_email=sender_email, - recipient_email=recipient_email + sender_name=sender_name, ) if send: diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index c73c4d1ff7e..48afbee0988 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -16,6 +16,7 @@ # under the License. from datetime import date +import email.utils from pathlib import Path import time import sys @@ -382,11 +383,14 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, queue.fetch() job = queue.get(job_name) + report = Report(job) email_report = EmailReport( - report=Report(job), - sender_name=sender_name, + date=email.utils.formatdate(), + message_id=email.utils.make_msgid(), + recipient_email=recipient_email, + report=report, sender_email=sender_email, - recipient_email=recipient_email + sender_name=sender_name, ) if poll: @@ -645,15 +649,18 @@ def __init__(self, token_expiration_date, days_left): self.token_expiration_date = token_expiration_date self.days_left = days_left + report = TokenExpirationReport( + token_expiration_date or "ALREADY_EXPIRED", days_left) email_report = EmailReport( - report=TokenExpirationReport( - token_expiration_date or "ALREADY_EXPIRED", days_left), - sender_name=sender_name, + date=email.utils.formatdate(), + message_id=email.utils.make_msgid(), + recipient_email=recipient_email, + report=report, sender_email=sender_email, - recipient_email=recipient_email + sender_name=sender_name, ) - message = email_report.render("token_expiration").strip() + message = email_report.render("token_expiration") if send: ReportUtils.send_email( smtp_user=smtp_user, diff --git a/dev/archery/archery/crossbow/reports.py b/dev/archery/archery/crossbow/reports.py index 32962410d6e..38d0a66aae6 100644 --- a/dev/archery/archery/crossbow/reports.py +++ b/dev/archery/archery/crossbow/reports.py @@ -277,10 +277,12 @@ class EmailReport(JinjaReport): 'workflow_report': 'email_workflow_report.txt.j2', } fields = [ + 'date', + 'message_id', + 'recipient_email', 'report', - 'sender_name', 'sender_email', - 'recipient_email', + 'sender_name', ] diff --git a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt index c29cafd3938..bab93e6f89d 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt +++ b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt @@ -1,3 +1,7 @@ +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Message-Id: +Date: Mon, 22 Dec 2025 22:12:00 -0000 From: Sender Reporter To: recipient@arrow.com Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending diff --git a/dev/archery/archery/crossbow/tests/test_reports.py b/dev/archery/archery/crossbow/tests/test_reports.py index 620b4c78bbc..829c1e833e6 100644 --- a/dev/archery/archery/crossbow/tests/test_reports.py +++ b/dev/archery/archery/crossbow/tests/test_reports.py @@ -76,9 +76,12 @@ def test_crossbow_email_report(load_fixture): job = load_fixture('crossbow-job.yaml', decoder=yaml.load) report = Report(job) assert report.tasks_by_state is not None - email_report = EmailReport(report=report, sender_name="Sender Reporter", + email_report = EmailReport(date="Mon, 22 Dec 2025 22:12:00 -0000", + message_id="", + recipient_email="recipient@arrow.com", + report=report, sender_email="sender@arrow.com", - recipient_email="recipient@arrow.com") + sender_name="Sender Reporter") assert ( email_report.render("nightly_report") == textwrap.dedent(expected_msg) diff --git a/dev/archery/archery/templates/email_nightly_report.txt.j2 b/dev/archery/archery/templates/email_nightly_report.txt.j2 index bc040734b03..959738467f9 100644 --- a/dev/archery/archery/templates/email_nightly_report.txt.j2 +++ b/dev/archery/archery/templates/email_nightly_report.txt.j2 @@ -15,9 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -{%- if True -%} -{%- endif -%} +-#} +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Message-Id: {{ message_id }} +Date: {{ date }} From: {{ sender_name }} <{{ sender_email }}> To: {{ recipient_email }} Subject: [NIGHTLY] Arrow Build Report for Job {{report.job.branch}}: {{ (report.tasks_by_state["error"] | length) + (report.tasks_by_state["failure"] | length) }} failed, {{ report.tasks_by_state["pending"] | length }} pending @@ -58,4 +60,4 @@ Succeeded Tasks: - {{ task_name }} {{ report.task_url(task) }} {% endfor %} -{%- endif -%} \ No newline at end of file +{%- endif -%} diff --git a/dev/archery/archery/templates/email_token_expiration.txt.j2 b/dev/archery/archery/templates/email_token_expiration.txt.j2 index 54c2005e57e..2af8aaaea85 100644 --- a/dev/archery/archery/templates/email_token_expiration.txt.j2 +++ b/dev/archery/archery/templates/email_token_expiration.txt.j2 @@ -15,7 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} +-#} +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Message-Id: {{ message_id }} +Date: {{ date }} From: {{ sender_name }} <{{ sender_email }}> To: {{ recipient_email }} Subject: [CI] Arrow Crossbow Token Expiration in {{ report.token_expiration_date }} diff --git a/dev/archery/archery/templates/email_workflow_report.txt.j2 b/dev/archery/archery/templates/email_workflow_report.txt.j2 index 193856c1806..370eb557ca3 100644 --- a/dev/archery/archery/templates/email_workflow_report.txt.j2 +++ b/dev/archery/archery/templates/email_workflow_report.txt.j2 @@ -15,9 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -{%- if True -%} -{%- endif -%} +-#} +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Message-Id: {{ message_id }} +Date: {{ date }} From: {{ sender_name }} <{{ sender_email }}> To: {{ recipient_email }} Subject: [{{ report.datetime.strftime('%Y-%m-%d') }}] Arrow Build Report for {{ report.name }}: {{ report.failed_jobs() | length }} failed @@ -42,4 +44,4 @@ Succeeded Jobs: - {{ job.name }} {{ job.url }} {% endfor %} -{%- endif -%} \ No newline at end of file +{%- endif -%} From 7796e994f304dd9aa39847b4667711d096920632 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 24 Dec 2025 17:08:27 +0900 Subject: [PATCH 2/2] Use email.message.EmailMessage --- dev/archery/archery/ci/cli.py | 28 ++++++---- dev/archery/archery/crossbow/cli.py | 53 +++++++++++-------- dev/archery/archery/crossbow/reports.py | 7 +-- .../crossbow/tests/fixtures/email-report.txt | 8 --- .../archery/crossbow/tests/test_reports.py | 7 +-- .../templates/email_nightly_report.txt.j2 | 8 --- .../templates/email_token_expiration.txt.j2 | 8 --- .../templates/email_workflow_report.txt.j2 | 8 --- 8 files changed, 52 insertions(+), 75 deletions(-) diff --git a/dev/archery/archery/ci/cli.py b/dev/archery/archery/ci/cli.py index 5d34fd582ca..d5c0a2c1f83 100644 --- a/dev/archery/archery/ci/cli.py +++ b/dev/archery/archery/ci/cli.py @@ -16,6 +16,7 @@ # under the License. import click +import email.message import email.utils from .core import Workflow @@ -108,14 +109,23 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email, workflow = Workflow(workflow_id, repository, ignore_job=ignore, gh_token=obj['github_token']) - email_report = EmailReport( - date=email.utils.formatdate(workflow.datetime), - message_id=email.utils.make_msgid(), - recipient_email=recipient_email, - report=workflow, - sender_email=sender_email, - sender_name=sender_name, + email_report = EmailReport(report=workflow) + message = email.message.EmailMessage() + message.set_charset('utf-8') + message['Message-Id'] = email.utils.make_msgid() + message['Date'] = email.utils.formatdate(workflow.datetime) + message['From'] = f'{sender_name} <{sender_email}>' + message['To'] = recipient_email + n_errors = len(workflow.tasks_by_state['error']) + n_failures = len(workflow.tasks_by_state['failure']) + n_pendings = len(workflow.tasks_by_state['pending']) + subject = ( + f'[NIGHTLY] Arrow Build Report for Job {workflow.job.branch}: ' + f'{n_errors + n_failures} failed, ' + f'{n_pendings} pending' ) + message['Subject'] = subject + message.set_content(email_report.render('workflow_report')) if send: ReportUtils.send_email( @@ -124,7 +134,7 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email, smtp_server=smtp_server, smtp_port=smtp_port, recipient_email=recipient_email, - message=email_report.render("workflow_report") + message=message ) else: - output.write(email_report.render("workflow_report")) + output.write(str(message)) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index 48afbee0988..62f39ad723f 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -16,6 +16,7 @@ # under the License. from datetime import date +import email.message import email.utils from pathlib import Path import time @@ -384,14 +385,19 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, job = queue.get(job_name) report = Report(job) - email_report = EmailReport( - date=email.utils.formatdate(), - message_id=email.utils.make_msgid(), - recipient_email=recipient_email, - report=report, - sender_email=sender_email, - sender_name=sender_name, + email_report = EmailReport(report=report) + message = email.message.EmailMessage() + message.set_charset('utf-8') + message['Message-Id'] = email.utils.make_msgid() + message['Date'] = email.utils.formatdate() + message['From'] = f'{sender_name} <{sender_email}>' + message['To'] = recipient_email + date = report.datetime.strftime('%Y-%m-%d') + message['Subject'] = ( + f'[{date}] Arrow Build Report for {report.name}: ' + f'{len(report.failed_jobs())} failed' ) + message.set_content(email_report.render('nightly_report')) if poll: job.wait_until_finished( @@ -406,10 +412,10 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, smtp_server=smtp_server, smtp_port=smtp_port, recipient_email=recipient_email, - message=email_report.render("nightly_report") + message=message ) else: - output.write(email_report.render("nightly_report")) + output.write(str(message)) @crossbow.command() @@ -645,22 +651,25 @@ def notify_token_expiration(obj, days, sender_name, sender_email, return class TokenExpirationReport: - def __init__(self, token_expiration_date, days_left): - self.token_expiration_date = token_expiration_date + def __init__(self, days_left): self.days_left = days_left - report = TokenExpirationReport( - token_expiration_date or "ALREADY_EXPIRED", days_left) - email_report = EmailReport( - date=email.utils.formatdate(), - message_id=email.utils.make_msgid(), - recipient_email=recipient_email, - report=report, - sender_email=sender_email, - sender_name=sender_name, + if not token_expiration_date: + token_expiration_date = 'ALREADY_EXPIRED' + report = TokenExpirationReport(days_left) + email_report = EmailReport(report) + + message = email.message.EmailMessage() + message.set_charset('utf-8') + message['Message-Id'] = email.utils.make_msgid() + message['Date'] = email.utils.formatdate() + message['From'] = f'{sender_name} <{sender_email}>' + message['To'] = recipient_email + message['Subject'] = ( + f'[CI] Arrow Crossbow Token Expiration in {token_expiration_date}' ) + message.set_content(email_report.render('token_expiration')) - message = email_report.render("token_expiration") if send: ReportUtils.send_email( smtp_user=smtp_user, @@ -671,4 +680,4 @@ def __init__(self, token_expiration_date, days_left): message=message ) else: - output.write(message) + output.write(str(message)) diff --git a/dev/archery/archery/crossbow/reports.py b/dev/archery/archery/crossbow/reports.py index 38d0a66aae6..f2efd8623f8 100644 --- a/dev/archery/archery/crossbow/reports.py +++ b/dev/archery/archery/crossbow/reports.py @@ -259,7 +259,7 @@ def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port, else: smtp.starttls() smtp.login(smtp_user, smtp_password) - smtp.sendmail(smtp_user, recipient_email, message) + smtp.send_message(smtp_user, recipient_email, message) @classmethod def write_csv(cls, report, add_headers=True): @@ -277,12 +277,7 @@ class EmailReport(JinjaReport): 'workflow_report': 'email_workflow_report.txt.j2', } fields = [ - 'date', - 'message_id', - 'recipient_email', 'report', - 'sender_email', - 'sender_name', ] diff --git a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt index bab93e6f89d..16409dc38ef 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt +++ b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt @@ -1,11 +1,3 @@ -MIME-Version: 1.0 -Content-Type: text/plain; charset="utf-8" -Message-Id: -Date: Mon, 22 Dec 2025 22:12:00 -0000 -From: Sender Reporter -To: recipient@arrow.com -Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending - Arrow Build Report for Job ursabot-1 See https://s3.amazonaws.com/arrow-data/index.html for more information. diff --git a/dev/archery/archery/crossbow/tests/test_reports.py b/dev/archery/archery/crossbow/tests/test_reports.py index 829c1e833e6..4ed40a7793c 100644 --- a/dev/archery/archery/crossbow/tests/test_reports.py +++ b/dev/archery/archery/crossbow/tests/test_reports.py @@ -76,12 +76,7 @@ def test_crossbow_email_report(load_fixture): job = load_fixture('crossbow-job.yaml', decoder=yaml.load) report = Report(job) assert report.tasks_by_state is not None - email_report = EmailReport(date="Mon, 22 Dec 2025 22:12:00 -0000", - message_id="", - recipient_email="recipient@arrow.com", - report=report, - sender_email="sender@arrow.com", - sender_name="Sender Reporter") + email_report = EmailReport(report=report) assert ( email_report.render("nightly_report") == textwrap.dedent(expected_msg) diff --git a/dev/archery/archery/templates/email_nightly_report.txt.j2 b/dev/archery/archery/templates/email_nightly_report.txt.j2 index 959738467f9..7b43d7c867e 100644 --- a/dev/archery/archery/templates/email_nightly_report.txt.j2 +++ b/dev/archery/archery/templates/email_nightly_report.txt.j2 @@ -16,14 +16,6 @@ # specific language governing permissions and limitations # under the License. -#} -MIME-Version: 1.0 -Content-Type: text/plain; charset="utf-8" -Message-Id: {{ message_id }} -Date: {{ date }} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [NIGHTLY] Arrow Build Report for Job {{report.job.branch}}: {{ (report.tasks_by_state["error"] | length) + (report.tasks_by_state["failure"] | length) }} failed, {{ report.tasks_by_state["pending"] | length }} pending - Arrow Build Report for Job {{ report.job.branch }} See https://s3.amazonaws.com/arrow-data/index.html for more information. diff --git a/dev/archery/archery/templates/email_token_expiration.txt.j2 b/dev/archery/archery/templates/email_token_expiration.txt.j2 index 2af8aaaea85..096026fa3a2 100644 --- a/dev/archery/archery/templates/email_token_expiration.txt.j2 +++ b/dev/archery/archery/templates/email_token_expiration.txt.j2 @@ -16,14 +16,6 @@ # specific language governing permissions and limitations # under the License. -#} -MIME-Version: 1.0 -Content-Type: text/plain; charset="utf-8" -Message-Id: {{ message_id }} -Date: {{ date }} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [CI] Arrow Crossbow Token Expiration in {{ report.token_expiration_date }} - The Arrow Crossbow Token will expire in {{ report.days_left }} days. Please generate a new Token. Send it to Apache INFRA to update the CROSSBOW_GITHUB_TOKEN. diff --git a/dev/archery/archery/templates/email_workflow_report.txt.j2 b/dev/archery/archery/templates/email_workflow_report.txt.j2 index 370eb557ca3..6668d6c67ee 100644 --- a/dev/archery/archery/templates/email_workflow_report.txt.j2 +++ b/dev/archery/archery/templates/email_workflow_report.txt.j2 @@ -16,14 +16,6 @@ # specific language governing permissions and limitations # under the License. -#} -MIME-Version: 1.0 -Content-Type: text/plain; charset="utf-8" -Message-Id: {{ message_id }} -Date: {{ date }} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [{{ report.datetime.strftime('%Y-%m-%d') }}] Arrow Build Report for {{ report.name }}: {{ report.failed_jobs() | length }} failed - Arrow Build Report for {{ report.name }} Workflow URL: {{ report.url }}