GH-48623: [CI][Archery][Dev] Add missing headers to email reports

dev/archery/archery/ci/cli.py

@@ -16,6 +16,8 @@
 # under the License.
 
 import click
+import email.message
+import email.utils
 
 from .core import Workflow
 from ..crossbow.reports import ChatReport, EmailReport, ReportUtils
@@ -105,13 +107,25 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email,
     """
     output = obj['output']
 
-    email_report = EmailReport(
-        report=Workflow(workflow_id, repository,
-                        ignore_job=ignore, gh_token=obj['github_token']),
-        sender_name=sender_name,
-        sender_email=sender_email,
-        recipient_email=recipient_email
+    workflow = Workflow(workflow_id, repository,
+                        ignore_job=ignore, gh_token=obj['github_token'])
+    email_report = EmailReport(report=workflow)
+    message = email.message.EmailMessage()
+    message.set_charset('utf-8')
+    message['Message-Id'] = email.utils.make_msgid()
+    message['Date'] = email.utils.formatdate(workflow.datetime)
+    message['From'] = f'{sender_name} <{sender_email}>'
+    message['To'] = recipient_email
+    n_errors = len(workflow.tasks_by_state['error'])
+    n_failures = len(workflow.tasks_by_state['failure'])
+    n_pendings = len(workflow.tasks_by_state['pending'])
+    subject = (
+        f'[NIGHTLY] Arrow Build Report for Job {workflow.job.branch}: '
+        f'{n_errors + n_failures} failed, '
+        f'{n_pendings} pending'
     )
+    message['Subject'] = subject
+    message.set_content(email_report.render('workflow_report'))
 
     if send:
         ReportUtils.send_email(
@@ -120,7 +134,7 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email,
             smtp_server=smtp_server,
             smtp_port=smtp_port,
             recipient_email=recipient_email,
-            message=email_report.render("workflow_report")
+            message=message
         )
     else:
-        output.write(email_report.render("workflow_report"))
+        output.write(str(message))

dev/archery/archery/crossbow/cli.py

@@ -16,6 +16,8 @@
 # under the License.
 
 from datetime import date
+import email.message
+import email.utils
 from pathlib import Path
 import time
 import sys
@@ -382,12 +384,20 @@ def report(obj, job_name, sender_name, sender_email, recipient_email,
         queue.fetch()
 
     job = queue.get(job_name)
-    email_report = EmailReport(
-        report=Report(job),
-        sender_name=sender_name,
-        sender_email=sender_email,
-        recipient_email=recipient_email
+    report = Report(job)
+    email_report = EmailReport(report=report)
+    message = email.message.EmailMessage()
+    message.set_charset('utf-8')
+    message['Message-Id'] = email.utils.make_msgid()
+    message['Date'] = email.utils.formatdate()
+    message['From'] = f'{sender_name} <{sender_email}>'
+    message['To'] = recipient_email
+    date = report.datetime.strftime('%Y-%m-%d')
+    message['Subject'] = (
+        f'[{date}] Arrow Build Report for {report.name}: '
+        f'{len(report.failed_jobs())} failed'
     )
+    message.set_content(email_report.render('nightly_report'))
 
     if poll:
         job.wait_until_finished(
@@ -402,10 +412,10 @@ def report(obj, job_name, sender_name, sender_email, recipient_email,
             smtp_server=smtp_server,
             smtp_port=smtp_port,
             recipient_email=recipient_email,
-            message=email_report.render("nightly_report")
+            message=message
         )
     else:
-        output.write(email_report.render("nightly_report"))
+        output.write(str(message))
 
 
 @crossbow.command()
@@ -641,19 +651,25 @@ def notify_token_expiration(obj, days, sender_name, sender_email,
             return
 
     class TokenExpirationReport:
-        def __init__(self, token_expiration_date, days_left):
-            self.token_expiration_date = token_expiration_date
+        def __init__(self, days_left):
             self.days_left = days_left
 
-    email_report = EmailReport(
-        report=TokenExpirationReport(
-            token_expiration_date or "ALREADY_EXPIRED", days_left),
-        sender_name=sender_name,
-        sender_email=sender_email,
-        recipient_email=recipient_email
+    if not token_expiration_date:
+        token_expiration_date = 'ALREADY_EXPIRED'
+    report = TokenExpirationReport(days_left)
+    email_report = EmailReport(report)
+
+    message = email.message.EmailMessage()
+    message.set_charset('utf-8')
+    message['Message-Id'] = email.utils.make_msgid()
+    message['Date'] = email.utils.formatdate()
+    message['From'] = f'{sender_name} <{sender_email}>'
+    message['To'] = recipient_email
+    message['Subject'] = (
+        f'[CI] Arrow Crossbow Token Expiration in {token_expiration_date}'
     )
+    message.set_content(email_report.render('token_expiration'))
 
-    message = email_report.render("token_expiration").strip()
     if send:
         ReportUtils.send_email(
             smtp_user=smtp_user,
@@ -664,4 +680,4 @@ def __init__(self, token_expiration_date, days_left):
             message=message
         )
     else:
-        output.write(message)
+        output.write(str(message))

dev/archery/archery/crossbow/reports.py

@@ -259,7 +259,7 @@ def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port,
             else:
                 smtp.starttls()
             smtp.login(smtp_user, smtp_password)
-            smtp.sendmail(smtp_user, recipient_email, message)
+            smtp.send_message(smtp_user, recipient_email, message)
 
     @classmethod
     def write_csv(cls, report, add_headers=True):
@@ -278,9 +278,6 @@ class EmailReport(JinjaReport):
     }
     fields = [
         'report',
-        'sender_name',
-        'sender_email',
-        'recipient_email',
     ]
 
 

dev/archery/archery/crossbow/tests/fixtures/email-report.txt

@@ -1,7 +1,3 @@
-From: Sender Reporter <sender@arrow.com>
-To: recipient@arrow.com
-Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending
-
 Arrow Build Report for Job ursabot-1
 
 See https://s3.amazonaws.com/arrow-data/index.html for more information.

dev/archery/archery/crossbow/tests/test_reports.py

@@ -76,9 +76,7 @@ def test_crossbow_email_report(load_fixture):
     job = load_fixture('crossbow-job.yaml', decoder=yaml.load)
     report = Report(job)
     assert report.tasks_by_state is not None
-    email_report = EmailReport(report=report, sender_name="Sender Reporter",
-                               sender_email="sender@arrow.com",
-                               recipient_email="recipient@arrow.com")
+    email_report = EmailReport(report=report)
 
     assert (
         email_report.render("nightly_report") == textwrap.dedent(expected_msg)

dev/archery/archery/templates/email_nightly_report.txt.j2

@@ -15,13 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#}
-{%- if True -%}
-{%- endif -%}
-From: {{ sender_name }} <{{ sender_email }}>
-To: {{ recipient_email }}
-Subject: [NIGHTLY] Arrow Build Report for Job {{report.job.branch}}: {{ (report.tasks_by_state["error"] | length) +  (report.tasks_by_state["failure"] | length) }} failed, {{ report.tasks_by_state["pending"] | length }} pending
-
+-#}
 Arrow Build Report for Job {{ report.job.branch }}
 
 See https://s3.amazonaws.com/arrow-data/index.html for more information.
@@ -58,4 +52,4 @@ Succeeded Tasks:
 - {{ task_name }}
   {{ report.task_url(task) }}
 {% endfor %}
-{%- endif -%}
+{%- endif -%}

dev/archery/archery/templates/email_token_expiration.txt.j2

@@ -15,11 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#}
-From: {{ sender_name }} <{{ sender_email }}>
-To: {{ recipient_email }}
-Subject: [CI] Arrow Crossbow Token Expiration in {{ report.token_expiration_date }}
-
+-#}
 The Arrow Crossbow Token will expire in {{ report.days_left }} days.
 
 Please generate a new Token. Send it to Apache INFRA to update the CROSSBOW_GITHUB_TOKEN.

dev/archery/archery/templates/email_workflow_report.txt.j2

@@ -15,13 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#}
-{%- if True -%}
-{%- endif -%}
-From: {{ sender_name }} <{{ sender_email }}>
-To: {{ recipient_email }}
-Subject: [{{ report.datetime.strftime('%Y-%m-%d') }}] Arrow Build Report for {{ report.name }}: {{ report.failed_jobs() | length }} failed
-
+-#}
 Arrow Build Report for {{ report.name }}
 
 Workflow URL: {{ report.url }}
@@ -42,4 +36,4 @@ Succeeded Jobs:
 - {{ job.name }}
   {{ job.url }}
 {% endfor %}
-{%- endif -%}
+{%- endif -%}