diff --git a/dev/archery/archery/ci/cli.py b/dev/archery/archery/ci/cli.py index bf7b68d5327..d5c0a2c1f83 100644 --- a/dev/archery/archery/ci/cli.py +++ b/dev/archery/archery/ci/cli.py @@ -16,6 +16,8 @@ # under the License. import click +import email.message +import email.utils from .core import Workflow from ..crossbow.reports import ChatReport, EmailReport, ReportUtils @@ -105,13 +107,25 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email, """ output = obj['output'] - email_report = EmailReport( - report=Workflow(workflow_id, repository, - ignore_job=ignore, gh_token=obj['github_token']), - sender_name=sender_name, - sender_email=sender_email, - recipient_email=recipient_email + workflow = Workflow(workflow_id, repository, + ignore_job=ignore, gh_token=obj['github_token']) + email_report = EmailReport(report=workflow) + message = email.message.EmailMessage() + message.set_charset('utf-8') + message['Message-Id'] = email.utils.make_msgid() + message['Date'] = email.utils.formatdate(workflow.datetime) + message['From'] = f'{sender_name} <{sender_email}>' + message['To'] = recipient_email + n_errors = len(workflow.tasks_by_state['error']) + n_failures = len(workflow.tasks_by_state['failure']) + n_pendings = len(workflow.tasks_by_state['pending']) + subject = ( + f'[NIGHTLY] Arrow Build Report for Job {workflow.job.branch}: ' + f'{n_errors + n_failures} failed, ' + f'{n_pendings} pending' ) + message['Subject'] = subject + message.set_content(email_report.render('workflow_report')) if send: ReportUtils.send_email( @@ -120,7 +134,7 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email, smtp_server=smtp_server, smtp_port=smtp_port, recipient_email=recipient_email, - message=email_report.render("workflow_report") + message=message ) else: - output.write(email_report.render("workflow_report")) + output.write(str(message)) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index c73c4d1ff7e..62f39ad723f 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -16,6 +16,8 @@ # under the License. from datetime import date +import email.message +import email.utils from pathlib import Path import time import sys @@ -382,12 +384,20 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, queue.fetch() job = queue.get(job_name) - email_report = EmailReport( - report=Report(job), - sender_name=sender_name, - sender_email=sender_email, - recipient_email=recipient_email + report = Report(job) + email_report = EmailReport(report=report) + message = email.message.EmailMessage() + message.set_charset('utf-8') + message['Message-Id'] = email.utils.make_msgid() + message['Date'] = email.utils.formatdate() + message['From'] = f'{sender_name} <{sender_email}>' + message['To'] = recipient_email + date = report.datetime.strftime('%Y-%m-%d') + message['Subject'] = ( + f'[{date}] Arrow Build Report for {report.name}: ' + f'{len(report.failed_jobs())} failed' ) + message.set_content(email_report.render('nightly_report')) if poll: job.wait_until_finished( @@ -402,10 +412,10 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, smtp_server=smtp_server, smtp_port=smtp_port, recipient_email=recipient_email, - message=email_report.render("nightly_report") + message=message ) else: - output.write(email_report.render("nightly_report")) + output.write(str(message)) @crossbow.command() @@ -641,19 +651,25 @@ def notify_token_expiration(obj, days, sender_name, sender_email, return class TokenExpirationReport: - def __init__(self, token_expiration_date, days_left): - self.token_expiration_date = token_expiration_date + def __init__(self, days_left): self.days_left = days_left - email_report = EmailReport( - report=TokenExpirationReport( - token_expiration_date or "ALREADY_EXPIRED", days_left), - sender_name=sender_name, - sender_email=sender_email, - recipient_email=recipient_email + if not token_expiration_date: + token_expiration_date = 'ALREADY_EXPIRED' + report = TokenExpirationReport(days_left) + email_report = EmailReport(report) + + message = email.message.EmailMessage() + message.set_charset('utf-8') + message['Message-Id'] = email.utils.make_msgid() + message['Date'] = email.utils.formatdate() + message['From'] = f'{sender_name} <{sender_email}>' + message['To'] = recipient_email + message['Subject'] = ( + f'[CI] Arrow Crossbow Token Expiration in {token_expiration_date}' ) + message.set_content(email_report.render('token_expiration')) - message = email_report.render("token_expiration").strip() if send: ReportUtils.send_email( smtp_user=smtp_user, @@ -664,4 +680,4 @@ def __init__(self, token_expiration_date, days_left): message=message ) else: - output.write(message) + output.write(str(message)) diff --git a/dev/archery/archery/crossbow/reports.py b/dev/archery/archery/crossbow/reports.py index 32962410d6e..f2efd8623f8 100644 --- a/dev/archery/archery/crossbow/reports.py +++ b/dev/archery/archery/crossbow/reports.py @@ -259,7 +259,7 @@ def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port, else: smtp.starttls() smtp.login(smtp_user, smtp_password) - smtp.sendmail(smtp_user, recipient_email, message) + smtp.send_message(smtp_user, recipient_email, message) @classmethod def write_csv(cls, report, add_headers=True): @@ -278,9 +278,6 @@ class EmailReport(JinjaReport): } fields = [ 'report', - 'sender_name', - 'sender_email', - 'recipient_email', ] diff --git a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt index c29cafd3938..16409dc38ef 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt +++ b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt @@ -1,7 +1,3 @@ -From: Sender Reporter -To: recipient@arrow.com -Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending - Arrow Build Report for Job ursabot-1 See https://s3.amazonaws.com/arrow-data/index.html for more information. diff --git a/dev/archery/archery/crossbow/tests/test_reports.py b/dev/archery/archery/crossbow/tests/test_reports.py index 620b4c78bbc..4ed40a7793c 100644 --- a/dev/archery/archery/crossbow/tests/test_reports.py +++ b/dev/archery/archery/crossbow/tests/test_reports.py @@ -76,9 +76,7 @@ def test_crossbow_email_report(load_fixture): job = load_fixture('crossbow-job.yaml', decoder=yaml.load) report = Report(job) assert report.tasks_by_state is not None - email_report = EmailReport(report=report, sender_name="Sender Reporter", - sender_email="sender@arrow.com", - recipient_email="recipient@arrow.com") + email_report = EmailReport(report=report) assert ( email_report.render("nightly_report") == textwrap.dedent(expected_msg) diff --git a/dev/archery/archery/templates/email_nightly_report.txt.j2 b/dev/archery/archery/templates/email_nightly_report.txt.j2 index bc040734b03..7b43d7c867e 100644 --- a/dev/archery/archery/templates/email_nightly_report.txt.j2 +++ b/dev/archery/archery/templates/email_nightly_report.txt.j2 @@ -15,13 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -{%- if True -%} -{%- endif -%} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [NIGHTLY] Arrow Build Report for Job {{report.job.branch}}: {{ (report.tasks_by_state["error"] | length) + (report.tasks_by_state["failure"] | length) }} failed, {{ report.tasks_by_state["pending"] | length }} pending - +-#} Arrow Build Report for Job {{ report.job.branch }} See https://s3.amazonaws.com/arrow-data/index.html for more information. @@ -58,4 +52,4 @@ Succeeded Tasks: - {{ task_name }} {{ report.task_url(task) }} {% endfor %} -{%- endif -%} \ No newline at end of file +{%- endif -%} diff --git a/dev/archery/archery/templates/email_token_expiration.txt.j2 b/dev/archery/archery/templates/email_token_expiration.txt.j2 index 54c2005e57e..096026fa3a2 100644 --- a/dev/archery/archery/templates/email_token_expiration.txt.j2 +++ b/dev/archery/archery/templates/email_token_expiration.txt.j2 @@ -15,11 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [CI] Arrow Crossbow Token Expiration in {{ report.token_expiration_date }} - +-#} The Arrow Crossbow Token will expire in {{ report.days_left }} days. Please generate a new Token. Send it to Apache INFRA to update the CROSSBOW_GITHUB_TOKEN. diff --git a/dev/archery/archery/templates/email_workflow_report.txt.j2 b/dev/archery/archery/templates/email_workflow_report.txt.j2 index 193856c1806..6668d6c67ee 100644 --- a/dev/archery/archery/templates/email_workflow_report.txt.j2 +++ b/dev/archery/archery/templates/email_workflow_report.txt.j2 @@ -15,13 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -{%- if True -%} -{%- endif -%} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [{{ report.datetime.strftime('%Y-%m-%d') }}] Arrow Build Report for {{ report.name }}: {{ report.failed_jobs() | length }} failed - +-#} Arrow Build Report for {{ report.name }} Workflow URL: {{ report.url }} @@ -42,4 +36,4 @@ Succeeded Jobs: - {{ job.name }} {{ job.url }} {% endfor %} -{%- endif -%} \ No newline at end of file +{%- endif -%}