From 5d22f52841dbd5ca261a2f118fe311fcfc37b8a0 Mon Sep 17 00:00:00 2001 From: Virginia Dooley Date: Mon, 18 Dec 2023 17:10:51 +0000 Subject: [PATCH 1/9] Clean party names --- .../management/commands/candidatebot_import_next_ppcs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index 5ec0a071e6..3880e3a403 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -101,7 +101,13 @@ def get_party_from_line(self, line): party_id = line["Party ID"] if party_id not in self.party_cache: try: - self.party_cache[party_id] = Party.objects.get(ec_id=party_id) + if "ynmp-party:2" in party_id: + ec_id = party_id + elif "-" in party_id and party_id != "ynmp-party:2": + ec_id = "joint-party:" + party_id + else: + ec_id = "PP" + party_id + self.party_cache[party_id] = Party.objects.get(ec_id=ec_id) except Party.DoesNotExist: print(line) raise ValueError("Party not found in line") From f5cfc5c7bb864981419194b7f3eaf1c4599f7e26 Mon Sep 17 00:00:00 2001 From: Virginia Dooley Date: Mon, 18 Dec 2023 17:11:15 +0000 Subject: [PATCH 2/9] Match column name --- .../management/commands/candidatebot_import_next_ppcs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index 3880e3a403..18852e8a6c 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -118,9 +118,8 @@ def line_has_values(self, line): Some lines exist that only have a ballot ID and no actual membership info """ - return any( - (line["Candidate Name"], line["Existing Candidate Profile URL"]) - ) + + return any((line["Candidate Name"], line["Existing Candidate Profile"])) def add_contact_details(self, bot, person, line): if not person.get_email and line["Email"]: From a0d7d701d43a43aec1992927b162fbfe63b71450 Mon Sep 17 00:00:00 2001 From: Virginia Dooley Date: Mon, 18 Dec 2023 17:14:28 +0000 Subject: [PATCH 3/9] Update README with PPCS importer instructions --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ec9c1bac6..b06f71d415 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ Sentry is used to report errors in production. We have added a url for `sentry-d ``` # Pre-election Tasks - # Enable Candidate Leaderboard @@ -70,4 +69,6 @@ We take a slice of edits in YNR and assign them to a election leaderboard. This is defined here: https://github.com/DemocracyClub/yournextrepresentative/blob/master/ynr/apps/candidates/views/mixins.py#L20 -We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy] \ No newline at end of file +We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy] + +If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with `python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL` \ No newline at end of file From cb6c1d9b64e8a05c8ed354d6373f931b91facbd2 Mon Sep 17 00:00:00 2001 From: Sym Roe Date: Thu, 25 Jan 2024 12:47:59 +0000 Subject: [PATCH 4/9] Update clean_twitter_username to support x.com --- ynr/apps/people/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ynr/apps/people/helpers.py b/ynr/apps/people/helpers.py index db5fbf81e5..a8df99634c 100644 --- a/ynr/apps/people/helpers.py +++ b/ynr/apps/people/helpers.py @@ -109,9 +109,9 @@ def clean_mastodon_username(username): def clean_twitter_username(username): # Remove any URL bits around it: username = username.strip() - m = re.search(r"^.*twitter.com/(\w+)", username) + m = re.search(r"^.*(twitter.com|x.com)/(\@?)(\w+)", username) if m: - username = m.group(1) + username = m.group(3) # If there's a leading '@', strip that off: username = re.sub(r"^@", "", username) if not re.search(r"^\w*$", username): From a03bf4a5b09d203849b781f84474dbe941d63f5f Mon Sep 17 00:00:00 2001 From: Sym Roe Date: Thu, 25 Jan 2024 12:50:04 +0000 Subject: [PATCH 5/9] Match ballots based on constituency name --- .../commands/candidatebot_import_next_ppcs.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index 18852e8a6c..62bbe0848e 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -2,12 +2,14 @@ import requests from candidatebot.helpers import CandidateBot +from candidates.models import Ballot from django.core.management.base import BaseCommand from django.db import transaction from elections.models import Election from parties.models import Party from people.models import Person from popolo.models import Membership, NotStandingValidationError +from slugify import slugify class Command(BaseCommand): @@ -86,16 +88,20 @@ def get_source_from_line(self, line): return line.get("Source", "PPC sheet importer") def get_ballot_from_line(self, line): - ballot_paper_start = ( - ".".join(line["Ballot paper ID"].split(".")[0:-1]) + "." - ) - if ballot_paper_start not in self.ballot_cache: - self.ballot_cache[ - ballot_paper_start - ] = self.election.ballot_set.get( - ballot_paper_id__startswith=ballot_paper_start - ) - return self.ballot_cache[ballot_paper_start] + constituency = line["Constituency"].strip() + + if constituency not in self.ballot_cache: + try: + ballot = Ballot.objects.get( + election=self.election, post__label=constituency + ) + except Ballot.DoesNotExist: + print(slugify(constituency)) + ballot = Ballot.objects.get( + election=self.election, post__slug=slugify(constituency) + ) + self.ballot_cache[constituency] = ballot + return self.ballot_cache[constituency] def get_party_from_line(self, line): party_id = line["Party ID"] From cbd511e31229a5f42bc8342183c332af53e39b7b Mon Sep 17 00:00:00 2001 From: Sym Roe Date: Thu, 25 Jan 2024 12:50:28 +0000 Subject: [PATCH 6/9] Take election date from command line --- .../management/commands/candidatebot_import_next_ppcs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index 62bbe0848e..bb7f194026 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -17,10 +17,12 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("--sheet-url", action="store", required=True) + parser.add_argument("--election-date", action="store", required=True) def get_next_parl_election(self): election = ( Election.objects.filter(slug__contains="parl.") + .filter(election_date=self.election_date) .future() .order_by("election_date") .first() @@ -33,6 +35,7 @@ def get_next_parl_election(self): def handle(self, *args, **options): self.ballot_cache = {} self.party_cache = {} + self.election_date = options["election_date"] self.election = self.get_next_parl_election() req = requests.get(options["sheet_url"]) req.raise_for_status() From c575cab20b28b4b0c69f674a16c204ee06cb93a1 Mon Sep 17 00:00:00 2001 From: Sym Roe Date: Thu, 25 Jan 2024 12:52:24 +0000 Subject: [PATCH 7/9] Decode CSV properly --- .../management/commands/candidatebot_import_next_ppcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index bb7f194026..c3b04af822 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -39,11 +39,11 @@ def handle(self, *args, **options): self.election = self.get_next_parl_election() req = requests.get(options["sheet_url"]) req.raise_for_status() - self.parse_csv(req.text) raise ValueError("Still testing") + self.parse_csv(req.content.decode("utf8").splitlines()) def parse_csv(self, in_file): - csv_data = csv.DictReader(in_file.splitlines()) + csv_data = csv.DictReader(in_file) for line in csv_data: self.import_line(line) From ccd1b0fe52a0c2e42d7fbe1f273ccd704da111c8 Mon Sep 17 00:00:00 2001 From: Sym Roe Date: Thu, 25 Jan 2024 12:52:43 +0000 Subject: [PATCH 8/9] Remove email source --- .../management/commands/candidatebot_import_next_ppcs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index c3b04af822..37e7aa398d 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -133,9 +133,6 @@ def line_has_values(self, line): def add_contact_details(self, bot, person, line): if not person.get_email and line["Email"]: bot.add_email(line["Email"]) - if line["Email Source"] and line["Source"] != line["Email Source"]: - # The source for the email is different, save now - bot.save(line["Email Source"]) if line["Twitter"] and not person.tmp_person_identifiers.filter( value=line["Twitter"] From 53210e3d16ab78c138f9fbb154c849fd3f211a19 Mon Sep 17 00:00:00 2001 From: Sym Roe Date: Thu, 25 Jan 2024 12:53:01 +0000 Subject: [PATCH 9/9] Remove final exception --- .../management/commands/candidatebot_import_next_ppcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index 37e7aa398d..75c8466e28 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -39,8 +39,8 @@ def handle(self, *args, **options): self.election = self.get_next_parl_election() req = requests.get(options["sheet_url"]) req.raise_for_status() - raise ValueError("Still testing") self.parse_csv(req.content.decode("utf8").splitlines()) + # raise ValueError("Still testing") def parse_csv(self, in_file): csv_data = csv.DictReader(in_file)