From 659d49825d0fe9177491c83dd5c45e263bc5832f Mon Sep 17 00:00:00 2001 From: Agatha Zeren Date: Fri, 8 Mar 2024 01:12:18 -0500 Subject: [PATCH 1/3] First draft of cartesian tool --- tools/api-client/python/cartesian | 133 +++++++++++++++++++++++++++ tools/api-client/python/lib/bmapi.py | 42 ++++++++- 2 files changed, 173 insertions(+), 2 deletions(-) create mode 100755 tools/api-client/python/cartesian diff --git a/tools/api-client/python/cartesian b/tools/api-client/python/cartesian new file mode 100755 index 000000000..986f026dd --- /dev/null +++ b/tools/api-client/python/cartesian @@ -0,0 +1,133 @@ +#!/bin/python + +import configparser +import argparse +import os +import sys +import json +import time + +# Import Button Men utilities. + +sys.path.append('lib') +import bmutils + +def parse_arguments(): + parser = argparse.ArgumentParser(description="Determine and create games for projects of completing certain cartesian products of play counts.") + + parser.add_argument("specs", + nargs = "*", + help = "button or set names to include") + + parser.add_argument("--limit", + default = 5, + help = "target game count") + + parser.add_argument("--mirrors", + default = False, + help = "include mirror matches") + + parser.add_argument("--include-active", + default = True, + help = "include active games") + + + parser.add_argument("--stale-threshold", + default = 14, + help = "days until a game is considered stale") + + + parser.add_argument('--site', + default = 'www', + help = "site to check ('www' by default)") + + parser.add_argument('--config', '-c', + default='~/.bmrc', + help="config file containing site parameters") + + return parser.parse_args() + +def establish_conn(args): + try: + bmconn = bmutils.BMClientParser(os.path.expanduser(args.config), args.site) + except configparser.NoSectionError as e: + print("ERROR: {0} doesn't seem to have a '{1}' section".format(args.config, args.site)) + print("(Exception: {0}: {1})".format(e.__module__, e.message)) + sys.exit(1) + + if not bmconn.verify_login(): + print("ERROR: Could not log in to {0}".format(args.site)) + sys.exit(1) + + return bmconn + +def count(bmconn, args, buttonA, buttonB): + completedCount = bmconn.client.search_game_history({ + "buttonNameA": buttonA, + "buttonNameB": buttonB, + "status": "COMPLETE", + "sortColumn": "lastMove", + "sortDirection": "DESC", + "numberOfResults": 0, + "page": 1, + }).data["summary"]["matchesFound"] + + if completedCount >= int(args.limit) or not args.include_active: + return completedCount + + activeCount = bmconn.client.search_game_history({ + "buttonNameA": buttonA, + "buttonNameB": buttonB, + "status": "ACTIVE", + "lastMoveMin": int(time.time() - (60 * 60 * 24 * args.stale_threshold)), + "sortColumn": "lastMove", + "sortDirection": "DESC", + "numberOfResults": 0, + "page": 1, + }).data["summary"]["matchesFound"] + + return completedCount + activeCount + +def load_buttons(bmconn, args): + allButtons = bmconn.wrap_load_button_names() + + buttons = [] + + for spec in args.specs: + if spec in allButtons: + buttons.append(spec) + else: + data = bmconn.client.load_button_set_data(spec) + if data.status == "ok": + for button in data.data[0]["buttons"]: + buttons.append(button["buttonName"]) + else: + raise RuntimeError("Unknown spec: " + spec) + return buttons + + + +def main(): + args = parse_arguments() + bmconn = establish_conn(args) + + buttons = load_buttons(bmconn, args) + + for idx, buttonA in enumerate(buttons): + print(buttonA) + print("---") + for buttonB in buttons[idx:]: + if buttonA == buttonB and not args.mirrors: + continue + c = count(bmconn, args, buttonA, buttonB); + if c < int(args.limit): + print(buttonB, c) + print() + + + # print(json.dumps(bmconn.wrap_load_completed_games())) + +if __name__ == "__main__": + main() + + diff --git a/tools/api-client/python/lib/bmapi.py b/tools/api-client/python/lib/bmapi.py index f9b57f985..003877aff 100644 --- a/tools/api-client/python/lib/bmapi.py +++ b/tools/api-client/python/lib/bmapi.py @@ -9,8 +9,8 @@ # Import stuff from the future. from __future__ import absolute_import, division, print_function, unicode_literals -from future import standard_library -standard_library.install_aliases() +# from future import standard_library +# standard_library.install_aliases() # Import regular stuff. @@ -267,3 +267,41 @@ def choose_auxiliary_dice(self, gameId, action, dieIdx=None): if dieIdx is not None: args['dieIdx'] = dieIdx return self._make_request(args) + + def search_game_history(self, params): + args = { + 'type': 'searchGameHistory', + } + searchKeys = [ + "gameId", + "playerNameA", + "playerNameB", + "buttonNameA", + "buttonNameB", + "gameStartMin", + "gameStartMax", + "lastMoveMin", + "lastMoveMax", + "winningPlayer", + "status", + "sortColumn", + "sortDirection", + "numberOfResults", + "page", + ] + for key in searchKeys: + try: + args[key] = params[key] + except KeyError: + pass + + return self._make_request(args) + + def load_button_set_data(self, buttonSet=None): + args = { + 'type': 'loadButtonSetData', + } + if buttonSet: + args['buttonSet'] = buttonSet + + return self._make_request(args) From 2072ae4a4047a550924b190049780be094619cef Mon Sep 17 00:00:00 2001 From: Agatha Zeren Date: Mon, 1 Apr 2024 13:30:46 -0400 Subject: [PATCH 2/3] [tools/api-client/python/cartesian] add total summary Counts the total number of games required to meet the limit value for all included buttons. --- tools/api-client/python/cartesian | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/api-client/python/cartesian b/tools/api-client/python/cartesian index 986f026dd..2515ea2b8 100755 --- a/tools/api-client/python/cartesian +++ b/tools/api-client/python/cartesian @@ -113,6 +113,8 @@ def main(): buttons = load_buttons(bmconn, args) + total = 0 + for idx, buttonA in enumerate(buttons): print(buttonA) print("---") @@ -121,8 +123,11 @@ def main(): continue c = count(bmconn, args, buttonA, buttonB); if c < int(args.limit): + total += int(args.limit) - c print(buttonB, c) print() + + print(f"Total: {total}"); # print(json.dumps(bmconn.wrap_load_completed_games())) From 797c9cdc8da1dd8392262390cffc477ce9460695 Mon Sep 17 00:00:00 2001 From: Chaos Date: Wed, 21 Feb 2024 20:42:12 -0500 Subject: [PATCH 3/3] Containers should mount an EFS filesystem and use it for SSL certs and backups --- deploy/docker/buttonmen_ecs_config.json | 3 ++ deploy/docker/deploy_buttonmen_site | 22 ++++++++++++- deploy/docker/startup.sh | 31 +++++++++++++++++++ .../apache/templates/setup_certbot.erb | 7 ++--- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/deploy/docker/buttonmen_ecs_config.json b/deploy/docker/buttonmen_ecs_config.json index 8cf0f7138..37ccca8fc 100644 --- a/deploy/docker/buttonmen_ecs_config.json +++ b/deploy/docker/buttonmen_ecs_config.json @@ -3,6 +3,7 @@ "bmsite_fqdn": "FIXME", "network_subnet": "FIXME", "network_security_group": "FIXME", + "filesystem_id": "FIXME", "log_group": "FIXME", "nlb_arn_port_80": "FIXME", "nlb_arn_port_443": "FIXME", @@ -13,6 +14,7 @@ "bmsite_fqdn": "FIXME", "network_subnet": "FIXME", "network_security_group": "FIXME", + "filesystem_id": "FIXME", "log_group": "FIXME", "nlb_arn_port_80": "FIXME", "nlb_arn_port_443": "FIXME", @@ -23,6 +25,7 @@ "bmsite_fqdn_suffix": "FIXME", "network_subnet": "FIXME", "network_security_group": "FIXME", + "filesystem_id": "FIXME", "log_group": "FIXME", "dns_update_script_path": "FIXME", "load_database_path": "FIXME" diff --git a/deploy/docker/deploy_buttonmen_site b/deploy/docker/deploy_buttonmen_site index b0f13990e..8532d77ed 100644 --- a/deploy/docker/deploy_buttonmen_site +++ b/deploy/docker/deploy_buttonmen_site @@ -102,6 +102,8 @@ def add_ecs_config(git_info, args): raise ValueError(f"ECS config file {BUTTONMEN_ECS_CONFIG_FILE} is missing a valid 'network_security_group' entry for key {key}") if not git_info['config'].get('log_group', None): raise ValueError(f"ECS config file {BUTTONMEN_ECS_CONFIG_FILE} is missing a valid 'log_group' entry for key {key}") + if not git_info['config'].get('filesystem_id', None): + raise ValueError(f"ECS config file {BUTTONMEN_ECS_CONFIG_FILE} is missing a valid 'filesystem_id' entry for key {key}") # Non-dev branches always use a remote database; dev branches do if it's requested as a CLI git_info['config']['use_remote_database'] = args['use_remote_database_for_dev'] or key != 'development' @@ -342,6 +344,12 @@ def update_ecs_task_definition(git_info, repo_uri, ecs_client): 'protocol': 'tcp', }, ], + 'mountPoints': [ + { + "containerPath": "/mnt/efs", + "sourceVolume": "buttonmen-efs", + } + ], 'logConfiguration': { 'logDriver': 'awslogs', 'options': { @@ -352,6 +360,14 @@ def update_ecs_task_definition(git_info, repo_uri, ecs_client): }, }, ], + volumes=[ + { + 'name': 'buttonmen-efs', + 'efsVolumeConfiguration': { + 'fileSystemId': git_info['config']['filesystem_id'], + }, + }, + ], cpu="256", memory="512", requiresCompatibilities=[ @@ -480,11 +496,15 @@ def get_site_public_ipv4(eni_id, ec2_client): def configure_dns(git_info, public_ipv4): + bmsite_fqdn = buttonmen_site_fqdn(git_info) dns_update_script = git_info['config'].get('dns_update_script_path', None) + use_elastic_ip = git_info['config']['use_elastic_ip'] + if use_elastic_ip: + print(f"Not configuring DNS for this target - it uses an elastic IP for {bmsite_fqdn}") + return if not dns_update_script: print("Not configuring DNS for this target - dns_update_script_path is not defined in the config file") return - bmsite_fqdn = buttonmen_site_fqdn(git_info) cmdargs = f"{dns_update_script} {bmsite_fqdn} {public_ipv4}" print(f"About to run DNS update script: {cmdargs}") retcode = os.system(cmdargs) diff --git a/deploy/docker/startup.sh b/deploy/docker/startup.sh index ccd6cdc6e..f19155c78 100644 --- a/deploy/docker/startup.sh +++ b/deploy/docker/startup.sh @@ -10,6 +10,37 @@ set -x /etc/init.d/ssh start /etc/init.d/postfix start +## Remote filesystem setup +FQDN=$(cat /usr/local/etc/bmsite_fqdn) +MNT_DIR="/mnt/efs/${FQDN}" +if [ ! -e "${MNT_DIR}" ]; then + mkdir ${MNT_DIR} +fi + +# Replace the /srv/backup in the image with a remotely-mounted one +if [ ! -e "${MNT_DIR}/backup" ]; then + mkdir ${MNT_DIR}/backup + chown root:adm ${MNT_DIR}/backup + chmod 750 ${MNT_DIR}/backup +fi +rmdir /srv/backup +ln -s ${MNT_DIR}/backup /srv/backup + +# Replace the /etc/letsencrypt in the image with a remotely-mounted one +if [ ! -e "${MNT_DIR}/letsencrypt" ]; then + mkdir ${MNT_DIR}/letsencrypt +fi +mv /etc/letsencrypt/* ${MNT_DIR}/letsencrypt/ +rmdir /etc/letsencrypt +ln -s ${MNT_DIR}/letsencrypt /etc/letsencrypt + +# If /etc/letsencrypt/live exists, there's an existing cert for +# this domain, and we need to install it for apache. +# (If it doesn't exist, we may not have DNS yet, so it's not safe to run certbot.) +if [ -d /etc/letsencrypt/live ]; then + /usr/local/bin/apache_setup_certbot +fi + # Buttonmen services /etc/init.d/apache2 start if [ -f /etc/init.d/mysql ]; then diff --git a/deploy/vagrant/modules/apache/templates/setup_certbot.erb b/deploy/vagrant/modules/apache/templates/setup_certbot.erb index c35cf6c7a..394876871 100644 --- a/deploy/vagrant/modules/apache/templates/setup_certbot.erb +++ b/deploy/vagrant/modules/apache/templates/setup_certbot.erb @@ -18,10 +18,9 @@ if [ "${FQDN}" = "${SANDBOX_FQDN}" ]; then exit 0 fi -if [ -d "/etc/letsencrypt/live" ]; then - echo "Directory /etc/letsencrypt/live is already populated - nothing to do" - exit 0 +if [ -d "/etc/letsencrypt/live/${FQDN}" ]; then + echo "Directory /etc/letsencrypt/live is already populated; continuing in case we need to reinstall the cert to apache" fi echo "Running certbot to configure this site as FQDN ${FQDN}" -/usr/bin/certbot --apache -d ${FQDN} -n --email help@buttonweavers.com --agree-tos +/usr/bin/certbot --apache -d ${FQDN} -n --email help@buttonweavers.com --agree-tos --reinstall