Implement cartesian tool for projects of getting sets of buttons to game counts

deploy/docker/buttonmen_ecs_config.json

@@ -3,6 +3,7 @@
     "bmsite_fqdn": "FIXME",
     "network_subnet": "FIXME",
     "network_security_group": "FIXME",
+    "filesystem_id": "FIXME",
     "log_group": "FIXME",
     "nlb_arn_port_80": "FIXME",
     "nlb_arn_port_443": "FIXME",
@@ -13,6 +14,7 @@
     "bmsite_fqdn": "FIXME",
     "network_subnet": "FIXME",
     "network_security_group": "FIXME",
+    "filesystem_id": "FIXME",
     "log_group": "FIXME",
     "nlb_arn_port_80": "FIXME",
     "nlb_arn_port_443": "FIXME",
@@ -23,6 +25,7 @@
     "bmsite_fqdn_suffix": "FIXME",
     "network_subnet": "FIXME",
     "network_security_group": "FIXME",
+    "filesystem_id": "FIXME",
     "log_group": "FIXME",
     "dns_update_script_path": "FIXME",
     "load_database_path": "FIXME"

deploy/docker/deploy_buttonmen_site

@@ -102,6 +102,8 @@ def add_ecs_config(git_info, args):
     raise ValueError(f"ECS config file {BUTTONMEN_ECS_CONFIG_FILE} is missing a valid 'network_security_group' entry for key {key}")
   if not git_info['config'].get('log_group', None):
     raise ValueError(f"ECS config file {BUTTONMEN_ECS_CONFIG_FILE} is missing a valid 'log_group' entry for key {key}")
+  if not git_info['config'].get('filesystem_id', None):
+    raise ValueError(f"ECS config file {BUTTONMEN_ECS_CONFIG_FILE} is missing a valid 'filesystem_id' entry for key {key}")
 
   # Non-dev branches always use a remote database; dev branches do if it's requested as a CLI
   git_info['config']['use_remote_database'] = args['use_remote_database_for_dev'] or key != 'development'
@@ -342,6 +344,12 @@ def update_ecs_task_definition(git_info, repo_uri, ecs_client):
             'protocol': 'tcp',
           },
         ],
+        'mountPoints': [
+          {
+            "containerPath": "/mnt/efs",
+            "sourceVolume": "buttonmen-efs",
+          }
+        ],
         'logConfiguration': {
           'logDriver': 'awslogs',
           'options': {
@@ -352,6 +360,14 @@ def update_ecs_task_definition(git_info, repo_uri, ecs_client):
         },
       },
     ],
+    volumes=[
+      {
+        'name': 'buttonmen-efs',
+        'efsVolumeConfiguration': {
+          'fileSystemId': git_info['config']['filesystem_id'],
+        },
+      },
+    ],
     cpu="256",
     memory="512",
     requiresCompatibilities=[
@@ -480,11 +496,15 @@ def get_site_public_ipv4(eni_id, ec2_client):
 
 
 def configure_dns(git_info, public_ipv4):
+  bmsite_fqdn = buttonmen_site_fqdn(git_info)
   dns_update_script = git_info['config'].get('dns_update_script_path', None)
+  use_elastic_ip = git_info['config']['use_elastic_ip']
+  if use_elastic_ip:
+    print(f"Not configuring DNS for this target - it uses an elastic IP for {bmsite_fqdn}")
+    return
   if not dns_update_script:
     print("Not configuring DNS for this target - dns_update_script_path is not defined in the config file")
     return
-  bmsite_fqdn = buttonmen_site_fqdn(git_info)
   cmdargs = f"{dns_update_script} {bmsite_fqdn} {public_ipv4}"
   print(f"About to run DNS update script: {cmdargs}")
   retcode = os.system(cmdargs)

deploy/docker/startup.sh

@@ -10,6 +10,37 @@ set -x
 /etc/init.d/ssh start
 /etc/init.d/postfix start
 
+## Remote filesystem setup
+FQDN=$(cat /usr/local/etc/bmsite_fqdn)
+MNT_DIR="/mnt/efs/${FQDN}"
+if [ ! -e "${MNT_DIR}" ]; then
+  mkdir ${MNT_DIR}
+fi
+
+# Replace the /srv/backup in the image with a remotely-mounted one
+if [ ! -e "${MNT_DIR}/backup" ]; then
+  mkdir ${MNT_DIR}/backup
+  chown root:adm ${MNT_DIR}/backup
+  chmod 750 ${MNT_DIR}/backup
+fi
+rmdir /srv/backup
+ln -s ${MNT_DIR}/backup /srv/backup
+
+# Replace the /etc/letsencrypt in the image with a remotely-mounted one
+if [ ! -e "${MNT_DIR}/letsencrypt" ]; then
+  mkdir ${MNT_DIR}/letsencrypt
+fi
+mv /etc/letsencrypt/* ${MNT_DIR}/letsencrypt/
+rmdir /etc/letsencrypt
+ln -s ${MNT_DIR}/letsencrypt /etc/letsencrypt
+
+# If /etc/letsencrypt/live exists, there's an existing cert for
+# this domain, and we need to install it for apache.
+# (If it doesn't exist, we may not have DNS yet, so it's not safe to run certbot.)
+if [ -d /etc/letsencrypt/live ]; then
+  /usr/local/bin/apache_setup_certbot
+fi
+
 # Buttonmen services
 /etc/init.d/apache2 start
 if [ -f /etc/init.d/mysql ]; then

deploy/vagrant/modules/apache/templates/setup_certbot.erb

@@ -18,10 +18,9 @@ if [ "${FQDN}" = "${SANDBOX_FQDN}" ]; then
   exit 0
 fi
 
-if [ -d "/etc/letsencrypt/live" ]; then
-  echo "Directory /etc/letsencrypt/live is already populated - nothing to do"
-  exit 0
+if [ -d "/etc/letsencrypt/live/${FQDN}" ]; then
+  echo "Directory /etc/letsencrypt/live is already populated; continuing in case we need to reinstall the cert to apache"
 fi
 
 echo "Running certbot to configure this site as FQDN ${FQDN}"
-/usr/bin/certbot --apache -d ${FQDN} -n --email help@buttonweavers.com --agree-tos
+/usr/bin/certbot --apache -d ${FQDN} -n --email help@buttonweavers.com --agree-tos --reinstall

tools/api-client/python/cartesian

@@ -0,0 +1,138 @@
+#!/bin/python
+
+import configparser
+import argparse
+import os
+import sys
+import json
+import time
+
+# Import Button Men utilities.
+
+sys.path.append('lib')
+import bmutils
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Determine and create games for projects of completing certain cartesian products of play counts.")
+
+    parser.add_argument("specs",
+                        nargs = "*",
+                        help = "button or set names to include")
+
+    parser.add_argument("--limit",
+                        default = 5,
+                        help = "target game count")
+
+    parser.add_argument("--mirrors",
+                        default = False,
+                        help = "include mirror matches")
+
+    parser.add_argument("--include-active",
+                        default = True,
+                        help = "include active games")
+
+
+    parser.add_argument("--stale-threshold",
+                        default = 14,
+                        help = "days until a game is considered stale")
+
+
+    parser.add_argument('--site',
+                      default = 'www',
+                      help = "site to check ('www' by default)")
+
+    parser.add_argument('--config', '-c',
+                      default='~/.bmrc',
+                      help="config file containing site parameters")
+
+    return parser.parse_args()
+
+def establish_conn(args):
+    try:
+        bmconn = bmutils.BMClientParser(os.path.expanduser(args.config), args.site)
+    except configparser.NoSectionError as e:
+        print("ERROR: {0} doesn't seem to have a '{1}' section".format(args.config, args.site))
+        print("(Exception: {0}: {1})".format(e.__module__, e.message))
+        sys.exit(1)
+
+    if not bmconn.verify_login():
+        print("ERROR: Could not log in to {0}".format(args.site))
+        sys.exit(1)
+
+    return bmconn
+
+def count(bmconn, args, buttonA, buttonB):
+    completedCount =  bmconn.client.search_game_history({
+        "buttonNameA": buttonA,
+        "buttonNameB": buttonB,
+        "status": "COMPLETE",
+        "sortColumn": "lastMove",
+        "sortDirection": "DESC",
+        "numberOfResults": 0,
+        "page": 1,
+    }).data["summary"]["matchesFound"]
+
+    if completedCount >= int(args.limit) or not args.include_active:
+        return completedCount
+
+    activeCount = bmconn.client.search_game_history({
+        "buttonNameA": buttonA,
+        "buttonNameB": buttonB,
+        "status": "ACTIVE",
+        "lastMoveMin": int(time.time() - (60 * 60 * 24 * args.stale_threshold)),
+        "sortColumn": "lastMove",
+        "sortDirection": "DESC",
+        "numberOfResults": 0,
+        "page": 1,
+    }).data["summary"]["matchesFound"]
+
+    return completedCount + activeCount
+
+def load_buttons(bmconn, args):
+    allButtons = bmconn.wrap_load_button_names()
+
+    buttons = []
+
+    for spec in args.specs:
+        if spec in allButtons:
+            buttons.append(spec)
+        else:
+            data = bmconn.client.load_button_set_data(spec)
+            if data.status == "ok":
+                for button in data.data[0]["buttons"]:
+                    buttons.append(button["buttonName"])
+            else:
+                raise RuntimeError("Unknown spec: " + spec)
+    return buttons
+
+
+
+def main():
+    args = parse_arguments()
+    bmconn = establish_conn(args)
+
+    buttons = load_buttons(bmconn, args)
+
+    total = 0
+
+    for idx, buttonA in enumerate(buttons):
+        print(buttonA)
+        print("---")
+        for buttonB in buttons[idx:]:
+            if buttonA == buttonB and not args.mirrors:
+                continue
+            c = count(bmconn, args, buttonA, buttonB);
+            if c < int(args.limit):
+                total += int(args.limit) - c
+                print(buttonB, c)
+        print()
+
+    print(f"Total: {total}");
+
+
+    # print(json.dumps(bmconn.wrap_load_completed_games()))
+
+if __name__ == "__main__":
+    main()
+
+

tools/api-client/python/lib/bmapi.py

@@ -9,8 +9,8 @@
 # Import stuff from the future.
 
 from __future__ import absolute_import, division, print_function, unicode_literals
-from future import standard_library
-standard_library.install_aliases()
+# from future import standard_library
+# standard_library.install_aliases()
 
 # Import regular stuff.
 
@@ -267,3 +267,41 @@ def choose_auxiliary_dice(self, gameId, action, dieIdx=None):
     if dieIdx is not None:
       args['dieIdx'] = dieIdx
     return self._make_request(args)
+
+  def search_game_history(self, params):
+    args = {
+      'type': 'searchGameHistory',
+    }
+    searchKeys = [
+      "gameId",
+      "playerNameA",
+      "playerNameB",
+      "buttonNameA",
+      "buttonNameB",
+      "gameStartMin",
+      "gameStartMax",
+      "lastMoveMin",
+      "lastMoveMax",
+      "winningPlayer",
+      "status",
+      "sortColumn",
+      "sortDirection",
+      "numberOfResults",
+      "page",
+    ]
+    for key in searchKeys:
+      try:
+        args[key] = params[key]
+      except KeyError:
+        pass
+
+    return self._make_request(args)
+
+  def load_button_set_data(self, buttonSet=None):
+    args = {
+      'type': 'loadButtonSetData',
+    }
+    if buttonSet:
+      args['buttonSet'] = buttonSet
+
+    return self._make_request(args)