From a66104fdbd84dd3769e7346d7a1a07a2e3a48e7b Mon Sep 17 00:00:00 2001 From: Jason Spafford Date: Sun, 27 May 2018 17:03:08 -0700 Subject: [PATCH 1/5] WIP --- api/migrations/0002_add_rcon_fields.py | 45 +++++++ api/models/server.py | 12 +- api/sync/__init__.py | 1 + api/sync/serverrcon.py | 157 +++++++++++++++++++++++++ api/tasks.py | 21 ++-- requirements.txt | 1 + serverthrallapi/settings.py | 23 +++- startworker.sh | 1 + 8 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 api/migrations/0002_add_rcon_fields.py create mode 100644 api/sync/serverrcon.py diff --git a/api/migrations/0002_add_rcon_fields.py b/api/migrations/0002_add_rcon_fields.py new file mode 100644 index 0000000..a7b5c38 --- /dev/null +++ b/api/migrations/0002_add_rcon_fields.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.2 on 2018-05-28 00:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0001_initial_squash'), + ] + + operations = [ + migrations.AddField( + model_name='server', + name='rcon_host', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='server', + name='rcon_password', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='server', + name='rcon_port', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='server', + name='sync_rcon', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='server', + name='ip_address', + field=models.TextField(null=True), + ), + migrations.AlterField( + model_name='server', + name='name', + field=models.TextField(null=True), + ), + ] diff --git a/api/models/server.py b/api/models/server.py index 952770a..b2ad295 100644 --- a/api/models/server.py +++ b/api/models/server.py @@ -3,13 +3,17 @@ class Server(models.Model): - name = models.TextField() - ip_address = models.TextField(default='') + private_secret = models.UUIDField() + last_sync = models.DateTimeField(null=True) + sync_rcon = models.BooleanField(default=False) + rcon_host = models.TextField(null=True) + rcon_port = models.IntegerField(null=True) + rcon_password = models.TextField(null=True) + name = models.TextField(null=True) + ip_address = models.TextField(null=True) version = models.TextField(null=True) query_port = models.TextField(null=True) max_players = models.IntegerField(null=True) tick_rate = models.IntegerField(null=True) - private_secret = models.UUIDField() - last_sync = models.DateTimeField(null=True) objects = ServerManager.as_manager() diff --git a/api/sync/__init__.py b/api/sync/__init__.py index cd9f9fd..dc19625 100644 --- a/api/sync/__init__.py +++ b/api/sync/__init__.py @@ -1,3 +1,4 @@ # flake8: noqa from .ginfo import sync_ginfo from .serverdata import sync_server_data +from .serverrcon import sync_server_rcon diff --git a/api/sync/serverrcon.py b/api/sync/serverrcon.py new file mode 100644 index 0000000..a7a023a --- /dev/null +++ b/api/sync/serverrcon.py @@ -0,0 +1,157 @@ +from api.models import Server, ServerSyncData +from valve.rcon import RCON, RCONError +import json + +def from_int_bool(v): + return bool(int(v)) + +def execute_rcon(rcon_host, rcon_port, rcon_password, command): + print('Executing on %s:%s %s' % (rcon_host, rcon_port, command)) + + try: + with RCON((rcon_host, rcon_port), rcon_password) as rcon: + return rcon.execute(command) + except RCONError: + print('Error sending command ' + command) + except Exception as ex: + print('Error when exceuting RCON command') + raise ex + +def execute_rcon_sql(server, sql, type_map): + response = execute_rcon( + rcon_host=server.rcon_host, + rcon_port=server.rcon_port, + rcon_password=server.rcon_password, + command='sql %s' % sql) + + if response is None: + return False, [] + + return True, parse_response_into_rows(response, type_map) + +def parse_response_into_rows(response, type_map): + print "RESPNOSE", response.text + print 'TYPE-MAP', type_map + + rows = [] + + for row_index, line in enumerate(response.text.splitlines()): + + if row_index == 0: + continue + + print "ROW", row_index, line + + row = [] + for col_index, col in enumerate(line.split('|')): + + if col_index == 0: + col = col.replace('#%s' % (row_index - 1), '') + + type_transform = type_map[col_index] + row.append(type_transform(col.strip())) + + rows.append(row) + + return rows + +def get_characters(server): + is_success, rows = execute_rcon_sql(server, ''' + SELECT + acc.online, + ch.id, + ch.char_name, + ch.level, + ch.playerId, + ch.lastTimeOnline, + ch.killerName, + ch.guild, + act.x, + act.y, + act.z + FROM characters AS ch + LEFT JOIN account AS acc ON ch.playerId = acc.user + LEFT JOIN actor_position AS act ON ch.id = act.id; + ''', [from_int_bool, int, unicode, int, unicode, int, unicode, int, float, float, float]) + + if not is_success: + return None + + characters = [] + + for row in rows: + characters.append({ + 'name': row[2], + 'level': row[3], + 'is_online': row[0], + 'steam_id': row[4], + 'conan_id': row[1], + 'last_killed_by': row[6], + 'last_online': row[5], + 'clan_id': row[7], + 'x': row[8], + 'y': row[9], + 'z': row[10]}) + + return characters + +def get_clans(server): + is_success, rows = execute_rcon_sql(server, ''' + SELECT guildId, name, owner, messageOfTheDay FROM guilds + ''', [int, unicode, int, unicode]) + + if not is_success: + return None + + guilds = [] + + for row in rows: + guilds.append({ + 'id': row[0], + 'name': row[1], + 'owner_id': row[2], + 'motd': row[3] + }) + + return guilds + +def sync_server_rcon(server_id): + server = (Server.objects + .filter(id=server_id) + .first()) + + is_valid = ( + server is not None and + server.rcon_host is not None and + server.rcon_password is not None and + server.rcon_port is not None) + + if not is_valid: + return + + characters = get_characters(server) + clans = get_clans(server) + + # if clans is not None: + # for character in characters: + # print character + + # if clans is not None: + # for clan in clans: + # print clan + + is_valid_sync = ( + clans is not None and + characters is not None) + + if not is_valid_sync: + return + + sync_data = ServerSyncData.objects.create(server=server, data=json.dumps({ + 'version': 'api', + 'characters': characters, + 'clans': clans + })) + + from api.tasks import sync_server_data_task + sync_server_data_task(sync_data.id, {}) diff --git a/api/tasks.py b/api/tasks.py index 0152f37..85c3b2d 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,7 +1,5 @@ -from .sync import sync_server_data, sync_ginfo -from api.models import CharacterHistory -from datetime import timedelta -from django.utils import timezone +from .sync import sync_server_data, sync_ginfo, sync_server_rcon +from api.models import Server from serverthrallapi.celery import app @@ -16,6 +14,15 @@ def sync_server_data_task(sync_data_id, request_get_params): @app.task() -def delete_old_history_task(): - history_threshold = timezone.now() - timedelta(days=5) - CharacterHistory.objects.filter(created__lt=history_threshold).delete() +def sync_server_rcon_task(server_id): + sync_server_rcon(server_id) + + +@app.task() +def sync_all_rcon_servers_task(): + server_ids = (Server.objects + .filter(rcon_host__isnull=False) + .values_list('id', flat=True)) + + for server_id in server_ids: + sync_server_rcon_task(server_id) diff --git a/requirements.txt b/requirements.txt index ac1b583..2ccc9c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ psycopg2==2.6.2 pytz==2017.2 requests==2.18.1 serpy==0.1.1 +python-valve==0.2.1 diff --git a/serverthrallapi/settings.py b/serverthrallapi/settings.py index 095bd87..2657c7d 100644 --- a/serverthrallapi/settings.py +++ b/serverthrallapi/settings.py @@ -1,6 +1,6 @@ import os -import json import dj_database_url +from kombu import Exchange, Queue ENVIRONMENT = os.environ.get('ENVIRONMENT', 'DEVELOPMENT') @@ -91,8 +91,29 @@ # Celery CELERY_TASK_ALWAYS_EAGER = DEBUG +CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + CELERY_BROKER_URL = os.environ.get('RABBITMQ_BIGWIG_URL') +CELERY_TASK_DEFAULT_QUEUE = 'default' +CELERY_TASK_DEFAULT_EXCHANGE = 'default' +CELERY_TASK_DEFAULT_ROUTING_KEY = 'default' + +CELLERY_QUEUES = ( + Queue('default', Exchange('default'), routing_key='default'), + Queue('sync_rest', Exchange('default'), routing_key='sync_rest'), + Queue('sync_rcon', Exchange('default'), routing_key='sync_rcon'), +) + +CELERY_BEAT_SCHEDULE = { + 'sync_all_rcon_servers': { + 'task': 'api.tasks.sync_all_rcon_servers_task', + 'schedule': 2 * 60, + 'args': [], + 'options': {'queue': 'sync_rcon'} + } +} ST_ENABLE_HISTORY = os.environ.get('ST_ENABLE_HISTORY', 'false').lower().startswith('true') diff --git a/startworker.sh b/startworker.sh index f9a8540..d91db82 100644 --- a/startworker.sh +++ b/startworker.sh @@ -7,3 +7,4 @@ newrelic-admin run-program \ --app=serverthrallapi.celery_app \ --beat \ --scheduler django + --queues default,sync_rest,sync_rcon From 6a3e93b43e583760204e2f7e7e1789938586cb40 Mon Sep 17 00:00:00 2001 From: Jason Spafford Date: Sun, 27 May 2018 17:39:56 -0700 Subject: [PATCH 2/5] WIP2 --- api/management.py/__init__.py | 0 api/management.py/commands/__init__.py | 0 api/management.py/commands/syncrcon.py | 0 ...0001_initial_squash.py => 0001_initial.py} | 20 ------------------- api/migrations/0002_add_rcon_fields.py | 4 ++-- 5 files changed, 2 insertions(+), 22 deletions(-) create mode 100644 api/management.py/__init__.py create mode 100644 api/management.py/commands/__init__.py create mode 100644 api/management.py/commands/syncrcon.py rename api/migrations/{0001_initial_squash.py => 0001_initial.py} (89%) diff --git a/api/management.py/__init__.py b/api/management.py/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management.py/commands/__init__.py b/api/management.py/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management.py/commands/syncrcon.py b/api/management.py/commands/syncrcon.py new file mode 100644 index 0000000..e69de29 diff --git a/api/migrations/0001_initial_squash.py b/api/migrations/0001_initial.py similarity index 89% rename from api/migrations/0001_initial_squash.py rename to api/migrations/0001_initial.py index fb8ee02..6ec52e6 100644 --- a/api/migrations/0001_initial_squash.py +++ b/api/migrations/0001_initial.py @@ -6,28 +6,8 @@ import django.db.models.deletion -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# api.migrations.0002_add_test_data - class Migration(migrations.Migration): - replaces = [ - (b'api', '0001_initial'), - (b'api', '0002_add_test_data'), - (b'api', '0003_remove_server_public_secret'), - (b'api', '0004_characterhistory'), - (b'api', '0005_ginfocharacter'), - (b'api', '0006_clan'), - (b'api', 'add_clan_id'), - (b'api', '0001_server_ip_address'), - (b'api', '0002_clean_data_schema'), - (b'api', '0003_add_server_fields'), - (b'api', '0004_server_version'), - (b'api', '0005_make_location_nullable') - ] - initial = True dependencies = [ diff --git a/api/migrations/0002_add_rcon_fields.py b/api/migrations/0002_add_rcon_fields.py index a7b5c38..25122ca 100644 --- a/api/migrations/0002_add_rcon_fields.py +++ b/api/migrations/0002_add_rcon_fields.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.2 on 2018-05-28 00:02 +# Generated by Django 1.11.2 on 2018-05-28 00:17 from __future__ import unicode_literals from django.db import migrations, models @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ - ('api', '0001_initial_squash'), + ('api', '0001_initial'), ] operations = [ From 2368f85bb81037b4c2013e13b5116771ba2b4907 Mon Sep 17 00:00:00 2001 From: Jason Spafford Date: Mon, 28 May 2018 21:16:08 -0700 Subject: [PATCH 3/5] WIP: --- api/management.py/commands/syncrcon.py | 0 api/{management.py => management}/__init__.py | 0 .../commands/__init__.py | 0 api/management/commands/sync_rcon.py | 12 ++ api/sync/serverdata.py | 3 - api/sync/serverrcon.py | 110 ++++++++++++------ 6 files changed, 84 insertions(+), 41 deletions(-) delete mode 100644 api/management.py/commands/syncrcon.py rename api/{management.py => management}/__init__.py (100%) rename api/{management.py => management}/commands/__init__.py (100%) create mode 100644 api/management/commands/sync_rcon.py diff --git a/api/management.py/commands/syncrcon.py b/api/management.py/commands/syncrcon.py deleted file mode 100644 index e69de29..0000000 diff --git a/api/management.py/__init__.py b/api/management/__init__.py similarity index 100% rename from api/management.py/__init__.py rename to api/management/__init__.py diff --git a/api/management.py/commands/__init__.py b/api/management/commands/__init__.py similarity index 100% rename from api/management.py/commands/__init__.py rename to api/management/commands/__init__.py diff --git a/api/management/commands/sync_rcon.py b/api/management/commands/sync_rcon.py new file mode 100644 index 0000000..20e1a54 --- /dev/null +++ b/api/management/commands/sync_rcon.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand +from api.tasks import sync_server_rcon_task + +class Command(BaseCommand): + help = 'Sync RCON for a specific server' + + def add_arguments(self, parser): + parser.add_argument('server_id', nargs='+', type=int) + + def handle(self, *args, **options): + for server_id in options['server_id']: + sync_server_rcon_task(server_id) diff --git a/api/sync/serverdata.py b/api/sync/serverdata.py index 5841d67..fd334b6 100644 --- a/api/sync/serverdata.py +++ b/api/sync/serverdata.py @@ -183,6 +183,3 @@ def sync_server_data(sync_data_id, request_get_params): sync_ginfo_task.delay(changed_character_ids, request_get_params['ginfo_group_uid'], request_get_params['ginfo_access_token']) - - from api.tasks import delete_old_history_task - delete_old_history_task.delay() diff --git a/api/sync/serverrcon.py b/api/sync/serverrcon.py index a7a023a..9b83f11 100644 --- a/api/sync/serverrcon.py +++ b/api/sync/serverrcon.py @@ -17,62 +17,93 @@ def execute_rcon(rcon_host, rcon_port, rcon_password, command): print('Error when exceuting RCON command') raise ex -def execute_rcon_sql(server, sql, type_map): - response = execute_rcon( - rcon_host=server.rcon_host, - rcon_port=server.rcon_port, - rcon_password=server.rcon_password, - command='sql %s' % sql) +def execute_rcon_sql(server, build_sql, paginate_predicate, col_type_map): + last_col_paginate = -1 + results = [] - if response is None: - return False, [] + while True: + response = execute_rcon( + rcon_host=server.rcon_host, + rcon_port=server.rcon_port, + rcon_password=server.rcon_password, + command='sql %s' % build_sql(last_col_paginate)) - return True, parse_response_into_rows(response, type_map) + if response is None: + return False, [] -def parse_response_into_rows(response, type_map): - print "RESPNOSE", response.text - print 'TYPE-MAP', type_map + rows = parse_response_into_rows(response, col_type_map) + if len(rows) == 0: + break + + last_col_paginate = paginate_predicate(rows[len(rows) - 1]) + results = results + rows + + return True, results + +def parse_response_into_rows(response, col_type_map): rows = [] - for row_index, line in enumerate(response.text.splitlines()): + for row_index, line in enumerate(response.body.splitlines()): + + print line if row_index == 0: continue - print "ROW", row_index, line + # print "ROW `%s`" % (line) row = [] for col_index, col in enumerate(line.split('|')): + col = col.strip() + + if col_index >= len(col_type_map): + continue if col_index == 0: - col = col.replace('#%s' % (row_index - 1), '') + col = col.replace('#%s' % (row_index - 1), '').strip() + + if col == 'void': + col = None - type_transform = type_map[col_index] - row.append(type_transform(col.strip())) + if col is not None: + type_transform = col_type_map[col_index] + col = type_transform(col) + + row.append(col) rows.append(row) return rows def get_characters(server): - is_success, rows = execute_rcon_sql(server, ''' - SELECT - acc.online, - ch.id, - ch.char_name, - ch.level, - ch.playerId, - ch.lastTimeOnline, - ch.killerName, - ch.guild, - act.x, - act.y, - act.z - FROM characters AS ch - LEFT JOIN account AS acc ON ch.playerId = acc.user - LEFT JOIN actor_position AS act ON ch.id = act.id; - ''', [from_int_bool, int, unicode, int, unicode, int, unicode, int, float, float, float]) + def build_sql(paginate_value): + return ''' + SELECT + acc.online, + ch.id, + ch.char_name, + ch.level, + ch.playerId, + ch.lastTimeOnline, + ch.killerName, + ch.guild, + act.x, + act.y, + act.z + FROM characters AS ch + LEFT JOIN account AS acc ON ch.playerId = acc.user + LEFT JOIN actor_position AS act ON ch.id = act.id + WHERE ch.id > %s + ORDER BY ch.id + LIMIT 20 + ''' % paginate_value + + def paginate_predicate(r): + return r[1] + + is_success, rows = execute_rcon_sql(server, build_sql, paginate_predicate, + [from_int_bool, int, str, int, str, int, str, int, float, float, float]) if not is_success: return None @@ -81,13 +112,13 @@ def get_characters(server): for row in rows: characters.append({ + 'is_online': row[0], + 'conan_id': row[1], 'name': row[2], 'level': row[3], - 'is_online': row[0], 'steam_id': row[4], - 'conan_id': row[1], - 'last_killed_by': row[6], 'last_online': row[5], + 'last_killed_by': row[6], 'clan_id': row[7], 'x': row[8], 'y': row[9], @@ -121,6 +152,7 @@ def sync_server_rcon(server_id): .first()) is_valid = ( + server.sync_rcon and server is not None and server.rcon_host is not None and server.rcon_password is not None and @@ -130,7 +162,7 @@ def sync_server_rcon(server_id): return characters = get_characters(server) - clans = get_clans(server) + # clans = get_clans(server) # if clans is not None: # for character in characters: @@ -140,6 +172,8 @@ def sync_server_rcon(server_id): # for clan in clans: # print clan + return + is_valid_sync = ( clans is not None and characters is not None) From a5161f6b4feddcc55c848f6ff9c8c50029f0cfed Mon Sep 17 00:00:00 2001 From: Jason Spafford Date: Mon, 28 May 2018 21:41:16 -0700 Subject: [PATCH 4/5] Add edit stuff --- api/views/server.py | 19 ++++++++++++++++--- api/views/servers.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/api/views/server.py b/api/views/server.py index a878170..d48e046 100644 --- a/api/views/server.py +++ b/api/views/server.py @@ -3,6 +3,7 @@ from api.serializers import ServerSerializer, ServerAdminSerializer from .base import BaseView +import json class ServerView(BaseView): @@ -20,16 +21,28 @@ def post(self, request, server_id): if 'private_secret' not in request.GET: return HttpResponse('missing required param private_secret') - data = request.GET server = self.get_server_private(request, server_id) if server is None: return HttpResponse('server does not exist', status=404) - if 'name' in data: - server.name = data['name'] + data = json.loads(request.body) + + if 'sync_rcon' in data: + server.sync_rcon = data['sync_rcon'] + if 'rcon_host' in data: + server.rcon_host = data['rcon_host'] + server.ip_address = data['rcon_host'] + if 'rcon_port' in data: + server.rcon_port = data['rcon_port'] + if 'rcon_password' in data: + server.rcon_password = data['rcon_password'] + + # TODO: If sync_rcon is true, and credentials are different + # check RCON credentials work before creating or editing the server server.save() + server.refresh_from_db() serialized = ServerAdminSerializer(server).data return JsonResponse(serialized, status=200) diff --git a/api/views/servers.py b/api/views/servers.py index 35a0d91..0a11932 100644 --- a/api/views/servers.py +++ b/api/views/servers.py @@ -6,6 +6,7 @@ from api.serializers import ServerAdminSerializer, ServerSerializer from .base import BaseView +import json class ServersView(BaseView): @@ -18,6 +19,19 @@ def get(self, request): def post(self, request): server = Server() server.private_secret = uuid1() + + data = json.loads(request.body) + + if 'sync_rcon' in data: + server.sync_rcon = data['sync_rcon'] + if 'rcon_host' in data: + server.rcon_host = data['rcon_host'] + server.ip_address = data['rcon_host'] + if 'rcon_port' in data: + server.rcon_port = data['rcon_port'] + if 'rcon_password' in data: + server.rcon_password = data['rcon_password'] + server.save() server = self.get_server_public(request, server.id) From 2e4c68d200e38068b6cdf19e5f09e045cc2bb4b3 Mon Sep 17 00:00:00 2001 From: Jason Spafford Date: Tue, 29 May 2018 00:04:52 -0700 Subject: [PATCH 5/5] Add RCON information to admin serializer --- api/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/serializers.py b/api/serializers.py index 8b64582..37a86bf 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -78,4 +78,8 @@ def get_online_count(self, server): class ServerAdminSerializer(ServerSerializer): - private_secret = serpy.Field() + private_secret = serpy.Field() + sync_rcon = serpy.Field() + rcon_host = serpy.Field() + rcon_password = serpy.Field() + rcon_port = serpy.Field()