diff --git a/api/management/__init__.py b/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/__init__.py b/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/sync_rcon.py b/api/management/commands/sync_rcon.py new file mode 100644 index 0000000..20e1a54 --- /dev/null +++ b/api/management/commands/sync_rcon.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand +from api.tasks import sync_server_rcon_task + +class Command(BaseCommand): + help = 'Sync RCON for a specific server' + + def add_arguments(self, parser): + parser.add_argument('server_id', nargs='+', type=int) + + def handle(self, *args, **options): + for server_id in options['server_id']: + sync_server_rcon_task(server_id) diff --git a/api/migrations/0001_initial_squash.py b/api/migrations/0001_initial.py similarity index 89% rename from api/migrations/0001_initial_squash.py rename to api/migrations/0001_initial.py index fb8ee02..6ec52e6 100644 --- a/api/migrations/0001_initial_squash.py +++ b/api/migrations/0001_initial.py @@ -6,28 +6,8 @@ import django.db.models.deletion -# Functions from the following migrations need manual copying. -# Move them and any dependencies into this file, then update the -# RunPython operations to refer to the local versions: -# api.migrations.0002_add_test_data - class Migration(migrations.Migration): - replaces = [ - (b'api', '0001_initial'), - (b'api', '0002_add_test_data'), - (b'api', '0003_remove_server_public_secret'), - (b'api', '0004_characterhistory'), - (b'api', '0005_ginfocharacter'), - (b'api', '0006_clan'), - (b'api', 'add_clan_id'), - (b'api', '0001_server_ip_address'), - (b'api', '0002_clean_data_schema'), - (b'api', '0003_add_server_fields'), - (b'api', '0004_server_version'), - (b'api', '0005_make_location_nullable') - ] - initial = True dependencies = [ diff --git a/api/migrations/0002_add_rcon_fields.py b/api/migrations/0002_add_rcon_fields.py new file mode 100644 index 0000000..25122ca --- /dev/null +++ b/api/migrations/0002_add_rcon_fields.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.2 on 2018-05-28 00:17 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='server', + name='rcon_host', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='server', + name='rcon_password', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='server', + name='rcon_port', + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name='server', + name='sync_rcon', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='server', + name='ip_address', + field=models.TextField(null=True), + ), + migrations.AlterField( + model_name='server', + name='name', + field=models.TextField(null=True), + ), + ] diff --git a/api/models/server.py b/api/models/server.py index 952770a..b2ad295 100644 --- a/api/models/server.py +++ b/api/models/server.py @@ -3,13 +3,17 @@ class Server(models.Model): - name = models.TextField() - ip_address = models.TextField(default='') + private_secret = models.UUIDField() + last_sync = models.DateTimeField(null=True) + sync_rcon = models.BooleanField(default=False) + rcon_host = models.TextField(null=True) + rcon_port = models.IntegerField(null=True) + rcon_password = models.TextField(null=True) + name = models.TextField(null=True) + ip_address = models.TextField(null=True) version = models.TextField(null=True) query_port = models.TextField(null=True) max_players = models.IntegerField(null=True) tick_rate = models.IntegerField(null=True) - private_secret = models.UUIDField() - last_sync = models.DateTimeField(null=True) objects = ServerManager.as_manager() diff --git a/api/serializers.py b/api/serializers.py index 8b64582..37a86bf 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -78,4 +78,8 @@ def get_online_count(self, server): class ServerAdminSerializer(ServerSerializer): - private_secret = serpy.Field() + private_secret = serpy.Field() + sync_rcon = serpy.Field() + rcon_host = serpy.Field() + rcon_password = serpy.Field() + rcon_port = serpy.Field() diff --git a/api/sync/__init__.py b/api/sync/__init__.py index cd9f9fd..dc19625 100644 --- a/api/sync/__init__.py +++ b/api/sync/__init__.py @@ -1,3 +1,4 @@ # flake8: noqa from .ginfo import sync_ginfo from .serverdata import sync_server_data +from .serverrcon import sync_server_rcon diff --git a/api/sync/serverdata.py b/api/sync/serverdata.py index 5841d67..fd334b6 100644 --- a/api/sync/serverdata.py +++ b/api/sync/serverdata.py @@ -183,6 +183,3 @@ def sync_server_data(sync_data_id, request_get_params): sync_ginfo_task.delay(changed_character_ids, request_get_params['ginfo_group_uid'], request_get_params['ginfo_access_token']) - - from api.tasks import delete_old_history_task - delete_old_history_task.delay() diff --git a/api/sync/serverrcon.py b/api/sync/serverrcon.py new file mode 100644 index 0000000..9b83f11 --- /dev/null +++ b/api/sync/serverrcon.py @@ -0,0 +1,191 @@ +from api.models import Server, ServerSyncData +from valve.rcon import RCON, RCONError +import json + +def from_int_bool(v): + return bool(int(v)) + +def execute_rcon(rcon_host, rcon_port, rcon_password, command): + print('Executing on %s:%s %s' % (rcon_host, rcon_port, command)) + + try: + with RCON((rcon_host, rcon_port), rcon_password) as rcon: + return rcon.execute(command) + except RCONError: + print('Error sending command ' + command) + except Exception as ex: + print('Error when exceuting RCON command') + raise ex + +def execute_rcon_sql(server, build_sql, paginate_predicate, col_type_map): + last_col_paginate = -1 + results = [] + + while True: + response = execute_rcon( + rcon_host=server.rcon_host, + rcon_port=server.rcon_port, + rcon_password=server.rcon_password, + command='sql %s' % build_sql(last_col_paginate)) + + if response is None: + return False, [] + + rows = parse_response_into_rows(response, col_type_map) + + if len(rows) == 0: + break + + last_col_paginate = paginate_predicate(rows[len(rows) - 1]) + results = results + rows + + return True, results + +def parse_response_into_rows(response, col_type_map): + rows = [] + + for row_index, line in enumerate(response.body.splitlines()): + + print line + + if row_index == 0: + continue + + # print "ROW `%s`" % (line) + + row = [] + for col_index, col in enumerate(line.split('|')): + col = col.strip() + + if col_index >= len(col_type_map): + continue + + if col_index == 0: + col = col.replace('#%s' % (row_index - 1), '').strip() + + if col == 'void': + col = None + + if col is not None: + type_transform = col_type_map[col_index] + col = type_transform(col) + + row.append(col) + + rows.append(row) + + return rows + +def get_characters(server): + def build_sql(paginate_value): + return ''' + SELECT + acc.online, + ch.id, + ch.char_name, + ch.level, + ch.playerId, + ch.lastTimeOnline, + ch.killerName, + ch.guild, + act.x, + act.y, + act.z + FROM characters AS ch + LEFT JOIN account AS acc ON ch.playerId = acc.user + LEFT JOIN actor_position AS act ON ch.id = act.id + WHERE ch.id > %s + ORDER BY ch.id + LIMIT 20 + ''' % paginate_value + + def paginate_predicate(r): + return r[1] + + is_success, rows = execute_rcon_sql(server, build_sql, paginate_predicate, + [from_int_bool, int, str, int, str, int, str, int, float, float, float]) + + if not is_success: + return None + + characters = [] + + for row in rows: + characters.append({ + 'is_online': row[0], + 'conan_id': row[1], + 'name': row[2], + 'level': row[3], + 'steam_id': row[4], + 'last_online': row[5], + 'last_killed_by': row[6], + 'clan_id': row[7], + 'x': row[8], + 'y': row[9], + 'z': row[10]}) + + return characters + +def get_clans(server): + is_success, rows = execute_rcon_sql(server, ''' + SELECT guildId, name, owner, messageOfTheDay FROM guilds + ''', [int, unicode, int, unicode]) + + if not is_success: + return None + + guilds = [] + + for row in rows: + guilds.append({ + 'id': row[0], + 'name': row[1], + 'owner_id': row[2], + 'motd': row[3] + }) + + return guilds + +def sync_server_rcon(server_id): + server = (Server.objects + .filter(id=server_id) + .first()) + + is_valid = ( + server.sync_rcon and + server is not None and + server.rcon_host is not None and + server.rcon_password is not None and + server.rcon_port is not None) + + if not is_valid: + return + + characters = get_characters(server) + # clans = get_clans(server) + + # if clans is not None: + # for character in characters: + # print character + + # if clans is not None: + # for clan in clans: + # print clan + + return + + is_valid_sync = ( + clans is not None and + characters is not None) + + if not is_valid_sync: + return + + sync_data = ServerSyncData.objects.create(server=server, data=json.dumps({ + 'version': 'api', + 'characters': characters, + 'clans': clans + })) + + from api.tasks import sync_server_data_task + sync_server_data_task(sync_data.id, {}) diff --git a/api/tasks.py b/api/tasks.py index 0152f37..85c3b2d 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -1,7 +1,5 @@ -from .sync import sync_server_data, sync_ginfo -from api.models import CharacterHistory -from datetime import timedelta -from django.utils import timezone +from .sync import sync_server_data, sync_ginfo, sync_server_rcon +from api.models import Server from serverthrallapi.celery import app @@ -16,6 +14,15 @@ def sync_server_data_task(sync_data_id, request_get_params): @app.task() -def delete_old_history_task(): - history_threshold = timezone.now() - timedelta(days=5) - CharacterHistory.objects.filter(created__lt=history_threshold).delete() +def sync_server_rcon_task(server_id): + sync_server_rcon(server_id) + + +@app.task() +def sync_all_rcon_servers_task(): + server_ids = (Server.objects + .filter(rcon_host__isnull=False) + .values_list('id', flat=True)) + + for server_id in server_ids: + sync_server_rcon_task(server_id) diff --git a/api/views/server.py b/api/views/server.py index a878170..d48e046 100644 --- a/api/views/server.py +++ b/api/views/server.py @@ -3,6 +3,7 @@ from api.serializers import ServerSerializer, ServerAdminSerializer from .base import BaseView +import json class ServerView(BaseView): @@ -20,16 +21,28 @@ def post(self, request, server_id): if 'private_secret' not in request.GET: return HttpResponse('missing required param private_secret') - data = request.GET server = self.get_server_private(request, server_id) if server is None: return HttpResponse('server does not exist', status=404) - if 'name' in data: - server.name = data['name'] + data = json.loads(request.body) + + if 'sync_rcon' in data: + server.sync_rcon = data['sync_rcon'] + if 'rcon_host' in data: + server.rcon_host = data['rcon_host'] + server.ip_address = data['rcon_host'] + if 'rcon_port' in data: + server.rcon_port = data['rcon_port'] + if 'rcon_password' in data: + server.rcon_password = data['rcon_password'] + + # TODO: If sync_rcon is true, and credentials are different + # check RCON credentials work before creating or editing the server server.save() + server.refresh_from_db() serialized = ServerAdminSerializer(server).data return JsonResponse(serialized, status=200) diff --git a/api/views/servers.py b/api/views/servers.py index 35a0d91..0a11932 100644 --- a/api/views/servers.py +++ b/api/views/servers.py @@ -6,6 +6,7 @@ from api.serializers import ServerAdminSerializer, ServerSerializer from .base import BaseView +import json class ServersView(BaseView): @@ -18,6 +19,19 @@ def get(self, request): def post(self, request): server = Server() server.private_secret = uuid1() + + data = json.loads(request.body) + + if 'sync_rcon' in data: + server.sync_rcon = data['sync_rcon'] + if 'rcon_host' in data: + server.rcon_host = data['rcon_host'] + server.ip_address = data['rcon_host'] + if 'rcon_port' in data: + server.rcon_port = data['rcon_port'] + if 'rcon_password' in data: + server.rcon_password = data['rcon_password'] + server.save() server = self.get_server_public(request, server.id) diff --git a/requirements.txt b/requirements.txt index ac1b583..2ccc9c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ psycopg2==2.6.2 pytz==2017.2 requests==2.18.1 serpy==0.1.1 +python-valve==0.2.1 diff --git a/serverthrallapi/settings.py b/serverthrallapi/settings.py index 095bd87..2657c7d 100644 --- a/serverthrallapi/settings.py +++ b/serverthrallapi/settings.py @@ -1,6 +1,6 @@ import os -import json import dj_database_url +from kombu import Exchange, Queue ENVIRONMENT = os.environ.get('ENVIRONMENT', 'DEVELOPMENT') @@ -91,8 +91,29 @@ # Celery CELERY_TASK_ALWAYS_EAGER = DEBUG +CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + CELERY_BROKER_URL = os.environ.get('RABBITMQ_BIGWIG_URL') +CELERY_TASK_DEFAULT_QUEUE = 'default' +CELERY_TASK_DEFAULT_EXCHANGE = 'default' +CELERY_TASK_DEFAULT_ROUTING_KEY = 'default' + +CELLERY_QUEUES = ( + Queue('default', Exchange('default'), routing_key='default'), + Queue('sync_rest', Exchange('default'), routing_key='sync_rest'), + Queue('sync_rcon', Exchange('default'), routing_key='sync_rcon'), +) + +CELERY_BEAT_SCHEDULE = { + 'sync_all_rcon_servers': { + 'task': 'api.tasks.sync_all_rcon_servers_task', + 'schedule': 2 * 60, + 'args': [], + 'options': {'queue': 'sync_rcon'} + } +} ST_ENABLE_HISTORY = os.environ.get('ST_ENABLE_HISTORY', 'false').lower().startswith('true') diff --git a/startworker.sh b/startworker.sh index f9a8540..d91db82 100644 --- a/startworker.sh +++ b/startworker.sh @@ -7,3 +7,4 @@ newrelic-admin run-program \ --app=serverthrallapi.celery_app \ --beat \ --scheduler django + --queues default,sync_rest,sync_rcon