From 0189e1f3cefd92d962f1fea64e11f7ad7ed42589 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 28 May 2021 17:35:42 +0200 Subject: [PATCH 01/38] Adding a few features --- openstack_lease_it/config/config.ini | 6 +- .../lease_it/backend/OpenstackConnection.py | 21 +++++- .../lease_it/backend/TestConnection.py | 45 +++++++++--- .../lease_it/backend/__init__.py | 6 +- openstack_lease_it/lease_it/client/run.py | 9 ++- .../lease_it/datastore/ModelAccess.py | 4 +- .../lease_it/datastore/__init__.py | 2 +- .../lease_it/notification/MailNotification.py | 69 ++++++++++++++----- openstack_lease_it/lease_it/tests.py | 61 ++++++++++++++++ openstack_lease_it/lease_it/views.py | 1 + .../openstack_lease_it/config.py | 44 +++++++++--- .../openstack_lease_it/settings.py | 5 ++ requirements.txt | 12 +++- 13 files changed, 237 insertions(+), 48 deletions(-) diff --git a/openstack_lease_it/config/config.ini b/openstack_lease_it/config/config.ini index 6077c12..6f2ab1f 100644 --- a/openstack_lease_it/config/config.ini +++ b/openstack_lease_it/config/config.ini @@ -29,4 +29,8 @@ password = secret email_header = admin@example.com subject=Cloud@VD notification link=https://lease-it.lal.in2p3.fr -default_domain=default.example.com \ No newline at end of file +default_domain=default.example.com + +[special] +exclude_projects = ['LAL', 'project-01'] +excluded_users_id = [2] \ No newline at end of file diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 9934a8a..e970a15 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -2,6 +2,7 @@ """ This module manage interaction between application and OpenStack cloud infrastructure + """ import math @@ -288,8 +289,20 @@ def lease_instance(request, instance_id): not request.user.is_superuser: raise PermissionDenied(request.user.id, instance_id) InstancesAccess.lease(data_instances[instance_id]) + InstancesAccess.heartbeat(data_instances[instance_id]) return data_instances[instance_id] + def delete(self, instance_id): + """ + Deletes the VM with the id given in parameter + + :param instance_id: id of instance to delete + :return: void + """ + nova = nvclient.Client(NOVA_VERSION, session=self.session) + to_delete = nova.servers.list(search_opts={'all_tenants': 'true', 'id': instance_id}) + to_delete.delete() + def spy_instances(self): """ spy_instances is started by instance_spy module and check all running VM + notify user @@ -319,10 +332,12 @@ def spy_instances(self): "Instance: %s will be notify %s and %s", data_instances[instance]['id'], first_notification_date, - second_notification_date + second_notification_date, ) - # If lease as expire we tag it as delete - if lease_end < now: + # If lease has expired and it's not in the excluded projects, we tag it as delete + if lease_end < now \ + and data_instances[instance]['project_id'] not in GLOBAL_CONFIG['EXCLUDED_PROJECTS']\ + and str(data_instances[instance]['user_id']) not in GLOBAL_CONFIG['EXCLUDED_USERS_ID']: response['delete'].append(data_instances[instance]) elif first_notification_date == now or \ second_notification_date == now or \ diff --git a/openstack_lease_it/lease_it/backend/TestConnection.py b/openstack_lease_it/lease_it/backend/TestConnection.py index ebdfd47..f268729 100644 --- a/openstack_lease_it/lease_it/backend/TestConnection.py +++ b/openstack_lease_it/lease_it/backend/TestConnection.py @@ -13,6 +13,7 @@ - _users() - _projects() """ + from django.utils.dateparse import parse_datetime from django.core.cache import cache @@ -20,13 +21,16 @@ PROJECTS_CACHE_TIMEOUT, USERS_CACHE_TIMEOUT, FLAVOR_CACHE_TIMEOUT from lease_it.backend.Exceptions import PermissionDenied +from openstack_lease_it.settings import GLOBAL_CONFIG + class TestConnection(OpenstackConnection): """ - This class is only used for developement. This will - return false value formated as expected by views. + This class is only used for development. This will + return plausible values formatted as expected by views. """ def __init__(self): + self.session = None def _instances(self): @@ -36,7 +40,10 @@ def _instances(self): :return: dict() """ - response = cache.get('instances') + if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + response = False + else: + response = cache.get('instances') if not response: response = { 'instance-01': { @@ -44,18 +51,18 @@ def _instances(self): 'project_id': 'project-01', 'id': 'instance-01', 'name': 'instance-name-01', - 'created_at': parse_datetime('2017-04-29T17:40:26Z').date() + 'created_at': parse_datetime('2016-04-29T17:40:26Z').date() }, 'instance-02': { 'user_id': 1, - 'project_id': 'project-01', + 'project_id': 'project-02', 'id': 'instance-02', 'name': 'instance-name-02', 'created_at': parse_datetime('2017-10-29T17:40:26Z').date() }, 'instance-03': { 'user_id': 2, - 'project_id': 'project-01', + 'project_id': 'project-02', 'id': 'instance-03', 'name': 'instance-name-03', 'created_at': parse_datetime('2016-04-29T17:40:26Z').date() @@ -85,7 +92,11 @@ def _flavors(self): :return: dict() """ - response = cache.get('flavors') + + if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + response = False + else: + response = cache.get('flavors') if not response: response = { 'flavor.01': { @@ -138,7 +149,10 @@ def _domains(self): :return: dict() """ - response = cache.get('domains') + if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + response = False + else: + response = cache.get('domains') if not response: response = { 'domain-01': { @@ -160,7 +174,10 @@ def _users(self): :return: dict() """ - response = cache.get('users') + if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + response = False + else: + response = cache.get('users') if not response: response = { 1: { @@ -186,4 +203,12 @@ def _projects(self): :return: dict() """ - return dict() \ No newline at end of file + return dict() + + def delete(self, instance_id): + """ + Notifies in the terminal the instance that tried to be deleted, as it can't because it's a test connection + + :return: void + """ + print("Deleted the instance " + instance_id + " from Openstack") diff --git a/openstack_lease_it/lease_it/backend/__init__.py b/openstack_lease_it/lease_it/backend/__init__.py index be8b218..30baa92 100644 --- a/openstack_lease_it/lease_it/backend/__init__.py +++ b/openstack_lease_it/lease_it/backend/__init__.py @@ -1,6 +1,6 @@ # pylint: skip-file # -*- coding: utf-8 -*- -from OpenstackConnection import * -from TestConnection import * -from Exceptions import * +from lease_it.backend.OpenstackConnection import * +from lease_it.backend.TestConnection import * +from lease_it.backend.Exceptions import * diff --git a/openstack_lease_it/lease_it/client/run.py b/openstack_lease_it/lease_it/client/run.py index 3adb54f..141110a 100644 --- a/openstack_lease_it/lease_it/client/run.py +++ b/openstack_lease_it/lease_it/client/run.py @@ -5,10 +5,14 @@ """ import os from collections import defaultdict -from lease_it.notification.MailNotification import MailNotification + os.environ['DJANGO_SETTINGS_MODULE'] = "openstack_lease_it.settings" +from lease_it.notification.MailNotification import MailNotification +from lease_it.datastore.ModelAccess import InstancesAccess + + from lease_it import backend # pylint: disable=wrong-import-position from openstack_lease_it.settings import GLOBAL_CONFIG # pylint: disable=wrong-import-position @@ -30,6 +34,9 @@ def instance_spy(): notification[notification_type] = defaultdict(list) for instance in instances[notification_type]: notification[notification_type][instance['user_id']].append(instance) + if notification_type == "delete": + InstancesAccess.delete(instance['id']) + BACKEND.delete(instance['id']) notify = MailNotification(users) notify.send(notification) diff --git a/openstack_lease_it/lease_it/datastore/ModelAccess.py b/openstack_lease_it/lease_it/datastore/ModelAccess.py index 88b618e..bb07bd2 100644 --- a/openstack_lease_it/lease_it/datastore/ModelAccess.py +++ b/openstack_lease_it/lease_it/datastore/ModelAccess.py @@ -2,6 +2,7 @@ """ ModelAccess module is a interface between Django model and view """ +import os from dateutil.relativedelta import relativedelta @@ -133,7 +134,8 @@ def delete(instance_id): """ try: model = Instances.objects.get(id=instance_id) # pylint: disable=no-member - if model.heartbeat_at + relativedelta(days=+HEARTBEAT_TIMEOUT) > timezone.now().date(): + if model.leased_at + relativedelta(days=+model.lease_duration) >\ + timezone.now().date(): raise StillRunning(model.id, model.heartbeat_at) model.delete() except ObjectDoesNotExist: diff --git a/openstack_lease_it/lease_it/datastore/__init__.py b/openstack_lease_it/lease_it/datastore/__init__.py index 946a895..217a58c 100644 --- a/openstack_lease_it/lease_it/datastore/__init__.py +++ b/openstack_lease_it/lease_it/datastore/__init__.py @@ -1,4 +1,4 @@ # pylint: skip-file # -*- coding: utf-8 -*- -from ModelAccess import * +from lease_it.datastore.ModelAccess import * diff --git a/openstack_lease_it/lease_it/notification/MailNotification.py b/openstack_lease_it/lease_it/notification/MailNotification.py index 5dc5c09..29a04ba 100644 --- a/openstack_lease_it/lease_it/notification/MailNotification.py +++ b/openstack_lease_it/lease_it/notification/MailNotification.py @@ -8,24 +8,33 @@ from email.mime.text import MIMEText from openstack_lease_it.settings import GLOBAL_CONFIG, EMAIL_REGEXP, LOGGER_NOTIFICATION + class MailNotification(object): # pylint: disable=too-few-public-methods """ A class to abstract e-mail notification """ def __init__(self, users): """ - Not yet implemented + Not implemented yet """ self.users = users - self.smtp = smtplib.SMTP_SSL(GLOBAL_CONFIG['NOTIFICATION_SMTP']) - self.smtp.login(GLOBAL_CONFIG['NOTIFICATION_USERNAME'], - GLOBAL_CONFIG['NOTIFICATION_PASSWORD']) - delete_content = open(GLOBAL_CONFIG['NOTIFICATION_DELETE_CONTENT'], 'r') - lease_content = open(GLOBAL_CONFIG['NOTIFICATION_LEASE_CONTENT'], 'r') - self.notification = { - 'delete': delete_content.read(), - 'notify': lease_content.read() - } + try: + self.smtp = smtplib.SMTP_SSL(GLOBAL_CONFIG['NOTIFICATION_SMTP']) + self.smtp.login(GLOBAL_CONFIG['NOTIFICATION_USERNAME'], + GLOBAL_CONFIG['NOTIFICATION_PASSWORD']) + delete_content = open(GLOBAL_CONFIG['NOTIFICATION_DELETE_CONTENT'], 'r') + lease_content = open(GLOBAL_CONFIG['NOTIFICATION_LEASE_CONTENT'], 'r') + self.notification = { + 'delete': delete_content.read(), + 'notify': lease_content.read() + } + except: + self.notification = { + 'delete': open("/dev/null").read(), + 'notify': open("/dev/null").read() + } + + pass @staticmethod def format_user_instances(user): @@ -54,9 +63,13 @@ def format_mail(self, user, notification_type, instances): LOGGER_NOTIFICATION.info("User %s as not be found", user) core_text = self.notification[notification_type] instances_text = self.format_user_instances(instances) - return core_text.format(username=user_name, + try: + return core_text.format(username=user_name, link=GLOBAL_CONFIG['NOTIFICATION_LINK'], instances=instances_text) + except KeyError: + LOGGER_NOTIFICATION.info("Notification link not found") + pass def send(self, notifications): """ @@ -68,13 +81,24 @@ def send(self, notifications): for notification in notifications: for user in notifications[notification]: mail_text = self.format_mail(user, notification, notifications[notification][user]) - mail = MIMEText(mail_text) - mail['Subject'] = GLOBAL_CONFIG['NOTIFICATION_SUBJECT'] - mail['From'] = GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER'] + try: + mail = MIMEText(mail_text) + except AttributeError: + LOGGER_NOTIFICATION.info("Mail text None type") + mail = dict() + try: + mail['Subject'] = GLOBAL_CONFIG['NOTIFICATION_SUBJECT'] + except KeyError: + LOGGER_NOTIFICATION.info("Email subject not found") + mail['Subject'] = "/" + try: + mail['From'] = GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER'] + except KeyError: + LOGGER_NOTIFICATION.info("Email header not found") + mail['From'] = "/" try: email = self.users[user]['email'] - if not re.match(EMAIL_REGEXP, email) and \ - GLOBAL_CONFIG['NOTIFICATION_DOMAIN'] != "": + if not re.match(EMAIL_REGEXP, email) and GLOBAL_CONFIG['NOTIFICATION_DOMAIN'] != "": LOGGER_NOTIFICATION.info("email %s not match a email format" " (name@domain.com). Add @%s", email, GLOBAL_CONFIG['NOTIFICATION_DOMAIN']) @@ -88,9 +112,16 @@ def send(self, notifications): email] except KeyError: LOGGER_NOTIFICATION.info("email field of %s as not be found", user) - mail['To'] = GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER'] - recipient = [GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER']] + try: + mail['To'] = GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER'] + recipient = [GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER']] + except KeyError: + LOGGER_NOTIFICATION.info("Email header not found") + recipient = "/" LOGGER_NOTIFICATION.info("Notification %s to %s", notification, ''.join(recipient)) - self.smtp.sendmail(GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER'], + try: + self.smtp.sendmail(GLOBAL_CONFIG['NOTIFICATION_EMAIL_HEADER'], recipient, mail.as_string()) + except AttributeError: + LOGGER_NOTIFICATION.info("Smtp not found") diff --git a/openstack_lease_it/lease_it/tests.py b/openstack_lease_it/lease_it/tests.py index 98e7d3d..404225d 100644 --- a/openstack_lease_it/lease_it/tests.py +++ b/openstack_lease_it/lease_it/tests.py @@ -1,4 +1,65 @@ +""" +This file tests the instances model with some plausible values. +The database created for the test is destroyed right after. +""" + # pylint: skip-file +import django.core.exceptions from django.test import TestCase +from lease_it.models import Instances +from lease_it.client.run import instance_spy +from openstack_lease_it.settings import GLOBAL_CONFIG, load_config # Create your tests here. + + +class InstancesTest(TestCase): + def setUp(self): + """ + Filling the database with plausible values + + :return: void + """ + Instances.objects.create(id="instance-01", heartbeat_at="2021-08-28", leased_at="2021-05-28", lease_duration=90) + Instances.objects.create(id="instance-02", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) + Instances.objects.create(id="instance-03", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) + Instances.objects.create(id="instance-04", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) + Instances.objects.create(id="instance-05", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) + + def test_instance_spy(self): + """ + Testing the instance_spy function + + :return: void + """ + + instance_spy() + + # Only the instance with id "instance-02" should be deleted, as it's not excluded nor leased + self.assertIsInstance(Instances.objects.get(id="instance-01"),Instances) + self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", + Instances.objects.get, id="instance-02") + self.assertIsInstance(Instances.objects.get(id="instance-03"), Instances) + self.assertIsInstance(Instances.objects.get(id="instance-04"), Instances) + self.assertIsInstance(Instances.objects.get(id="instance-05"), Instances) + + load_config() + GLOBAL_CONFIG['EXCLUDED_USERS_ID'] = [] + instance_spy() + # Now instance_03 should be deleted, because it no longer benefit from the user exclusion + self.assertIsInstance(Instances.objects.get(id="instance-01"), Instances) + self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", + Instances.objects.get, id="instance-03") + self.assertIsInstance(Instances.objects.get(id="instance-04"), Instances) + self.assertIsInstance(Instances.objects.get(id="instance-05"), Instances) + + load_config() + GLOBAL_CONFIG['EXCLUDED_PROJECTS'] = ['LAL'] + GLOBAL_CONFIG['EXCLUDED_USERS_ID'] = [] + instance_spy() + # Now that all the exclusions are disabled, instance 04 and 05 should be deleted + self.assertIsInstance(Instances.objects.get(id="instance-01"), Instances) + self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", + Instances.objects.get, id="instance-04") + self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", + Instances.objects.get, id="instance-05") diff --git a/openstack_lease_it/lease_it/views.py b/openstack_lease_it/lease_it/views.py index 61e46cf..8e43c0f 100644 --- a/openstack_lease_it/lease_it/views.py +++ b/openstack_lease_it/lease_it/views.py @@ -164,6 +164,7 @@ def database(request, instance_id): # pylint: disable=unused-argument } try: InstancesAccess.delete(instance_id) + BACKEND.delete(instance_id) except StillRunning as error: response = { 'status': 'failed', diff --git a/openstack_lease_it/openstack_lease_it/config.py b/openstack_lease_it/openstack_lease_it/config.py index baeacdc..8b7acd3 100644 --- a/openstack_lease_it/openstack_lease_it/config.py +++ b/openstack_lease_it/openstack_lease_it/config.py @@ -7,7 +7,7 @@ This module also provide **GLOBAL_CONFIG** variable used to share configuration across module / django apps. """ -import ConfigParser +import configparser import os BASE_CONFIG_DIR = '/etc/openstack-lease-it' @@ -53,7 +53,12 @@ 'NOTIFICATION_DEBUG': 'False', 'NOTIFICATION_DOMAIN': '', 'NOTIFICATION_DELETE_CONTENT': BASE_CONFIG_DIR + '/delete-notification.txt', - 'NOTIFICATION_LEASE_CONTENT': BASE_CONFIG_DIR + '/lease-notification.txt' + 'NOTIFICATION_LEASE_CONTENT': BASE_CONFIG_DIR + '/lease-notification.txt', + + # special parameter + 'EXCLUDED_PROJECTS': [], + 'EXCLUDED_USERS_ID': [], + 'RESET_CACHE_INSTANCES': False } """ We use the global variable GLOBAL_CONFIG to share openstack-lease-it configuration to all user. Some @@ -147,12 +152,24 @@ """ +SPECIAL_OPTIONS = { + 'EXCLUDED_PROJECTS': 'excluded_projects', + 'EXCLUDED_USERS_ID': 'excluded_users_id', + 'RESET_CACHE_INSTANCES': 'reset_cache_instances' +} +""" + - **EXCLUDED_PROJECTS**: List of the projects excluded by the delete call + - **EXCLUDED_USERS_ID**: List of the users' id excluded by the delete call + - **RESET_CACHE_INSTANCES**: Enable / Disable reset of the cache when loading instances +""" + SECTIONS = { 'django': DJANGO_OPTIONS, 'openstack': OPENSTACK_OPTIONS, 'memcached': MEMCACHED_OPTIONS, 'plugins': PLUGINS_OPTIONS, - 'notification': NOTIFICATION_OPTIONS + 'notification': NOTIFICATION_OPTIONS, + 'special': SPECIAL_OPTIONS } """ @@ -161,6 +178,7 @@ - **memcached**: section [memcached] - **plugins**: section [plugins] - **notification**: section [notification] + - **special**: section [special] """ @@ -177,11 +195,21 @@ def load_config_option(config, section): options = SECTIONS[section] for option in options: try: - GLOBAL_CONFIG[option] = config.get(section, - options[option]) - except ConfigParser.NoSectionError: + config_to_add = config.get(section, options[option]) + if config_to_add[0] != "[": + GLOBAL_CONFIG[option] = config_to_add + else: + formalized_config_to_add = config_to_add.split(',') + len_config_to_add = len(formalized_config_to_add) + formalized_config_to_add = [formalized_config_to_add[i].strip() + for i in range(len_config_to_add)] + formalized_config_to_add[0] = formalized_config_to_add[0][1:] + formalized_config_to_add[len_config_to_add - 1] = formalized_config_to_add[len_config_to_add-1][:-1] + GLOBAL_CONFIG[option] = formalized_config_to_add + + except configparser.NoSectionError: pass - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass @@ -191,7 +219,7 @@ def load_config(): :return: void """ - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() for config_file in CONFIG_FILES: config.read(config_file) diff --git a/openstack_lease_it/openstack_lease_it/settings.py b/openstack_lease_it/openstack_lease_it/settings.py index 533191e..11a6c95 100644 --- a/openstack_lease_it/openstack_lease_it/settings.py +++ b/openstack_lease_it/openstack_lease_it/settings.py @@ -14,9 +14,11 @@ import os import ast import logging +import django from openstack_lease_it.config import GLOBAL_CONFIG, load_config + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Load configuration @@ -25,6 +27,7 @@ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = GLOBAL_CONFIG['DJANGO_SECRET_KEY'] + # SECURITY WARNING: don't run with debug turned on in production! DEBUG = ast.literal_eval(GLOBAL_CONFIG['DJANGO_DEBUG']) @@ -211,3 +214,5 @@ LOGGER = logging.getLogger('main') LOGGER_NOTIFICATION = logging.getLogger('notification') LOGGER_INSTANCES = logging.getLogger('instances') + +django.setup() diff --git a/requirements.txt b/requirements.txt index 4be24e9..abe94d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,35 @@ alabaster==0.7.10 appdirs==1.4.3 Babel==2.4.0 +certifi==2020.12.5 +chardet==4.0.0 +configparser==5.0.2 debtcollector==1.13.0 Django==1.8 django-openstack-auth==3.1.1 docutils==0.14 funcsigs==1.0.2 +idna==2.10 imagesize==0.7.1 iso8601==0.1.11 Jinja2==2.9.6 keystoneauth1==2.19.0 -MarkupSafe==1.0 +MarkupSafe==2.0.1 monotonic==1.3 +msgpack==1.0.2 msgpack-python==0.4.8 netaddr==0.7.19 netifaces==0.10.5 +os-service-types==1.7.0 oslo.config==3.24.0 +oslo.context==3.2.0 oslo.i18n==3.15.0 oslo.policy==1.22.0 oslo.serialization==2.18.0 oslo.utils==3.25.0 packaging==16.8 pbr==2.0.0 +pkg-resources==0.0.0 positional==1.1.1 prettytable==0.7.2 Pygments==2.2.0 @@ -38,7 +46,9 @@ simplejson==3.10.0 six==1.10.0 snowballstemmer==1.2.1 Sphinx==1.6.5 +sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-websupport==1.0.1 stevedore==1.21.0 typing==3.6.2 +urllib3==1.26.4 wrapt==1.10.10 From ad36ee7a91f4d06e47125b21627030f04bdd9b26 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Wed, 9 Jun 2021 15:40:17 +0200 Subject: [PATCH 02/38] Features and fixes --- openstack_lease_it/config/config.ini | 9 +- .../lease_it/backend/OpenstackConnection.py | 75 ++-- .../lease_it/backend/TestConnection.py | 23 +- .../lease_it/datastore/ModelAccess.py | 4 +- .../lease_it/fixtures/load_test_models.py | 340 ++++++++++++++++++ .../fixtures/test_models explanations | 35 ++ .../lease_it/fixtures/test_models.json | 1 + .../lease_it/fixtures/test_models_setup.json | 53 +++ .../lease_it/static/js/database.js | 6 +- openstack_lease_it/lease_it/tests.py | 48 ++- .../openstack_lease_it/config.py | 59 +-- 11 files changed, 567 insertions(+), 86 deletions(-) create mode 100644 openstack_lease_it/lease_it/fixtures/load_test_models.py create mode 100644 openstack_lease_it/lease_it/fixtures/test_models explanations create mode 100644 openstack_lease_it/lease_it/fixtures/test_models.json create mode 100644 openstack_lease_it/lease_it/fixtures/test_models_setup.json diff --git a/openstack_lease_it/config/config.ini b/openstack_lease_it/config/config.ini index 6f2ab1f..ead0ad6 100644 --- a/openstack_lease_it/config/config.ini +++ b/openstack_lease_it/config/config.ini @@ -31,6 +31,9 @@ subject=Cloud@VD notification link=https://lease-it.lal.in2p3.fr default_domain=default.example.com -[special] -exclude_projects = ['LAL', 'project-01'] -excluded_users_id = [2] \ No newline at end of file +[project-01] +exclude = True + +[user_id_2] +exclude = True +duration = 120 \ No newline at end of file diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index e970a15..97befc8 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -21,6 +21,8 @@ from lease_it.datastore import InstancesAccess, LEASE_DURATION from lease_it.backend.Exceptions import PermissionDenied +from lease_it.models import Instances + # Define nova client version as a constant NOVA_VERSION = 2 @@ -306,41 +308,60 @@ def delete(self, instance_id): def spy_instances(self): """ spy_instances is started by instance_spy module and check all running VM + notify user - if a VM is close to its lease time + if a VM is close to its lease time + update lease duration according to the lease duration settings :return: dict() """ now = date.today() data_instances = InstancesAccess.show(self._instances()) + users = self._users() response = { 'delete': list(), # List of instance we must delete 'notify': list() # List of instance we must notify user to renew the lease } for instance in data_instances: - # We mark the VM as showed - InstancesAccess.heartbeat(data_instances[instance]) - leased_at = data_instances[instance]['leased_at'] - lease_end = data_instances[instance]['lease_end'] - # If it's a new instance, we put lease value as today - # it's not necessary to lease on model as heartbeat should have create and - # lease the virtual machine - if leased_at is None: - lease_end = now + relativedelta(days=+LEASE_DURATION) - first_notification_date = lease_end - relativedelta(days=+LEASE_DURATION/3) - second_notification_date = lease_end - relativedelta(days=+LEASE_DURATION/6) - LOGGER_INSTANCES.info( - "Instance: %s will be notify %s and %s", - data_instances[instance]['id'], - first_notification_date, - second_notification_date, - ) - # If lease has expired and it's not in the excluded projects, we tag it as delete - if lease_end < now \ - and data_instances[instance]['project_id'] not in GLOBAL_CONFIG['EXCLUDED_PROJECTS']\ - and str(data_instances[instance]['user_id']) not in GLOBAL_CONFIG['EXCLUDED_USERS_ID']: - response['delete'].append(data_instances[instance]) - elif first_notification_date == now or \ - second_notification_date == now or \ - lease_end < now - relativedelta(days=-6): - response['notify'].append(data_instances[instance]) + user_name = users[data_instances[instance]['user_id']]['name'] + if data_instances[instance]['project_id'] not in GLOBAL_CONFIG["EXCLUDE"] and \ + user_name not in GLOBAL_CONFIG["EXCLUDE"]: + # We mark the VM as shown + InstancesAccess.heartbeat(data_instances[instance]) + leased_at = data_instances[instance]['leased_at'] + lease_end = data_instances[instance]['lease_end'] + lease_duration = LEASE_DURATION + # If the instance benefits from a special lease (from its user_name, instance_id or project_id), + # we update the lease_duration (used to determine whether to delete it or not) + # and the instance's lease duration + if user_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][user_name] + model = Instances.objects.get(id=data_instances[instance]['id']) + model.lease_duration = lease_duration + elif data_instances[instance]['project_id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['project_id']] + model = Instances.objects.get(id=data_instances[instance]['id']) + model.lease_duration = lease_duration + elif data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] + model = Instances.objects.get(id=data_instances[instance]['id']) + model.lease_duration = lease_duration + + # If it's a new instance, we put lease value as today + # it's not necessary to lease on model as heartbeat should have create and + # lease the virtual machine + if leased_at is None: + lease_end = now + relativedelta(days=+lease_duration) + first_notification_date = lease_end - relativedelta(days=+lease_duration/3) + second_notification_date = lease_end - relativedelta(days=+lease_duration/6) + LOGGER_INSTANCES.info( + "Instance: %s will be notify %s and %s", + data_instances[instance]['id'], + first_notification_date, + second_notification_date, + ) + # If lease has expired and it's not in the excluded projects, we tag it as delete + if lease_end < now: + response['delete'].append(data_instances[instance]) + elif first_notification_date == now or \ + second_notification_date == now or \ + lease_end < now - relativedelta(days=-6): + response['notify'].append(data_instances[instance]) return response diff --git a/openstack_lease_it/lease_it/backend/TestConnection.py b/openstack_lease_it/lease_it/backend/TestConnection.py index f268729..34d55a8 100644 --- a/openstack_lease_it/lease_it/backend/TestConnection.py +++ b/openstack_lease_it/lease_it/backend/TestConnection.py @@ -21,6 +21,7 @@ PROJECTS_CACHE_TIMEOUT, USERS_CACHE_TIMEOUT, FLAVOR_CACHE_TIMEOUT from lease_it.backend.Exceptions import PermissionDenied + from openstack_lease_it.settings import GLOBAL_CONFIG @@ -30,7 +31,6 @@ class TestConnection(OpenstackConnection): return plausible values formatted as expected by views. """ def __init__(self): - self.session = None def _instances(self): @@ -40,9 +40,8 @@ def _instances(self): :return: dict() """ - if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: - response = False - else: + response = None + if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: response = cache.get('instances') if not response: response = { @@ -92,10 +91,8 @@ def _flavors(self): :return: dict() """ - - if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: - response = False - else: + response = None + if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: response = cache.get('flavors') if not response: response = { @@ -149,9 +146,8 @@ def _domains(self): :return: dict() """ - if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: - response = False - else: + response = None + if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: response = cache.get('domains') if not response: response = { @@ -174,9 +170,8 @@ def _users(self): :return: dict() """ - if GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: - response = False - else: + response = None + if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: response = cache.get('users') if not response: response = { diff --git a/openstack_lease_it/lease_it/datastore/ModelAccess.py b/openstack_lease_it/lease_it/datastore/ModelAccess.py index bb07bd2..1c03702 100644 --- a/openstack_lease_it/lease_it/datastore/ModelAccess.py +++ b/openstack_lease_it/lease_it/datastore/ModelAccess.py @@ -14,6 +14,7 @@ from lease_it.datastore.Exceptions import StillRunning from openstack_lease_it.settings import LOGGER_INSTANCES +from openstack_lease_it.settings import GLOBAL_CONFIG # Default lease duration in day LEASE_DURATION = 90 @@ -24,7 +25,7 @@ class InstancesAccess(object): # pylint: disable=too-few-public-methods """ ModelAccess is a class will abstract model access for application. It - will get / save / ... informations in a format expected by views + will get / save / ... information in a format expected by views """ @staticmethod def get(instance): @@ -43,6 +44,7 @@ def get(instance): model.leased_at = timezone.now() model.heartbeat_at = timezone.now() model.lease_duration = LEASE_DURATION + # The different lease durations will be adapted once spy_instance has run once return model @staticmethod diff --git a/openstack_lease_it/lease_it/fixtures/load_test_models.py b/openstack_lease_it/lease_it/fixtures/load_test_models.py new file mode 100644 index 0000000..0da66d1 --- /dev/null +++ b/openstack_lease_it/lease_it/fixtures/load_test_models.py @@ -0,0 +1,340 @@ +""" +Executing this file updates the test_models.json with the data provided by test_models_setup.json. +This simplifies the instances' management during the tests. +""" +import json + +# Basic database dictionary +test_models_base = [ + { + "model": "contenttypes.contenttype", + "fields": { + "app_label": "openstack_auth", + "model": "user"}, + "pk": 1 + }, + { + "model": "contenttypes.contenttype", + "fields": { + "app_label": "admin", "model": "logentry" + }, + "pk": 2 + }, + { + "model": "contenttypes.contenttype", + "fields": { + "app_label": "auth", "model": "permission" + }, + "pk": 3 + }, + { + "model": "contenttypes.contenttype", + "fields": { + "app_label": "auth", "model": "group" + }, + "pk": 4 + }, + { + "model": "contenttypes.contenttype", + "fields": { + "app_label": "auth", "model": "user" + }, + "pk": 5 + }, + { + "model": "contenttypes.contenttype", + "fields": + { + "app_label": "contenttypes", + "model": "contenttype" + }, + "pk": 6 + }, + { + "model": "contenttypes.contenttype", + "fields": + { + "app_label": "sessions", "model": "session" + }, + "pk": 7 + }, + { + "model": "contenttypes.contenttype", + "fields": + { + "app_label": "lease_it", + "model": "instances" + }, + "pk": 8 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add user", + "content_type": 1, + "codename": "add_user" + }, + "pk": 1 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change user", + "content_type": 1, + "codename": "change_user" + }, + "pk": 2 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete user", + "content_type": 1, + "codename": "delete_user" + }, + "pk": 3 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add log entry", + "content_type": 2, + "codename": "add_logentry" + }, + "pk": 4 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change log entry", + "content_type": 2, + "codename": "change_logentry" + }, + "pk": 5 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete log entry", + "content_type": 2, + "codename": "delete_logentry" + }, + "pk": 6 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add permission", + "content_type": 3, + "codename": "add_permission" + }, + "pk": 7 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change permission", + "content_type": 3, + "codename": "change_permission" + }, + "pk": 8 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete permission", + "content_type": 3, + "codename": "delete_permission" + }, + "pk": 9 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add group", + "content_type": 4, + "codename": "add_group" + }, + "pk": 10 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change group", + "content_type": 4, + "codename": "change_group" + }, + "pk": 11 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete group", + "content_type": 4, + "codename": "delete_group" + }, + "pk": 12 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add user", + "content_type": 5, + "codename": "add_user" + }, + "pk": 13 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change user", + "content_type": 5, + "codename": "change_user" + }, + "pk": 14 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete user", + "content_type": 5, + "codename": "delete_user" + }, + "pk": 15 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add content type", + "content_type": 6, + "codename": "add_contenttype" + }, + "pk": 16 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change content type", + "content_type": 6, + "codename": "change_contenttype" + }, + "pk": 17 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete content type", + "content_type": 6, + "codename": "delete_contenttype" + }, + "pk": 18 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add session", + "content_type": 7, + "codename": "add_session" + }, + "pk": 19 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change session", + "content_type": 7, + "codename": "change_session" + }, + "pk": 20 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete session", + "content_type": 7, + "codename": "delete_session" + }, + "pk": 21 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can add instances", + "content_type": 8, + "codename": "add_instances" + }, + "pk": 22 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can change instances", + "content_type": 8, + "codename": "change_instances" + }, + "pk": 23 + }, + { + "model": "auth.permission", + "fields": + { + "name": "Can delete instances", + "content_type": 8, + "codename": "delete_instances" + }, + "pk": 24 + }, + { + "model": "auth.user", + "fields": + { + "password": "pbkdf2_sha256$20000$f0kZKtEf9D78$XvUaXte8o9kP5Nu5stefJUXCWltiQRFl7iZD1mB5onI=", + "last_login": "2021-05-26T11:28:57.063Z", + "is_superuser": True, + "username": "yann", + "first_name": "", + "last_name": "", + "email": "test@test.fr", + "is_staff": True, + "is_active": True, + "date_joined": "2021-05-25T10:37:21.225Z", + "groups": [], + "user_permissions": [] + }, + "pk": 1 + } +] + +# Adding the instances to the basic database dictionary +test_models_setup_file = open("./lease_it/fixtures/test_models_setup.json") +test_models = open("./lease_it/fixtures/test_models.json", "w") +test_models_setup = json.loads(test_models_setup_file.read()) + +for test_model in test_models_setup: + test_models_base.append(test_model) + +json.dump(test_models_base, test_models) + diff --git a/openstack_lease_it/lease_it/fixtures/test_models explanations b/openstack_lease_it/lease_it/fixtures/test_models explanations new file mode 100644 index 0000000..dfb83ff --- /dev/null +++ b/openstack_lease_it/lease_it/fixtures/test_models explanations @@ -0,0 +1,35 @@ + +File used to load a database locally corresponding to the data from TestConnection + +To load this database, use the django command : + manage.py loaddata test_models.json + +To manage instances loaded parameters, set them in test_models_setup.json, +then execute the Python file load_test_models.py + +The instances generated originally have the following attributes : + + instance-01: + heartbeat_at: 2021-05-24 + leased_at: 2021-05-24 + lease_duration: 90 + + instance-02: + heartbeat_at: 2020-08-24 + leased_at: 2020-05-24 + lease_duration: 90 + + instance-03: + heartbeat_at: 2020-08-24 + leased_at: 2020-05-24 + lease_duration: 120 + + instance-04: + heartbeat_at: 2020-08-24 + leased_at: 2020-05-24 + lease_duration: 120 + + instance-05: + heartbeat_at: 2020-08-24 + leased_at: 2020-05-24 + lease_duration: 90 diff --git a/openstack_lease_it/lease_it/fixtures/test_models.json b/openstack_lease_it/lease_it/fixtures/test_models.json new file mode 100644 index 0000000..c940c6a --- /dev/null +++ b/openstack_lease_it/lease_it/fixtures/test_models.json @@ -0,0 +1 @@ +[{"model": "contenttypes.contenttype", "fields": {"app_label": "openstack_auth", "model": "user"}, "pk": 1}, {"model": "contenttypes.contenttype", "fields": {"app_label": "admin", "model": "logentry"}, "pk": 2}, {"model": "contenttypes.contenttype", "fields": {"app_label": "auth", "model": "permission"}, "pk": 3}, {"model": "contenttypes.contenttype", "fields": {"app_label": "auth", "model": "group"}, "pk": 4}, {"model": "contenttypes.contenttype", "fields": {"app_label": "auth", "model": "user"}, "pk": 5}, {"model": "contenttypes.contenttype", "fields": {"app_label": "contenttypes", "model": "contenttype"}, "pk": 6}, {"model": "contenttypes.contenttype", "fields": {"app_label": "sessions", "model": "session"}, "pk": 7}, {"model": "contenttypes.contenttype", "fields": {"app_label": "lease_it", "model": "instances"}, "pk": 8}, {"model": "auth.permission", "fields": {"name": "Can add user", "content_type": 1, "codename": "add_user"}, "pk": 1}, {"model": "auth.permission", "fields": {"name": "Can change user", "content_type": 1, "codename": "change_user"}, "pk": 2}, {"model": "auth.permission", "fields": {"name": "Can delete user", "content_type": 1, "codename": "delete_user"}, "pk": 3}, {"model": "auth.permission", "fields": {"name": "Can add log entry", "content_type": 2, "codename": "add_logentry"}, "pk": 4}, {"model": "auth.permission", "fields": {"name": "Can change log entry", "content_type": 2, "codename": "change_logentry"}, "pk": 5}, {"model": "auth.permission", "fields": {"name": "Can delete log entry", "content_type": 2, "codename": "delete_logentry"}, "pk": 6}, {"model": "auth.permission", "fields": {"name": "Can add permission", "content_type": 3, "codename": "add_permission"}, "pk": 7}, {"model": "auth.permission", "fields": {"name": "Can change permission", "content_type": 3, "codename": "change_permission"}, "pk": 8}, {"model": "auth.permission", "fields": {"name": "Can delete permission", "content_type": 3, "codename": "delete_permission"}, "pk": 9}, {"model": "auth.permission", "fields": {"name": "Can add group", "content_type": 4, "codename": "add_group"}, "pk": 10}, {"model": "auth.permission", "fields": {"name": "Can change group", "content_type": 4, "codename": "change_group"}, "pk": 11}, {"model": "auth.permission", "fields": {"name": "Can delete group", "content_type": 4, "codename": "delete_group"}, "pk": 12}, {"model": "auth.permission", "fields": {"name": "Can add user", "content_type": 5, "codename": "add_user"}, "pk": 13}, {"model": "auth.permission", "fields": {"name": "Can change user", "content_type": 5, "codename": "change_user"}, "pk": 14}, {"model": "auth.permission", "fields": {"name": "Can delete user", "content_type": 5, "codename": "delete_user"}, "pk": 15}, {"model": "auth.permission", "fields": {"name": "Can add content type", "content_type": 6, "codename": "add_contenttype"}, "pk": 16}, {"model": "auth.permission", "fields": {"name": "Can change content type", "content_type": 6, "codename": "change_contenttype"}, "pk": 17}, {"model": "auth.permission", "fields": {"name": "Can delete content type", "content_type": 6, "codename": "delete_contenttype"}, "pk": 18}, {"model": "auth.permission", "fields": {"name": "Can add session", "content_type": 7, "codename": "add_session"}, "pk": 19}, {"model": "auth.permission", "fields": {"name": "Can change session", "content_type": 7, "codename": "change_session"}, "pk": 20}, {"model": "auth.permission", "fields": {"name": "Can delete session", "content_type": 7, "codename": "delete_session"}, "pk": 21}, {"model": "auth.permission", "fields": {"name": "Can add instances", "content_type": 8, "codename": "add_instances"}, "pk": 22}, {"model": "auth.permission", "fields": {"name": "Can change instances", "content_type": 8, "codename": "change_instances"}, "pk": 23}, {"model": "auth.permission", "fields": {"name": "Can delete instances", "content_type": 8, "codename": "delete_instances"}, "pk": 24}, {"model": "auth.user", "fields": {"password": "pbkdf2_sha256$20000$f0kZKtEf9D78$XvUaXte8o9kP5Nu5stefJUXCWltiQRFl7iZD1mB5onI=", "last_login": "2021-05-26T11:28:57.063Z", "is_superuser": true, "username": "yann", "first_name": "", "last_name": "", "email": "test@test.fr", "is_staff": true, "is_active": true, "date_joined": "2021-05-25T10:37:21.225Z", "groups": [], "user_permissions": []}, "pk": 1}, {"model": "lease_it.instances", "fields": {"heartbeat_at": "2021-05-24", "leased_at": "2021-05-24", "lease_duration": 120}, "pk": "instance-01"}, {"model": "lease_it.instances", "fields": {"heartbeat_at": "2020-08-24", "leased_at": "2020-05-24", "lease_duration": 90}, "pk": "instance-02"}, {"model": "lease_it.instances", "fields": {"heartbeat_at": "2020-08-24", "leased_at": "2020-05-24", "lease_duration": 120}, "pk": "instance-03"}, {"model": "lease_it.instances", "fields": {"heartbeat_at": "2020-08-24", "leased_at": "2020-05-24", "lease_duration": 120}, "pk": "instance-04"}, {"model": "lease_it.instances", "fields": {"heartbeat_at": "2021-08-24", "leased_at": "2021-05-24", "lease_duration": 90}, "pk": "instance-05"}] \ No newline at end of file diff --git a/openstack_lease_it/lease_it/fixtures/test_models_setup.json b/openstack_lease_it/lease_it/fixtures/test_models_setup.json new file mode 100644 index 0000000..e914aa8 --- /dev/null +++ b/openstack_lease_it/lease_it/fixtures/test_models_setup.json @@ -0,0 +1,53 @@ + +[ + { + "model": "lease_it.instances", + "fields": + { + "heartbeat_at": "2021-05-24", + "leased_at": "2021-05-24", + "lease_duration": 120 + }, + "pk": "instance-01" + }, + { + "model": "lease_it.instances", + "fields": + { + "heartbeat_at": "2020-08-24", + "leased_at": "2020-05-24", + "lease_duration": 90 + }, + "pk": "instance-02" + }, + { + "model": "lease_it.instances", + "fields": + { + "heartbeat_at": "2020-08-24", + "leased_at": "2020-05-24", + "lease_duration": 120 + }, + "pk": "instance-03" + }, + { + "model": "lease_it.instances", + "fields": + { + "heartbeat_at": "2020-08-24", + "leased_at": "2020-05-24", + "lease_duration": 120 + }, + "pk": "instance-04" + }, + { + "model": "lease_it.instances", + "fields": + { + "heartbeat_at": "2021-08-24", + "leased_at": "2021-05-24", + "lease_duration": 90 + }, + "pk": "instance-05" + } +] diff --git a/openstack_lease_it/lease_it/static/js/database.js b/openstack_lease_it/lease_it/static/js/database.js index 4155ca4..2472417 100644 --- a/openstack_lease_it/lease_it/static/js/database.js +++ b/openstack_lease_it/lease_it/static/js/database.js @@ -26,9 +26,9 @@ function buildDatabaseView(div_name) { targets: [0], render: function ( data, type, row ) { var now = new Date(); - var heartbeat_date = new Date(row.heartbeat_at); - // If a VM as not been seen since 1 week, we can delete it - if (heartbeat_date < now - HEARTBEAT_TIMEOUT) { + var lease_end = new Date(row.lease_end); + // If a VM has a lease_end date before today, we can delete it + if (lease_end < now) { return buildDatabaseRowMenu(data) + formatText(data, MAX_STRING_LENGTH); } else { diff --git a/openstack_lease_it/lease_it/tests.py b/openstack_lease_it/lease_it/tests.py index 404225d..1c5b41f 100644 --- a/openstack_lease_it/lease_it/tests.py +++ b/openstack_lease_it/lease_it/tests.py @@ -1,17 +1,24 @@ """ This file tests the instances model with some plausible values. The database created for the test is destroyed right after. +To test further, test_models creates a database that doesn't get destroyed. +To proceed to the tests, run the django command : + manage.py test """ # pylint: skip-file import django.core.exceptions from django.test import TestCase +from datetime import date +from dateutil.relativedelta import relativedelta from lease_it.models import Instances from lease_it.client.run import instance_spy from openstack_lease_it.settings import GLOBAL_CONFIG, load_config # Create your tests here. +now = date.today() + class InstancesTest(TestCase): def setUp(self): @@ -20,11 +27,16 @@ def setUp(self): :return: void """ - Instances.objects.create(id="instance-01", heartbeat_at="2021-08-28", leased_at="2021-05-28", lease_duration=90) - Instances.objects.create(id="instance-02", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) - Instances.objects.create(id="instance-03", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) - Instances.objects.create(id="instance-04", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) - Instances.objects.create(id="instance-05", heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) + Instances.objects.create(id="instance-01", + heartbeat_at=now, leased_at=now - relativedelta(days=+119), lease_duration=120) + Instances.objects.create(id="instance-02", + heartbeat_at="2020-08-28", leased_at="2020-05-28", lease_duration=90) + Instances.objects.create(id="instance-03", + heartbeat_at="2020-09-25", leased_at="2020-05-28", lease_duration=120) + Instances.objects.create(id="instance-04", + heartbeat_at="2020-09-25", leased_at="2020-05-28", lease_duration=120) + Instances.objects.create(id="instance-05", + heartbeat_at="2021-08-28", leased_at=now - relativedelta(days=+89), lease_duration=90) def test_instance_spy(self): """ @@ -34,9 +46,8 @@ def test_instance_spy(self): """ instance_spy() - # Only the instance with id "instance-02" should be deleted, as it's not excluded nor leased - self.assertIsInstance(Instances.objects.get(id="instance-01"),Instances) + self.assertIsInstance(Instances.objects.get(id="instance-01"), Instances) self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", Instances.objects.get, id="instance-02") self.assertIsInstance(Instances.objects.get(id="instance-03"), Instances) @@ -44,22 +55,33 @@ def test_instance_spy(self): self.assertIsInstance(Instances.objects.get(id="instance-05"), Instances) load_config() - GLOBAL_CONFIG['EXCLUDED_USERS_ID'] = [] + GLOBAL_CONFIG['EXCLUDE'].remove("Jane Smith") instance_spy() - # Now instance_03 should be deleted, because it no longer benefit from the user exclusion + # Now instance_03 should be deleted, because it no longer benefits from the user exclusion self.assertIsInstance(Instances.objects.get(id="instance-01"), Instances) self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", Instances.objects.get, id="instance-03") self.assertIsInstance(Instances.objects.get(id="instance-04"), Instances) self.assertIsInstance(Instances.objects.get(id="instance-05"), Instances) - load_config() - GLOBAL_CONFIG['EXCLUDED_PROJECTS'] = ['LAL'] - GLOBAL_CONFIG['EXCLUDED_USERS_ID'] = [] + GLOBAL_CONFIG['EXCLUDE'].remove("project-01") instance_spy() - # Now that all the exclusions are disabled, instance 04 and 05 should be deleted + # Now that all the exclusions are disabled, instance 04 should be deleted self.assertIsInstance(Instances.objects.get(id="instance-01"), Instances) self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", Instances.objects.get, id="instance-04") + self.assertIsInstance(Instances.objects.get(id="instance-05"), Instances) + + Instances.objects.filter(id="instance-01").update(leased_at=now - relativedelta(days=+91)) + Instances.objects.filter(id="instance-05").update(leased_at=now - relativedelta(days=+91)) + instance_spy() + # Only instance-05 should be deleted, as its lease duration is 90 days and instance-01's is 120 + self.assertIsInstance(Instances.objects.get(id="instance-01"), Instances) self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", Instances.objects.get, id="instance-05") + + Instances.objects.filter(id="instance-01").update(leased_at=now - relativedelta(days=+121)) + instance_spy() + # Now that we are over instance-01's lease duration, it should get deleted + self.assertRaisesMessage(django.core.exceptions.ObjectDoesNotExist, "Instances matching query does not exist.", + Instances.objects.get, id="instance-01") diff --git a/openstack_lease_it/openstack_lease_it/config.py b/openstack_lease_it/openstack_lease_it/config.py index 8b7acd3..4b51da0 100644 --- a/openstack_lease_it/openstack_lease_it/config.py +++ b/openstack_lease_it/openstack_lease_it/config.py @@ -55,10 +55,12 @@ 'NOTIFICATION_DELETE_CONTENT': BASE_CONFIG_DIR + '/delete-notification.txt', 'NOTIFICATION_LEASE_CONTENT': BASE_CONFIG_DIR + '/lease-notification.txt', - # special parameter - 'EXCLUDED_PROJECTS': [], - 'EXCLUDED_USERS_ID': [], - 'RESET_CACHE_INSTANCES': False + # Exclude initialisation : list of exclusions (projects, users,...) + 'EXCLUDE': [], + + # Lease durations dictionary initialisation + # It takes the form {"user_id_" + user_id: lease_duration} + 'SPECIAL_LEASE_DURATION': dict() } """ We use the global variable GLOBAL_CONFIG to share openstack-lease-it configuration to all user. Some @@ -69,13 +71,15 @@ 'DJANGO_SECRET_KEY': 'secret_key', 'DJANGO_DEBUG': 'debug', 'DJANGO_LOGDIR': 'log_dir', - 'DJANGO_LOGLEVEL': 'log_level' + 'DJANGO_LOGLEVEL': 'log_level', + 'RESET_CACHE_INSTANCES': 'reset_cache_instances' } """ - **DJANGO_SECRET_KEY**: The secret key used by django (file option: *secret_key*) - **DJANGO_DEBUG**: The DEBUG value for django (file option: *debug*) - **DJANGO_LOGDIR**: Directory where log file will be write (file option: *log_dir*) - **DJANGO_LOGLEVEL**: The log level used for django (file option: *log_level*) + - **RESET_CACHE_INSTANCES**: Enable / Disable reset of the cache when loading instances (for TestConnection) """ @@ -152,15 +156,10 @@ """ -SPECIAL_OPTIONS = { - 'EXCLUDED_PROJECTS': 'excluded_projects', - 'EXCLUDED_USERS_ID': 'excluded_users_id', - 'RESET_CACHE_INSTANCES': 'reset_cache_instances' +LISTS_OPTIONS = { } """ - - **EXCLUDED_PROJECTS**: List of the projects excluded by the delete call - - **EXCLUDED_USERS_ID**: List of the users' id excluded by the delete call - - **RESET_CACHE_INSTANCES**: Enable / Disable reset of the cache when loading instances + """ SECTIONS = { @@ -169,7 +168,7 @@ 'memcached': MEMCACHED_OPTIONS, 'plugins': PLUGINS_OPTIONS, 'notification': NOTIFICATION_OPTIONS, - 'special': SPECIAL_OPTIONS + 'lists': LISTS_OPTIONS } """ @@ -178,7 +177,7 @@ - **memcached**: section [memcached] - **plugins**: section [plugins] - **notification**: section [notification] - - **special**: section [special] + - **lists**: lists [lists] """ @@ -192,20 +191,29 @@ def load_config_option(config, section): :param section: The section of configuration file we compute :return: void """ - options = SECTIONS[section] + section_exists = True + try: + options = SECTIONS[section] + except KeyError: + # The section is unknown, we must create the options dictionary to go through + section_exists = False + options = config.options(section) + options = dict(zip(options, options)) for option in options: try: config_to_add = config.get(section, options[option]) - if config_to_add[0] != "[": + if section == "lists": + config_to_add = list(filter(None, [x.strip() for x in config_to_add.splitlines()])) + # If the first element is detected to be a tuple, we assume the whole list is a list of tuples + if config_to_add[0][0] == "(" and config_to_add[0][-1] == ")": + config_to_add = [tuple(x[1:-1].strip().split(',')) for x in config_to_add] + if section_exists: GLOBAL_CONFIG[option] = config_to_add else: - formalized_config_to_add = config_to_add.split(',') - len_config_to_add = len(formalized_config_to_add) - formalized_config_to_add = [formalized_config_to_add[i].strip() - for i in range(len_config_to_add)] - formalized_config_to_add[0] = formalized_config_to_add[0][1:] - formalized_config_to_add[len_config_to_add - 1] = formalized_config_to_add[len_config_to_add-1][:-1] - GLOBAL_CONFIG[option] = formalized_config_to_add + if "exclude" == option and bool(config_to_add) and section not in GLOBAL_CONFIG["EXCLUDE"]: + GLOBAL_CONFIG["EXCLUDE"].append(section) + if "duration" == option: + GLOBAL_CONFIG["SPECIAL_LEASE_DURATION"][section] = int(config_to_add) except configparser.NoSectionError: pass @@ -223,5 +231,6 @@ def load_config(): for config_file in CONFIG_FILES: config.read(config_file) - for section in SECTIONS: - load_config_option(config, section) + sections = config.sections() + for section in sections: + load_config_option(config, section) \ No newline at end of file From 7c24091e9f886d603ad1ee48673a99b0fae7c030 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Wed, 9 Jun 2021 15:41:15 +0200 Subject: [PATCH 03/38] Features and fixes --- .../fixtures/test_models explanations | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/openstack_lease_it/lease_it/fixtures/test_models explanations b/openstack_lease_it/lease_it/fixtures/test_models explanations index dfb83ff..5d45602 100644 --- a/openstack_lease_it/lease_it/fixtures/test_models explanations +++ b/openstack_lease_it/lease_it/fixtures/test_models explanations @@ -6,30 +6,3 @@ To load this database, use the django command : To manage instances loaded parameters, set them in test_models_setup.json, then execute the Python file load_test_models.py - -The instances generated originally have the following attributes : - - instance-01: - heartbeat_at: 2021-05-24 - leased_at: 2021-05-24 - lease_duration: 90 - - instance-02: - heartbeat_at: 2020-08-24 - leased_at: 2020-05-24 - lease_duration: 90 - - instance-03: - heartbeat_at: 2020-08-24 - leased_at: 2020-05-24 - lease_duration: 120 - - instance-04: - heartbeat_at: 2020-08-24 - leased_at: 2020-05-24 - lease_duration: 120 - - instance-05: - heartbeat_at: 2020-08-24 - leased_at: 2020-05-24 - lease_duration: 90 From a590f1c5d09359d886a243915fefbf2f9ea32ee5 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Mon, 21 Jun 2021 09:49:54 +0200 Subject: [PATCH 04/38] Modify url for new version of Django --- openstack_lease_it/lease_it/urls.py | 16 +++++++++------- openstack_lease_it/openstack_lease_it/urls.py | 5 +++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/openstack_lease_it/lease_it/urls.py b/openstack_lease_it/lease_it/urls.py index 2d57481..56fdfc1 100644 --- a/openstack_lease_it/lease_it/urls.py +++ b/openstack_lease_it/lease_it/urls.py @@ -14,21 +14,23 @@ """ from django.conf.urls import url +from lease_it import views + urlpatterns = [ # pylint: disable=invalid-name # Default dashboard view - url(r'^$', 'lease_it.views.dashboard', name='dashboard'), + url(r'^$', views.dashboard, name='dashboard'), # Flavors view - url(r'^flavors', 'lease_it.views.flavors', name='flavors'), + url(r'^flavors', views.flavors, name='flavors'), # Instances view - url(r'^instances[/]?$', 'lease_it.views.instances', name='instances'), - url(r'^instances/(?P[\w-]+)$', 'lease_it.views.instance', name='instance'), + url(r'^instances[/]?$', views.instances, name='instances'), + url(r'^instances/(?P[\w-]+)$', views.instance, name='instance'), # Database view - url(r'^database[/]?$', 'lease_it.views.databases', name='databases'), - url(r'^database/(?P[\w-]+)$', 'lease_it.views.database', name='database'), + url(r'^database[/]?$', views.databases, name='databases'), + url(r'^database/(?P[\w-]+)$', views.database, name='database'), # Users view - url(r'^users', 'lease_it.views.users', name='users') + url(r'^users', views.users, name='users') ] diff --git a/openstack_lease_it/openstack_lease_it/urls.py b/openstack_lease_it/openstack_lease_it/urls.py index c96b314..734cf32 100644 --- a/openstack_lease_it/openstack_lease_it/urls.py +++ b/openstack_lease_it/openstack_lease_it/urls.py @@ -17,6 +17,7 @@ from django.contrib import admin from openstack_lease_it.settings import GLOBAL_CONFIG +from openstack_lease_it import views if GLOBAL_CONFIG['BACKEND_PLUGIN'] == 'Openstack': urlpatterns = [ # pylint: disable=invalid-name @@ -30,8 +31,8 @@ ] else: urlpatterns = [ # pylint: disable=invalid-name - url(r'^login', 'openstack_lease_it.views.login', name='login'), - url(r'^logout', 'openstack_lease_it.views.logout', name='logout'), + url(r'^login', views.login, name='login'), + url(r'^logout', views.logout, name='logout'), # We add default django admin view in case of Test backend to allow easiest user management url(r'^admin', include(admin.site.urls)), url(r'^', include('lease_it.urls')) From 9f496b979200f1b3dcfec04422f4a74173197626 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Mon, 21 Jun 2021 09:51:31 +0200 Subject: [PATCH 05/38] Update requirements.txt --- requirements.txt | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index abe94d1..cac633d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,20 @@ alabaster==0.7.10 appdirs==1.4.3 +asgiref==3.3.4 Babel==2.4.0 certifi==2020.12.5 chardet==4.0.0 configparser==5.0.2 debtcollector==1.13.0 -Django==1.8 -django-openstack-auth==3.1.1 +Django==1.11.29 +django-openstack-auth==3.6.1 docutils==0.14 funcsigs==1.0.2 idna==2.10 imagesize==0.7.1 iso8601==0.1.11 Jinja2==2.9.6 -keystoneauth1==2.19.0 +keystoneauth1==4.3.1 MarkupSafe==2.0.1 monotonic==1.3 msgpack==1.0.2 @@ -21,13 +22,13 @@ msgpack-python==0.4.8 netaddr==0.7.19 netifaces==0.10.5 os-service-types==1.7.0 -oslo.config==3.24.0 +oslo.config==8.7.0 oslo.context==3.2.0 -oslo.i18n==3.15.0 -oslo.policy==1.22.0 +oslo.i18n==5.0.1 +oslo.policy==3.8.0 oslo.serialization==2.18.0 -oslo.utils==3.25.0 -packaging==16.8 +oslo.utils==4.9.0 +packaging==20.9 pbr==2.0.0 pkg-resources==0.0.0 positional==1.1.1 @@ -39,15 +40,16 @@ python-keystoneclient==3.10.0 python-memcached==1.58 python-novaclient==8.0.0 pytz==2017.2 -PyYAML==3.12 -requests==2.12.5 -rfc3986==0.4.1 +PyYAML==5.4.1 +requests==2.25.1 +rfc3986==1.5.0 simplejson==3.10.0 six==1.10.0 snowballstemmer==1.2.1 Sphinx==1.6.5 sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-websupport==1.0.1 +sqlparse==0.4.1 stevedore==1.21.0 typing==3.6.2 urllib3==1.26.4 From e2d3eda0fe2f167c9a2e1e75f60900371ce1f003 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Mon, 21 Jun 2021 16:50:27 +0200 Subject: [PATCH 06/38] Fixing API version in config & requirements, adding a delete config for Openstack --- .../lease_it/backend/OpenstackConnection.py | 7 ++-- .../openstack_lease_it/config.py | 32 +++++++++++-------- requirements.txt | 1 - 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 97befc8..47504d5 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -301,9 +301,10 @@ def delete(self, instance_id): :param instance_id: id of instance to delete :return: void """ - nova = nvclient.Client(NOVA_VERSION, session=self.session) - to_delete = nova.servers.list(search_opts={'all_tenants': 'true', 'id': instance_id}) - to_delete.delete() + if bool(GLOBAL_CONFIG['OS_IDENTITY_API_VERSION']): + nova = nvclient.Client(NOVA_VERSION, session=self.session) + to_delete = nova.servers.list(search_opts={'all_tenants': 'true', 'id': instance_id}) + to_delete.delete() def spy_instances(self): """ diff --git a/openstack_lease_it/openstack_lease_it/config.py b/openstack_lease_it/openstack_lease_it/config.py index 4b51da0..0a18d62 100644 --- a/openstack_lease_it/openstack_lease_it/config.py +++ b/openstack_lease_it/openstack_lease_it/config.py @@ -37,10 +37,11 @@ 'OS_PASSWORD': 'admin_password', # Must be defined to allow sphinx to run 'OS_PROJECT_NAME': 'admin', 'OS_AUTH_URL': 'https://keystone.example.com', # Must be defined to allow sphinx to run - 'OS_IDENTITY_API_VERSION': '3', + 'OS_IDENTITY_API_VERSION': 3, 'OS_USER_DOMAIN_NAME': 'default', 'OS_PROJECT_DOMAIN_NAME': 'default', 'OS_CACERT': None, # If certificate is signed by a legit CA, we don't need to define it + 'OS_DELETE': True, # Actually deletes the VMs from Openstack (turn False for testing) # memcached parameter 'MEMCACHED_HOST': '127.0.0.1', @@ -99,7 +100,8 @@ 'OS_CACERT': 'OS_CACERT', 'OS_IDENTITY_API_VERSION': 'OS_IDENTITY_API_VERSION', 'OS_PROJECT_DOMAIN_NAME': 'OS_PROJECT_DOMAIN_NAME', - 'OS_USER_DOMAIN_NAME': 'OS_USER_DOMAIN_NAME' + 'OS_USER_DOMAIN_NAME': 'OS_USER_DOMAIN_NAME', + 'OS_DELETE': 'OS_DELETE' } """ - **OS_USERNAME**: Openstack admin username (file option: *OS_USERNAME*) @@ -111,6 +113,7 @@ - **OS_IDENTITY_API_VERSION**: Keystone version (file option: *OS_IDENTITY_API_VERSION*) - **OS_PROJECT_DOMAIN_NAME**: project domain name (file option: *OS_PROJECT_DOMAIN_NAME*) - **OS_USER_DOMAIN_NAME**: user domain name (file option: *OS_USER_DOMAIN_NAME*) + - **OS_DELETE**: Deletes the VMs (turn False for testing) """ @@ -202,18 +205,21 @@ def load_config_option(config, section): for option in options: try: config_to_add = config.get(section, options[option]) - if section == "lists": - config_to_add = list(filter(None, [x.strip() for x in config_to_add.splitlines()])) - # If the first element is detected to be a tuple, we assume the whole list is a list of tuples - if config_to_add[0][0] == "(" and config_to_add[0][-1] == ")": - config_to_add = [tuple(x[1:-1].strip().split(',')) for x in config_to_add] - if section_exists: - GLOBAL_CONFIG[option] = config_to_add + if option == 'OS_IDENTITY_API_VERSION': + GLOBAL_CONFIG[option] = int(config_to_add) else: - if "exclude" == option and bool(config_to_add) and section not in GLOBAL_CONFIG["EXCLUDE"]: - GLOBAL_CONFIG["EXCLUDE"].append(section) - if "duration" == option: - GLOBAL_CONFIG["SPECIAL_LEASE_DURATION"][section] = int(config_to_add) + if section == "lists": + config_to_add = list(filter(None, [x.strip() for x in config_to_add.splitlines()])) + # If the first element is detected to be a tuple, we assume the whole list is a list of tuples + if config_to_add[0][0] == "(" and config_to_add[0][-1] == ")": + config_to_add = [tuple(x[1:-1].strip().split(',')) for x in config_to_add] + if section_exists: + GLOBAL_CONFIG[option] = config_to_add + else: + if "exclude" == option and bool(config_to_add) and section not in GLOBAL_CONFIG["EXCLUDE"]: + GLOBAL_CONFIG["EXCLUDE"].append(section) + if "duration" == option: + GLOBAL_CONFIG["SPECIAL_LEASE_DURATION"][section] = int(config_to_add) except configparser.NoSectionError: pass diff --git a/requirements.txt b/requirements.txt index cac633d..c36d4f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,6 @@ oslo.serialization==2.18.0 oslo.utils==4.9.0 packaging==20.9 pbr==2.0.0 -pkg-resources==0.0.0 positional==1.1.1 prettytable==0.7.2 Pygments==2.2.0 From 17b80a42bb82a5ca81bd8ba035417d4480693dd3 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 22 Jun 2021 17:31:35 +0200 Subject: [PATCH 07/38] Optimizing code in spy_instance --- .../lease_it/backend/OpenstackConnection.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 47504d5..e4f8581 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -334,16 +334,13 @@ def spy_instances(self): # and the instance's lease duration if user_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][user_name] - model = Instances.objects.get(id=data_instances[instance]['id']) - model.lease_duration = lease_duration elif data_instances[instance]['project_id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['project_id']] - model = Instances.objects.get(id=data_instances[instance]['id']) - model.lease_duration = lease_duration elif data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] - model = Instances.objects.get(id=data_instances[instance]['id']) - model.lease_duration = lease_duration + model = Instances.objects.get(id=data_instances[instance]['id']) + model.lease_duration = lease_duration + model.save() # If it's a new instance, we put lease value as today # it's not necessary to lease on model as heartbeat should have create and From 1696bc5676c1d4030069139c302215d0ce416448 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 22 Jun 2021 17:31:51 +0200 Subject: [PATCH 08/38] Optimizing code in spy_instance --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index e4f8581..fbb11bc 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -340,7 +340,6 @@ def spy_instances(self): lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] model = Instances.objects.get(id=data_instances[instance]['id']) model.lease_duration = lease_duration - model.save() # If it's a new instance, we put lease value as today # it's not necessary to lease on model as heartbeat should have create and From 6b0da42e699dc67eb2a4265e109851231a1fb5bc Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Wed, 23 Jun 2021 16:32:18 +0200 Subject: [PATCH 09/38] Fix delete + model save --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index fbb11bc..6766a7e 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -301,7 +301,7 @@ def delete(self, instance_id): :param instance_id: id of instance to delete :return: void """ - if bool(GLOBAL_CONFIG['OS_IDENTITY_API_VERSION']): + if bool(GLOBAL_CONFIG['OS_DELETE']): nova = nvclient.Client(NOVA_VERSION, session=self.session) to_delete = nova.servers.list(search_opts={'all_tenants': 'true', 'id': instance_id}) to_delete.delete() @@ -340,6 +340,7 @@ def spy_instances(self): lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] model = Instances.objects.get(id=data_instances[instance]['id']) model.lease_duration = lease_duration + model.save() # If it's a new instance, we put lease value as today # it's not necessary to lease on model as heartbeat should have create and From 04f645f53337ee19734d83924fd1a2b380a6096d Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 24 Jun 2021 10:52:48 +0200 Subject: [PATCH 10/38] Fix delete OS --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 6766a7e..b3a54f6 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -304,7 +304,8 @@ def delete(self, instance_id): if bool(GLOBAL_CONFIG['OS_DELETE']): nova = nvclient.Client(NOVA_VERSION, session=self.session) to_delete = nova.servers.list(search_opts={'all_tenants': 'true', 'id': instance_id}) - to_delete.delete() + for instance in to_delete: + instance.delete() def spy_instances(self): """ From 99f7cb0fd8bd4e7217fa2ef294a74eebe0dfe87f Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 24 Jun 2021 11:43:06 +0200 Subject: [PATCH 11/38] Delete notification --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index b3a54f6..0e1c741 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -306,6 +306,8 @@ def delete(self, instance_id): to_delete = nova.servers.list(search_opts={'all_tenants': 'true', 'id': instance_id}) for instance in to_delete: instance.delete() + else : + print("Deleted the instance " + instance_id + " from Openstack") def spy_instances(self): """ From 67e18c92e189306b99cffe4e037c26acdffc8f52 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 24 Jun 2021 15:51:31 +0200 Subject: [PATCH 12/38] Adjustments in OpenstackConnection.py --- .../lease_it/backend/OpenstackConnection.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 0e1c741..afa7e17 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -303,10 +303,10 @@ def delete(self, instance_id): """ if bool(GLOBAL_CONFIG['OS_DELETE']): nova = nvclient.Client(NOVA_VERSION, session=self.session) - to_delete = nova.servers.list(search_opts={'all_tenants': 'true', 'id': instance_id}) + to_delete = nova.servers.list(search_opts={'id': instance_id}) for instance in to_delete: instance.delete() - else : + else: print("Deleted the instance " + instance_id + " from Openstack") def spy_instances(self): @@ -324,11 +324,12 @@ def spy_instances(self): 'notify': list() # List of instance we must notify user to renew the lease } for instance in data_instances: + # We mark the VM as shown + InstancesAccess.heartbeat(data_instances[instance]) user_name = users[data_instances[instance]['user_id']]['name'] + instance_name = data_instances[instance]['name'] if data_instances[instance]['project_id'] not in GLOBAL_CONFIG["EXCLUDE"] and \ user_name not in GLOBAL_CONFIG["EXCLUDE"]: - # We mark the VM as shown - InstancesAccess.heartbeat(data_instances[instance]) leased_at = data_instances[instance]['leased_at'] lease_end = data_instances[instance]['lease_end'] lease_duration = LEASE_DURATION @@ -337,6 +338,8 @@ def spy_instances(self): # and the instance's lease duration if user_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][user_name] + if instance_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][instance_name] elif data_instances[instance]['project_id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['project_id']] elif data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: From 93571f8f6b323784cabe53b81cc3c1ec853fa219 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 24 Jun 2021 16:42:06 +0200 Subject: [PATCH 13/38] Fix delete --- .../lease_it/backend/OpenstackConnection.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index afa7e17..50ea1f8 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -303,9 +303,11 @@ def delete(self, instance_id): """ if bool(GLOBAL_CONFIG['OS_DELETE']): nova = nvclient.Client(NOVA_VERSION, session=self.session) - to_delete = nova.servers.list(search_opts={'id': instance_id}) - for instance in to_delete: - instance.delete() + instance_list = nova.servers.list() + for instance in instance_list: + if instance.id == instance_id: + instance.delete() + break else: print("Deleted the instance " + instance_id + " from Openstack") From bafb0cdc6b15e811025f3f9206c10036968237b5 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 24 Jun 2021 17:21:33 +0200 Subject: [PATCH 14/38] Reset cache in delete --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 50ea1f8..ac71196 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -303,13 +303,14 @@ def delete(self, instance_id): """ if bool(GLOBAL_CONFIG['OS_DELETE']): nova = nvclient.Client(NOVA_VERSION, session=self.session) - instance_list = nova.servers.list() + instance_list = nova.servers.list(search_opts={'all_tenants': 'true', 'uuid': instance_id}) for instance in instance_list: if instance.id == instance_id: instance.delete() break else: print("Deleted the instance " + instance_id + " from Openstack") + cache.delete('instances') def spy_instances(self): """ From 28ab5f4739a70c6c04f25e84256a6412b6cbea49 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 25 Jun 2021 13:05:21 +0200 Subject: [PATCH 15/38] Optimizing VM deleting --- .../lease_it/backend/OpenstackConnection.py | 11 +++++------ openstack_lease_it/lease_it/client/run.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index ac71196..0a0c39f 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -294,22 +294,21 @@ def lease_instance(request, instance_id): InstancesAccess.heartbeat(data_instances[instance_id]) return data_instances[instance_id] - def delete(self, instance_id): + def delete(self, instances_to_delete): """ Deletes the VM with the id given in parameter - :param instance_id: id of instance to delete + :param instances_to_delete: list of instances to delete :return: void """ if bool(GLOBAL_CONFIG['OS_DELETE']): nova = nvclient.Client(NOVA_VERSION, session=self.session) - instance_list = nova.servers.list(search_opts={'all_tenants': 'true', 'uuid': instance_id}) + instance_list = nova.servers.list(search_opts={'all_tenants': 'true'}) for instance in instance_list: - if instance.id == instance_id: + if instance in instances_to_delete: instance.delete() - break else: - print("Deleted the instance " + instance_id + " from Openstack") + print("Deleted the instances from Openstack") cache.delete('instances') def spy_instances(self): diff --git a/openstack_lease_it/lease_it/client/run.py b/openstack_lease_it/lease_it/client/run.py index 141110a..9baa3d5 100644 --- a/openstack_lease_it/lease_it/client/run.py +++ b/openstack_lease_it/lease_it/client/run.py @@ -36,7 +36,7 @@ def instance_spy(): notification[notification_type][instance['user_id']].append(instance) if notification_type == "delete": InstancesAccess.delete(instance['id']) - BACKEND.delete(instance['id']) + BACKEND.delete(instances['delete']) notify = MailNotification(users) notify.send(notification) From a8788c3b3fb92efb25855ae5c779d525d344a137 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 25 Jun 2021 13:54:47 +0200 Subject: [PATCH 16/38] Fix delete optimizing --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 0a0c39f..34dca56 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -305,8 +305,9 @@ def delete(self, instances_to_delete): nova = nvclient.Client(NOVA_VERSION, session=self.session) instance_list = nova.servers.list(search_opts={'all_tenants': 'true'}) for instance in instance_list: - if instance in instances_to_delete: - instance.delete() + for to_delete in instances_to_delete: + if instance.id == to_delete['id']: + instance.delete() else: print("Deleted the instances from Openstack") cache.delete('instances') From 50ceab433f7fd43c5a0b4ac86e77a540febc0c2c Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 25 Jun 2021 15:44:32 +0200 Subject: [PATCH 17/38] Adapting delete in views --- openstack_lease_it/lease_it/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstack_lease_it/lease_it/views.py b/openstack_lease_it/lease_it/views.py index 8e43c0f..201fdae 100644 --- a/openstack_lease_it/lease_it/views.py +++ b/openstack_lease_it/lease_it/views.py @@ -164,7 +164,7 @@ def database(request, instance_id): # pylint: disable=unused-argument } try: InstancesAccess.delete(instance_id) - BACKEND.delete(instance_id) + BACKEND.delete([{'id': instance_id}]) except StillRunning as error: response = { 'status': 'failed', From e71e2f09ca95e7a2ebe169a5c9a254cca2285671 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 25 Jun 2021 16:04:48 +0200 Subject: [PATCH 18/38] Fix exclusion projects --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 34dca56..d8a49e6 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -322,6 +322,7 @@ def spy_instances(self): now = date.today() data_instances = InstancesAccess.show(self._instances()) users = self._users() + projects = self._projects() response = { 'delete': list(), # List of instance we must delete 'notify': list() # List of instance we must notify user to renew the lease @@ -331,8 +332,10 @@ def spy_instances(self): InstancesAccess.heartbeat(data_instances[instance]) user_name = users[data_instances[instance]['user_id']]['name'] instance_name = data_instances[instance]['name'] - if data_instances[instance]['project_id'] not in GLOBAL_CONFIG["EXCLUDE"] and \ - user_name not in GLOBAL_CONFIG["EXCLUDE"]: + project_name = projects[data_instances[instance]['project_id']]['name'] + if project_name not in GLOBAL_CONFIG["EXCLUDE"] and \ + user_name not in GLOBAL_CONFIG["EXCLUDE"] and \ + instance_name not in GLOBAL_CONFIG["EXCLUDE"]: leased_at = data_instances[instance]['leased_at'] lease_end = data_instances[instance]['lease_end'] lease_duration = LEASE_DURATION From 07ef53ba68ab48a66047c4bbf550ee0ffc2cec2b Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 25 Jun 2021 16:11:37 +0200 Subject: [PATCH 19/38] Fix exclusion projects --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index d8a49e6..a16b3ee 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -322,7 +322,7 @@ def spy_instances(self): now = date.today() data_instances = InstancesAccess.show(self._instances()) users = self._users() - projects = self._projects() + projects = self.projects() response = { 'delete': list(), # List of instance we must delete 'notify': list() # List of instance we must notify user to renew the lease @@ -375,3 +375,4 @@ def spy_instances(self): lease_end < now - relativedelta(days=-6): response['notify'].append(data_instances[instance]) return response + From 001851c667ad7b11824bb8337620257d0da57734 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 25 Jun 2021 16:53:08 +0200 Subject: [PATCH 20/38] Changing exclusion rules --- .../lease_it/backend/OpenstackConnection.py | 77 +++++++++---------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index a16b3ee..c8ce0ff 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -333,46 +333,45 @@ def spy_instances(self): user_name = users[data_instances[instance]['user_id']]['name'] instance_name = data_instances[instance]['name'] project_name = projects[data_instances[instance]['project_id']]['name'] - if project_name not in GLOBAL_CONFIG["EXCLUDE"] and \ + leased_at = data_instances[instance]['leased_at'] + lease_end = data_instances[instance]['lease_end'] + lease_duration = LEASE_DURATION + # If the instance benefits from a special lease (from its user_name, instance_id or project_id), + # we update the lease_duration (used to determine whether to delete it or not) + # and the instance's lease duration + if user_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][user_name] + if instance_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][instance_name] + elif data_instances[instance]['project_id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['project_id']] + elif data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] + model = Instances.objects.get(id=data_instances[instance]['id']) + model.lease_duration = lease_duration + model.save() + + # If it's a new instance, we put lease value as today + # it's not necessary to lease on model as heartbeat should have create and + # lease the virtual machine + if leased_at is None: + lease_end = now + relativedelta(days=+lease_duration) + first_notification_date = lease_end - relativedelta(days=+lease_duration/3) + second_notification_date = lease_end - relativedelta(days=+lease_duration/6) + LOGGER_INSTANCES.info( + "Instance: %s will be notify %s and %s", + data_instances[instance]['id'], + first_notification_date, + second_notification_date, + ) + # If lease has expired and it's not in the excluded projects, we tag it as delete + if lease_end < now and project_name not in GLOBAL_CONFIG["EXCLUDE"] and \ user_name not in GLOBAL_CONFIG["EXCLUDE"] and \ instance_name not in GLOBAL_CONFIG["EXCLUDE"]: - leased_at = data_instances[instance]['leased_at'] - lease_end = data_instances[instance]['lease_end'] - lease_duration = LEASE_DURATION - # If the instance benefits from a special lease (from its user_name, instance_id or project_id), - # we update the lease_duration (used to determine whether to delete it or not) - # and the instance's lease duration - if user_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: - lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][user_name] - if instance_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: - lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][instance_name] - elif data_instances[instance]['project_id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: - lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['project_id']] - elif data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: - lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] - model = Instances.objects.get(id=data_instances[instance]['id']) - model.lease_duration = lease_duration - model.save() - - # If it's a new instance, we put lease value as today - # it's not necessary to lease on model as heartbeat should have create and - # lease the virtual machine - if leased_at is None: - lease_end = now + relativedelta(days=+lease_duration) - first_notification_date = lease_end - relativedelta(days=+lease_duration/3) - second_notification_date = lease_end - relativedelta(days=+lease_duration/6) - LOGGER_INSTANCES.info( - "Instance: %s will be notify %s and %s", - data_instances[instance]['id'], - first_notification_date, - second_notification_date, - ) - # If lease has expired and it's not in the excluded projects, we tag it as delete - if lease_end < now: - response['delete'].append(data_instances[instance]) - elif first_notification_date == now or \ - second_notification_date == now or \ - lease_end < now - relativedelta(days=-6): - response['notify'].append(data_instances[instance]) + response['delete'].append(data_instances[instance]) + elif first_notification_date == now or \ + second_notification_date == now or \ + lease_end < now - relativedelta(days=-6): + response['notify'].append(data_instances[instance]) return response From 51701644d4c07eff28f55386dca55205c2d1c78b Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 29 Jun 2021 09:43:35 +0200 Subject: [PATCH 21/38] Loading config during spy_instance --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index c8ce0ff..d95a34b 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -18,6 +18,7 @@ from novaclient import client as nvclient from openstack_lease_it.settings import GLOBAL_CONFIG, LOGGER_INSTANCES +from openstack_lease_it.config import load_config from lease_it.datastore import InstancesAccess, LEASE_DURATION from lease_it.backend.Exceptions import PermissionDenied @@ -319,6 +320,7 @@ def spy_instances(self): :return: dict() """ + load_config() now = date.today() data_instances = InstancesAccess.show(self._instances()) users = self._users() @@ -373,5 +375,6 @@ def spy_instances(self): second_notification_date == now or \ lease_end < now - relativedelta(days=-6): response['notify'].append(data_instances[instance]) + cache.delete("instances") return response From 4c9752b9ed9600af61afa0f8dcd4a116a2684c89 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 29 Jun 2021 12:11:26 +0200 Subject: [PATCH 22/38] Adding reset cache setting --- .../lease_it/backend/OpenstackConnection.py | 18 ++++++++++++------ .../lease_it/backend/TestConnection.py | 8 ++++---- .../openstack_lease_it/config.py | 5 +++-- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index d95a34b..80a2292 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -18,7 +18,6 @@ from novaclient import client as nvclient from openstack_lease_it.settings import GLOBAL_CONFIG, LOGGER_INSTANCES -from openstack_lease_it.config import load_config from lease_it.datastore import InstancesAccess, LEASE_DURATION from lease_it.backend.Exceptions import PermissionDenied @@ -66,7 +65,9 @@ def _instances(self): List of instances actually launched :return: dict() """ - response = cache.get('instances') + response = None + if not bool(GLOBAL_CONFIG['RESET_CACHE']): + response = cache.get('instances') if not response: response = dict() nova = nvclient.Client(NOVA_VERSION, session=self.session) @@ -108,7 +109,9 @@ def _flavors(self): List of flavors and their details """ # We retrieve information from memcached - response = cache.get('flavors') + response = None + if not bool(GLOBAL_CONFIG['RESET_CACHE']): + response = cache.get('flavors') if not response: response = dict() nova = nvclient.Client(NOVA_VERSION, session=self.session) @@ -128,7 +131,9 @@ def _domains(self): List all domains available :return: dict() """ - response = cache.get('domains') + response = None + if not bool(GLOBAL_CONFIG['RESET_CACHE']): + response = cache.get('domains') if not response: response = dict() keystone = ksclient.Client(session=self.session) @@ -150,7 +155,9 @@ def _users(self): so we return a None object :return: dict() """ - response = cache.get('users') + response = None + if not bool(GLOBAL_CONFIG['RESET_CACHE']): + response = cache.get('users') if not response: response = dict() keystone = ksclient.Client(session=self.session) @@ -320,7 +327,6 @@ def spy_instances(self): :return: dict() """ - load_config() now = date.today() data_instances = InstancesAccess.show(self._instances()) users = self._users() diff --git a/openstack_lease_it/lease_it/backend/TestConnection.py b/openstack_lease_it/lease_it/backend/TestConnection.py index 34d55a8..7556383 100644 --- a/openstack_lease_it/lease_it/backend/TestConnection.py +++ b/openstack_lease_it/lease_it/backend/TestConnection.py @@ -41,7 +41,7 @@ def _instances(self): :return: dict() """ response = None - if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('instances') if not response: response = { @@ -92,7 +92,7 @@ def _flavors(self): :return: dict() """ response = None - if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('flavors') if not response: response = { @@ -147,7 +147,7 @@ def _domains(self): :return: dict() """ response = None - if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('domains') if not response: response = { @@ -171,7 +171,7 @@ def _users(self): :return: dict() """ response = None - if not GLOBAL_CONFIG['RESET_CACHE_INSTANCES']: + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('users') if not response: response = { diff --git a/openstack_lease_it/openstack_lease_it/config.py b/openstack_lease_it/openstack_lease_it/config.py index 0a18d62..c792f41 100644 --- a/openstack_lease_it/openstack_lease_it/config.py +++ b/openstack_lease_it/openstack_lease_it/config.py @@ -30,6 +30,7 @@ 'DJANGO_LOGDIR': '/var/log/openstack-lease-it/', 'DJANGO_LOGLEVEL': 'INFO', 'DJANGO_SECRET_KEY': 'Must_be_defined', # Must be defined to allow sphinx to run + 'RESET_CACHE': False, # OpenStack parameters 'OS_USERNAME': 'admin', @@ -73,14 +74,14 @@ 'DJANGO_DEBUG': 'debug', 'DJANGO_LOGDIR': 'log_dir', 'DJANGO_LOGLEVEL': 'log_level', - 'RESET_CACHE_INSTANCES': 'reset_cache_instances' + 'RESET_CACHE': 'reset_cache' } """ - **DJANGO_SECRET_KEY**: The secret key used by django (file option: *secret_key*) - **DJANGO_DEBUG**: The DEBUG value for django (file option: *debug*) - **DJANGO_LOGDIR**: Directory where log file will be write (file option: *log_dir*) - **DJANGO_LOGLEVEL**: The log level used for django (file option: *log_level*) - - **RESET_CACHE_INSTANCES**: Enable / Disable reset of the cache when loading instances (for TestConnection) + - **RESET_CACHE**: Enable / Disable reset of the cache when loading instances """ From eb029feaa1a010b4644f815d4287d739e8560062 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 29 Jun 2021 14:33:08 +0200 Subject: [PATCH 23/38] Inverting reset cache config: True = reset --- .../lease_it/backend/OpenstackConnection.py | 8 ++++---- openstack_lease_it/lease_it/backend/TestConnection.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 80a2292..a2f77aa 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -66,7 +66,7 @@ def _instances(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('instances') if not response: response = dict() @@ -110,7 +110,7 @@ def _flavors(self): """ # We retrieve information from memcached response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('flavors') if not response: response = dict() @@ -132,7 +132,7 @@ def _domains(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('domains') if not response: response = dict() @@ -156,7 +156,7 @@ def _users(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('users') if not response: response = dict() diff --git a/openstack_lease_it/lease_it/backend/TestConnection.py b/openstack_lease_it/lease_it/backend/TestConnection.py index 7556383..0b0d15d 100644 --- a/openstack_lease_it/lease_it/backend/TestConnection.py +++ b/openstack_lease_it/lease_it/backend/TestConnection.py @@ -41,7 +41,7 @@ def _instances(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('instances') if not response: response = { @@ -92,7 +92,7 @@ def _flavors(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('flavors') if not response: response = { @@ -147,7 +147,7 @@ def _domains(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('domains') if not response: response = { @@ -171,7 +171,7 @@ def _users(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('users') if not response: response = { From a82dac87a3ad4315f2eb93ef82786aa7268a42e0 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 29 Jun 2021 14:35:25 +0200 Subject: [PATCH 24/38] Inverting reset cache config: True = reset --- .../lease_it/backend/OpenstackConnection.py | 8 ++++---- openstack_lease_it/lease_it/backend/TestConnection.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index a2f77aa..80a2292 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -66,7 +66,7 @@ def _instances(self): :return: dict() """ response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('instances') if not response: response = dict() @@ -110,7 +110,7 @@ def _flavors(self): """ # We retrieve information from memcached response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('flavors') if not response: response = dict() @@ -132,7 +132,7 @@ def _domains(self): :return: dict() """ response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('domains') if not response: response = dict() @@ -156,7 +156,7 @@ def _users(self): :return: dict() """ response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('users') if not response: response = dict() diff --git a/openstack_lease_it/lease_it/backend/TestConnection.py b/openstack_lease_it/lease_it/backend/TestConnection.py index 0b0d15d..7556383 100644 --- a/openstack_lease_it/lease_it/backend/TestConnection.py +++ b/openstack_lease_it/lease_it/backend/TestConnection.py @@ -41,7 +41,7 @@ def _instances(self): :return: dict() """ response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('instances') if not response: response = { @@ -92,7 +92,7 @@ def _flavors(self): :return: dict() """ response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('flavors') if not response: response = { @@ -147,7 +147,7 @@ def _domains(self): :return: dict() """ response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('domains') if not response: response = { @@ -171,7 +171,7 @@ def _users(self): :return: dict() """ response = None - if bool(GLOBAL_CONFIG['RESET_CACHE']): + if not bool(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('users') if not response: response = { From e462b46de17653c9b2bc5b760d116ab2032e14b4 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 29 Jun 2021 15:07:46 +0200 Subject: [PATCH 25/38] Rearranging priority for special lease durations --- .../lease_it/backend/OpenstackConnection.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 80a2292..60869ac 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -347,14 +347,19 @@ def spy_instances(self): # If the instance benefits from a special lease (from its user_name, instance_id or project_id), # we update the lease_duration (used to determine whether to delete it or not) # and the instance's lease duration - if user_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: - lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][user_name] - if instance_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + # Special lease are ordered in the following priority order : + # instance_id > instance_name > user_name > project_id > project_name + if data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']:\ + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] + elif instance_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][instance_name] + elif user_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][user_name] elif data_instances[instance]['project_id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['project_id']] - elif data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: - lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] + elif project_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][project_name] + model = Instances.objects.get(id=data_instances[instance]['id']) model.lease_duration = lease_duration model.save() From 155e7c32e309581939291ce732d204b33771d9ee Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 29 Jun 2021 15:32:10 +0200 Subject: [PATCH 26/38] Adding special lease instance differentiation for new instances --- openstack_lease_it/lease_it/backend/OpenstackConnection.py | 2 +- openstack_lease_it/lease_it/datastore/ModelAccess.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 60869ac..161fdc1 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -349,7 +349,7 @@ def spy_instances(self): # and the instance's lease duration # Special lease are ordered in the following priority order : # instance_id > instance_name > user_name > project_id > project_name - if data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']:\ + if data_instances[instance]['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][data_instances[instance]['id']] elif instance_name in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][instance_name] diff --git a/openstack_lease_it/lease_it/datastore/ModelAccess.py b/openstack_lease_it/lease_it/datastore/ModelAccess.py index 1c03702..3bc97bc 100644 --- a/openstack_lease_it/lease_it/datastore/ModelAccess.py +++ b/openstack_lease_it/lease_it/datastore/ModelAccess.py @@ -44,7 +44,12 @@ def get(instance): model.leased_at = timezone.now() model.heartbeat_at = timezone.now() model.lease_duration = LEASE_DURATION - # The different lease durations will be adapted once spy_instance has run once + if instance['id'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + model.lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][instance['id']] + elif instance['name'] in GLOBAL_CONFIG['SPECIAL_LEASE_DURATION']: + model.lease_duration = GLOBAL_CONFIG['SPECIAL_LEASE_DURATION'][instance['name']] + # The different lease durations according to users and projects + # will be adapted once spy_instance has run once return model @staticmethod From c21a511245c079c8171be5dc7c1bb2bcbabed67a Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 29 Jun 2021 17:16:44 +0200 Subject: [PATCH 27/38] Fix bool evaluation and heartbeat timeout --- .../lease_it/backend/OpenstackConnection.py | 10 +++++----- openstack_lease_it/lease_it/backend/TestConnection.py | 8 ++++---- openstack_lease_it/lease_it/datastore/ModelAccess.py | 4 ---- openstack_lease_it/lease_it/static/js/database.js | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/openstack_lease_it/lease_it/backend/OpenstackConnection.py b/openstack_lease_it/lease_it/backend/OpenstackConnection.py index 161fdc1..ef28a8f 100644 --- a/openstack_lease_it/lease_it/backend/OpenstackConnection.py +++ b/openstack_lease_it/lease_it/backend/OpenstackConnection.py @@ -66,7 +66,7 @@ def _instances(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('instances') if not response: response = dict() @@ -110,7 +110,7 @@ def _flavors(self): """ # We retrieve information from memcached response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('flavors') if not response: response = dict() @@ -132,7 +132,7 @@ def _domains(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('domains') if not response: response = dict() @@ -156,7 +156,7 @@ def _users(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('users') if not response: response = dict() @@ -309,7 +309,7 @@ def delete(self, instances_to_delete): :param instances_to_delete: list of instances to delete :return: void """ - if bool(GLOBAL_CONFIG['OS_DELETE']): + if eval(GLOBAL_CONFIG['OS_DELETE']): nova = nvclient.Client(NOVA_VERSION, session=self.session) instance_list = nova.servers.list(search_opts={'all_tenants': 'true'}) for instance in instance_list: diff --git a/openstack_lease_it/lease_it/backend/TestConnection.py b/openstack_lease_it/lease_it/backend/TestConnection.py index 7556383..9e3dee2 100644 --- a/openstack_lease_it/lease_it/backend/TestConnection.py +++ b/openstack_lease_it/lease_it/backend/TestConnection.py @@ -41,7 +41,7 @@ def _instances(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('instances') if not response: response = { @@ -92,7 +92,7 @@ def _flavors(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('flavors') if not response: response = { @@ -147,7 +147,7 @@ def _domains(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('domains') if not response: response = { @@ -171,7 +171,7 @@ def _users(self): :return: dict() """ response = None - if not bool(GLOBAL_CONFIG['RESET_CACHE']): + if not eval(GLOBAL_CONFIG['RESET_CACHE']): response = cache.get('users') if not response: response = { diff --git a/openstack_lease_it/lease_it/datastore/ModelAccess.py b/openstack_lease_it/lease_it/datastore/ModelAccess.py index 3bc97bc..600ba44 100644 --- a/openstack_lease_it/lease_it/datastore/ModelAccess.py +++ b/openstack_lease_it/lease_it/datastore/ModelAccess.py @@ -2,8 +2,6 @@ """ ModelAccess module is a interface between Django model and view """ -import os - from dateutil.relativedelta import relativedelta from django.utils import timezone @@ -18,8 +16,6 @@ # Default lease duration in day LEASE_DURATION = 90 -# Number of day we keep instance in database -HEARTBEAT_TIMEOUT = 7 class InstancesAccess(object): # pylint: disable=too-few-public-methods diff --git a/openstack_lease_it/lease_it/static/js/database.js b/openstack_lease_it/lease_it/static/js/database.js index 2472417..76913af 100644 --- a/openstack_lease_it/lease_it/static/js/database.js +++ b/openstack_lease_it/lease_it/static/js/database.js @@ -28,7 +28,7 @@ function buildDatabaseView(div_name) { var now = new Date(); var lease_end = new Date(row.lease_end); // If a VM has a lease_end date before today, we can delete it - if (lease_end < now) { + if (lease_end < now - HEARTBEAT_TIMEOUT) { return buildDatabaseRowMenu(data) + formatText(data, MAX_STRING_LENGTH); } else { From a7ace5b32f3241c8cb39dc1fe0cd0d2ee23e4907 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Wed, 30 Jun 2021 11:44:56 +0200 Subject: [PATCH 28/38] Test button on instance view --- .../lease_it/static/js/instances.js | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index 85a59b0..d2a00ba 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -13,6 +13,9 @@ */ const MAX_STRING_LENGTH = 30; +// 7 * 25 * 60 * 60 * 1000 = 604800000 +const HEARTBEAT_TIMEOUT = 604800000; + /* buildInstancesView create a full display of Instance on div_name */ @@ -46,7 +49,16 @@ function buildInstancesView(div_name, get_option, show_user){ { targets: [0, 1, 2], render: function ( data, type, row ) { + //return formatText(data, MAX_STRING_LENGTH); + var now = new Date(); + var lease_end = new Date(row.lease_end); + // If a VM has a lease_end date before today, we can delete it + if (lease_end < now - HEARTBEAT_TIMEOUT) { + return buildDatabaseRowMenu(data) + + formatText(data, MAX_STRING_LENGTH); + } else { return formatText(data, MAX_STRING_LENGTH); + } } }], drawCallback: function(settings, json) { @@ -86,4 +98,30 @@ function formatLeaseBtn(date, instance) { ' new badge hoverable"' + ' data-badge-caption="new lease" onClick="updateLease(\''+ instance + '\')">'; +} + + +/* + buildDatabaseRowMenu build a menu for each row of Database Table +*/ +function buildDatabaseRowMenu(data) { + var menu = '' + + 'chevron_right ' + + ' '; + return menu; +} + +/* + swapDatabaseRowMenu swap on/off the delete button +*/ +function swapDatabaseRowMenu(button) { + if ($('#database-delete-' + button).css('display') == 'none') { + $('#database-icon-' + button).text('chevron_left'); + } else { + $('#database-icon-' + button).text('chevron_right'); + } + $('#database-delete-' + button).toggle(); } \ No newline at end of file From b80b6dd6e03a941cd82c67fc5c5848e2db083596 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Wed, 30 Jun 2021 11:58:15 +0200 Subject: [PATCH 29/38] Test button on instance view --- openstack_lease_it/lease_it/static/js/instances.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index d2a00ba..43e6cf0 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -124,4 +124,14 @@ function swapDatabaseRowMenu(button) { $('#database-icon-' + button).text('chevron_right'); } $('#database-delete-' + button).toggle(); +} + +/* + deleteDatabase delete entry in database +*/ +function deleteDatabase(instance) { + return $.getJSON("/database/" + instance, function(data){ + }).success(function(data){ + notify(data); + }); } \ No newline at end of file From d5c16cdb2bd505dc859b2aab9070e8777773d1cf Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 1 Jul 2021 09:03:12 +0200 Subject: [PATCH 30/38] Test button on instance view --- .../lease_it/static/js/instances.js | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index 43e6cf0..d0de9d3 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -13,9 +13,6 @@ */ const MAX_STRING_LENGTH = 30; -// 7 * 25 * 60 * 60 * 1000 = 604800000 -const HEARTBEAT_TIMEOUT = 604800000; - /* buildInstancesView create a full display of Instance on div_name */ @@ -49,16 +46,7 @@ function buildInstancesView(div_name, get_option, show_user){ { targets: [0, 1, 2], render: function ( data, type, row ) { - //return formatText(data, MAX_STRING_LENGTH); - var now = new Date(); - var lease_end = new Date(row.lease_end); - // If a VM has a lease_end date before today, we can delete it - if (lease_end < now - HEARTBEAT_TIMEOUT) { - return buildDatabaseRowMenu(data) + - formatText(data, MAX_STRING_LENGTH); - } else { return formatText(data, MAX_STRING_LENGTH); - } } }], drawCallback: function(settings, json) { @@ -99,39 +87,3 @@ function formatLeaseBtn(date, instance) { ' data-badge-caption="new lease" onClick="updateLease(\''+ instance + '\')">'; } - - -/* - buildDatabaseRowMenu build a menu for each row of Database Table -*/ -function buildDatabaseRowMenu(data) { - var menu = '' + - 'chevron_right ' + - ' '; - return menu; -} - -/* - swapDatabaseRowMenu swap on/off the delete button -*/ -function swapDatabaseRowMenu(button) { - if ($('#database-delete-' + button).css('display') == 'none') { - $('#database-icon-' + button).text('chevron_left'); - } else { - $('#database-icon-' + button).text('chevron_right'); - } - $('#database-delete-' + button).toggle(); -} - -/* - deleteDatabase delete entry in database -*/ -function deleteDatabase(instance) { - return $.getJSON("/database/" + instance, function(data){ - }).success(function(data){ - notify(data); - }); -} \ No newline at end of file From f9d5e393b16a784c67c2652b967ca0264a5fa2e1 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 1 Jul 2021 09:40:43 +0200 Subject: [PATCH 31/38] Test button on instance view --- .../lease_it/static/js/instances.js | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index d0de9d3..cf5759b 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -46,7 +46,7 @@ function buildInstancesView(div_name, get_option, show_user){ { targets: [0, 1, 2], render: function ( data, type, row ) { - return formatText(data, MAX_STRING_LENGTH); + return buildInstanceRowMenu(data) + formatText(data, MAX_STRING_LENGTH); } }], drawCallback: function(settings, json) { @@ -87,3 +87,39 @@ function formatLeaseBtn(date, instance) { ' data-badge-caption="new lease" onClick="updateLease(\''+ instance + '\')">'; } + + +/* + buildDatabaseRowMenu build a menu for each row of Database Table +*/ +function buildInstanceRowMenu(data) { + var menu = '' + + 'chevron_right ' + + ' '; + return menu; +} + +/* + swapDatabaseRowMenu swap on/off the delete button +*/ +function swapDatabaseRowMenu(button) { + if ($('#database-delete-' + button).css('display') == 'none') { + $('#database-icon-' + button).text('chevron_left'); + } else { + $('#database-icon-' + button).text('chevron_right'); + } + $('#database-delete-' + button).toggle(); +} + +/* + deleteDatabase delete entry in database +*/ +function deleteDatabase(instance) { + return $.getJSON("/database/" + instance, function(data){ + }).success(function(data){ + notify(data); + }); +} From d244ea5ccccc99780fe5eda1f769a0fb3c8c8e1f Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 2 Jul 2021 14:36:38 +0200 Subject: [PATCH 32/38] Test button on instance view --- .../lease_it/static/js/instances.js | 38 +------------------ 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index cf5759b..e1c9289 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -46,7 +46,7 @@ function buildInstancesView(div_name, get_option, show_user){ { targets: [0, 1, 2], render: function ( data, type, row ) { - return buildInstanceRowMenu(data) + formatText(data, MAX_STRING_LENGTH); + return buildDatabaseRowMenu(data) + formatText(data, MAX_STRING_LENGTH); } }], drawCallback: function(settings, json) { @@ -87,39 +87,3 @@ function formatLeaseBtn(date, instance) { ' data-badge-caption="new lease" onClick="updateLease(\''+ instance + '\')">'; } - - -/* - buildDatabaseRowMenu build a menu for each row of Database Table -*/ -function buildInstanceRowMenu(data) { - var menu = '' + - 'chevron_right ' + - ' '; - return menu; -} - -/* - swapDatabaseRowMenu swap on/off the delete button -*/ -function swapDatabaseRowMenu(button) { - if ($('#database-delete-' + button).css('display') == 'none') { - $('#database-icon-' + button).text('chevron_left'); - } else { - $('#database-icon-' + button).text('chevron_right'); - } - $('#database-delete-' + button).toggle(); -} - -/* - deleteDatabase delete entry in database -*/ -function deleteDatabase(instance) { - return $.getJSON("/database/" + instance, function(data){ - }).success(function(data){ - notify(data); - }); -} From 260e33681125085e2e513d731a9552940ac31356 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 6 Jul 2021 16:50:26 +0200 Subject: [PATCH 33/38] Test button on instance view --- .../lease_it/static/js/database.js | 2 +- .../lease_it/static/js/instances.js | 59 +++++++++++++++++-- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/openstack_lease_it/lease_it/static/js/database.js b/openstack_lease_it/lease_it/static/js/database.js index 76913af..dddaacd 100644 --- a/openstack_lease_it/lease_it/static/js/database.js +++ b/openstack_lease_it/lease_it/static/js/database.js @@ -49,7 +49,7 @@ function buildDatabaseRowMenu(data) { var menu = '' + 'chevron_right ' + ' '; return menu; diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index e1c9289..4b2dba8 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -16,14 +16,14 @@ const MAX_STRING_LENGTH = 30; /* buildInstancesView create a full display of Instance on div_name */ -function buildInstancesView(div_name, get_option, show_user){ +function buildInstancesView(div_name, get_option, is_admin){ var table_columns = [ { data: 'name' }, { data: 'project' }, { data: 'created_at' }, { data: 'lease_end' } ]; - if (show_user) { + if (is_admin) { table_columns.unshift({data: 'user'}); } $('#table-' + div_name).DataTable({ @@ -44,9 +44,14 @@ function buildInstancesView(div_name, get_option, show_user){ pageLength: 25, columnDefs: [ { - targets: [0, 1, 2], - render: function ( data, type, row ) { - return buildDatabaseRowMenu(data) + formatText(data, MAX_STRING_LENGTH); + targets: [0, 1, 2], + render: function ( data, type, row, meta ) { + if (meta.col == 0) { + return buildInstanceRowMenu(data, row, is_admin) + formatText(data, MAX_STRING_LENGTH); + } + else { + return formatText(data, MAX_STRING_LENGTH); + }; } }], drawCallback: function(settings, json) { @@ -87,3 +92,47 @@ function formatLeaseBtn(date, instance) { ' data-badge-caption="new lease" onClick="updateLease(\''+ instance + '\')">'; } + +/* + buildInstanceRowMenu build a menu for each row of Instance Table +*/ +function buildInstanceRowMenu(data, row, is_admin) { + if (is_admin) { + var menu = '' + + 'chevron_right ' + + ' '; + } + else { + var menu = '' + + 'chevron_right ' + + ' '; + + }; + return menu; +} + +/* + swapInstanceRowMenu swap on/off the delete button +*/ +function swapInstanceRowMenu(button) { + if ($('#instance-delete-' + button).css('display') == 'none') { + $('#instance-icon-' + button).text('chevron_left'); + } + if ($('#instance-admin-delete-' + button).css('display') == 'none') { + $('#instance-admin-icon-' + button).text('chevron_left'); + } + else { + $('#instance-icon-' + button).text('chevron_right'); + $('#instance-admin-icon-' + button).text('chevron_right'); + + } + $('#instance-delete-' + button).toggle(); + $('#instance-admin-delete-' + button).toggle(); +} + From 134992fd689db682727ec45b0758390124a37def Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 6 Jul 2021 17:14:33 +0200 Subject: [PATCH 34/38] Test button on instance view --- openstack_lease_it/lease_it/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openstack_lease_it/lease_it/views.py b/openstack_lease_it/lease_it/views.py index 201fdae..bd25984 100644 --- a/openstack_lease_it/lease_it/views.py +++ b/openstack_lease_it/lease_it/views.py @@ -9,6 +9,7 @@ from django.shortcuts import render from django.http import JsonResponse from django.contrib.auth.decorators import login_required +from django.core.cache import cache from lease_it import backend from lease_it.backend import Exceptions as bckExceptions # pylint: disable=ungrouped-imports @@ -165,6 +166,7 @@ def database(request, instance_id): # pylint: disable=unused-argument try: InstancesAccess.delete(instance_id) BACKEND.delete([{'id': instance_id}]) + cache.delete('instances') except StillRunning as error: response = { 'status': 'failed', From afb117f4508e7851ffb778b8d5b1e7dc0c319c3a Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 6 Jul 2021 17:15:57 +0200 Subject: [PATCH 35/38] Test button on instance view --- openstack_lease_it/lease_it/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstack_lease_it/lease_it/views.py b/openstack_lease_it/lease_it/views.py index bd25984..dbb3b95 100644 --- a/openstack_lease_it/lease_it/views.py +++ b/openstack_lease_it/lease_it/views.py @@ -166,7 +166,7 @@ def database(request, instance_id): # pylint: disable=unused-argument try: InstancesAccess.delete(instance_id) BACKEND.delete([{'id': instance_id}]) - cache.delete('instances') + cache.delete('instances') except StillRunning as error: response = { 'status': 'failed', From 8e27d649ec9e9a0fe5a79835dde89920e0f23bca Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Tue, 6 Jul 2021 17:29:14 +0200 Subject: [PATCH 36/38] Test button on instance view --- openstack_lease_it/lease_it/views.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openstack_lease_it/lease_it/views.py b/openstack_lease_it/lease_it/views.py index dbb3b95..cef2d1a 100644 --- a/openstack_lease_it/lease_it/views.py +++ b/openstack_lease_it/lease_it/views.py @@ -150,7 +150,7 @@ def databases(request): # pylint: disable=unused-argument return JsonResponse(response, safe=False) -@superuser_required +@login_required def database(request, instance_id): # pylint: disable=unused-argument """ This view is used to delete instance from database @@ -164,9 +164,12 @@ def database(request, instance_id): # pylint: disable=unused-argument 'instance': {'id': instance_id} } try: - InstancesAccess.delete(instance_id) - BACKEND.delete([{'id': instance_id}]) - cache.delete('instances') + # We retrieve data from backend + user_instances = BACKEND.instances(request, filtered) + if request.user.is_superuser or instance_id in [user_instance['id'] for user_instance in user_instances]: + InstancesAccess.delete(instance_id) + BACKEND.delete([{'id': instance_id}]) + cache.delete('instances') except StillRunning as error: response = { 'status': 'failed', From ee93227275ec2b8f346de564827d5c22dc922b2a Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Thu, 8 Jul 2021 15:31:06 +0200 Subject: [PATCH 37/38] Test button on instance view --- .../lease_it/static/js/instances.js | 20 ++++++++++--------- openstack_lease_it/lease_it/views.py | 5 +++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index 4b2dba8..976666d 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -46,7 +46,9 @@ function buildInstancesView(div_name, get_option, is_admin){ { targets: [0, 1, 2], render: function ( data, type, row, meta ) { - if (meta.col == 0) { + var now = new Date(); + var lease_end = new Date(row.lease_end); + if (meta.col == 0 && lease_end < now - HEARTBEAT_TIMEOUT) { return buildInstanceRowMenu(data, row, is_admin) + formatText(data, MAX_STRING_LENGTH); } else { @@ -105,15 +107,15 @@ function buildInstanceRowMenu(data, row, is_admin) { 'onClick="deleteDatabase(\'' + row.id + '\')">' + 'delete '; } - else { - var menu = '' + - 'chevron_right ' + - ' '; + else { + var menu = '' + + 'chevron_right ' + + ' '; - }; + }; return menu; } diff --git a/openstack_lease_it/lease_it/views.py b/openstack_lease_it/lease_it/views.py index cef2d1a..7e51e4f 100644 --- a/openstack_lease_it/lease_it/views.py +++ b/openstack_lease_it/lease_it/views.py @@ -165,11 +165,12 @@ def database(request, instance_id): # pylint: disable=unused-argument } try: # We retrieve data from backend - user_instances = BACKEND.instances(request, filtered) - if request.user.is_superuser or instance_id in [user_instance['id'] for user_instance in user_instances]: + user_instances = BACKEND.instances(request, True) + if request.user.is_superuser or instance_id in [k for k in user_instances]: InstancesAccess.delete(instance_id) BACKEND.delete([{'id': instance_id}]) cache.delete('instances') + except StillRunning as error: response = { 'status': 'failed', From 3a506a85b914b5b7cda537bb4fd2c7cb4a24a7f1 Mon Sep 17 00:00:00 2001 From: Yann Renard Date: Fri, 30 Jul 2021 15:23:38 +0200 Subject: [PATCH 38/38] Button on instance view --- openstack_lease_it/lease_it/datastore/ModelAccess.py | 7 ++++--- openstack_lease_it/lease_it/static/js/database.js | 9 ++++++--- openstack_lease_it/lease_it/static/js/instances.js | 4 ++-- openstack_lease_it/lease_it/views.py | 5 +++++ 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/openstack_lease_it/lease_it/datastore/ModelAccess.py b/openstack_lease_it/lease_it/datastore/ModelAccess.py index 600ba44..968de2e 100644 --- a/openstack_lease_it/lease_it/datastore/ModelAccess.py +++ b/openstack_lease_it/lease_it/datastore/ModelAccess.py @@ -137,9 +137,10 @@ def delete(instance_id): """ try: model = Instances.objects.get(id=instance_id) # pylint: disable=no-member - if model.leased_at + relativedelta(days=+model.lease_duration) >\ - timezone.now().date(): - raise StillRunning(model.id, model.heartbeat_at) + # To let the users delete their own instances, we have to disable the StillRunning error + #if model.leased_at + relativedelta(days=+model.lease_duration) >\ + # timezone.now().date(): + # raise StillRunning(model.id, model.heartbeat_at) model.delete() except ObjectDoesNotExist: LOGGER_INSTANCES.info('Instance %s does not exist', instance_id) diff --git a/openstack_lease_it/lease_it/static/js/database.js b/openstack_lease_it/lease_it/static/js/database.js index dddaacd..b21aba3 100644 --- a/openstack_lease_it/lease_it/static/js/database.js +++ b/openstack_lease_it/lease_it/static/js/database.js @@ -70,9 +70,12 @@ function swapDatabaseRowMenu(button) { /* deleteDatabase delete entry in database */ -function deleteDatabase(instance) { - return $.getJSON("/database/" + instance, function(data){ +function deleteDatabase(id) { + confirm("Instance of id :" + id + " will be deleted") + return $.getJSON("/database/" + id, function(data){ }).success(function(data){ - notify(data); + var notification = data; + notification["instance"]["id"] = "Instance of id : " + notification["instance"]["id"] + " has been deleted"; + notify(notification); }); } \ No newline at end of file diff --git a/openstack_lease_it/lease_it/static/js/instances.js b/openstack_lease_it/lease_it/static/js/instances.js index 976666d..020a07c 100644 --- a/openstack_lease_it/lease_it/static/js/instances.js +++ b/openstack_lease_it/lease_it/static/js/instances.js @@ -47,8 +47,8 @@ function buildInstancesView(div_name, get_option, is_admin){ targets: [0, 1, 2], render: function ( data, type, row, meta ) { var now = new Date(); - var lease_end = new Date(row.lease_end); - if (meta.col == 0 && lease_end < now - HEARTBEAT_TIMEOUT) { + var lease_end = new Date(row.lease_end.slice(0,10)); + if (meta.col == 0 && (div_name == "admin-instances" || div_name == "instances")) { return buildInstanceRowMenu(data, row, is_admin) + formatText(data, MAX_STRING_LENGTH); } else { diff --git a/openstack_lease_it/lease_it/views.py b/openstack_lease_it/lease_it/views.py index 7e51e4f..ddeaded 100644 --- a/openstack_lease_it/lease_it/views.py +++ b/openstack_lease_it/lease_it/views.py @@ -170,6 +170,11 @@ def database(request, instance_id): # pylint: disable=unused-argument InstancesAccess.delete(instance_id) BACKEND.delete([{'id': instance_id}]) cache.delete('instances') + else : + response = { + 'status': 'not allowed', + 'instance': {'id': instance_id} + } except StillRunning as error: response = {