From b79df685ec7c3edbed16af58a4be41564f8e7044 Mon Sep 17 00:00:00 2001 From: Konstantin Pauly Date: Mon, 18 Dec 2023 11:29:50 +0100 Subject: [PATCH 1/2] Skip instance_type selection if type is already set --- igvm/vm.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/igvm/vm.py b/igvm/vm.py index e389737a..a4241004 100644 --- a/igvm/vm.py +++ b/igvm/vm.py @@ -28,7 +28,7 @@ from fabric.contrib.files import upload_template from fabric.exceptions import NetworkError from json.decoder import JSONDecodeError -from urllib.error import HTTPError +from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen from igvm.exceptions import ConfigError, HypervisorError, RemoteCommandError, VMError @@ -713,12 +713,21 @@ def aws_build(self, :raises: VMError: Generic exception for VM errors of all kinds """ - vm_types_overview = self.aws_get_instances_overview() - if vm_types_overview: - vm_types = self.aws_get_fitting_vm_types(vm_types_overview) + # The current solution for figuring out the best instance_type is not + # scalable for the disaster recovery case because we are parsing a + # 70 MB big json file in parallel for every VM. We are currently working + # on a different solution to prefill the instance_type for the disaster + # recovery case. We'll keep the functionality in igvm for now to be able + # to build VMs in AWS without the need of a prefill. + if self.dataset_obj['aws_instance_type']: + vm_types = [self.dataset_obj['aws_instance_type']] else: - vm_types = [AWS_FALLBACK_INSTANCE_TYPE] - self.dataset_obj['aws_instance_type'] = vm_types[0] + vm_types_overview = self.aws_get_instances_overview() + if vm_types_overview: + vm_types = self.aws_get_fitting_vm_types(vm_types_overview) + else: + vm_types = [AWS_FALLBACK_INSTANCE_TYPE] + self.dataset_obj['aws_instance_type'] = vm_types[0] self.check_serveradmin_config() @@ -1221,7 +1230,7 @@ def aws_get_instances_overview( f.write(content) return json.loads(content) - except (HTTPError, JSONDecodeError) as e: + except (HTTPError, JSONDecodeError, URLError) as e: log.warning('Could not retrieve instances overview') log.warning(e) log.info('Proceeding with instance_type: ' From 6139c3c3f3bee6a291cca4612a977e7cd7ee1dc5 Mon Sep 17 00:00:00 2001 From: Konstantin Pauly Date: Thu, 21 Dec 2023 07:33:26 +0100 Subject: [PATCH 2/2] Move aws_get_instances_overview to utils --- igvm/utils.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ igvm/vm.py | 66 ++++----------------------------------------------- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/igvm/utils.py b/igvm/utils.py index 8370a0e4..46a000fb 100644 --- a/igvm/utils.py +++ b/igvm/utils.py @@ -5,15 +5,27 @@ from __future__ import division +import json import logging import socket import time from concurrent import futures +from json import JSONDecodeError from os import path +from pathlib import Path +from typing import Union, List +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen from paramiko import SSHConfig from igvm.exceptions import TimeoutError +from igvm.settings import ( + AWS_INSTANCES_OVERVIEW_URL, + AWS_INSTANCES_OVERVIEW_FILE, + AWS_INSTANCES_OVERVIEW_FILE_ETAG, + AWS_FALLBACK_INSTANCE_TYPE +) _SIZE_FACTORS = { 'T': 1024 ** 4, @@ -219,3 +231,54 @@ def parallel( results.append(result) return results + + +def aws_get_instances_overview(timeout: int = 5) -> Union[List, None]: + """AWS Get Instances Overview + + Load or download the latest instances.json, which contains + a complete overview about all instance_types, their configuration, + performance and pricing. + + :param: timeout: Timeout value for the head/get request + + :return: VM types overview as list + or None, if the parsing/download failed + """ + + url = AWS_INSTANCES_OVERVIEW_URL + file = Path.home() / AWS_INSTANCES_OVERVIEW_FILE + etag_file = Path.home() / AWS_INSTANCES_OVERVIEW_FILE_ETAG + + try: + head_req = Request(url, method='HEAD') + resp = urlopen(head_req, timeout=timeout) + if resp.status == 200: + etag = dict(resp.info())['ETag'] + else: + log.warning('Could not retrieve ETag from {}'.format(url)) + etag = None + if file.exists() and etag_file.exists() and etag: + with open(etag_file, 'r+') as f: + prev_etag = f.read() + if etag == prev_etag: + with open(file, 'r+') as f: + return json.load(f) + + resp = urlopen(url, timeout=timeout) + if etag: + with open(etag_file, 'w+') as f: + f.write(etag) + with open(file, 'w+') as f: + content = resp.read().decode('utf-8') + f.write(content) + + return json.loads(content) + except (HTTPError, JSONDecodeError, URLError) as e: + log.warning('Could not retrieve instances overview') + log.warning(e) + log.info('Proceeding with instance_type: ' + f'{AWS_FALLBACK_INSTANCE_TYPE}' + ) + + return None diff --git a/igvm/vm.py b/igvm/vm.py index a4241004..9d9ba3ef 100644 --- a/igvm/vm.py +++ b/igvm/vm.py @@ -17,9 +17,8 @@ from grp import getgrnam from hashlib import sha1, sha256 from io import BytesIO -from pathlib import Path from re import compile as re_compile -from typing import Optional, List, Union +from typing import Optional, List from uuid import uuid4 import boto3 @@ -27,9 +26,6 @@ from fabric.api import cd, get, hide, put, run, settings from fabric.contrib.files import upload_template from fabric.exceptions import NetworkError -from json.decoder import JSONDecodeError -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen from igvm.exceptions import ConfigError, HypervisorError, RemoteCommandError, VMError from igvm.host import Host @@ -37,12 +33,9 @@ AWS_ECU_FACTOR, AWS_FALLBACK_INSTANCE_TYPE, AWS_RETURN_CODES, - AWS_INSTANCES_OVERVIEW_FILE, - AWS_INSTANCES_OVERVIEW_FILE_ETAG, - AWS_INSTANCES_OVERVIEW_URL, ) from igvm.transaction import Transaction -from igvm.utils import parse_size, wait_until +from igvm.utils import parse_size, wait_until, aws_get_instances_overview from igvm.puppet import clean_cert if typing.TYPE_CHECKING: @@ -722,7 +715,7 @@ def aws_build(self, if self.dataset_obj['aws_instance_type']: vm_types = [self.dataset_obj['aws_instance_type']] else: - vm_types_overview = self.aws_get_instances_overview() + vm_types_overview = aws_get_instances_overview() if vm_types_overview: vm_types = self.aws_get_fitting_vm_types(vm_types_overview) else: @@ -1169,7 +1162,7 @@ def performance_value(self) -> float: :return: performance_value of VM as float """ - # Serveradmin can not handle floats right now so we safe them as + # Serveradmin can not handle floats right now so we save them as # multiple ones of thousand and just divide them here again. vm_load_99 = self.dataset_obj['load_99'] / 1000 # Default 0 vm_num_cpu = self.dataset_obj['num_cpu'] @@ -1188,57 +1181,6 @@ def performance_value(self) -> float: return float(estimated_load) - def aws_get_instances_overview( - self, timeout: int = 5) -> Union[List, None]: - """AWS Get Instances Overview - - Load or download the latest instances.json, which contains - a complete overview about all instance_types, their configuration, - performance and pricing. - - :param: timeout: Timeout value for the head/get request - - :return: VM types overview as list - or None, if the parsing/download failed - """ - - url = AWS_INSTANCES_OVERVIEW_URL - file = Path.home() / AWS_INSTANCES_OVERVIEW_FILE - etag_file = Path.home() / AWS_INSTANCES_OVERVIEW_FILE_ETAG - - try: - head_req = Request(url, method='HEAD') - resp = urlopen(head_req, timeout=timeout) - if resp.status == 200: - etag = dict(resp.info())['ETag'] - else: - log.warning('Could not retrieve ETag from {}'.format(url)) - etag = None - if file.exists() and etag_file.exists() and etag: - with open(etag_file, 'r+') as f: - prev_etag = f.read() - if etag == prev_etag: - with open(file, 'r+') as f: - return json.load(f) - - resp = urlopen(url, timeout=timeout) - if etag: - with open(etag_file, 'w+') as f: - f.write(etag) - with open(file, 'w+') as f: - content = resp.read().decode('utf-8') - f.write(content) - - return json.loads(content) - except (HTTPError, JSONDecodeError, URLError) as e: - log.warning('Could not retrieve instances overview') - log.warning(e) - log.info('Proceeding with instance_type: ' - f'{AWS_FALLBACK_INSTANCE_TYPE}' - ) - - return None - def aws_get_fitting_vm_types(self, overview: List) -> List: """AWS Get Fitting VM types