From 7f5f58612ccc4b88cb12fd51f62cbb3b55255cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 Mar 2023 10:37:06 -0500 Subject: [PATCH 01/23] Define an API to retrieve available domains --- .../src/components/cluster/ClusterEditor.vue | 36 +++++++++++-------- .../src/repositories/DomainsRepository.js | 9 +++++ mchub/__init__.py | 12 +++++-- mchub/models/cloud/cloud_manager.py | 10 ++---- mchub/resources/api_view.py | 2 +- mchub/resources/available_resources_api.py | 4 +-- mchub/resources/domains_api.py | 9 +++++ mchub/resources/magic_castle_api.py | 4 +-- mchub/resources/progress_api.py | 4 +-- mchub/resources/project_api.py | 4 +-- mchub/resources/template_api.py | 4 +-- mchub/resources/user_api.py | 4 +-- 12 files changed, 65 insertions(+), 37 deletions(-) create mode 100644 frontend/src/repositories/DomainsRepository.js create mode 100644 mchub/resources/domains_api.py diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index 6e28f280..fffa6786 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -28,12 +28,7 @@ /> - + @@ -306,6 +301,7 @@ import ResourceUsageDisplay from "@/components/ui/ResourceUsageDisplay"; import TypeSelect from "./TypeSelect"; import CodeEditor from "@/components/ui/CodeEditor"; import AvailableResourcesRepository from "@/repositories/AvailableResourcesRepository"; +import DomainsRepository from "@/repositories/DomainsRepository"; import ProjectRepository from "@/repositories/ProjectRepository"; import UserRepository from "@/repositories/UserRepository"; @@ -359,6 +355,7 @@ export default { promise: null, quotas: null, possibleResources: null, + domains: null, resourceDetails: null, projects: [], }; @@ -367,18 +364,19 @@ export default { promise() { this.$emit("loading", this.loading); }, - possibleResources(possibleResources) { + domains(domains) { // We set default values for select boxes based on possible resources fetched from the API // Domain if (this.localSpecs.domain === null) { try { - this.localSpecs.domain = possibleResources.domain[0]; - this.initialSpecs.domain = possibleResources.domain[0]; + this.localSpecs.domain = domains[0]; + this.initialSpecs.domain = domains[0]; } catch (err) { console.log("No domain available"); } } - + }, + possibleResources(possibleResources) { // Image if (this.localSpecs.image === null) { try { @@ -497,10 +495,7 @@ export default { return true; }, domainRule() { - return ( - (this.possibleResources && this.possibleResources.domain.includes(this.localSpecs.domain)) || - "Invalid domain provided" - ); + return (this.domains && this.domains.includes(this.localSpecs.domain)) || "Invalid domain provided"; }, volumeCountRule() { return this.volumeCountUsed <= this.volumeCountMax || "Volume number quota exceeded"; @@ -654,6 +649,13 @@ export default { return fieldPath.split(".").reduce((acc, x) => acc[x], this.possibleResources); } }, + getPossibleDomains() { + if (this.domains === null) { + return []; + } else { + return this.domains; + } + }, getInstanceDetail(instanceType, detailName, defaultValue = 0) { const matchingInstances = this.resourceDetails.instance_types.filter( (instanceTypeDetails) => instanceTypeDetails.name === instanceType @@ -808,6 +810,12 @@ export default { this.resourceDetails = data.resource_details; this.promise = null; }); + this.promise_dns = DomainsRepository.getDomains(); + this.promise_dns.then((response) => { + const data = response.data; + this.domains = data.domains; + this.promise_dns = null; + }); return this.promise; }, }, diff --git a/frontend/src/repositories/DomainsRepository.js b/frontend/src/repositories/DomainsRepository.js new file mode 100644 index 00000000..6f9f4f10 --- /dev/null +++ b/frontend/src/repositories/DomainsRepository.js @@ -0,0 +1,9 @@ +import Repository from "./Repository"; + +const resource = "/domains"; + +export default { + getDomains() { + return Repository.get(`${resource}/`); + }, +}; diff --git a/mchub/__init__.py b/mchub/__init__.py index 19ece202..f49593be 100644 --- a/mchub/__init__.py +++ b/mchub/__init__.py @@ -10,7 +10,8 @@ def create_app(db_path=None): from .database import db from .resources.magic_castle_api import MagicCastleAPI from .resources.progress_api import ProgressAPI - from .resources.available_resources_api import AvailableResourcesApi + from .resources.available_resources_api import AvailableResourcesAPI + from .resources.domains_api import DomainsAPI from .resources.user_api import UserAPI from .resources.project_api import ProjectAPI from .resources.template_api import TemplateAPI @@ -54,7 +55,14 @@ def create_app(db_path=None): methods=["GET"], ) - available_resources_view = AvailableResourcesApi.as_view("available_resources") + domains_view = DomainsAPI.as_view("domains") + app.add_url_rule( + "/api/domains/", + view_func=domains_view, + methods=["GET"], + ) + + available_resources_view = AvailableResourcesAPI.as_view("available_resources") app.add_url_rule( "/api/available-resources/host/", view_func=available_resources_view, diff --git a/mchub/models/cloud/cloud_manager.py b/mchub/models/cloud/cloud_manager.py index a8b83a76..4842448e 100644 --- a/mchub/models/cloud/cloud_manager.py +++ b/mchub/models/cloud/cloud_manager.py @@ -1,5 +1,4 @@ from ..cloud.openstack_manager import OpenStackManager -from ..cloud.dns_manager import DnsManager MANAGER_CLASSES = { "openstack": OpenStackManager, @@ -16,11 +15,6 @@ def __init__(self, project, **kwargs): @property def available_resources(self): """ - Retrieves the available cloud resources including resources from OpenStack - and available domains. + Retrieves the available cloud resources from the cloud provider. """ - available_resources = self.manager.available_resources - available_resources["possible_resources"][ - "domain" - ] = DnsManager.get_available_domains() - return available_resources + return self.manager.available_resources diff --git a/mchub/resources/api_view.py b/mchub/resources/api_view.py index c3444299..517a7af3 100644 --- a/mchub/resources/api_view.py +++ b/mchub/resources/api_view.py @@ -126,7 +126,7 @@ def decorator(**kwargs): return decorator -class ApiView(MethodView): +class APIView(MethodView): """ Configures all child classes to use the default decorators on all route handlers. """ diff --git a/mchub/resources/available_resources_api.py b/mchub/resources/available_resources_api.py index 51f99bf1..3a73b6ac 100644 --- a/mchub/resources/available_resources_api.py +++ b/mchub/resources/available_resources_api.py @@ -1,4 +1,4 @@ -from ..resources.api_view import ApiView +from ..resources.api_view import APIView from ..models.cloud.cloud_manager import CloudManager from ..models.user import User from ..models.cloud.project import Project @@ -9,7 +9,7 @@ from ..database import db -class AvailableResourcesApi(ApiView): +class AvailableResourcesAPI(APIView): def get(self, user: User, hostname, cloud_id): if hostname: orm = db.session.execute( diff --git a/mchub/resources/domains_api.py b/mchub/resources/domains_api.py new file mode 100644 index 00000000..48dadd46 --- /dev/null +++ b/mchub/resources/domains_api.py @@ -0,0 +1,9 @@ +from ..resources.api_view import APIView +from ..models.cloud.dns_manager import DnsManager +from ..models.user import User + +class DomainsAPI(APIView): + def get(self, user: User): + return { + "domains": DnsManager.get_available_domains(), + } \ No newline at end of file diff --git a/mchub/resources/magic_castle_api.py b/mchub/resources/magic_castle_api.py index bf6adbc5..85b35141 100644 --- a/mchub/resources/magic_castle_api.py +++ b/mchub/resources/magic_castle_api.py @@ -1,6 +1,6 @@ from operator import imod from flask import request -from .api_view import ApiView +from .api_view import APIView from ..exceptions.invalid_usage_exception import ( ClusterNotFoundException, InvalidUsageException, @@ -11,7 +11,7 @@ from ..database import db -class MagicCastleAPI(ApiView): +class MagicCastleAPI(APIView): def get(self, user: User, hostname): if hostname: orm = db.session.execute( diff --git a/mchub/resources/progress_api.py b/mchub/resources/progress_api.py index 74e743d9..56ea5807 100644 --- a/mchub/resources/progress_api.py +++ b/mchub/resources/progress_api.py @@ -1,11 +1,11 @@ -from .api_view import ApiView +from .api_view import APIView from ..models.magic_castle.cluster_status_code import ClusterStatusCode from ..models.user import User from ..models.magic_castle.magic_castle import MagicCastleORM, MagicCastle from ..database import db -class ProgressAPI(ApiView): +class ProgressAPI(APIView): def get(self, user: User, hostname): orm = db.session.execute( db.select(MagicCastleORM).filter_by(hostname=hostname) diff --git a/mchub/resources/project_api.py b/mchub/resources/project_api.py index abdbc707..9facfe60 100644 --- a/mchub/resources/project_api.py +++ b/mchub/resources/project_api.py @@ -1,6 +1,6 @@ from flask import request -from .api_view import ApiView +from .api_view import APIView from ..database import db from ..models.user import User, UserORM from ..models.cloud.project import Project, Provider, ENV_VALIDATORS @@ -9,7 +9,7 @@ ) -class ProjectAPI(ApiView): +class ProjectAPI(APIView): def get(self, user: User, id: int = None): if id is not None: project = db.session.get(Project, id) diff --git a/mchub/resources/template_api.py b/mchub/resources/template_api.py index 20345dec..ba56981a 100644 --- a/mchub/resources/template_api.py +++ b/mchub/resources/template_api.py @@ -1,8 +1,8 @@ -from .api_view import ApiView +from .api_view import APIView from ..models.template import DEFAULT from ..models.user import User -class TemplateAPI(ApiView): +class TemplateAPI(APIView): def get(self, user: User, template_name): return DEFAULT diff --git a/mchub/resources/user_api.py b/mchub/resources/user_api.py index 20c8e366..191b0934 100644 --- a/mchub/resources/user_api.py +++ b/mchub/resources/user_api.py @@ -1,8 +1,8 @@ -from .api_view import ApiView +from .api_view import APIView from ..models.user import User -class UserAPI(ApiView): +class UserAPI(APIView): def get(self, user: User): return { "username": user.username, From e113a0c6fcbb753d7cb9a1cca27d2d800d32131b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 Mar 2023 12:44:21 -0500 Subject: [PATCH 02/23] Load domains from API only when the cluster does not exist --- .../src/components/cluster/ClusterEditor.vue | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index fffa6786..dc262f05 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -353,6 +353,7 @@ export default { nowDate: new Date().toISOString().slice(0, 10), tomorrowDate: new Date(new Date().getTime() + 24 * 60 * 60 * 1000).toISOString().slice(0, 10), promise: null, + promise_dns: null, quotas: null, possibleResources: null, domains: null, @@ -448,7 +449,7 @@ export default { }, computed: { loading() { - return this.promise !== null; + return this.promise !== null && this.promise_dns !== null; }, localSpecs: { get() { @@ -808,15 +809,15 @@ export default { this.possibleResources = data.possible_resources; this.quotas = data.quotas; this.resourceDetails = data.resource_details; - this.promise = null; }); - this.promise_dns = DomainsRepository.getDomains(); - this.promise_dns.then((response) => { - const data = response.data; - this.domains = data.domains; - this.promise_dns = null; - }); - return this.promise; + if (!this.existingCluster) { + this.promise_dns = DomainsRepository.getDomains(); + this.promise_dns.then((response) => { + const data = response.data; + this.domains = data.domains; + }); + } + return [this.promise, this.promise_dns]; }, }, }; From 1aca808b3135d6728bafcacde944e87e39faaea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 Mar 2023 12:59:37 -0500 Subject: [PATCH 03/23] Fix frontend test --- .../src/components/cluster/ClusterEditor.vue | 12 ++--- .../components/cluster/ClusterEditor.spec.js | 48 ++++++++++++------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index dc262f05..18c0607c 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -810,13 +810,11 @@ export default { this.quotas = data.quotas; this.resourceDetails = data.resource_details; }); - if (!this.existingCluster) { - this.promise_dns = DomainsRepository.getDomains(); - this.promise_dns.then((response) => { - const data = response.data; - this.domains = data.domains; - }); - } + this.promise_dns = DomainsRepository.getDomains(); + this.promise_dns.then((response) => { + const data = response.data; + this.domains = data.domains; + }); return [this.promise, this.promise_dns]; }, }, diff --git a/frontend/tests/unit/components/cluster/ClusterEditor.spec.js b/frontend/tests/unit/components/cluster/ClusterEditor.spec.js index 458c45c2..1be9e0c9 100644 --- a/frontend/tests/unit/components/cluster/ClusterEditor.spec.js +++ b/frontend/tests/unit/components/cluster/ClusterEditor.spec.js @@ -8,6 +8,10 @@ import { cloneDeep } from "lodash"; import moxios from 'moxios'; import Repository from "@/repositories/Repository"; +import { jest } from "@jest/globals"; + +jest.setTimeout(100); + Vue.use(Vuetify); const localVue = createLocalVue(); @@ -58,6 +62,9 @@ const DEFAULT_POSSIBLE_RESOURCES = Object.freeze({ tag_types: {"mgmt": ["p4-6gb", "c2-7.5gb-31"], "login": ["p2-3gb", "p4-6gb"], "node": ["p2-3gb", "p4-6gb"]}, "types": ["p1-1.5gb", "p2-3gb", "p4-6gb"], volumes: {}, +}); + +const DEFAULT_DOMAINS = Object.freeze({ domain: ["magic-castle.cloud", "mc.ca"] }); @@ -102,23 +109,30 @@ describe("ClusterEditor", () => { // import and pass your custom axios instance to this method moxios.install(Repository) moxios.wait(function () { - let request = moxios.requests.mostRecent(); - if(request.url.includes("/available-resources")) { - request.respondWith({ - status: 200, - response: { - 'possible_resources': DEFAULT_POSSIBLE_RESOURCES, - 'quotas': DEFAULT_QUOTAS, - 'resource_details': DEFAULT_RESOURCE_DETAILS - } - }) - } else if (request.url.includes("/users/me")) { - request.respondWith({ - status: 200, - response: DEFAULT_USER - }) - } else { - console.log(request.url); + for(let i = 0; i < moxios.requests.count(); i++) { + let request = moxios.requests.at(i); + if(request.url.includes("/available-resources")) { + request.respondWith({ + status: 200, + response: { + 'possible_resources': DEFAULT_POSSIBLE_RESOURCES, + 'quotas': DEFAULT_QUOTAS, + 'resource_details': DEFAULT_RESOURCE_DETAILS + } + }) + } else if (request.url.includes("/users/me")) { + request.respondWith({ + status: 200, + response: DEFAULT_USER + }) + } else if (request.url.includes("/domains")) { + request.respondWith({ + status: 200, + response: DEFAULT_DOMAINS + }) + } else { + console.log(request.url); + } } }) }) From 933ae8bd65556722e24c9b52d3057ec3604c1f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 Mar 2023 13:07:57 -0500 Subject: [PATCH 04/23] Add missing await in frontend test --- frontend/tests/unit/components/cluster/ClusterEditor.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/tests/unit/components/cluster/ClusterEditor.spec.js b/frontend/tests/unit/components/cluster/ClusterEditor.spec.js index 1be9e0c9..094e0fe4 100644 --- a/frontend/tests/unit/components/cluster/ClusterEditor.spec.js +++ b/frontend/tests/unit/components/cluster/ClusterEditor.spec.js @@ -100,6 +100,7 @@ async function getDefaultClusterEditorWrapper(existingCluster=true, hostname="te } }); await wrapper.vm.promise; + await wrapper.vm.promise_dns; return wrapper; } From 4e9ccbbb32e8b9a6bdfbc8a08cc366c7336cf9ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 Mar 2023 13:30:12 -0500 Subject: [PATCH 05/23] Fix loading --- frontend/src/components/cluster/ClusterEditor.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index 18c0607c..8736bc5e 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -809,11 +809,13 @@ export default { this.possibleResources = data.possible_resources; this.quotas = data.quotas; this.resourceDetails = data.resource_details; + this.promise = null; }); this.promise_dns = DomainsRepository.getDomains(); this.promise_dns.then((response) => { const data = response.data; this.domains = data.domains; + this.promise_dns = null; }); return [this.promise, this.promise_dns]; }, From 0a9493c2f9f8ac44c47a880987b18dafaff7a8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 11:34:27 -0500 Subject: [PATCH 06/23] Use standard library cache decorator --- mchub/models/cloud/openstack_manager.py | 96 +++++++++---------------- 1 file changed, 35 insertions(+), 61 deletions(-) diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index a4f89394..98e14340 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -1,5 +1,6 @@ import openstack +from functools import cache from os import environ, path from re import match, IGNORECASE, compile @@ -35,18 +36,12 @@ class OpenStackManager: """ __slots__ = [ - "_con", - "_project_id", "project", "__pre_allocated_instance_count", "__pre_allocated_cores", "__pre_allocated_ram", "__pre_allocated_volume_count", "__pre_allocated_volume_size", - "_volume_quotas", - "_compute_quotas", - "_network_quotas", - "_available_flavors", ] def __init__( @@ -59,8 +54,6 @@ def __init__( pre_allocated_volume_count=0, pre_allocated_volume_size=0, ): - self._con = None - self._project_id = None self.project = project self.__pre_allocated_instance_count = pre_allocated_instance_count self.__pre_allocated_cores = pre_allocated_cores @@ -68,27 +61,19 @@ def __init__( self.__pre_allocated_volume_count = pre_allocated_volume_count self.__pre_allocated_volume_size = pre_allocated_volume_size - self._volume_quotas = None - self._compute_quotas = None - self._network_quotas = None - - self._available_flavors = None @property + @cache def connection(self): - if self._con is None: - # Convert OS_* environment variable in keyword arguments - kargs = {key[3:].lower(): value for key, value in self.project.env.items()} - kargs["auth_type"] = "v3applicationcredential" - self._con = openstack.connect(**kargs) - - return self._con + # Convert OS_* environment variable in keyword arguments + kargs = {key[3:].lower(): value for key, value in self.project.env.items()} + kargs["auth_type"] = "v3applicationcredential" + return openstack.connect(**kargs) @property + @cache def project_id(self): - if self._project_id is None: - self._project_id = self.connection.current_project_id - return self._project_id + return self.connection.current_project_id @property def env(self): @@ -149,6 +134,7 @@ def resource_details(self): } @property + @cache def images(self): images = [] for image in self.connection.image.images(): @@ -160,11 +146,11 @@ def images(self): return [image[1] for image in images] @property + @cache def available_flavors(self): - if self._available_flavors is None: - self._available_flavors = list(self.connection.compute.flavors()) - self._available_flavors.sort(key=lambda flavor: (flavor.ram, flavor.vcpus)) - return self._available_flavors + available_flavors = list(self.connection.compute.flavors()) + available_flavors.sort(key=lambda flavor: (flavor.ram, flavor.vcpus)) + return available_flavors @property def available_tags(self): @@ -223,46 +209,34 @@ def available_floating_ip_count(self): ) @property + @cache def volume_quotas(self): - if self._volume_quotas is None: - # Normally, we should use self.__connection.get_volume_quotas(...) from openstack sdk. - # However, this method executes the action - # identity:list_projects from the identity api which is forbidden - # to some users. - # - # API documentation: - # https://docs.openstack.org/api-ref/block-storage/v3/index.html?expanded=show-quotas-for-a-project-detail#show-quotas-for-a-project - self._volume_quotas = self.connection.block_storage.get( - f"/os-quota-sets/{self.project_id}?usage=true" + # Normally, we should use self.__connection.get_volume_quotas(...) from openstack sdk. + # However, this method executes the action + # identity:list_projects from the identity api which is forbidden + # to some users. + # + # API documentation: + # https://docs.openstack.org/api-ref/block-storage/v3/index.html?expanded=show-quotas-for-a-project-detail#show-quotas-for-a-project + return self.connection.block_storage.get( + f"/os-quota-sets/{self.project_id}?usage=true" ).json()["quota_set"] - return self._volume_quotas @property + @cache def compute_quotas(self): - if self._compute_quotas is None: - # Normally, we should use self.__connection.get_compute_quotas(...) from openstack sdk. - # However, this method executes the action - # identity:list_projects from the identity api which is forbidden - # to some users. - # - # API documentation: - # https://docs.openstack.org/api-ref/compute/?expanded=show-a-quota-detail#show-a-quota - self._compute_quotas = self.connection.compute.get( - f"/os-quota-sets/{self.project_id}/detail" + # Normally, we should use self.__connection.get_compute_quotas(...) from openstack sdk. + # However, this method executes the action + # identity:list_projects from the identity api which is forbidden + # to some users. + # + # API documentation: + # https://docs.openstack.org/api-ref/compute/?expanded=show-a-quota-detail#show-a-quota + return self.connection.compute.get( + f"/os-quota-sets/{self.project_id}/detail" ).json()["quota_set"] - return self._compute_quotas @property + @cache def network_quotas(self): - if self._network_quotas is None: - # Normally, we should use self.__connection.get_network_quotas(...) from openstack sdk. - # However, this method executes the action - # identity:list_projects from the identity api which is forbidden - # to some users. - # - # API documentation: - # https://docs.openstack.org/api-ref/network/v2/?expanded=show-quota-details-for-a-tenant-detail#show-quota-details-for-a-tenant - self._network_quotas = self.connection.network.get( - f"/quotas/{self.project_id}/details.json" - ).json()["quota"] - return self._network_quotas + return self.connection.network.get_quota(self.project_id, details=True) From bdf2358fc5db04cb5a327b5e04396ec3adaee54c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 13:38:58 -0500 Subject: [PATCH 07/23] Simplify pre_allocated logic --- mchub/models/cloud/cloud_manager.py | 26 ++++++++++++--- mchub/models/cloud/openstack_manager.py | 33 ++++++-------------- mchub/models/magic_castle/magic_castle.py | 18 ++++------- mchub/resources/available_resources_api.py | 19 +++-------- tests/unit/magic_castle/test_magic_castle.py | 30 +++++++++--------- 5 files changed, 56 insertions(+), 70 deletions(-) diff --git a/mchub/models/cloud/cloud_manager.py b/mchub/models/cloud/cloud_manager.py index 4842448e..d84db3cb 100644 --- a/mchub/models/cloud/cloud_manager.py +++ b/mchub/models/cloud/cloud_manager.py @@ -5,12 +5,15 @@ } class CloudManager: - def __init__(self, project, **kwargs): - manager_class = MANAGER_CLASSES.get(project.provider) - if manager_class: - self.manager = manager_class(project=project, **kwargs) + def __init__(self, project, allocated_resources): + if project: + manager_class = MANAGER_CLASSES.get(project.provider) + if manager_class: + self.manager = manager_class(project, allocated_resources) + else: + raise ValueError(f"Unknown cloud provider {project.provider}") else: - raise ValueError("Invalid cloud provider") + self.manager = DefaultCloudManager(project, allocated_resources) @property def available_resources(self): @@ -18,3 +21,16 @@ def available_resources(self): Retrieves the available cloud resources from the cloud provider. """ return self.manager.available_resources + +class DefaultCloudManager: + def __init__(self, project, allocated_resources): + self.project = project + self.allocated_resources = allocated_resources + + @property + def available_resources(self): + return { + "quotas": {}, + "possible_resources": {}, + "resource_details": {}, + } diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index 98e14340..fa6fc893 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -37,30 +37,16 @@ class OpenStackManager: __slots__ = [ "project", - "__pre_allocated_instance_count", - "__pre_allocated_cores", - "__pre_allocated_ram", - "__pre_allocated_volume_count", - "__pre_allocated_volume_size", + "allocated_resources", ] def __init__( self, project, - *, - pre_allocated_instance_count=0, - pre_allocated_cores=0, - pre_allocated_ram=0, - pre_allocated_volume_count=0, - pre_allocated_volume_size=0, + allocated_resources, ): self.project = project - self.__pre_allocated_instance_count = pre_allocated_instance_count - self.__pre_allocated_cores = pre_allocated_cores - self.__pre_allocated_ram = pre_allocated_ram - self.__pre_allocated_volume_count = pre_allocated_volume_count - self.__pre_allocated_volume_size = pre_allocated_volume_size - + self.allocated_resources = allocated_resources @property @cache @@ -164,7 +150,7 @@ def available_tags(self): @property def available_instance_count(self): return ( - self.__pre_allocated_instance_count + self.allocated_resources.get("instance_count", 0) + self.compute_quotas["instances"]["limit"] - self.compute_quotas["instances"]["in_use"] ) @@ -172,7 +158,7 @@ def available_instance_count(self): @property def available_ram(self): return ( - self.__pre_allocated_ram + self.allocated_resources.get("ram", 0) + self.compute_quotas["ram"]["limit"] - self.compute_quotas["ram"]["in_use"] ) @@ -180,7 +166,7 @@ def available_ram(self): @property def available_vcpus(self): return ( - self.__pre_allocated_cores + self.allocated_resources.get("cores", 0) + self.compute_quotas["cores"]["limit"] - self.compute_quotas["cores"]["in_use"] ) @@ -188,7 +174,7 @@ def available_vcpus(self): @property def available_volume_count(self): return ( - self.__pre_allocated_volume_count + self.allocated_resources.get("volumes", 0) + self.volume_quotas["volumes"]["limit"] - self.volume_quotas["volumes"]["in_use"] ) @@ -196,7 +182,7 @@ def available_volume_count(self): @property def available_volume_size(self): return ( - self.__pre_allocated_volume_size + self.allocated_resources.get("volume_size", 0) + self.volume_quotas["gigabytes"]["limit"] - self.volume_quotas["gigabytes"]["in_use"] ) @@ -204,7 +190,8 @@ def available_volume_size(self): @property def available_floating_ip_count(self): return ( - self.network_quotas["floatingip"]["limit"] + self.allocated_resources.get("floatingip", 0) + + self.network_quotas["floatingip"]["limit"] - self.network_quotas["floatingip"]["used"] ) diff --git a/mchub/models/magic_castle/magic_castle.py b/mchub/models/magic_castle/magic_castle.py index 86ceffb5..f456b5ae 100644 --- a/mchub/models/magic_castle/magic_castle.py +++ b/mchub/models/magic_castle/magic_castle.py @@ -345,20 +345,14 @@ def allocated_resources(self): if self.tf_state is not None: return dict( - pre_allocated_instance_count=self.tf_state.instance_count, - pre_allocated_ram=self.tf_state.ram, - pre_allocated_cores=self.tf_state.cores, - pre_allocated_volume_count=self.tf_state.volume_count, - pre_allocated_volume_size=self.tf_state.volume_size, + instance_count=self.tf_state.instance_count, + ram=self.tf_state.ram, + cores=self.tf_state.cores, + volume_count=self.tf_state.volume_count, + volume_size=self.tf_state.volume_size, ) else: - return dict( - pre_allocated_instance_count=0, - pre_allocated_ram=0, - pre_allocated_cores=0, - pre_allocated_volume_count=0, - pre_allocated_volume_size=0, - ) + return {} @property def is_busy(self): diff --git a/mchub/resources/available_resources_api.py b/mchub/resources/available_resources_api.py index 3a73b6ac..e04449ca 100644 --- a/mchub/resources/available_resources_api.py +++ b/mchub/resources/available_resources_api.py @@ -21,20 +21,9 @@ def get(self, user: User, hostname, cloud_id): raise ClusterNotFoundException project = mc.project allocated_resources = mc.allocated_resources - elif cloud_id: + else: project = db.session.get(Project, cloud_id) - if project is None or project not in user.projects: - return { - "quotas": {}, - "possible_resources": {}, - "resource_details": {}, - } + if project not in user.projects: + project = None allocated_resources = {} - else: - return { - "quotas": {}, - "possible_resources": {}, - "resource_details": {}, - } - cloud = CloudManager(project=project, **allocated_resources) - return cloud.available_resources + return CloudManager(project, allocated_resources).available_resources diff --git a/tests/unit/magic_castle/test_magic_castle.py b/tests/unit/magic_castle/test_magic_castle.py index ac416987..6988c55e 100644 --- a/tests/unit/magic_castle/test_magic_castle.py +++ b/tests/unit/magic_castle/test_magic_castle.py @@ -255,11 +255,11 @@ def test_allocated_resources_valid(app): ) magic_castle = MagicCastle(orm=orm) assert magic_castle.allocated_resources == { - "pre_allocated_instance_count": 3, - "pre_allocated_ram": 15360, - "pre_allocated_cores": 10, - "pre_allocated_volume_count": 3, - "pre_allocated_volume_size": 200, + "instance_count": 3, + "ram": 15360, + "cores": 10, + "volume_count": 3, + "volume_size": 200, } @@ -283,11 +283,11 @@ def test_allocated_resources_missing_nodes(app): ) magic_castle = MagicCastle(orm=orm) assert magic_castle.allocated_resources == { - "pre_allocated_instance_count": 0, - "pre_allocated_ram": 0, - "pre_allocated_cores": 0, - "pre_allocated_volume_count": 3, - "pre_allocated_volume_size": 200, + "instance_count": 0, + "ram": 0, + "cores": 0, + "volume_count": 3, + "volume_size": 200, } @@ -301,9 +301,9 @@ def test_allocated_resources_not_found(app): magic_castle = MagicCastle() assert magic_castle.allocated_resources == { - "pre_allocated_instance_count": 0, - "pre_allocated_ram": 0, - "pre_allocated_cores": 0, - "pre_allocated_volume_count": 0, - "pre_allocated_volume_size": 0, + "instance_count": 0, + "ram": 0, + "cores": 0, + "volume_count": 0, + "volume_size": 0, } From 70af36796d32bdafd14ec783c4917c9ce72de110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 14:28:29 -0500 Subject: [PATCH 08/23] Reintegrate floatingip counting from terraform state --- frontend/src/components/cluster/ClusterEditor.vue | 7 ------- mchub/models/cloud/openstack_manager.py | 2 +- mchub/models/magic_castle/magic_castle.py | 1 + mchub/models/terraform/terraform_state.py | 9 +++++++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index 8736bc5e..114c9b17 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -616,13 +616,6 @@ export default { newPublicIP += self.localSpecs.instances[key].count; } } - if (self.initialSpecs) { - for (let key in self.initialSpecs.instances) { - if (self.initialSpecs.instances[key].tags.includes("public")) { - newPublicIP -= self.initialSpecs.instances[key].count; - } - } - } return newPublicIP <= self.ipsCountMax || "Public IP quota exceeded"; } return true; diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index fa6fc893..aa992b9d 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -190,7 +190,7 @@ def available_volume_size(self): @property def available_floating_ip_count(self): return ( - self.allocated_resources.get("floatingip", 0) + self.allocated_resources.get("public_ip", 0) + self.network_quotas["floatingip"]["limit"] - self.network_quotas["floatingip"]["used"] ) diff --git a/mchub/models/magic_castle/magic_castle.py b/mchub/models/magic_castle/magic_castle.py index f456b5ae..58c62acd 100644 --- a/mchub/models/magic_castle/magic_castle.py +++ b/mchub/models/magic_castle/magic_castle.py @@ -350,6 +350,7 @@ def allocated_resources(self): cores=self.tf_state.cores, volume_count=self.tf_state.volume_count, volume_size=self.tf_state.volume_size, + public_ip=self.tf_state.public_ip, ) else: return {} diff --git a/mchub/models/terraform/terraform_state.py b/mchub/models/terraform/terraform_state.py index 21cf006e..b70810bb 100644 --- a/mchub/models/terraform/terraform_state.py +++ b/mchub/models/terraform/terraform_state.py @@ -2,7 +2,7 @@ CLOUD_PARSER = { "openstack": { - "instance_count": parse( + "instances": parse( "resources[?type=openstack_compute_flavor_v2].instances[*].attributes.id" ), "cores": parse( @@ -17,6 +17,9 @@ "instance_volumes": parse( "resources[?type=openstack_compute_instance_v2].instances[*].attributes.block_device[*].volume_size" ), + "floatingip": parse( + "resources[?type=openstack_compute_floatingip_associate_v2].instances[*].attributes.id" + ), } } @@ -38,12 +41,13 @@ class TerraformState: "volume_count", "volume_size", "image", + "public_ip", "freeipa_passwd", ] def __init__(self, tf_state: object, cloud="openstack"): parser = CLOUD_PARSER[cloud] - self.instance_count = len(parser["instance_count"].find(tf_state)) + self.instance_count = len(parser["instances"].find(tf_state)) self.cores = sum([cores.value for cores in parser["cores"].find(tf_state)]) self.ram = sum([ram.value for ram in parser["ram"].find(tf_state)]) @@ -53,6 +57,7 @@ def __init__(self, tf_state: object, cloud="openstack"): self.volume_size = sum(vol.value for vol in volumes) + sum( vol.value for vol in inst_volumes ) + self.public_ip = len(parser["floatingip"].find(tf_state)) try: self.image = IMAGE_PARSER.find(tf_state)[0].value From bfb1780eb1fbb33e605ed802c43ca207ff2a06ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 14:31:33 -0500 Subject: [PATCH 09/23] Rename available_floating_ip_count to available_floating_ip --- mchub/models/cloud/openstack_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index aa992b9d..5fc6b127 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -81,7 +81,7 @@ def quotas(self): "vcpus": {"max": self.available_vcpus}, "volume_count": {"max": self.available_volume_count}, "volume_size": {"max": self.available_volume_size}, - "ips": {"max": self.available_floating_ip_count}, + "ips": {"max": self.available_floating_ip}, } @property @@ -188,7 +188,7 @@ def available_volume_size(self): ) @property - def available_floating_ip_count(self): + def available_floating_ip(self): return ( self.allocated_resources.get("public_ip", 0) + self.network_quotas["floatingip"]["limit"] From fb5358c3c02ad5187ae6a89d675192aff5022dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 14:36:10 -0500 Subject: [PATCH 10/23] Remove "max" from quota and return value directly --- frontend/src/components/cluster/ClusterEditor.vue | 12 ++++++------ mchub/models/cloud/openstack_manager.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index 114c9b17..fcc4e5b8 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -508,10 +508,10 @@ export default { return this.usedResourcesLoaded ? this.instances.reduce((acc, instance) => acc + instance.count, 0) : 0; }, instanceCountMax() { - return this.quotas ? this.quotas.instance_count.max : 0; + return this.quotas ? this.quotas.instance_count : 0; }, ipsCountMax() { - return this.quotas ? this.quotas.ips.max : 0; + return this.quotas ? this.quotas.ips : 0; }, ramRule() { return this.ramGbUsed <= this.ramGbMax || "Ram quota exceeded"; @@ -528,7 +528,7 @@ export default { : 0; }, ramGbMax() { - return this.quotas ? this.quotas.ram.max / MB_PER_GB : 0; + return this.quotas ? this.quotas.ram / MB_PER_GB : 0; }, vcpuUsed() { return this.usedResourcesLoaded @@ -539,7 +539,7 @@ export default { : 0; }, vcpuMax() { - return this.quotas ? this.quotas.vcpus.max : 0; + return this.quotas ? this.quotas.vcpus : 0; }, volumeCountUsed() { return this.usedResourcesLoaded @@ -550,7 +550,7 @@ export default { : 0; }, volumeCountMax() { - return this.quotas ? this.quotas.volume_count.max : 0; + return this.quotas ? this.quotas.volume_count : 0; }, volumeSizeUsed() { return this.usedResourcesLoaded @@ -559,7 +559,7 @@ export default { : 0; }, volumeSizeMax() { - return this.quotas ? this.quotas.volume_size.max : 0; + return this.quotas ? this.quotas.volume_size : 0; }, instancesVolumeSizeUsed() { return this.instances.reduce( diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index 5fc6b127..93243a2c 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -76,12 +76,12 @@ def available_resources(self): @property def quotas(self): return { - "instance_count": {"max": self.available_instance_count}, - "ram": {"max": self.available_ram}, - "vcpus": {"max": self.available_vcpus}, - "volume_count": {"max": self.available_volume_count}, - "volume_size": {"max": self.available_volume_size}, - "ips": {"max": self.available_floating_ip}, + "instance_count": self.available_instance_count, + "ram": self.available_ram, + "vcpus": self.available_vcpus, + "volume_count": self.available_volume_count, + "volume_size": self.available_volume_size, + "ips": self.available_floating_ip, } @property From bfb8a991eb15ff780bf1f6d420d0152fcde30130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 14:46:56 -0500 Subject: [PATCH 11/23] Add port and security group to OSManager --- mchub/models/cloud/openstack_manager.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index 93243a2c..7c4e7834 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -82,6 +82,8 @@ def quotas(self): "volume_count": self.available_volume_count, "volume_size": self.available_volume_size, "ips": self.available_floating_ip, + "ports": self.available_ports, + "security_groups": self.available_security_groups, } @property @@ -195,6 +197,22 @@ def available_floating_ip(self): - self.network_quotas["floatingip"]["used"] ) + @property + def available_ports(self): + return ( + self.allocated_resources.get("ports", 0) + + self.network_quotas["ports"]["limit"] + - self.network_quotas["ports"]["used"] + ) + + @property + def available_security_groups(self): + return ( + self.allocated_resources.get("security_groups", 0) + + self.network_quotas["security_groups"]["limit"] + - self.network_quotas["security_groups"]["used"] + ) + @property @cache def volume_quotas(self): From 78e724791caa6df485c8613e0f7b36896c70b4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 14:55:46 -0500 Subject: [PATCH 12/23] Add parsing of ports and security groups to tf parser --- mchub/models/terraform/terraform_state.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mchub/models/terraform/terraform_state.py b/mchub/models/terraform/terraform_state.py index b70810bb..764a0d8f 100644 --- a/mchub/models/terraform/terraform_state.py +++ b/mchub/models/terraform/terraform_state.py @@ -17,9 +17,15 @@ "instance_volumes": parse( "resources[?type=openstack_compute_instance_v2].instances[*].attributes.block_device[*].volume_size" ), - "floatingip": parse( + "public_ips": parse( "resources[?type=openstack_compute_floatingip_associate_v2].instances[*].attributes.id" ), + "ports": parse( + "resources[?type=openstack_networking_port_v2].instances[*].attributes.id" + ), + "security_groups": parse( + "resources[?type=openstack_compute_secgroup_v2].instances[*].attributes.id" + ), } } @@ -57,7 +63,10 @@ def __init__(self, tf_state: object, cloud="openstack"): self.volume_size = sum(vol.value for vol in volumes) + sum( vol.value for vol in inst_volumes ) - self.public_ip = len(parser["floatingip"].find(tf_state)) + + self.public_ip = len(parser["public_ips"].find(tf_state)) + self.ports = len(parser["ports"].find(tf_state)) + self.security_groups = len(parser["security_groups"].find(tf_state)) try: self.image = IMAGE_PARSER.find(tf_state)[0].value From 6a5c6359385f6a969d4656ef4a5873fe6f52e510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 15:08:03 -0500 Subject: [PATCH 13/23] Include ports in instance quota computation --- frontend/src/components/cluster/ClusterEditor.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index fcc4e5b8..3f19972c 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -508,7 +508,7 @@ export default { return this.usedResourcesLoaded ? this.instances.reduce((acc, instance) => acc + instance.count, 0) : 0; }, instanceCountMax() { - return this.quotas ? this.quotas.instance_count : 0; + return this.quotas ? Math.min(this.quotas.instance_count, this.quotas.ports) : 0; }, ipsCountMax() { return this.quotas ? this.quotas.ips : 0; From 97ca84f9906f01eed8d034fd7a03eaecca470fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 15:11:29 -0500 Subject: [PATCH 14/23] Fix tests --- mchub/models/terraform/terraform_state.py | 2 ++ tests/unit/magic_castle/test_magic_castle.py | 10 +++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/mchub/models/terraform/terraform_state.py b/mchub/models/terraform/terraform_state.py index 764a0d8f..8c234a36 100644 --- a/mchub/models/terraform/terraform_state.py +++ b/mchub/models/terraform/terraform_state.py @@ -49,6 +49,8 @@ class TerraformState: "image", "public_ip", "freeipa_passwd", + "ports", + "security_groups", ] def __init__(self, tf_state: object, cloud="openstack"): diff --git a/tests/unit/magic_castle/test_magic_castle.py b/tests/unit/magic_castle/test_magic_castle.py index 6988c55e..ed3be34f 100644 --- a/tests/unit/magic_castle/test_magic_castle.py +++ b/tests/unit/magic_castle/test_magic_castle.py @@ -256,6 +256,7 @@ def test_allocated_resources_valid(app): magic_castle = MagicCastle(orm=orm) assert magic_castle.allocated_resources == { "instance_count": 3, + "public_ip": 1, "ram": 15360, "cores": 10, "volume_count": 3, @@ -284,6 +285,7 @@ def test_allocated_resources_missing_nodes(app): magic_castle = MagicCastle(orm=orm) assert magic_castle.allocated_resources == { "instance_count": 0, + "public_ip": 1, "ram": 0, "cores": 0, "volume_count": 3, @@ -300,10 +302,4 @@ def test_allocated_resources_not_found(app): from mchub.models.magic_castle.magic_castle import MagicCastle magic_castle = MagicCastle() - assert magic_castle.allocated_resources == { - "instance_count": 0, - "ram": 0, - "cores": 0, - "volume_count": 0, - "volume_size": 0, - } + assert magic_castle.allocated_resources == {} From 2a11123c0afb9827e4e25ab5cce0d93e6f686ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 15:14:02 -0500 Subject: [PATCH 15/23] Fix frontend tests --- .../unit/components/cluster/ClusterEditor.spec.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frontend/tests/unit/components/cluster/ClusterEditor.spec.js b/frontend/tests/unit/components/cluster/ClusterEditor.spec.js index 094e0fe4..2c2565c7 100644 --- a/frontend/tests/unit/components/cluster/ClusterEditor.spec.js +++ b/frontend/tests/unit/components/cluster/ClusterEditor.spec.js @@ -69,12 +69,13 @@ const DEFAULT_DOMAINS = Object.freeze({ }); const DEFAULT_QUOTAS = Object.freeze({ - instance_count: { max: 115 }, - ram: { max: 221184 }, - vcpus: { max: 224 }, - volume_count: { max: 114 }, - volume_size: { max: 490 }, - ips: { max: 3 }, + instance_count: 115, + ram: 221184, + vcpus: 224, + ports: 200, + volume_count: 114, + volume_size: 490, + ips: 3, }); const DEFAULT_RESOURCE_DETAILS = Object.freeze({ From f48f59e8980b73ef6a425284ab5a83c6dc78fa64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 Mar 2023 16:00:33 -0500 Subject: [PATCH 16/23] Remove image from terraform state --- .../src/components/cluster/ClustersList.vue | 17 ---- mchub/models/magic_castle/magic_castle.py | 17 +--- mchub/models/terraform/terraform_state.py | 18 +---- tests/data/__init__.py | 13 --- .../terraform_plan.json | 80 ------------------- .../terraform_plan.log | 15 ---- .../terraform_apply.log | 2 - .../terraform_plan.json | 80 ------------------- .../missingnodes.mc.ca/terraform.tfstate | 30 ------- .../terraform.tfstate | 30 ------- .../terraform_apply.log | 17 ---- .../terraform.tfstate | 30 ------- .../terraform_apply.log | 2 - tests/unit/magic_castle/test_magic_castle.py | 4 + 14 files changed, 7 insertions(+), 348 deletions(-) diff --git a/frontend/src/components/cluster/ClustersList.vue b/frontend/src/components/cluster/ClustersList.vue index 8a073da4..369ea993 100644 --- a/frontend/src/components/cluster/ClustersList.vue +++ b/frontend/src/components/cluster/ClustersList.vue @@ -42,23 +42,6 @@ centos - - FreeIPA admin username - - - admin - - - FreeIPA admin password - - - not available - - Guest usernames diff --git a/mchub/models/magic_castle/magic_castle.py b/mchub/models/magic_castle/magic_castle.py index 58c62acd..e0ee681f 100644 --- a/mchub/models/magic_castle/magic_castle.py +++ b/mchub/models/magic_castle/magic_castle.py @@ -321,7 +321,6 @@ def state(self): **(self.applied_config if self.applied_config else self.config), "hostname": self.hostname, "status": self.status, - "freeipa_passwd": self.freeipa_passwd, "age": self.age, "expiration_date": self.expiration_date, "cloud": {"name": self.project.name, "id": self.project.id}, @@ -331,27 +330,13 @@ def state(self): def tf_state(self): return self.orm.tf_state - @property - def freeipa_passwd(self): - if self.tf_state is not None: - return self.tf_state.freeipa_passwd - else: - return None - @property def allocated_resources(self): if self.is_busy: raise BusyClusterException if self.tf_state is not None: - return dict( - instance_count=self.tf_state.instance_count, - ram=self.tf_state.ram, - cores=self.tf_state.cores, - volume_count=self.tf_state.volume_count, - volume_size=self.tf_state.volume_size, - public_ip=self.tf_state.public_ip, - ) + return self.tf_state.to_dict() else: return {} diff --git a/mchub/models/terraform/terraform_state.py b/mchub/models/terraform/terraform_state.py index 8c234a36..0c77cada 100644 --- a/mchub/models/terraform/terraform_state.py +++ b/mchub/models/terraform/terraform_state.py @@ -29,12 +29,6 @@ } } -IMAGE_PARSER = parse("resources[?name=image].instances[0].attributes.name") -FREEIPA_PASSWD_PARSER = parse( - "resources[?name=freeipa_passwd].instances[0].attributes.result" -) - - class TerraformState: """ TerraformState holds the state file of a cluster, i.e. the terraform.tfstate file. @@ -46,9 +40,7 @@ class TerraformState: "ram", "volume_count", "volume_size", - "image", "public_ip", - "freeipa_passwd", "ports", "security_groups", ] @@ -70,11 +62,5 @@ def __init__(self, tf_state: object, cloud="openstack"): self.ports = len(parser["ports"].find(tf_state)) self.security_groups = len(parser["security_groups"].find(tf_state)) - try: - self.image = IMAGE_PARSER.find(tf_state)[0].value - except: - self.image = "" - try: - self.freeipa_passwd = FREEIPA_PASSWD_PARSER.find(tf_state)[0].value - except: - self.freeipa_passwd = None + def to_dict(self): + return { key : getattr(self, key) for key in self.__slots__ } \ No newline at end of file diff --git a/tests/data/__init__.py b/tests/data/__init__.py index 5b95e049..7bc5ac8e 100644 --- a/tests/data/__init__.py +++ b/tests/data/__init__.py @@ -77,7 +77,6 @@ "public_keys": ["ssh-rsa FAKE"], "status": "provisioning_success", "hostname": "valid1.magic-castle.cloud", - "freeipa_passwd": "FAKE", "expiration_date": "2029-01-01", "age": "a moment", } @@ -321,7 +320,6 @@ "expiration_date": "2029-01-01", "hostname": "buildplanning.magic-castle.cloud", "status": "plan_running", - "freeipa_passwd": None, "age": "a moment", }, "created.magic-castle.cloud": { @@ -330,7 +328,6 @@ "expiration_date": "2029-01-01", "hostname": "created.magic-castle.cloud", "status": "created", - "freeipa_passwd": None, "age": "a moment", }, "valid1.magic-castle.cloud": { @@ -339,7 +336,6 @@ "expiration_date": "2029-01-01", "hostname": "valid1.magic-castle.cloud", "status": "provisioning_success", - "freeipa_passwd": "FAKE", "age": "a moment", }, "empty-state.magic-castle.cloud": { @@ -348,7 +344,6 @@ "hostname": "empty-state.magic-castle.cloud", "expiration_date": "2029-01-01", "status": "build_error", - "freeipa_passwd": None, "age": "a moment", }, "missingfloatingips.mc.ca": { @@ -357,7 +352,6 @@ "hostname": "missingfloatingips.mc.ca", "expiration_date": "2029-01-01", "status": "build_running", - "freeipa_passwd": None, "age": "a moment", }, "missingnodes.mc.ca": { @@ -366,7 +360,6 @@ "hostname": "missingnodes.mc.ca", "expiration_date": "2029-01-01", "status": "build_error", - "freeipa_passwd": "FAKE", "age": "a moment", }, "noowner.magic-castle.cloud": { @@ -375,7 +368,6 @@ "hostname": "noowner.magic-castle.cloud", "expiration_date": "2029-01-01", "status": "provisioning_success", - "freeipa_passwd": "FAKE", "age": "a moment", }, } @@ -389,11 +381,6 @@ "type": "null_resource", "change": {"actions": ["create"], "progress": "queued"}, }, - { - "address": "module.openstack.module.cluster_config.random_string.freeipa_passwd", - "type": "random_string", - "change": {"actions": ["create"], "progress": "queued"}, - }, { "address": "module.openstack.module.cluster_config.random_string.munge_key", "type": "random_string", diff --git a/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.json b/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.json index 0c9e9286..775d0ba5 100644 --- a/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.json +++ b/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.json @@ -1017,28 +1017,6 @@ "triggers": {} } }, - { - "address": "module.openstack.module.cluster_config.random_string.freeipa_passwd", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider_name": "registry.terraform.io/hashicorp/random", - "schema_version": 1, - "values": { - "keepers": null, - "length": 16, - "lower": true, - "min_lower": 0, - "min_numeric": 0, - "min_special": 0, - "min_upper": 0, - "number": true, - "override_special": null, - "special": false, - "upper": true - }, - "sensitive_values": {} - }, { "address": "module.openstack.module.cluster_config.random_string.munge_key", "mode": "managed", @@ -1106,39 +1084,6 @@ } } }, - { - "address": "module.openstack.module.cluster_config.random_string.freeipa_passwd", - "module_address": "module.openstack.module.cluster_config", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider_name": "registry.terraform.io/hashicorp/random", - "change": { - "actions": [ - "create" - ], - "before": null, - "after": { - "keepers": null, - "length": 16, - "lower": true, - "min_lower": 0, - "min_numeric": 0, - "min_special": 0, - "min_upper": 0, - "number": true, - "override_special": null, - "special": false, - "upper": true - }, - "after_unknown": { - "id": true, - "result": true - }, - "before_sensitive": false, - "after_sensitive": {} - } - }, { "address": "module.openstack.module.cluster_config.random_string.munge_key", "module_address": "module.openstack.module.cluster_config", @@ -3474,7 +3419,6 @@ "var.nb_users", "module.cluster_config.guest_passwd", "module.cluster_config", - "module.cluster_config.freeipa_passwd", "module.cluster_config", "var.sudoer_username" ] @@ -4022,14 +3966,6 @@ }, "module": { "outputs": { - "freeipa_passwd": { - "expression": { - "references": [ - "random_string.freeipa_passwd.result", - "random_string.freeipa_passwd" - ] - } - }, "guest_passwd": { "expression": { "references": [ @@ -4153,22 +4089,6 @@ ] } }, - { - "address": "random_string.freeipa_passwd", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider_config_key": "cluster_config:random", - "expressions": { - "length": { - "constant_value": 16 - }, - "special": { - "constant_value": false - } - }, - "schema_version": 1 - }, { "address": "random_string.munge_key", "mode": "managed", diff --git a/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.log b/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.log index 7f8f4f26..204f417f 100644 --- a/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.log +++ b/tests/data/mock-clusters/created.magic-castle.cloud/terraform_plan.log @@ -650,21 +650,6 @@ Terraform will perform the following actions: + triggers = (known after apply) } - # module.openstack.module.cluster_config.random_string.freeipa_passwd will be created - + resource "random_string" "freeipa_passwd" { - + id = (known after apply) - + length = 16 - + lower = true - + min_lower = 0 - + min_numeric = 0 - + min_special = 0 - + min_upper = 0 - + number = true - + result = (known after apply) - + special = false - + upper = true - } - # module.openstack.module.cluster_config.random_string.munge_key will be created + resource "random_string" "munge_key" { + id = (known after apply) diff --git a/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_apply.log b/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_apply.log index 3ed20e26..285e5f09 100644 --- a/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_apply.log +++ b/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_apply.log @@ -11,14 +11,12 @@ module.openstack.random_string.munge_key: Creating... module.openstack.random_string.puppetmaster_password: Creating... module.openstack.random_uuid.consul_token: Creating... module.openstack.openstack_compute_keypair_v2.keypair: Creating... -module.openstack.random_string.freeipa_passwd: Creating... module.openstack.random_uuid.consul_token: Creation complete after 0s [id=6e9d031e-9a60-f5f1-37a8-4dec6196b1fd] module.openstack.random_pet.guest_passwd[0]: Creation complete after 0s [id=curiously.typically.proper.kit] module.openstack.openstack_compute_secgroup_v2.secgroup_1: Creating... module.openstack.openstack_blockstorage_volume_v2.project[0]: Creating... module.openstack.random_string.munge_key: Creation complete after 0s [id=qLvFrWBgTYyyPL86Me3CroeXvoHM3Tyn] module.openstack.random_string.puppetmaster_password: Creation complete after 0s [id=IBXAHZNHRKn5AOBGKjm2oS45xFnht1Kj] -module.openstack.random_string.freeipa_passwd: Creation complete after 0s [id=enJqlYeF8vmPOXID] module.openstack.openstack_blockstorage_volume_v2.home[0]: Creating... module.openstack.openstack_networking_floatingip_v2.fip[0]: Creating... module.openstack.openstack_blockstorage_volume_v2.scratch[0]: Creating... diff --git a/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_plan.json b/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_plan.json index 3a1177e1..f616b162 100644 --- a/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_plan.json +++ b/tests/data/mock-clusters/missingfloatingips.mc.ca/terraform_plan.json @@ -839,28 +839,6 @@ "triggers": {} } }, - { - "address": "module.openstack.module.cluster_config.random_string.freeipa_passwd", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider_name": "registry.terraform.io/hashicorp/random", - "schema_version": 1, - "values": { - "keepers": null, - "length": 16, - "lower": true, - "min_lower": 0, - "min_numeric": 0, - "min_special": 0, - "min_upper": 0, - "number": true, - "override_special": null, - "special": false, - "upper": true - }, - "sensitive_values": {} - }, { "address": "module.openstack.module.cluster_config.random_string.munge_key", "mode": "managed", @@ -928,39 +906,6 @@ } } }, - { - "address": "module.openstack.module.cluster_config.random_string.freeipa_passwd", - "module_address": "module.openstack.module.cluster_config", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider_name": "registry.terraform.io/hashicorp/random", - "change": { - "actions": [ - "create" - ], - "before": null, - "after": { - "keepers": null, - "length": 16, - "lower": true, - "min_lower": 0, - "min_numeric": 0, - "min_special": 0, - "min_upper": 0, - "number": true, - "override_special": null, - "special": false, - "upper": true - }, - "after_unknown": { - "id": true, - "result": true - }, - "before_sensitive": false, - "after_sensitive": {} - } - }, { "address": "module.openstack.module.cluster_config.random_string.munge_key", "module_address": "module.openstack.module.cluster_config", @@ -2908,7 +2853,6 @@ "var.nb_users", "module.cluster_config.guest_passwd", "module.cluster_config", - "module.cluster_config.freeipa_passwd", "module.cluster_config", "var.sudoer_username" ] @@ -3456,14 +3400,6 @@ }, "module": { "outputs": { - "freeipa_passwd": { - "expression": { - "references": [ - "random_string.freeipa_passwd.result", - "random_string.freeipa_passwd" - ] - } - }, "guest_passwd": { "expression": { "references": [ @@ -3587,22 +3523,6 @@ ] } }, - { - "address": "random_string.freeipa_passwd", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider_config_key": "cluster_config:random", - "expressions": { - "length": { - "constant_value": 16 - }, - "special": { - "constant_value": false - } - }, - "schema_version": 1 - }, { "address": "random_string.munge_key", "mode": "managed", diff --git a/tests/data/mock-clusters/missingnodes.mc.ca/terraform.tfstate b/tests/data/mock-clusters/missingnodes.mc.ca/terraform.tfstate index 4dce714f..ce5a17d1 100644 --- a/tests/data/mock-clusters/missingnodes.mc.ca/terraform.tfstate +++ b/tests/data/mock-clusters/missingnodes.mc.ca/terraform.tfstate @@ -610,7 +610,6 @@ "module.openstack.data.openstack_compute_flavor_v2.flavors", "module.openstack.data.openstack_images_image_v2.image", "module.openstack.module.cluster_config.random_pet.guest_passwd", - "module.openstack.module.cluster_config.random_string.freeipa_passwd", "module.openstack.module.cluster_config.random_string.munge_key", "module.openstack.module.cluster_config.random_uuid.consul_token", "module.openstack.module.instance_config.random_string.puppetserver_password", @@ -625,35 +624,6 @@ } ] }, - { - "module": "module.openstack.module.cluster_config", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider": "provider[\"registry.terraform.io/hashicorp/random\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "id": "FAKE", - "keepers": null, - "length": 16, - "lower": true, - "min_lower": 0, - "min_numeric": 0, - "min_special": 0, - "min_upper": 0, - "number": true, - "override_special": null, - "result": "FAKE", - "special": false, - "upper": true - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, { "module": "module.openstack.module.cluster_config", "mode": "managed", diff --git a/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform.tfstate b/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform.tfstate index 6e135fe4..793307bb 100644 --- a/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform.tfstate +++ b/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform.tfstate @@ -1124,7 +1124,6 @@ "module.openstack.data.openstack_networking_network_v2.int_network", "module.openstack.data.openstack_networking_subnet_v2.subnet", "module.openstack.module.cluster_config.random_pet.guest_passwd", - "module.openstack.module.cluster_config.random_string.freeipa_passwd", "module.openstack.module.cluster_config.random_string.munge_key", "module.openstack.module.cluster_config.random_uuid.consul_token", "module.openstack.module.instance_config.random_string.puppetserver_password", @@ -1140,35 +1139,6 @@ } ] }, - { - "module": "module.openstack.module.cluster_config", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider": "provider[\"registry.terraform.io/hashicorp/random\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "id": "FAKE", - "keepers": null, - "length": 16, - "lower": true, - "min_lower": 0, - "min_numeric": 0, - "min_special": 0, - "min_upper": 0, - "number": true, - "override_special": null, - "result": "FAKE", - "special": false, - "upper": true - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, { "module": "module.openstack.module.cluster_config", "mode": "managed", diff --git a/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform_apply.log b/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform_apply.log index 000d94ed..d15da3d2 100644 --- a/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform_apply.log +++ b/tests/data/mock-clusters/noowner.magic-castle.cloud/terraform_apply.log @@ -386,21 +386,6 @@ Terraform will perform the following actions: + triggers = (known after apply) } - # module.openstack.module.cluster_config.random_string.freeipa_passwd will be created - + resource "random_string" "freeipa_passwd" { - + id = (known after apply) - + length = 16 - + lower = true - + min_lower = 0 - + min_numeric = 0 - + min_special = 0 - + min_upper = 0 - + number = true - + result = (known after apply) - + special = false - + upper = true - } - # module.openstack.module.cluster_config.random_string.munge_key will be created + resource "random_string" "munge_key" { + id = (known after apply) @@ -491,10 +476,8 @@ module.openstack.module.instance_config.tls_private_key.rsa_hostkeys["node"]: Cr module.openstack.module.cluster_config.random_uuid.consul_token: Creating... module.openstack.module.instance_config.tls_private_key.rsa_hostkeys["login"]: Creating... module.openstack.module.cluster_config.random_string.munge_key: Creating... -module.openstack.module.cluster_config.random_string.freeipa_passwd: Creating... module.openstack.module.instance_config.tls_private_key.ssh[0]: Creating... module.openstack.module.instance_config.tls_private_key.rsa_hostkeys["mgmt"]: Creating... -module.openstack.module.cluster_config.random_string.freeipa_passwd: Creation complete after 0s [id=iOKS9XldZCuN4CCU] module.openstack.module.cluster_config.random_string.munge_key: Creation complete after 0s [id=95xTXFn0WRBsQ1VLOxt320hMaS1otBkR] module.openstack.module.cluster_config.random_uuid.consul_token: Creation complete after 0s [id=a67dcbbe-ddc1-c2d4-392f-c7a9ba1b59f9] module.openstack.module.instance_config.random_string.puppetserver_password: Creation complete after 0s [id=FQJO7pC9IXcx6rucXf1mU8D8doRu7DAF] diff --git a/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform.tfstate b/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform.tfstate index 9217a597..af86c6c6 100644 --- a/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform.tfstate +++ b/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform.tfstate @@ -1124,7 +1124,6 @@ "module.openstack.data.openstack_networking_network_v2.int_network", "module.openstack.data.openstack_networking_subnet_v2.subnet", "module.openstack.module.cluster_config.random_pet.guest_passwd", - "module.openstack.module.cluster_config.random_string.freeipa_passwd", "module.openstack.module.cluster_config.random_string.munge_key", "module.openstack.module.cluster_config.random_uuid.consul_token", "module.openstack.module.instance_config.random_string.puppetserver_password", @@ -1140,35 +1139,6 @@ } ] }, - { - "module": "module.openstack.module.cluster_config", - "mode": "managed", - "type": "random_string", - "name": "freeipa_passwd", - "provider": "provider[\"registry.terraform.io/hashicorp/random\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "id": "FAKE", - "keepers": null, - "length": 16, - "lower": true, - "min_lower": 0, - "min_numeric": 0, - "min_special": 0, - "min_upper": 0, - "number": true, - "override_special": null, - "result": "FAKE", - "special": false, - "upper": true - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, { "module": "module.openstack.module.cluster_config", "mode": "managed", diff --git a/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform_apply.log b/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform_apply.log index 0aa00510..d0ea8c0d 100644 --- a/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform_apply.log +++ b/tests/data/mock-clusters/valid1.magic-castle.cloud/terraform_apply.log @@ -11,14 +11,12 @@ module.openstack.random_string.munge_key: Creating... module.openstack.random_string.puppetmaster_password: Creating... module.openstack.random_uuid.consul_token: Creating... module.openstack.openstack_compute_keypair_v2.keypair: Creating... -module.openstack.random_string.freeipa_passwd: Creating... module.openstack.random_uuid.consul_token: Creation complete after 0s [id=6e9d031e-9a60-f5f1-37a8-4dec6196b1fd] module.openstack.random_pet.guest_passwd[0]: Creation complete after 0s [id=curiously.typically.proper.kit] module.openstack.openstack_compute_secgroup_v2.secgroup_1: Creating... module.openstack.openstack_blockstorage_volume_v2.project[0]: Creating... module.openstack.random_string.munge_key: Creation complete after 0s [id=qLvFrWBgTYyyPL86Me3CroeXvoHM3Tyn] module.openstack.random_string.puppetmaster_password: Creation complete after 0s [id=IBXAHZNHRKn5AOBGKjm2oS45xFnht1Kj] -module.openstack.random_string.freeipa_passwd: Creation complete after 0s [id=enJqlYeF8vmPOXID] module.openstack.openstack_blockstorage_volume_v2.home[0]: Creating... module.openstack.openstack_blockstorage_volume_v2.scratch[0]: Creating... module.openstack.tls_private_key.login_rsa: Creating... diff --git a/tests/unit/magic_castle/test_magic_castle.py b/tests/unit/magic_castle/test_magic_castle.py index ed3be34f..c581e700 100644 --- a/tests/unit/magic_castle/test_magic_castle.py +++ b/tests/unit/magic_castle/test_magic_castle.py @@ -261,6 +261,8 @@ def test_allocated_resources_valid(app): "cores": 10, "volume_count": 3, "volume_size": 200, + "ports": 3, + "security_groups": 1, } @@ -290,6 +292,8 @@ def test_allocated_resources_missing_nodes(app): "cores": 0, "volume_count": 3, "volume_size": 200, + "ports": 0, + "security_groups": 1, } From 694fb4874294969a6ed2489f44432095aa5154d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Fri, 10 Mar 2023 12:20:07 -0500 Subject: [PATCH 17/23] Remove available_tags --- mchub/models/cloud/openstack_manager.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index 7c4e7834..e7617aa6 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -140,15 +140,6 @@ def available_flavors(self): available_flavors.sort(key=lambda flavor: (flavor.ram, flavor.vcpus)) return available_flavors - @property - def available_tags(self): - tags = { - "mgmt": ["mgmt", "nfs", "puppet"], - "login": ["login", "proxy", "public"], - "node": ["node"], - } - return tags - @property def available_instance_count(self): return ( From 333a8946a94d22e90c71bcb36f85f55ae24f9090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Fri, 10 Mar 2023 13:03:10 -0500 Subject: [PATCH 18/23] Use dot notation for possibleResources in frontend --- frontend/src/components/cluster/ClusterEditor.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index 3f19972c..8ecfb0a4 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -627,9 +627,9 @@ export default { } // Retrieve all available types // Then filter based on the selected tags - let inst_types = this.possibleResources["types"]; + let inst_types = this.possibleResources.types; for (const tag of tags) { - if (tag in this.possibleResources["tag_types"]) { + if (tag in this.possibleResources.tag_types) { const tag_types = new Set(this.possibleResources["tag_types"][tag]); inst_types = inst_types.filter((x) => tag_types.has(x)); } From 56131058066974d9ca19e69ec0671e69c2b89897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Fri, 10 Mar 2023 13:03:29 -0500 Subject: [PATCH 19/23] Add volume type to possible resources in OpenStack manager --- mchub/models/cloud/openstack_manager.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index e7617aa6..ffe0932d 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -99,7 +99,7 @@ def possible_resources(self): for tag in TAG_MINIMUM_REQUIREMENTS }, "types": [flavor.name for flavor in self.available_flavors], - "volumes": {}, + "volumes": [type.name for type in self.available_volume_types], } @property @@ -140,6 +140,11 @@ def available_flavors(self): available_flavors.sort(key=lambda flavor: (flavor.ram, flavor.vcpus)) return available_flavors + @property + @cache + def available_volume_types(self): + return list(self.connection.block_storage.types()) + @property def available_instance_count(self): return ( @@ -216,7 +221,7 @@ def volume_quotas(self): # https://docs.openstack.org/api-ref/block-storage/v3/index.html?expanded=show-quotas-for-a-project-detail#show-quotas-for-a-project return self.connection.block_storage.get( f"/os-quota-sets/{self.project_id}?usage=true" - ).json()["quota_set"] + ).json()["quota_set"] @property @cache @@ -230,7 +235,7 @@ def compute_quotas(self): # https://docs.openstack.org/api-ref/compute/?expanded=show-a-quota-detail#show-a-quota return self.connection.compute.get( f"/os-quota-sets/{self.project_id}/detail" - ).json()["quota_set"] + ).json()["quota_set"] @property @cache From 649f3363ba18628723501787935a578de4214b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Fri, 10 Mar 2023 15:03:37 -0500 Subject: [PATCH 20/23] Clean imports in openstackmanager --- mchub/models/cloud/openstack_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mchub/models/cloud/openstack_manager.py b/mchub/models/cloud/openstack_manager.py index ffe0932d..b88d36fa 100644 --- a/mchub/models/cloud/openstack_manager.py +++ b/mchub/models/cloud/openstack_manager.py @@ -1,8 +1,7 @@ import openstack from functools import cache -from os import environ, path -from re import match, IGNORECASE, compile +from re import IGNORECASE, compile VALID_IMAGES_REGEX_ARRAY = [ compile(r"rocky-8", IGNORECASE), From 59ce4d1eb85a5dac3b2dce51eb43e3a2100238cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Fri, 10 Mar 2023 15:10:41 -0500 Subject: [PATCH 21/23] Remove unused imports --- mchub/models/magic_castle/magic_castle.py | 2 -- mchub/models/magic_castle/magic_castle_configuration.py | 4 +--- mchub/models/user.py | 1 - mchub/resources/magic_castle_api.py | 1 - 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/mchub/models/magic_castle/magic_castle.py b/mchub/models/magic_castle/magic_castle.py index e0ee681f..e7297947 100644 --- a/mchub/models/magic_castle/magic_castle.py +++ b/mchub/models/magic_castle/magic_castle.py @@ -13,8 +13,6 @@ from sqlalchemy.sql import func from sqlalchemy.exc import IntegrityError -from mchub.models.cloud.cloud_manager import CloudManager - from .magic_castle_configuration import MagicCastleConfiguration from .cluster_status_code import ClusterStatusCode from .plan_type import PlanType diff --git a/mchub/models/magic_castle/magic_castle_configuration.py b/mchub/models/magic_castle/magic_castle_configuration.py index 0f1d53fe..1074bad4 100644 --- a/mchub/models/magic_castle/magic_castle_configuration.py +++ b/mchub/models/magic_castle/magic_castle_configuration.py @@ -4,9 +4,7 @@ from collections.abc import Mapping import marshmallow -from marshmallow import fields, ValidationError, EXCLUDE - -from copy import deepcopy +from marshmallow import fields, EXCLUDE from ..cloud.dns_manager import DnsManager from ...configuration.magic_castle import ( diff --git a/mchub/models/user.py b/mchub/models/user.py index fe00802d..98e72734 100644 --- a/mchub/models/user.py +++ b/mchub/models/user.py @@ -1,5 +1,4 @@ from subprocess import getoutput -from typing import List from getpass import getuser from .magic_castle.magic_castle import MagicCastle, MagicCastleORM diff --git a/mchub/resources/magic_castle_api.py b/mchub/resources/magic_castle_api.py index 85b35141..5520105b 100644 --- a/mchub/resources/magic_castle_api.py +++ b/mchub/resources/magic_castle_api.py @@ -1,4 +1,3 @@ -from operator import imod from flask import request from .api_view import APIView from ..exceptions.invalid_usage_exception import ( From 7119f5efa3026d977ed3d9e413b67c4f2e8bdbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Tue, 14 Mar 2023 12:24:08 -0400 Subject: [PATCH 22/23] Refactor form row and add region --- .../src/components/cluster/ClusterEditor.vue | 254 ++++++++++-------- mchub/models/magic_castle/magic_castle.py | 6 +- 2 files changed, 142 insertions(+), 118 deletions(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index 8ecfb0a4..8c6dae16 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -4,14 +4,28 @@ General configuration - + + + + + + + + @@ -57,68 +71,73 @@ - - + + + - - - - - - - - - - - - + + + + + + + + + + + + + +
- - - - - - - - - - - - - - - mdi-delete - - + + + + + + + + + + + + + + + + mdi-delete + + +
@@ -129,65 +148,66 @@ - - - - - - - - + + + + + + + + + +
- - - - - - - - - - - - - - mdi-delete - - - + + + + + + + + + + + + + + + mdi-delete + + +
@@ -499,10 +519,10 @@ export default { return (this.domains && this.domains.includes(this.localSpecs.domain)) || "Invalid domain provided"; }, volumeCountRule() { - return this.volumeCountUsed <= this.volumeCountMax || "Volume number quota exceeded"; + return this.volumeCountUsed <= this.volumeCountMax || "quota exceeded"; }, volumeSizeRule() { - return this.volumeSizeUsed <= this.volumeSizeMax || "Volume size quota exceeded"; + return this.volumeSizeUsed <= this.volumeSizeMax || "quota exceeded"; }, instanceCountUsed() { return this.usedResourcesLoaded ? this.instances.reduce((acc, instance) => acc + instance.count, 0) : 0; diff --git a/mchub/models/magic_castle/magic_castle.py b/mchub/models/magic_castle/magic_castle.py index e7297947..c3d73de0 100644 --- a/mchub/models/magic_castle/magic_castle.py +++ b/mchub/models/magic_castle/magic_castle.py @@ -321,7 +321,11 @@ def state(self): "status": self.status, "age": self.age, "expiration_date": self.expiration_date, - "cloud": {"name": self.project.name, "id": self.project.id}, + "cloud": { + "name": self.project.name, + "id": self.project.id, + "provider": self.project.provider, + }, } @property From 2e0c99b0ff35823c9e57f018ff34cfaa04b730a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Tue, 6 Jun 2023 10:13:59 -0400 Subject: [PATCH 23/23] Add cloud.name before cloud.provider in title Add regions variable --- .../src/components/cluster/ClusterEditor.vue | 26 +++++++++++++++++-- mchub/models/template.py | 6 ++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/cluster/ClusterEditor.vue b/frontend/src/components/cluster/ClusterEditor.vue index 8c6dae16..c5daccd6 100644 --- a/frontend/src/components/cluster/ClusterEditor.vue +++ b/frontend/src/components/cluster/ClusterEditor.vue @@ -30,7 +30,7 @@ Cloud project - {{ localSpecs.cloud.name }} + {{ localSpecs.cloud.provider }} - {{ localSpecs.cloud.name }} @@ -194,7 +194,13 @@ :readonly="stateful && id in initialSpecs.volumes.nfs" /> - + + +