diff --git a/image/cli/app-root/src/.bashrc b/image/cli/app-root/src/.bashrc index 8cf16b03b12..38de336e434 100644 --- a/image/cli/app-root/src/.bashrc +++ b/image/cli/app-root/src/.bashrc @@ -43,5 +43,6 @@ if [ $arch != "s390x" ] && [ $arch != "ppc64le" ]; then echo " - ${TEXT_BOLD}${COLOR_GREEN}mas provision-fyre${TEXT_RESET} to provision an OCP cluster on IBM DevIT Fyre (internal)" echo "AI Service (Standalone) Management:" echo " - ${TEXT_BOLD}${COLOR_GREEN}mas aiservice-install${TEXT_RESET} to install a new AI Service instance" + echo " - ${TEXT_BOLD}${COLOR_GREEN}mas aiservice-upgrade${TEXT_RESET} to upgrade a existing AI Service instance" echo fi diff --git a/image/cli/mascli/mas b/image/cli/mascli/mas index 8d8ca829a8e..60d86f8f575 100755 --- a/image/cli/mascli/mas +++ b/image/cli/mascli/mas @@ -230,6 +230,16 @@ case $1 in mas-cli aiservice-install "$@" ;; + aiservice-upgrade) + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> $LOGFILE + echo "!! aiservice-upgrade !!" >> $LOGFILE + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> $LOGFILE + # Take the first parameter off (it will be "aiservice-upgrade") + shift + # Run the new Python-based aiservice-upgrade + mas-cli aiservice-upgrade "$@" + ;; + update) echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> $LOGFILE echo "!! update !!" >> $LOGFILE diff --git a/python/src/mas-cli b/python/src/mas-cli index fe0f5b8492b..8d8a7781584 100644 --- a/python/src/mas-cli +++ b/python/src/mas-cli @@ -16,6 +16,7 @@ from sys import argv from mas.cli import __version__ as VERSION from mas.cli.install.app import InstallApp from mas.cli.aiservice.install.app import AiServiceInstallApp +from mas.cli.aiservice.upgrade.app import AiServiceUpgradeApp from mas.cli.update.app import UpdateApp from mas.cli.upgrade.app import UpgradeApp from mas.cli.uninstall.app import UninstallApp @@ -58,6 +59,9 @@ if __name__ == '__main__': elif function == "aiservice-install": app = AiServiceInstallApp() app.install(argv[2:]) + elif function == "aiservice-upgrade": + app = AiServiceUpgradeApp() + app.upgrade(argv[2:]) elif function == "uninstall": app = UninstallApp() app.uninstall(argv[2:]) diff --git a/python/src/mas/cli/aiservice/upgrade/__init__.py b/python/src/mas/cli/aiservice/upgrade/__init__.py new file mode 100644 index 00000000000..07779332557 --- /dev/null +++ b/python/src/mas/cli/aiservice/upgrade/__init__.py @@ -0,0 +1,11 @@ +# ***************************************************************************** +# Copyright (c) 2024, 2025 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +from ...cli import BaseApp # noqa: F401 diff --git a/python/src/mas/cli/aiservice/upgrade/app.py b/python/src/mas/cli/aiservice/upgrade/app.py new file mode 100644 index 00000000000..7aa9989151d --- /dev/null +++ b/python/src/mas/cli/aiservice/upgrade/app.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# ***************************************************************************** +# Copyright (c) 2024, 2025 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +import sys +import logging +import logging.handlers +from prompt_toolkit import prompt, print_formatted_text, HTML +from prompt_toolkit.completion import WordCompleter + +from halo import Halo + +from ...cli import BaseApp +from ...validators import AiserviceInstanceIDValidator +from .argParser import upgradeArgParser + +from mas.devops.ocp import createNamespace +from mas.devops.mas import listAiServiceInstances, getAiserviceChannel +from mas.devops.tekton import installOpenShiftPipelines, updateTektonDefinitions, launchAiServiceUpgradePipeline +from openshift.dynamic.exceptions import ResourceNotFoundError +logger = logging.getLogger(__name__) + + +class AiServiceUpgradeApp(BaseApp): + def upgrade(self, argv): + """ + Upgrade AI Service instance + """ + args = upgradeArgParser.parse_args(args=argv) + aiserviceInstanceId = args.aiservice_instance_id + self.noConfirm = args.no_confirm + self.skipPreCheck = args.skip_pre_check + self.licenseAccepted = args.accept_license + self.devMode = args.dev_mode + + if aiserviceInstanceId is None: + self.printH1("Set Target OpenShift Cluster") + # Connect to the target cluster + self.connect() + else: + logger.debug("AI Service instance ID is set, so we assume already connected to the desired OCP") + # Need to lookup target architecture because configDb2 will try to access self.architecture + self.lookupTargetArchitecture() + + if self.dynamicClient is None: + print_formatted_text(HTML("Error: The Kubernetes dynamic Client is not available. See log file for details")) + sys.exit(1) + + if aiserviceInstanceId is None: + # Interactive mode + self.printH1("AI Service Instance Selection") + print_formatted_text(HTML("Select a AI Service instance to upgrade from the list below:")) + try: + aiserviceInstances = listAiServiceInstances(self.dynamicClient) + except ResourceNotFoundError: + aiserviceInstances = [] + aiserviceOptions = [] + + if len(aiserviceInstances) == 0: + print_formatted_text(HTML("Error: No AI Service instances detected on this cluster")) + sys.exit(1) + + for aiservice in aiserviceInstances: + print_formatted_text(HTML(f"- {aiservice['metadata']['name']} v{aiservice['status']['versions']['reconciled']}")) + aiserviceOptions.append(aiservice['metadata']['name']) + + aiserviceCompleter = WordCompleter(aiserviceOptions) + print() + aiserviceInstanceId = prompt(HTML('Enter AI Service instance ID: '), completer=aiserviceCompleter, validator=AiserviceInstanceIDValidator(), validate_while_typing=False) + + currentAiserviceChannel = getAiserviceChannel(self.dynamicClient, aiserviceInstanceId) + if currentAiserviceChannel is not None: + if self.devMode: + # this enables upgrade of custom channel for AI service + nextAiserviceChannel = prompt(HTML('Custom channel ')) + else: + if currentAiserviceChannel not in self.upgrade_path: + self.fatalError(f"No upgrade available, {aiserviceInstanceId} is are already on the latest release {currentAiserviceChannel}") + nextAiserviceChannel = self.upgrade_path[currentAiserviceChannel] + + if not self.licenseAccepted and not self.devMode: + self.printH1("License Terms") + self.printDescription([ + "To continue with the upgrade, you must accept the license terms:", + self.licenses[nextAiserviceChannel] + ]) + + if self.noConfirm: + self.fatalError("You must accept the license terms with --accept-license when using the --no-confirm flag") + else: + if not self.yesOrNo("Do you accept the license terms"): + exit(1) + + self.printH1("Review Settings") + print_formatted_text(HTML(f"AI Service Instance ID ..................... {aiserviceInstanceId}")) + print_formatted_text(HTML(f"Current AI Service Channel ............. {currentAiserviceChannel}")) + print_formatted_text(HTML(f"Next AI Service Channel ................ {nextAiserviceChannel}")) + print_formatted_text(HTML(f"Skip Pre-Upgrade Checks ......... {self.skipPreCheck}")) + + if not self.noConfirm: + print() + continueWithUpgrade = self.yesOrNo("Proceed with these settings") + + if self.noConfirm or continueWithUpgrade: + self.createTektonFileWithDigest() + + self.printH1("Launch Upgrade") + pipelinesNamespace = f"aiservice-{aiserviceInstanceId}-pipelines" + + with Halo(text='Validating OpenShift Pipelines installation', spinner=self.spinner) as h: + installOpenShiftPipelines(self.dynamicClient) + h.stop_and_persist(symbol=self.successIcon, text="OpenShift Pipelines Operator is installed and ready to use") + + with Halo(text=f'Preparing namespace ({pipelinesNamespace})', spinner=self.spinner) as h: + createNamespace(self.dynamicClient, pipelinesNamespace) + h.stop_and_persist(symbol=self.successIcon, text=f"Namespace is ready ({pipelinesNamespace})") + + with Halo(text=f'Installing latest Tekton definitions (v{self.version})', spinner=self.spinner) as h: + updateTektonDefinitions(pipelinesNamespace, self.tektonDefsPath) + h.stop_and_persist(symbol=self.successIcon, text=f"Latest Tekton definitions are installed (v{self.version})") + + with Halo(text='Submitting PipelineRun for {aiserviceInstanceId} upgrade', spinner=self.spinner) as h: + pipelineURL = launchAiServiceUpgradePipeline(self.dynamicClient, aiserviceInstanceId, self.skipPreCheck, aiserviceChannel=nextAiserviceChannel, params=self.params) + if pipelineURL is not None: + h.stop_and_persist(symbol=self.successIcon, text=f"PipelineRun for {aiserviceInstanceId} upgrade submitted") + print_formatted_text(HTML(f"\nView progress:\n {pipelineURL}\n")) + else: + h.stop_and_persist(symbol=self.failureIcon, text=f"Failed to submit PipelineRun for {aiserviceInstanceId} upgrade, see log file for details") + print() diff --git a/python/src/mas/cli/aiservice/upgrade/argParser.py b/python/src/mas/cli/aiservice/upgrade/argParser.py new file mode 100644 index 00000000000..66270a0ac33 --- /dev/null +++ b/python/src/mas/cli/aiservice/upgrade/argParser.py @@ -0,0 +1,69 @@ +# ***************************************************************************** +# Copyright (c) 2024, 2025 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +import argparse + +from ... import __version__ as packageVersion +from ...cli import getHelpFormatter + +upgradeArgParser = argparse.ArgumentParser( + prog='mas aiservice-upgrade', + description="\n".join([ + f"IBM Maximo Application Suite Admin CLI v{packageVersion}", + "Upgrade AI Service by configuring and launching the AI Service Upgrade Tekton Pipeline.\n", + "Interactive Mode:", + "Omitting the --aiservice-instance-id option will trigger an interactive prompt" + ]), + epilog="Refer to the online documentation for more information: https://ibm-mas.github.io/cli/", + formatter_class=getHelpFormatter(), + add_help=False +) + +masArgGroup = upgradeArgParser.add_argument_group('MAS Instance Selection') +masArgGroup.add_argument( + '--aiservice-instance-id', + required=False, + help="The AI Service Instance ID to be upgraded" +) + +otherArgGroup = upgradeArgParser.add_argument_group('More') +otherArgGroup.add_argument( + '--skip-pre-check', + required=False, + action='store_true', + default=False, + help="Disable the 'pre-upgrade-check' and 'post-upgrade-verify' tasks in the upgrade pipeline" +) +otherArgGroup.add_argument( + '--no-confirm', + required=False, + action='store_true', + default=False, + help="Launch the upgrade without prompting for confirmation", +) +otherArgGroup.add_argument( + "--accept-license", + action="store_true", + default=False, + help="Accept all license terms without prompting" +) +otherArgGroup.add_argument( + "--dev-mode", + required=False, + action="store_true", + default=False, + help="Configure upgrade for development mode", +) +otherArgGroup.add_argument( + '-h', "--help", + action='help', + default=False, + help="Show this help message and exit", +) diff --git a/python/src/mas/cli/validators.py b/python/src/mas/cli/validators.py index 195415b2ac0..e9f8b54f6dd 100644 --- a/python/src/mas/cli/validators.py +++ b/python/src/mas/cli/validators.py @@ -20,7 +20,7 @@ from prompt_toolkit.validation import Validator, ValidationError from mas.devops.ocp import getStorageClass -from mas.devops.mas import verifyMasInstance +from mas.devops.mas import verifyMasInstance, verifyAiServiceInstance import logging @@ -85,6 +85,20 @@ def validate(self, document): raise ValidationError(message='Not a valid MAS instance ID on this cluster', cursor_position=len(instanceId)) +class AiserviceInstanceIDValidator(Validator): + def validate(self, document): + """ + Validate that a AI Service instance ID exists on the target cluster + """ + instanceId = document.text + + dynClient = dynamic.DynamicClient( + api_client.ApiClient(configuration=config.load_kube_config()) + ) + if not verifyAiServiceInstance(dynClient, instanceId): + raise ValidationError(message='Not a valid AI Service instance ID on this cluster', cursor_position=len(instanceId)) + + class StorageClassValidator(Validator): def validate(self, document): """ diff --git a/tekton/generate-tekton-pipelines.yml b/tekton/generate-tekton-pipelines.yml index 323de034953..8da57851850 100644 --- a/tekton/generate-tekton-pipelines.yml +++ b/tekton/generate-tekton-pipelines.yml @@ -54,6 +54,7 @@ - mas-ivt-manage # AI Service Pipelines - aiservice-install + - aiservice-upgrade # AI Service FVT Pipelines - aiservice-fvt-launcher - aiservice-fvt diff --git a/tekton/generate-tekton-tasks.yml b/tekton/generate-tekton-tasks.yml index 0d3dd5a8371..1797264b1cd 100644 --- a/tekton/generate-tekton-tasks.yml +++ b/tekton/generate-tekton-tasks.yml @@ -43,9 +43,9 @@ - turbonomic - uds - # 3. Generate Tasks (AI Broker) + # 3. Generate Tasks (AI Service) # ------------------------------------------------------------------------- - - name: Generate Tasks (AI Broker) + - name: Generate Tasks (AI Service) ansible.builtin.template: src: "{{ task_src_dir }}/aiservice/{{ item }}.yml.j2" dest: "{{ task_target_dir }}/{{ item }}.yaml" @@ -55,6 +55,7 @@ - odh - aiservice-tenant - aiservice-post-verify + - aiservice-upgrade # 4. Generate Tasks (FVT) # ------------------------------------------------------------------------- diff --git a/tekton/src/pipelines/aiservice-upgrade.yml.j2 b/tekton/src/pipelines/aiservice-upgrade.yml.j2 new file mode 100644 index 00000000000..1754f3a0402 --- /dev/null +++ b/tekton/src/pipelines/aiservice-upgrade.yml.j2 @@ -0,0 +1,94 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: aiservice-upgrade +spec: + workspaces: + # The generated configuration files + - name: shared-configs + # PodTemplates configurations + - name: shared-pod-templates + params: + # 1. Common Parameters + # ------------------------------------------------------------------------- + {{ lookup('template', params_src_dir ~ '/common.yml.j2') | indent(4) }} + + - name: aiservice_instance_id + type: string + description: AI Service Instance ID + - name: aiservice_channel + type: string + description: AI Service Channnel + - name: skip_pre_check + type: string + default: "" + + # 2. Upgrade + # ------------------------------------------------------------------------- + - name: skip_compatibility_check + type: string + default: "False" + description: Skip performing compatiblity checks before upgrade + + tasks: + # 1. Wait for approval & verify health of the cluster before we change anything + # ------------------------------------------------------------------------- + - name: waitfor-approval + timeout: "0" + taskRef: + kind: Task + name: mas-devops-wait-for-configmap-v2 + params: + - name: image_pull_policy + value: $(params.image_pull_policy) + - name: configmap_name + value: approval-upgrade + + {{ lookup('template', 'taskdefs/cluster-setup/ocp-verify-all.yml.j2', template_vars={'name': 'pre-upgrade-check', 'devops_suite_name': 'pre-upgrade-check'}) | indent(4) }} + runAfter: + - waitfor-approval + + # 2. AI Service Upgrade + # ------------------------------------------------------------------------- + - name: aiservice-upgrade + timeout: "0" + params: + - name: aiservice_instance_id + value: $(params.aiservice_instance_id) + - name: aiservice_channel + value: $(params.aiservice_channel) + - name: skip_compatibility_check + value: $(params.skip_compatibility_check) + taskRef: + kind: Task + name: mas-devops-aiservice-upgrade + runAfter: + - pre-upgrade-check + + # 3. Verify health of the cluster after upgrade + # ------------------------------------------------------------------------- + {{ lookup('template', 'taskdefs/cluster-setup/ocp-verify-all.yml.j2', template_vars={ + 'name': 'post-upgrade-verify', + 'devops_suite_name': 'post-upgrade-verify' + }) | indent(4) }} + runAfter: + - aiservice-upgrade + + finally: + # Update synchronization configmap + # ------------------------------------------------------------------------- + - name: sync-upgrade + timeout: "0" + taskRef: + kind: Task + name: mas-devops-update-configmap + params: + - name: image_pull_policy + value: $(params.image_pull_policy) + - name: configmap_name + value: sync-upgrade + - name: configmap_value + # An aggregate status of all the pipelineTasks under the tasks section (excluding the finally section). + # This variable is only available in the finally tasks and can have any one of the values (Succeeded, Failed, Completed, or None) + value: $(tasks.status) diff --git a/tekton/src/tasks/aiservice/aiservice-upgrade.yml.j2 b/tekton/src/tasks/aiservice/aiservice-upgrade.yml.j2 new file mode 100644 index 00000000000..829f270ef54 --- /dev/null +++ b/tekton/src/tasks/aiservice/aiservice-upgrade.yml.j2 @@ -0,0 +1,37 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: mas-devops-aiservice-upgrade +spec: + params: + {{ lookup('template', task_src_dir ~ '/common/cli-params.yml.j2') | indent(4) }} + + - name: aiservice_instance_id + type: string + description: AI Service Instance ID + - name: aiservice_channel + type: string + description: AI Service Channnel + + - name: skip_compatibility_check + type: string + default: "False" + + stepTemplate: + env: + {{ lookup('template', task_src_dir ~ '/common/cli-env.yml.j2') | indent(6) }} + + - name: AISERVICE_INSTANCE_ID + value: $(params.aiservice_instance_id) + - name: AISERVICE_CHANNEL + value: $(params.aiservice_channel) + - name: SKIP_COMPATIBILITY_CHECK + value: $(params.skip_compatibility_check) + steps: + - name: aiservice-upgrade + command: + - /opt/app-root/src/run-role.sh + - aiservice_upgrade + image: quay.io/ibmmas/cli:latest + imagePullPolicy: $(params.image_pull_policy)