diff --git a/src/connectedk8s/azext_connectedk8s/_utils.py b/src/connectedk8s/azext_connectedk8s/_utils.py index bdbe223ba25..bbcf9baa84d 100644 --- a/src/connectedk8s/azext_connectedk8s/_utils.py +++ b/src/connectedk8s/azext_connectedk8s/_utils.py @@ -21,7 +21,6 @@ from tabulate import tabulate # pylint: disable=import-error import tempfile - from knack.log import get_logger from knack.prompting import NoTTYException, prompt_y_n from azure.cli.core.commands.client_factory import get_subscription_id @@ -728,6 +727,16 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, if kube_context: kubectl_prior.extend(["--context", kube_context]) + fd, temp_yaml_path = tempfile.mkstemp() + temp_yaml_file = os.fdopen(fd, 'w+t') + deployment_yaml = urlopen( + "https://raw.githubusercontent.com/Azure/aks-periscope/latest/deployment/aks-periscope.yaml").read().decode() + + if not storage_account_name or not sas_token: + print('No storage account specified. Downloading logs to local machine.') + apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path) + copy_and_zip_periscope_files(kubectl_prior) + return readonly_sas_token = readonly_sas_token.strip('?') from knack.prompting import prompt_y_n @@ -746,12 +755,32 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, sas_token = sas_token.strip('?') - deployment_yaml = urlopen( - "https://raw.githubusercontent.com/Azure/aks-periscope/latest/deployment/aks-periscope.yaml").read().decode() deployment_yaml = deployment_yaml.replace("# ", (base64.b64encode(bytes(storage_account_name, 'ascii'))).decode('ascii')) deployment_yaml = deployment_yaml.replace("# ", (base64.b64encode(bytes("?" + sas_token, 'ascii'))).decode('ascii')) + apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path) + print() + # log_storage_account_url = f"https://{storage_account_name}.blob.core.windows.net/" + + print(f'{colorama.Fore.GREEN}Your logs are being uploaded to storage account {format_bright(storage_account_name)}...') + + print() + print(f'You can download Azure Storage Explorer here ' + f'{format_hyperlink("https://azure.microsoft.com/en-us/features/storage-explorer/")}' + f' to check the logs by accessing the storage account {storage_account_name}.') + # f' to check the logs by adding the storage account using the following URL:') + # print(f'{format_hyperlink(log_storage_account_url)}') + + print() + if not prompt_y_n('Do you want to see analysis results now?', default="n"): + print(f"You can rerun 'az connectedk8s troubleshoot -g {resource_group_name} -n {name}' " + f"anytime to check the analysis results.") + else: + display_diagnostics_report(kubectl_prior) + + +def apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path): container_logs = "azure-arc" kube_objects = "azure-arc/pod azure-arc/service azure-arc/deployment" yaml_lines = deployment_yaml.splitlines() @@ -765,8 +794,7 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, deployment_yaml = '\n'.join(yaml_lines) - fd, temp_yaml_path = tempfile.mkstemp() - temp_yaml_file = os.fdopen(fd, 'w+t') + try: temp_yaml_file.write(deployment_yaml) temp_yaml_file.flush() @@ -791,31 +819,52 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, print() print(f"{colorama.Fore.GREEN}Deploying diagnostic container on the K8s cluster...") - subprocess_cmd = kubectl_prior + ["apply", "-f", temp_yaml_path, "-n", "aks-periscope"] + subprocess_cmd = kubectl_prior + ["apply", "-f", temp_yaml_path, "-n", "aks-periscope", "--validate=false"] subprocess.check_output(subprocess_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: raise CLIInternalError(err.output) finally: os.remove(temp_yaml_path) - print() - # log_storage_account_url = f"https://{storage_account_name}.blob.core.windows.net/" - print(f'{colorama.Fore.GREEN}Your logs are being uploaded to storage account {format_bright(storage_account_name)}...') - - print() - print(f'You can download Azure Storage Explorer here ' - f'{format_hyperlink("https://azure.microsoft.com/en-us/features/storage-explorer/")}' - f' to check the logs by accessing the storage account {storage_account_name}.') - # f' to check the logs by adding the storage account using the following URL:') - # print(f'{format_hyperlink(log_storage_account_url)}') +def copy_and_zip_periscope_files(kubectl_prior): + periscope_files = [] + time.sleep(5) + subprocess_cmd = kubectl_prior + ["get", "pods", "-n", "aks-periscope", "-o", "wide", "--no-headers"] + subprocess.call(subprocess_cmd, stderr=subprocess.STDOUT) + pods = subprocess.check_output( + subprocess_cmd, + universal_newlines=True) + pod_lines = pods.splitlines() + for line in pod_lines: + time.sleep(5) + info = line.split() + pod = info[0] + status = info[2] + node = info[6] + if status != "Running": + continue + file = "periscope-logs-"+node + + subprocess_cmd = kubectl_prior + ["cp", "-n", "aks-periscope", pod+":"+file, file] + subprocess.call(subprocess_cmd, stderr=subprocess.STDOUT) + periscope_files.append(file) + try: + periscope_zip = os.path.join(os.path.expanduser('~'), '.azure', 'periscope_output.tar.gz') + periscope_log_path = os.path.join(os.path.expanduser('~'), '.azure') + # Creating the .tar.gz for logs and deleting the actual log file + import tarfile + with tarfile.open(periscope_zip , "w:gz") as tar: + for file in periscope_files: + zipfilename = file + '.zip' + shutil.make_archive(file, "zip", file) + tar.add(zipfilename, zipfilename) + shutil.rmtree(file) + os.remove(zipfilename) - print() - if not prompt_y_n('Do you want to see analysis results now?', default="n"): - print(f"You can rerun 'az connectedk8s troubleshoot -g {resource_group_name} -n {name}' " - f"anytime to check the analysis results.") - else: - display_diagnostics_report(kubectl_prior) + except Exception as ex: + logger.error("Error occured while archiving the periscope logs: {}".format(str(ex))) + print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}You can find the unarchived periscope logs at '{periscope_log_path}'.") def which(binary): @@ -836,13 +885,15 @@ def which(binary): def try_archive_log_file(troubleshoot_log_path, output_file): try: + periscope_zip = os.path.join(os.path.expanduser('~'), '.azure', 'periscope_output.tar.gz') # Creating the .tar.gz for logs and deleting the actual log file import tarfile with tarfile.open(output_file, "w:gz") as tar: tar.add(troubleshoot_log_path, 'connected8s_troubleshoot.log') + tar.add(periscope_zip, 'periscope_output.tar.gz') logging.shutdown() # To release log file handler, so that the actual log file can be removed after archiving os.remove(troubleshoot_log_path) - print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}Some diagnostic logs have been collected and archived at '{output_file}'.") + os.remove(periscope_zip) except Exception as ex: logger.error("Error occured while archiving the log file: {}".format(str(ex))) print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}You can find the unarchived log file at '{troubleshoot_log_path}'.") diff --git a/src/connectedk8s/azext_connectedk8s/custom.py b/src/connectedk8s/azext_connectedk8s/custom.py index 962db60632b..bbb5d6db0be 100644 --- a/src/connectedk8s/azext_connectedk8s/custom.py +++ b/src/connectedk8s/azext_connectedk8s/custom.py @@ -1500,11 +1500,9 @@ def troubleshoot(cmd, client, resource_group_name, cluster_name, kube_config=Non storage_account_name, sas_token, readonly_sas_token = utils.setup_validate_storage_account(cmd.cli_ctx, storage_account, sas_token, resource_group_name) if storage_account_name: # When validated the storage account utils.try_upload_log_file(cluster_name, storage_account_name, sas_token, troubleshoot_log_path) - utils.try_archive_log_file(troubleshoot_log_path, output_file) # token_in_storage_account_url = readonly_sas_token if readonly_sas_token is not None else sas_token. - utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) - else: - utils.try_archive_log_file(troubleshoot_log_path, output_file) + utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) + utils.try_archive_log_file(troubleshoot_log_path, output_file) except Exception as ex: tr_logger.error("Exception caught while running troubleshoot: {}".format(str(ex)), exc_info=True)