From 671b6e6053a67cf2ae02eb8c5389015eb285fa97 Mon Sep 17 00:00:00 2001 From: Sophie Zhao Date: Sun, 6 Jun 2021 20:36:13 -0400 Subject: [PATCH 1/3] kubectl cp changes in progress --- src/connectedk8s/azext_connectedk8s/_utils.py | 93 +++++++++++++++---- src/connectedk8s/azext_connectedk8s/custom.py | 3 +- 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/src/connectedk8s/azext_connectedk8s/_utils.py b/src/connectedk8s/azext_connectedk8s/_utils.py index bdbe223ba25..8902a90b5b0 100644 --- a/src/connectedk8s/azext_connectedk8s/_utils.py +++ b/src/connectedk8s/azext_connectedk8s/_utils.py @@ -728,6 +728,16 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, if kube_context: kubectl_prior.extend(["--context", kube_context]) + fd, temp_yaml_path = tempfile.mkstemp() + temp_yaml_file = os.fdopen(fd, 'w+t') + deployment_yaml = urlopen( + "https://raw.githubusercontent.com/Azure/aks-periscope/latest/deployment/aks-periscope.yaml").read().decode() + + if not storage_account_name or not sas_token: + print('No storage account specified. Downloading logs to local machine.') + apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path) + copy_and_zip_periscope_files(kubectl_prior) + return readonly_sas_token = readonly_sas_token.strip('?') from knack.prompting import prompt_y_n @@ -746,12 +756,32 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, sas_token = sas_token.strip('?') - deployment_yaml = urlopen( - "https://raw.githubusercontent.com/Azure/aks-periscope/latest/deployment/aks-periscope.yaml").read().decode() deployment_yaml = deployment_yaml.replace("# ", (base64.b64encode(bytes(storage_account_name, 'ascii'))).decode('ascii')) deployment_yaml = deployment_yaml.replace("# ", (base64.b64encode(bytes("?" + sas_token, 'ascii'))).decode('ascii')) + apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path) + print() + # log_storage_account_url = f"https://{storage_account_name}.blob.core.windows.net/" + + print(f'{colorama.Fore.GREEN}Your logs are being uploaded to storage account {format_bright(storage_account_name)}...') + + print() + print(f'You can download Azure Storage Explorer here ' + f'{format_hyperlink("https://azure.microsoft.com/en-us/features/storage-explorer/")}' + f' to check the logs by accessing the storage account {storage_account_name}.') + # f' to check the logs by adding the storage account using the following URL:') + # print(f'{format_hyperlink(log_storage_account_url)}') + + print() + if not prompt_y_n('Do you want to see analysis results now?', default="n"): + print(f"You can rerun 'az connectedk8s troubleshoot -g {resource_group_name} -n {name}' " + f"anytime to check the analysis results.") + else: + display_diagnostics_report(kubectl_prior) + + +def apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path): container_logs = "azure-arc" kube_objects = "azure-arc/pod azure-arc/service azure-arc/deployment" yaml_lines = deployment_yaml.splitlines() @@ -765,8 +795,7 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, deployment_yaml = '\n'.join(yaml_lines) - fd, temp_yaml_path = tempfile.mkstemp() - temp_yaml_file = os.fdopen(fd, 'w+t') + try: temp_yaml_file.write(deployment_yaml) temp_yaml_file.flush() @@ -791,31 +820,52 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None, print() print(f"{colorama.Fore.GREEN}Deploying diagnostic container on the K8s cluster...") - subprocess_cmd = kubectl_prior + ["apply", "-f", temp_yaml_path, "-n", "aks-periscope"] + subprocess_cmd = kubectl_prior + ["apply", "-f", temp_yaml_path, "-n", "aks-periscope", "--validate=false"] subprocess.check_output(subprocess_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: raise CLIInternalError(err.output) finally: os.remove(temp_yaml_path) - print() - # log_storage_account_url = f"https://{storage_account_name}.blob.core.windows.net/" - print(f'{colorama.Fore.GREEN}Your logs are being uploaded to storage account {format_bright(storage_account_name)}...') - print() - print(f'You can download Azure Storage Explorer here ' - f'{format_hyperlink("https://azure.microsoft.com/en-us/features/storage-explorer/")}' - f' to check the logs by accessing the storage account {storage_account_name}.') - # f' to check the logs by adding the storage account using the following URL:') - # print(f'{format_hyperlink(log_storage_account_url)}') +def copy_and_zip_periscope_files(kubectl_prior): + periscope_files = [] + time.sleep(5) + subprocess_cmd = kubectl_prior + ["get", "pods", "-n", "aks-periscope", "-o", "wide", "--no-headers"] + subprocess.call(subprocess_cmd, stderr=subprocess.STDOUT) + pods = subprocess.check_output( + subprocess_cmd, + universal_newlines=True) + pod_lines = pods.splitlines() + for line in pod_lines: + info = line.split() + pod = info[0] + status = info[2] + node = info[6] + if status != "Running": + continue + file = node+".zip" + + time.sleep(5) + subprocess_cmd = kubectl_prior + ["cp", "-n", "aks-periscope", pod+":"+file, file] + subprocess.call(subprocess_cmd, stderr=subprocess.STDOUT) + periscope_files.append(file) + try: + periscope_zip = os.path.join(os.path.expanduser('~'), '.azure', 'periscope_output.tar.gz') + periscope_log_path = os.path.join(os.path.expanduser('~'), '.azure') + # Creating the .tar.gz for logs and deleting the actual log file + import tarfile + with tarfile.open(periscope_zip , "w:gz") as tar: + for file in periscope_files: + tar.add(file, file) + #logging.shutdown() # To release log file handler, so that the actual log file can be removed after archiving + os.remove(file) - print() - if not prompt_y_n('Do you want to see analysis results now?', default="n"): - print(f"You can rerun 'az connectedk8s troubleshoot -g {resource_group_name} -n {name}' " - f"anytime to check the analysis results.") - else: - display_diagnostics_report(kubectl_prior) + print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}Some diagnostic logs have been collected and archived at '{periscope_zip}'.") + except Exception as ex: + logger.error("Error occured while archiving the periscope logs: {}".format(str(ex))) + print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}You can find the unarchived periscope logs at '{periscope_log_path}'.") def which(binary): @@ -836,12 +886,15 @@ def which(binary): def try_archive_log_file(troubleshoot_log_path, output_file): try: + periscope_zip = os.path.join(os.path.expanduser('~'), '.azure', 'periscope_output.tar.gz') # Creating the .tar.gz for logs and deleting the actual log file import tarfile with tarfile.open(output_file, "w:gz") as tar: tar.add(troubleshoot_log_path, 'connected8s_troubleshoot.log') + tar.add(periscope_zip, 'periscope_output.tar.gz') logging.shutdown() # To release log file handler, so that the actual log file can be removed after archiving os.remove(troubleshoot_log_path) + os.remove(periscope_zip) print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}Some diagnostic logs have been collected and archived at '{output_file}'.") except Exception as ex: logger.error("Error occured while archiving the log file: {}".format(str(ex))) diff --git a/src/connectedk8s/azext_connectedk8s/custom.py b/src/connectedk8s/azext_connectedk8s/custom.py index 962db60632b..ad19bc2902e 100644 --- a/src/connectedk8s/azext_connectedk8s/custom.py +++ b/src/connectedk8s/azext_connectedk8s/custom.py @@ -1500,10 +1500,11 @@ def troubleshoot(cmd, client, resource_group_name, cluster_name, kube_config=Non storage_account_name, sas_token, readonly_sas_token = utils.setup_validate_storage_account(cmd.cli_ctx, storage_account, sas_token, resource_group_name) if storage_account_name: # When validated the storage account utils.try_upload_log_file(cluster_name, storage_account_name, sas_token, troubleshoot_log_path) + utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) utils.try_archive_log_file(troubleshoot_log_path, output_file) # token_in_storage_account_url = readonly_sas_token if readonly_sas_token is not None else sas_token. - utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) else: + utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) utils.try_archive_log_file(troubleshoot_log_path, output_file) except Exception as ex: From 590e841d2fd2796d5a281225069fdf842afbabf0 Mon Sep 17 00:00:00 2001 From: Sophie Zhao Date: Sat, 12 Jun 2021 12:17:35 -0400 Subject: [PATCH 2/3] recent kubectl cp changes --- src/connectedk8s/azext_connectedk8s/_utils.py | 18 +++++++++--------- src/connectedk8s/azext_connectedk8s/custom.py | 7 ++----- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/connectedk8s/azext_connectedk8s/_utils.py b/src/connectedk8s/azext_connectedk8s/_utils.py index 8902a90b5b0..1ec35f6cae9 100644 --- a/src/connectedk8s/azext_connectedk8s/_utils.py +++ b/src/connectedk8s/azext_connectedk8s/_utils.py @@ -20,7 +20,8 @@ from six.moves.urllib.request import urlopen # pylint: disable=import-error from tabulate import tabulate # pylint: disable=import-error import tempfile - +from zipfile import ZipFile +from os.path import basename from knack.log import get_logger from knack.prompting import NoTTYException, prompt_y_n @@ -828,7 +829,6 @@ def apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_ya os.remove(temp_yaml_path) - def copy_and_zip_periscope_files(kubectl_prior): periscope_files = [] time.sleep(5) @@ -839,15 +839,15 @@ def copy_and_zip_periscope_files(kubectl_prior): universal_newlines=True) pod_lines = pods.splitlines() for line in pod_lines: + time.sleep(5) info = line.split() pod = info[0] status = info[2] node = info[6] if status != "Running": continue - file = node+".zip" + file = "periscope-logs-"+node - time.sleep(5) subprocess_cmd = kubectl_prior + ["cp", "-n", "aks-periscope", pod+":"+file, file] subprocess.call(subprocess_cmd, stderr=subprocess.STDOUT) periscope_files.append(file) @@ -858,11 +858,12 @@ def copy_and_zip_periscope_files(kubectl_prior): import tarfile with tarfile.open(periscope_zip , "w:gz") as tar: for file in periscope_files: - tar.add(file, file) - #logging.shutdown() # To release log file handler, so that the actual log file can be removed after archiving - os.remove(file) + zipfilename = file + '.zip' + shutil.make_archive(file, "zip", file) + tar.add(zipfilename, zipfilename) + shutil.rmtree(file) + os.remove(zipfilename) - print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}Some diagnostic logs have been collected and archived at '{periscope_zip}'.") except Exception as ex: logger.error("Error occured while archiving the periscope logs: {}".format(str(ex))) print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}You can find the unarchived periscope logs at '{periscope_log_path}'.") @@ -895,7 +896,6 @@ def try_archive_log_file(troubleshoot_log_path, output_file): logging.shutdown() # To release log file handler, so that the actual log file can be removed after archiving os.remove(troubleshoot_log_path) os.remove(periscope_zip) - print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}Some diagnostic logs have been collected and archived at '{output_file}'.") except Exception as ex: logger.error("Error occured while archiving the log file: {}".format(str(ex))) print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}You can find the unarchived log file at '{troubleshoot_log_path}'.") diff --git a/src/connectedk8s/azext_connectedk8s/custom.py b/src/connectedk8s/azext_connectedk8s/custom.py index ad19bc2902e..bbb5d6db0be 100644 --- a/src/connectedk8s/azext_connectedk8s/custom.py +++ b/src/connectedk8s/azext_connectedk8s/custom.py @@ -1500,12 +1500,9 @@ def troubleshoot(cmd, client, resource_group_name, cluster_name, kube_config=Non storage_account_name, sas_token, readonly_sas_token = utils.setup_validate_storage_account(cmd.cli_ctx, storage_account, sas_token, resource_group_name) if storage_account_name: # When validated the storage account utils.try_upload_log_file(cluster_name, storage_account_name, sas_token, troubleshoot_log_path) - utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) - utils.try_archive_log_file(troubleshoot_log_path, output_file) # token_in_storage_account_url = readonly_sas_token if readonly_sas_token is not None else sas_token. - else: - utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) - utils.try_archive_log_file(troubleshoot_log_path, output_file) + utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config) + utils.try_archive_log_file(troubleshoot_log_path, output_file) except Exception as ex: tr_logger.error("Exception caught while running troubleshoot: {}".format(str(ex)), exc_info=True) From 414269b5270b958d3e94fbfc9bf3c93074cab0ef Mon Sep 17 00:00:00 2001 From: Sophie Zhao Date: Thu, 17 Jun 2021 16:36:36 -0700 Subject: [PATCH 3/3] remove unused imports --- src/connectedk8s/azext_connectedk8s/_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/connectedk8s/azext_connectedk8s/_utils.py b/src/connectedk8s/azext_connectedk8s/_utils.py index 1ec35f6cae9..bbcf9baa84d 100644 --- a/src/connectedk8s/azext_connectedk8s/_utils.py +++ b/src/connectedk8s/azext_connectedk8s/_utils.py @@ -20,8 +20,6 @@ from six.moves.urllib.request import urlopen # pylint: disable=import-error from tabulate import tabulate # pylint: disable=import-error import tempfile -from zipfile import ZipFile -from os.path import basename from knack.log import get_logger from knack.prompting import NoTTYException, prompt_y_n