Skip to content

kubectl cp changes #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alphaWizard wants to merge 3 commits into
base: troubleshoot
Choose a base branch
from
Open

kubectl cp changes #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
671b6e6
kubectl cp changes in progress
Jun 7, 2021
590e841
recent kubectl cp changes
Jun 12, 2021
414269b
remove unused imports
Jun 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 74 additions & 23 deletions src/connectedk8s/azext_connectedk8s/_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
from tabulate import tabulate # pylint: disable=import-error
import tempfile


from knack.log import get_logger
from knack.prompting import NoTTYException, prompt_y_n
from azure.cli.core.commands.client_factory import get_subscription_id
Expand Down Expand Up @@ -728,6 +727,16 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None,
if kube_context:
kubectl_prior.extend(["--context", kube_context])

fd, temp_yaml_path = tempfile.mkstemp()
temp_yaml_file = os.fdopen(fd, 'w+t')
deployment_yaml = urlopen(
"https://raw.githubusercontent.com/Azure/aks-periscope/latest/deployment/aks-periscope.yaml").read().decode()

if not storage_account_name or not sas_token:
print('No storage account specified. Downloading logs to local machine.')
apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path)
copy_and_zip_periscope_files(kubectl_prior)
return
readonly_sas_token = readonly_sas_token.strip('?')

from knack.prompting import prompt_y_n
Expand All @@ -746,12 +755,32 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None,

sas_token = sas_token.strip('?')

deployment_yaml = urlopen(
"https://raw.githubusercontent.com/Azure/aks-periscope/latest/deployment/aks-periscope.yaml").read().decode()
deployment_yaml = deployment_yaml.replace("# <accountName, base64 encoded>",
(base64.b64encode(bytes(storage_account_name, 'ascii'))).decode('ascii'))
deployment_yaml = deployment_yaml.replace("# <saskey, base64 encoded>",
(base64.b64encode(bytes("?" + sas_token, 'ascii'))).decode('ascii'))
apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path)
print()
# log_storage_account_url = f"https://{storage_account_name}.blob.core.windows.net/"

print(f'{colorama.Fore.GREEN}Your logs are being uploaded to storage account {format_bright(storage_account_name)}...')

print()
print(f'You can download Azure Storage Explorer here '
f'{format_hyperlink("https://azure.microsoft.com/en-us/features/storage-explorer/")}'
f' to check the logs by accessing the storage account {storage_account_name}.')
# f' to check the logs by adding the storage account using the following URL:')
# print(f'{format_hyperlink(log_storage_account_url)}')

print()
if not prompt_y_n('Do you want to see analysis results now?', default="n"):
print(f"You can rerun 'az connectedk8s troubleshoot -g {resource_group_name} -n {name}' "
f"anytime to check the analysis results.")
else:
display_diagnostics_report(kubectl_prior)


def apply_periscope_yaml(kubectl_prior, deployment_yaml, temp_yaml_file, temp_yaml_path):
container_logs = "azure-arc"
kube_objects = "azure-arc/pod azure-arc/service azure-arc/deployment"
yaml_lines = deployment_yaml.splitlines()
Expand All @@ -765,8 +794,7 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None,

deployment_yaml = '\n'.join(yaml_lines)

fd, temp_yaml_path = tempfile.mkstemp()
temp_yaml_file = os.fdopen(fd, 'w+t')

try:
temp_yaml_file.write(deployment_yaml)
temp_yaml_file.flush()
Expand All @@ -791,31 +819,52 @@ def collect_periscope_logs(resource_group_name, name, storage_account_name=None,

print()
print(f"{colorama.Fore.GREEN}Deploying diagnostic container on the K8s cluster...")
subprocess_cmd = kubectl_prior + ["apply", "-f", temp_yaml_path, "-n", "aks-periscope"]
subprocess_cmd = kubectl_prior + ["apply", "-f", temp_yaml_path, "-n", "aks-periscope", "--validate=false"]
subprocess.check_output(subprocess_cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
raise CLIInternalError(err.output)
finally:
os.remove(temp_yaml_path)

print()
# log_storage_account_url = f"https://{storage_account_name}.blob.core.windows.net/"

print(f'{colorama.Fore.GREEN}Your logs are being uploaded to storage account {format_bright(storage_account_name)}...')

print()
print(f'You can download Azure Storage Explorer here '
f'{format_hyperlink("https://azure.microsoft.com/en-us/features/storage-explorer/")}'
f' to check the logs by accessing the storage account {storage_account_name}.')
# f' to check the logs by adding the storage account using the following URL:')
# print(f'{format_hyperlink(log_storage_account_url)}')
def copy_and_zip_periscope_files(kubectl_prior):
periscope_files = []
time.sleep(5)
subprocess_cmd = kubectl_prior + ["get", "pods", "-n", "aks-periscope", "-o", "wide", "--no-headers"]
subprocess.call(subprocess_cmd, stderr=subprocess.STDOUT)
pods = subprocess.check_output(
subprocess_cmd,
universal_newlines=True)
pod_lines = pods.splitlines()
for line in pod_lines:
time.sleep(5)
Copy link
Owner Author

@alphaWizard alphaWizard Jun 14, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have we intentionally kept time sleep inside loop(for each pod). We can keep it outside the loop?

Copy link
Collaborator

@sophsoph321 sophsoph321 Jun 14, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I can increase the length of time sleep outside of the loop and test out what happens.

info = line.split()
pod = info[0]
status = info[2]
node = info[6]
if status != "Running":
Copy link
Owner Author

@alphaWizard alphaWizard Jun 14, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the status is terminated, then also logs wont be captured?

Copy link
Collaborator

@sophsoph321 sophsoph321 Jun 14, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I can check if the logs will be captured when the status is terminated.

continue
file = "periscope-logs-"+node

subprocess_cmd = kubectl_prior + ["cp", "-n", "aks-periscope", pod+":"+file, file]
subprocess.call(subprocess_cmd, stderr=subprocess.STDOUT)
periscope_files.append(file)
try:
periscope_zip = os.path.join(os.path.expanduser('~'), '.azure', 'periscope_output.tar.gz')
periscope_log_path = os.path.join(os.path.expanduser('~'), '.azure')
# Creating the .tar.gz for logs and deleting the actual log file
import tarfile
with tarfile.open(periscope_zip , "w:gz") as tar:
for file in periscope_files:
zipfilename = file + '.zip'
shutil.make_archive(file, "zip", file)
tar.add(zipfilename, zipfilename)
shutil.rmtree(file)
os.remove(zipfilename)

print()
if not prompt_y_n('Do you want to see analysis results now?', default="n"):
print(f"You can rerun 'az connectedk8s troubleshoot -g {resource_group_name} -n {name}' "
f"anytime to check the analysis results.")
else:
display_diagnostics_report(kubectl_prior)
except Exception as ex:
logger.error("Error occured while archiving the periscope logs: {}".format(str(ex)))
print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}You can find the unarchived periscope logs at '{periscope_log_path}'.")


def which(binary):
Expand All @@ -836,13 +885,15 @@ def which(binary):

def try_archive_log_file(troubleshoot_log_path, output_file):
try:
periscope_zip = os.path.join(os.path.expanduser('~'), '.azure', 'periscope_output.tar.gz')
# Creating the .tar.gz for logs and deleting the actual log file
import tarfile
with tarfile.open(output_file, "w:gz") as tar:
tar.add(troubleshoot_log_path, 'connected8s_troubleshoot.log')
tar.add(periscope_zip, 'periscope_output.tar.gz')
logging.shutdown() # To release log file handler, so that the actual log file can be removed after archiving
os.remove(troubleshoot_log_path)
print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}Some diagnostic logs have been collected and archived at '{output_file}'.")
os.remove(periscope_zip)
except Exception as ex:
logger.error("Error occured while archiving the log file: {}".format(str(ex)))
print(f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}You can find the unarchived log file at '{troubleshoot_log_path}'.")
6 changes: 2 additions & 4 deletions src/connectedk8s/azext_connectedk8s/custom.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1500,11 +1500,9 @@ def troubleshoot(cmd, client, resource_group_name, cluster_name, kube_config=Non
storage_account_name, sas_token, readonly_sas_token = utils.setup_validate_storage_account(cmd.cli_ctx, storage_account, sas_token, resource_group_name)
if storage_account_name: # When validated the storage account
utils.try_upload_log_file(cluster_name, storage_account_name, sas_token, troubleshoot_log_path)
utils.try_archive_log_file(troubleshoot_log_path, output_file)
# token_in_storage_account_url = readonly_sas_token if readonly_sas_token is not None else sas_token.
utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config)
else:
utils.try_archive_log_file(troubleshoot_log_path, output_file)
utils.collect_periscope_logs(resource_group_name, cluster_name, storage_account_name, sas_token, readonly_sas_token, kube_context, kube_config)
utils.try_archive_log_file(troubleshoot_log_path, output_file)

except Exception as ex:
tr_logger.error("Exception caught while running troubleshoot: {}".format(str(ex)), exc_info=True)
Expand Down