From cafaeb24108846d052a5145a63d9bbdeb4fcee4d Mon Sep 17 00:00:00 2001 From: bia1708 Date: Thu, 4 Sep 2025 16:15:44 +0300 Subject: [PATCH 1/2] gparse: parser: Add support for multilevel jobs Signed-off-by: bia1708 --- telemetry/gparser/parser.py | 116 ++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/telemetry/gparser/parser.py b/telemetry/gparser/parser.py index ad0e78a..2bbb01a 100644 --- a/telemetry/gparser/parser.py +++ b/telemetry/gparser/parser.py @@ -60,9 +60,6 @@ def __init__(self, url, grabber=None): def initialize(self): url_ = urlparse(self.url) - self.multilevel = False - if len(url_.path.split('/job/')) > 2: - self.multilevel = True # initialize attributes self.server = url_.scheme + '://' + url_.netloc + '/' + url_.path.split('/')[1] @@ -86,29 +83,40 @@ def show_info(self): def get_job_info(self): '''returns jenkins project name, job no and job date''' - if self.multilevel: - url = urlparse(self.url) - job=url.path.split('/')[3] + '/' + url.path.split('/')[5] - job_no=url.path.split('/')[6] - # TODO: get job date using jenkins api - job_date=None - return (job,job_no,job_date) - raise Exception("Does not support non multilevel yet!") + url = urlparse(self.url) + parts = [p for p in url.path.split("/") if p] # split URL + + # Jenkins URL paths look like: /job/folder/job/my-job/123/ + # Extract all job segments + job_parts = [] + job_no = None + job_date = None + + i = 0 + while i < len(parts): + if parts[i] == "job": + if i + 1 < len(parts): + job_parts.append(parts[i + 1]) + i += 2 + continue + # Check if the current part is the build number + if parts[i].isdigit(): + job_no = parts[i] + i += 1 + + job = "/".join(job_parts) + return (job, job_no, job_date) def get_file_info(self): '''returns file name, file info, target_board, artifact_info_type''' - if self.multilevel: - url = urlparse(self.url) - file_name = url.path.split('/')[-1] - file_info = file_name.split('_') - target_board=file_info[0] - artifact_info_type=file_info[1] + '_' + file_info[2] - artifact_info_type = remove_suffix(artifact_info_type,".log") - return (file_name, file_info, target_board, artifact_info_type) - - raise Exception("Does not support non multilevel yet!") - + url = urlparse(self.url) + file_name = url.path.split('/')[-1] + file_info = file_name.split('_') + target_board=file_info[0] + artifact_info_type=file_info[1] + '_' + file_info[2] + artifact_info_type = remove_suffix(artifact_info_type,".log") + return (file_name, file_info, target_board, artifact_info_type) def get_payload_raw(self): payload = [] @@ -148,18 +156,16 @@ def __init__(self, url, grabber): def get_file_info(self): '''returns file name, file info, target_board, artifact_info_type''' - if self.multilevel: - url = urlparse(self.url) - file_name = url.path.split('/')[-1] - file_info = file_name.split('_') - target_board=file_info[1] - artifact_info_type=file_info[0] - if len(file_info) == 3: - artifact_info_type += '_' + file_info[2] - artifact_info_type = remove_suffix(artifact_info_type,".log") - return (file_name, file_info, target_board, artifact_info_type) - - raise Exception("Does not support non multilevel yet!") + url = urlparse(self.url) + file_name = url.path.split('/')[-1] + file_info = file_name.split('_') + target_board=file_info[1] + artifact_info_type=file_info[0] + if len(file_info) == 3: + artifact_info_type += '_' + file_info[2] + artifact_info_type = remove_suffix(artifact_info_type,".log") + return (file_name, file_info, target_board, artifact_info_type) + class DmesgError(Dmesg): @@ -187,21 +193,18 @@ def __init__(self, url, grabber): def get_file_info(self): '''returns file name, file info, target_board, artifact_info_type''' - if self.multilevel: - url = urlparse(self.url) - url_path = url.path.split('/') - file_name = url_path[-1] - parser_type = type(self).__name__ - x = [i for i, c in enumerate(parser_type) if c.isupper()] - file_info = (parser_type[:x[1]]+'_'+parser_type[x[1]:]).lower() - target_board = file_name.replace('_','-') - target_board = remove_suffix(target_board,"-reports.xml") - target_board = remove_suffix(target_board,"-HWTestResults.xml") - artifact_info_type=file_info - return (file_name, file_info, target_board, artifact_info_type) - - raise Exception("Does not support non multilevel yet!") - + url = urlparse(self.url) + url_path = url.path.split('/') + file_name = url_path[-1] + parser_type = type(self).__name__ + x = [i for i, c in enumerate(parser_type) if c.isupper()] + file_info = (parser_type[:x[1]]+'_'+parser_type[x[1]:]).lower() + target_board = file_name.replace('_','-') + target_board = remove_suffix(target_board,"-reports.xml") + target_board = remove_suffix(target_board,"-HWTestResults.xml") + artifact_info_type=file_info + return (file_name, file_info, target_board, artifact_info_type) + def get_payload_raw(self): payload = [] try: @@ -422,15 +425,12 @@ def __init__(self, url, grabber): def get_file_info(self): '''returns file name, file info, target_board, artifact_info_type''' - if self.multilevel: - url = urlparse(self.url) - file_name = url.path.split('/')[-1] - file_info = "NA" - target_board="NA" - artifact_info_type = "info_txt" - return (file_name, file_info, target_board, artifact_info_type) - - raise Exception("Does not support non multilevel yet!") + url = urlparse(self.url) + file_name = url.path.split('/')[-1] + file_info = "NA" + target_board="NA" + artifact_info_type = "info_txt" + return (file_name, file_info, target_board, artifact_info_type) def get_payload_raw(self): payload = [] From fb7614e8549638875aaf53877f087af5f8ac6b8b Mon Sep 17 00:00:00 2001 From: bia1708 Date: Wed, 17 Sep 2025 17:18:48 +0300 Subject: [PATCH 2/2] gparser: Implement get build date functionality Obtain the base URL of the current log file. Call the Jenkins API to retrieve the commit date. Convert date from UNIX timestamp format to human-readable date format. Signed-off-by: bia1708 --- telemetry/gparser/grabber.py | 15 +++++++++++++++ telemetry/gparser/parser.py | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/telemetry/gparser/grabber.py b/telemetry/gparser/grabber.py index 9ddcab8..e415749 100644 --- a/telemetry/gparser/grabber.py +++ b/telemetry/gparser/grabber.py @@ -3,6 +3,7 @@ from tqdm import tqdm import os import requests +from datetime import datetime, timezone class Grabber: '''Class providing grabber functionality''' @@ -12,6 +13,20 @@ def __init__(self, auth=None): if auth: self.sess.auth = auth + def get_date(self, url): + '''Calls the Jenkins API endpoint and returns the date of a build.''' + base_url = url.strip() + api_url = base_url.rstrip('/') + '/api/json' + resp = self.sess.get(api_url) + if not resp.ok: + raise Exception(f"Failed to call API {url} (status={resp.status_code})") + + data = resp.json() + ts = data.get('timestamp') + date = datetime.fromtimestamp(ts / 1000.0, tz=timezone.utc) + + return date + def download_file(self, url, filename): '''Downloads file''' resp = self.sess.get(url, stream=True) diff --git a/telemetry/gparser/parser.py b/telemetry/gparser/parser.py index 2bbb01a..7ee67f6 100644 --- a/telemetry/gparser/parser.py +++ b/telemetry/gparser/parser.py @@ -93,6 +93,7 @@ def get_job_info(self): job_no = None job_date = None + base_url = url.scheme + "://" + url.netloc + "/" i = 0 while i < len(parts): if parts[i] == "job": @@ -103,9 +104,13 @@ def get_job_info(self): # Check if the current part is the build number if parts[i].isdigit(): job_no = parts[i] + # Get the base URL (up until the build no) in order + # to access the build's API endpoint + base_url += "/".join(parts[:i + 1]) i += 1 job = "/".join(job_parts) + job_date = self.grabber.get_date(base_url) return (job, job_no, job_date) def get_file_info(self):