From 40d4134d6158de1b25592613751bf1a35c1f67cf Mon Sep 17 00:00:00 2001 From: ProcrastinateGuy Date: Wed, 6 Aug 2025 14:35:10 -0700 Subject: [PATCH 1/4] testing query for detection missing entries --- system_status/server.py | 2 +- system_status/templates/my_template.html | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/system_status/server.py b/system_status/server.py index 6a4447e..03bed75 100644 --- a/system_status/server.py +++ b/system_status/server.py @@ -91,7 +91,7 @@ def get_prometheus_data() -> list[PrometheusData]: url = urljoin(args.target, "api/v1/query_range") now = datetime.datetime.now() params = { - "query": 'min_over_time(up{job!=""}[1h])', + "query": 'min_over_time(up{job!=""}[1h]) or vector(-1)', "start": int((now - datetime.timedelta(hours=23)).timestamp()), "end": int(now.timestamp()), "step": "1h", diff --git a/system_status/templates/my_template.html b/system_status/templates/my_template.html index 44a2049..384406b 100644 --- a/system_status/templates/my_template.html +++ b/system_status/templates/my_template.html @@ -124,8 +124,10 @@

Fetch Time: {{ fetch_time }}

{% for value in item.values %} {% if value.value == "1" %} ✅ - {% else %} + {% elif value.value == "0" %} ❌ + {% else %} + ⚠️ {% endif %} {# the below if-statement adds a vertical line for every four entries #} {% if (not loop.index % 4) and loop.index < 24%} From b7f96b4c4512b104c226061cc1bcba03660989d9 Mon Sep 17 00:00:00 2001 From: ProcrastinateGuy Date: Sun, 10 Aug 2025 12:22:42 -0700 Subject: [PATCH 2/4] change the query to fill in the gaps when an entry is missing --- system_status/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_status/server.py b/system_status/server.py index 03bed75..f30a7e9 100644 --- a/system_status/server.py +++ b/system_status/server.py @@ -91,7 +91,7 @@ def get_prometheus_data() -> list[PrometheusData]: url = urljoin(args.target, "api/v1/query_range") now = datetime.datetime.now() params = { - "query": 'min_over_time(up{job!=""}[1h]) or vector(-1)', + "query": '(min_over_time(up{job!=""}[1h])) or (vector(-1) * on(instance, job) ((up{job!=""} * 0) + 1))', "start": int((now - datetime.timedelta(hours=23)).timestamp()), "end": int(now.timestamp()), "step": "1h", From 264adeb666e1116aa1e887d274e39f7ad7df4dfe Mon Sep 17 00:00:00 2001 From: ProcrastinateGuy Date: Sun, 10 Aug 2025 15:21:53 -0700 Subject: [PATCH 3/4] change the query to fill in the gaps when an entry is missing --- system_status/server.py | 23 ++++++++++++++++++----- system_status/templates/my_template.html | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/system_status/server.py b/system_status/server.py index f30a7e9..f2d03e5 100644 --- a/system_status/server.py +++ b/system_status/server.py @@ -64,7 +64,7 @@ def get_args() -> argparse.Namespace: args = get_args() - +@app.get("/page_raw") def get_prometheus_data() -> list[PrometheusData]: """Sends a PromQL query to Prometheus and returns the results.""" """ @@ -89,7 +89,8 @@ def get_prometheus_data() -> list[PrometheusData]: } """ url = urljoin(args.target, "api/v1/query_range") - now = datetime.datetime.now() + now = datetime.datetime(2025, 8, 6, 11, 38, 49) + #now = datetime.datetime.now() params = { "query": '(min_over_time(up{job!=""}[1h])) or (vector(-1) * on(instance, job) ((up{job!=""} * 0) + 1))', "start": int((now - datetime.timedelta(hours=23)).timestamp()), @@ -103,6 +104,12 @@ def get_prometheus_data() -> list[PrometheusData]: response_json = response.json() result_list = response_json.get("data", {}).get("result", []) + key_list = [] # make a list of expected timestamps + for i in range(24): + key_list.append(int(now.timestamp()) - (23 - i) * 3600) + key_list[i] = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(key_list[i])) + #print(key_list) + for service_dict in result_list: maybe_instance = service_dict.get("metric", {}).get( "instance", "NO INSTANCE AVAILABLE" @@ -111,9 +118,15 @@ def get_prometheus_data() -> list[PrometheusData]: maybe_values = service_dict.get("values", []) timestamps_and_values = [] - for epoch_time, value in maybe_values: - timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(epoch_time)) - timestamps_and_values.append(TimestampAndValuePair(timestamp, value)) + value_ptr = 0 + for i in range(len(key_list)): + timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(maybe_values[value_ptr][0])) + if timestamp == key_list[i]: + timestamps_and_values.append(TimestampAndValuePair(key_list[i], maybe_values[value_ptr][1])) + value_ptr += 1 + else: + #fill in the gaps + timestamps_and_values.append(TimestampAndValuePair(key_list[i], -1)) # the service is up if the maximum timestamp's value is "1" # prometheus returns data with the greatest timestamp last diff --git a/system_status/templates/my_template.html b/system_status/templates/my_template.html index 384406b..4a183f0 100644 --- a/system_status/templates/my_template.html +++ b/system_status/templates/my_template.html @@ -167,7 +167,7 @@

Fetch Time: {{ fetch_time }}

// Format the string with leading zeroes const dayStr = `${year.toString()}-${month.toString().padStart(2, '0')}-${date.toString().padStart(2, '0')}` const clockStr = `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`; - timeElement.innerText = `${dayStr} ${clockStr} - PDT/PST`; + timeElement.innerText = `${dayStr} ${clockStr} - Local Time`; } // Call initially to display time immediately From ebcfdeae889768d48852085e7a813755c537f0f1 Mon Sep 17 00:00:00 2001 From: ProcrastinateGuy Date: Sun, 10 Aug 2025 15:33:00 -0700 Subject: [PATCH 4/4] change the scraping timestamp back to normal --- system_status/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system_status/server.py b/system_status/server.py index f2d03e5..9bae03d 100644 --- a/system_status/server.py +++ b/system_status/server.py @@ -64,7 +64,6 @@ def get_args() -> argparse.Namespace: args = get_args() -@app.get("/page_raw") def get_prometheus_data() -> list[PrometheusData]: """Sends a PromQL query to Prometheus and returns the results.""" """ @@ -89,10 +88,11 @@ def get_prometheus_data() -> list[PrometheusData]: } """ url = urljoin(args.target, "api/v1/query_range") - now = datetime.datetime(2025, 8, 6, 11, 38, 49) - #now = datetime.datetime.now() + #the below date has a rare data for testing purpose + #now = datetime.datetime(2025, 8, 6, 11, 38, 49) + now = datetime.datetime.now() params = { - "query": '(min_over_time(up{job!=""}[1h])) or (vector(-1) * on(instance, job) ((up{job!=""} * 0) + 1))', + "query": '(min_over_time(up{job!=""}[1h]))', "start": int((now - datetime.timedelta(hours=23)).timestamp()), "end": int(now.timestamp()), "step": "1h",