Skip to content

0.3.0 #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tbotnz merged 5 commits into from
Oct 20, 2025
Merged

0.3.0 #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
897dda4
updates
tbotnz Oct 20, 2025
e085849
.
tbotnz Oct 20, 2025
cb10e9b
.
tbotnz Oct 20, 2025
a5613e8
.
tbotnz Oct 20, 2025
9f1bc73
update response
tbotnz Oct 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions .github/workflows/lint.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Lint

on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]

jobs:
lint:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'

- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: latest
virtualenvs-create: true
virtualenvs-in-project: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root

- name: Install project
run: poetry install --no-interaction

- name: Run Ruff Linter
run: poetry run ruff check .

- name: Run Ruff Formatter Check
run: poetry run ruff format --check .
4 changes: 3 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
*.pyc
.vscode/
.vscode/settings.json
.vscode/settings.json
poetry.lock
__pycache__/
4 changes: 2 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ simple micro service for converting your RRD's to web services
### getting started
- ensure you have ```rrdtool``` installed and you can access the rrd files from the server
- git clone the project ``` git clone https://github.com/tbotnz/RRDReST && cd RRDReST ```
- install the requirements ```pip3 install -r requirements.txt```
- run the app with uvicorn ```uvicorn rrdrest:rrd_rest --host "0.0.0.0" --port 9000```
- install dependencies ```poetry install```
- run the app with poetry ```poetry run uvicorn rrdrest:rrd_rest --host "0.0.0.0" --port 9000```
- access the swagger documentation via ```http://127.0.0.1:9000/docs```

### examples
Expand Down
5 changes: 5 additions & 0 deletions __version__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
__version__ = "0.3.0"


def get_version():
return __version__
64 changes: 29 additions & 35 deletions backend/RRD_parse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@


class RRD_parser:

def __init__(self, rrd_file=None, start_time=None, end_time=None):
self.rrd_file = rrd_file
self.ds = None
Expand All @@ -19,23 +18,21 @@ def __init__(self, rrd_file=None, start_time=None, end_time=None):
self.end_time = end_time

def check_dependc(self):
result = subprocess.check_output(
"rrdtool --version",
shell=True
).decode('utf-8')
result = subprocess.check_output("rrdtool --version", shell=True).decode(
"utf-8"
)
if "RRDtool 1." not in result:
raise Exception("RRDtool version not found, check rrdtool installed")

def get_data_source(self):
""" gets datasources from rrd tool """
"""gets datasources from rrd tool"""

STEP_VAL = None
DS_VALS = []

result = subprocess.check_output(
f"rrdtool info {self.rrd_file}",
shell=True
).decode('utf-8')
f"rrdtool info {self.rrd_file}", shell=True
).decode("utf-8")

temp_arr = result.split("\n")

Expand All @@ -48,7 +45,7 @@ def get_data_source(self):
STEP_VAL = raw_val

if ("ds[" in raw_key) and ("]." in raw_key):
match_obj = re.match(r'^ds\[(.*)\]', raw_key)
match_obj = re.match(r"^ds\[(.*)\]", raw_key)
if match_obj:
ds_val = match_obj.group(1)
if ds_val not in DS_VALS:
Expand All @@ -57,47 +54,47 @@ def get_data_source(self):
self.ds = DS_VALS

def get_rrd_json(self, ds):
""" gets RRD json from rrd tool """
"""gets RRD json from rrd tool"""

rrd_xport_command = f"rrdtool xport --step {self.step} DEF:data={self.rrd_file}:{ds}:AVERAGE XPORT:data:{ds} --showtime"
if self.start_time:
rrd_xport_command = f"rrdtool xport DEF:data={self.rrd_file}:{ds}:AVERAGE XPORT:data:{ds} --showtime --start {self.start_time} --end {self.end_time}"
result = subprocess.check_output(
rrd_xport_command,
shell=True
).decode('utf-8')
result = subprocess.check_output(rrd_xport_command, shell=True).decode("utf-8")
json_result = json.dumps(xmltodict.parse(result), indent=4)
# replace rrdtool v key with the ds
replace_val = "\""+ds.lower()+"\": "
temp_result_one = re.sub("\"v\": ", replace_val, json_result)
replace_val = '"' + ds.lower() + '": '
temp_result_one = re.sub('"v": ', replace_val, json_result)
return json.loads(temp_result_one)

def cleanup_payload(self, payload):
""" cleans up / transforms response payload """
"""cleans up / transforms response payload"""

# convert timezones and floats
for count, temp_obj in enumerate(payload["data"]):
epoch_time = temp_obj["t"]
utc_time = datetime.datetime.fromtimestamp(
int(epoch_time)
).strftime(self.time_format)
utc_time = datetime.datetime.fromtimestamp(int(epoch_time)).strftime(
self.time_format
)
payload["data"][count]["t"] = utc_time
for key in payload["data"][count]:
temp_val = ""
if "e+" in payload["data"][count][key] or "e-" in payload["data"][count][key]:
if (
"e+" in payload["data"][count][key]
or "e-" in payload["data"][count][key]
):
temp_val = payload["data"][count][key]
payload["data"][count][key] = float(temp_val)
pl = json.dumps(payload)

# convert ints, floats
pl = re.sub(r'\"(\d+)\"', r'\1', f"{pl}")
pl = re.sub(r'\"(\d+\.\d+)\"', r'\1', f"{pl}")
pl = re.sub(r"\"(\d+)\"", r"\1", f"{pl}")
pl = re.sub(r"\"(\d+\.\d+)\"", r"\1", f"{pl}")

# convert NaN to null
pl = re.sub(r'\"NaN\"', "null", f"{pl}")
pl = re.sub(r"\"NaN\"", "null", f"{pl}")

# replace "t" with time
pl = re.sub(r'\"t\"', r'"time"', f"{pl}")
pl = re.sub(r"\"t\"", r'"time"', f"{pl}")

# return response as JSON obj
return json.loads(pl)
Expand All @@ -111,10 +108,9 @@ def compile_result(self):
"step": "",
"end": "",
"rows": "",
"data_sources": []
"data_sources": [],
},
"data": [],

}

collector = defaultdict(dict)
Expand All @@ -123,19 +119,17 @@ def compile_result(self):
r = self.get_rrd_json(ds=d)
master_result["meta"]["start"] = datetime.datetime.fromtimestamp(
int(r["xport"]["meta"]["start"])
).strftime(self.time_format)
).strftime(self.time_format)
master_result["meta"]["step"] = r["xport"]["meta"]["step"]
master_result["meta"]["end"] = datetime.datetime.fromtimestamp(
int(r["xport"]["meta"]["end"])
).strftime(self.time_format)
).strftime(self.time_format)
master_result["meta"]["rows"] = 0
master_result["meta"]["data_sources"].append(
r["xport"]["meta"]["legend"]["entry"]
)
)

for collectible in chain(
master_result["data"], r["xport"]["data"]["row"]
):
for collectible in chain(master_result["data"], r["xport"]["data"]["row"]):
collector[collectible["t"]].update(collectible.items())

# combine objs, add row_count
Expand Down
Empty file removed backend/security.py
View file
Open in desktop
Empty file.
35 changes: 35 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
[tool.poetry]
name = "rrdrest"
version = "0.0.0" # Will be replaced by dynamic versioning
description = "Makes RRD files API-able"
authors = ["tbotnz <tonynealon1989@gmail.com>"]
readme = "README.md"
packages = [{include = "rrdrest.py"}, {include = "backend"}, {include = "__version__.py"}]

[tool.poetry.dependencies]
python = "^3.11"
fastapi = "^0.119.1"
xmltodict = "^1.0.2"
pydantic = "^2.12.3"
uvicorn = "^0.38.0"
uvloop = "^0.22.1"
gunicorn = "^23.0.0"
httptools = "^0.7.1"
ruff = "^0.14.1"

[tool.poetry-dynamic-versioning]
enable = true
vcs = "any"
style = "pep440"
metadata = false
format-jinja = "{{serialize_pep440(bump_version(base))}}"
# Read version from __version__.py
format = "{base}"
pattern = "default-unprefixed"

[tool.poetry-dynamic-versioning.substitution]
files = ["__version__.py"]

[build-system]
requires = ["poetry-core>=2.0.0,<3.0.0", "poetry-dynamic-versioning"]
build-backend = "poetry_dynamic_versioning.backend"
7 changes: 0 additions & 7 deletions requirements.txt
View file
Open in desktop

This file was deleted.

87 changes: 64 additions & 23 deletions rrdrest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,34 +1,75 @@
from fastapi import FastAPI, HTTPException
from pydantic import Field

from backend.RRD_parse import RRD_parser
from __version__ import __version__

from typing import Optional

from typing import Optional, Annotated
import os

rrd_rest = FastAPI(
title="RRDReST",
description="Makes RRD files API-able",
version="0.2",
version=__version__,
)


@rrd_rest.get(
"/",
summary="Get the data from a RRD file, takes in a rrd file path"
)
async def get_rrd(rrd_path: str, epoch_start_time: Optional[int] = None, epoch_end_time: Optional[int] = None):
is_file = os.path.isfile(rrd_path)
if is_file:
if (epoch_start_time and not epoch_end_time) or (epoch_end_time and not epoch_start_time):
raise HTTPException(status_code=500, detail="If epoch start or end time is specified both start and end time MUST be specified")
try:
rr = RRD_parser(
rrd_file=rrd_path,
start_time=epoch_start_time,
end_time=epoch_end_time
)
r = rr.compile_result()
return r
except Exception as e:
HTTPException(status_code=500, detail=f"{e}")
raise HTTPException(status_code=404, detail="RRD not found")
def _validate_time_params(
epoch_start_time: Optional[int], epoch_end_time: Optional[int]
):
if (epoch_start_time and not epoch_end_time) or (
epoch_end_time and not epoch_start_time
):
raise HTTPException(
status_code=400,
detail="If epoch start or end time is specified both start and end time MUST be specified",
)


def _parse_rrd_file(
rrd_path: Annotated[
str, Field(description="RRD file path", examples=["/path/to/file.rrd"])
],
epoch_start_time: Optional[int],
epoch_end_time: Optional[int],
):
if not os.path.isfile(rrd_path):
raise HTTPException(status_code=404, detail=f"RRD file not found: {rrd_path}")

_validate_time_params(epoch_start_time, epoch_end_time)

try:
rr = RRD_parser(
rrd_file=rrd_path, start_time=epoch_start_time, end_time=epoch_end_time
)
return rr.compile_result()
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error parsing RRD file: {e}")


@rrd_rest.get("/", summary="Get data from a RRD file, takes in a rrd file path")
async def get_rrd(
rrd_path: str,
epoch_start_time: Optional[int] = None,
epoch_end_time: Optional[int] = None,
):
return _parse_rrd_file(rrd_path, epoch_start_time, epoch_end_time)


@rrd_rest.post("/", summary="Get multiple RRD files, takes in a list of rrd file paths")
async def get_rrd_multiple(
rrd_paths: list[
Annotated[
str, Field(description="RRD file path", examples=["/path/to/file.rrd"])
]
],
epoch_start_time: Optional[int] = None,
epoch_end_time: Optional[int] = None,
):
results = {}
for rrd_path in rrd_paths:
result = _parse_rrd_file(rrd_path, epoch_start_time, epoch_end_time)
results[rrd_path] = result
if not results:
raise HTTPException(status_code=404, detail="No RRD Data Found")
return results