From 0526dee6f4a51c79a8ec5f456197ce3fa82db97d Mon Sep 17 00:00:00 2001 From: clock Date: Mon, 8 Dec 2025 17:07:09 -0600 Subject: [PATCH 1/4] Update NRP page --- _services/nrp.html | 77 ++++++++++++++++- assets/js/pages/ospool-nrp-projects-v1.js | 101 ++++++++++++++++++---- 2 files changed, 161 insertions(+), 17 deletions(-) diff --git a/_services/nrp.html b/_services/nrp.html index 7ecec4a9..2e66f8b6 100644 --- a/_services/nrp.html +++ b/_services/nrp.html @@ -53,7 +53,80 @@

NRP contributes to PATh by:

  • Making it easier for institutions to contribute capacity to the OSPool due to the system infrastructure that NRP can run on a contributing institution's behalf.
  • Providing access for its community to PATh's Open Science Data Federation (OSDF) and Pelican services for data access and staging.
    • - + + + +
    +
    +
    +
    + +
    +
    +
    Day
    +
    +
    +

    CPU

    +

    Loading

    +
    +
    + Memory: + [Memory Placeholder] +
    +
    +

    GPU

    +

    Loading

    +
    +
    +
    +
    + +
    +
    +
    Week
    +
    +
    +

    CPU

    +

    Loading

    +
    +
    + Memory: + [Memory Placeholder] +
    +
    +

    GPU

    +

    Loading

    +
    +
    +
    +
    + +
    +
    +
    Month
    +
    +
    +

    CPU

    +

    Loading

    +
    +
    + Memory: + [Memory Placeholder] +
    +
    +

    GPU

    +

    Loading

    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    The below projects used NRP resources on the OSPool to advance their research in the past year and ran more than 100 jobs. To run your own research on the OSPool sign up now on the OSG Portal. @@ -66,7 +139,7 @@

    NRP contributes to PATh by:

    -
    +
    diff --git a/assets/js/pages/ospool-nrp-projects-v1.js b/assets/js/pages/ospool-nrp-projects-v1.js index a2b9b213..4126603f 100644 --- a/assets/js/pages/ospool-nrp-projects-v1.js +++ b/assets/js/pages/ospool-nrp-projects-v1.js @@ -1,10 +1,26 @@ import { fetchForBackup, fetchWithBackup } from "/assets/js/backup.js"; import {getInstitutionsOverview, getProjects} from "/assets/js/adstash.mjs" -import {locale_int_string_sort, string_sort} from "/assets/js/util.js"; +import {byteStringToBytes, formatBytes, locale_int_string_sort, sortByteString, string_sort} from "/assets/js/util.js"; import {PieChart} from "/assets/js/components/pie-chart.js"; import ProjectDisplay from "/assets/js/components/ProjectDisplay.mjs"; import {Grid, PluginPosition, BaseComponent, h} from "https://unpkg.com/gridjs@5.1.0/dist/gridjs.module.js" +const formatOrg = (cell, row, column) => { + + let url = row?._cells?.[5]?.['data'] + + // If url has no protocol add https + if(url !== undefined && !url.startsWith("http")) { + url = "https://" + url + } + + if(url !== undefined) { + return h('td', {}, h('a', {href: url, target: '_blank'}, cell)) + } + + return h('td', {}, cell) +} + class Table { constructor(wrapper, data_function, updateProjectDisplay){ this.grid = undefined @@ -12,12 +28,6 @@ class Table { this.wrapper = wrapper this.updateProjectDisplay = updateProjectDisplay this.columns = [ - { - id: 'numJobs', - name: 'Jobs Ran', - data: (row) => Math.floor(row.numJobs).toLocaleString(), - sort: { compare: locale_int_string_sort } - }, { id: 'projectName', name: 'Name', @@ -26,11 +36,24 @@ class Table { className: "gridjs-th gridjs-td pointer gridjs-th-sort text-start" } }, { - id: 'PIName', - name: 'PI Name', - sort: { compare: string_sort }, - attributes: { - className: "gridjs-th gridjs-td pointer gridjs-th-sort text-start" + id: 'numJobs', + name: 'Jobs Ran', + data: (row) => Math.floor(row.numJobs).toLocaleString(), + sort: { compare: locale_int_string_sort } + }, { + id: 'osdfByteTransferCount', + name: 'Bytes Transferred', + sort: { compare: sortByteString }, + data: (row) => formatBytes(row.osdfByteTransferCount), + attributes: (cell, row, column) => { + if(cell !== null && table?.data !== undefined){ + const data = table.data + const maxByteCount = Math.max(...Object.values(data).map(x => x.osdfByteTransferCount)) + const cellValue = byteStringToBytes(cell) + const colorValue = Math.min(1, 1 - Math.log(4 * (cellValue / maxByteCount) + 1)) + const color = whiteorange(colorValue) // 1 - Math.log((cellValue / maxFileCount)) + return {style: {backgroundColor: color}, className: "text-end"} + } } }, { id: 'Organization', @@ -38,7 +61,8 @@ class Table { sort: { compare: string_sort }, attributes: { className: "gridjs-th gridjs-td pointer gridjs-th-sort text-start" - } + }, + formatter: formatOrg }, { id: 'detailedFieldOfScience', name: 'Field Of Science', @@ -46,6 +70,9 @@ class Table { attributes: { className: "gridjs-th gridjs-td pointer gridjs-th-sort text-start" } + }, { + id: 'projectInstitutionIpedsWebsiteAddress', + hidden: true } ] @@ -65,7 +92,7 @@ class Table { data: async () => Object.values(await table.data_function()).sort((a, b) => b.numJobs - a.numJobs), pagination: { enabled: true, - limit: 5, + limit: 10, buttonsCount: 1 }, style: { @@ -83,7 +110,7 @@ class Table { row_click = async (PointerEvent, e) => { let data = await this.data_function() - let row_name = e["cells"][1].data + let row_name = e["cells"][0].data let project = data[row_name] this.updateProjectDisplay({...project, FieldOfScience: project.detailedFieldOfScience}) } @@ -257,9 +284,53 @@ class ProjectPage{ } } +class NrpOverview { + + constructor() { + this.initialize() + } + + async initialize() { + const timeConfigs = [ + {timespan: 1, cardSuffix: 'day'}, + {timespan: 7, cardSuffix: 'week'}, + {timespan: 30, cardSuffix: 'month'} + ]; + // Fetch all data in parallel + const results = await Promise.all(timeConfigs.map(cfg => this.getData(cfg.timespan))); + // Update card sections + results.forEach((data, idx) => { + const cardSuffix = timeConfigs[idx].cardSuffix; + if (document.querySelector(`.card-cpu-${cardSuffix}`)) { + document.querySelector(`.card-cpu-${cardSuffix}`).textContent = data['cpu'] ? Number(data['cpu']).toLocaleString() : 'N/A'; + } + if (document.querySelector(`.card-memory-${cardSuffix}`)) { + document.querySelector(`.card-memory-${cardSuffix}`).textContent = data['memory'] ? Number(data['memory']).toLocaleString() : 'N/A'; + } + if (document.querySelector(`.card-gpu-${cardSuffix}`)) { + document.querySelector(`.card-gpu-${cardSuffix}`).textContent = data['gpu'] ? Number(data['gpu']).toLocaleString() : 'N/A'; + } + }); + } + + async getData(timespan) { + + const currentTimestamp = Math.floor(Date.now() / 1000) + const response = await fetch(`https://thanos.nrp-nautilus.io/api/v1/query?query=sum%20by(resource)%20(sum_over_time(namespace_used_resources{namespace=~%22osg-opportunistic|icecube-ml|osg-icecube|osg-ligo|osg-nrao%22}[${timespan}d:1h]@${currentTimestamp}))&dedup=true&partial_response=false`) + const data = await response.json() + return data['data']['result'].reduce((acc, item) => { + return { + ...acc, + [item['metric']['resource']]: item['value'][1] + } + }, {}) + } +} + const tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) const tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { return new bootstrap.Tooltip(tooltipTriggerEl) }) const project_page = new ProjectPage() +const nrp_overview = new NrpOverview() From 241dee42bca7d27ab05fb4aca97b25d25f826a2c Mon Sep 17 00:00:00 2001 From: clock Date: Wed, 10 Dec 2025 16:16:55 -0600 Subject: [PATCH 2/4] Update NRP page --- .../workflows/deploy-production-same-repo.yml | 8 ++ Gemfile.lock | 2 +- _services/nrp.html | 122 ++++++++++++++---- assets/css/style.scss | 4 + assets/data/backups/placeholder.txt | 1 + assets/js/backup.js | 37 +++--- assets/js/nrp.mjs | 16 +++ assets/js/pages/ospool-nrp-projects-v1.js | 104 ++++----------- 8 files changed, 167 insertions(+), 127 deletions(-) create mode 100644 assets/data/backups/placeholder.txt create mode 100644 assets/js/nrp.mjs diff --git a/.github/workflows/deploy-production-same-repo.yml b/.github/workflows/deploy-production-same-repo.yml index 3c2367cf..992e590a 100644 --- a/.github/workflows/deploy-production-same-repo.yml +++ b/.github/workflows/deploy-production-same-repo.yml @@ -43,6 +43,14 @@ jobs: ruby-version: 3.4 bundler-cache: true + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: 'npm' + + - name: Download Backup Data + run: npm run backup + - name: Build and Deploy run: | # Build the production version of website diff --git a/Gemfile.lock b/Gemfile.lock index 9e195cda..13ef42fc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -311,4 +311,4 @@ DEPENDENCIES jekyll-sitemap BUNDLED WITH - 2.7.1 + 4.0.1 diff --git a/_services/nrp.html b/_services/nrp.html index 2e66f8b6..0eb685ed 100644 --- a/_services/nrp.html +++ b/_services/nrp.html @@ -46,7 +46,7 @@

    PATh and National Research Platform (NRP) collaborate and share a common mission to build open cyberinfrastructure and expand services for scientific research affiliated with US academic institutions. Both projects are supported by the National Science Foundation (NSF)'s Office of Advanced Cyberinfrastructure (OAC) award.

    -

    NRP contributes to PATh by:

    +

    NRP contributes to PATh by:

    • Increasing PATh's overall computing capacity by contributing to the OSPool, a pool of shared computing capacity accessible to researchers affiliated with a US academic institution.
    • Sharing their global Kubernetes infrastructure, which manages the deployment of PATh services.
      • @@ -56,26 +56,38 @@

      NRP contributes to PATh by:

    -
    +
    +
    +

    + Totals +

    +

    NRP contributions to the OSPool.

    +
    - +
    -
    Day
    +
    Month
    -

    CPU

    -

    Loading

    +

    CPU

    +
    +

    Loading

    + Core Hours +
    Memory: - [Memory Placeholder] + [Memory Placeholder]
    -

    GPU

    -

    Loading

    +

    GPU

    +
    +

    Loading

    + Hours +
    @@ -86,36 +98,48 @@

    Loading

    Week
    -

    CPU

    -

    Loading

    +

    CPU

    +
    +

    Loading

    + Core Hours +
    Memory: [Memory Placeholder]
    -

    GPU

    -

    Loading

    +

    GPU

    +
    +

    Loading

    + Hours +
    - +
    -
    Month
    +
    Day
    -

    CPU

    -

    Loading

    +

    CPU

    +
    +

    Loading

    + Core Hours +
    Memory: - [Memory Placeholder] + [Memory Placeholder]
    -

    GPU

    -

    Loading

    +

    GPU

    +
    +

    Loading

    + Hours +
    @@ -124,20 +148,66 @@

    Loading

    -
    +
    -

    - The below projects used NRP resources on the OSPool to advance their research in the past year and ran more than 100 jobs. - To run your own research on the OSPool sign up now on the OSG Portal. -

    +

    + NRP Contributions by Project +

    +

    NRP contributions to specific projects via the OSPool.

    +
    +
    +
    +
    +
    +
    +
    +
    LIGO
    +
    +
    +

    CPU

    +
    +

    Loading

    + Core Hours +
    +
    +
    +

    GPU

    +
    +

    Loading

    + Hours +
    +
    +
    +
    +
    +
    +
    +
    IceCube
    +
    +
    +

    CPU

    +
    +

    Loading

    + Core Hours +
    +
    +
    +

    GPU

    +
    +

    Loading

    + Hours +
    +
    +
    +
    -
    +
    diff --git a/assets/css/style.scss b/assets/css/style.scss index 4bcf03b2..d79bf859 100755 --- a/assets/css/style.scss +++ b/assets/css/style.scss @@ -339,3 +339,7 @@ main { // not collapsed, hide gradient display: none; } + +.fs-7 { + font-size: 0.875rem !important; +} diff --git a/assets/data/backups/placeholder.txt b/assets/data/backups/placeholder.txt new file mode 100644 index 00000000..d223ed68 --- /dev/null +++ b/assets/data/backups/placeholder.txt @@ -0,0 +1 @@ +Used to hold the directory structure for backups. \ No newline at end of file diff --git a/assets/js/backup.js b/assets/js/backup.js index 8eac3f53..8215ed9e 100644 --- a/assets/js/backup.js +++ b/assets/js/backup.js @@ -3,9 +3,9 @@ import {generateHash} from './util.js'; import { - getProjects, getInstitutionOverview, getProjectOverview, getLatestOSPoolOverview, - getInstitutionsOverview + getProjects, getProjectOverview } from './adstash.mjs'; +import {getNrpPrometheusData, updateTotals} from "./nrp.mjs"; const BACKUP_DIRECTORY = '/assets/data/backups/' @@ -26,34 +26,29 @@ const backupMap = async () => { function: fetchForBackup, args: ["https://topology.opensciencegrid.org/miscproject/json"], }, - { - function: fetchForBackup, - args: ["https://osg-htc.org/ospool-data/data/daily_reports/latest.json"], - }, - { - function: getLatestOSPoolOverview - }, { function: getProjects, }, - { - function: getInstitutions, - }, - { - function: getInstitutionsOverview - }, + { + function: updateTotals, + args: [[ + {timespan: 1, cardSuffix: 'day'}, + {timespan: 7, cardSuffix: 'week'}, + {timespan: 30, cardSuffix: 'month'} + ]] + }, ...( Object.values(await getProjects()).map( project => ({ function: getProjectOverview, args: [project.projectName], }) ) ), - ...( - Object.values(await getInstitutions()).map( institution => ({ - function: getInstitutionOverview, - args: [institution.institutionName], - }) ) - ), + ...( + ['icecube', 'ligo'].map(org => ({ + function: getNrpPrometheusData, + args: [365, `%22osg-${org}%22`] + })) + ) ] } diff --git a/assets/js/nrp.mjs b/assets/js/nrp.mjs new file mode 100644 index 00000000..1d2a5f6d --- /dev/null +++ b/assets/js/nrp.mjs @@ -0,0 +1,16 @@ +export const updateTotals = async (timeConfigs) => { + // Fetch all data in parallel + return await Promise.all(timeConfigs.map(cfg => getNrpPrometheusData(cfg.timespan, "~%22osg-opportunistic|icecube-ml|osg-icecube|osg-ligo|osg-nrao%22"))); +} + +export const getNrpPrometheusData = async (timespan, namespaceFilter=null) => { + const currentTimestamp = Math.floor(Date.now() / 1000) + const response = await fetch(`https://thanos.nrp-nautilus.io/api/v1/query?query=sum%20by(resource)%20(sum_over_time(namespace_used_resources{namespace=${namespaceFilter}}[${timespan}d:1h]@${currentTimestamp}))&dedup=true&partial_response=false`) + const data = await response.json() + return data['data']['result'].reduce((acc, item) => { + return { + ...acc, + [item['metric']['resource']]: item['value'][1] + } + }, {}) +} diff --git a/assets/js/pages/ospool-nrp-projects-v1.js b/assets/js/pages/ospool-nrp-projects-v1.js index 4126603f..113fc80e 100644 --- a/assets/js/pages/ospool-nrp-projects-v1.js +++ b/assets/js/pages/ospool-nrp-projects-v1.js @@ -1,9 +1,9 @@ import { fetchForBackup, fetchWithBackup } from "/assets/js/backup.js"; -import {getInstitutionsOverview, getProjects} from "/assets/js/adstash.mjs" +import {getProjects} from "/assets/js/adstash.mjs" import {byteStringToBytes, formatBytes, locale_int_string_sort, sortByteString, string_sort} from "/assets/js/util.js"; -import {PieChart} from "/assets/js/components/pie-chart.js"; import ProjectDisplay from "/assets/js/components/ProjectDisplay.mjs"; -import {Grid, PluginPosition, BaseComponent, h} from "https://unpkg.com/gridjs@5.1.0/dist/gridjs.module.js" +import {Grid, h} from "https://unpkg.com/gridjs@5.1.0/dist/gridjs.module.js" +import {getNrpPrometheusData, updateTotals} from "../nrp.mjs"; const formatOrg = (cell, row, column) => { @@ -81,7 +81,7 @@ class Table { columns: table.columns, sort: true, search: { - enabled: true + enabled: false }, className: { container: "", @@ -128,16 +128,6 @@ class DataManager { this.consumerToggles.forEach(f => f()) } - addFilter = (name, filter) => { - this.filters[name] = filter - this.toggleConsumers() - } - - removeFilter = (name) => { - delete this.filters[name] - this.toggleConsumers() - } - getData = async () => { if(!this.data) { this.data = this._getData() @@ -234,53 +224,10 @@ class ProjectPage{ this.wrapper = document.getElementById("wrapper") this.table = new Table(this.wrapper, this.dataManager.getFilteredData, this.projectDisplay.update.bind(this.projectDisplay)) - this.dataManager.addFilter("search", this.search.filter) - let urlProject = new URLSearchParams(window.location.search).get('project') if(urlProject){ this.projectDisplay.update((await this.dataManager.getData())[urlProject]) } - - this.orgPieChart = new PieChart( - "project-fos-cpu-summary", - this.dataManager.reduceByKey.bind(this.dataManager, "detailedFieldOfScience", "cpuHours"), - "# of CPU Hours by Field of Science" - ) - this.FosPieChart = new PieChart( - "project-fos-job-summary", - this.dataManager.reduceByKey.bind(this.dataManager, "detailedFieldOfScience", "numJobs"), - "# of Jobs by Field Of Science" - ) - this.jobPieChart = new PieChart( - "project-job-summary", - this.dataManager.reduceByKey.bind(this.dataManager, "projectName", "numJobs"), - "# of Jobs by Project", - ({label, value}) => { - this.table.updateProjectDisplay(this.dataManager.data[label]) - } - ) - this.cpuPieChart = new PieChart( - "project-cpu-summary", - this.dataManager.reduceByKey.bind(this.dataManager, "projectName", "cpuHours"), - "# of CPU Hours by Project", - ({label, value}) => { - this.table.updateProjectDisplay(this.dataManager.data[label]) - } - ) - this.gpuPieChart = new PieChart( - "project-gpu-summary", - this.dataManager.reduceByKey.bind(this.dataManager, "projectName", "gpuHours"), - "# of GPU Hours by Project", - ({label, value}) => { - this.table.updateProjectDisplay(this.dataManager.data[label]) - } - ) - - this.dataManager.consumerToggles.push(this.orgPieChart.update) - this.dataManager.consumerToggles.push(this.FosPieChart.update) - this.dataManager.consumerToggles.push(this.jobPieChart.update) - this.dataManager.consumerToggles.push(this.cpuPieChart.update) - this.dataManager.consumerToggles.push(this.gpuPieChart.update) } } @@ -291,40 +238,39 @@ class NrpOverview { } async initialize() { + await Promise.all([this.updateIndividualOrgs(), this.updateTotals()]) + } + + async updateIndividualOrgs() { + for(const org of ['icecube', 'ligo']) { + const orgData = (await fetchWithBackup(getNrpPrometheusData, 365, `%22osg-${org}%22`))['data'] + for(const key of ['cpu', 'memory', 'nvidia_com_gpu']) { + const htmlNode = document.querySelector(`#card-${org}-${key}`); + if (htmlNode) { + htmlNode.textContent = orgData[key] ? Math.floor(Number(orgData[key])).toLocaleString() : 'N/A'; + } + } + } + } + + async updateTotals() { const timeConfigs = [ {timespan: 1, cardSuffix: 'day'}, {timespan: 7, cardSuffix: 'week'}, {timespan: 30, cardSuffix: 'month'} ]; // Fetch all data in parallel - const results = await Promise.all(timeConfigs.map(cfg => this.getData(cfg.timespan))); + const results = (await fetchWithBackup(updateTotals, timeConfigs))['data'] // Update card sections results.forEach((data, idx) => { const cardSuffix = timeConfigs[idx].cardSuffix; - if (document.querySelector(`.card-cpu-${cardSuffix}`)) { - document.querySelector(`.card-cpu-${cardSuffix}`).textContent = data['cpu'] ? Number(data['cpu']).toLocaleString() : 'N/A'; - } - if (document.querySelector(`.card-memory-${cardSuffix}`)) { - document.querySelector(`.card-memory-${cardSuffix}`).textContent = data['memory'] ? Number(data['memory']).toLocaleString() : 'N/A'; - } - if (document.querySelector(`.card-gpu-${cardSuffix}`)) { - document.querySelector(`.card-gpu-${cardSuffix}`).textContent = data['gpu'] ? Number(data['gpu']).toLocaleString() : 'N/A'; + for(const key of ['cpu', 'memory', 'nvidia_com_gpu']) { + if (document.querySelector(`#card-${key}-${cardSuffix}`)) { + document.querySelector(`#card-${key}-${cardSuffix}`).textContent = data[key] ? Math.floor(Number(data[key])).toLocaleString() : 'N/A'; + } } }); } - - async getData(timespan) { - - const currentTimestamp = Math.floor(Date.now() / 1000) - const response = await fetch(`https://thanos.nrp-nautilus.io/api/v1/query?query=sum%20by(resource)%20(sum_over_time(namespace_used_resources{namespace=~%22osg-opportunistic|icecube-ml|osg-icecube|osg-ligo|osg-nrao%22}[${timespan}d:1h]@${currentTimestamp}))&dedup=true&partial_response=false`) - const data = await response.json() - return data['data']['result'].reduce((acc, item) => { - return { - ...acc, - [item['metric']['resource']]: item['value'][1] - } - }, {}) - } } const tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) From 0857f0fd63ac967e4ec324bcbde684dec1433d8f Mon Sep 17 00:00:00 2001 From: clock Date: Wed, 10 Dec 2025 16:21:50 -0600 Subject: [PATCH 3/4] Fix text to be correctly phrased --- _services/nrp.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_services/nrp.html b/_services/nrp.html index 0eb685ed..f0a93d56 100644 --- a/_services/nrp.html +++ b/_services/nrp.html @@ -62,7 +62,7 @@

    NRP contributes to PATh by:

    Totals

    -

    NRP contributions to the OSPool.

    +

    OSPool usage of NRP Contributions.

    @@ -152,9 +152,9 @@

    Loading

    - NRP Contributions by Project + NRP Contribution use by Project

    -

    NRP contributions to specific projects via the OSPool.

    +

    Project usage of NRP resource contributions.

    From 85276c785a92e6cca297e0437deda12eb412bb2d Mon Sep 17 00:00:00 2001 From: clock Date: Wed, 10 Dec 2025 16:22:22 -0600 Subject: [PATCH 4/4] Fix text to be correctly phrased --- _services/nrp.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_services/nrp.html b/_services/nrp.html index f0a93d56..c3e30cbc 100644 --- a/_services/nrp.html +++ b/_services/nrp.html @@ -154,7 +154,7 @@

    Loading

    NRP Contribution use by Project

    -

    Project usage of NRP resource contributions.

    +

    Project usage of NRP resource contributions last year.