From 628af1c6e279e770a29e3471f6cccd59dc7f3465 Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Fri, 16 May 2025 22:52:36 +0000 Subject: [PATCH 1/8] Composite Time Series Design document. --- docs/source/design/composite-time-series.rst | 220 +++++++++++++++++++ docs/source/design/index.rst | 12 + docs/source/index.rst | 2 + docs/source/libraries/java.rst | 2 - 4 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 docs/source/design/composite-time-series.rst create mode 100644 docs/source/design/index.rst diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst new file mode 100644 index 000000000..8e49aef49 --- /dev/null +++ b/docs/source/design/composite-time-series.rst @@ -0,0 +1,220 @@ +##################### +Composite Time Series +##################### + +Purpose +======= + +It is a challenge for users to identity what the correct authoritative time series is for a given measurement at a location. Additionally these time series often change over time, either being completely new or changing their interval as newer technologies become available. + +Gather an entire Period of Record for the value at a location is also rather difficult. And the POR record and "authoritative timeseries" may be one-in-the same. + + +Need +==== + +#. CWMS and Access-2-Water require a simple mechanism to allow users of data to retrieve the Authoritative Period of Record data for a given measurement without having to understand all of the possible component time series that may be involved. +#. Period-of-Record time series *should* not be created by duplicating data from the component time series and merging them into a new one. +#. The naming of the time series should fit within the excepting CWMS Time Series Identifier design and not unreasonably interfere with existing usages. + + +Caveats +======= + +#. It is assumed that CWMS-Vue will, as-always, require updates to handle what is created here. + #. e.g. we're not going to let any current limitations of CWMS-Vue hinder our design. + + +Proposal +======== + +Description +----------- + +CDA should handle a concept of a "Composite Time Series". Whether a Time Series is considered composite will be determined by a specific element of the Time Series Identifier. +Data Administrators will configure which Time Series, and the range there-in, are part of the composite time series. +CDA will use this stored information to build the Time Series per the question. + +Additional names not used +------------------------- + +#. Virtual Time Series +#. Period of Record Time Series + +Both names have been discarded. We use "Virtual" in too many other places with a more direct meaning of that word. +For Period-of-Record, while that is the primary use-case, the concept is useful in other situations as well. + +Hence generically have have a "composite time series" + +Axioms +------ + +#. Composite Time Series are Irregular +#. The definition of the composite time series is stored within the CWMS database +#. The members of a composite time series define a continuous range + #. The date ranges of a composite time series *MUST* not overlap + #. The date ranges of a composite time series *MUST* have any gaps + #. Data may have gaps, an explanation range should be provided. +#. The members of a composite time measure the same thing. (e.g. all members are Elevation, not some are elevation and some are stage.) + + +Time Series Naming +------------------ + +Option 1 +~~~~~~~~ + +`...Composite.var.` + ++----------------------+------------------------------------------------------------------------------------------------------------------------+ +| Element | Description | ++----------------------+------------------------------------------------------------------------------------------------------------------------+ +|Location Id |As the normal CWMS TS ID, the location for this measure | ++----------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter |As the normal CWMS TS ID, the measurement (e.g. Stage, Precip, Elevation, flow, etc) | ++----------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter Type |As Normal CWMS TS ID, Instantaneous, average, total, etc | ++----------------------+------------------------------------------------------------------------------------------------------------------------+ +|Interval -\> Composite| Marker that this time series does not have a fix information and is build of various member time series. | ++----------------------+------------------------------------------------------------------------------------------------------------------------+ +|Duration -\> var |Duration of average or total may change over time with new members, duration will be indicated in the member definition | ++----------------------+------------------------------------------------------------------------------------------------------------------------+ +|Version |As Normal CWMS TS ID | ++----------------------+------------------------------------------------------------------------------------------------------------------------+ + + +Option 2 +~~~~~~~~ + +`..Composite.0.0.` + + ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +| Element | Description | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Location Id |As the normal CWMS TS ID, the location for this measure | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter |As the normal CWMS TS ID, the measurement (e.g. Stage, Precip, Elevation, flow, etc) | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter Type Composite|Marker that this time series does not have a fix information and is build of various member time series. | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Interval -\> 0 |Interval of data elements. may change over time with new members, duration will be indicated in the member definition | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Duration -\> 0 |Duration of average or total. may change over time with new members, duration will be indicated in the member definition| ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Version |As Normal CWMS TS ID | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ + +The zero's could also be var + +Composite Time Series Definition +================================ + +.. code-block::jsonc + + { + "office": "", + "name": "", + "is-period-of-record": true, // or is authoritative. to distinguish between other possible use-cases? + "members": [ + { + "time-series-id": "TS ID for this range", + "start": "start date of this", + "end": "end date of this range", + "notes": "text", + "version", "version date", // maybe not? could just use POR or period-of-record in the ts id version + // if values that equals the start or end timestamp are included + "start-inclusive": true, + "end-inclusive": false + // suggest default of "start-inclusive": true, "end-inclusive": false + // it may also make sense to just make this *always* the above and not let the user set it. + // alternatively if this is always [) then only start is required. + // however, the class would required and end field as the actual Time Series output needs to know the actual end. + } + ] + // array above *should* be sorted by start when provided to user. + } + + +Operations required: + +* Create +* Remove member (ts id + range) +* Add member +* List members +* Replace all members? +* Delete + + +Composite Time Series Response +============================== + +.. code-block::jsonc + + { + // ... as current TimeSeries JSON + "composite-members-present": [ + // member definition from above + ] + } + + +Supported Operations: + +* Get, through existing TimeSeries classes. + +Does it make sense to support writing directly to a composite time series. While the write of each element *could* be sent to the underlying member, this seems ripe for error when editing or updating any data. It is likely that any edits would always be to the most recent time series, and configured in some other system. + +Storage of member information +================================ + +#. Store in Clob as we refine the design +#. Create appropriate tables once the design is stable. + +System responsibility for "knowing" to process composite. +========================================================= + +Time Series Catalog +------------------- + +Time Series Catalog should show composite time series and allow searching by "authoritative" + +TimeSeries DTO +-------------- + +Add nullable "members" property. + +TimeSeriesDao +------------- + +If the system sees the "Composite" marker retrieve the members for the range and build the time series. + +NOTE: considering the user may request the *entire* Period-of-record, this is a good opportunity to see that, start the retrieval in a job queue, and return a status URL to the user for future download. I have see such mechanism for bulk data in other systems. Maybe return an "I'm working on it variant" that the controller can know how to format. + +Error handling and other conditions. +==================================== + +Versioned (date) time series +---------------------------- + +As the composite time series is comprised of multiple other time series should this always be an error to specify? +The marker for always latest or always first may make sense to allow, however, at the time series is supposed to be authoritative, that would add ambiguity. + +Datum conversions +----------------- + +Retrievers of the Period-of-Record *SHOULD* be able to retrieve the data as a single datum. Composite retrieval should respond as https://github.com/USACE/cwms-data-api/issues/1102 and convert each member as appropriate + + +On the saving of a composite definition +--------------------------------------- + +The even if only a single member is added, the full definition needs to be check to ensure the ranges are still overlapping and continuous. + +References +========== + +#. https://github.com/USACE/cwms-data-api/discussions/956 +#. https://github.com/USACE/cwms-data-api/issues/955 +#. https://www.hec.usace.army.mil/confluence/spaces/CWMS/pages/290456000/Virtual+Timeseries +#. https://discourse.hecdev.net/t/period-of-record-timeseries/3859/2 \ No newline at end of file diff --git a/docs/source/design/index.rst b/docs/source/design/index.rst new file mode 100644 index 000000000..8f91deba5 --- /dev/null +++ b/docs/source/design/index.rst @@ -0,0 +1,12 @@ +################ +Design Documents +################ + +The follow pages formally document current and proposed designs +relating to operations and usage of data. + +.. toctree:: + :maxdepth: 2 + :caption: Introduction + + Composite Time Series <./composite-time-series.rst> diff --git a/docs/source/index.rst b/docs/source/index.rst index c2bfdf963..1c001280d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -43,3 +43,5 @@ Welcome to CWMS Data API documentation! Alternative Topics <./alternative-topics/index.rst> + + Design Documents <./design/index.rst> \ No newline at end of file diff --git a/docs/source/libraries/java.rst b/docs/source/libraries/java.rst index f94f6e318..ee1a88f88 100644 --- a/docs/source/libraries/java.rst +++ b/docs/source/libraries/java.rst @@ -4,5 +4,3 @@ CWMS Java Client Library - cwmsjava ===================================== This page is coming soon. Please check back later for updates and new content. - - From d2d5306674ac9e6e290096faf4189885af28061c Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Tue, 19 Aug 2025 20:18:57 +0000 Subject: [PATCH 2/8] Updates from PR comments. --- docs/source/design/composite-time-series.rst | 124 +++++++++++++++---- 1 file changed, 101 insertions(+), 23 deletions(-) diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst index 8e49aef49..0f74e1539 100644 --- a/docs/source/design/composite-time-series.rst +++ b/docs/source/design/composite-time-series.rst @@ -44,7 +44,7 @@ Additional names not used Both names have been discarded. We use "Virtual" in too many other places with a more direct meaning of that word. For Period-of-Record, while that is the primary use-case, the concept is useful in other situations as well. -Hence generically have have a "composite time series" +Hence generically we have a "composite time series" Axioms ------ @@ -52,10 +52,11 @@ Axioms #. Composite Time Series are Irregular #. The definition of the composite time series is stored within the CWMS database #. The members of a composite time series define a continuous range - #. The date ranges of a composite time series *MUST* not overlap - #. The date ranges of a composite time series *MUST* have any gaps + #. The date ranges of a member *MUST* not overlap + #. The date ranges of a member *MUST* not have any gaps #. Data may have gaps, an explanation range should be provided. #. The members of a composite time measure the same thing. (e.g. all members are Elevation, not some are elevation and some are stage.) +#. The interval and duration of each member *MAY* be different. Time Series Naming @@ -107,6 +108,87 @@ Option 2 The zero's could also be var + +Option 3 +~~~~~~~~ + +`.....Composite` + + ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +| Element | Description | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Location Id |As the normal CWMS TS ID, the location for this measure | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter |As the normal CWMS TS ID, the measurement (e.g. Stage, Precip, Elevation, flow, etc) | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter Type |Marker that this time series does not have a fix information and is build of various member time series. | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Interval |Interval of data elements. may change over time with new members, duration will be indicated in the member definition | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Duration |Duration of average or total. may change over time with new members, duration will be indicated in the member definition| ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Version |Composite or POR ... or check for composite at the front/back? | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ + +From Daniel + +Argument Against: the "Version" field it freeform and we often encode other information in it. +Argument Against above argument: That said, perhaps forcing the version to be "clean" is the right choice here. + + +Option 4 +~~~~~~~~ + +`.[Composite]....` + + ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +| Element | Description | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Location Id |As the normal CWMS TS ID, the location for this measure | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter |As the normal CWMS TS ID, the measurement (e.g. Stage, Precip, Elevation, flow, etc) | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter Type |Marker that this time series does not have a fix information and is build of various member time series. | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Interval |Interval of data elements. may change over time with new members, duration will be indicated in the member definition | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Duration |Duration of average or total. may change over time with new members, duration will be indicated in the member definition| ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Version |As Normal CWMS TS ID | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ + + +This form with something in [] has been discussed for embedded TimeZone and Offset information into the interval. Arguably this code go in any field. + + +Option 4 +~~~~~~~~ + +`.....` and/or arbitrary TS "alias" + + ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +| Element | Description | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Location Id |As the normal CWMS TS ID, the location for this measure | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter |As the normal CWMS TS ID, the measurement (e.g. Stage, Precip, Elevation, flow, etc) | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Parameter Type |Marker that this time series does not have a fix information and is build of various member time series. | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Interval |Interval of data elements. may change over time with new members, duration will be indicated in the member definition | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Duration |Duration of average or total. may change over time with new members, duration will be indicated in the member definition| ++------------------------+------------------------------------------------------------------------------------------------------------------------+ +|Version |As Normal CWMS TS ID | ++------------------------+------------------------------------------------------------------------------------------------------------------------+ + +However, on request for the timeseries the list of composite time series is consulted and used if present, otherwise passthrough to normal +time series retrieval. + + Composite Time Series Definition ================================ @@ -115,21 +197,13 @@ Composite Time Series Definition { "office": "", "name": "", - "is-period-of-record": true, // or is authoritative. to distinguish between other possible use-cases? + "is-authoritative": true, // or is authoritative. to distinguish between other possible use-cases? "members": [ { "time-series-id": "TS ID for this range", - "start": "start date of this", - "end": "end date of this range", - "notes": "text", - "version", "version date", // maybe not? could just use POR or period-of-record in the ts id version - // if values that equals the start or end timestamp are included - "start-inclusive": true, - "end-inclusive": false - // suggest default of "start-inclusive": true, "end-inclusive": false - // it may also make sense to just make this *always* the above and not let the user set it. - // alternatively if this is always [) then only start is required. - // however, the class would required and end field as the actual Time Series output needs to know the actual end. + "start": "start date of this", // Inclusive + "end": "end date of this range", // Exclusive + "notes": "text", } ] // array above *should* be sorted by start when provided to user. @@ -163,13 +237,12 @@ Supported Operations: * Get, through existing TimeSeries classes. -Does it make sense to support writing directly to a composite time series. While the write of each element *could* be sent to the underlying member, this seems ripe for error when editing or updating any data. It is likely that any edits would always be to the most recent time series, and configured in some other system. Storage of member information ================================ -#. Store in Clob as we refine the design -#. Create appropriate tables once the design is stable. +#. Store in Clob as we refine the design - cache appropriately in member to avoid any major performance issues. +#. Create appropriate tables once the design is stable - still cache things. System responsibility for "knowing" to process composite. ========================================================= @@ -187,9 +260,14 @@ Add nullable "members" property. TimeSeriesDao ------------- -If the system sees the "Composite" marker retrieve the members for the range and build the time series. +If the system sees the "Composite" marker/determines is composite retrieve the members for the range and build the time series. + +.. NOTE:: + Considering the user may request the *entire* Period-of-record, this is a good opportunity to see that, + start the retrieval in a job queue, and return a status URL to the user for future download. I have see such mechanism + for bulk data in other systems. Maybe return an "I'm working on it variant" that the controller can know how to format. -NOTE: considering the user may request the *entire* Period-of-record, this is a good opportunity to see that, start the retrieval in a job queue, and return a status URL to the user for future download. I have see such mechanism for bulk data in other systems. Maybe return an "I'm working on it variant" that the controller can know how to format. + Perhaps we do this for data beyond "x amount"? Error handling and other conditions. ==================================== @@ -197,13 +275,13 @@ Error handling and other conditions. Versioned (date) time series ---------------------------- -As the composite time series is comprised of multiple other time series should this always be an error to specify? -The marker for always latest or always first may make sense to allow, however, at the time series is supposed to be authoritative, that would add ambiguity. +It is an error to specify a Version (date) when requesting composite data. Datum conversions ----------------- -Retrievers of the Period-of-Record *SHOULD* be able to retrieve the data as a single datum. Composite retrieval should respond as https://github.com/USACE/cwms-data-api/issues/1102 and convert each member as appropriate +Retrievers of the Period-of-Record *SHOULD* be able to retrieve the data as a single datum. Composite retrieval should respond + as https://github.com/USACE/cwms-data-api/issues/1102 and convert each member as appropriate On the saving of a composite definition From eb12e8cd194b61f1f4c61ae490cee581fd1d8119 Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Fri, 5 Sep 2025 15:37:27 +0000 Subject: [PATCH 3/8] Update from additional comments. --- docs/source/design/composite-time-series.rst | 36 +++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst index 0f74e1539..a0c5ca7a2 100644 --- a/docs/source/design/composite-time-series.rst +++ b/docs/source/design/composite-time-series.rst @@ -7,7 +7,7 @@ Purpose It is a challenge for users to identity what the correct authoritative time series is for a given measurement at a location. Additionally these time series often change over time, either being completely new or changing their interval as newer technologies become available. -Gather an entire Period of Record for the value at a location is also rather difficult. And the POR record and "authoritative timeseries" may be one-in-the same. +Gather an entire Period of Record for the value at a location is also rather difficult. And the Period of Record (POR) and "authoritative timeseries" may be one-in-the same. Need @@ -15,7 +15,7 @@ Need #. CWMS and Access-2-Water require a simple mechanism to allow users of data to retrieve the Authoritative Period of Record data for a given measurement without having to understand all of the possible component time series that may be involved. #. Period-of-Record time series *should* not be created by duplicating data from the component time series and merging them into a new one. -#. The naming of the time series should fit within the excepting CWMS Time Series Identifier design and not unreasonably interfere with existing usages. +#. The naming of the time series should fit within the existing CWMS Time Series Identifier design and not unreasonably interfere with existing usages. Caveats @@ -31,9 +31,9 @@ Proposal Description ----------- -CDA should handle a concept of a "Composite Time Series". Whether a Time Series is considered composite will be determined by a specific element of the Time Series Identifier. +CWMS-Data-API (CDA) should handle a concept of a "Composite Time Series". Whether a Time Series is considered composite will be determined by some means (see naming options below). Data Administrators will configure which Time Series, and the range there-in, are part of the composite time series. -CDA will use this stored information to build the Time Series per the question. +CDA will use this composite time series definition to build an expand Time Series per query for the range of time requested. Additional names not used ------------------------- @@ -53,8 +53,10 @@ Axioms #. The definition of the composite time series is stored within the CWMS database #. The members of a composite time series define a continuous range #. The date ranges of a member *MUST* not overlap - #. The date ranges of a member *MUST* not have any gaps - #. Data may have gaps, an explanation range should be provided. + #. Each member *MUST* have a start date + #. The last member *MAY* have an end date indicating no more data will be available for this location and measure. + #. Data may have gaps, an explanation range *SHOULD* be provided. For data with regular gaps, e.g. season gauges + a description should be provided in the notes. A Link to a Location Level can be provided if the timing is well known. #. The members of a composite time measure the same thing. (e.g. all members are Elevation, not some are elevation and some are stage.) #. The interval and duration of each member *MAY* be different. @@ -163,8 +165,8 @@ Option 4 This form with something in [] has been discussed for embedded TimeZone and Offset information into the interval. Arguably this code go in any field. -Option 4 -~~~~~~~~ +Option 5 (Currently preferred) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ `.....` and/or arbitrary TS "alias" @@ -197,12 +199,14 @@ Composite Time Series Definition { "office": "", "name": "", - "is-authoritative": true, // or is authoritative. to distinguish between other possible use-cases? + "is-authoritative": true|false, + "seasonal-information": "/", // Optional, reference to location level that returns 0 for out-of-service + // or 1 for in-service "members": [ { "time-series-id": "TS ID for this range", "start": "start date of this", // Inclusive - "end": "end date of this range", // Exclusive + "end": "end date of this range", // Exclusive, can be null "notes": "text", } ] @@ -217,8 +221,14 @@ Operations required: * Add member * List members * Replace all members? +* Update member * Delete +Immutable fields: + +Fields marked immutable above cannot be updated. At this time no field are thought to be immutable. + + Composite Time Series Response ============================== @@ -241,7 +251,7 @@ Supported Operations: Storage of member information ================================ -#. Store in Clob as we refine the design - cache appropriately in member to avoid any major performance issues. +#. Store in Clob as we refine the design - cache appropriately to avoid any major performance issues. #. Create appropriate tables once the design is stable - still cache things. System responsibility for "knowing" to process composite. @@ -263,7 +273,7 @@ TimeSeriesDao If the system sees the "Composite" marker/determines is composite retrieve the members for the range and build the time series. .. NOTE:: - Considering the user may request the *entire* Period-of-record, this is a good opportunity to see that, + Considering the user may request the *entire* Period-of-record, start the retrieval in a job queue, and return a status URL to the user for future download. I have see such mechanism for bulk data in other systems. Maybe return an "I'm working on it variant" that the controller can know how to format. @@ -287,7 +297,7 @@ Retrievers of the Period-of-Record *SHOULD* be able to retrieve the data as a si On the saving of a composite definition --------------------------------------- -The even if only a single member is added, the full definition needs to be check to ensure the ranges are still overlapping and continuous. +The even if only a single member is added, the full definition needs to be check to ensure the ranges are still non overlapping. References ========== From 6e6e03fc56e08c22be305a39a2b50f52ae54b916 Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Wed, 24 Sep 2025 21:01:29 +0000 Subject: [PATCH 4/8] Add formal definitions of Composite, POR, and Authoritative. --- docs/source/design/composite-time-series.rst | 71 ++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst index a0c5ca7a2..7b8b48fd2 100644 --- a/docs/source/design/composite-time-series.rst +++ b/docs/source/design/composite-time-series.rst @@ -46,6 +46,77 @@ For Period-of-Record, while that is the primary use-case, the concept is useful Hence generically we have a "composite time series" +Definitions +----------- + +Composite Time Series +~~~~~~~~~~~~~~~~~~~~~ + +A Time Series that is comprised of multiple same measure time series. For example, a river gage that has two sensors +only one of which is valid for certain conditions. + +Period of Record +~~~~~~~~~~~~~~~~ + +The Period of Record (POR, period-of-record) for a measurement (such as the Stage at a river or the pool elevation of a +dam) is *ALL* available (time,value) pairs since recording began until either recording as ended or the most recent available +regardless of changes in intervals and, if unavoidable changes in averaging. + +The POR is the "best" available combined dataset that would be desired for studies requiring all values for a given +location. + +However, "best" is subjective. The POR of data used to make a given decision may not be the same as data that has formal +validation. Additionally having a POR of the raw, unedited data may be what + + +Naming Option 3, below, is selected for the path forward. A future design document will develop appropriate naming +to communicate the intent of any given composite time series, include period-of-record. + +Authoritative +~~~~~~~~~~~~~ + +An authoritative time series is a period-of-record time series with the additional constraint that is contains the best +official data. In other words the data that is determined to be "correct" by appropriate methods of validation. + +The data provide will have validated or corrected after events when additional information become available. +The data may not match what was used at the moment a decision was made. + +.. NOTE:: + + At time of design we are considering a boolean flag to indicate whether a time series is "the authoritative correct" + time series or an arbitrary period-of-record. This may change in the future after the above mentioned group + determines an appropriate naming scheme. + +.. COMMENT:: + Responses to discussion that the above is derived from. + + + Yeah I agree that period of record is everything you have (or best available) for that site for as long as you have it. + To me, it makes sense if it's all Inst data (e.g. 8 am readings, 1 hour DCP, 15 minute DCP). + I think there could be an argument about mixing in averaged data....that's a harder sell for me. + For mixing and matching sensors, though that doesn't bother me. We are already taking huge leaps + of faith by using a single gage to represent the entire storage of reservoirs + + As far as providing daily average timeseries as an official period of record, you run into the issue + of a lot of years where you are averaging one instantaneous point which is not a great average. + So my dream was to just composite all the inst data. However, the load times in cwms for people wanting to quick view period of record probably won't allow that + + In LRL if we use "period of record" it's almost always referring to a flood control project and encompasses all recorded data from time of impoundment to present. + + If I am a user grabbing data from us, which I have done many time both for published research papers and in consulting. + I would want the POR to be the best available data for each time point from beginning of measurements to now. + We should provide the full record with mixed intervals. + And like the USGS we should provide the Daily Avg values, but that is a second step to what we are doing. + This is what your district is saying the water level was since you started recording data to today. + I don't care how you got the data unless I am going to do a detailed study of different sensors. + All of your sensors should be calibrated. If I need that information the user should be able to also see + the meta data for the composite timeseries and what individual timeseries it came from. + You are the the expert and should be able to provide the best available stage values and combine them into a single time series. + If someone came to you and said what was the level on XX Date/time what value would you give them? + That is the period of record. but for all dates and times. + + + Axioms ------ From 4a31a8d4425cfeb1808ab2692ce4491c10ed1df1 Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Mon, 17 Nov 2025 16:22:26 +0000 Subject: [PATCH 5/8] Reponse to some comments. --- docs/source/design/composite-time-series.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst index 7b8b48fd2..aea08fcec 100644 --- a/docs/source/design/composite-time-series.rst +++ b/docs/source/design/composite-time-series.rst @@ -286,6 +286,7 @@ Composite Time Series Definition Operations required: +-------------------- * Create * Remove member (ts id + range) @@ -299,7 +300,13 @@ Immutable fields: Fields marked immutable above cannot be updated. At this time no field are thought to be immutable. +Operations Prohibited: +---------------------- +* Any direct manipulation of the underlying time series members. + +Example: one cannot `POST` values to a composite time series. Doing so will result in an HTTP 405 - Method Not Allowed +error. Composite Time Series Response ============================== From d108edae99f3623efbe041d956ef5981f0fbd749 Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Wed, 3 Dec 2025 17:33:25 +0000 Subject: [PATCH 6/8] Review updates. --- docs/source/design/composite-time-series.rst | 32 ++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst index aea08fcec..aa0772dcc 100644 --- a/docs/source/design/composite-time-series.rst +++ b/docs/source/design/composite-time-series.rst @@ -58,15 +58,15 @@ only one of which is valid for certain conditions. Period of Record ~~~~~~~~~~~~~~~~ -The Period of Record (POR, period-of-record) for a measurement (such as the Stage at a river or the pool elevation of a -dam) is *ALL* available (time,value) pairs since recording began until either recording as ended or the most recent available -regardless of changes in intervals and, if unavoidable changes in averaging. +The Period of Record (POR, period-of-record) for a measurement (such as the Stage at a river or the pool elevation of +a dam) is ALL available (time,value) pairs since recording began until recording has ended or the most recent available +pair, regardless of changes in intervals or unavoidable changes in averaging. The POR is the "best" available combined dataset that would be desired for studies requiring all values for a given location. However, "best" is subjective. The POR of data used to make a given decision may not be the same as data that has formal -validation. Additionally having a POR of the raw, unedited data may be what +validation. Additionally having a POR of the raw, unedited data, may be what a user studying data validation requires. Naming Option 3, below, is selected for the path forward. A future design document will develop appropriate naming @@ -78,7 +78,7 @@ Authoritative An authoritative time series is a period-of-record time series with the additional constraint that is contains the best official data. In other words the data that is determined to be "correct" by appropriate methods of validation. -The data provide will have validated or corrected after events when additional information become available. +The data provide will have been validated or corrected after events when additional information become available. The data may not match what was used at the moment a decision was made. .. NOTE:: @@ -127,9 +127,11 @@ Axioms #. Each member *MUST* have a start date #. The last member *MAY* have an end date indicating no more data will be available for this location and measure. #. Data may have gaps, an explanation range *SHOULD* be provided. For data with regular gaps, e.g. season gauges - a description should be provided in the notes. A Link to a Location Level can be provided if the timing is well known. -#. The members of a composite time measure the same thing. (e.g. all members are Elevation, not some are elevation and some are stage.) -#. The interval and duration of each member *MAY* be different. + a description should be provided in the notes. + Example: A Link to a Location Level can be provided if the seasonal timing is well known. This would let users + of the data now if the gap is missing data "an error" or if just out-of-service. +#. The members of a composite time series measure the same thing. (e.g. all members are Elevation, not some are elevation and some are stage.) +#. The parameter type (e.g. Instantaneous; Average; etc), interval, and duration of each member *MAY* be different. Time Series Naming @@ -296,6 +298,17 @@ Operations required: * Update member * Delete + +Events Requires: +---------------- + +* Composite Time Series created +* Composite Time Series deleted +* Composite Time Series modified + * Member added + * Member modified + * Member removed + Immutable fields: Fields marked immutable above cannot be updated. At this time no field are thought to be immutable. @@ -363,7 +376,8 @@ Error handling and other conditions. Versioned (date) time series ---------------------------- -It is an error to specify a Version (date) when requesting composite data. +It is an error to specify a Version (date) when requesting composite data. Only the latest data of each member will +be returned. Datum conversions ----------------- From 6eb43e01ace6da94654667a28212237e3976a3e0 Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Wed, 3 Dec 2025 17:44:19 +0000 Subject: [PATCH 7/8] Correct TOC and code block. --- docs/source/design/composite-time-series.rst | 2 +- docs/source/index.rst | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst index aa0772dcc..fdcd41912 100644 --- a/docs/source/design/composite-time-series.rst +++ b/docs/source/design/composite-time-series.rst @@ -267,7 +267,7 @@ time series retrieval. Composite Time Series Definition ================================ -.. code-block::jsonc +.. code-block:: jsonc { "office": "", diff --git a/docs/source/index.rst b/docs/source/index.rst index 1c001280d..c5249f851 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -38,10 +38,13 @@ Welcome to CWMS Data API documentation! .. toctree:: - :maxdepth: 1 - :caption: Alternative Topics + :maxdepth: 1 + :caption: Alternative Topics - Alternative Topics <./alternative-topics/index.rst> + Alternative Topics <./alternative-topics/index.rst> +.. toctree:: + :maxdepth: 1 + :caption: Design Documents Design Documents <./design/index.rst> \ No newline at end of file From 237abcd0bcd372c77af8c92cdc526cfe869f658a Mon Sep 17 00:00:00 2001 From: Mike Neilson Date: Wed, 3 Dec 2025 17:53:53 +0000 Subject: [PATCH 8/8] Correct code block, add example. --- docs/source/design/composite-time-series.rst | 37 ++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/source/design/composite-time-series.rst b/docs/source/design/composite-time-series.rst index fdcd41912..6cd1b1018 100644 --- a/docs/source/design/composite-time-series.rst +++ b/docs/source/design/composite-time-series.rst @@ -267,7 +267,8 @@ time series retrieval. Composite Time Series Definition ================================ -.. code-block:: jsonc +.. code-block:: json + :caption: Schema { "office": "", @@ -278,7 +279,7 @@ Composite Time Series Definition "members": [ { "time-series-id": "TS ID for this range", - "start": "start date of this", // Inclusive + "start": "start date of this", // Inclusive, ISO 8601 Full "end": "end date of this range", // Exclusive, can be null "notes": "text", } @@ -287,6 +288,38 @@ Composite Time Series Definition } +.. code-block:: json + :caption: Example (Using Option 5) + + // NOTE: While a specific location was used, the dates and time series are arbitrary and may not exist. + { + "office": "SPK", + "name": "Alder Springs.Precip-Cumulative.Inst.0.0.Best", + "is-authoritative": true, + "seasonal-information": null, // This location's measurement is not seasonal + "members": [ + { + "time-series-id": "Alder Springs.Precip-Cumulative.Inst.1DayLocal.0.Best", + "start": "1948-04-01T14:14:00Z", // Inclusive + "end": null, // Exclusive, can be null + "notes": "Drum Chart Recorder, Catch Tube", + }, + { + "time-series-id": "Alder Springs.Precip-Cumulative.Inst.1Hour.0.Best", + "start": "1962-04-01T14:14:00Z", // Inclusive + "end": null, // Exclusive, can be null + "notes": "Punched Tape Recorder, Catch Tube", + }, + { + "time-series-id": "Alder Springs.Precip-Cumulative.Inst.1DayLocal.0.Best", + "start": "1990-04-01T14:14:00Z", // Inclusive + "end": null, // Exclusive, can be null + "notes": "Digital Logger, Catch Tube", + }, + ] + // array above *WILL* be sorted by start when provided to user. + } + Operations required: --------------------