From d8d09860559c51768408164ab57fb195253c9db7 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Tue, 4 Oct 2022 10:45:15 +0200 Subject: [PATCH 01/35] update dependencies and pin ruby version --- Dockerfile | 2 +- Gemfile | 2 +- Gemfile.lock | 33 ++++++++++++++++----------------- transload | 0 4 files changed, 18 insertions(+), 19 deletions(-) mode change 100644 => 100755 transload diff --git a/Dockerfile b/Dockerfile index a6d0de2..5bd93ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Ruby 2.6 is the recommended version -FROM ruby:2.6 +FROM ruby:2.6.6 # Set the language to ensure encodings are handled properly. # Otherwise the transloader tool will have errors parsing. diff --git a/Gemfile b/Gemfile index a7dabdd..8fe23a9 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ gem "deep_merge", "~> 1.2" gem "nokogiri", "~> 1.10" gem "parallel", "~> 1.19" gem "pg", "~> 1.2" -gem "rdf-turtle", "~> 3.0" +gem "rdf-turtle", "~> 3.2" gem "semantic_logger", "~> 4.5" gem "spreadsheet", "~> 1.2" diff --git a/Gemfile.lock b/Gemfile.lock index 3c5aa68..e2be9c9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,27 +14,26 @@ GEM zeitwerk (~> 2.3) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) - amazing_print (1.3.0) + amazing_print (1.4.0) coderay (1.1.3) concurrent-ruby (1.1.8) crack (0.4.5) rexml deep_merge (1.2.1) diff-lcs (1.4.4) - ebnf (2.1.3) - amazing_print (~> 1.2) + ebnf (2.3.0) + amazing_print (~> 1.4) htmlentities (~> 4.3) - rdf (~> 3.1) + rdf (~> 3.2) scanf (~> 1.0) - sxp (~> 1.1) - unicode-types (~> 1.6) - hamster (3.0.0) - concurrent-ruby (~> 1.0) + sxp (~> 1.2) + unicode-types (~> 1.7) hashdiff (1.0.1) htmlentities (4.3.4) i18n (1.8.10) concurrent-ruby (~> 1.0) link_header (0.0.8) + matrix (0.4.2) method_source (1.0.0) mini_portile2 (2.5.3) minitest (5.14.4) @@ -48,12 +47,11 @@ GEM method_source (~> 1.0) public_suffix (4.0.6) racc (1.5.2) - rdf (3.1.14) - hamster (~> 3.0) + rdf (3.2.3) link_header (~> 0.0, >= 0.0.8) - rdf-turtle (3.1.3) - ebnf (~> 2.1) - rdf (~> 3.1, >= 3.1.8) + rdf-turtle (3.2.0) + ebnf (~> 2.3) + rdf (~> 3.2) rexml (3.2.5) rspec (3.10.0) rspec-core (~> 3.10.0) @@ -75,12 +73,13 @@ GEM concurrent-ruby (~> 1.0) spreadsheet (1.2.9) ruby-ole - sxp (1.1.0) - rdf (~> 3.1) + sxp (1.2.1) + matrix + rdf (~> 3.2) timecop (0.9.4) tzinfo (2.0.4) concurrent-ruby (~> 1.0) - unicode-types (1.6.0) + unicode-types (1.7.0) vcr (5.1.0) webmock (3.13.0) addressable (>= 2.3.6) @@ -98,7 +97,7 @@ DEPENDENCIES parallel (~> 1.19) pg (~> 1.2) pry (~> 0.12) - rdf-turtle (~> 3.0) + rdf-turtle (~> 3.2) rspec (~> 3.9) ruby-prof (~> 1.4) semantic_logger (~> 4.5) diff --git a/transload b/transload old mode 100644 new mode 100755 From 5a3daf1ff4d1246c3dbd6689599f0d0bd08309f9 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Tue, 4 Oct 2022 10:46:12 +0200 Subject: [PATCH 02/35] store a link to datagarrison page in sensors --- lib/transloader/data_garrison/station.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/transloader/data_garrison/station.rb b/lib/transloader/data_garrison/station.rb index 47601f6..a1e5b86 100644 --- a/lib/transloader/data_garrison/station.rb +++ b/lib/transloader/data_garrison/station.rb @@ -64,7 +64,7 @@ def download_metadata(override_metadata: nil) html.xpath('/html/body/table/tr[position()=2]/td/table/tr/td[position()=2]/div[position()=2]/table/tr[position()=2]/td/table/tr/td/font/a').collect do |element| href = element.attr('href') data_desc = element.text.to_s - filename = data_desc.sub(/ /, '_') + filename = data_desc.gsub(/ /, '_') if filename == "" # Without a filename, the file cannot be downloaded as a TSV @@ -273,6 +273,8 @@ def upload_metadata(server_url, options = {}) # Create Sensor entities sensor = build_sensor("Station #{@id} #{stream[:name]} Sensor", "#{NAME} #{@id} #{stream[:name]} Sensor") + sensor.encoding_type = "text/html" + sensor.metadata = "https://datagarrison.com/users/#{@user_id}/#{@id}/" # Upload entity and parse response sensor.upload_to(server_url) From 0a0164c81597e9c3f75b202d7f8071a1490f9658 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Tue, 4 Oct 2022 12:17:45 +0200 Subject: [PATCH 03/35] fix ontology for minimum temperature values --- lib/ontologies/etl-ontology.ttl | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/ontologies/etl-ontology.ttl b/lib/ontologies/etl-ontology.ttl index 751110d..be12f2f 100644 --- a/lib/ontologies/etl-ontology.ttl +++ b/lib/ontologies/etl-ontology.ttl @@ -639,21 +639,21 @@ etl-ontology:EnvironmentCanada:max_wnd_spd_pst1hr_tm rdf:type owl:NamedIndividua ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:min_air_temp_pst1hr etl-ontology:EnvironmentCanada:min_air_temp_pst1hr rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; - etl-ontology:matchesObservedProperty etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:min_air_temp_pst24hrs etl-ontology:EnvironmentCanada:min_air_temp_pst24hrs rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; - etl-ontology:matchesObservedProperty etl-ontology:Property:Maximum_Air_Temperature_over_past_24_hours ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Minimum_Air_Temperature_over_past_24_hours ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:min_air_temp_pst6hrs etl-ontology:EnvironmentCanada:min_air_temp_pst6hrs rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; - etl-ontology:matchesObservedProperty etl-ontology:Property:Maximum_Air_Temperature_over_past_6_hours ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Minimum_Air_Temperature_over_past_6_hours ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . @@ -1308,6 +1308,29 @@ etl-ontology:Property:Mean_sea_level_pressure rdf:type owl:NamedIndividual , etl-ontology:description "Air pressure is the force per unit area which would be exerted when the moving gas molecules of which the air is composed strike a theoretical surface of any orientation."^^xsd:string ; rdfs:label "Mean Sea Level Pressure" . +### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_1_hour +etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Minimum Air Temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_24_hours +etl-ontology:Property:Minimum_Air_Temperature_over_past_24_hours rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Minimum Air Temperature over past 24 hours" . + + +### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_6_hours +etl-ontology:Property:Minimum_Air_Temperature_over_past_6_hours rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Minimum Air Temperature over past 6 hours" . + + ### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Battery_Voltage etl-ontology:Property:Minimum_Battery_Voltage rdf:type owl:NamedIndividual , From e49c50e3536b766cac8e16df3de2cdccb6c3839a Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Tue, 4 Oct 2022 12:18:58 +0200 Subject: [PATCH 04/35] fix property access in envvironment_canada provider --- lib/transloader/environment_canada/station.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/transloader/environment_canada/station.rb b/lib/transloader/environment_canada/station.rb index a2173d9..31b1a2f 100644 --- a/lib/transloader/environment_canada/station.rb +++ b/lib/transloader/environment_canada/station.rb @@ -286,7 +286,7 @@ def get_swob_data(timestamp = nil) # "AUTO" or "MANNED" will be determined automatically from the # station metadata. def get_swob_data_from_url(url) - case @metadata["properties"]["AUTO/MAN"] + case @metadata[:properties][:"AUTO/MAN"] when "AUTO", "Auto", "Manned/Auto" type = "AUTO" when "MAN", "Manned" From 9b12c4e1c15e947e962161d8b7930e429267ed59 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Tue, 4 Oct 2022 12:20:20 +0200 Subject: [PATCH 05/35] fix datastream selection --- lib/transloader/observation_property_cache.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/transloader/observation_property_cache.rb b/lib/transloader/observation_property_cache.rb index e0ddb18..fc08d82 100644 --- a/lib/transloader/observation_property_cache.rb +++ b/lib/transloader/observation_property_cache.rb @@ -13,11 +13,14 @@ def initialize(datastream_names) # it if it is missing. If it already exists, nothing is changed. def cache_observation_property(property_name) if !@matches.key?(property_name) - matching_datastream = @datastream_names.find do |datastream| - property_name.include?(datastream) + # first check if we have an exact match + if @datastream_names.include?(property_name) + @matches[property_name] = property_name + else + @matches[property_name] = @datastream_names.find do |datastream| + property_name.include?(datastream) + end end - - @matches[property_name] = matching_datastream end end From 612f964b0b986d9b9c456200db6ff213fc2cea51 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Thu, 20 Oct 2022 15:17:59 +0200 Subject: [PATCH 06/35] add cronjob --- docker/cron/Dockerfile | 37 +++++++++++++++++++++++++++++++++++++ docker/cron/cronjob.sh | 15 +++++++++++++++ docker/cron/entrypoint.sh | 20 ++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 docker/cron/Dockerfile create mode 100755 docker/cron/cronjob.sh create mode 100644 docker/cron/entrypoint.sh diff --git a/docker/cron/Dockerfile b/docker/cron/Dockerfile new file mode 100644 index 0000000..8734754 --- /dev/null +++ b/docker/cron/Dockerfile @@ -0,0 +1,37 @@ +# Ruby 2.6 is the recommended version +FROM ruby:2.6.6 + +# Set the language to ensure encodings are handled properly. +# Otherwise the transloader tool will have errors parsing. +ENV LC_ALL C.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +WORKDIR /usr/src/app + +# Set up the gems for this image early, as it is slower +COPY Gemfile Gemfile.lock ./ +RUN bundle install + +# Copy contents of the repository into the image +COPY . . + +# install cron +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends cron \ + && rm -rf /var/lib/apt/lists/* \ + && chmod +x docker/cron/*.sh + + +ENV \ + SCHEDULE="@daily" \ + DATA_DIR=/data \ + MOVING_WINDOW="2 days" \ + STA_URL= \ + OVERWRITE_METADATA=false + + +VOLUME /data + +ENTRYPOINT [ "./docker/cron/entrypoint.sh" ] \ No newline at end of file diff --git a/docker/cron/cronjob.sh b/docker/cron/cronjob.sh new file mode 100755 index 0000000..cb4cbf4 --- /dev/null +++ b/docker/cron/cronjob.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -ex + +STA_URL="${STA_URL:?"Error: STA_URL not set"}" +DATABASE_URL="file://${DATA_DIR}" +INTERVAL="$(date -u -d "${MOVING_WINDOW} ago" "+%Y-%m-%dT%TZ")/$(date -u "+%Y-%m-%dT%TZ")" + +if [ "$OVERWRITE_METADATA" = "true" ]; then + ruby transload get metadata --database_url "${DATABASE_URL}" --overwrite "$@" +fi + +ruby transload put metadata --database_url "${DATABASE_URL}" --destination "${STA_URL}" "$@" +ruby transload get observations --database_url "${DATABASE_URL}" "$@" +ruby transload put observations --database_url "${DATABASE_URL}" --destination "${STA_URL}" --date "${INTERVAL}" "$@" diff --git a/docker/cron/entrypoint.sh b/docker/cron/entrypoint.sh new file mode 100644 index 0000000..6965619 --- /dev/null +++ b/docker/cron/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +# fail early +STA_URL=${STA_URL:?"Error: STA_URL not set"} + +# build the command by quoting addtional parameters +COMMAND="cd $(pwd) && ./docker/cron/cronjob.sh" +for PARAM in "$@"; do + COMMAND="$COMMAND ${PARAM@Q}" +done + +# build the crontab +env > /etc/cron.d/transloader-job +echo >> /etc/cron.d/transloader-job +echo "${SCHEDULE:-"@daily"} $(id -un) ${COMMAND} > /proc/\$(cat /var/run/crond.pid)/fd/1 2>/proc/\$(cat /var/run/crond.pid)/fd/2" >> /etc/cron.d/transloader-job + +# give over to cron +exec cron -f -L 15 \ No newline at end of file From 768fdf789b9384437e0bbb89bf8e600257305074 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Thu, 20 Oct 2022 16:26:41 +0200 Subject: [PATCH 07/35] add coordinats to environment canada stations --- lib/transloader/environment_canada/station.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/transloader/environment_canada/station.rb b/lib/transloader/environment_canada/station.rb index 31b1a2f..12021f9 100644 --- a/lib/transloader/environment_canada/station.rb +++ b/lib/transloader/environment_canada/station.rb @@ -73,6 +73,8 @@ def download_metadata(override_metadata: {}) name: "#{NAME} #{@id}", description: "#{LONG_NAME} #{properties["Name"]}", elevation: xml.xpath('//po:element[@name="stn_elev"]', NAMESPACES).first.attribute('value').value, + longitude: properties[:Longitude], + latitude: properties[:Latitude], updated_at: Time.now, datastreams: datastreams, procedure: xml.xpath('//om:procedure/@xlink:href', NAMESPACES).text, From e850e5d0efa12fd8247acb5b289363efe8116b9b Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Tue, 15 Nov 2022 09:23:11 -0600 Subject: [PATCH 08/35] add docker-compose.yaml --- docker-compose.yaml | 244 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..c151700 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,244 @@ +version: "3" + +x-defaults: + cronjob: &cron-job-defaults + image: data-transloader-job:latest + build: + context: . + dockerfile: docker/cron/Dockerfile + volumes: + - data:/data + networks: + - sta + restart: unless-stopped + environment: + # the STA endpoint to load data to + STA_URL: http://web:8080/FROST-Server/v1.0/ + # the moving window to download data for (optional, defaults to 2 days) + MOVING_WINDOW: "2 days" + # the schedule of the job (optional, defaults to @daily) + SCHEDULE: "@hourly" + DATA_DIR: /data + OVERWRITE_METADATA: "false" + + get-metadata-environment: &get-metadata-environment + DATA_DIR: /data + + get-metadata-job-defaults: &get-metadata-job-defaults + image: data-transloader:latest + build: + context: . + dockerfile: Dockerfile + environment: *get-metadata-environment + restart: "no" + volumes: + - data:/data + + data-garrision-get-metadata-job-defaults: + &data-garrision-get-metadata-job-defaults + <<: *get-metadata-job-defaults + entrypoint: [/bin/sh, -e, -x, -c] + command: + - | + ruby transload get metadata \ + --provider data_garrison \ + --user_id "$${USER_ID}" \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" \ + --overwrite + sed -i "$${DATA_DIR}/data_garrison/metadata/$${USER_ID}-$${STATION_ID}.json" \ + -e "s/\"latitude\": null/\"latitude\": $${LATITUDE}/g" \ + -e "s/\"longitude\": null/\"longitude\": $${LONGITUDE}/g" \ + -e "s/\"timezone_offset\": null/\"timezone_offset\": \"$${TIMEZONE_OFFSET}\"/g" + + environment-canada-get-metadata-job-defaults: + &environment-canada-get-metadata-job-defaults + <<: *get-metadata-job-defaults + entrypoint: [/bin/sh, -e, -x, -c] + command: + - | + ruby transload get metadata --provider environment_canada --station_id "$${STATION_ID}" --database_url "file://$${DATA_DIR}" --overwrite + +services: + get-metadata-environment-canada-CWCF: # Berens, River + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWCF + + get-metadata-environment-canada-CWZE: # Carberry, CS + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWZE + + get-metadata-environment-canada-CWZT: # Dauphin, Airport + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWZT + + get-metadata-environment-canada-CWSZ: # Fisher, Branch, (auto) + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWSZ + + get-metadata-environment-canada-CWWS: # George, Island + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWWS + + get-metadata-environment-canada-CPGH: # Gimli, Climate + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CPGH + + get-metadata-environment-canada-CWJD: # Grand, Rapids, (auto) + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWJD + + get-metadata-environment-canada-CWOO: # McCreary + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWOO + + get-metadata-environment-canada-CWOJ: # Oak, Point, Marine + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWOJ + + get-metadata-environment-canada-CWPG: # Portage, Southport + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWPG + + get-metadata-environment-canada-CWEQ: # Swan, River, RCS + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWEQ + + get-metadata-environment-canada-CWII: # victoria, Beach, (auto) + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWII + + get-metadata-environment-canada-CWWP: # Wasagaming + <<: *environment-canada-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + STATION_ID: CWWP + + get-metadata-data-garrison-MMF1: + <<: *data-garrision-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + USER_ID: "300234068013390" + STATION_ID: "300534063017060" + LATITUDE: "50.40768" + LONGITUDE: "-97.96212" + TIMEZONE_OFFSET: "-06:00" + + get-metadata-data-garrison-MMF2: + <<: *data-garrision-get-metadata-job-defaults + environment: + <<: *get-metadata-environment + USER_ID: "300234068013390" + STATION_ID: "300534061454190" + LATITUDE: "52.977050" + LONGITUDE: "-100.979630" + TIMEZONE_OFFSET: "-06:00" + + transload-data-data-garrison-MMF1: + <<: *cron-job-defaults + command: + [ + --provider, + data_garrison, + --user_id, + "300234068013390", + --station_id, + "300534063017060", + ] + + transload-data-data-garrison-MMF2: + <<: *cron-job-defaults + command: + [ + --provider, + data_garrison, + --user_id, + "300234068013390", + --station_id, + "300534061454190", + ] + + transload-data-environment-canada-CWCF: # Berens, River + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWCF] + + transload-data-environment-canada-CWZE: # Carberry, CS + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWZE] + + transload-data-environment-canada-CWZT: # Dauphin, Airport + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWZT] + + transload-data-environment-canada-CWSZ: # Fisher, Branch, (auto) + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWSZ] + + transload-data-environment-canada-CWWS: # George, Island + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWWS] + + transload-data-environment-canada-CPGH: # Gimli, Climate + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CPGH] + + transload-data-environment-canada-CWJD: # Grand, Rapids, (auto) + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWJD] + + transload-data-environment-canada-CWOO: # McCreary + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWOO] + + transload-data-environment-canada-CWOJ: # Oak, Point, Marine + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWOJ] + + transload-data-environment-canada-CWPG: # Portage, Southport + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWPG] + + transload-data-environment-canada-CWEQ: # Swan, River, RCS + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWEQ] + + transload-data-environment-canada-CWII: # victoria, Beach, (auto) + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWII] + + transload-data-environment-canada-CWWP: # Wasagaming + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWWP] + +volumes: + data: + driver: local + +networks: + sta: + external: + name: frost-server_default From 12a83d8c8abbdfb2145234e381bddc71ab8b496f Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 25 Jan 2023 13:57:29 +0100 Subject: [PATCH 09/35] add commnets and readme --- docker-compose.yaml | 21 +++++++++++++++++++++ docker/cron/README.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 docker/cron/README.md diff --git a/docker-compose.yaml b/docker-compose.yaml index c151700..0255836 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,20 @@ +# +# Docker compose setup for transloading observations from +# environment canada and data garrison to a SensorThings API +# +# There are two types of services: +# +# * get-metadata-jobs that fetch the metadata from the provider for a station and push it +# to the STA. The jobs are meant to run once and then exit. +# +# * transload-data-jobs are cronjobs that fetch new data from the provider and transload +# it to the STA. There is one cronjob per station. +# version: "3" +# Default values/templates for the different services x-defaults: + # Defaults for the transloader cron job cronjob: &cron-job-defaults image: data-transloader-job:latest build: @@ -24,6 +38,7 @@ x-defaults: get-metadata-environment: &get-metadata-environment DATA_DIR: /data + # Defaults for getting metadata get-metadata-job-defaults: &get-metadata-job-defaults image: data-transloader:latest build: @@ -34,6 +49,11 @@ x-defaults: volumes: - data:/data + # Defaults for getting metadata from data garrison + # + # geolocation and timezone offset can not determined from + # data garrison and have to be supplied using environment + # variables data-garrision-get-metadata-job-defaults: &data-garrision-get-metadata-job-defaults <<: *get-metadata-job-defaults @@ -51,6 +71,7 @@ x-defaults: -e "s/\"longitude\": null/\"longitude\": $${LONGITUDE}/g" \ -e "s/\"timezone_offset\": null/\"timezone_offset\": \"$${TIMEZONE_OFFSET}\"/g" + # Defaults for getting metadata from environment canada environment-canada-get-metadata-job-defaults: &environment-canada-get-metadata-job-defaults <<: *get-metadata-job-defaults diff --git a/docker/cron/README.md b/docker/cron/README.md new file mode 100644 index 0000000..9c56121 --- /dev/null +++ b/docker/cron/README.md @@ -0,0 +1,30 @@ +# Arctic Sensor Web Data Transloader Cron Job + +The Docker image runs `cron` as PID 1 and regularly executes the following script. Provider specific command line options should be supplied as the Docker command. + +```shell +ruby transload get metadata ... ${COMMAND} # optional +ruby transload put metadata ... ${COMMAND} +ruby transload get observations ... ${COMMAND} +ruby transload put observations ... ${COMMAND} +``` + +Example invocation: + +```sh +docker run -d -e STA_URL="..." -e SCHEDULE="@hourly" -e MOVING_WINDOW="1 day" \ + data-transloader-job:latest --provider environment_canada --station_id CWCF +``` + +## Environment Variables + +| Environment Variable | Description | Default Value | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------- | ------------- | +| `SCHEDULE` | A valid cron time specification (see [here][man5crontab]) | `@daily` | +| `DATA_DIR` | The directory to store data. Defaults to the volume `/data`. | `/data` | +| `MOVING WINDOW` | The moving time window of observations to push to th STA, supplied to [`date -d`][man1date] as `"${MOVING_WINDOW} ago"` | `2 days` | +| `STA_URL` | The URL of SensorThings API to push metadata and observations | `` | +| `OVERWRITE_METADATA` | Whether `get metadata` should be executed with each `cron` invocation. | `false` | + +[man1date]: https://man7.org/linux/man-pages/man1/date.1.html +[man5crontab]: https://man7.org/linux/man-pages/man5/crontab.5.html From 5f88828d4c14336d961e1b581ddbbe897fc252fb Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 25 Jan 2023 20:18:58 +0100 Subject: [PATCH 10/35] use transload set metadata instead of sed --- docker-compose.yaml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0255836..376ab98 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -66,10 +66,27 @@ x-defaults: --station_id "$${STATION_ID}" \ --database_url "file://$${DATA_DIR}" \ --overwrite - sed -i "$${DATA_DIR}/data_garrison/metadata/$${USER_ID}-$${STATION_ID}.json" \ - -e "s/\"latitude\": null/\"latitude\": $${LATITUDE}/g" \ - -e "s/\"longitude\": null/\"longitude\": $${LONGITUDE}/g" \ - -e "s/\"timezone_offset\": null/\"timezone_offset\": \"$${TIMEZONE_OFFSET}\"/g" + ruby transload set metadata \ + --provider data_garrison \ + --user_id "$${USER_ID}" \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" \ + --key longitude \ + --value "$${LONGITUDE}" + ruby transload set metadata \ + --provider data_garrison \ + --user_id "$${USER_ID}" \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" \ + --key latitude \ + --value "$${LATITUDE}" + ruby transload set metadata \ + --provider data_garrison \ + --user_id "$${USER_ID}" \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" \ + --key timezone_offset \ + --value "$${TIMEZONE_OFFSET}" # Defaults for getting metadata from environment canada environment-canada-get-metadata-job-defaults: From a3cc73df2d6d5bdd36578d4ecfef3c774721920b Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Mon, 6 Feb 2023 12:02:54 +0100 Subject: [PATCH 11/35] Update README.md --- docker/cron/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/cron/README.md b/docker/cron/README.md index 9c56121..5f00099 100644 --- a/docker/cron/README.md +++ b/docker/cron/README.md @@ -16,6 +16,14 @@ docker run -d -e STA_URL="..." -e SCHEDULE="@hourly" -e MOVING_WINDOW="1 day" \ data-transloader-job:latest --provider environment_canada --station_id CWCF ``` +## Building + +The build has to be done in the root directory of this repository: + +``` +docker build -t data-transloader-job:latest -f docker/cron/Dockerfile . +``` + ## Environment Variables | Environment Variable | Description | Default Value | From 95762b5441b3ad459971097cd95444f5a663bc1f Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Fri, 10 Feb 2023 17:41:17 +0100 Subject: [PATCH 12/35] Fix whitespace errors --- docker/cron/cronjob.sh | 2 +- docker/cron/entrypoint.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/cron/cronjob.sh b/docker/cron/cronjob.sh index cb4cbf4..41d4f78 100755 --- a/docker/cron/cronjob.sh +++ b/docker/cron/cronjob.sh @@ -8,7 +8,7 @@ INTERVAL="$(date -u -d "${MOVING_WINDOW} ago" "+%Y-%m-%dT%TZ")/$(date -u "+%Y-%m if [ "$OVERWRITE_METADATA" = "true" ]; then ruby transload get metadata --database_url "${DATABASE_URL}" --overwrite "$@" -fi +fi ruby transload put metadata --database_url "${DATABASE_URL}" --destination "${STA_URL}" "$@" ruby transload get observations --database_url "${DATABASE_URL}" "$@" diff --git a/docker/cron/entrypoint.sh b/docker/cron/entrypoint.sh index 6965619..b96ed38 100644 --- a/docker/cron/entrypoint.sh +++ b/docker/cron/entrypoint.sh @@ -7,9 +7,9 @@ STA_URL=${STA_URL:?"Error: STA_URL not set"} # build the command by quoting addtional parameters COMMAND="cd $(pwd) && ./docker/cron/cronjob.sh" -for PARAM in "$@"; do +for PARAM in "$@"; do COMMAND="$COMMAND ${PARAM@Q}" -done +done # build the crontab env > /etc/cron.d/transloader-job From d5c6e141b5f2af317c8069b5a2b5c580a72a2fd0 Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Fri, 10 Feb 2023 17:41:36 +0100 Subject: [PATCH 13/35] Sort env in crontab for better readability --- docker/cron/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cron/entrypoint.sh b/docker/cron/entrypoint.sh index b96ed38..b3ef338 100644 --- a/docker/cron/entrypoint.sh +++ b/docker/cron/entrypoint.sh @@ -12,7 +12,7 @@ for PARAM in "$@"; do done # build the crontab -env > /etc/cron.d/transloader-job +env | sort > /etc/cron.d/transloader-job echo >> /etc/cron.d/transloader-job echo "${SCHEDULE:-"@daily"} $(id -un) ${COMMAND} > /proc/\$(cat /var/run/crond.pid)/fd/1 2>/proc/\$(cat /var/run/crond.pid)/fd/2" >> /etc/cron.d/transloader-job From e3b1a65d469a3b3abf1a3a6376a539800863757e Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Fri, 10 Feb 2023 17:42:03 +0100 Subject: [PATCH 14/35] add support for basic auth protected STA instances --- docker/cron/README.md | 4 ++++ docker/cron/cronjob.sh | 18 +++++++++++++++--- docker/cron/entrypoint.sh | 4 ++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docker/cron/README.md b/docker/cron/README.md index 5f00099..d4450fb 100644 --- a/docker/cron/README.md +++ b/docker/cron/README.md @@ -32,7 +32,11 @@ docker build -t data-transloader-job:latest -f docker/cron/Dockerfile . | `DATA_DIR` | The directory to store data. Defaults to the volume `/data`. | `/data` | | `MOVING WINDOW` | The moving time window of observations to push to th STA, supplied to [`date -d`][man1date] as `"${MOVING_WINDOW} ago"` | `2 days` | | `STA_URL` | The URL of SensorThings API to push metadata and observations | `` | +| `STA_USER` | *optional*: username to be used if STA is write protected using basic auth. | | +| `STA_PASSWORD` | *optional*: password to be used if STA is write protected using basic auth. | | | `OVERWRITE_METADATA` | Whether `get metadata` should be executed with each `cron` invocation. | `false` | +`STA_USER` and `STA_PASSWORD` must be set both, or none will be used. + [man1date]: https://man7.org/linux/man-pages/man1/date.1.html [man5crontab]: https://man7.org/linux/man-pages/man5/crontab.5.html diff --git a/docker/cron/cronjob.sh b/docker/cron/cronjob.sh index 41d4f78..c0230bd 100755 --- a/docker/cron/cronjob.sh +++ b/docker/cron/cronjob.sh @@ -1,6 +1,8 @@ #!/bin/bash -set -ex +set -e +# for debugging +# set -x STA_URL="${STA_URL:?"Error: STA_URL not set"}" DATABASE_URL="file://${DATA_DIR}" @@ -10,6 +12,16 @@ if [ "$OVERWRITE_METADATA" = "true" ]; then ruby transload get metadata --database_url "${DATABASE_URL}" --overwrite "$@" fi -ruby transload put metadata --database_url "${DATABASE_URL}" --destination "${STA_URL}" "$@" +if [ -z "${STA_USER}" ] || [ -z "${STA_PASSWORD}" ]; then + ruby transload put metadata --database_url "${DATABASE_URL}" --destination "${STA_URL}" "$@" +else + ruby transload put metadata --database_url "${DATABASE_URL}" --destination "${STA_URL}" --user "${STA_USER}:${STA_PASSWORD}" "$@" +fi + ruby transload get observations --database_url "${DATABASE_URL}" "$@" -ruby transload put observations --database_url "${DATABASE_URL}" --destination "${STA_URL}" --date "${INTERVAL}" "$@" + +if [ -z "${STA_USER}" ] || [ -z "${STA_PASSWORD}" ]; then + ruby transload put observations --database_url "${DATABASE_URL}" --destination "${STA_URL}" --date "${INTERVAL}" "$@" +else + ruby transload put observations --database_url "${DATABASE_URL}" --destination "${STA_URL}" --date "${INTERVAL}" --user "${STA_USER}:${STA_PASSWORD}" "$@" +fi diff --git a/docker/cron/entrypoint.sh b/docker/cron/entrypoint.sh index b3ef338..37e1720 100644 --- a/docker/cron/entrypoint.sh +++ b/docker/cron/entrypoint.sh @@ -16,5 +16,9 @@ env | sort > /etc/cron.d/transloader-job echo >> /etc/cron.d/transloader-job echo "${SCHEDULE:-"@daily"} $(id -un) ${COMMAND} > /proc/\$(cat /var/run/crond.pid)/fd/1 2>/proc/\$(cat /var/run/crond.pid)/fd/2" >> /etc/cron.d/transloader-job +# for debugging +#cat /etc/cron.d/transloader-job +#date + # give over to cron exec cron -f -L 15 \ No newline at end of file From 1bba540d702b409bf5afa9496a7922b8b86cff13 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 10:28:27 +0100 Subject: [PATCH 15/35] ignore docker-compose during docker build --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 605503a..e64afa1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,6 @@ /spec /test /tmp +docker-compose.yaml +docker-compose.*.yaml +.dockerignore \ No newline at end of file From 0fc39f6268e95e610a91fe2bab47931b4f78cb76 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 10:28:44 +0100 Subject: [PATCH 16/35] ontology updates --- lib/ontologies/etl-ontology.ttl | 338 +++++++++++++++++++++++++++----- 1 file changed, 289 insertions(+), 49 deletions(-) diff --git a/lib/ontologies/etl-ontology.ttl b/lib/ontologies/etl-ontology.ttl index be12f2f..30ebd68 100644 --- a/lib/ontologies/etl-ontology.ttl +++ b/lib/ontologies/etl-ontology.ttl @@ -273,13 +273,42 @@ etl-ontology:DataGarrison:Wind_Speed rdf:type owl:NamedIndividual , etl-ontology:matchesObservedProperty etl-ontology:Property:Wind_Speed ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:kilometer_per_hour . +### http://gswlab.ca/ontologies/etl-ontology#DataGarrison:PAR +etl-ontology:DataGarrison:PAR rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:PAR ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:mikroeinstein_per_square_meter_per_second . +### http://gswlab.ca/ontologies/etl-ontology#DataGarrison:Rain +etl-ontology:DataGarrison:Rain rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Precipitation ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:millimeter . + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:air_temp etl-ontology:EnvironmentCanada:air_temp rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; etl-ontology:matchesObservedProperty etl-ontology:Property:Air_Temperature ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:air_temp_1 +etl-ontology:EnvironmentCanada:air_temp_1 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Air_Temperature_1 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:air_temp_2 +etl-ontology:EnvironmentCanada:air_temp_2 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Air_Temperature_2 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:air_temp_3 +etl-ontology:EnvironmentCanada:air_temp_3 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Air_Temperature_3 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:air_temp_12hrs_ago etl-ontology:EnvironmentCanada:air_temp_12hrs_ago rdf:type owl:NamedIndividual , @@ -299,6 +328,24 @@ etl-ontology:EnvironmentCanada:avg_air_temp_pst1hr rdf:type owl:NamedIndividual etl-ontology:matchesObservedProperty etl-ontology:Property:Average_Air_Temperature_over_past_1_hour ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_air_temp_pst1hr_1 +etl-ontology:EnvironmentCanada:avg_air_temp_pst1hr_1 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Average_Air_Temperature_over_past_1_hour ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_air_temp_pst1hr_2 +etl-ontology:EnvironmentCanada:avg_air_temp_pst1hr_2 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Average_Air_Temperature_over_past_1_hour ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_air_temp_pst1hr_3 +etl-ontology:EnvironmentCanada:avg_air_temp_pst1hr_3 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Average_Air_Temperature_over_past_1_hour ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_cum_pcpn_gag_wt_fltrd_mt55-60 etl-ontology:EnvironmentCanada:avg_cum_pcpn_gag_wt_fltrd_mt55-60 rdf:type owl:NamedIndividual , @@ -348,6 +395,24 @@ etl-ontology:EnvironmentCanada:avg_snw_dpth_pst5mts rdf:type owl:NamedIndividual etl-ontology:matchesObservedProperty etl-ontology:Property:Average_snow_depth_over_past_5_minutes ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:centimeter . +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_snw_dpth_pst5mts_1 +etl-ontology:EnvironmentCanada:avg_snw_dpth_pst5mts_1 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Average_snow_depth_over_past_5_minutes_1 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:centimeter . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_snw_dpth_pst5mts_2 +etl-ontology:EnvironmentCanada:avg_snw_dpth_pst5mts_2 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Average_snow_depth_over_past_5_minutes_2 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:centimeter . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_snw_dpth_pst5mts_3 +etl-ontology:EnvironmentCanada:avg_snw_dpth_pst5mts_3 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Average_snow_depth_over_past_5_minutes_3 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:centimeter . + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_wnd_dir_10m_mt50-60 etl-ontology:EnvironmentCanada:avg_wnd_dir_10m_mt50-60 rdf:type owl:NamedIndividual , @@ -551,6 +616,24 @@ etl-ontology:EnvironmentCanada:max_air_temp_pst1hr rdf:type owl:NamedIndividual etl-ontology:matchesObservedProperty etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:max_air_temp_pst1hr_1 +etl-ontology:EnvironmentCanada:max_air_temp_pst1hr_1 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour_1 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:max_air_temp_pst1hr_2 +etl-ontology:EnvironmentCanada:max_air_temp_pst1hr_2 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour_2 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:max_air_temp_pst1hr_3 +etl-ontology:EnvironmentCanada:max_air_temp_pst1hr_3 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour_3 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:max_air_temp_pst24hrs etl-ontology:EnvironmentCanada:max_air_temp_pst24hrs rdf:type owl:NamedIndividual , @@ -642,6 +725,25 @@ etl-ontology:EnvironmentCanada:min_air_temp_pst1hr rdf:type owl:NamedIndividual etl-ontology:matchesObservedProperty etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:min_air_temp_pst1hr_1 +etl-ontology:EnvironmentCanada:min_air_temp_pst1hr_1 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour_1 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:min_air_temp_pst1hr_2 +etl-ontology:EnvironmentCanada:min_air_temp_pst1hr_2 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour_2 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:min_air_temp_pst1hr_3 +etl-ontology:EnvironmentCanada:min_air_temp_pst1hr_3 rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour_3 ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:min_air_temp_pst24hrs etl-ontology:EnvironmentCanada:min_air_temp_pst24hrs rdf:type owl:NamedIndividual , @@ -992,7 +1094,7 @@ etl-ontology:KLRSHistoricalWeather:WindDir_SD1_WVT rdf:type owl:NamedIndividual ### http://gswlab.ca/ontologies/etl-ontology#Property:Air_Pressure etl-ontology:Property:Air_Pressure rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_pressure"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0015/"^^xsd:anyURI ; etl-ontology:description "Air pressure is the force per unit area which would be exerted when the moving gas molecules of which the air is composed strike a theoretical surface of any orientation."^^xsd:string ; rdfs:label "Air Pressure" . @@ -1000,7 +1102,28 @@ etl-ontology:Property:Air_Pressure rdf:type owl:NamedIndividual , ### http://gswlab.ca/ontologies/etl-ontology#Property:Air_Temperature etl-ontology:Property:Air_Temperature rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Air Temperature" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Air_Temperature_1 +etl-ontology:Property:Air_Temperature_1 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Air Temperature" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Air_Temperature_2 +etl-ontology:Property:Air_Temperature_2 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Air Temperature" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Air_Temperature_3 +etl-ontology:Property:Air_Temperature_3 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Air Temperature" . @@ -1008,7 +1131,7 @@ etl-ontology:Property:Air_Temperature rdf:type owl:NamedIndividual , ### http://gswlab.ca/ontologies/etl-ontology#Property:Air_Temperature_12_hours_ago etl-ontology:Property:Air_Temperature_12_hours_ago rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Air Temperature 12 hours ago" . @@ -1016,7 +1139,7 @@ etl-ontology:Property:Air_Temperature_12_hours_ago rdf:type owl:NamedIndividual ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_10m_wind_speed_over_past_10_minutes etl-ontology:Property:Average_10m_wind_speed_over_past_10_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity."^^xsd:string ; rdfs:label "Average 10 meter wind speed over past 10 minutes" . @@ -1024,7 +1147,7 @@ etl-ontology:Property:Average_10m_wind_speed_over_past_10_minutes rdf:type owl:N ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_10m_wind_speed_over_past_1_hour etl-ontology:Property:Average_10m_wind_speed_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity."^^xsd:string ; rdfs:label "Average 10 meter wind speed over past 1 hour" . @@ -1032,7 +1155,7 @@ etl-ontology:Property:Average_10m_wind_speed_over_past_1_hour rdf:type owl:Named ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_10m_wind_speed_over_past_2_minutes etl-ontology:Property:Average_10m_wind_speed_over_past_2_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity."^^xsd:string ; rdfs:label "Average 10 meter wind speed over past 2 minutes" . @@ -1040,7 +1163,28 @@ etl-ontology:Property:Average_10m_wind_speed_over_past_2_minutes rdf:type owl:Na ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_Air_Temperature_over_past_1_hour etl-ontology:Property:Average_Air_Temperature_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Average air temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Average_Air_Temperature_over_past_1_hour_1 +etl-ontology:Property:Average_Air_Temperature_over_past_1_hour_1 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Average air temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Average_Air_Temperature_over_past_1_hour_2 +etl-ontology:Property:Average_Air_Temperature_over_past_1_hour_2 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Average air temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Average_Air_Temperature_over_past_1_hour_3 +etl-ontology:Property:Average_Air_Temperature_over_past_1_hour_3 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Average air temperature over past 1 hour" . @@ -1052,6 +1196,13 @@ etl-ontology:Property:Average_Battery_Voltage rdf:type owl:NamedIndividual , etl-ontology:description "The battery voltage of a datalogger or sensing system, often recorded as an indicator of data reliability."^^xsd:string ; rdfs:label "Average Battery Voltage over an interval"^^xsd:string . +### http://gswlab.ca/ontologies/etl-ontology#Property:batteryVoltage +etl-ontology:Property:batteryVoltage rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "https://mmisw.org/ont/CUAHSI/WaterQualityCore2/batteryVoltage"^^xsd:anyURI ; + etl-ontology:description "The battery voltage of a datalogger or sensing system, often recorded as an indicator of data reliability."^^xsd:string ; + rdfs:label "Battery Voltage"^^xsd:string . + ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_snow_depth_over_past_5_minutes etl-ontology:Property:Average_snow_depth_over_past_5_minutes rdf:type owl:NamedIndividual , @@ -1060,11 +1211,32 @@ etl-ontology:Property:Average_snow_depth_over_past_5_minutes rdf:type owl:NamedI etl-ontology:description "Average snow depth over past 5 minutes." ; rdfs:label "Average snow depth over past 5 minutes" . +### http://gswlab.ca/ontologies/etl-ontology#Property:Average_snow_depth_over_past_5_minutes_1 +etl-ontology:Property:Average_snow_depth_over_past_5_minutes_1 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://mmisw.org/ont/ioos_test/parameters_regional_web_services/snowdepth"^^xsd:anyURI ; + etl-ontology:description "Average snow depth over past 5 minutes." ; + rdfs:label "Average snow depth over past 5 minutes" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Average_snow_depth_over_past_5_minutes_2 +etl-ontology:Property:Average_snow_depth_over_past_5_minutes_2 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://mmisw.org/ont/ioos_test/parameters_regional_web_services/snowdepth"^^xsd:anyURI ; + etl-ontology:description "Average snow depth over past 5 minutes." ; + rdfs:label "Average snow depth over past 5 minutes" . + + ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_snow_depth_over_past_5_minutes_3 +etl-ontology:Property:Average_snow_depth_over_past_5_minutes_3 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://mmisw.org/ont/ioos_test/parameters_regional_web_services/snowdepth"^^xsd:anyURI ; + etl-ontology:description "Average snow depth over past 5 minutes." ; + rdfs:label "Average snow depth over past 5 minutes" . + ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_wind_direction_at_10_meters_over_past_2_minutes etl-ontology:Property:Average_wind_direction_at_10_meters_over_past_2_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming." ; rdfs:label "Average wind direction at 10 meters over past 2 minutes" . @@ -1072,7 +1244,7 @@ etl-ontology:Property:Average_wind_direction_at_10_meters_over_past_2_minutes rd ### http://gswlab.ca/ontologies/etl-ontology#Property:Average_wind_speed_at_precipitation_gauge_over_past_10_minutes etl-ontology:Property:Average_wind_speed_at_precipitation_gauge_over_past_10_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity." ; rdfs:label "Average wind speed at precipitation gauge over past 10 minutes" . @@ -1224,7 +1396,7 @@ etl-ontology:Property:Max_wind_speed_past_1_hour_timestamp rdf:type owl:NamedInd ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_10m_peak_wind_speed_over_past_1_hour etl-ontology:Property:Maximum_10m_peak_wind_speed_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity."^^xsd:string ; rdfs:label "Maximum 10 meter peak wind speed over past 1 hour" . @@ -1232,7 +1404,7 @@ etl-ontology:Property:Maximum_10m_peak_wind_speed_over_past_1_hour rdf:type owl: ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_10m_wind_speed_over_past_10_minutes etl-ontology:Property:Maximum_10m_wind_speed_over_past_10_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity."^^xsd:string ; rdfs:label "Maximum 10 meter wind speed over past 10 minutes" . @@ -1240,7 +1412,7 @@ etl-ontology:Property:Maximum_10m_wind_speed_over_past_10_minutes rdf:type owl:N ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_10m_wind_speed_over_past_1_hour etl-ontology:Property:Maximum_10m_wind_speed_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity."^^xsd:string ; rdfs:label "Maximum 10 meter wind speed over past 1 hour" . @@ -1248,7 +1420,28 @@ etl-ontology:Property:Maximum_10m_wind_speed_over_past_1_hour rdf:type owl:Named ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Air_Temperature_over_past_1_hour etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Maximum Air Temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Air_Temperature_over_past_1_hour_1 +etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour_1 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Maximum Air Temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Air_Temperature_over_past_1_hour_2 +etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour_2 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Maximum Air Temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Air_Temperature_over_past_1_hour_3 +etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour_3 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Maximum Air Temperature over past 1 hour" . @@ -1256,7 +1449,7 @@ etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour rdf:type owl:Name ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Air_Temperature_over_past_24_hours etl-ontology:Property:Maximum_Air_Temperature_over_past_24_hours rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Maximum Air Temperature over past 24 hours" . @@ -1264,7 +1457,7 @@ etl-ontology:Property:Maximum_Air_Temperature_over_past_24_hours rdf:type owl:Na ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Air_Temperature_over_past_6_hours etl-ontology:Property:Maximum_Air_Temperature_over_past_6_hours rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Maximum Air Temperature over past 6 hours" . @@ -1288,7 +1481,7 @@ etl-ontology:Property:Maximum_Battery_Voltage_Past_1_Hour rdf:type owl:NamedIndi ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Relative_Humidity_over_past_1_hour etl-ontology:Property:Maximum_Relative_Humidity_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/relative_humidity"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0413/"^^xsd:anyURI ; etl-ontology:description "Amount of moisture in the air as vapor relative to how much it can possibly hold at the same temperature."^^xsd:string ; rdfs:label "Maximum Relative Humidity over past 1 hour" . @@ -1304,21 +1497,46 @@ etl-ontology:Property:Maximum_visibility_over_past_10_minutes rdf:type owl:Named ### http://gswlab.ca/ontologies/etl-ontology#Property:Mean_sea_level_pressure etl-ontology:Property:Mean_sea_level_pressure rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_pressure_at_mean_sea_level"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0015/_at_mean_sea_level"^^xsd:anyURI ; etl-ontology:description "Air pressure is the force per unit area which would be exerted when the moving gas molecules of which the air is composed strike a theoretical surface of any orientation."^^xsd:string ; rdfs:label "Mean Sea Level Pressure" . ### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_1_hour etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Minimum Air Temperature over past 1 hour" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_1_hour_1 +etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour_1 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Minimum Air Temperature over past 1 hour" . + + +### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_1_hour_2 +etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour_2 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; + etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; + rdfs:label "Minimum Air Temperature over past 1 hour" . + + +### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_1_hour_3 +etl-ontology:Property:Minimum_Air_Temperature_over_past_1_hour_3 rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Minimum Air Temperature over past 1 hour" . + + ### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_24_hours etl-ontology:Property:Minimum_Air_Temperature_over_past_24_hours rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Minimum Air Temperature over past 24 hours" . @@ -1326,7 +1544,7 @@ etl-ontology:Property:Minimum_Air_Temperature_over_past_24_hours rdf:type owl:Na ### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Air_Temperature_over_past_6_hours etl-ontology:Property:Minimum_Air_Temperature_over_past_6_hours rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_temperature"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/"^^xsd:anyURI ; etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Minimum Air Temperature over past 6 hours" . @@ -1359,7 +1577,7 @@ etl-ontology:Property:Minimum_Battery_Voltage_Timestamp rdf:type owl:NamedIndivi ### http://gswlab.ca/ontologies/etl-ontology#Property:Minimum_Relative_Humidity_over_past_1_hour etl-ontology:Property:Minimum_Relative_Humidity_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/relative_humidity"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0413/"^^xsd:anyURI ; etl-ontology:description "Amount of moisture in the air as vapor relative to how much it can possibly hold at the same temperature."^^xsd:string ; rdfs:label "Minimum Relative Humidity over past 1 hour" . @@ -1423,7 +1641,7 @@ etl-ontology:Property:Non-cumulative_cloud_cover_or_amount_layer_3 rdf:type owl: ### http://gswlab.ca/ontologies/etl-ontology#Property:Precipitation_Amount_over_past_1_hour etl-ontology:Property:Precipitation_Amount_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "Precipitation Amount over past 1 hour" . @@ -1431,7 +1649,7 @@ etl-ontology:Property:Precipitation_Amount_over_past_1_hour rdf:type owl:NamedIn ### http://gswlab.ca/ontologies/etl-ontology#Property:Precipitation_Amount_over_past_24_hours etl-ontology:Property:Precipitation_Amount_over_past_24_hours rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "Precipitation amount over past 24 hours" . @@ -1439,7 +1657,7 @@ etl-ontology:Property:Precipitation_Amount_over_past_24_hours rdf:type owl:Named ### http://gswlab.ca/ontologies/etl-ontology#Property:Precipitation_Amount_over_past_3_hours etl-ontology:Property:Precipitation_Amount_over_past_3_hours rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "Precipitation amount over past 3 hours" . @@ -1447,7 +1665,7 @@ etl-ontology:Property:Precipitation_Amount_over_past_3_hours rdf:type owl:NamedI ### http://gswlab.ca/ontologies/etl-ontology#Property:Precipitation_Amount_over_past_6_hours etl-ontology:Property:Precipitation_Amount_over_past_6_hours rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "Precipitation amount over past 6 hours" . @@ -1455,10 +1673,24 @@ etl-ontology:Property:Precipitation_Amount_over_past_6_hours rdf:type owl:NamedI ### http://gswlab.ca/ontologies/etl-ontology#Property:Precipitation_Amount_since_last_synoptic_hour etl-ontology:Property:Precipitation_Amount_since_last_synoptic_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "Precipitation Amount since last synoptic hour" . +### http://gswlab.ca/ontologies/etl-ontology#Property:Precipitation +etl-ontology:Property:Precipitation rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; + etl-ontology:description "Thickness of precipitation amount (liquid water equivalent) in the atmosphere by in-situ rain gauge."^^xsd:string ; + rdfs:label "Precipitation" . + +### http://gswlab.ca/ontologies/etl-ontology#Property:PAR +etl-ontology:Property:PAR rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P14/current/GVAR0627/"^^xsd:anyURI ; + etl-ontology:description "Photosynthetically active radiation (PAR) is a measure of light. It is the intensity of the part of sunlight that plants can use to support new growth, and also the wavelengths of light that our eyes are sensitive to."^^xsd:string ; + rdfs:label "Photosynthetically Active Radiation" . + ### http://gswlab.ca/ontologies/etl-ontology#Property:Present_Weather etl-ontology:Property:Present_Weather rdf:type owl:NamedIndividual , @@ -1471,7 +1703,7 @@ etl-ontology:Property:Present_Weather rdf:type owl:NamedIndividual , ### http://gswlab.ca/ontologies/etl-ontology#Property:Rainfall_amount_over_past_1_hour etl-ontology:Property:Rainfall_amount_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "Rainfall amount over past 1 hour" . @@ -1479,7 +1711,7 @@ etl-ontology:Property:Rainfall_amount_over_past_1_hour rdf:type owl:NamedIndivid ### http://gswlab.ca/ontologies/etl-ontology#Property:Rainfall_since_last_synoptic_hour etl-ontology:Property:Rainfall_since_last_synoptic_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "Rainfall since last synoptic hour" . @@ -1487,8 +1719,8 @@ etl-ontology:Property:Rainfall_since_last_synoptic_hour rdf:type owl:NamedIndivi ### http://gswlab.ca/ontologies/etl-ontology#Property:Relative_Humidity etl-ontology:Property:Relative_Humidity rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/relative_humidity"^^xsd:anyURI ; - etl-ontology:description "Amount of moisture in the air as vapor relative to how much it can possibly hold at the same temperature."^^xsd:string ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0413/"^^xsd:anyURI ; + etl-ontology:description "The ratio of the amount of water vapour in the air compared to the maximum amount of water vapour that can theoretically be held at the air's temperature."^^xsd:string ; rdfs:label "Relative Humidity" . @@ -1551,7 +1783,7 @@ etl-ontology:Property:Solar_Panel_Temperature_Minimum rdf:type owl:NamedIndividu ### http://gswlab.ca/ontologies/etl-ontology#Property:Station_Pressure etl-ontology:Property:Station_Pressure rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/air_pressure"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0015/"^^xsd:anyURI ; etl-ontology:description "Air pressure is the force per unit area which would be exerted when the moving gas molecules of which the air is composed strike a theoretical surface of any orientation."^^xsd:string ; rdfs:label "Station Pressure" . @@ -1567,7 +1799,7 @@ etl-ontology:Property:Total_global_solar_radiation_over_past_1_hour rdf:type owl ### http://gswlab.ca/ontologies/etl-ontology#Property:Vector_average_10m_wind_direction_over_past_1_hour etl-ontology:Property:Vector_average_10m_wind_direction_over_past_1_hour rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; rdfs:label "Vector average 10 meter wind direction over past 1 hour" . @@ -1575,7 +1807,7 @@ etl-ontology:Property:Vector_average_10m_wind_direction_over_past_1_hour rdf:typ ### http://gswlab.ca/ontologies/etl-ontology#Property:Vectoral_average_10m_wind_direction_over_past_10_minutes etl-ontology:Property:Vectoral_average_10m_wind_direction_over_past_10_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; rdfs:label "Vectoral average 10 meter wind direction over past 10 minutes" . @@ -1583,7 +1815,7 @@ etl-ontology:Property:Vectoral_average_10m_wind_direction_over_past_10_minutes r ### http://gswlab.ca/ontologies/etl-ontology#Property:Vectoral_average_10m_wind_direction_over_past_2_minutes etl-ontology:Property:Vectoral_average_10m_wind_direction_over_past_2_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; rdfs:label "Vectoral average 10 meter wind direction over past 2 minutes" . @@ -1615,7 +1847,7 @@ etl-ontology:Property:Wet_bulb_temperature rdf:type owl:NamedIndividual , ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_Direction_Average etl-ontology:Property:Wind_Direction_Average rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction" ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/" ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming." ; rdfs:label "Average wind direction over an interval" . @@ -1623,7 +1855,7 @@ etl-ontology:Property:Wind_Direction_Average rdf:type owl:NamedIndividual , ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_Direction_Standard_Deviation etl-ontology:Property:Wind_Direction_Standard_Deviation rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction" ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/" ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming." ; rdfs:label "Standard deviation of wind direction over an interval" . @@ -1631,23 +1863,23 @@ etl-ontology:Property:Wind_Direction_Standard_Deviation rdf:type owl:NamedIndivi ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_From_Direction etl-ontology:Property:Wind_From_Direction rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; - etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; + etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. (Vertical motion in the atmosphere has the standard name upward_air_velocity.) In meteorological reports, the direction of the wind vector is usually (but not always) given as the direction from which it is blowing (wind_from_direction) (westerly, northerly, etc.). In other contexts, such as atmospheric modelling, it is often natural to give the direction in the usual manner of vectors as the heading or the direction to which it is blowing (wind_to_direction) (eastward, southward, etc.) 'from_direction' is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; rdfs:label "Wind From Direction" . ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_Speed etl-ontology:Property:Wind_Speed rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed"^^xsd:anyURI ; - etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity."^^xsd:string ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; + etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. (Vertical motion in the atmosphere has the standard name upward_air_velocity.) The wind speed is the magnitude of the wind velocity."^^xsd:string ; rdfs:label "Wind Speed" . ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_Speed_Average etl-ontology:Property:Wind_Speed_Average rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed" ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/" ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity." ; rdfs:label "Average wind speed over an interval" . @@ -1655,7 +1887,7 @@ etl-ontology:Property:Wind_Speed_Average rdf:type owl:NamedIndividual , ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_Speed_Maximum etl-ontology:Property:Wind_Speed_Maximum rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed" ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/" ; etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity." ; rdfs:label "Maximum wind speed over an interval" . @@ -1663,7 +1895,7 @@ etl-ontology:Property:Wind_Speed_Maximum rdf:type owl:NamedIndividual , ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_direction_associated_with_the_past_1-hour_maximum_wind_speed_at_10m etl-ontology:Property:Wind_direction_associated_with_the_past_1-hour_maximum_wind_speed_at_10m rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; rdfs:label "Wind direction associated with the maximum wind speed at 10 meters over past 1 hour" . @@ -1671,7 +1903,7 @@ etl-ontology:Property:Wind_direction_associated_with_the_past_1-hour_maximum_win ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_direction_associated_with_the_past_1-hour_peak_wind_speed_at_10m etl-ontology:Property:Wind_direction_associated_with_the_past_1-hour_peak_wind_speed_at_10m rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; rdfs:label "Wind direction associated with the peak wind speed at 10 meters over past 1 hour" . @@ -1679,7 +1911,7 @@ etl-ontology:Property:Wind_direction_associated_with_the_past_1-hour_peak_wind_s ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_direction_associated_with_the_past_10-minute_maximum_wind_speed_at_10m etl-ontology:Property:Wind_direction_associated_with_the_past_10-minute_maximum_wind_speed_at_10m rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_from_direction"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0036/"^^xsd:anyURI ; etl-ontology:description "Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. \"from_direction\" is used in the construction X_from_direction and indicates the direction from which the velocity vector of X is coming."^^xsd:string ; rdfs:label "Wind direction associated with the maximum wind speed at 10 meters over past 10 minutes" . @@ -1687,15 +1919,15 @@ etl-ontology:Property:Wind_direction_associated_with_the_past_10-minute_maximum_ ### http://gswlab.ca/ontologies/etl-ontology#Property:Wind_speed_of_gust etl-ontology:Property:Wind_speed_of_gust rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed_of_gust"^^xsd:anyURI ; - etl-ontology:description "Speed is the magnitude of velocity. A gust is a sudden brief period of high wind speed." ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0039/"^^xsd:anyURI ; + etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. (Vertical motion in the atmosphere has the standard name upward_air_velocity.) The wind speed is the magnitude of the wind velocity. A gust is a sudden brief period of high wind speed. In an observed timeseries of wind speed, the gust wind speed can be indicated by a cell_methods of maximum for the time-interval. In an atmospheric model which has a parametrised calculation of gustiness, the gust wind speed may be separately diagnosed from the wind speed." ; rdfs:label "Wind speed of gust" . ### http://gswlab.ca/ontologies/etl-ontology#Property:maximum_10m_wind_gust_speed_over_past_10_minutes etl-ontology:Property:maximum_10m_wind_gust_speed_over_past_10_minutes rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/cf/parameter/wind_speed_of_gust"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0039/"^^xsd:anyURI ; etl-ontology:description "A gust is a sudden brief period of high wind speed."^^xsd:string ; rdfs:label "Maximum 10 meter wind gust speed over past 10 minutes" . @@ -1751,7 +1983,7 @@ etl-ontology:Property:maximum_10m_wind_gust_speed_over_past_10_minutes rdf:type ### http://gswlab.ca/ontologies/etl-ontology#Property:5-minute_cumulative_precipitation_gauge_filtered_weight_for_minutes_55_to_60 rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://mmisw.org/ont/ioos/parameter/precipitation_amount"^^xsd:anyURI ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P01/current/CPRPRG01/"^^xsd:anyURI ; etl-ontology:description "The deposit of atmospheric moisture as rain, hail, sleet, snow, frost or dew. Precipition amount is often measured as depth of liquid equivalent moisture at a point but also can be expressed as liquid volume or mass per unit area."^^xsd:string ; rdfs:label "5-minute cumulative precipitation gauge filtered weight for minutes 55 to 60" . @@ -1919,6 +2151,14 @@ etl-ontology:Unit:millimeter rdf:type owl:NamedIndividual , rdfs:label "millimeter"^^xsd:string . +### http://gswlab.ca/ontologies/etl-ontology#Unit:mikroeinstein_per_square_meter_per_second +etl-ontology:Unit:mikroeinstein_per_square_meter_per_second rdf:type owl:NamedIndividual , + etl-ontology:UnitOfMeasurement ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P06/current/UMES/"^^xsd:anyURI ; + etl-ontology:observationType "http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement"^^xsd:anyURI ; + etl-ontology:symbol "uE/m^2/s"^^xsd:string ; + rdfs:label "MicroEinsteins per square metre per second"^^xsd:string . + ### http://gswlab.ca/ontologies/etl-ontology#Unit:obscuring_phenomena etl-ontology:Unit:obscuring_phenomena rdf:type owl:NamedIndividual , etl-ontology:UnitOfMeasurement ; From 735940ae1738d84243455202b9abb4d79235b4ca Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 10:30:24 +0100 Subject: [PATCH 17/35] split put metadata from cronjob --- ...> docker-compose.get-and-put-metadata.yaml | 159 +++++++----------- docker-compose.get-and-put-observations.yaml | 124 ++++++++++++++ docker/cron/README.md | 2 +- docker/cron/cronjob.sh | 2 +- update-sensors.sh | 62 +++++++ 5 files changed, 246 insertions(+), 103 deletions(-) rename docker-compose.yaml => docker-compose.get-and-put-metadata.yaml (63%) create mode 100644 docker-compose.get-and-put-observations.yaml create mode 100755 update-sensors.sh diff --git a/docker-compose.yaml b/docker-compose.get-and-put-metadata.yaml similarity index 63% rename from docker-compose.yaml rename to docker-compose.get-and-put-metadata.yaml index 376ab98..a26dee7 100644 --- a/docker-compose.yaml +++ b/docker-compose.get-and-put-metadata.yaml @@ -15,28 +15,12 @@ version: "3" # Default values/templates for the different services x-defaults: # Defaults for the transloader cron job - cronjob: &cron-job-defaults - image: data-transloader-job:latest - build: - context: . - dockerfile: docker/cron/Dockerfile - volumes: - - data:/data - networks: - - sta - restart: unless-stopped - environment: - # the STA endpoint to load data to - STA_URL: http://web:8080/FROST-Server/v1.0/ - # the moving window to download data for (optional, defaults to 2 days) - MOVING_WINDOW: "2 days" - # the schedule of the job (optional, defaults to @daily) - SCHEDULE: "@hourly" - DATA_DIR: /data - OVERWRITE_METADATA: "false" get-metadata-environment: &get-metadata-environment DATA_DIR: /data + # the STA endpoint to load data to + STA_URL: http://sta:8080/FROST-Server/v1.0/ + INSTRUMENT_DETAILS_BASE_URL: https://canwin-datahub.ad.umanitoba.ca/data/instrument_details # Defaults for getting metadata get-metadata-job-defaults: &get-metadata-job-defaults @@ -45,9 +29,12 @@ x-defaults: context: . dockerfile: Dockerfile environment: *get-metadata-environment - restart: "no" + restart: "on-failure" + networks: + - sta volumes: - data:/data + - ./update-sensors.sh:/usr/local/bin/update-sensors # Defaults for getting metadata from data garrison # @@ -60,33 +47,60 @@ x-defaults: entrypoint: [/bin/sh, -e, -x, -c] command: - | + # getting the metadata from the provider ruby transload get metadata \ --provider data_garrison \ --user_id "$${USER_ID}" \ --station_id "$${STATION_ID}" \ --database_url "file://$${DATA_DIR}" \ --overwrite + + # customize the metadata ruby transload set metadata \ --provider data_garrison \ --user_id "$${USER_ID}" \ --station_id "$${STATION_ID}" \ --database_url "file://$${DATA_DIR}" \ --key longitude \ - --value "$${LONGITUDE}" + --value "$${LONGITUDE}" > /dev/null ruby transload set metadata \ --provider data_garrison \ --user_id "$${USER_ID}" \ --station_id "$${STATION_ID}" \ --database_url "file://$${DATA_DIR}" \ --key latitude \ - --value "$${LATITUDE}" + --value "$${LATITUDE}" > /dev/null ruby transload set metadata \ --provider data_garrison \ --user_id "$${USER_ID}" \ --station_id "$${STATION_ID}" \ --database_url "file://$${DATA_DIR}" \ --key timezone_offset \ - --value "$${TIMEZONE_OFFSET}" + --value "$${TIMEZONE_OFFSET}" > /dev/null + ruby transload set metadata \ + --provider data_garrison \ + --user_id "$${USER_ID}" \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" \ + --key name \ + --value "$${NAME}" > /dev/null + ruby transload set metadata \ + --provider data_garrison \ + --user_id "$${USER_ID}" \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" \ + --key description \ + --value "$${DESCRIPTION}" > /dev/null + + # push the metadata to the STA + ruby transload put metadata \ + --database_url "file://$${DATA_DIR}" \ + --destination "$${STA_URL}" \ + --provider data_garrison \ + --user_id "$${USER_ID}" \ + --station_id "$${STATION_ID}" + + update-sensors "$${STATION_ID}" "$${STATION_SENSOR_NAME}" "$${INSTRUMENT_DETAILS}" # Defaults for getting metadata from environment canada environment-canada-get-metadata-job-defaults: @@ -95,7 +109,18 @@ x-defaults: entrypoint: [/bin/sh, -e, -x, -c] command: - | - ruby transload get metadata --provider environment_canada --station_id "$${STATION_ID}" --database_url "file://$${DATA_DIR}" --overwrite + # getting the metadata from the provider + ruby transload get metadata --provider environment_canada \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" \ + --overwrite + + # push the metadata to the STA + ruby transload put metadata \ + --provider environment_canada \ + --destination "$${STA_URL}" \ + --station_id "$${STATION_ID}" \ + --database_url "file://$${DATA_DIR}" services: get-metadata-environment-canada-CWCF: # Berens, River @@ -185,6 +210,10 @@ services: LATITUDE: "50.40768" LONGITUDE: "-97.96212" TIMEZONE_OFFSET: "-06:00" + NAME: St Laurent Li Taan Aen Staansyoon_17060 + STATION_SENSOR_NAME: Dawson Bay Li Taan Aen Staansyoon + INSTRUMENT_DETAILS: dawbay-met-sensors + DESCRIPTION: Manitoba Metis Federation weather station located in St.Laurent get-metadata-data-garrison-MMF2: <<: *data-garrision-get-metadata-job-defaults @@ -195,82 +224,10 @@ services: LATITUDE: "52.977050" LONGITUDE: "-100.979630" TIMEZONE_OFFSET: "-06:00" - - transload-data-data-garrison-MMF1: - <<: *cron-job-defaults - command: - [ - --provider, - data_garrison, - --user_id, - "300234068013390", - --station_id, - "300534063017060", - ] - - transload-data-data-garrison-MMF2: - <<: *cron-job-defaults - command: - [ - --provider, - data_garrison, - --user_id, - "300234068013390", - --station_id, - "300534061454190", - ] - - transload-data-environment-canada-CWCF: # Berens, River - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWCF] - - transload-data-environment-canada-CWZE: # Carberry, CS - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWZE] - - transload-data-environment-canada-CWZT: # Dauphin, Airport - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWZT] - - transload-data-environment-canada-CWSZ: # Fisher, Branch, (auto) - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWSZ] - - transload-data-environment-canada-CWWS: # George, Island - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWWS] - - transload-data-environment-canada-CPGH: # Gimli, Climate - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CPGH] - - transload-data-environment-canada-CWJD: # Grand, Rapids, (auto) - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWJD] - - transload-data-environment-canada-CWOO: # McCreary - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWOO] - - transload-data-environment-canada-CWOJ: # Oak, Point, Marine - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWOJ] - - transload-data-environment-canada-CWPG: # Portage, Southport - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWPG] - - transload-data-environment-canada-CWEQ: # Swan, River, RCS - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWEQ] - - transload-data-environment-canada-CWII: # victoria, Beach, (auto) - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWII] - - transload-data-environment-canada-CWWP: # Wasagaming - <<: *cron-job-defaults - command: [--provider, environment_canada, --station_id, CWWP] + NAME: Dawson Bay Li Taan Aen Staansyoon_54190 + STATION_SENSOR_NAME: St Laurent Li Taan Aen Staansyoon + INSTRUMENT_DETAILS: st-laurent-met-sensors + DESCRIPTION: Manitoba Metis Federation weather station located in Dawson Bay volumes: data: @@ -279,4 +236,4 @@ volumes: networks: sta: external: - name: frost-server_default + name: sta_default diff --git a/docker-compose.get-and-put-observations.yaml b/docker-compose.get-and-put-observations.yaml new file mode 100644 index 0000000..310d72e --- /dev/null +++ b/docker-compose.get-and-put-observations.yaml @@ -0,0 +1,124 @@ +# +# Docker compose setup for transloading observations from +# environment canada and data garrison to a SensorThings API +# +# There are two types of services: +# +# * get-metadata-jobs that fetch the metadata from the provider for a station and push it +# to the STA. The jobs are meant to run once and then exit. +# +# * transload-data-jobs are cronjobs that fetch new data from the provider and transload +# it to the STA. There is one cronjob per station. +# +version: "3" + +# Default values/templates for the different services +x-defaults: + # Defaults for the transloader cron job + cronjob: &cron-job-defaults + image: data-transloader-job:latest + build: + context: . + dockerfile: docker/cron/Dockerfile + volumes: + - data:/data + networks: + - sta + restart: unless-stopped + environment: + # the STA endpoint to load data to + STA_URL: http://sta:8080/FROST-Server/v1.0/ + # the moving window to download data for (optional, defaults to 2 days) + MOVING_WINDOW: "2 days" + # the schedule of the job (optional, defaults to @daily) + SCHEDULE: "34 * * * *" + DATA_DIR: /data + OVERWRITE_METADATA: "false" + + +services: + + transload-data-data-garrison-MMF1: + <<: *cron-job-defaults + command: + [ + --provider, + data_garrison, + --user_id, + "300234068013390", + --station_id, + "300534063017060", + ] + + transload-data-data-garrison-MMF2: + <<: *cron-job-defaults + command: + [ + --provider, + data_garrison, + --user_id, + "300234068013390", + --station_id, + "300534061454190", + ] + + transload-data-environment-canada-CWCF: # Berens, River + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWCF] + + transload-data-environment-canada-CWZE: # Carberry, CS + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWZE] + + transload-data-environment-canada-CWZT: # Dauphin, Airport + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWZT] + + transload-data-environment-canada-CWSZ: # Fisher, Branch, (auto) + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWSZ] + + transload-data-environment-canada-CWWS: # George, Island + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWWS] + + transload-data-environment-canada-CPGH: # Gimli, Climate + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CPGH] + + transload-data-environment-canada-CWJD: # Grand, Rapids, (auto) + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWJD] + + transload-data-environment-canada-CWOO: # McCreary + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWOO] + + transload-data-environment-canada-CWOJ: # Oak, Point, Marine + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWOJ] + + transload-data-environment-canada-CWPG: # Portage, Southport + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWPG] + + transload-data-environment-canada-CWEQ: # Swan, River, RCS + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWEQ] + + transload-data-environment-canada-CWII: # victoria, Beach, (auto) + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWII] + + transload-data-environment-canada-CWWP: # Wasagaming + <<: *cron-job-defaults + command: [--provider, environment_canada, --station_id, CWWP] + +volumes: + data: + driver: local + +networks: + sta: + external: + name: sta_default diff --git a/docker/cron/README.md b/docker/cron/README.md index 9c56121..82cadda 100644 --- a/docker/cron/README.md +++ b/docker/cron/README.md @@ -4,7 +4,7 @@ The Docker image runs `cron` as PID 1 and regularly executes the following scrip ```shell ruby transload get metadata ... ${COMMAND} # optional -ruby transload put metadata ... ${COMMAND} +ruby transload put metadata ... ${COMMAND} # optional ruby transload get observations ... ${COMMAND} ruby transload put observations ... ${COMMAND} ``` diff --git a/docker/cron/cronjob.sh b/docker/cron/cronjob.sh index cb4cbf4..aa86636 100755 --- a/docker/cron/cronjob.sh +++ b/docker/cron/cronjob.sh @@ -8,8 +8,8 @@ INTERVAL="$(date -u -d "${MOVING_WINDOW} ago" "+%Y-%m-%dT%TZ")/$(date -u "+%Y-%m if [ "$OVERWRITE_METADATA" = "true" ]; then ruby transload get metadata --database_url "${DATABASE_URL}" --overwrite "$@" + ruby transload put metadata --database_url "${DATABASE_URL}" --destination "${STA_URL}" "$@" fi -ruby transload put metadata --database_url "${DATABASE_URL}" --destination "${STA_URL}" "$@" ruby transload get observations --database_url "${DATABASE_URL}" "$@" ruby transload put observations --database_url "${DATABASE_URL}" --destination "${STA_URL}" --date "${INTERVAL}" "$@" diff --git a/update-sensors.sh b/update-sensors.sh new file mode 100755 index 0000000..af89d72 --- /dev/null +++ b/update-sensors.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -ex + +#STA_URL="http://localhost:8080/FROST-Server/v1.0" +#INSTRUMENT_DETAILS_BASE_URL="https://canwin-datahub.ad.umanitoba.ca/data/instrument_details" + +function get-sensor-id() { + local NAME="$1" + curl -sG "${STA_URL}/Sensors" --data-urlencode "\$filter=name eq '${NAME}'" | jq -r '.value[0]["@iot.id"]'ss +} + +function update-sensors() { + local STATION_ID="$1" + local STATION_SENSOR_NAME="$2" + local INSTRUMENT_DETAILS="$3" + local METADATA_URL="${INSTRUMENT_DETAILS_BASE_URL}/${INSTRUMENT_DETAILS}" + declare -A NAMES DESCRIPTIONS + + + NAMES["Station ${STATION_ID} PAR Sensor"]="${STATION_SENSOR_NAME} HOBO PAR Sensor" + NAMES["Station ${STATION_ID} Rain Sensor"]="${STATION_SENSOR_NAME} HOBO Rain Gauge" + NAMES["Station ${STATION_ID} Temperature Sensor"]="${STATION_SENSOR_NAME} HOBO Temperature/Relative Humidity Smart Sensor- Temperature Sensor" + NAMES["Station ${STATION_ID} RH Sensor"]="${STATION_SENSOR_NAME} HOBO Temperature/Relative Humidity Smart Sensor- Relative Humidity (RH Sensor)" + NAMES["Station ${STATION_ID} Wind Speed Sensor"]="${STATION_SENSOR_NAME} R M Young Wind sensor/monitor - Wind Speed Sensor" + NAMES["Station ${STATION_ID} Gust Speed Sensor"]="${STATION_SENSOR_NAME} R M Young Wind sensor/monitor - Gust Speed Sensor" + + DESCRIPTIONS["Station ${STATION_ID} PAR Sensor"]="Photosynthetically active radiation (PAR) is a measure of light. It is the intensity of the part of sunlight that plants can use to support new growth, and also the wavelengths of light that our eyes are sensitive to. It is measured on a small white disc that records the sum of light falling directly from the sun plus the light scattered by the sky and clouds. It is reported in a unit called a micro-Einstein." + DESCRIPTIONS["Station ${STATION_ID} Rain Sensor"]="Rain is reported in millimetres accumulated every 15 minutes in a gauge set about 1 m above the ground. Rain falling into a 6-inch diameter funnel drips onto one of a pair of “buckets”, mounted on a teeter-totter. When one bucket is filled, the teeter-totter flips, the first bucket is emptied and rain then drips into the other “bucket” until it is filled, and so on. Each flip is equal to 0.2 mm of rain." + DESCRIPTIONS["Station ${STATION_ID} Temperature Sensor"]="We report temperature in the shade, in Centigrade degrees. It is measured using an electronic thermometer shielded from direct sunlight and mounted inside an enclosure with louvered walls to allow free air flow past the sensor." + DESCRIPTIONS["Station ${STATION_ID} RH Sensor"]="Relative humidity is the amount of water vapour in the air reported as a percentage of the amount that would saturate it at the air temperature. Warmer air can hold more water vapour than cooler air." + DESCRIPTIONS["Station ${STATION_ID} Wind Speed Sensor"]="Wind speed is measured using a spinning anemometer mounted on the wind vane at the top of the tower, about 3 m above the ground. We report the average speed for every 15 minute period, in kilometres per hour." + DESCRIPTIONS["Station ${STATION_ID} Gust Speed Sensor"]="Wind gusts are the highest wind speed recorded in each 15 minute period, and are also recorded in kilometres per hour." + + + for OLDNAME in "${!NAMES[@]}"; do + local SENSOR_ID="$(get-sensor-id "${OLDNAME}")" + local NAME="${NAMES[$OLDNAME]}" + local DESCRIPTION="${DESCRIPTIONS[${OLDNAME}]}" + if [ "${SENSOR_ID}" != "null" ]; then + curl -X PATCH -H "Content-Type: application/json" "${STA_URL}/Sensors(${SENSOR_ID})" -d "{ + \"name\": \"${NAME}\", + \"description\": \"${DESCRIPTION}\", + \"metadata\": \"${METADATA_URL}\" + }" + + fi + done +} + + +#STATION_ID="300534061454190" +#STATION_SENSOR_NAME="Dawson Bay Li Taan Aen Staansyoon" +#INSTRUMENT_DETAILS="dawbay-met-sensors" +#update-sensors "${STATION_ID}" "${STATION_SENSOR_NAME}" "${INSTRUMENT_DETAILS}" + +#STATION_ID="300534063017060" +#STATION_SENSOR_NAME="St Laurent Li Taan Aen Staansyoon" +#INSTRUMENT_DETAILS="st-laurent-met-sensors" +#update-sensors "${STATION_ID}" "${STATION_SENSOR_NAME}" "${INSTRUMENT_DETAILS}" + +update-sensors "$1" "$2" "$3" From b7b9fdca9775783f9aff6d6c2cb92fd07631b3ab Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 12:50:23 +0100 Subject: [PATCH 18/35] adjust update-sensors to STA_URL ending with a / --- update-sensors.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/update-sensors.sh b/update-sensors.sh index e7b03ca..017fcef 100755 --- a/update-sensors.sh +++ b/update-sensors.sh @@ -2,12 +2,13 @@ set -ex -#STA_URL="http://localhost:8080/FROST-Server/v1.0" +# STA_URL ends with an slash!!! +#STA_URL="http://localhost:8080/FROST-Server/v1.0/" #INSTRUMENT_DETAILS_BASE_URL="https://canwin-datahub.ad.umanitoba.ca/data/instrument_details" function get-sensor-id() { local NAME="$1" - curl -sG "${STA_URL}/Sensors" --data-urlencode "\$filter=name eq '${NAME}'" | jq -r '.value[0]["@iot.id"]'ss + curl -sG "${STA_URL}Sensors" --data-urlencode "\$filter=name eq '${NAME}'" | jq -r '.value[0]["@iot.id"]' } function update-sensors() { @@ -39,17 +40,19 @@ function update-sensors() { local DESCRIPTION="${DESCRIPTIONS[${OLDNAME}]}" if [ "${SENSOR_ID}" != "null" ]; then if [ -z "${STA_USER}" ] || [ -z "${STA_PASSWORD}" ]; then - curl -X PATCH -H "Content-Type: application/json" \ - "${STA_URL}/Sensors(${SENSOR_ID})" \ + curl -X PATCH \ + -H "Content-Type: application/json" \ + "${STA_URL}Sensors(${SENSOR_ID})" \ -d "{ \"name\": \"${NAME}\", \"description\": \"${DESCRIPTION}\", \"metadata\": \"${METADATA_URL}\" }" else - curl -X PATCH -u "$${STA_USER}:$${STA_PASSWORD}" \ + curl -X PATCH \ + -u "${STA_USER}:${STA_PASSWORD}" \ -H "Content-Type: application/json" \ - "${STA_URL}/Sensors(${SENSOR_ID})" \ + "${STA_URL}Sensors(${SENSOR_ID})" \ -d "{ \"name\": \"${NAME}\", \"description\": \"${DESCRIPTION}\", From fc986d5ae071b1a1832318a62dd30d6386c0beb5 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 12:50:37 +0100 Subject: [PATCH 19/35] add missing sensor names/descriptions --- update-sensors.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/update-sensors.sh b/update-sensors.sh index 017fcef..2674495 100755 --- a/update-sensors.sh +++ b/update-sensors.sh @@ -18,20 +18,25 @@ function update-sensors() { local METADATA_URL="${INSTRUMENT_DETAILS_BASE_URL}/${INSTRUMENT_DETAILS}" declare -A NAMES DESCRIPTIONS - + NAMES["Station ${STATION_ID} Pressure Sensor"]="${STATION_SENSOR_NAME} Barometric Pressure Sensor" NAMES["Station ${STATION_ID} PAR Sensor"]="${STATION_SENSOR_NAME} HOBO PAR Sensor" - NAMES["Station ${STATION_ID} Rain Sensor"]="${STATION_SENSOR_NAME} HOBO Rain Gauge" NAMES["Station ${STATION_ID} Temperature Sensor"]="${STATION_SENSOR_NAME} HOBO Temperature/Relative Humidity Smart Sensor- Temperature Sensor" NAMES["Station ${STATION_ID} RH Sensor"]="${STATION_SENSOR_NAME} HOBO Temperature/Relative Humidity Smart Sensor- Relative Humidity (RH Sensor)" + NAMES["Station ${STATION_ID} Rain Sensor"]="${STATION_SENSOR_NAME} HOBO Rain Gauge" NAMES["Station ${STATION_ID} Wind Speed Sensor"]="${STATION_SENSOR_NAME} R M Young Wind sensor/monitor - Wind Speed Sensor" + NAMES["Station ${STATION_ID} Wind Direction Sensor"]="${STATION_SENSOR_NAME} R M Young Wind sensor/monitor - Wind direction Sensor" NAMES["Station ${STATION_ID} Gust Speed Sensor"]="${STATION_SENSOR_NAME} R M Young Wind sensor/monitor - Gust Speed Sensor" + NAMES["Station ${STATION_ID} Backup Batteries Sensor"]="${STATION_SENSOR_NAME} Backup Batteries Sensor" + DESCRIPTIONS["Station ${STATION_ID} Pressure Sensor"]="Pressure measured using a barometer mounted inside the enclosure that houses the air temperature sensor. It is reported in millibars, where 1000 millibars is the average air pressure at sea level. Changing pressure often indicates a coming change in the weather. Increasing pressure is associated with clearing skies. Decreasing pressure is associated with increasing cloudiness, and possibly an approaching storm" DESCRIPTIONS["Station ${STATION_ID} PAR Sensor"]="Photosynthetically active radiation (PAR) is a measure of light. It is the intensity of the part of sunlight that plants can use to support new growth, and also the wavelengths of light that our eyes are sensitive to. It is measured on a small white disc that records the sum of light falling directly from the sun plus the light scattered by the sky and clouds. It is reported in a unit called a micro-Einstein." - DESCRIPTIONS["Station ${STATION_ID} Rain Sensor"]="Rain is reported in millimetres accumulated every 15 minutes in a gauge set about 1 m above the ground. Rain falling into a 6-inch diameter funnel drips onto one of a pair of “buckets”, mounted on a teeter-totter. When one bucket is filled, the teeter-totter flips, the first bucket is emptied and rain then drips into the other “bucket” until it is filled, and so on. Each flip is equal to 0.2 mm of rain." DESCRIPTIONS["Station ${STATION_ID} Temperature Sensor"]="We report temperature in the shade, in Centigrade degrees. It is measured using an electronic thermometer shielded from direct sunlight and mounted inside an enclosure with louvered walls to allow free air flow past the sensor." DESCRIPTIONS["Station ${STATION_ID} RH Sensor"]="Relative humidity is the amount of water vapour in the air reported as a percentage of the amount that would saturate it at the air temperature. Warmer air can hold more water vapour than cooler air." + DESCRIPTIONS["Station ${STATION_ID} Rain Sensor"]="Rain is reported in millimetres accumulated every 15 minutes in a gauge set about 1 m above the ground. Rain falling into a 6-inch diameter funnel drips onto one of a pair of “buckets”, mounted on a teeter-totter. When one bucket is filled, the teeter-totter flips, the first bucket is emptied and rain then drips into the other “bucket” until it is filled, and so on. Each flip is equal to 0.2 mm of rain." DESCRIPTIONS["Station ${STATION_ID} Wind Speed Sensor"]="Wind speed is measured using a spinning anemometer mounted on the wind vane at the top of the tower, about 3 m above the ground. We report the average speed for every 15 minute period, in kilometres per hour." + DESCRIPTIONS["Station ${STATION_ID} Wind Direction Sensor"]="Wind direction is measured using a wind vane mounted at the top of the tower. As we do with wind speed, we report the average direction for every 15 minutes, in compass degrees from true north. It is then converted to a direction." DESCRIPTIONS["Station ${STATION_ID} Gust Speed Sensor"]="Wind gusts are the highest wind speed recorded in each 15 minute period, and are also recorded in kilometres per hour." + DESCRIPTIONS["Station ${STATION_ID} Backup Batteries Sensor"]="This sensor is measuring the voltage from the 12 volt battery that acts as a backup to the solar panels." for OLDNAME in "${!NAMES[@]}"; do From 1aed23f662402c9aaed9089db57170c70602243a Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 12:50:51 +0100 Subject: [PATCH 20/35] add .env --- .env | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..7817581 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +STA_USER= +STA_PASSWORD= \ No newline at end of file From 5b9be13e77be46f138f38c4b1825eee058a45da8 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 12:51:13 +0100 Subject: [PATCH 21/35] add script to start get/put metadata services sequentially --- start-sequentially.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 start-sequentially.sh diff --git a/start-sequentially.sh b/start-sequentially.sh new file mode 100755 index 0000000..dc000f9 --- /dev/null +++ b/start-sequentially.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Requirements: +# * https://github.com/kislyuk/yq + +set -e + +COMPOSE_FILE="docker-compose.get-and-put-metadata.yaml" + +yq -r '.services | keys | .[]' "${COMPOSE_FILE}" | \ +while read service; do + docker-compose -f "${COMPOSE_FILE}" up "$service" +done \ No newline at end of file From b113824afe76b797ed930abc4920e25d7bcdb974 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 12:57:18 +0100 Subject: [PATCH 22/35] do not print credentials --- update-sensors.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update-sensors.sh b/update-sensors.sh index 2674495..c299f81 100755 --- a/update-sensors.sh +++ b/update-sensors.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -e # STA_URL ends with an slash!!! #STA_URL="http://localhost:8080/FROST-Server/v1.0/" From b8713253bc1777f0db93f2cf976b5a0b035323d8 Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Wed, 15 Feb 2023 12:57:34 +0100 Subject: [PATCH 23/35] guard against empty SENSOR_ID --- update-sensors.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update-sensors.sh b/update-sensors.sh index c299f81..a25b0be 100755 --- a/update-sensors.sh +++ b/update-sensors.sh @@ -43,7 +43,7 @@ function update-sensors() { local SENSOR_ID="$(get-sensor-id "${OLDNAME}")" local NAME="${NAMES[$OLDNAME]}" local DESCRIPTION="${DESCRIPTIONS[${OLDNAME}]}" - if [ "${SENSOR_ID}" != "null" ]; then + if [ -n "${SENSOR_ID}" -a "${SENSOR_ID}" != "null" ]; then if [ -z "${STA_USER}" ] || [ -z "${STA_PASSWORD}" ]; then curl -X PATCH \ -H "Content-Type: application/json" \ From 26d1c29ba6a2f7058f5533ab429aa0ead9f28df9 Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Wed, 15 Feb 2023 17:05:44 +0100 Subject: [PATCH 24/35] Minor changes --- docker-compose.get-and-put-metadata.yaml | 2 +- docker-compose.get-and-put-observations.yaml | 2 +- docker/cron/README.md | 2 +- lib/ontologies/README.markdown | 4 ++-- start-sequentially.sh | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.get-and-put-metadata.yaml b/docker-compose.get-and-put-metadata.yaml index 64a688f..0689ec6 100644 --- a/docker-compose.get-and-put-metadata.yaml +++ b/docker-compose.get-and-put-metadata.yaml @@ -56,7 +56,7 @@ x-defaults: --station_id "$${STATION_ID}" \ --database_url "file://$${DATA_DIR}" \ --overwrite - + # customize the metadata ruby transload set metadata \ --provider data_garrison \ diff --git a/docker-compose.get-and-put-observations.yaml b/docker-compose.get-and-put-observations.yaml index d441589..2129867 100644 --- a/docker-compose.get-and-put-observations.yaml +++ b/docker-compose.get-and-put-observations.yaml @@ -25,7 +25,7 @@ x-defaults: networks: - sta restart: unless-stopped - environment: + environment: # the STA endpoint to load data to STA_URL: http://sta:8080/FROST-Server/v1.0/ # the moving window to download data for (optional, defaults to 2 days) diff --git a/docker/cron/README.md b/docker/cron/README.md index cac5ff3..8fe78b0 100644 --- a/docker/cron/README.md +++ b/docker/cron/README.md @@ -20,7 +20,7 @@ docker run -d -e STA_URL="..." -e SCHEDULE="@hourly" -e MOVING_WINDOW="1 day" \ The build has to be done in the root directory of this repository: -``` +```shell docker build -t data-transloader-job:latest -f docker/cron/Dockerfile . ``` diff --git a/lib/ontologies/README.markdown b/lib/ontologies/README.markdown index a8ad3bc..36aa68c 100644 --- a/lib/ontologies/README.markdown +++ b/lib/ontologies/README.markdown @@ -1,12 +1,12 @@ # Ontologies -The [RDF][] ontology files in this directory provide semantic mappings for metadata from the data providers into a standard vocabulary of entities for SensorThings API. +The [RDF] ontology files in this directory provide semantic mappings for metadata from the data providers into a standard vocabulary of entities for SensorThings API. Observed Properties from a provider are namespaced and matched to a single Unit of Measurement and Observed Property. A class in the Data Transloader Ruby library will provide an API for looking up these mappings. [RDF]: https://en.wikipedia.org/wiki/Resource_Description_Framework -### Example +## Example The source data provider `Data Garrison` provides observations for a sensor with the property `Pressure` and units `mbar`. This maps to the RDF Individual `DataGarrison:Pressure`. Properties are namespaced by their provider in case different providers use the same names. diff --git a/start-sequentially.sh b/start-sequentially.sh index dc000f9..f6a1082 100755 --- a/start-sequentially.sh +++ b/start-sequentially.sh @@ -8,5 +8,5 @@ COMPOSE_FILE="docker-compose.get-and-put-metadata.yaml" yq -r '.services | keys | .[]' "${COMPOSE_FILE}" | \ while read service; do - docker-compose -f "${COMPOSE_FILE}" up "$service" -done \ No newline at end of file + docker compose -f "${COMPOSE_FILE}" up "$service" +done From 7373b1278b1aeb80cb20bbf730f9aa7e500917a8 Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Wed, 15 Feb 2023 17:06:36 +0100 Subject: [PATCH 25/35] Fix for https://github.com/docker/compose-cli/issues/1856 --- docker-compose.get-and-put-metadata.yaml | 4 ++-- docker-compose.get-and-put-observations.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.get-and-put-metadata.yaml b/docker-compose.get-and-put-metadata.yaml index 0689ec6..95c6062 100644 --- a/docker-compose.get-and-put-metadata.yaml +++ b/docker-compose.get-and-put-metadata.yaml @@ -257,5 +257,5 @@ volumes: networks: sta: - external: - name: sta_default + name: uom-frost_default + external: true diff --git a/docker-compose.get-and-put-observations.yaml b/docker-compose.get-and-put-observations.yaml index 2129867..e05586e 100644 --- a/docker-compose.get-and-put-observations.yaml +++ b/docker-compose.get-and-put-observations.yaml @@ -122,5 +122,5 @@ volumes: networks: sta: - external: - name: sta_default + name: uom-frost_default + external: true From be71fb1c04da7b66594dabe9f84d04e58934e439 Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Wed, 15 Feb 2023 17:07:52 +0100 Subject: [PATCH 26/35] Markdown lint --- DOCKER.markdown | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCKER.markdown b/DOCKER.markdown index 093f2c6..03950c8 100644 --- a/DOCKER.markdown +++ b/DOCKER.markdown @@ -7,7 +7,7 @@ There is currently no official Docker image on Docker Hub. Please build a Docker ## Building the Image ```terminal -$ docker build -t data-transloader:latest . +docker build -t data-transloader:latest . ``` ## Running a Container @@ -15,7 +15,7 @@ $ docker build -t data-transloader:latest . To test that the container runs: ```terminal -$ docker run -it --rm data-transloader:latest +docker run -it --rm data-transloader:latest ``` This should print the "help" information for the tool. We use `--rm` to delete the container after running, as it is an ephemeral one-time run. @@ -23,7 +23,7 @@ This should print the "help" information for the tool. We use `--rm` to delete t To run the tool in actual usage, a Docker volume should be used to store the data and metadata. ([A bind mount][binds] to the host filesystem may also be used.) ```terminal -$ docker volume create etl-data +docker volume create etl-data ``` The tool can then be ran with the usual parameters; be sure to mount the volume so that the metadata is re-used. From ca4a8b28f37cb53e519c9bb4caf6bc59a696417e Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Mon, 6 Mar 2023 13:33:13 +0100 Subject: [PATCH 27/35] do not include empty env variables in crontab --- docker/cron/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cron/entrypoint.sh b/docker/cron/entrypoint.sh index 37e1720..ce4b78b 100644 --- a/docker/cron/entrypoint.sh +++ b/docker/cron/entrypoint.sh @@ -12,7 +12,7 @@ for PARAM in "$@"; do done # build the crontab -env | sort > /etc/cron.d/transloader-job +env | sort | grep -Ev '^[^=]+=$' > /etc/cron.d/transloader-job echo >> /etc/cron.d/transloader-job echo "${SCHEDULE:-"@daily"} $(id -un) ${COMMAND} > /proc/\$(cat /var/run/crond.pid)/fd/1 2>/proc/\$(cat /var/run/crond.pid)/fd/2" >> /etc/cron.d/transloader-job From c40e9935c833a941b8b4c8c5c78c6bc7c0eb503a Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Mon, 6 Mar 2023 14:02:42 +0100 Subject: [PATCH 28/35] escape environment variables --- docker-compose.get-and-put-metadata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.get-and-put-metadata.yaml b/docker-compose.get-and-put-metadata.yaml index 64a688f..2a897d1 100644 --- a/docker-compose.get-and-put-metadata.yaml +++ b/docker-compose.get-and-put-metadata.yaml @@ -95,7 +95,7 @@ x-defaults: --value "$${DESCRIPTION}" > /dev/null # push the metadata to the STA - if [ -z "${STA_USER}" ] || [ -z "${STA_PASSWORD}" ]; then + if [ -z "$${STA_USER}" ] || [ -z "$${STA_PASSWORD}" ]; then ruby transload put metadata \ --database_url "file://$${DATA_DIR}" \ --destination "$${STA_URL}" \ @@ -128,7 +128,7 @@ x-defaults: --overwrite # push the metadata to the STA - if [ -z "${STA_USER}" ] || [ -z "${STA_PASSWORD}" ]; then + if [ -z "$${STA_USER}" ] || [ -z "$${STA_PASSWORD}" ]; then ruby transload put metadata \ --provider environment_canada \ --destination "$${STA_URL}" \ From 154a555f0ad39ad1879d6aa772a86f6a5574dffb Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Mon, 6 Mar 2023 14:05:40 +0100 Subject: [PATCH 29/35] update ontology --- lib/ontologies/etl-ontology.ttl | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/ontologies/etl-ontology.ttl b/lib/ontologies/etl-ontology.ttl index 30ebd68..a553cfe 100644 --- a/lib/ontologies/etl-ontology.ttl +++ b/lib/ontologies/etl-ontology.ttl @@ -231,6 +231,12 @@ etl-ontology:DataGarrison:Backup_Batteries rdf:type owl:NamedIndividual , etl-ontology:matchesObservedProperty etl-ontology:Property:Battery_Voltage ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:Volt . +### http://gswlab.ca/ontologies/etl-ontology#DataGarrison:Backup_Batts +etl-ontology:DataGarrison:Backup_Batts rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Battery_Voltage ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:Volt . + ### http://gswlab.ca/ontologies/etl-ontology#DataGarrison:Gust_Speed etl-ontology:DataGarrison:Gust_Speed rdf:type owl:NamedIndividual , @@ -487,10 +493,24 @@ etl-ontology:EnvironmentCanada:avg_wnd_spd_10m_pst2mts rdf:type owl:NamedIndivid ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_wnd_spd_pcpn_gag_mt50-60 etl-ontology:EnvironmentCanada:avg_wnd_spd_pcpn_gag_mt50-60 rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; - etl-ontology:matchesObservedProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:10-minute_average_wind_speed_at_the_height_of_the_precipitation_gauge ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:kilometer_per_hour . +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:max_pk_wnd_tm_pst1hr +etl-ontology:EnvironmentCanada:max_pk_wnd_tm_pst1hr rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Max_wind_speed_past_1_hour_timestamp ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:timestamp . + + +### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:max_wnd_gst_spd_10m_pst10mts +etl-ontology:EnvironmentCanada:max_wnd_gst_spd_10m_pst10mts rdf:type owl:NamedIndividual , + etl-ontology:SourceProperty ; + etl-ontology:matchesObservedProperty etl-ontology:Property:Max_10m_wind_gust_speed_over_past_10_minutes ; + etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:kilometer_per_hour . + + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:avg_wnd_spd_pcpn_gag_pst10mts etl-ontology:EnvironmentCanada:avg_wnd_spd_pcpn_gag_pst10mts rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; @@ -1248,6 +1268,12 @@ etl-ontology:Property:Average_wind_speed_at_precipitation_gauge_over_past_10_min etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity." ; rdfs:label "Average wind speed at precipitation gauge over past 10 minutes" . +### http://gswlab.ca/ontologies/etl-ontology#Property:Max_10m_wind_gust_speed_over_past_10_minutes +etl-ontology:Property:Max_10m_wind_gust_speed_over_past_10_minutes rdf:type owl:NamedIndividual , + etl-ontology:ObservedProperty ; + etl-ontology:definition "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0038/"^^xsd:anyURI ; + etl-ontology:description "Speed is the magnitude of velocity. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. The wind speed is the magnitude of the wind velocity." ; + rdfs:label "Max wind speed over the last 10 minutes of the hour is >= 15 knots AND the max speed exceeds the 2-minute mean wind speed by >= 5 knots" . ### http://gswlab.ca/ontologies/etl-ontology#Property:Battery_Voltage etl-ontology:Property:Battery_Voltage rdf:type owl:NamedIndividual , From b19d999ebde01c70a42976940380baa6d93ee33a Mon Sep 17 00:00:00 2001 From: Christian Autermann Date: Fri, 10 Mar 2023 11:49:58 +0100 Subject: [PATCH 30/35] show more details on data garrison page --- lib/transloader/data_garrison/station.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/transloader/data_garrison/station.rb b/lib/transloader/data_garrison/station.rb index a1e5b86..d5a1fa1 100644 --- a/lib/transloader/data_garrison/station.rb +++ b/lib/transloader/data_garrison/station.rb @@ -40,7 +40,7 @@ def initialize(options = {}) properties = options[:properties] || {} @metadata[:properties].merge!(properties) - @base_path = "https://datagarrison.com/users/#{@user_id}/#{@id}/index.php?sens_details=127&details=7" + @base_path = "https://datagarrison.com/users/#{@user_id}/#{@id}/index.php?sens_details=65535&details=7" end # Download and extract metadata from HTML, use to build metadata From d66625f27daa911ca3bd9840f80a376076aa6193 Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Tue, 18 Apr 2023 14:47:19 +0200 Subject: [PATCH 31/35] Enable fine grained scheduling of data upload jobs --- docker-compose.get-and-put-observations.env | 11 +++++ docker-compose.get-and-put-observations.yaml | 45 ++++++++++++++------ 2 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 docker-compose.get-and-put-observations.env diff --git a/docker-compose.get-and-put-observations.env b/docker-compose.get-and-put-observations.env new file mode 100644 index 0000000..a1cb519 --- /dev/null +++ b/docker-compose.get-and-put-observations.env @@ -0,0 +1,11 @@ +# the STA endpoint to load data to +STA_URL: http://sta:8080/FROST-Server/v1.0/ +# the moving window to download data for (optional, defaults to 2 days) +MOVING_WINDOW: "2 days" +# the schedule of the job (optional, defaults to @daily) +# currently configured in docker-compose.get-and-put-observations.yaml +# SCHEDULE: "34 * * * *" +DATA_DIR: /data +OVERWRITE_METADATA: "false" +STA_USER: ${STA_USER} +STA_PASSWORD: ${STA_PASSWORD} \ No newline at end of file diff --git a/docker-compose.get-and-put-observations.yaml b/docker-compose.get-and-put-observations.yaml index e05586e..6ff9ad7 100644 --- a/docker-compose.get-and-put-observations.yaml +++ b/docker-compose.get-and-put-observations.yaml @@ -9,6 +9,9 @@ # # * transload-data-jobs are cronjobs that fetch new data from the provider and transload # it to the STA. There is one cronjob per station. +# The scheduling of the transload-data-jobs in done manually using the environment +# variable SCHEDULE. The current approach triggers two jobs every 5 minutes for +# keeping the load on the hosting machine lower.# # version: "3" @@ -25,23 +28,15 @@ x-defaults: networks: - sta restart: unless-stopped - environment: - # the STA endpoint to load data to - STA_URL: http://sta:8080/FROST-Server/v1.0/ - # the moving window to download data for (optional, defaults to 2 days) - MOVING_WINDOW: "2 days" - # the schedule of the job (optional, defaults to @daily) - SCHEDULE: "34 * * * *" - DATA_DIR: /data - OVERWRITE_METADATA: "false" - STA_USER: ${STA_USER} - STA_PASSWORD: ${STA_PASSWORD} - + env_file: + - docker-compose.get-and-put-observations.env services: transload-data-data-garrison-MMF1: <<: *cron-job-defaults + environment: + SCHEDULE: "5 * * * *" command: [ --provider, @@ -66,53 +61,79 @@ services: transload-data-environment-canada-CWCF: # Berens, River <<: *cron-job-defaults + environment: + SCHEDULE: "15 * * * *" command: [--provider, environment_canada, --station_id, CWCF] transload-data-environment-canada-CWZE: # Carberry, CS <<: *cron-job-defaults + environment: + SCHEDULE: "15 * * * *" command: [--provider, environment_canada, --station_id, CWZE] transload-data-environment-canada-CWZT: # Dauphin, Airport + environment: + SCHEDULE: "20 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWZT] transload-data-environment-canada-CWSZ: # Fisher, Branch, (auto) + environment: + SCHEDULE: "20 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWSZ] transload-data-environment-canada-CWWS: # George, Island + environment: + SCHEDULE: "25 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWWS] transload-data-environment-canada-CPGH: # Gimli, Climate + environment: + SCHEDULE: "25 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CPGH] transload-data-environment-canada-CWJD: # Grand, Rapids, (auto) + environment: + SCHEDULE: "30 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWJD] transload-data-environment-canada-CWOO: # McCreary + environment: + SCHEDULE: "30 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWOO] transload-data-environment-canada-CWOJ: # Oak, Point, Marine + environment: + SCHEDULE: "35 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWOJ] transload-data-environment-canada-CWPG: # Portage, Southport + environment: + SCHEDULE: "35 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWPG] transload-data-environment-canada-CWEQ: # Swan, River, RCS + environment: + SCHEDULE: "40 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWEQ] transload-data-environment-canada-CWII: # victoria, Beach, (auto) + environment: + SCHEDULE: "40 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWII] transload-data-environment-canada-CWWP: # Wasagaming + environment: + SCHEDULE: "45 * * * *" <<: *cron-job-defaults command: [--provider, environment_canada, --station_id, CWWP] From 430c930ef39102cb3aa927085879ea01dc9378ca Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Tue, 18 Apr 2023 14:55:10 +0200 Subject: [PATCH 32/35] Fix metadata error between MMF stations --- docker-compose.get-and-put-metadata.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker-compose.get-and-put-metadata.yaml b/docker-compose.get-and-put-metadata.yaml index bf2b34e..11210bc 100644 --- a/docker-compose.get-and-put-metadata.yaml +++ b/docker-compose.get-and-put-metadata.yaml @@ -143,7 +143,6 @@ x-defaults: --user "$${STA_USER}:$${STA_PASSWORD}" fi - services: get-metadata-environment-canada-CWCF: # Berens, River <<: *environment-canada-get-metadata-job-defaults @@ -233,8 +232,8 @@ services: LONGITUDE: "-97.96212" TIMEZONE_OFFSET: "-06:00" NAME: St Laurent Li Taan Aen Staansyoon_17060 - STATION_SENSOR_NAME: Dawson Bay Li Taan Aen Staansyoon - INSTRUMENT_DETAILS: dawbay-met-sensors + STATION_SENSOR_NAME: St Laurent Li Taan Aen Staansyoon + INSTRUMENT_DETAILS: st-laurent-met-sensors DESCRIPTION: Manitoba Metis Federation weather station located in St.Laurent get-metadata-data-garrison-MMF2: @@ -247,8 +246,8 @@ services: LONGITUDE: "-100.979630" TIMEZONE_OFFSET: "-06:00" NAME: Dawson Bay Li Taan Aen Staansyoon_54190 - STATION_SENSOR_NAME: St Laurent Li Taan Aen Staansyoon - INSTRUMENT_DETAILS: st-laurent-met-sensors + STATION_SENSOR_NAME: Dawson Bay Li Taan Aen Staansyoon + INSTRUMENT_DETAILS: dawbay-met-sensors DESCRIPTION: Manitoba Metis Federation weather station located in Dawson Bay volumes: From 30df4d194d791b550b7f8679c60fb8061039c0c8 Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Tue, 18 Apr 2023 15:39:21 +0200 Subject: [PATCH 33/35] Updates - fix env file syntax - add .env with STA credentials - add missing schedule for MMF2 --- docker-compose.get-and-put-observations.env | 12 +++++------- docker-compose.get-and-put-observations.yaml | 3 +++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docker-compose.get-and-put-observations.env b/docker-compose.get-and-put-observations.env index a1cb519..379f97c 100644 --- a/docker-compose.get-and-put-observations.env +++ b/docker-compose.get-and-put-observations.env @@ -1,11 +1,9 @@ # the STA endpoint to load data to -STA_URL: http://sta:8080/FROST-Server/v1.0/ +STA_URL=http://sta:8080/FROST-Server/v1.0/ # the moving window to download data for (optional, defaults to 2 days) -MOVING_WINDOW: "2 days" +MOVING_WINDOW="2 days" # the schedule of the job (optional, defaults to @daily) # currently configured in docker-compose.get-and-put-observations.yaml -# SCHEDULE: "34 * * * *" -DATA_DIR: /data -OVERWRITE_METADATA: "false" -STA_USER: ${STA_USER} -STA_PASSWORD: ${STA_PASSWORD} \ No newline at end of file +# SCHEDULE="34 * * * *" +DATA_DIR=/data +OVERWRITE_METADATA="false" diff --git a/docker-compose.get-and-put-observations.yaml b/docker-compose.get-and-put-observations.yaml index 6ff9ad7..5b021b7 100644 --- a/docker-compose.get-and-put-observations.yaml +++ b/docker-compose.get-and-put-observations.yaml @@ -29,6 +29,7 @@ x-defaults: - sta restart: unless-stopped env_file: + - .env - docker-compose.get-and-put-observations.env services: @@ -49,6 +50,8 @@ services: transload-data-data-garrison-MMF2: <<: *cron-job-defaults + environment: + SCHEDULE: "5 * * * *" command: [ --provider, From 4f185929156b71b00d8c853607992b207949ea32 Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Mon, 22 May 2023 14:55:01 +0200 Subject: [PATCH 34/35] Update definitions of observed properties Updated definition of - Data Availability - Derived peak wind remark - Timestamp of maximum wind speed over past hour from http://dd.weather.gc.ca/observations/doc/ to https://dd.weather.gc.ca/observations/doc/SWOB-ML_Product_User_Guide_V8.9_e.pdf --- lib/ontologies/etl-ontology.ttl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/ontologies/etl-ontology.ttl b/lib/ontologies/etl-ontology.ttl index a553cfe..2c923bd 100644 --- a/lib/ontologies/etl-ontology.ttl +++ b/lib/ontologies/etl-ontology.ttl @@ -231,6 +231,7 @@ etl-ontology:DataGarrison:Backup_Batteries rdf:type owl:NamedIndividual , etl-ontology:matchesObservedProperty etl-ontology:Property:Battery_Voltage ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:Volt . + ### http://gswlab.ca/ontologies/etl-ontology#DataGarrison:Backup_Batts etl-ontology:DataGarrison:Backup_Batts rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; @@ -297,6 +298,7 @@ etl-ontology:EnvironmentCanada:air_temp rdf:type owl:NamedIndividual , etl-ontology:matchesObservedProperty etl-ontology:Property:Air_Temperature ; etl-ontology:matchesUnitOfMeasurement etl-ontology:Unit:degree_Celsius . + ### http://gswlab.ca/ontologies/etl-ontology#EnvironmentCanada:air_temp_1 etl-ontology:EnvironmentCanada:air_temp_1 rdf:type owl:NamedIndividual , etl-ontology:SourceProperty ; @@ -1334,7 +1336,7 @@ etl-ontology:Property:Cloud_type_obscuring_phenomena_for_layer_3 rdf:type owl:Na ### http://gswlab.ca/ontologies/etl-ontology#Property:Data_Availability etl-ontology:Property:Data_Availability rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://dd.weather.gc.ca/observations/doc/"^^xsd:anyURI ; + etl-ontology:definition "https://dd.weather.gc.ca/observations/doc/SWOB-ML_Product_User_Guide_V8.9_e.pdf"^^xsd:anyURI ; etl-ontology:description "Data availability"^^xsd:string ; rdfs:label "Data Availability" . @@ -1350,7 +1352,7 @@ etl-ontology:Property:Datalogger_panel_temperature rdf:type owl:NamedIndividual ### http://gswlab.ca/ontologies/etl-ontology#Property:Derived_peak_wind_remark etl-ontology:Property:Derived_peak_wind_remark rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://dd.weather.gc.ca/observations/doc/"^^xsd:anyURI ; + etl-ontology:definition "https://dd.weather.gc.ca/observations/doc/SWOB-ML_Product_User_Guide_V8.9_e.pdf"^^xsd:anyURI ; etl-ontology:description "Derived peak wind remark"^^xsd:string ; rdfs:label "Derived peak wind remark"^^xsd:string . @@ -1414,7 +1416,7 @@ etl-ontology:Property:High_Data_Rate_transmitter_reflected_power rdf:type owl:Na ### http://gswlab.ca/ontologies/etl-ontology#Property:Max_wind_speed_past_1_hour_timestamp etl-ontology:Property:Max_wind_speed_past_1_hour_timestamp rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; - etl-ontology:definition "http://dd.weather.gc.ca/observations/doc/"^^xsd:anyURI ; + etl-ontology:definition "https://dd.weather.gc.ca/observations/doc/SWOB-ML_Product_User_Guide_V8.9_e.pdf"^^xsd:anyURI ; etl-ontology:description "Timestamp of maximum wind speed over past hour." ; rdfs:label "Timestamp of maximum wind speed over past hour" . @@ -1450,6 +1452,7 @@ etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour rdf:type owl:Name etl-ontology:description "Air temperature is the bulk temperature of the air, not the surface (skin) temperature."^^xsd:string ; rdfs:label "Maximum Air Temperature over past 1 hour" . + ### http://gswlab.ca/ontologies/etl-ontology#Property:Maximum_Air_Temperature_over_past_1_hour_1 etl-ontology:Property:Maximum_Air_Temperature_over_past_1_hour_1 rdf:type owl:NamedIndividual , etl-ontology:ObservedProperty ; From ff4ef32965ec2b21209df0c5f5bf07167cc3d83f Mon Sep 17 00:00:00 2001 From: EHJ-52n Date: Sun, 29 Oct 2023 17:31:28 +0100 Subject: [PATCH 35/35] Update README.markdown --- README.markdown | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.markdown b/README.markdown index 084c587..ee616a3 100644 --- a/README.markdown +++ b/README.markdown @@ -1,3 +1,7 @@ +**The latest state of the 52North UoM branch of the data transloader is migrated to https://cwincloud.cc.umanitoba.ca/ccadi/transloader (a private repository!) and will be maintained there.** +**Hence, this public fork of the data-transloader by 52N will be archived.** + + # Arctic Sensor Web Data Transloader [![Build Status](https://travis-ci.org/GeoSensorWebLab/data-transloader.svg?branch=master)](https://travis-ci.org/GeoSensorWebLab/data-transloader)