From 4c8aa09ecfd2cd4b3edc68caaac9a623ae90bd91 Mon Sep 17 00:00:00 2001 From: Robin Gower Date: Wed, 22 Apr 2020 17:26:52 +0200 Subject: [PATCH] Regenerate regional-trade example csvw Adds a script to regenerate the csvw for this example Changes some output filenames to avoid clobbering between pipelines Introduces util/csvw-url function which allows csvw:urls to be specified relatively (i.e. csv file relative to location of json) so the examples don't have absolute paths Minor consistency edits to tests --- examples/rebuild-examples.sh | 45 ++++++++++ .../csvw/component-specifications.csv | 8 +- .../csvw/component-specifications.json | 34 ------- examples/regional-trade/csvw/components.json | 88 +------------------ .../csvw/data-structure-definition.json | 27 ------ examples/regional-trade/csvw/dataset.json | 24 ----- .../regional-trade/csvw/flow-directions.csv | 6 +- .../regional-trade/csvw/flow-directions.json | 64 +------------- .../regional-trade/csvw/observations.json | 68 -------------- .../csvw/regional-trade-metadata.json | 1 + .../regional-trade/csvw/sitc-sections.csv | 24 ++--- .../regional-trade/csvw/sitc-sections.json | 64 +------------- examples/regional-trade/csvw/units.csv | 6 +- examples/regional-trade/csvw/units.json | 42 +-------- .../csvw/used-codes-codelists.json | 27 ------ .../regional-trade/csvw/used-codes-codes.json | 49 ----------- src/table2qb/pipelines/codelist.clj | 10 ++- src/table2qb/pipelines/components.clj | 5 +- src/table2qb/pipelines/cube.clj | 19 ++-- src/table2qb/util.clj | 8 ++ test/table2qb/pipelines/codelist_test.clj | 4 +- test/table2qb/pipelines/cube_test.clj | 83 +++++++++-------- test/table2qb/util_test.clj | 13 +++ 23 files changed, 162 insertions(+), 557 deletions(-) create mode 100755 examples/rebuild-examples.sh delete mode 100644 examples/regional-trade/csvw/component-specifications.json delete mode 100644 examples/regional-trade/csvw/data-structure-definition.json delete mode 100644 examples/regional-trade/csvw/dataset.json delete mode 100644 examples/regional-trade/csvw/observations.json create mode 100644 examples/regional-trade/csvw/regional-trade-metadata.json delete mode 100644 examples/regional-trade/csvw/used-codes-codelists.json delete mode 100644 examples/regional-trade/csvw/used-codes-codes.json create mode 100644 test/table2qb/util_test.clj diff --git a/examples/rebuild-examples.sh b/examples/rebuild-examples.sh new file mode 100755 index 0000000..35b627a --- /dev/null +++ b/examples/rebuild-examples.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +PROJECT_DIR=$(dirname $(dirname "$0")) +JAR="$PROJECT_DIR/target/table2qb.jar" +TABLE2QB="java -jar $JAR" +#TABLE2QB="lein run" + +BASE_URI="http://gss-data.org.uk/" + +EXAMPLE_DIR="$PROJECT_DIR/examples/regional-trade" +CSV_DIR="$EXAMPLE_DIR/csv" +CSVW_DIR="$EXAMPLE_DIR/csvw" + +rm -f "$CSVW_DIR/*" +$TABLE2QB csvw components-pipeline \ + --input-csv "$CSV_DIR/components.csv" \ + --base-uri $BASE_URI \ + --output-directory "$CSVW_DIR" +$TABLE2QB csvw codelist-pipeline \ + --codelist-csv "$CSV_DIR/flow-directions.csv" \ + --codelist-name "Flow Directions Codelist" \ + --codelist-slug "flow-directions" \ + --base-uri $BASE_URI \ + --output-directory "$CSVW_DIR" +$TABLE2QB csvw codelist-pipeline \ + --codelist-csv "$CSV_DIR/sitc-sections.csv" \ + --codelist-name "SITC Sections Codelist" \ + --codelist-slug "sitc-sections" \ + --base-uri $BASE_URI \ + --output-directory "$CSVW_DIR" +$TABLE2QB csvw codelist-pipeline \ + --codelist-csv "$CSV_DIR/units.csv" \ + --codelist-name "Units" \ + --codelist-slug "units" \ + --base-uri $BASE_URI \ + --output-directory "$CSVW_DIR" +$TABLE2QB csvw cube-pipeline \ + --column-config "$EXAMPLE_DIR/columns.csv" \ + --input-csv "$CSV_DIR/input.csv" \ + --dataset-name "Regional Trade" \ + --dataset-slug "regional-trade" \ + --base-uri $BASE_URI \ + --output-directory "$CSVW_DIR" + diff --git a/examples/regional-trade/csvw/component-specifications.csv b/examples/regional-trade/csvw/component-specifications.csv index 573b957..c0893a6 100644 --- a/examples/regional-trade/csvw/component-specifications.csv +++ b/examples/regional-trade/csvw/component-specifications.csv @@ -1,9 +1,9 @@ component_slug,component_attachment,component_property date,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refPeriod +sitc_section,qb:dimension,http://gss-data.org.uk/def/dimension/sitc-section flow,qb:dimension,http://gss-data.org.uk/def/dimension/flow -gbp_total,qb:measure,http://gss-data.org.uk/def/measure/gbp-total -geography,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refArea measure_type,qb:dimension,http://purl.org/linked-data/cube#measureType -net_mass,qb:measure,http://gss-data.org.uk/def/measure/net-mass -sitc_section,qb:dimension,http://gss-data.org.uk/def/dimension/sitc-section +geography,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refArea unit,qb:attribute,http://purl.org/linked-data/sdmx/2009/attribute#unitMeasure +net_mass,qb:measure,http://gss-data.org.uk/def/measure/net-mass +gbp_total,qb:measure,http://gss-data.org.uk/def/measure/gbp-total diff --git a/examples/regional-trade/csvw/component-specifications.json b/examples/regional-trade/csvw/component-specifications.json deleted file mode 100644 index 5a4967a..0000000 --- a/examples/regional-trade/csvw/component-specifications.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "url": "regional-trade.slugged.normalised.csv", - "dc:title": "Regional Trade Component Specifications", - "tableSchema": { - "columns": [{ - "name": "component_slug", - "titles": "component_slug", - "datatype": "string", - "suppressOutput": true - },{ - "name": "component_attachment", - "titles": "component_attachment", - "datatype": "string", - "suppressOutput": true - },{ - "name": "component_property", - "titles": "component_property", - "datatype": "string", - "propertyUrl": "{+component_attachment}", - "valueUrl": "{+component_property}" - },{ - "name": "type", - "virtual": true, - "propertyUrl": "rdf:type", - "valueUrl": "qb:ComponentSpecification" - },{ - "name": "codes_used", - "virtual": true, - "propertyUrl": "http://publishmydata.com/def/qb/codesUsed", - "valueUrl": "http://gss-data.org.uk/data/regional-trade/codes-used/{component_slug}" - }], - "aboutUrl": "http://gss-data.org.uk/data/regional-trade/component/{component_slug}" - } -} diff --git a/examples/regional-trade/csvw/components.json b/examples/regional-trade/csvw/components.json index c80e384..34158e7 100644 --- a/examples/regional-trade/csvw/components.json +++ b/examples/regional-trade/csvw/components.json @@ -1,87 +1 @@ -{ - "@context": [ - "http://www.w3.org/ns/csvw", - { - "@language": "en" - } - ], - "@id": "http://gss-data.org.uk/def/ontology/components", - "url": "components.csv", - "dc:title": "Components Ontology", - "rdfs:label": "Components Ontology", - "rdf:type": { - "@id": "owl:Ontology" - }, - "tableSchema": { - "columns": [ - { - "name": "label", - "titles": "label", - "datatype": "string", - "propertyUrl": "rdfs:label" - }, - { - "name": "description", - "titles": "description", - "datatype": "string", - "propertyUrl": "dc:description" - }, - { - "name": "component_type", - "titles": "component_type", - "propertyUrl": "rdf:type", - "valueUrl": "{+component_type}" - }, - { - "name": "codelist", - "titles": "codelist", - "datatype": "string", - "propertyUrl": "qb:codeList", - "valueUrl": "{+codelist}" - }, - { - "name": "notation", - "titles": "notation", - "datatype": "string", - "propertyUrl": "skos:notation" - }, - { - "name": "component_type_slug", - "titles": "component_type_slug", - "datatype": "string", - "suppressOutput": true - }, - { - "name": "property_slug", - "titles": "property_slug", - "datatype": "string", - "suppressOutput": true - }, - { - "name": "class_slug", - "titles": "class_slug", - "datatype": "string", - "propertyUrl": "rdfs:range", - "valueUrl": "http://gss-data.org.uk/def/{class_slug}" - }, - { - "name": "parent_property", - "titles": "parent_property", - "datatype": "string", - "propertyUrl": "rdfs:subPropertyOf", - "valueUrl": "{+parent_property}" - }, - { - "propertyUrl": "rdfs:isDefinedBy", - "virtual": true, - "valueUrl": "http://gss-data.org.uk/def/ontology/components" - }, - { - "propertyUrl": "rdf:type", - "virtual": true, - "valueUrl": "rdf:Property" - } - ], - "aboutUrl": "http://gss-data.org.uk/def/{component_type_slug}/{notation}" - } -} +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/def\/ontology\/components","url":"components.csv","dc:title":"Components Ontology","rdfs:label":"Components Ontology","rdf:type":{"@id":"owl:Ontology"},"tableSchema":{"columns":[{"name":"label","titles":"label","datatype":"string","propertyUrl":"rdfs:label"},{"name":"description","titles":"description","datatype":"string","propertyUrl":"dc:description"},{"name":"component_type","titles":"component_type","propertyUrl":"rdf:type","valueUrl":"{+component_type}"},{"name":"codelist","titles":"codelist","datatype":"string","propertyUrl":"qb:codeList","valueUrl":"{+codelist}"},{"name":"notation","titles":"notation","datatype":"string","propertyUrl":"skos:notation"},{"name":"component_type_slug","titles":"component_type_slug","datatype":"string","suppressOutput":true},{"name":"property_slug","titles":"property_slug","datatype":"string","suppressOutput":true},{"name":"class_slug","titles":"class_slug","datatype":"string","propertyUrl":"rdfs:range","valueUrl":"http:\/\/gss-data.org.uk\/def\/{class_slug}"},{"name":"parent_property","titles":"parent_property","datatype":"string","propertyUrl":"rdfs:subPropertyOf","valueUrl":"{+parent_property}"},{"propertyUrl":"rdfs:isDefinedBy","virtual":true,"valueUrl":"http:\/\/gss-data.org.uk\/def\/ontology\/components"},{"propertyUrl":"rdf:type","virtual":true,"valueUrl":"rdf:Property"}],"aboutUrl":"http:\/\/gss-data.org.uk\/def\/{component_type_slug}\/{notation}"}} \ No newline at end of file diff --git a/examples/regional-trade/csvw/data-structure-definition.json b/examples/regional-trade/csvw/data-structure-definition.json deleted file mode 100644 index 03488a7..0000000 --- a/examples/regional-trade/csvw/data-structure-definition.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "@id": "http://gss-data.org.uk/data/regional-trade/structure", - "url": "regional-trade.slugged.normalised.csv", - "dc:title": "Regional Trade (Data Structure Definition)", - "rdf:type": {"@id": "qb:DataStructureDefinition"}, - "rdfs:label": "Regional Trade (Data Structure Definition)", - "tableSchema": { - "columns": [{ - "name": "component_slug", - "titles": "component_slug", - "datatype": "string", - "propertyUrl": "qb:component", - "valueUrl": "http://gss-data.org.uk/data/regional-trade/component/{component_slug}" - },{ - "name": "component_attachment", - "titles": "component_attachment", - "datatype": "string", - "suppressOutput": true - },{ - "name": "component_property", - "titles": "component_property", - "datatype": "string", - "suppressOutput": true - }], - "aboutUrl": "http://gss-data.org.uk/data/regional-trade/structure" - } -} diff --git a/examples/regional-trade/csvw/dataset.json b/examples/regional-trade/csvw/dataset.json deleted file mode 100644 index a326146..0000000 --- a/examples/regional-trade/csvw/dataset.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "@id": "http://gss-data.org.uk/data/regional-trade", - "url": "regional-trade.slugged.normalised.csv", - "dc:title": "Regional Trade", - "rdfs:label": "Regional Trade", - "tableSchema": { - "columns": [ - {"titles": "component_slug", "name": "component_slug", "suppressOutput": true}, - {"titles": "component_attachment", "name": "component_attachment", "suppressOutput": true}, - {"titles": "component_property", "name": "component_property", "suppressOutput": true}, - { - "name": "type", - "virtual": true, - "propertyUrl": "rdf:type", - "valueUrl": "qb:DataSet" - },{ - "name": "structure", - "virtual": true, - "propertyUrl": "qb:structure", - "valueUrl": "http://gss-data.org.uk/data/regional-trade/structure" - }], - "aboutUrl": "http://gss-data.org.uk/data/regional-trade" - } -} diff --git a/examples/regional-trade/csvw/flow-directions.csv b/examples/regional-trade/csvw/flow-directions.csv index 7ddb2a2..c733dcf 100644 --- a/examples/regional-trade/csvw/flow-directions.csv +++ b/examples/regional-trade/csvw/flow-directions.csv @@ -1,3 +1,3 @@ -Label,Notation,Parent Notation -Export,export, -Import,import, +label,notation,parent_notation,sort_priority,description,top_concept_of,has_top_concept,pref_label +Export,export,,,,yes,yes,Export +Import,import,,,,yes,yes,Import diff --git a/examples/regional-trade/csvw/flow-directions.json b/examples/regional-trade/csvw/flow-directions.json index decfb64..738f1a7 100644 --- a/examples/regional-trade/csvw/flow-directions.json +++ b/examples/regional-trade/csvw/flow-directions.json @@ -1,63 +1 @@ -{ - "@context": ["http://www.w3.org/ns/csvw", {"@language": "en"}], - "@id": "http://gss-data.org.uk/def/concept-scheme/flow-directions", - "url": "flow-directions-codelist.csv", - "dc:title": "Flow Directions Codelist", - "rdfs:label": "Flow Directions Codelist", - "rdf:type": { "@id": "skos:ConceptScheme" }, - "tableSchema": { - "aboutUrl": "http://gss-data.org.uk/def/concept/flow-directions/{notation}", - "columns": [{ - "name": "label", - "titles": "label", - "datatype": "string", - "propertyUrl": "rdfs:label" - },{ - "name": "notation", - "titles": "notation", - "datatype": "string", - "propertyUrl": "skos:notation" - },{ - "name": "parent_notation", - "titles": "parent_notation", - "datatype": "string", - "propertyUrl": "skos:broader", - "valueUrl": "http://gss-data.org.uk/def/concept/flow-directions/{parent_notation}" - },{ - "name": "sort_priority", - "titles": "sort_priority", - "datatype": "integer", - "propertyUrl": "http://www.w3.org/ns/ui#sortPriority" - },{ - "name": "description", - "titles": "description", - "datatype": "string", - "propertyUrl": "rdfs:comment" - },{ - "name": "top_concept_of", - "titles": "top_concept_of", - "propertyUrl": "skos:topConceptOf", - "valueUrl": "http://gss-data.org.uk/def/concept-scheme/flow-directions" - },{ - "name": "has_top_concept", - "titles" "has_top_concept", - "aboutUrl": "http://gss-data.org.uk/def/concept-scheme/flow-directions", - "propertyUrl": "skos:hasTopConcept", - "valueUrl": "http://gss-data.org.uk/def/concept/flow-directions/{notation}" - }, - { - "name": "pref_label", - "titles": "pref_label", - "propertyUrl": "skos:prefLabel" - }, - { - "propertyUrl": "skos:inScheme", - "valueUrl": "http://gss-data.org.uk/def/concept-scheme/flow-directions", - "virtual": true - },{ - "propertyUrl": "rdf:type", - "valueUrl": "skos:Concept", - "virtual": true - }] - } -} +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/flow-directions","url":"flow-directions.csv","dc:title":"Flow Directions Codelist","rdfs:label":"Flow Directions Codelist","rdf:type":{"@id":"skos:ConceptScheme"},"tableSchema":{"aboutUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{notation}","columns":[{"name":"label","titles":"label","datatype":"string","propertyUrl":"rdfs:label"},{"name":"notation","titles":"notation","datatype":"string","propertyUrl":"skos:notation"},{"name":"parent_notation","titles":"parent_notation","datatype":"string","propertyUrl":"skos:broader","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{parent_notation}"},{"name":"sort_priority","titles":"sort_priority","datatype":"integer","propertyUrl":"http:\/\/www.w3.org\/ns\/ui#sortPriority"},{"name":"description","titles":"description","datatype":"string","propertyUrl":"rdfs:comment"},{"name":"top_concept_of","titles":"top_concept_of","propertyUrl":"skos:topConceptOf","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/flow-directions"},{"name":"has_top_concept","titles":"has_top_concept","aboutUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/flow-directions","propertyUrl":"skos:hasTopConcept","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{notation}"},{"name":"pref_label","titles":"pref_label","propertyUrl":"skos:prefLabel"},{"propertyUrl":"skos:inScheme","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/flow-directions","virtual":true},{"propertyUrl":"rdf:type","valueUrl":"skos:Concept","virtual":true}]}} \ No newline at end of file diff --git a/examples/regional-trade/csvw/observations.json b/examples/regional-trade/csvw/observations.json deleted file mode 100644 index a873ac8..0000000 --- a/examples/regional-trade/csvw/observations.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "url": "observations.csv", - "tableSchema": { - "columns": [ - { - "name": "geography", - "titles": "geography", - "datatype": "string", - "propertyUrl": "http://purl.org/linked-data/sdmx/2009/dimension#refArea", - "valueUrl": "http://statistics.data.gov.uk/id/statistical-geography/{geography}" - }, - { - "name": "date", - "titles": "date", - "datatype": "string", - "propertyUrl": "http://purl.org/linked-data/sdmx/2009/dimension#refPeriod", - "valueUrl": "http://reference.data.gov.uk/id/year/{date}" - }, - { - "name": "sitc_section", - "titles": "sitc_section", - "datatype": "string", - "propertyUrl": "http://gss-data.org.uk/def/dimension/sitc-section", - "valueUrl": "http://gss-data.org.uk/def/concept/sitc-sections/{sitc_section}" - }, - { - "name": "flow", - "titles": "flow", - "datatype": "string", - "propertyUrl": "http://gss-data.org.uk/def/dimension/flow", - "valueUrl": "http://gss-data.org.uk/def/concept/flow-directions/{flow}" - }, - { - "name": "measure_type", - "titles": "measure_type", - "datatype": "string", - "propertyUrl": "http://purl.org/linked-data/cube#measureType", - "valueUrl": "http://gss-data.org.uk/def/measure/{measure_type}" - }, - { - "name": "unit", - "titles": "unit", - "datatype": "string", - "propertyUrl": "http://purl.org/linked-data/sdmx/2009/attribute#unitMeasure", - "valueUrl": "http://gss-data.org.uk/def/concept/measurement-units/{unit}" - }, - { - "name": "value", - "titles": "value", - "datatype": "number", - "propertyUrl": "http://gss-data.org.uk/def/measure/{measure_type}" - }, - { - "name": "Observation", - "virtual": true, - "propertyUrl": "rdf:type", - "valueUrl": "qb:Observation" - }, - { - "name": "DataSet", - "virtual": true, - "propertyUrl": "qb:dataSet", - "valueUrl": "http://gss-data.org.uk/data/regional-trade" - } - ], - "aboutUrl": "http://gss-data.org.uk/data/regional-trade/{+geography}/{+date}/{+sitc_section}/{+flow}/{+measure_type}" - } -} diff --git a/examples/regional-trade/csvw/regional-trade-metadata.json b/examples/regional-trade/csvw/regional-trade-metadata.json new file mode 100644 index 0000000..44eed5d --- /dev/null +++ b/examples/regional-trade/csvw/regional-trade-metadata.json @@ -0,0 +1 @@ +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"tables":[{"@id":"http:\/\/gss-data.org.uk\/data\/regional-trade","url":"component-specifications.csv","dc:title":"Regional Trade","rdfs:label":"Regional Trade","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","suppressOutput":true},{"name":"component_property","titles":"component_property","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"},{"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/structure"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade"}},{"@id":"http:\/\/gss-data.org.uk\/data\/regional-trade\/structure","url":"component-specifications.csv","dc:title":"Regional Trade (Data Structure Definition)","rdf:type":{"@id":"qb:DataStructureDefinition"},"rdfs:label":"Regional Trade (Data Structure Definition)","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/component\/{component_slug}"},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/structure"}},{"url":"component-specifications.csv","dc:title":"Regional Trade","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"},{"name":"codes_used","virtual":true,"propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/codes-used\/{component_slug}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/component\/{component_slug}"}},{"url":"component-specifications.csv","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"skos:Collection"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/codes-used\/{component_slug}"}},{"url":"observations.csv","tableSchema":{"columns":[{"name":"geography","titles":"geography","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/statistics.data.gov.uk\/id\/statistical-geography\/{geography}"},{"name":"date","titles":"date","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/reference.data.gov.uk\/id\/year\/{date}"},{"name":"sitc_section","titles":"sitc_section","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/sitc-sections\/{sitc_section}"},{"name":"flow","titles":"flow","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{flow}"},{"name":"measure_type","titles":"measure_type","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"unit","titles":"unit","datatype":"string","propertyUrl":"skos:member","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/measurement-units\/{unit}"},{"name":"value","titles":"value","datatype":"number","propertyUrl":"skos:member","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/codes-used\/{_name}"}},{"url":"observations.csv","tableSchema":{"columns":[{"name":"geography","titles":"geography","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/dimension#refArea","valueUrl":"http:\/\/statistics.data.gov.uk\/id\/statistical-geography\/{geography}"},{"name":"date","titles":"date","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/dimension#refPeriod","valueUrl":"http:\/\/reference.data.gov.uk\/id\/year\/{date}"},{"name":"sitc_section","titles":"sitc_section","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/dimension\/sitc-section","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/sitc-sections\/{sitc_section}"},{"name":"flow","titles":"flow","datatype":"string","propertyUrl":"http:\/\/gss-data.org.uk\/def\/dimension\/flow","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/flow-directions\/{flow}"},{"name":"measure_type","titles":"measure_type","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/cube#measureType","valueUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"unit","titles":"unit","datatype":"string","propertyUrl":"http:\/\/purl.org\/linked-data\/sdmx\/2009\/attribute#unitMeasure","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/measurement-units\/{unit}"},{"name":"value","titles":"value","datatype":"number","propertyUrl":"http:\/\/gss-data.org.uk\/def\/measure\/{measure_type}"},{"name":"Observation","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:Observation"},{"name":"DataSet","virtual":true,"propertyUrl":"qb:dataSet","valueUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/regional-trade\/{+geography}\/{+date}\/{+sitc_section}\/{+flow}\/{+measure_type}"}}]} \ No newline at end of file diff --git a/examples/regional-trade/csvw/sitc-sections.csv b/examples/regional-trade/csvw/sitc-sections.csv index 0e69ef4..74f1f3e 100644 --- a/examples/regional-trade/csvw/sitc-sections.csv +++ b/examples/regional-trade/csvw/sitc-sections.csv @@ -1,12 +1,12 @@ -Label,Notation,Parent Notation -0 Food and Live Animals,0-food-and-live-animals,total -1 Beverages and Tobacco,1-beverages-and-tobacco,total -2 Crude Materials,2-crude-materials,total -3 Mineral Fuels,3-mineral-fuels,total -4 Animal and Vegetable Oils,4-animal-and-vegetable-oils,total -5 Chemicals,5-chemicals,total -6 Manufactured Goods,6-manufactured-goods,total -7 Machinery and Transport,7-machinery-and-transport,total -8 Miscellaneous Manufactures,8-miscellaneous-manufactures,total -9 Other commodities nes,9-other-commodities-nes,total -Total,total, +label,notation,parent_notation,sort_priority,description,top_concept_of,has_top_concept,pref_label +0 Food and Live Animals,0-food-and-live-animals,total,0,lorem ipsum,,,0 Food and Live Animals +1 Beverages and Tobacco,1-beverages-and-tobacco,total,1,lorem ipsum,,,1 Beverages and Tobacco +2 Crude Materials,2-crude-materials,total,2,lorem ipsum,,,2 Crude Materials +3 Mineral Fuels,3-mineral-fuels,total,3,lorem ipsum,,,3 Mineral Fuels +4 Animal and Vegetable Oils,4-animal-and-vegetable-oils,total,4,lorem ipsum,,,4 Animal and Vegetable Oils +5 Chemicals,5-chemicals,total,5,lorem ipsum,,,5 Chemicals +6 Manufactured Goods,6-manufactured-goods,total,6,lorem ipsum,,,6 Manufactured Goods +7 Machinery and Transport,7-machinery-and-transport,total,7,lorem ipsum,,,7 Machinery and Transport +8 Miscellaneous Manufactures,8-miscellaneous-manufactures,total,8,lorem ipsum,,,8 Miscellaneous Manufactures +9 Other commodities nes,9-other-commodities-nes,total,9,lorem ipsum,,,9 Other commodities nes +Total,total,,10,lorem ipsum,yes,yes,Total diff --git a/examples/regional-trade/csvw/sitc-sections.json b/examples/regional-trade/csvw/sitc-sections.json index da306de..22f9eb8 100644 --- a/examples/regional-trade/csvw/sitc-sections.json +++ b/examples/regional-trade/csvw/sitc-sections.json @@ -1,63 +1 @@ -{ - "@context": ["http://www.w3.org/ns/csvw", {"@language": "en"}], - "@id": "http://gss-data.org.uk/def/concept-scheme/sitc-sections", - "url": "sitc-sections-codelist.csv", - "dc:title": "SITC Sections Codelist", - "rdfs:label": "SITC Sections Codelist", - "rdf:type": { "@id": "skos:ConceptScheme" }, - "tableSchema": { - "aboutUrl": "http://gss-data.org.uk/def/concept/sitc-sections/{notation}", - "columns": [{ - "name": "label", - "titles": "label", - "datatype": "string", - "propertyUrl": "rdfs:label" - },{ - "name": "notation", - "titles": "notation", - "datatype": "string", - "propertyUrl": "skos:notation" - },{ - "name": "parent_notation", - "titles": "parent_notation", - "datatype": "string", - "propertyUrl": "skos:broader", - "valueUrl": "http://gss-data.org.uk/def/concept/sitc-sections/{parent_notation}" - },{ - "name": "sort_priority", - "titles": "sort_priority", - "datatype": "integer", - "propertyUrl": "http://www.w3.org/ns/ui#sortPriority" - },{ - "name": "description", - "titles": "description", - "datatype": "string", - "propertyUrl": "rdfs:comment" - },{ - "name": "top_concept_of", - "titles": "top_concept_of", - "propertyUrl": "skos:topConceptOf", - "valueUrl": "http://gss-data.org.uk/def/concept-scheme/sitc-sections" - },{ - "name": "has_top_concept", - "titles": "has_top_concept", - "aboutUrl": "http://gss-data.org.uk/def/concept-scheme/sitc-sections", - "propertyUrl": "skos:hasTopConcept", - "valueUrl": "http://gss-data.org.uk/def/concept/sitc-sections/{notation}" - }, - { - "name": "pref_label", - "titles": "pref_label", - "propertyUrl": "skos:prefLabel" - }, - { - "propertyUrl": "skos:inScheme", - "valueUrl": "http://gss-data.org.uk/def/concept-scheme/sitc-sections", - "virtual": true - },{ - "propertyUrl": "rdf:type", - "valueUrl": "skos:Concept", - "virtual": true - }] - } -} +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/sitc-sections","url":"sitc-sections.csv","dc:title":"SITC Sections Codelist","rdfs:label":"SITC Sections Codelist","rdf:type":{"@id":"skos:ConceptScheme"},"tableSchema":{"aboutUrl":"http:\/\/gss-data.org.uk\/def\/concept\/sitc-sections\/{notation}","columns":[{"name":"label","titles":"label","datatype":"string","propertyUrl":"rdfs:label"},{"name":"notation","titles":"notation","datatype":"string","propertyUrl":"skos:notation"},{"name":"parent_notation","titles":"parent_notation","datatype":"string","propertyUrl":"skos:broader","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/sitc-sections\/{parent_notation}"},{"name":"sort_priority","titles":"sort_priority","datatype":"integer","propertyUrl":"http:\/\/www.w3.org\/ns\/ui#sortPriority"},{"name":"description","titles":"description","datatype":"string","propertyUrl":"rdfs:comment"},{"name":"top_concept_of","titles":"top_concept_of","propertyUrl":"skos:topConceptOf","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/sitc-sections"},{"name":"has_top_concept","titles":"has_top_concept","aboutUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/sitc-sections","propertyUrl":"skos:hasTopConcept","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/sitc-sections\/{notation}"},{"name":"pref_label","titles":"pref_label","propertyUrl":"skos:prefLabel"},{"propertyUrl":"skos:inScheme","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/sitc-sections","virtual":true},{"propertyUrl":"rdf:type","valueUrl":"skos:Concept","virtual":true}]}} \ No newline at end of file diff --git a/examples/regional-trade/csvw/units.csv b/examples/regional-trade/csvw/units.csv index c975722..f6f5c09 100644 --- a/examples/regional-trade/csvw/units.csv +++ b/examples/regional-trade/csvw/units.csv @@ -1,3 +1,3 @@ -Label,Notation,Parent Notation -£ Million,gbp-million, -Tonnes,tonnes, +label,notation,parent_notation,sort_priority,description,top_concept_of,has_top_concept,pref_label +£ Million,gbp-million,,,,yes,yes,£ Million +Tonnes,tonnes,,,,yes,yes,Tonnes diff --git a/examples/regional-trade/csvw/units.json b/examples/regional-trade/csvw/units.json index 0df7680..2f3a94b 100644 --- a/examples/regional-trade/csvw/units.json +++ b/examples/regional-trade/csvw/units.json @@ -1,41 +1 @@ -{ - "@context": ["http://www.w3.org/ns/csvw", {"@language": "en"}], - "@id": "http://gss-data.org.uk/def/concept-scheme/units", - "url": "units-codelist.csv", - "dc:title": "Units Codelist", - "rdfs:label": "Units Codelist", - "rdf:type": { "@id": "skos:ConceptScheme" }, - "tableSchema": { - "aboutUrl": "http://gss-data.org.uk/def/concept/{notation}", - "columns": [{ - "name": "label", - "titles": "Label", - "datatype": "string", - "propertyUrl": "rdfs:label" - },{ - "name": "notation", - "titles": "Notation", - "datatype": "string", - "propertyUrl": "skos:notation" - },{ - "name": "parent_notation", - "titles": "Parent Notation", - "datatype": "string", - "propertyUrl": "skos:broader", - "valueUrl": "http://gss-data.org.uk/def/concept/{parent_notation}" - },{ - "propertyUrl": "skos:inScheme", - "valueUrl": "http://gss-data.org.uk/def/concept-scheme/units", - "virtual": true - },{ - "propertyUrl": "skos:topConceptOf", - "valueUrl": "http://gss-data.org.uk/def/concept-scheme/units", - "how-to-make-this-only-apply-when-parent-is-null?": "perhaps reasoning?", - "virtual": true - },{ - "propertyUrl": "skos:prefLabel", - "value": "{label}", - "virtual": true - }] - } -} +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/units","url":"units.csv","dc:title":"Units","rdfs:label":"Units","rdf:type":{"@id":"skos:ConceptScheme"},"tableSchema":{"aboutUrl":"http:\/\/gss-data.org.uk\/def\/concept\/units\/{notation}","columns":[{"name":"label","titles":"label","datatype":"string","propertyUrl":"rdfs:label"},{"name":"notation","titles":"notation","datatype":"string","propertyUrl":"skos:notation"},{"name":"parent_notation","titles":"parent_notation","datatype":"string","propertyUrl":"skos:broader","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/units\/{parent_notation}"},{"name":"sort_priority","titles":"sort_priority","datatype":"integer","propertyUrl":"http:\/\/www.w3.org\/ns\/ui#sortPriority"},{"name":"description","titles":"description","datatype":"string","propertyUrl":"rdfs:comment"},{"name":"top_concept_of","titles":"top_concept_of","propertyUrl":"skos:topConceptOf","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/units"},{"name":"has_top_concept","titles":"has_top_concept","aboutUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/units","propertyUrl":"skos:hasTopConcept","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept\/units\/{notation}"},{"name":"pref_label","titles":"pref_label","propertyUrl":"skos:prefLabel"},{"propertyUrl":"skos:inScheme","valueUrl":"http:\/\/gss-data.org.uk\/def\/concept-scheme\/units","virtual":true},{"propertyUrl":"rdf:type","valueUrl":"skos:Concept","virtual":true}]}} \ No newline at end of file diff --git a/examples/regional-trade/csvw/used-codes-codelists.json b/examples/regional-trade/csvw/used-codes-codelists.json deleted file mode 100644 index c841219..0000000 --- a/examples/regional-trade/csvw/used-codes-codelists.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "url": "regional-trade.slugged.normalised.csv", - "tableSchema": { - "columns": [{ - "name": "component_slug", - "titles": "component_slug", - "datatype": "string", - "suppressOutput": true - },{ - "name": "component_attachment", - "titles": "component_attachment", - "datatype": "string", - "suppressOutput": true - },{ - "name": "component_property", - "titles": "component_property", - "datatype": "string", - "suppressOutput": true - },{ - "name": "type", - "virtual": true, - "propertyUrl": "rdf:type", - "valueUrl": "skos:Collection" - }], - "aboutUrl": "http://gss-data.org.uk/data/regional-trade/codes-used/{component_slug}" - } -} diff --git a/examples/regional-trade/csvw/used-codes-codes.json b/examples/regional-trade/csvw/used-codes-codes.json deleted file mode 100644 index 2d2a313..0000000 --- a/examples/regional-trade/csvw/used-codes-codes.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "url": "regional-trade.slugged.csv", - "tableSchema": { - "columns": [{ - "name": "geography", - "titles": "geography", - "datatype": "string", - "propertyUrl": "skos:member", - "valueUrl": "http://statistics.data.gov.uk/id/statistical-geography/{geography}" - },{ - "name": "date", - "titles": "date", - "datatype": "string", - "propertyUrl": "skos:member", - "valueUrl": "http://reference.data.gov.uk/id/year/{date}" - },{ - "name": "sitc_section", - "titles": "sitc_section", - "datatype": "string", - "propertyUrl": "skos:member", - "valueUrl": "http://gss-data.org.uk/def/concept/sitc-sections/{sitc_section}" - },{ - "name": "flow", - "titles": "flow", - "datatype": "string", - "propertyUrl": "skos:member", - "valueUrl": "http://gss-data.org.uk/def/concept/flow-directions/{flow}" - },{ - "name": "measure_type", - "titles": "measure_type", - "datatype": "string", - "propertyUrl": "skos:member", - "valueUrl": "http://gss-data.org.uk/def/measure/{measure_type}" - },{ - "name": "unit", - "titles": "unit", - "datatype": "string", - "propertyUrl": "skos:member", - "valueUrl": "http://gss-data.org.uk/def/concept/measurement-units/{unit}" - },{ - "name": "value", - "titles": "value", - "datatype": "number", - "propertyUrl": "skos:member", - "suppressOutput": true - }], - "aboutUrl": "http://gss-data.org.uk/data/regional-trade/codes-used/{_name}" - } -} diff --git a/src/table2qb/pipelines/codelist.clj b/src/table2qb/pipelines/codelist.clj index fe398ca..bb4b438 100644 --- a/src/table2qb/pipelines/codelist.clj +++ b/src/table2qb/pipelines/codelist.clj @@ -118,11 +118,13 @@ (defn codelist-pipeline "Generates a codelist from a CSV file describing its members" [output-directory {:keys [codelist-csv codelist-name codelist-slug base-uri]}] - (let [metadata-file (io/file output-directory "metadata.json") + (let [metadata-file (io/file output-directory (str codelist-slug ".json")) domain-def (uri-config/domain-def base-uri) - output-csv (io/file output-directory "codelist.csv") - metadata (codelist-schema (.toURI output-csv) domain-def codelist-name codelist-slug)] - (codelist->csvw codelist-csv output-csv) + output-filename (str codelist-slug ".csv") + output-file (io/file output-directory output-filename) + csv-url (util/csvw-url output-directory output-filename) + metadata (codelist-schema csv-url domain-def codelist-name codelist-slug)] + (codelist->csvw codelist-csv output-file) (util/write-json-file metadata-file metadata) {:metadata-file metadata-file})) diff --git a/src/table2qb/pipelines/components.clj b/src/table2qb/pipelines/components.clj index 92679da..b6ce79c 100644 --- a/src/table2qb/pipelines/components.clj +++ b/src/table2qb/pipelines/components.clj @@ -115,10 +115,11 @@ "Generates component specifications." [output-dir {:keys [input-csv base-uri]}] (let [components-csv (io/file output-dir "components.csv") - metadata-file (io/file output-dir "metadata.json") + csv-url (util/csvw-url output-dir "components.csv") + metadata-file (io/file output-dir "components.json") domain-def (uri-config/domain-def base-uri)] (components->csvw input-csv components-csv) - (util/write-json-file metadata-file (components-schema (.toURI components-csv) domain-def)) + (util/write-json-file metadata-file (components-schema csv-url domain-def)) {:metadata-file metadata-file})) (defmethod ig/init-key :table2qb.pipelines.components/components-pipeline [_ opts] diff --git a/src/table2qb/pipelines/cube.clj b/src/table2qb/pipelines/cube.clj index 7112028..5307b7e 100644 --- a/src/table2qb/pipelines/cube.clj +++ b/src/table2qb/pipelines/cube.clj @@ -198,10 +198,11 @@ [output-directory {:keys [input-csv dataset-name dataset-slug column-config base-uri]}] (let [cube-config (cube-config/get-cube-configuration input-csv column-config) domain-data (uri-config/domain-data base-uri) - metadata-file (io/file output-directory "metadata.json") - + metadata-file (io/file output-directory (str dataset-slug "-metadata.json")) component-specifications-csv (io/file output-directory "component-specifications.csv") - observations-csv (io/file output-directory "observations.csv")] + component-specifications-url (util/csvw-url output-directory "component-specifications.csv") + observations-csv (io/file output-directory "observations.csv") + observations-url (util/csvw-url output-directory "observations.csv")] ;;write csv files (components->csvw component-specifications-csv cube-config) (observations->csvw input-csv observations-csv cube-config) @@ -209,12 +210,12 @@ (util/write-json-file metadata-file {"@context" ["http://www.w3.org/ns/csvw" {"@language" "en"}] - "tables" [(dataset-schema (.toURI component-specifications-csv) domain-data dataset-name dataset-slug) - (data-structure-definition-schema (.toURI component-specifications-csv) domain-data dataset-name dataset-slug) - (component-specification-schema (.toURI component-specifications-csv) domain-data dataset-name dataset-slug) - (used-codes-codelists-schema (.toURI component-specifications-csv) domain-data dataset-slug) - (used-codes-codes-schema (.toURI observations-csv) domain-data dataset-slug cube-config) - (observations-schema (.toURI observations-csv) domain-data dataset-slug cube-config)]}) + "tables" [(dataset-schema component-specifications-url domain-data dataset-name dataset-slug) + (data-structure-definition-schema component-specifications-url domain-data dataset-name dataset-slug) + (component-specification-schema component-specifications-url domain-data dataset-name dataset-slug) + (used-codes-codelists-schema component-specifications-url domain-data dataset-slug) + (used-codes-codes-schema observations-url domain-data dataset-slug cube-config) + (observations-schema observations-url domain-data dataset-slug cube-config)]}) {:metadata-file metadata-file})) diff --git a/src/table2qb/util.clj b/src/table2qb/util.clj index b899a74..cf88d5c 100644 --- a/src/table2qb/util.clj +++ b/src/table2qb/util.clj @@ -21,6 +21,14 @@ (defn map-by [f s] (into {} (map (fn [v] [(f v) v]) s))) +(defn csvw-url [output-directory output-filename] + "Returns a URI for the csv file that is specified relatively or absolutely + depending on how the output directory is specified. Absolute URIs include + the 'file:/' scheme part." + (if (.isAbsolute (io/file output-directory)) + (.toURI (io/file output-directory output-filename)) + (java.net.URI. output-filename))) + (defn csv-file->metadata-uri [csv-file-str] (let [meta-file (str csv-file-str "meta.json")] (URI. meta-file))) diff --git a/test/table2qb/pipelines/codelist_test.clj b/test/table2qb/pipelines/codelist_test.clj index 11c87c6..9d3ccac 100644 --- a/test/table2qb/pipelines/codelist_test.clj +++ b/test/table2qb/pipelines/codelist_test.clj @@ -23,7 +23,7 @@ (testing "json metadata" (maps-match? (util/read-json (example-csvw "regional-trade" "flow-directions.json")) (codelist-schema - "flow-directions-codelist.csv" + "flow-directions.csv" test-domain-def "Flow Directions Codelist" "flow-directions")))) @@ -41,7 +41,7 @@ (testing "json metadata" (maps-match? (util/read-json (example-csvw "regional-trade" "sitc-sections.json")) (codelist-schema - "sitc-sections-codelist.csv" + "sitc-sections.csv" test-domain-def "SITC Sections Codelist" "sitc-sections")))) diff --git a/test/table2qb/pipelines/cube_test.clj b/test/table2qb/pipelines/cube_test.clj index f1eae99..5e7265e 100644 --- a/test/table2qb/pipelines/cube_test.clj +++ b/test/table2qb/pipelines/cube_test.clj @@ -45,7 +45,20 @@ dimension-names ["year" "flow" "code"]] (is (= "http://example.com/data/test/{+year}/{+flow}/{+code}" (observation-template domain-data-prefix dataset-slug dimension-names))))) -(deftest read-component-specifications-test +(defn- example-csvw-schema-part [example-directory part-name] + "Extracts a table schema from the json metadata that covers the whole cube" + (let [metadata-file (example-csvw example-directory (str example-directory "-metadata.json")) + schema (util/read-json metadata-file) + part-number (case part-name + "dataset" 0 + "data-structure-defintion" 1 + "component-specifications" 2 + "used-codes-codelists" 3 + "used-codes-codes" 4 + "observations" 5)] + (-> schema (get "tables") (nth part-number)))) + +(deftest component-specification-records-test (testing "returns a dataset of component-specifications" (let [cube-config (cube-config/get-cube-configuration (example-csv "regional-trade" "input.csv") default-config) component-specifications (component-specification-records cube-config)] @@ -63,41 +76,41 @@ (set component-specifications))))) (testing "serialised contents match" (let [string-writer (StringWriter.)] - (csv/write-csv string-writer (sort-by :component_slug component-specifications)) + (csv/write-csv string-writer component-specifications) (is (= (slurp (example-csvw "regional-trade" "component-specifications.csv")) (str string-writer)))))) (testing "compare with component-specifications.json" (testing "parsed contents match" - (maps-match? (util/read-json (example-csvw "regional-trade" "component-specifications.json")) + (maps-match? (example-csvw-schema-part "regional-trade" "component-specifications") (component-specification-schema - "regional-trade.slugged.normalised.csv" - test-domain-data - "Regional Trade Component Specifications" - "regional-trade")))))) + "component-specifications.csv" + test-domain-data + "Regional Trade" + "regional-trade")))))) (testing "name is optional" (let [metadata (component-specification-schema "components.csv" test-domain-data "" "ds-slug")] (is (= nil (get metadata "dc:title")))))) (deftest dataset-test (testing "compare with dataset.json" - (maps-match? (util/read-json (example-csvw "regional-trade" "dataset.json")) + (maps-match? (example-csvw-schema-part "regional-trade" "dataset") (dataset-schema - "regional-trade.slugged.normalised.csv" - test-domain-data - "Regional Trade" - "regional-trade"))) - (testing "name is optional" - (let [metadata (dataset-schema "components.csv" test-domain-data "" "ds-slug")] - (is (= nil (get metadata "rdfs:label")))))) + "component-specifications.csv" + test-domain-data + "Regional Trade" + "regional-trade"))) + (testing "name is optional" + (let [metadata (dataset-schema "components.csv" test-domain-data "" "ds-slug")] + (is (= nil (get metadata "rdfs:label")))))) (deftest data-structure-definition-test (testing "compare with data-structure-definition.json" - (maps-match? (util/read-json (example-csvw "regional-trade" "data-structure-definition.json")) + (maps-match? (example-csvw-schema-part "regional-trade" "data-structure-defintion") (data-structure-definition-schema - "regional-trade.slugged.normalised.csv" - test-domain-data - "Regional Trade" - "regional-trade"))) + "component-specifications.csv" + test-domain-data + "Regional Trade" + "regional-trade"))) (testing "name is optional" (let [metadata (data-structure-definition-schema "components.csv" test-domain-data "" "ds-slug")] (is (= nil (get metadata "rdfs:label")))))) @@ -119,7 +132,7 @@ (is (= 7 (count observation)))) (testing "slugged columns" (are [expected actual] (= expected actual) - "gbp-total" (:measure_type observation) + "gbp-total" (:measure_type observation) "gbp-million" (:unit observation) "0-food-and-live-animals" (:sitc_section observation) "export" (:flow observation))))) @@ -132,7 +145,7 @@ (is (= 7 (count observation)))) (testing "slugged columns" (are [expected actual] (= expected actual) - "gbp-total" (:measure_type observation) + "gbp-total" (:measure_type observation) "gbp-million" (:unit observation) "cn#cn8_28399000" (:combined_nomenclature observation) "export" (:flow observation)))))) @@ -141,10 +154,10 @@ (let [obs-source (example-csv "regional-trade" "input.csv") cube-config (cube-config/get-cube-configuration obs-source default-config) obs-meta (observations-schema "observations.csv" - test-domain-data - "regional-trade" - cube-config)] - (maps-match? (util/read-json (example-csvw "regional-trade" "observations.json")) + test-domain-data + "regional-trade" + cube-config)] + (maps-match? (example-csvw-schema-part "regional-trade" "observations") obs-meta))) (testing "overseas trade example" (let [obs-source (example-csv "overseas-trade" "ots-cn-sample.csv") @@ -154,19 +167,19 @@ (deftest used-codes-test (testing "codelists metadata" - (maps-match? (util/read-json (example-csvw "regional-trade" "used-codes-codelists.json")) - (used-codes-codelists-schema "regional-trade.slugged.normalised.csv" - test-domain-data - "regional-trade"))) + (maps-match? (example-csvw-schema-part "regional-trade" "used-codes-codelists") + (used-codes-codelists-schema "component-specifications.csv" + test-domain-data + "regional-trade"))) (testing "codes metadata" (let [obs-source (example-csv "regional-trade" "input.csv") cube-config (cube-config/get-cube-configuration obs-source default-config) - expected-json (util/read-json (example-csvw "regional-trade" "used-codes-codes.json"))] + expected-json (example-csvw-schema-part "regional-trade" "used-codes-codes")] (maps-match? expected-json - (used-codes-codes-schema "regional-trade.slugged.csv" - test-domain-data - "regional-trade" - cube-config))))) + (used-codes-codes-schema "observations.csv" + test-domain-data + "regional-trade" + cube-config))))) (deftest cube-pipeline-test (let [dataset-name "Regional Trade" diff --git a/test/table2qb/util_test.clj b/test/table2qb/util_test.clj new file mode 100644 index 0000000..57065f2 --- /dev/null +++ b/test/table2qb/util_test.clj @@ -0,0 +1,13 @@ +(ns table2qb.util-test + (:require [table2qb.util :refer :all] + [clojure.test :refer :all] + [clojure.java.io :as io])) + +(deftest csvw-url-test + (testing "specifies a path to the csv") + (testing "as an absolute path when directory is specified absolutely" + (is (= "file:/absolute/file.csv" + (str (csvw-url "/absolute/" "file.csv"))))) + (testing "relative to the json when the output directory is specified relatively" + (is (= "file.csv" + (str (csvw-url "./relative/" "file.csv"))))))