diff --git a/examples/bop-quarterly/csvw/component-specifications.json b/examples/bop-quarterly/csvw/component-specifications.json index 504daff..0350562 100644 --- a/examples/bop-quarterly/csvw/component-specifications.json +++ b/examples/bop-quarterly/csvw/component-specifications.json @@ -1 +1,12 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"component-specifications.csv","dc:title":"BoP Quarterly Example","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"},{"name":"codes_used","virtual":true,"propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "url":"component-specifications.csv", + "dc:title":"BoP Quarterly Example", + "tableSchema": { + "columns": [ + {"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true}, + {"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true}, + {"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"}, + {"name":"codes_used", "titles": "codes_used","propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}, + {"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"}], + "aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"} +} \ No newline at end of file diff --git a/examples/bop-quarterly/csvw/data-structure-definition.json b/examples/bop-quarterly/csvw/data-structure-definition.json index 42f5eaa..5aabe0c 100644 --- a/examples/bop-quarterly/csvw/data-structure-definition.json +++ b/examples/bop-quarterly/csvw/data-structure-definition.json @@ -1 +1,14 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure","url":"component-specifications.csv","dc:title":"BoP Quarterly Example (Data Structure Definition)","rdf:type":{"@id":"qb:DataStructureDefinition"},"rdfs:label":"BoP Quarterly Example (Data Structure Definition)","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure", + "url":"component-specifications.csv", + "dc:title":"BoP Quarterly Example (Data Structure Definition)", + "rdf:type":{"@id":"qb:DataStructureDefinition"}, + "rdfs:label":"BoP Quarterly Example (Data Structure Definition)", + "tableSchema":{ + "columns":[ + {"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/component\/{component_slug}"}, + {"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true}, + {"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}, + {"name": "codes_used", "titles": "codes_used", "suppressOutput": true} + ], + "aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"}} \ No newline at end of file diff --git a/examples/bop-quarterly/csvw/dataset.json b/examples/bop-quarterly/csvw/dataset.json index cc6beb2..7a2dfed 100644 --- a/examples/bop-quarterly/csvw/dataset.json +++ b/examples/bop-quarterly/csvw/dataset.json @@ -1 +1,14 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example","url":"component-specifications.csv","dc:title":"BoP Quarterly Example","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","suppressOutput":true},{"name":"component_property","titles":"component_property","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"},{"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "@id":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example", + "url":"component-specifications.csv", + "dc:title":"BoP Quarterly Example", + "tableSchema":{ + "columns":[ + {"name":"component_slug","titles":"component_slug","suppressOutput":true}, + {"name":"component_attachment","titles":"component_attachment","suppressOutput":true}, + {"name":"component_property","titles":"component_property","suppressOutput":true}, + {"name": "codes_used", "titles": "codes_used", "suppressOutput": true}, + {"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"}, + {"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/structure"} + ], + "aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example"}} \ No newline at end of file diff --git a/examples/bop-quarterly/csvw/used-codes-codelists.json b/examples/bop-quarterly/csvw/used-codes-codelists.json index bf2af7b..223f34b 100644 --- a/examples/bop-quarterly/csvw/used-codes-codelists.json +++ b/examples/bop-quarterly/csvw/used-codes-codelists.json @@ -1 +1,10 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"component-specifications.csv","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"skos:ConceptScheme"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "url":"component-specifications.csv", + "tableSchema":{ + "columns":[ + {"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true}, + {"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true}, + {"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}, + {"name":"codes_used","titles":"codes_used","propertyUrl":"rdf:type","valueUrl":"skos:ConceptScheme"} + ], + "aboutUrl":"http:\/\/gss-data.org.uk\/data\/bop-quarterly-example\/codes-used\/{component_slug}"}} \ No newline at end of file diff --git a/examples/overseas-trade/csvw/component-specifications.json b/examples/overseas-trade/csvw/component-specifications.json index 9525559..2ad30f6 100644 --- a/examples/overseas-trade/csvw/component-specifications.json +++ b/examples/overseas-trade/csvw/component-specifications.json @@ -1 +1,12 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"component-specifications.csv","dc:title":"Overseas Trade Sample","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"},{"name":"codes_used","virtual":true,"propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/codes-used\/{component_slug}"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "url":"component-specifications.csv", + "dc:title":"Overseas Trade Sample", + "tableSchema":{ + "columns":[ + {"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true}, + {"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true}, + {"name":"component_property","titles":"component_property","datatype":"string","propertyUrl":"{+component_attachment}","valueUrl":"{+component_property}"}, + {"name":"codes_used","titles": "codes_used","propertyUrl":"http:\/\/publishmydata.com\/def\/qb\/codesUsed","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/codes-used\/{component_slug}"}, + {"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:ComponentSpecification"}], + "aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"} +} \ No newline at end of file diff --git a/examples/overseas-trade/csvw/data-structure-definition.json b/examples/overseas-trade/csvw/data-structure-definition.json index d89a68e..a0d1d32 100644 --- a/examples/overseas-trade/csvw/data-structure-definition.json +++ b/examples/overseas-trade/csvw/data-structure-definition.json @@ -1 +1,14 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure","url":"component-specifications.csv","dc:title":"Overseas Trade Sample (Data Structure Definition)","rdf:type":{"@id":"qb:DataStructureDefinition"},"rdfs:label":"Overseas Trade Sample (Data Structure Definition)","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure", + "url":"component-specifications.csv", + "dc:title":"Overseas Trade Sample (Data Structure Definition)", + "rdf:type":{"@id":"qb:DataStructureDefinition"}, + "rdfs:label":"Overseas Trade Sample (Data Structure Definition)", + "tableSchema":{ + "columns":[ + {"name":"component_slug","titles":"component_slug","datatype":"string","propertyUrl":"qb:component","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/component\/{component_slug}"}, + {"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true}, + {"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}, + {"name":"codes_used","titles":"codes_used","suppressOutput":true}, + ],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"} +} \ No newline at end of file diff --git a/examples/overseas-trade/csvw/dataset.json b/examples/overseas-trade/csvw/dataset.json index c6efad3..58a4d98 100644 --- a/examples/overseas-trade/csvw/dataset.json +++ b/examples/overseas-trade/csvw/dataset.json @@ -1 +1,14 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample","url":"component-specifications.csv","dc:title":"Overseas Trade Sample","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","suppressOutput":true},{"name":"component_property","titles":"component_property","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"},{"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "@id":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample", + "url":"component-specifications.csv", + "dc:title":"Overseas Trade Sample", + "tableSchema":{ + "columns":[ + {"name":"component_slug","titles":"component_slug","suppressOutput":true}, + {"name":"component_attachment","titles":"component_attachment","suppressOutput":true}, + {"name":"component_property","titles":"component_property","suppressOutput":true}, + {"name": "codes_used", "titles": "codes_used", "suppressOutput": true}, + {"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"qb:DataSet"}, + {"name":"structure","virtual":true,"propertyUrl":"qb:structure","valueUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/structure"} + ], + "aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample"}} \ No newline at end of file diff --git a/examples/overseas-trade/csvw/used-codes-codelists.json b/examples/overseas-trade/csvw/used-codes-codelists.json index 85db2a7..851e284 100644 --- a/examples/overseas-trade/csvw/used-codes-codelists.json +++ b/examples/overseas-trade/csvw/used-codes-codelists.json @@ -1 +1,10 @@ -{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}],"url":"component-specifications.csv","tableSchema":{"columns":[{"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true},{"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true},{"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true},{"name":"type","virtual":true,"propertyUrl":"rdf:type","valueUrl":"skos:ConceptScheme"}],"aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/codes-used\/{component_slug}"}} \ No newline at end of file +{"@context":["http:\/\/www.w3.org\/ns\/csvw",{"@language":"en"}], + "url":"component-specifications.csv", + "tableSchema":{ + "columns":[ + {"name":"component_slug","titles":"component_slug","datatype":"string","suppressOutput":true}, + {"name":"component_attachment","titles":"component_attachment","datatype":"string","suppressOutput":true}, + {"name":"component_property","titles":"component_property","datatype":"string","suppressOutput":true}, + {"name":"codes_used", "titles": "codes_used", "propertyUrl":"rdf:type","valueUrl":"skos:ConceptScheme"}, + ], + "aboutUrl":"http:\/\/gss-data.org.uk\/data\/overseas-trade-sample\/codes-used\/{component_slug}"}} \ No newline at end of file diff --git a/examples/regional-trade/csvw/component-specifications.csv b/examples/regional-trade/csvw/component-specifications.csv index 573b957..992cbe8 100644 --- a/examples/regional-trade/csvw/component-specifications.csv +++ b/examples/regional-trade/csvw/component-specifications.csv @@ -1,9 +1,9 @@ -component_slug,component_attachment,component_property -date,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refPeriod -flow,qb:dimension,http://gss-data.org.uk/def/dimension/flow -gbp_total,qb:measure,http://gss-data.org.uk/def/measure/gbp-total -geography,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refArea -measure_type,qb:dimension,http://purl.org/linked-data/cube#measureType -net_mass,qb:measure,http://gss-data.org.uk/def/measure/net-mass -sitc_section,qb:dimension,http://gss-data.org.uk/def/dimension/sitc-section -unit,qb:attribute,http://purl.org/linked-data/sdmx/2009/attribute#unitMeasure +component_slug,component_attachment,component_property,codes_used +date,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refPeriod,yes +flow,qb:dimension,http://gss-data.org.uk/def/dimension/flow,yes +gbp_total,qb:measure,http://gss-data.org.uk/def/measure/gbp-total, +geography,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refArea,yes +measure_type,qb:dimension,http://purl.org/linked-data/cube#measureType,yes +net_mass,qb:measure,http://gss-data.org.uk/def/measure/net-mass, +sitc_section,qb:dimension,http://gss-data.org.uk/def/dimension/sitc-section,yes +unit,qb:attribute,http://purl.org/linked-data/sdmx/2009/attribute#unitMeasure,yes diff --git a/examples/regional-trade/csvw/component-specifications.json b/examples/regional-trade/csvw/component-specifications.json index fe6490b..706d6a4 100644 --- a/examples/regional-trade/csvw/component-specifications.json +++ b/examples/regional-trade/csvw/component-specifications.json @@ -19,16 +19,16 @@ "datatype": "string", "propertyUrl": "{+component_attachment}", "valueUrl": "{+component_property}" + }{ + "name": "codes_used", + "titles": "codes_used", + "propertyUrl": "http://publishmydata.com/def/qb/codesUsed", + "valueUrl": "http://gss-data.org.uk/data/regional-trade/codes-used/{component_slug}" },{ "name": "type", "virtual": true, "propertyUrl": "rdf:type", "valueUrl": "qb:ComponentSpecification" - },{ - "name": "codes_used", - "virtual": true, - "propertyUrl": "http://publishmydata.com/def/qb/codesUsed", - "valueUrl": "http://gss-data.org.uk/data/regional-trade/codes-used/{component_slug}" }], "aboutUrl": "http://gss-data.org.uk/data/regional-trade/component/{component_slug}" } diff --git a/examples/regional-trade/csvw/data-structure-definition.json b/examples/regional-trade/csvw/data-structure-definition.json index 63e71ff..5dce4e5 100644 --- a/examples/regional-trade/csvw/data-structure-definition.json +++ b/examples/regional-trade/csvw/data-structure-definition.json @@ -22,6 +22,10 @@ "titles": "component_property", "datatype": "string", "suppressOutput": true + },{ + "name": "codes_used", + "titles": "codes_used", + "suppressOutput": true }], "aboutUrl": "http://gss-data.org.uk/data/regional-trade/structure" } diff --git a/examples/regional-trade/csvw/dataset.json b/examples/regional-trade/csvw/dataset.json index 80322de..2df442f 100644 --- a/examples/regional-trade/csvw/dataset.json +++ b/examples/regional-trade/csvw/dataset.json @@ -9,16 +9,17 @@ {"titles": "component_slug", "name": "component_slug", "suppressOutput": true}, {"titles": "component_attachment", "name": "component_attachment", "suppressOutput": true}, {"titles": "component_property", "name": "component_property", "suppressOutput": true}, + {"name": "codes_used", "titles": "codes_used", "suppressOutput": true}, { - "name": "type", - "virtual": true, - "propertyUrl": "rdf:type", - "valueUrl": "qb:DataSet" + "name": "type", + "virtual": true, + "propertyUrl": "rdf:type", + "valueUrl": "qb:DataSet" },{ - "name": "structure", - "virtual": true, - "propertyUrl": "qb:structure", - "valueUrl": "http://gss-data.org.uk/data/regional-trade/structure" + "name": "structure", + "virtual": true, + "propertyUrl": "qb:structure", + "valueUrl": "http://gss-data.org.uk/data/regional-trade/structure" }], "aboutUrl": "http://gss-data.org.uk/data/regional-trade" } diff --git a/examples/regional-trade/csvw/used-codes-codelists.json b/examples/regional-trade/csvw/used-codes-codelists.json index 483aabe..e8ecd37 100644 --- a/examples/regional-trade/csvw/used-codes-codelists.json +++ b/examples/regional-trade/csvw/used-codes-codelists.json @@ -18,8 +18,8 @@ "datatype": "string", "suppressOutput": true },{ - "name": "type", - "virtual": true, + "name": "codes_used", + "titles": "codes_used", "propertyUrl": "rdf:type", "valueUrl": "skos:Collection" }], diff --git a/examples/regional-trade/csvw/used-codes-codes.json b/examples/regional-trade/csvw/used-codes-codes.json index c9f6d6d..4bcbdd8 100644 --- a/examples/regional-trade/csvw/used-codes-codes.json +++ b/examples/regional-trade/csvw/used-codes-codes.json @@ -42,7 +42,6 @@ "name": "value", "titles": "value", "datatype": "number", - "propertyUrl": "skos:member", "suppressOutput": true }], "aboutUrl": "http://gss-data.org.uk/data/regional-trade/codes-used/{_name}" diff --git a/src/table2qb/configuration/column.clj b/src/table2qb/configuration/column.clj index f905150..47f3699 100644 --- a/src/table2qb/configuration/column.clj +++ b/src/table2qb/configuration/column.clj @@ -36,6 +36,12 @@ (:type column))) (def property-template :property_template) +(defn is-dimension-column? [column] + (= :dimension (:type column))) + +(defn is-attribute-column? [column] + (= :attribute (:type column))) + (defn- replace-symbols [s] (string/replace s #"£" "GBP")) diff --git a/src/table2qb/pipelines/cube.clj b/src/table2qb/pipelines/cube.clj index afcf889..8f5239b 100644 --- a/src/table2qb/pipelines/cube.clj +++ b/src/table2qb/pipelines/cube.clj @@ -10,12 +10,12 @@ [table2qb.configuration.column :as column]) (:import [java.io File])) -(defn suppress-value-column - "Suppresses the output of a metadata column definition if it corresponds to a value component" - [column is-value-p] - (if (is-value-p (keyword (get column "name"))) - (assoc column "suppressOutput" true) - column)) +(defn- component-has-codes-used-list? + "Indicates whether the DSD component for the given column has an associated pmd:codesUsed list." + [column] + ;;TODO: check dimension range? + (or (column/is-dimension-column? column) + (column/is-attribute-column? column))) (defn component->column [{:keys [name property_template value_template datatype]}] (let [col {"name" name @@ -26,14 +26,19 @@ (assoc col "valueUrl" value_template) col))) +(defn- code-list-metadata-column + "Returns a code list metadata CSVW column JSON for a cube column defintion." + [column] + (let [col (component->column column)] + (if (component-has-codes-used-list? column) + (assoc col "propertyUrl" "skos:member") + (-> col + (assoc "suppressOutput" true) + (dissoc "propertyUrl"))))) + (defn used-codes-codes-metadata [csv-url domain-data dataset-slug cube-config] (let [components (cube-config/ordered-columns cube-config) - columns (mapv (fn [comp] - (-> comp - (component->column) - (assoc "propertyUrl" "skos:member") - (suppress-value-column (cube-config/values cube-config)))) - components) + columns (mapv code-list-metadata-column components) codelist-uri (str domain-data dataset-slug "/codes-used/{_name}")] {"@context" ["http://www.w3.org/ns/csvw" {"@language" "en"}], "url" (str csv-url) @@ -89,9 +94,9 @@ "titles" "component_property", "datatype" "string", "suppressOutput" true} - {"name" "type", - "virtual" true, - "propertyUrl" "rdf:type", + {"name" "codes_used" + "titles" "codes_used" + "propertyUrl" "rdf:type" "valueUrl" "skos:Collection"}], "aboutUrl" codelist-uri}})) @@ -127,6 +132,9 @@ {"name" "component_property", "titles" "component_property", "datatype" "string", + "suppressOutput" true} + {"name" "codes_used" + "titles" "codes_used" "suppressOutput" true}], "aboutUrl" dsd-uri}})) @@ -135,28 +143,27 @@ "url" (str csv-url) "dc:title" (util/blank->nil dataset-name) "tableSchema" - {"columns" - [{"name" "component_slug", - "titles" "component_slug", - "datatype" "string", - "suppressOutput" true} - {"name" "component_attachment", - "titles" "component_attachment", - "datatype" "string", - "suppressOutput" true} - {"name" "component_property", - "titles" "component_property", - "datatype" "string", - "propertyUrl" "{+component_attachment}", - "valueUrl" "{+component_property}"} - {"name" "type", - "virtual" true, - "propertyUrl" "rdf:type", - "valueUrl" "qb:ComponentSpecification"} - {"name" "codes_used", - "virtual" true, - "propertyUrl" "http://publishmydata.com/def/qb/codesUsed", - "valueUrl" (str domain-data dataset-slug "/codes-used/{component_slug}")}], + {"columns" [{"name" "component_slug" + "titles" "component_slug" + "datatype" "string" + "suppressOutput" true} + {"name" "component_attachment" + "titles" "component_attachment" + "datatype" "string" + "suppressOutput" true} + {"name" "component_property" + "titles" "component_property" + "datatype" "string" + "propertyUrl" "{+component_attachment}" + "valueUrl" "{+component_property}"} + {"name" "codes_used" + "titles" "codes_used" + "propertyUrl" "http://publishmydata.com/def/qb/codesUsed" + "valueUrl" (str domain-data dataset-slug "/codes-used/{component_slug}")} + {"name" "type" + "virtual" true + "propertyUrl" "rdf:type" + "valueUrl" "qb:ComponentSpecification"}] "aboutUrl" (component-specification-template domain-data dataset-slug)}}) (defn dataset-metadata [csv-url domain-data dataset-name dataset-slug] @@ -173,6 +180,7 @@ [{"name" "component_slug", "titles" "component_slug", "suppressOutput" true} {"name" "component_attachment", "titles" "component_attachment", "suppressOutput" true} {"name" "component_property", "titles" "component_property", "suppressOutput" true} + {"name" "codes_used" "titles" "codes_used" "suppressOutput" true} {"name" "type","virtual" true,"propertyUrl" "rdf:type","valueUrl" "qb:DataSet"} {"name" "structure","virtual" true,"propertyUrl" "qb:structure","valueUrl" dsd-uri}], "aboutUrl" ds-uri}})) @@ -181,7 +189,8 @@ (map (fn [column] {:component_slug (column/column-name column) :component_attachment (column/component-attachment column) - :component_property (column/property-template column)}) + :component_property (column/property-template column) + :codes_used (if (component-has-codes-used-list? column) "yes" "")}) (cube-config/dimension-attribute-measure-columns cube-config))) (defn- components->csvw @@ -189,7 +198,7 @@ observations CSV file" [output-csv cube-config] (with-open [writer (io/writer output-csv)] - (write-csv-rows writer [:component_slug :component_attachment :component_property] (read-component-specifications cube-config)))) + (write-csv-rows writer [:component_slug :component_attachment :component_property :codes_used] (read-component-specifications cube-config)))) (defn- observations->csvw "Writes an intermediate observations CSV file for a given column configuration to the specified location." diff --git a/test/table2qb/pipelines/cube_test.clj b/test/table2qb/pipelines/cube_test.clj index 73683ff..48d51ca 100644 --- a/test/table2qb/pipelines/cube_test.clj +++ b/test/table2qb/pipelines/cube_test.clj @@ -17,19 +17,6 @@ (map #(get % "name")))] (is (= csv-columns# meta-columns#)))))) -(deftest suppress-value-column-test - (testing "value column" - (is (= {"name" "value" - "title" "Value" - "suppressOutput" true} - (suppress-value-column {"name" "value" - "title" "Value"} #{:value})))) - - (testing "non-value column" - (let [col {"name" "flow" - "title" "Flow"}] - (is (= col (suppress-value-column col #{:value})))))) - (deftest observation-template-test (let [domain-data-prefix "http://example.com/data/" dataset-slug "test" diff --git a/test/table2qb/pipelines/integration_test.clj b/test/table2qb/pipelines/integration_test.clj index 4529b6d..c90d09c 100644 --- a/test/table2qb/pipelines/integration_test.clj +++ b/test/table2qb/pipelines/integration_test.clj @@ -19,19 +19,18 @@ "./examples/vocabularies/qb.ttl" "./examples/overseas-trade/vocabularies/2012.rdf" "./examples/overseas-trade/vocabularies/CN_2015_20180206_105537.ttl")] - (let [stmts (concat - ;; Existing reference data - (codelist-pipeline "./examples/regional-trade/csv/flow-directions.csv" "Flow Directions" "flow-directions" test-domain) - (codelist-pipeline "./examples/regional-trade/csv/sitc-sections.csv" "Flow Directions" "sitc-sections" test-domain) - (codelist-pipeline "./examples/regional-trade/csv/units.csv" "Measurement Units" "measurement-units" test-domain) - (components-pipeline "./examples/regional-trade/csv/components.csv" test-domain) + (with-open [conn (repo/->connection repo)] + (rdf/add conn (concat + ;; Existing reference data + (codelist-pipeline "./examples/regional-trade/csv/flow-directions.csv" "Flow Directions" "flow-directions" test-domain) + (codelist-pipeline "./examples/regional-trade/csv/sitc-sections.csv" "Flow Directions" "sitc-sections" test-domain) + (codelist-pipeline "./examples/regional-trade/csv/units.csv" "Measurement Units" "measurement-units" test-domain) + (components-pipeline "./examples/regional-trade/csv/components.csv" test-domain) - ;; This dataset - (codelist-pipeline "./examples/overseas-trade/csv/countries.csv" "Countries" "countries" test-domain) - (components-pipeline "./examples/overseas-trade/csv/components.csv" test-domain) - (cube-pipeline "./examples/overseas-trade/csv/ots-cn-sample.csv" "Overseas Trade Sample" "overseas-trade-sample" default-config test-domain))] - (with-open [conn (repo/->connection repo)] - (rdf/add conn stmts))) + ;; This dataset + (codelist-pipeline "./examples/overseas-trade/csv/countries.csv" "Countries" "countries" test-domain) + (components-pipeline "./examples/overseas-trade/csv/components.csv" test-domain) + (cube-pipeline "./examples/overseas-trade/csv/ots-cn-sample.csv" "Overseas Trade Sample" "overseas-trade-sample" default-config test-domain)))) (testing "PMD Validation" (is (empty? (pmd/errors repo)))) (testing "PMD Dataset Validation" @@ -54,7 +53,25 @@ (testing "may be provided" (is (some #{"http://gss-data.org.uk/def/concept-scheme/sitc-sections"} schemes))) (testing "is optional" - (is (not-any? #{"http://gss-data.org.uk/def/concept-scheme/flow-directions"} schemes)))))))))) + (is (not-any? #{"http://gss-data.org.uk/def/concept-scheme/flow-directions"} schemes)))))) + + (testing "pmd:codesUsed" + (testing "should exist for dimensions" + (let [q (str "SELECT ?dim WHERE {" + " ?comp ?dim ." + " ?comp ?codes ." + "}") + dims (with-open [conn (repo/->connection repo)] + (mapv :dim (repo/query conn q)))] + (is (= 5 (count dims))))) + + (testing "should not exist for measures" + (let [q (str "ASK WHERE {" + " ?comp ?m ." + " ?comp ?codes ." + "}")] + (is (= false (with-open [conn (repo/->connection repo)] + (repo/query conn q))))))))))) ;; TODO: Vocabulary for pmd:usedCode