diff --git a/.gitignore b/.gitignore index a395c345..7f8e336a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ db/*.db .env .*.env .cfignore +cf-ssh.yml .vagrant .idea/ *profile* diff --git a/API.md b/API.md index 13874aa3..501e7e0e 100644 --- a/API.md +++ b/API.md @@ -210,7 +210,7 @@ By default, using the `_sort_` option returns records sorted into ascending orde When the dataset includes a `location` at the root level (`location.lat` and `location.lon`) then the documents will be indexed geographically. You can use the `_zip` and `_distance` options to narrow query results down to those within a geographic area. For example, `_zip=12345&_distance=10mi` will return only those results within 10 miles of the center of the given zip code. -**Note: Due to a bug, specifying `location` and its sub-fields in certain field or option parameters may cause an error.** This is a known problem and tracked in this GitHub issue: _[Querying for a school by name and returning location.lon does not work](https://github.com/18F/open-data-maker/issues/227)_ +Additionally, you can request `location.lat` and `location.lat` in a search that includes a `_fields` filter and it will return the record(s) with respective lat and/or lon coordinates. #### Additional Notes on Geographic Filtering diff --git a/lib/data_magic.rb b/lib/data_magic.rb index 1000d5cf..7b07da73 100644 --- a/lib/data_magic.rb +++ b/lib/data_magic.rb @@ -171,7 +171,7 @@ def self.nested_object_type(hash) def self.create_index(es_index_name = nil, field_types={}) logger.info "create_index field_types: #{field_types.inspect[0..500]}" es_index_name ||= self.config.scoped_index_name - field_types['location'] = 'geo_point' + field_types['location'] = 'lat_lon' # custom lat_lon type maps to geo_point with additional field options es_types = NestedHash.new.add(es_field_types(field_types)) nested_object_type(es_types) begin @@ -234,6 +234,11 @@ def self.es_field_types(field_types) index_analyzer: 'autocomplete_index', search_analyzer: 'autocomplete_search' }, + 'lat_lon' => { type: 'geo_point', + lat_lon: true, + store: true + + } } field_types.each_with_object({}) do |(key, type), result| result[key] = custom_type[type] diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb index df4ab0f0..86214b36 100644 --- a/lib/data_magic/config.rb +++ b/lib/data_magic/config.rb @@ -270,7 +270,6 @@ def field_types logger.info "field_types #{fields.inspect}" fields.each do |field_name, info| type = info['type'] || "string" - type = nil if field_name == 'location.lat' || field_name == 'location.lon' #logger.info "field #{field_name}: #{type.inspect}" @field_types[field_name] = type unless type.nil? if type == 'name' || type == 'autocomplete' diff --git a/spec/features/api_spec.rb b/spec/features/api_spec.rb index 641f64a6..7b20dbd1 100644 --- a/spec/features/api_spec.rb +++ b/spec/features/api_spec.rb @@ -141,8 +141,8 @@ expect(result.length).to eq 2 - expect(result[0]).to eq %w(id code name state population area.land area.water location.lat location.lon) - expect(result[1]).to eq %w(1714000 00428803 Chicago IL 2695598 227.635 6.479 41.837551 -87.681844) + expect(result[0]).to eq %w(id code name state population location.lat location.lon area.land area.water) + expect(result[1]).to eq %w(1714000 00428803 Chicago IL 2695598 41.837551 -87.681844 227.635 6.479 ) end end diff --git a/spec/lib/data_magic/config_field_types_spec.rb b/spec/lib/data_magic/config_field_types_spec.rb index 557e24fd..0cd8361c 100644 --- a/spec/lib/data_magic/config_field_types_spec.rb +++ b/spec/lib/data_magic/config_field_types_spec.rb @@ -72,13 +72,17 @@ end end - it "supports special case for location fields as nil" do - # special case for location in create_index + it "supports location.lat and location.lon fields" do + allow(config).to receive(:file_config).and_return([{'name' => 'one.csv'}]) allow(config).to receive(:dictionary).and_return( - IndifferentHash.new 'location.lat': {source:'LAT_COLUMN'}, - 'location.lon': {source:'LON_COLUMN'} - + IndifferentHash.new 'location.lat': {source:'LAT_COLUMN', type: 'float'}, + 'location.lon': {source:'LON_COLUMN', type: 'float'} + ) + expect(config.field_types).to eq( + { + 'location.lat'=>'float', + 'location.lon'=>'float' + } ) - expect(config.field_types).to eq({}) end end diff --git a/spec/lib/data_magic/search_spec.rb b/spec/lib/data_magic/search_spec.rb index ae8a9436..3c3b1a6a 100644 --- a/spec/lib/data_magic/search_spec.rb +++ b/spec/lib/data_magic/search_spec.rb @@ -191,6 +191,18 @@ expected['metadata']["total"] = expected["results"].length expect(result).to eq(expected) end + + it "#search with a fields filter can return location.lat and location.lon values" do + sf_location = { lat: 37.727239, lon: -123.032229 } + DataMagic.logger.debug "sfo_location[:lat] #{sf_location[:lat].class} #{sf_location[:lat].inspect}" + response = DataMagic.search({city: "San Francisco"}, {:fields => ["location.lat", "location.lon"]}) + result = response["results"][0] + expect(result.keys.length).to eq(2) + expect(result).to include("location.lat") + expect(result).to include("location.lat") + expect(result["location.lat"]).to eq sf_location[:lat] + expect(result["location.lon"]).to eq sf_location[:lon] + end end describe "with sample-data" do