From 98155355ac54e0463bcc7e2e99845451dd6e3ab1 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 25 Aug 2016 11:12:14 +0200 Subject: [PATCH 01/18] temp scripts to infer way and estimate power pole distances --- estimate_power_pole_distance.py | 64 ++++++++++++++++++++++++++++++++ infer_way.py | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 estimate_power_pole_distance.py create mode 100644 infer_way.py diff --git a/estimate_power_pole_distance.py b/estimate_power_pole_distance.py new file mode 100644 index 0000000..4ebd492 --- /dev/null +++ b/estimate_power_pole_distance.py @@ -0,0 +1,64 @@ +import psycopg2 +import json +import sys +import ast +import numpy as np +import scipy.stats as stats +import pylab as pl + +try: + conn = psycopg2.connect("dbname='gis' user='Munna' host='localhost' password=''") + cur = conn.cursor() +except: + print("I am unable to connect to the database") + +min_distances = [] +max_distances = [] + +get_powerlines_query = "SELECT id as id, properties->>'refs' as ref FROM powerline LIMIT 10" +cur.execute(get_powerlines_query) +powerlines = cur.fetchall() +for powerline in powerlines: + nodes_osmids = ast.literal_eval(powerline[1]) + nodes_osmids = tuple([str(i) for i in nodes_osmids]) + nodes_count_query = "SELECT count(DISTINCT id) FROM point WHERE properties->>'osmid' IN %s" + cur.execute(nodes_count_query, (nodes_osmids, )) + nodes_count = cur.fetchone() + # check if we have all the nodes for a powerline in our database + if nodes_count[0] == len(nodes_osmids): + nodes_distances_query = '''SELECT MIN(ST_Distance(a.geom, b.geom)), + MAX(ST_Distance(a.geom, b.geom)) + FROM + point a, + point b + WHERE a.id IN (SELECT DISTINCT id FROM point where properties->>'osmid' in %s) + AND b.id IN (SELECT DISTINCT id FROM point where properties->>'osmid' in %s) + AND a.id != b.id''' + cur.execute(nodes_distances_query, (nodes_osmids, nodes_osmids, )) + distances = cur.fetchone() + print(distances) + min_distances.append(distances[0]) + max_distances.append(distances[1]) + +print("Average minimum distance: ", ) +print(reduce(lambda x, y: x + y, min_distances) / len(min_distances)) +print("Average maximum distance: ", ) +print(reduce(lambda x, y: x + y, max_distances) / len(max_distances)) + +# h = sorted(min_distances) + +# fit = stats.norm.pdf(h, np.mean(h), np.std(h)) #this is a fitting indeed + +# f = pl.figure() +# pl.plot(h,fit,'-o') +# pl.hist(h,normed=True) #use this to draw histogram of your data +# f.savefig('min.pdf') + +# h = sorted(max_distances) + +# fit = stats.norm.pdf(h, np.mean(h), np.std(h)) #this is a fitting indeed + +# f = pl.figure() +# pl.plot(h,fit,'-o') +# pl.hist(h,normed=True) #use this to draw histogram of your data +# f.savefig('max.pdf') diff --git a/infer_way.py b/infer_way.py new file mode 100644 index 0000000..decd070 --- /dev/null +++ b/infer_way.py @@ -0,0 +1,66 @@ +import psycopg2 +import json +import sys +import ast + +try: + conn = psycopg2.connect("dbname='gis' user='Munna' host='localhost' password=''") + cur = conn.cursor() +except: + print("I am unable to connect to the database") + +nodes_osmids = ( +'3212196101', '3212196097', '3212196093', '3212196089', '3212196086', '3212196083', +'3212196077', '3212196075', '3212196071', '3212196045', '3212196012', '3212195977', +'3212195974', '3212195967', '3212195960', '3212195952', '3212195947', +'3212195940', '3212195935', '3212195931', '3212195926', '3212195925', +'3212195924', '3212195923', '3212195917', '3212195908', '3212195898', +'3212195884', '3212195874', '3212195866', '3212195869', '3212195878', +'3212195882', '3212195889', '3212195895', '3212195893', '3212195896' +) + +nodes_query = "SELECT id, geom, properties->>'osmid' FROM point "\ + "WHERE properties->>'osmid' in %s" +cur.execute(nodes_query, (nodes_osmids, )) + +nodes = cur.fetchall() + +print(len(nodes_osmids)) +print(len(nodes)) + + +processing_node = "3212196097" +processed_nodes = [] +processed_nodes.append(processing_node) + +is_complete = False + +while is_complete == False: + # print(processed_nodes) + # print "processing - ", + # print(processing_node) + fetch_closest_query = ''' + WITH current_point AS ( + SELECT id, ST_AsText(geom) AS geom FROM point WHERE (properties->>'osmid') = %s + ) + SELECT point.id, point.properties->>'osmid' FROM point, current_point + WHERE ST_Distance(ST_GeomFromText(current_point.geom), point.geom) > 0.002 + AND ST_Distance(ST_GeomFromText(current_point.geom), point.geom) < 0.09 + AND properties->>'osmid' IN %s + ORDER BY ST_Distance(ST_GeomFromText(current_point.geom), point.geom) ASC + LIMIT 1''' + cur.execute(fetch_closest_query, [ + processing_node, + tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) + ]) + closest_node = cur.fetchone() + if closest_node is not None and len(closest_node) > 0: + print "Closest is - ", + print(closest_node[1]) + processing_node = closest_node[1] + processed_nodes.append(processing_node) + else: + print("\n*********** IS COMPLETE **************\n") + is_complete = True + +print(processed_nodes) From 671dd2d43dbc117bc41dd896cf380c4f43c67571 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Sun, 28 Aug 2016 13:36:40 +0200 Subject: [PATCH 02/18] Way inference with initiation from cluster --- way_inference/cluster_wrapper.py | 28 ++++++++++++ infer_way.py => way_inference/infer_way.py | 41 ++++++++--------- way_inference/nodes_wrapper.py | 51 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 way_inference/cluster_wrapper.py rename infer_way.py => way_inference/infer_way.py (60%) create mode 100644 way_inference/nodes_wrapper.py diff --git a/way_inference/cluster_wrapper.py b/way_inference/cluster_wrapper.py new file mode 100644 index 0000000..6ec5b06 --- /dev/null +++ b/way_inference/cluster_wrapper.py @@ -0,0 +1,28 @@ +class ClusterWrapper: + bounds = [] + cur = None + + def __init__(self, sql_cur, bounds): + self.bounds = bounds + self.cur = sql_cur + + def getClusters(self): + self.cur.execute(self._clusters_query(), self._clusters_query_args()) + return self.cur.fetchall() + + def _clusters_query(self): + clusters_query = ''' + SELECT ST_AsText(unnest((ST_ClusterWithin(geom, 0.1)))) as cluster_geom + FROM point + WHERE ST_Intersects( + ST_MakeEnvelope(%s, %s, %s, %s), + geom + ) + LIMIT 1 + ''' + return clusters_query + + def _clusters_query_args(self): + args = [self.bounds[1], self.bounds[0], self.bounds[3], self.bounds[2]] + return args + diff --git a/infer_way.py b/way_inference/infer_way.py similarity index 60% rename from infer_way.py rename to way_inference/infer_way.py index decd070..d1bf05d 100644 --- a/infer_way.py +++ b/way_inference/infer_way.py @@ -2,6 +2,8 @@ import json import sys import ast +from cluster_wrapper import ClusterWrapper +from nodes_wrapper import NodesWrapper try: conn = psycopg2.connect("dbname='gis' user='Munna' host='localhost' password=''") @@ -19,17 +21,15 @@ '3212195882', '3212195889', '3212195895', '3212195893', '3212195896' ) -nodes_query = "SELECT id, geom, properties->>'osmid' FROM point "\ - "WHERE properties->>'osmid' in %s" -cur.execute(nodes_query, (nodes_osmids, )) +bounds = [28.212890625, -1.8261677821806805, 33.486328125, 0.8349313860427184] +clusterWrapper = ClusterWrapper(cur, bounds) +clusters = clusterWrapper.getClusters() -nodes = cur.fetchall() +for cluster in clusters: + nodes_osmids = NodesWrapper(cur, bounds).get_nodes_osmids_in_cluster(cluster[0]) -print(len(nodes_osmids)) -print(len(nodes)) - - -processing_node = "3212196097" +# processing_node = "3212196097" +processing_node = "2043615536" processed_nodes = [] processed_nodes.append(processing_node) @@ -39,21 +39,16 @@ # print(processed_nodes) # print "processing - ", # print(processing_node) - fetch_closest_query = ''' - WITH current_point AS ( - SELECT id, ST_AsText(geom) AS geom FROM point WHERE (properties->>'osmid') = %s - ) - SELECT point.id, point.properties->>'osmid' FROM point, current_point - WHERE ST_Distance(ST_GeomFromText(current_point.geom), point.geom) > 0.002 - AND ST_Distance(ST_GeomFromText(current_point.geom), point.geom) < 0.09 - AND properties->>'osmid' IN %s - ORDER BY ST_Distance(ST_GeomFromText(current_point.geom), point.geom) ASC - LIMIT 1''' - cur.execute(fetch_closest_query, [ + closest_nodes = NodesWrapper(cur, []).get_closest_nodes_to( processing_node, tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) - ]) - closest_node = cur.fetchone() + ) + closest_node = None + print(closest_nodes) + + if len(closest_nodes) > 0: + closest_node = closest_nodes[0] # There are already ordered by distance + if closest_node is not None and len(closest_node) > 0: print "Closest is - ", print(closest_node[1]) @@ -64,3 +59,5 @@ is_complete = True print(processed_nodes) +for node_osmid in processed_nodes: + print("node(%s);" % node_osmid) diff --git a/way_inference/nodes_wrapper.py b/way_inference/nodes_wrapper.py new file mode 100644 index 0000000..5346960 --- /dev/null +++ b/way_inference/nodes_wrapper.py @@ -0,0 +1,51 @@ +class NodesWrapper: + bounds = [] + cur = None + + def __init__(self, sql_cursor, bounds): + self.bounds = bounds + self.cur = sql_cursor + + def get_nodes_osmids_in_cluster(self, cluster_geom_text): + self.cur.execute(self._node_osmids_in_cluster_query(), [ + cluster_geom_text, + self.bounds[1], + self.bounds[0], + self.bounds[3], + self.bounds[2] + ]) + node_osmids_tuple = self.cur.fetchall() + # Return an array instead of a tuple. + def tmp(x): return x[0] + return list(map(tmp, node_osmids_tuple)) + + def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): + fetch_closest_query = ''' + WITH current_point AS ( + SELECT id, ST_AsText(geom) AS geom FROM point WHERE (properties->>'osmid') = %s + ) + SELECT point.id, point.properties->>'osmid', + ST_Distance(ST_GeomFromText(current_point.geom), point.geom) + FROM point, current_point + WHERE ST_Distance(ST_GeomFromText(current_point.geom), point.geom) > 0.0004 + AND ST_Distance(ST_GeomFromText(current_point.geom), point.geom) < 0.09 + AND properties->>'osmid' IN %s + ORDER BY ST_Distance(ST_GeomFromText(current_point.geom), point.geom) ASC + LIMIT 1 + ''' + self.cur.execute(fetch_closest_query, [ + for_node_osmid, + among_osm_ids + ]) + return self.cur.fetchall() + + def _node_osmids_in_cluster_query(self): + return ''' + SELECT DISTINCT properties->>'osmid' FROM point + WHERE ST_Intersects(ST_CollectionExtract(%s, 1), point.geom) + AND properties->'tags'->>'power'='tower' + AND ST_Intersects( + ST_MakeEnvelope(%s, %s, %s, %s), + point.geom + ); + ''' From 5d9b3cb5adcedba1a29396c639181f64c7c9ba10 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Sun, 28 Aug 2016 15:45:17 +0200 Subject: [PATCH 03/18] Save inferred way to database --- ...2066ce_create_inferred_powerlines_table.py | 30 +++++++++++++++ way_inference/infer_way.py | 28 ++++++++++---- way_inference/ways_wrapper.py | 38 +++++++++++++++++++ 3 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 migrations/versions/360ac2066ce_create_inferred_powerlines_table.py create mode 100644 way_inference/ways_wrapper.py diff --git a/migrations/versions/360ac2066ce_create_inferred_powerlines_table.py b/migrations/versions/360ac2066ce_create_inferred_powerlines_table.py new file mode 100644 index 0000000..88aab81 --- /dev/null +++ b/migrations/versions/360ac2066ce_create_inferred_powerlines_table.py @@ -0,0 +1,30 @@ +"""create inferred_powerlines table + +Revision ID: 360ac2066ce +Revises: 58ba1b9ec53 +Create Date: 2016-08-28 14:40:35.960705 + +""" + +# revision identifiers, used by Alembic. +revision = '360ac2066ce' +down_revision = '58ba1b9ec53' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # copy the structure of `powerline` table and create `inferred_powerlines` + # Note: The below approach won't copy quite some information like + # Foreignkeys and triggers. + # Refer: http://stackoverflow.com/a/1220942/976880 + op.execute("CREATE table inferred_powerlines("\ + "like powerline "\ + "including defaults "\ + "including constraints "\ + "including indexes"\ + ")") + +def downgrade(): + op.drop_table("inferred_powerlines") diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index d1bf05d..01efcc6 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -4,6 +4,7 @@ import ast from cluster_wrapper import ClusterWrapper from nodes_wrapper import NodesWrapper +from ways_wrapper import WaysWrapper try: conn = psycopg2.connect("dbname='gis' user='Munna' host='localhost' password=''") @@ -20,16 +21,18 @@ '3212195884', '3212195874', '3212195866', '3212195869', '3212195878', '3212195882', '3212195889', '3212195895', '3212195893', '3212195896' ) - -bounds = [28.212890625, -1.8261677821806805, 33.486328125, 0.8349313860427184] +bounds = [11.089153289794922, 4.593878359460639, 11.418743133544922, 4.759664350520704] +# bounds = [28.212890625, -1.8261677821806805, 33.486328125, 0.8349313860427184] clusterWrapper = ClusterWrapper(cur, bounds) clusters = clusterWrapper.getClusters() for cluster in clusters: nodes_osmids = NodesWrapper(cur, bounds).get_nodes_osmids_in_cluster(cluster[0]) +print(nodes_osmids) # processing_node = "3212196097" -processing_node = "2043615536" +# # processing_node = "2043615536" +processing_node = "3318502646" processed_nodes = [] processed_nodes.append(processing_node) @@ -39,12 +42,18 @@ # print(processed_nodes) # print "processing - ", # print(processing_node) - closest_nodes = NodesWrapper(cur, []).get_closest_nodes_to( - processing_node, - tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) - ) + unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) closest_node = None - print(closest_nodes) + + if len(unprocessed_nodes) > 0: + closest_nodes = NodesWrapper(cur, []).get_closest_nodes_to( + processing_node, + unprocessed_nodes + ) + print(closest_nodes) + else: + is_complete = True + continue if len(closest_nodes) > 0: closest_node = closest_nodes[0] # There are already ordered by distance @@ -61,3 +70,6 @@ print(processed_nodes) for node_osmid in processed_nodes: print("node(%s);" % node_osmid) + +WaysWrapper(cur).save_to_database(processed_nodes) +conn.commit() diff --git a/way_inference/ways_wrapper.py b/way_inference/ways_wrapper.py new file mode 100644 index 0000000..2c6e68b --- /dev/null +++ b/way_inference/ways_wrapper.py @@ -0,0 +1,38 @@ +import json + +class WaysWrapper: + + def __init__(self, sql_cur): + self.cur = sql_cur + + def save_to_database(self, nodes_osmids): + nodes = [] + flag = False + for node_osmid in nodes_osmids: + query = "SELECT ST_X(geom), ST_Y(geom) FROM point WHERE properties->>'osmid' = %s" + self.cur.execute(query, [str(node_osmid)] ) + node = self.cur.fetchone() + if node is None: + flag = True + else: + nodes.append(node) + + if flag: + return None + + if len(nodes) < 2: + return None + + linestring = "" + print(nodes) + for node in nodes: + linestring += "{} {},".format(node[0], node[1]) + + linestring = linestring[:-1] + print("INSERTING {}".format(linestring)) + query = "INSERT INTO inferred_powerlines(geom, properties) VALUES(%s, %s)" + self.cur.execute(query, [ + 'LINESTRING({})'.format(linestring), + json.dumps({ "tags": {}, "refs": nodes_osmids }) + ]) + From 37c97dd684fa0f0f7a16260aff0e1ef754518439 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Wed, 31 Aug 2016 12:11:44 +0200 Subject: [PATCH 04/18] Start inferring from one of the farthest points --- way_inference/cluster_wrapper.py | 12 +++++++--- way_inference/infer_way.py | 12 ++++++---- way_inference/nodes_wrapper.py | 40 +++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/way_inference/cluster_wrapper.py b/way_inference/cluster_wrapper.py index 6ec5b06..ffbe169 100644 --- a/way_inference/cluster_wrapper.py +++ b/way_inference/cluster_wrapper.py @@ -7,12 +7,17 @@ def __init__(self, sql_cur, bounds): self.cur = sql_cur def getClusters(self): - self.cur.execute(self._clusters_query(), self._clusters_query_args()) + self.cur.execute( + self._clusters_query(), + self._clusters_query_args() + ) return self.cur.fetchall() def _clusters_query(self): + # TODO: Get only the points that are not part of a powerline clusters_query = ''' - SELECT ST_AsText(unnest((ST_ClusterWithin(geom, 0.1)))) as cluster_geom + SELECT ST_AsText(unnest((ST_ClusterWithin(geom, 0.1)))) + AS cluster_geom FROM point WHERE ST_Intersects( ST_MakeEnvelope(%s, %s, %s, %s), @@ -23,6 +28,7 @@ def _clusters_query(self): return clusters_query def _clusters_query_args(self): - args = [self.bounds[1], self.bounds[0], self.bounds[3], self.bounds[2]] + args = [self.bounds[1], self.bounds[0], + self.bounds[3], self.bounds[2]] return args diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 01efcc6..4a491d1 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -21,8 +21,9 @@ '3212195884', '3212195874', '3212195866', '3212195869', '3212195878', '3212195882', '3212195889', '3212195895', '3212195893', '3212195896' ) -bounds = [11.089153289794922, 4.593878359460639, 11.418743133544922, 4.759664350520704] +# bounds = [11.089153289794922, 4.593878359460639, 11.418743133544922, 4.759664350520704] # bounds = [28.212890625, -1.8261677821806805, 33.486328125, 0.8349313860427184] +bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa clusterWrapper = ClusterWrapper(cur, bounds) clusters = clusterWrapper.getClusters() @@ -30,9 +31,12 @@ nodes_osmids = NodesWrapper(cur, bounds).get_nodes_osmids_in_cluster(cluster[0]) print(nodes_osmids) -# processing_node = "3212196097" -# # processing_node = "2043615536" -processing_node = "3318502646" + +farthest_nodes = NodesWrapper(cur, bounds).get_farthest_nodes_among_nodes(nodes_osmids) +print(farthest_nodes) + +# Start processing with one of the farthest nodes +processing_node = farthest_nodes[0] processed_nodes = [] processed_nodes.append(processing_node) diff --git a/way_inference/nodes_wrapper.py b/way_inference/nodes_wrapper.py index 5346960..ad00755 100644 --- a/way_inference/nodes_wrapper.py +++ b/way_inference/nodes_wrapper.py @@ -22,10 +22,14 @@ def tmp(x): return x[0] def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): fetch_closest_query = ''' WITH current_point AS ( - SELECT id, ST_AsText(geom) AS geom FROM point WHERE (properties->>'osmid') = %s + SELECT id, ST_AsText(geom) AS geom FROM point + WHERE (properties->>'osmid') = %s ) SELECT point.id, point.properties->>'osmid', - ST_Distance(ST_GeomFromText(current_point.geom), point.geom) + ST_Distance( + ST_GeomFromText(current_point.geom), + point.geom + ) FROM point, current_point WHERE ST_Distance(ST_GeomFromText(current_point.geom), point.geom) > 0.0004 AND ST_Distance(ST_GeomFromText(current_point.geom), point.geom) < 0.09 @@ -39,13 +43,37 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): ]) return self.cur.fetchall() + def get_farthest_nodes_among_nodes(self, among_osm_ids): + farthest_nodes_query = ''' + WITH point_ids AS ( + SELECT id FROM point + WHERE (point.properties->>'osmid') IN %s + ) + SELECT a.properties->>'osmid', + b.properties->>'osmid', + st_distance(a.geom, b.geom) + FROM point a, point b + WHERE a.id IN (SELECT id FROM point_ids) + AND b.id IN (SELECT id FROM point_ids) + AND a.id != b.id + ORDER BY st_distance(a.geom, b.geom) DESC + LIMIT 1; + ''' + self.cur.execute(farthest_nodes_query, [ + tuple(among_osm_ids) + ]) + return self.cur.fetchone() + def _node_osmids_in_cluster_query(self): return ''' SELECT DISTINCT properties->>'osmid' FROM point - WHERE ST_Intersects(ST_CollectionExtract(%s, 1), point.geom) + WHERE ST_Intersects( + ST_CollectionExtract(%s, 1), + point.geom + ) AND properties->'tags'->>'power'='tower' AND ST_Intersects( - ST_MakeEnvelope(%s, %s, %s, %s), - point.geom - ); + ST_MakeEnvelope(%s, %s, %s, %s), + point.geom + ); ''' From 291afdb9c4b9695a20119bc30d727f066c8b119b Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Wed, 31 Aug 2016 15:23:31 +0200 Subject: [PATCH 05/18] Ignore too close nodes during inference --- way_inference/infer_way.py | 25 ++++++++++++++----------- way_inference/nodes_wrapper.py | 26 ++++++++++++++++++++------ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 4a491d1..a34bc23 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -21,9 +21,8 @@ '3212195884', '3212195874', '3212195866', '3212195869', '3212195878', '3212195882', '3212195889', '3212195895', '3212195893', '3212195896' ) -# bounds = [11.089153289794922, 4.593878359460639, 11.418743133544922, 4.759664350520704] -# bounds = [28.212890625, -1.8261677821806805, 33.486328125, 0.8349313860427184] -bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa +bounds = [10.39529800415039, 4.050234320898018, 10.50516128540039, 4.109221809610561] # parallel lines +# bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa clusterWrapper = ClusterWrapper(cur, bounds) clusters = clusterWrapper.getClusters() @@ -39,6 +38,7 @@ processing_node = farthest_nodes[0] processed_nodes = [] processed_nodes.append(processing_node) +ignored_nodes = [] # nodes are ignored if there are too close to a node is_complete = False @@ -46,26 +46,29 @@ # print(processed_nodes) # print "processing - ", # print(processing_node) + # procesed nodes minus the all nodes unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) + + # unprocessed nodes minus the ignored node. + unprocessed_nodes = tuple(set(unprocessed_nodes) ^ set(tuple(ignored_nodes))) closest_node = None if len(unprocessed_nodes) > 0: - closest_nodes = NodesWrapper(cur, []).get_closest_nodes_to( + nodes_around = NodesWrapper(cur, []).get_closest_nodes_to( processing_node, unprocessed_nodes ) - print(closest_nodes) + + ignored_nodes = ignored_nodes + nodes_around['too_close_node_osmids'] + closest_node = nodes_around['closest_node_osmid'] else: is_complete = True continue - if len(closest_nodes) > 0: - closest_node = closest_nodes[0] # There are already ordered by distance - - if closest_node is not None and len(closest_node) > 0: + if closest_node is not None: print "Closest is - ", - print(closest_node[1]) - processing_node = closest_node[1] + print(closest_node) + processing_node = closest_node processed_nodes.append(processing_node) else: print("\n*********** IS COMPLETE **************\n") diff --git a/way_inference/nodes_wrapper.py b/way_inference/nodes_wrapper.py index ad00755..7debb0c 100644 --- a/way_inference/nodes_wrapper.py +++ b/way_inference/nodes_wrapper.py @@ -1,6 +1,8 @@ class NodesWrapper: bounds = [] cur = None + closest_min_distance = 0.0004 + closest_max_distance = 0.09 def __init__(self, sql_cursor, bounds): self.bounds = bounds @@ -29,19 +31,31 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): ST_Distance( ST_GeomFromText(current_point.geom), point.geom - ) + ) as distance FROM point, current_point - WHERE ST_Distance(ST_GeomFromText(current_point.geom), point.geom) > 0.0004 - AND ST_Distance(ST_GeomFromText(current_point.geom), point.geom) < 0.09 + WHERE ST_Distance(ST_GeomFromText(current_point.geom), point.geom) < %s AND properties->>'osmid' IN %s ORDER BY ST_Distance(ST_GeomFromText(current_point.geom), point.geom) ASC - LIMIT 1 ''' self.cur.execute(fetch_closest_query, [ for_node_osmid, - among_osm_ids + self.closest_max_distance, + among_osm_ids, ]) - return self.cur.fetchall() + closest_nodes = self.cur.fetchall() + result = { + 'too_close_node_osmids': [], + 'closest_node_osmid': None + } + + for node in closest_nodes: + if (node[2] < self.closest_min_distance): + result['too_close_node_osmids'].append(node[1]) + else: + result['closest_node_osmid'] = node[1] + break; + return result; + def get_farthest_nodes_among_nodes(self, among_osm_ids): farthest_nodes_query = ''' From c95bd13412c6a6f2a05b09fffd0a006e2dc709dc Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 1 Sep 2016 19:38:34 +0200 Subject: [PATCH 06/18] Instantiate wrappers only once --- way_inference/infer_way.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index a34bc23..6ca6e21 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -23,15 +23,18 @@ ) bounds = [10.39529800415039, 4.050234320898018, 10.50516128540039, 4.109221809610561] # parallel lines # bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa -clusterWrapper = ClusterWrapper(cur, bounds) -clusters = clusterWrapper.getClusters() + +nodesWrapper = NodesWrapper(cur, bounds) +clustersWrapper = ClusterWrapper(cur, bounds) + +clusters = clustersWrapper.getClusters() for cluster in clusters: - nodes_osmids = NodesWrapper(cur, bounds).get_nodes_osmids_in_cluster(cluster[0]) + nodes_osmids = nodesWrapper.get_nodes_osmids_in_cluster(cluster[0]) print(nodes_osmids) -farthest_nodes = NodesWrapper(cur, bounds).get_farthest_nodes_among_nodes(nodes_osmids) +farthest_nodes = nodesWrapper.get_farthest_nodes_among_nodes(nodes_osmids) print(farthest_nodes) # Start processing with one of the farthest nodes @@ -54,7 +57,7 @@ closest_node = None if len(unprocessed_nodes) > 0: - nodes_around = NodesWrapper(cur, []).get_closest_nodes_to( + nodes_around = nodesWrapper.get_closest_nodes_to( processing_node, unprocessed_nodes ) From 742613f1f8f6c571cacbb49c47e2c1152040e357 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 1 Sep 2016 20:27:19 +0200 Subject: [PATCH 07/18] Complete way on encountering a power building --- way_inference/infer_way.py | 21 ++++++++++++++------- way_inference/ways_wrapper.py | 23 ++++++++++++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 6ca6e21..2361bf3 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -21,11 +21,12 @@ '3212195884', '3212195874', '3212195866', '3212195869', '3212195878', '3212195882', '3212195889', '3212195895', '3212195893', '3212195896' ) -bounds = [10.39529800415039, 4.050234320898018, 10.50516128540039, 4.109221809610561] # parallel lines -# bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa +# bounds = [10.39529800415039, 4.050234320898018, 10.50516128540039, 4.109221809610561] # parallel lines +bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa nodesWrapper = NodesWrapper(cur, bounds) clustersWrapper = ClusterWrapper(cur, bounds) +waysWrapper = WaysWrapper(cur, bounds) clusters = clustersWrapper.getClusters() @@ -39,23 +40,24 @@ # Start processing with one of the farthest nodes processing_node = farthest_nodes[0] + processed_nodes = [] -processed_nodes.append(processing_node) ignored_nodes = [] # nodes are ignored if there are too close to a node +processed_nodes.append(processing_node) + is_complete = False while is_complete == False: - # print(processed_nodes) - # print "processing - ", - # print(processing_node) # procesed nodes minus the all nodes unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) # unprocessed nodes minus the ignored node. unprocessed_nodes = tuple(set(unprocessed_nodes) ^ set(tuple(ignored_nodes))) + closest_node = None + if len(unprocessed_nodes) > 0: nodes_around = nodesWrapper.get_closest_nodes_to( processing_node, @@ -73,6 +75,11 @@ print(closest_node) processing_node = closest_node processed_nodes.append(processing_node) + + # if the node that is just processed in + if waysWrapper.is_node_in_any_polygon(processing_node): + is_complete = True + else: print("\n*********** IS COMPLETE **************\n") is_complete = True @@ -81,5 +88,5 @@ for node_osmid in processed_nodes: print("node(%s);" % node_osmid) -WaysWrapper(cur).save_to_database(processed_nodes) +waysWrapper.save_to_database(processed_nodes) conn.commit() diff --git a/way_inference/ways_wrapper.py b/way_inference/ways_wrapper.py index 2c6e68b..dedf4d0 100644 --- a/way_inference/ways_wrapper.py +++ b/way_inference/ways_wrapper.py @@ -1,9 +1,30 @@ import json class WaysWrapper: + bounds = [] + cur = None - def __init__(self, sql_cur): + def __init__(self, sql_cur, bounds): self.cur = sql_cur + self.bounds = bounds + + def is_node_in_any_polygon(self, node_osmid): + query = ''' + WITH node AS ( + SELECT geom FROM point WHERE properties->>'osmid' = %s + ) + SELECT powerline.id + FROM powerline, node + WHERE ST_Contains(ST_MakePolygon(powerline.geom), node.geom) + AND ST_IsClosed(powerline.geom) + LIMIT 1 + ''' + self.cur.execute(query, [str(node_osmid)]) + powerline = self.cur.fetchone() + if powerline is not None: + return True + else: + return False def save_to_database(self, nodes_osmids): nodes = [] From f3524eccc4c97387feddb2f320663835cc53e30c Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Wed, 7 Sep 2016 10:39:03 +0200 Subject: [PATCH 08/18] Recursively infer for a cluster if needed --- way_inference/cluster_wrapper.py | 1 + way_inference/infer_way.py | 100 ++++++++++++++++++------------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/way_inference/cluster_wrapper.py b/way_inference/cluster_wrapper.py index ffbe169..6e3cdcb 100644 --- a/way_inference/cluster_wrapper.py +++ b/way_inference/cluster_wrapper.py @@ -23,6 +23,7 @@ def _clusters_query(self): ST_MakeEnvelope(%s, %s, %s, %s), geom ) + AND properties->>'tags' IS NOT NULL LIMIT 1 ''' return clusters_query diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 2361bf3..6832f8b 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -30,63 +30,77 @@ clusters = clustersWrapper.getClusters() -for cluster in clusters: - nodes_osmids = nodesWrapper.get_nodes_osmids_in_cluster(cluster[0]) -print(nodes_osmids) - +def infer_way_from_nodes(nodes_osmids): -farthest_nodes = nodesWrapper.get_farthest_nodes_among_nodes(nodes_osmids) -print(farthest_nodes) + farthest_nodes = nodesWrapper.get_farthest_nodes_among_nodes(nodes_osmids) -# Start processing with one of the farthest nodes -processing_node = farthest_nodes[0] + # Start processing with one of the farthest nodes + processing_node = farthest_nodes[0] -processed_nodes = [] -ignored_nodes = [] # nodes are ignored if there are too close to a node + processed_nodes = [] + ignored_nodes = [] # nodes are ignored if there are too close to a node -processed_nodes.append(processing_node) + processed_nodes.append(processing_node) -is_complete = False + is_complete = False -while is_complete == False: - # procesed nodes minus the all nodes - unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) + while is_complete == False: + # procesed nodes minus the all nodes + unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) - # unprocessed nodes minus the ignored node. - unprocessed_nodes = tuple(set(unprocessed_nodes) ^ set(tuple(ignored_nodes))) + # unprocessed nodes minus the ignored node. + unprocessed_nodes = tuple(set(unprocessed_nodes) ^ set(tuple(ignored_nodes))) - closest_node = None + closest_node = None + if len(unprocessed_nodes) > 0: + nodes_around = nodesWrapper.get_closest_nodes_to( + processing_node, + unprocessed_nodes + ) - if len(unprocessed_nodes) > 0: - nodes_around = nodesWrapper.get_closest_nodes_to( - processing_node, - unprocessed_nodes - ) + ignored_nodes = ignored_nodes + nodes_around['too_close_node_osmids'] + closest_node = nodes_around['closest_node_osmid'] + else: + is_complete = True + continue + + if closest_node is not None: + print "Closest is - ", + print(closest_node) + processing_node = closest_node + processed_nodes.append(processing_node) + + # if the node that is just processed in any polygon. + if waysWrapper.is_node_in_any_polygon(processing_node): + is_complete = True + else: + print("\n*********** IS COMPLETE **************\n") + is_complete = True - ignored_nodes = ignored_nodes + nodes_around['too_close_node_osmids'] - closest_node = nodes_around['closest_node_osmid'] + if len(processed_nodes) < 1: + # This means, we couldn't find any close nodes. + # End the iteration of the cluster. + return else: - is_complete = True - continue + print(processed_nodes) + for node_osmid in processed_nodes: + print("node(%s);" % node_osmid) - if closest_node is not None: - print "Closest is - ", - print(closest_node) - processing_node = closest_node - processed_nodes.append(processing_node) + waysWrapper.save_to_database(processed_nodes) + conn.commit() - # if the node that is just processed in - if waysWrapper.is_node_in_any_polygon(processing_node): - is_complete = True + # procesed nodes minus the all nodes + unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) - else: - print("\n*********** IS COMPLETE **************\n") - is_complete = True + # unprocessed nodes minus the ignored node. + unprocessed_nodes = tuple(set(unprocessed_nodes) ^ set(tuple(ignored_nodes))) + + if len(unprocessed_nodes) > 1: + # There are more nodes to be processed in this cluster. + infer_way_from_nodes(unprocessed_nodes) -print(processed_nodes) -for node_osmid in processed_nodes: - print("node(%s);" % node_osmid) +for cluster in clusters: + nodes_osmids = nodesWrapper.get_nodes_osmids_in_cluster(cluster[0]) + infer_way_from_nodes(nodes_osmids) -waysWrapper.save_to_database(processed_nodes) -conn.commit() From fe7f7e993ac9bbaba3ab33e5577f3e291a612be4 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Wed, 7 Sep 2016 10:53:00 +0200 Subject: [PATCH 09/18] Fetch points that have 'power' anywhere in tags --- way_inference/cluster_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/way_inference/cluster_wrapper.py b/way_inference/cluster_wrapper.py index 6e3cdcb..eaee51c 100644 --- a/way_inference/cluster_wrapper.py +++ b/way_inference/cluster_wrapper.py @@ -23,7 +23,7 @@ def _clusters_query(self): ST_MakeEnvelope(%s, %s, %s, %s), geom ) - AND properties->>'tags' IS NOT NULL + AND properties->>'tags' LIKE '%power%' LIMIT 1 ''' return clusters_query From 1b4c49e2dd594aba74ddf35371c94ec9e1d1f0f9 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 8 Sep 2016 16:45:06 +0200 Subject: [PATCH 10/18] Inferrence parallel lines flagging and faster queries * Identify possible parallel lines by determining if 1/4th of points of the line are below a predefined distance. * Getting farthest points from a list of points has been very slow when there are a lot of points. Improved it by using generating a ConvexHull and using it for distance. * Skip inferring cluster if there is only one point. --- way_inference/cluster_wrapper.py | 8 ++-- way_inference/infer_way.py | 45 ++++++++++++++------ way_inference/nodes_wrapper.py | 73 ++++++++++++++++++++++++++------ way_inference/ways_wrapper.py | 7 ++- 4 files changed, 99 insertions(+), 34 deletions(-) diff --git a/way_inference/cluster_wrapper.py b/way_inference/cluster_wrapper.py index eaee51c..622ecd7 100644 --- a/way_inference/cluster_wrapper.py +++ b/way_inference/cluster_wrapper.py @@ -11,11 +11,12 @@ def getClusters(self): self._clusters_query(), self._clusters_query_args() ) + print(self._clusters_query_args()) return self.cur.fetchall() def _clusters_query(self): # TODO: Get only the points that are not part of a powerline - clusters_query = ''' + clusters_query = """ SELECT ST_AsText(unnest((ST_ClusterWithin(geom, 0.1)))) AS cluster_geom FROM point @@ -23,9 +24,8 @@ def _clusters_query(self): ST_MakeEnvelope(%s, %s, %s, %s), geom ) - AND properties->>'tags' LIKE '%power%' - LIMIT 1 - ''' + AND properties->>'tags' LIKE '%%power%%' + """ return clusters_query def _clusters_query_args(self): diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 6832f8b..9438eb5 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -22,7 +22,11 @@ '3212195882', '3212195889', '3212195895', '3212195893', '3212195896' ) # bounds = [10.39529800415039, 4.050234320898018, 10.50516128540039, 4.109221809610561] # parallel lines -bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa +# bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa + +bounds = [15.003890991210938, -4.800890838853971, 15.663070678710938, -4.137558228375503] # bigger africa +# bounds = [14.793434143066406, -4.934987879630612, 15.452613830566406, -4.603460449699286] +bounds = [11.2060546875, -7.242597510949338, 16.4794921875, -1.9387168550573113] nodesWrapper = NodesWrapper(cur, bounds) clustersWrapper = ClusterWrapper(cur, bounds) @@ -30,15 +34,18 @@ clusters = clustersWrapper.getClusters() -def infer_way_from_nodes(nodes_osmids): - - farthest_nodes = nodesWrapper.get_farthest_nodes_among_nodes(nodes_osmids) +def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): - # Start processing with one of the farthest nodes + if cluster_geom_text is not None: + farthest_nodes = nodesWrapper.get_farthest_nodes_in_cluster(cluster_geom_text) + else: + farthest_nodes = nodesWrapper.get_farthest_nodes_among_nodes(nodes_osmids) + print(farthest_nodes) processing_node = farthest_nodes[0] processed_nodes = [] ignored_nodes = [] # nodes are ignored if there are too close to a node + possible_parallel_line_nodes = [] processed_nodes.append(processing_node) @@ -60,14 +67,14 @@ def infer_way_from_nodes(nodes_osmids): ) ignored_nodes = ignored_nodes + nodes_around['too_close_node_osmids'] + possible_parallel_line_nodes = possible_parallel_line_nodes + nodes_around['possible_parallel_line_nodes'] closest_node = nodes_around['closest_node_osmid'] else: is_complete = True continue if closest_node is not None: - print "Closest is - ", - print(closest_node) + print ".", processing_node = closest_node processed_nodes.append(processing_node) @@ -83,11 +90,17 @@ def infer_way_from_nodes(nodes_osmids): # End the iteration of the cluster. return else: - print(processed_nodes) - for node_osmid in processed_nodes: - print("node(%s);" % node_osmid) - - waysWrapper.save_to_database(processed_nodes) + # print(processed_nodes) + # for node_osmid in processed_nodes: + # print("node(%s);" % node_osmid) + + inferrence_notes = {} + if len(possible_parallel_line_nodes) >= (len(processed_nodes)/4): + inferrence_notes = { + 'possible_error': True, + 'notes': 'Posisble paralle lines' + } + waysWrapper.save_to_database(processed_nodes, inferrence_notes) conn.commit() # procesed nodes minus the all nodes @@ -98,9 +111,13 @@ def infer_way_from_nodes(nodes_osmids): if len(unprocessed_nodes) > 1: # There are more nodes to be processed in this cluster. - infer_way_from_nodes(unprocessed_nodes) + infer_way_from_nodes(nodes_osmids=unprocessed_nodes) for cluster in clusters: + print("************ Processing New Cluster **************") nodes_osmids = nodesWrapper.get_nodes_osmids_in_cluster(cluster[0]) - infer_way_from_nodes(nodes_osmids) + if len(nodes_osmids) > 1: + infer_way_from_nodes(nodes_osmids, cluster[0]) + else: + print("Not enough nodes in cluster! - SKIPPING") diff --git a/way_inference/nodes_wrapper.py b/way_inference/nodes_wrapper.py index 7debb0c..5e258b9 100644 --- a/way_inference/nodes_wrapper.py +++ b/way_inference/nodes_wrapper.py @@ -3,6 +3,7 @@ class NodesWrapper: cur = None closest_min_distance = 0.0004 closest_max_distance = 0.09 + parallel_line_nodes_max_distance = 0.0006 def __init__(self, sql_cursor, bounds): self.bounds = bounds @@ -45,9 +46,14 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): closest_nodes = self.cur.fetchall() result = { 'too_close_node_osmids': [], + 'possible_parallel_line_nodes': [], 'closest_node_osmid': None } + for node in closest_nodes: + if (node[2] < self.parallel_line_nodes_max_distance): + result['possible_parallel_line_nodes'].append(node[1]) + for node in closest_nodes: if (node[2] < self.closest_min_distance): result['too_close_node_osmids'].append(node[1]) @@ -56,24 +62,67 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): break; return result; + def get_node_osmids_intersecting_polygons(self, among_osm_ids): + query = ''' + SELECT point.properties->>'osmid' + FROM point + JOIN powerline + ON ST_Contains(ST_MakePolygon(powerline.geom), point.geom) + AND ST_IsClosed(powerline.geom) + AND point.properties->>'osmid' IN %s + ''' + self.cur.execute(query, [tuple(among_osm_ids)]) + return self.cur.fetchall() + + def get_farthest_nodes_in_cluster(self, cluster_geom_text): + query = ''' + WITH farthest AS ( + SELECT ST_Distance(a.geom, b.geom) AS distance, + a.geom AS a_geom, b.geom AS b_geom FROM + (SELECT (ST_DumpPoints(ST_ConvexHull(ST_GeomFromText(%s)))).geom) a, + (SELECT (ST_DumpPoints(ST_ConvexHull(ST_GeomFromText(%s)))).geom) b + ORDER BY ST_Distance(a.geom, b.geom) DESC + LIMIT 1 + ) + SELECT a.properties->>'osmid', + b.properties->>'osmid', + farthest.distance + FROM point a, point b, farthest + WHERE a.geom = farthest.a_geom + AND b.geom = farthest.b_geom + LIMIT 1; + ''' + self.cur.execute(query, [cluster_geom_text, cluster_geom_text]) + return self.cur.fetchone() + def get_farthest_nodes_among_nodes(self, among_osm_ids): - farthest_nodes_query = ''' - WITH point_ids AS ( - SELECT id FROM point - WHERE (point.properties->>'osmid') IN %s + # Using ST_ConvexHull for finding farthest nodes seems to be + # optimal. + # Refer: http://gis.stackexchange.com/a/25078/80804 + query = ''' + WITH farthest AS ( + SELECT ST_Distance(a.geom, b.geom) AS distance, + a.geom AS a_geom, b.geom AS b_geom FROM + (SELECT (ST_DumpPoints(ST_ConvexHull(ST_Collect(geom)))).geom FROM point WHERE properties->>'osmid' IN %s) a, + (SELECT (ST_DumpPoints(ST_ConvexHull(ST_Collect(geom)))).geom FROM point WHERE properties->>'osmid' IN %s) b + ORDER BY ST_Distance(a.geom, b.geom) DESC + LIMIT 1 ) SELECT a.properties->>'osmid', b.properties->>'osmid', - st_distance(a.geom, b.geom) - FROM point a, point b - WHERE a.id IN (SELECT id FROM point_ids) - AND b.id IN (SELECT id FROM point_ids) - AND a.id != b.id - ORDER BY st_distance(a.geom, b.geom) DESC + farthest.distance + FROM point a, point b, farthest + WHERE a.geom = farthest.a_geom + AND b.geom = farthest.b_geom + AND a.properties->>'osmid' IN %s + AND b.properties->>'osmid' IN %s LIMIT 1; ''' - self.cur.execute(farthest_nodes_query, [ + self.cur.execute(query, [ + tuple(among_osm_ids), + tuple(among_osm_ids), + tuple(among_osm_ids), tuple(among_osm_ids) ]) return self.cur.fetchone() @@ -85,7 +134,7 @@ def _node_osmids_in_cluster_query(self): ST_CollectionExtract(%s, 1), point.geom ) - AND properties->'tags'->>'power'='tower' + AND properties->>'tags' LIKE '%%power%%' AND ST_Intersects( ST_MakeEnvelope(%s, %s, %s, %s), point.geom diff --git a/way_inference/ways_wrapper.py b/way_inference/ways_wrapper.py index dedf4d0..5bfc165 100644 --- a/way_inference/ways_wrapper.py +++ b/way_inference/ways_wrapper.py @@ -26,7 +26,7 @@ def is_node_in_any_polygon(self, node_osmid): else: return False - def save_to_database(self, nodes_osmids): + def save_to_database(self, nodes_osmids, inferrence_notes): nodes = [] flag = False for node_osmid in nodes_osmids: @@ -45,15 +45,14 @@ def save_to_database(self, nodes_osmids): return None linestring = "" - print(nodes) for node in nodes: linestring += "{} {},".format(node[0], node[1]) linestring = linestring[:-1] - print("INSERTING {}".format(linestring)) + print("Saving powerline to database") query = "INSERT INTO inferred_powerlines(geom, properties) VALUES(%s, %s)" self.cur.execute(query, [ 'LINESTRING({})'.format(linestring), - json.dumps({ "tags": {}, "refs": nodes_osmids }) + json.dumps({ "tags": {"inferrence": inferrence_notes}, "refs": nodes_osmids, }) ]) From 562f8f52b5b6319b8736c71bb07ff1294cc66156 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Wed, 14 Sep 2016 18:28:46 +0200 Subject: [PATCH 11/18] Ability to infer ways considering angle of deviation variable to be set to True to consider angle of deviation. NodesWrapper calculates the angles of deviation for those points that are within a distance (a separate variable to be used just when angles are requested). --- way_inference/infer_way.py | 36 +++++++++++++---- way_inference/nodes_wrapper.py | 70 ++++++++++++++++++++++++++-------- 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 9438eb5..72f9ac5 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -24,9 +24,13 @@ # bounds = [10.39529800415039, 4.050234320898018, 10.50516128540039, 4.109221809610561] # parallel lines # bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa -bounds = [15.003890991210938, -4.800890838853971, 15.663070678710938, -4.137558228375503] # bigger africa -# bounds = [14.793434143066406, -4.934987879630612, 15.452613830566406, -4.603460449699286] -bounds = [11.2060546875, -7.242597510949338, 16.4794921875, -1.9387168550573113] +# bounds = [15.003890991210938, -4.800890838853971, 15.663070678710938, -4.137558228375503] # bigger africa +# bounds = [15.232973098754881, -4.554179759718965, 15.342836380004883, -4.495226724658142] +bounds = [15.070838928222654, -4.89223025116464, 15.510292053222656, -4.656501822101158] +# bounds = [15.22255539894104, -4.743145262934742, 15.23628830909729, -4.735778370815115] + +# settings +is_less_deviating_point_preferred = False nodesWrapper = NodesWrapper(cur, bounds) clustersWrapper = ClusterWrapper(cur, bounds) @@ -60,15 +64,33 @@ def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): closest_node = None + if is_less_deviating_point_preferred and len(processed_nodes) > 1 and processing_node != processed_nodes[-2]: + last_processed_node = processed_nodes[-2] + else: + last_processed_node = None + if len(unprocessed_nodes) > 0: nodes_around = nodesWrapper.get_closest_nodes_to( processing_node, - unprocessed_nodes + unprocessed_nodes, + last_processed_node ) ignored_nodes = ignored_nodes + nodes_around['too_close_node_osmids'] possible_parallel_line_nodes = possible_parallel_line_nodes + nodes_around['possible_parallel_line_nodes'] - closest_node = nodes_around['closest_node_osmid'] + + closest_node = None + + if is_less_deviating_point_preferred == True and len(nodes_around['angles']) > 0: + sorted_nodes = sorted(nodes_around['angles'].iteritems(), key=lambda (k,v): (v,k)) + print "From ", + print str(processing_node), + print " -> ", + print(sorted_nodes) + closest_node = sorted_nodes[0][0] + + if closest_node is None: + closest_node = nodes_around['closest_node_osmid'] else: is_complete = True continue @@ -91,8 +113,8 @@ def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): return else: # print(processed_nodes) - # for node_osmid in processed_nodes: - # print("node(%s);" % node_osmid) + for node_osmid in processed_nodes: + print("node(%s);" % node_osmid) inferrence_notes = {} if len(possible_parallel_line_nodes) >= (len(processed_nodes)/4): diff --git a/way_inference/nodes_wrapper.py b/way_inference/nodes_wrapper.py index 5e258b9..b04405e 100644 --- a/way_inference/nodes_wrapper.py +++ b/way_inference/nodes_wrapper.py @@ -3,6 +3,7 @@ class NodesWrapper: cur = None closest_min_distance = 0.0004 closest_max_distance = 0.09 + closest_max_distance_for_angles = 0.0055 # tells to get angles for those only within that distance. parallel_line_nodes_max_distance = 0.0006 def __init__(self, sql_cursor, bounds): @@ -10,7 +11,19 @@ def __init__(self, sql_cursor, bounds): self.cur = sql_cursor def get_nodes_osmids_in_cluster(self, cluster_geom_text): - self.cur.execute(self._node_osmids_in_cluster_query(), [ + query = ''' + SELECT DISTINCT properties->>'osmid' FROM point + WHERE ST_Intersects( + ST_CollectionExtract(%s, 1), + point.geom + ) + AND properties->>'tags' LIKE '%%power%%' + AND ST_Intersects( + ST_MakeEnvelope(%s, %s, %s, %s), + point.geom + ); + ''' + self.cur.execute(query, [ cluster_geom_text, self.bounds[1], self.bounds[0], @@ -22,7 +35,8 @@ def get_nodes_osmids_in_cluster(self, cluster_geom_text): def tmp(x): return x[0] return list(map(tmp, node_osmids_tuple)) - def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): + def get_closest_nodes_to(self, for_node_osmid, among_osm_ids, + previous_node_osmid=None): fetch_closest_query = ''' WITH current_point AS ( SELECT id, ST_AsText(geom) AS geom FROM point @@ -47,19 +61,26 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids): result = { 'too_close_node_osmids': [], 'possible_parallel_line_nodes': [], - 'closest_node_osmid': None + 'closest_node_osmid': None, + 'angles': {} } for node in closest_nodes: if (node[2] < self.parallel_line_nodes_max_distance): result['possible_parallel_line_nodes'].append(node[1]) + if (previous_node_osmid is not None) and (node[2] <= self.closest_max_distance_for_angles): + # Case to include angles formed. + angle = self.get_deviation_angle(previous_node_osmid, for_node_osmid, node[1]) + result['angles'][node[1]] = angle + for node in closest_nodes: if (node[2] < self.closest_min_distance): result['too_close_node_osmids'].append(node[1]) else: result['closest_node_osmid'] = node[1] break; + return result; def get_node_osmids_intersecting_polygons(self, among_osm_ids): @@ -127,16 +148,35 @@ def get_farthest_nodes_among_nodes(self, among_osm_ids): ]) return self.cur.fetchone() - def _node_osmids_in_cluster_query(self): - return ''' - SELECT DISTINCT properties->>'osmid' FROM point - WHERE ST_Intersects( - ST_CollectionExtract(%s, 1), - point.geom - ) - AND properties->>'tags' LIKE '%%power%%' - AND ST_Intersects( - ST_MakeEnvelope(%s, %s, %s, %s), - point.geom - ); + def get_deviation_angle(self, point1, intersection, point2): + # Returns the angle at which the line joining +intersection+ point + # with +point2+ devitates the line formed by joining point1 to + # intersection + query = ''' + WITH + point1 AS (SELECT geom FROM point WHERE properties->>'osmid' = %s), + intersection AS (SELECT geom FROM point WHERE properties->>'osmid' = %s), + point2 AS (SELECT geom FROM point WHERE properties->>'osmid' = %s) + + SELECT deg_i21i, CASE WHEN deg_i21i > 180 THEN (360 - deg_i21i) + ELSE deg_i21i END + FROM ( + SELECT + abs(round(degrees( + ST_Azimuth( + ST_Point(ST_y(intersection.geom), ST_x(intersection.geom)), + ST_Point(ST_y(point2.geom), ST_x(point2.geom)) + ) - + ST_Azimuth ( + ST_Point(ST_y(point1.geom), ST_x(point1.geom)), + ST_Point(ST_y(intersection.geom), ST_x(intersection.geom)) + ) + )::decimal, 2)) + AS deg_i21i + FROM point1, point2, intersection + ) s ''' + self.cur.execute(query, [str(point1), str(intersection), str(point2)]) + row = self.cur.fetchone() + return row[0] + From 11938fa87be87fb2736beea7c024c9c828cd4e77 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 15 Sep 2016 13:10:04 +0200 Subject: [PATCH 12/18] Better angle inferrence by ceiling angles We ceil the angle to a closest multiple of 5 and also round up the distance to 3 decimal points. This way, we will sort based on the angle and then by distance. --- way_inference/infer_way.py | 32 ++++++++++++++++++-------------- way_inference/nodes_wrapper.py | 18 ++++++++++++++---- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 72f9ac5..29355bc 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -26,11 +26,13 @@ # bounds = [15.003890991210938, -4.800890838853971, 15.663070678710938, -4.137558228375503] # bigger africa # bounds = [15.232973098754881, -4.554179759718965, 15.342836380004883, -4.495226724658142] -bounds = [15.070838928222654, -4.89223025116464, 15.510292053222656, -4.656501822101158] +# bounds = [15.070838928222654, -4.89223025116464, 15.510292053222656, -4.656501822101158] # bounds = [15.22255539894104, -4.743145262934742, 15.23628830909729, -4.735778370815115] +# bounds = [14.515686035156248, -5.103254918829327, 16.27349853515625, -4.160158150193397] # Africa Mbanza +bounds = [10.853462219238281, 49.238000036465465, 11.292915344238281, 49.392206057422044] # settings -is_less_deviating_point_preferred = False +is_less_deviating_point_preferred = True nodesWrapper = NodesWrapper(cur, bounds) clustersWrapper = ClusterWrapper(cur, bounds) @@ -57,10 +59,10 @@ def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): while is_complete == False: # procesed nodes minus the all nodes - unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) + unprocessed_nodes = tuple(set(nodes_osmids) - set(tuple(processed_nodes))) # unprocessed nodes minus the ignored node. - unprocessed_nodes = tuple(set(unprocessed_nodes) ^ set(tuple(ignored_nodes))) + unprocessed_nodes = tuple(set(unprocessed_nodes) - set(tuple(ignored_nodes))) closest_node = None @@ -82,12 +84,15 @@ def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): closest_node = None if is_less_deviating_point_preferred == True and len(nodes_around['angles']) > 0: - sorted_nodes = sorted(nodes_around['angles'].iteritems(), key=lambda (k,v): (v,k)) - print "From ", - print str(processing_node), - print " -> ", - print(sorted_nodes) - closest_node = sorted_nodes[0][0] + sorted_nodes = sorted( + nodes_around['angles'], + key=lambda x: (x[2], x[1]) + ) # sort by angle and then distance + + print("From -> %s: %s" % (processing_node, sorted_nodes)) + + if sorted_nodes[0][2] is not None: # angle could sometimes be none. + closest_node = sorted_nodes[0][0] if closest_node is None: closest_node = nodes_around['closest_node_osmid'] @@ -125,11 +130,10 @@ def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): waysWrapper.save_to_database(processed_nodes, inferrence_notes) conn.commit() - # procesed nodes minus the all nodes - unprocessed_nodes = tuple(set(tuple(processed_nodes)) ^ set(nodes_osmids)) - + # all nodes minus the processed nodes + unprocessed_nodes = tuple(set(nodes_osmids) - set(tuple(processed_nodes))) # unprocessed nodes minus the ignored node. - unprocessed_nodes = tuple(set(unprocessed_nodes) ^ set(tuple(ignored_nodes))) + unprocessed_nodes = tuple(set(unprocessed_nodes) - set(tuple(ignored_nodes))) if len(unprocessed_nodes) > 1: # There are more nodes to be processed in this cluster. diff --git a/way_inference/nodes_wrapper.py b/way_inference/nodes_wrapper.py index b04405e..bc8798e 100644 --- a/way_inference/nodes_wrapper.py +++ b/way_inference/nodes_wrapper.py @@ -1,10 +1,14 @@ +import math +from decimal import * + class NodesWrapper: bounds = [] cur = None closest_min_distance = 0.0004 closest_max_distance = 0.09 - closest_max_distance_for_angles = 0.0055 # tells to get angles for those only within that distance. + closest_max_distance_for_angles = 0.008# 0.0055 # tells to get angles for those only within that distance. parallel_line_nodes_max_distance = 0.0006 + angle_ceil_multiplier = 5 def __init__(self, sql_cursor, bounds): self.bounds = bounds @@ -49,6 +53,7 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids, ) as distance FROM point, current_point WHERE ST_Distance(ST_GeomFromText(current_point.geom), point.geom) < %s + AND ST_Distance(ST_GeomFromText(current_point.geom), point.geom) > 0 AND properties->>'osmid' IN %s ORDER BY ST_Distance(ST_GeomFromText(current_point.geom), point.geom) ASC ''' @@ -62,7 +67,7 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids, 'too_close_node_osmids': [], 'possible_parallel_line_nodes': [], 'closest_node_osmid': None, - 'angles': {} + 'angles': [] # contains tuple with (osmid, roundedup distance, ceiled angle) } for node in closest_nodes: @@ -72,7 +77,12 @@ def get_closest_nodes_to(self, for_node_osmid, among_osm_ids, if (previous_node_osmid is not None) and (node[2] <= self.closest_max_distance_for_angles): # Case to include angles formed. angle = self.get_deviation_angle(previous_node_osmid, for_node_osmid, node[1]) - result['angles'][node[1]] = angle + if angle is not None: + angle = (int(math.ceil(angle/ Decimal(self.angle_ceil_multiplier))) * self.angle_ceil_multiplier) + rounded_up_distance = math.ceil(node[2] * 1000)/1000 + node_dist_angle_tuple = (node[1], rounded_up_distance, angle) + result['angles'].append(node_dist_angle_tuple) + for node in closest_nodes: if (node[2] < self.closest_min_distance): @@ -158,7 +168,7 @@ def get_deviation_angle(self, point1, intersection, point2): intersection AS (SELECT geom FROM point WHERE properties->>'osmid' = %s), point2 AS (SELECT geom FROM point WHERE properties->>'osmid' = %s) - SELECT deg_i21i, CASE WHEN deg_i21i > 180 THEN (360 - deg_i21i) + SELECT CASE WHEN deg_i21i > 180 THEN (360 - deg_i21i) ELSE deg_i21i END FROM ( SELECT From b52bad496224fe4d38ffe4af543ade13c1331498 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 15 Sep 2016 14:48:40 +0200 Subject: [PATCH 13/18] Basic readme for way inferrence --- way_inference/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 way_inference/README.md diff --git a/way_inference/README.md b/way_inference/README.md new file mode 100644 index 0000000..8151843 --- /dev/null +++ b/way_inference/README.md @@ -0,0 +1,40 @@ +## Way Inference + +### How does it work? + +The idea was to take basic simple methods and make them work in co-ordination and show methods for way inference covering possible scenarios. +> These scripts at the moment can't perform very good inferrence. This is more like, prooving the possiblity. + + +#### Basic script. + +Takes in boundaries to run inferrence on that region of the map. +Gets clusters of point (discussed [here](#clustering)) and loops over each cluster. Each cluster is possibly a powerline. +For each cluster +* Start inferrence with a point the cluster. +* Fetch the database and choose a possible point. + * Here we have different options on how to choose a next point. Some are even ignored during this process (discussed later in this document). +* Once we have gone though all the points, we will save the points as a line into the database. + +As simple as that! + +##### Clustering + +Everything basically starts with [clustering](#clustering) in the way inference. +Given a boundary, we take clusters of points in that region from our database. +We take the points that are not marked as substations or stations or something similar. +We cluster points based on the distance between them. +For powerlines (at least) for those we had in the database, we calculated (using `../estimate_power_pole_distance.py` script) the average maximum distance between two points in a powerline is around 0.1 in postGIS distance units (projected units based on spatial ref - [See more](http://postgis.net/docs/ST_Distance.html)). +So, we cluster points that are in that distance of each other (at least with another point in that cluster). +You could see the `ClusterWrapper` for the code related to this. + +##### Choosing a point to start with +Lorem ipsum dolor sit amet, pro stet iisque commune no, vix tibique inimicus ut. Quando mandamus eu nec. Sed facilisi vituperata ea. Sea mundi choro dicam ad. + +Ludus integre menandri eu sea, ex pri dicam electram. Nibh homero ocurreret in ius. Et sit alia feugait noluisse, viderer dolorem voluptatum eum in. Vis purto noster virtute te, dicit populo ad nec, sed in purto dicta liber. Vel tale ullum oratio ea, ius luptatum vivendum quaerendum an. + +Sententiae delicatissimi ex vis, ea eos ferri torquatos appellantur. Lorem nostro philosophia cu sit, ne odio meliore vivendum mei, vidit impetus docendi ut his. Eu mel vitae congue pericula. Nam te cetero luptatum, no duo vero deterruisset. Vel dicat partiendo dissentiunt te. + +In quot dicunt maiorum mel. Nominavi sensibus sit in, per purto feugait forensibus ea. Mel ne agam mutat molestie, cum probo ipsum accusamus ut. Soluta instructior in per, per ad causae ocurreret patrioque. + +Nec vero iudicabit reprehendunt ne, has dolorum civibus electram et. His ea munere volumus accommodare, affert neglegentur te sit. Ad rebum consequat mel, et alii tritani per. Ne sit detracto mediocrem suscipiantur, civibus placerat consectetuer quo ei. Te everti persecuti eum, vis ne mutat errem. Pro tritani rationibus cu, virtute concludaturque mel an, et sit tota harum recusabo. Saepe maiorum atomorum qui eu. From 03245e788c7cf893b4ce1d1f9be605fd76a6b415 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 15 Sep 2016 15:07:01 +0200 Subject: [PATCH 14/18] README on choosing a starting point --- way_inference/README.md | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/way_inference/README.md b/way_inference/README.md index 8151843..fa64ca4 100644 --- a/way_inference/README.md +++ b/way_inference/README.md @@ -1,12 +1,12 @@ -## Way Inference +# Way Inference -### How does it work? +# How does it work? The idea was to take basic simple methods and make them work in co-ordination and show methods for way inference covering possible scenarios. > These scripts at the moment can't perform very good inferrence. This is more like, prooving the possiblity. -#### Basic script. +## Basic script Takes in boundaries to run inferrence on that region of the map. Gets clusters of point (discussed [here](#clustering)) and loops over each cluster. Each cluster is possibly a powerline. @@ -15,10 +15,12 @@ For each cluster * Fetch the database and choose a possible point. * Here we have different options on how to choose a next point. Some are even ignored during this process (discussed later in this document). * Once we have gone though all the points, we will save the points as a line into the database. +* Alternative to the above point, if we encounter a point that meets a polygon (building), we assume its the end of the line and save the line to database. +We will continue freshly with the remaining point from the cluster as new a new cluster. As simple as that! -##### Clustering +## Clustering Everything basically starts with [clustering](#clustering) in the way inference. Given a boundary, we take clusters of points in that region from our database. @@ -28,13 +30,26 @@ For powerlines (at least) for those we had in the database, we calculated (using So, we cluster points that are in that distance of each other (at least with another point in that cluster). You could see the `ClusterWrapper` for the code related to this. -##### Choosing a point to start with -Lorem ipsum dolor sit amet, pro stet iisque commune no, vix tibique inimicus ut. Quando mandamus eu nec. Sed facilisi vituperata ea. Sea mundi choro dicam ad. +## Choosing a point to start with -Ludus integre menandri eu sea, ex pri dicam electram. Nibh homero ocurreret in ius. Et sit alia feugait noluisse, viderer dolorem voluptatum eum in. Vis purto noster virtute te, dicit populo ad nec, sed in purto dicta liber. Vel tale ullum oratio ea, ius luptatum vivendum quaerendum an. +Within a cluster, we choose the points based on different methods. -Sententiae delicatissimi ex vis, ea eos ferri torquatos appellantur. Lorem nostro philosophia cu sit, ne odio meliore vivendum mei, vidit impetus docendi ut his. Eu mel vitae congue pericula. Nam te cetero luptatum, no duo vero deterruisset. Vel dicat partiendo dissentiunt te. +### Farthest points -In quot dicunt maiorum mel. Nominavi sensibus sit in, per purto feugait forensibus ea. Mel ne agam mutat molestie, cum probo ipsum accusamus ut. Soluta instructior in per, per ad causae ocurreret patrioque. +By default, we choose one of the two farthest points. +The distances are calculated and farthest points are fetched from SQL for performance leveraging the [postGIS](http://postgis.net) functions. +The script then takes one of those points as a starting point. -Nec vero iudicabit reprehendunt ne, has dolorum civibus electram et. His ea munere volumus accommodare, affert neglegentur te sit. Ad rebum consequat mel, et alii tritani per. Ne sit detracto mediocrem suscipiantur, civibus placerat consectetuer quo ei. Te everti persecuti eum, vis ne mutat errem. Pro tritani rationibus cu, virtute concludaturque mel an, et sit tota harum recusabo. Saepe maiorum atomorum qui eu. +### Points on a substation +**Optional** + +Some polygons are tagged with `power: substation` in the OpenStreetMaps data to mark substations. +If the option to select such points in the script, the script fetches points (from the cluster) that intersects/within such polygons. +This again uses the postGIS functions and such points are fecthed using SQL queries. +The script then takes one of those points as a starting point. +If no such points are found, falls back to using **Farthest points** + + +# Requirements +* Postgres with [postGIS](http://postgis.net) Version 2.2 +* Python 2.7 From 5d8b1db3ec62aa4b96405a9230fd5b2523c6ef49 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 15 Sep 2016 16:32:59 +0200 Subject: [PATCH 15/18] README with how to choose next point --- way_inference/README.md | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/way_inference/README.md b/way_inference/README.md index fa64ca4..92dfcac 100644 --- a/way_inference/README.md +++ b/way_inference/README.md @@ -49,6 +49,54 @@ This again uses the postGIS functions and such points are fecthed using SQL quer The script then takes one of those points as a starting point. If no such points are found, falls back to using **Farthest points** +## Choosing a possible next point + +This is one of the critical steps which would make a difference on how the lines are inferred. +A possible next point from a point is done based on different variables. +We have tested these variables seperately. They cannot all work together. +However, a few of them can work together. + +### Closest point in a ring + +By default, we choose the next closest point in a ring. +Its highly possible that a point has points that are too close (in case of parallelly running powerlines). +We don't want to join those _too-close_ points. +Similarly, there are always points around a point that are too far. +We used the `../estimate_power_pole_distance.py` script to find the average minimum and maximum distance between adjacent points among the powerlines in our database. +Using these minimum and maximum distance, we set the variables in the `NodeWrapper` and form an SQL query to fetch points within the ring with inner and outer radii as minimum and maximum distances respectively. +From these closest points we take the most closest point. + +**Issues:** +* Crossing powerlines: There usually are powerlines crossing each other and this method might fail by picking a point in supposed to be the other powerline. +* Lines joining at a substations: Powerlines usually join at a substation and this method might fail by picking points from other powerlines, similar to the _Crossing powerlines_ issue. +* Parallel lines: Powerlines tend to run parallel for quite some distances from a substation, such parallel lines are failed to be inferred correctly. It forms a zig-zag pattern by joining points from the possible two line alternatively. + +**Works best for:** +* Lone powerlines: Powerlines that run idependantly without any intersections and parallel neighbours. + +### Point causing less angle of deviation + +Not default. Set by setting the variable `is_less_deviating_point_preferred` to `True`. + +While building a line, this method will choose a point that will deviate a line with a lesser angle than all other points that are around it within a maximum distance of 0.008 units. +This maximum distance is needed so that, too far away nodes are not taken into consideration as quite often the farthest nodes create less deviation angle. +From a given point, we gather the possible angle formed by all the points within the maximum distance. +We will then ceil these angles to the closest 5th multiple and round up the distances upto third decimal point. +We sort these nodes by angle and then by distance so that all the nodes that are forming noticeable deviation (5 degree angle) form a group. +We will then choose the closest node from the group that forms the least deviation. + +The varibales that control angles' ceiling multiple, distance roundup decimal point and maximum distance for point to be considered in angle deviation calculation are within the `NodesWrapper` class. + +**Issues:** +* Lines joining at a substation: Such lines usually take sharp turns at substations but points from neighbouring lines might show less deviations. + +**Works better for:** +* Parallel lines +* Crossing powerlines + + +## Varibles that can work together +* Farthest points, Points on a substation, Next Closest within the ring, Point causing less deviation. # Requirements * Postgres with [postGIS](http://postgis.net) Version 2.2 From 46cd50cd90ac26e753f3f383ec313f167a56cfc5 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 15 Sep 2016 16:35:51 +0200 Subject: [PATCH 16/18] inferrence REAEDME updated --- way_inference/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/way_inference/README.md b/way_inference/README.md index 92dfcac..0f78e6d 100644 --- a/way_inference/README.md +++ b/way_inference/README.md @@ -44,6 +44,7 @@ The script then takes one of those points as a starting point. **Optional** Some polygons are tagged with `power: substation` in the OpenStreetMaps data to mark substations. +Also, lines that are taggest with `power: line` but are closed are also usually substations. If the option to select such points in the script, the script fetches points (from the cluster) that intersects/within such polygons. This again uses the postGIS functions and such points are fecthed using SQL queries. The script then takes one of those points as a starting point. From c29a885ddc9b2e5fccde18199dfae21619e027e9 Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Thu, 15 Sep 2016 16:44:58 +0200 Subject: [PATCH 17/18] way inferrence README links updated --- way_inference/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/way_inference/README.md b/way_inference/README.md index 0f78e6d..f646685 100644 --- a/way_inference/README.md +++ b/way_inference/README.md @@ -11,9 +11,9 @@ The idea was to take basic simple methods and make them work in co-ordination an Takes in boundaries to run inferrence on that region of the map. Gets clusters of point (discussed [here](#clustering)) and loops over each cluster. Each cluster is possibly a powerline. For each cluster -* Start inferrence with a point the cluster. -* Fetch the database and choose a possible point. - * Here we have different options on how to choose a next point. Some are even ignored during this process (discussed later in this document). +* Start inferrence with a point the cluster. How we pick a point to start with is discussed [Here](#choosing-a-point-to-start-with) +* Fetch the database and choose a [possible next point](#choosing-a-possible-next-point). + * Here we have different options on how to choose a next point. Some are even ignored during this process (discussed [here](#choosing-a-possible-next-point)). * Once we have gone though all the points, we will save the points as a line into the database. * Alternative to the above point, if we encounter a point that meets a polygon (building), we assume its the end of the line and save the line to database. We will continue freshly with the remaining point from the cluster as new a new cluster. @@ -22,7 +22,7 @@ As simple as that! ## Clustering -Everything basically starts with [clustering](#clustering) in the way inference. +Everything basically starts with clustering. Given a boundary, we take clusters of points in that region from our database. We take the points that are not marked as substations or stations or something similar. We cluster points based on the distance between them. From e30c88ebe97630d3caf05e4174fde7a2714db2ef Mon Sep 17 00:00:00 2001 From: Sri Vishnnu Totakura Date: Fri, 16 Sep 2016 15:50:17 +0200 Subject: [PATCH 18/18] Inference can pick start point intersection a polygon Polygon in sense, a powerstation (substation etc.) --- way_inference/infer_way.py | 33 +++++++++++++++++++++++---------- way_inference/nodes_wrapper.py | 12 +++++++++++- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/way_inference/infer_way.py b/way_inference/infer_way.py index 29355bc..c473ac0 100644 --- a/way_inference/infer_way.py +++ b/way_inference/infer_way.py @@ -22,17 +22,18 @@ '3212195882', '3212195889', '3212195895', '3212195893', '3212195896' ) # bounds = [10.39529800415039, 4.050234320898018, 10.50516128540039, 4.109221809610561] # parallel lines -# bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa +bounds = [15.496902465820312, -1.4843615162701949, 16.375808715820312, -1.0113763068489454] # the short line in the middle of africa # bounds = [15.003890991210938, -4.800890838853971, 15.663070678710938, -4.137558228375503] # bigger africa # bounds = [15.232973098754881, -4.554179759718965, 15.342836380004883, -4.495226724658142] # bounds = [15.070838928222654, -4.89223025116464, 15.510292053222656, -4.656501822101158] # bounds = [15.22255539894104, -4.743145262934742, 15.23628830909729, -4.735778370815115] -# bounds = [14.515686035156248, -5.103254918829327, 16.27349853515625, -4.160158150193397] # Africa Mbanza -bounds = [10.853462219238281, 49.238000036465465, 11.292915344238281, 49.392206057422044] +bounds = [14.515686035156248, -5.103254918829327, 16.27349853515625, -4.160158150193397] # Africa Mbanza +# bounds = [10.853462219238281, 49.238000036465465, 11.292915344238281, 49.392206057422044] # Nuremburg # settings is_less_deviating_point_preferred = True +prefer_nodes_from_substations = True nodesWrapper = NodesWrapper(cur, bounds) clustersWrapper = ClusterWrapper(cur, bounds) @@ -42,12 +43,20 @@ def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): - if cluster_geom_text is not None: - farthest_nodes = nodesWrapper.get_farthest_nodes_in_cluster(cluster_geom_text) - else: - farthest_nodes = nodesWrapper.get_farthest_nodes_among_nodes(nodes_osmids) - print(farthest_nodes) - processing_node = farthest_nodes[0] + processing_node = None + + if prefer_nodes_from_substations == True: + nodes_on_polygons = nodesWrapper.get_node_osmids_intersecting_polygons(nodes_osmids) + if len(nodes_on_polygons) > 0: + processing_node = nodes_on_polygons[0][0] + + if processing_node is None: + if cluster_geom_text is not None: + farthest_nodes = nodesWrapper.get_farthest_nodes_in_cluster(cluster_geom_text) + else: + farthest_nodes = nodesWrapper.get_farthest_nodes_among_nodes(nodes_osmids) + + processing_node = farthest_nodes[0] processed_nodes = [] ignored_nodes = [] # nodes are ignored if there are too close to a node @@ -143,7 +152,11 @@ def infer_way_from_nodes(nodes_osmids, cluster_geom_text=None): print("************ Processing New Cluster **************") nodes_osmids = nodesWrapper.get_nodes_osmids_in_cluster(cluster[0]) if len(nodes_osmids) > 1: - infer_way_from_nodes(nodes_osmids, cluster[0]) + if prefer_nodes_from_substations: + infer_way_from_nodes(nodes_osmids) + else: + infer_way_from_nodes(nodes_osmids, cluster[0]) + else: print("Not enough nodes in cluster! - SKIPPING") diff --git a/way_inference/nodes_wrapper.py b/way_inference/nodes_wrapper.py index bc8798e..c7864cb 100644 --- a/way_inference/nodes_wrapper.py +++ b/way_inference/nodes_wrapper.py @@ -101,8 +101,18 @@ def get_node_osmids_intersecting_polygons(self, among_osm_ids): ON ST_Contains(ST_MakePolygon(powerline.geom), point.geom) AND ST_IsClosed(powerline.geom) AND point.properties->>'osmid' IN %s + AND ST_Intersects( + ST_MakeEnvelope(%s, %s, %s, %s), + powerline.geom + ) ''' - self.cur.execute(query, [tuple(among_osm_ids)]) + self.cur.execute(query, [ + tuple(among_osm_ids), + self.bounds[1], + self.bounds[0], + self.bounds[3], + self.bounds[2] + ]) return self.cur.fetchall() def get_farthest_nodes_in_cluster(self, cluster_geom_text):