Dev 5 1 2015 knn

common/thrift/CatalogObjects.thrift

@@ -32,6 +32,7 @@ enum TCatalogObjectType {
   TABLE,
   VIEW,
   FUNCTION,
+  GLOBAL_INDEX,
   DATA_SOURCE,
   ROLE,
   PRIVILEGE,
@@ -265,6 +266,41 @@ struct TDataSourceTable {
   2: required string init_string
 }
 
+// Represents a point
+struct TPoint {
+  1: required double x
+  2: required double y
+}
+
+// Represents a rectangle used in a global index record for spatial tables.
+struct TRectangle {
+  1: required double x1
+  2: required double y1
+  3: required double x2
+  4: required double y2
+}
+
+// Represents a global index record for spatial tables.
+struct TGlobalIndexRecord {
+  // Id of the record.
+  1: required i32 id
+
+  // Tag of the record.
+  2: required string tag
+
+  // MBR of the record.
+  3: required TRectangle mbr
+}
+
+// Represents a global index used for spatial tables.
+struct TGlobalIndex {
+  // Name of the table that this global index belongs to.
+  1: required string tbl_name
+
+  // Map of global indexes' records.
+  2: required map<string, TGlobalIndexRecord> globalIndexMap
+}
+
 // Represents a table or view.
 struct TTable {
   // Name of the parent database. Case insensitive, expected to be stored as lowercase.
@@ -309,6 +345,9 @@ struct TTable {
 
   // Set iff this is a table from an external data source
   13: optional TDataSourceTable data_source_table
+
+  // Set iff this table is spatial with initialized global index.
+  14: optional TGlobalIndex globalIndex;
 }
 
 // Represents a database.

fe/src/main/cup/sql-parser.y

@@ -14,6 +14,11 @@
 
 package com.cloudera.impala.analysis;
 
+import org.gistic.spatialImpala.catalog.Point;
+import org.gistic.spatialImpala.catalog.Rectangle;
+import org.gistic.spatialImpala.analysis.SpatialPointInclusionStmt;
+import org.gistic.spatialImpala.analysis.SpatialKnnStmt;
+
 import com.cloudera.impala.catalog.Type;
 import com.cloudera.impala.catalog.ScalarType;
 import com.cloudera.impala.catalog.ArrayType;
@@ -239,19 +244,19 @@ terminal
   KW_FIRST, KW_FLOAT, KW_FOLLOWING, KW_FOR, KW_FORMAT, KW_FORMATTED, KW_FROM, KW_FULL,
   KW_FUNCTION, KW_FUNCTIONS, KW_GRANT, KW_GROUP, KW_HAVING, KW_IF, KW_IN, KW_INIT_FN,
   KW_INNER, KW_INPATH, KW_INSERT, KW_INT, KW_INTERMEDIATE, KW_INTERVAL, KW_INTO,
-  KW_INVALIDATE, KW_IS, KW_JOIN, KW_LAST, KW_LEFT, KW_LIKE, KW_LIMIT, KW_LINES, KW_LOAD,
+  KW_INVALIDATE, KW_IS, KW_JOIN, KW_KNN, KW_LAST, KW_LEFT, KW_LIKE, KW_LIMIT, KW_LINES, KW_LOAD,
   KW_LOCATION, KW_MAP, KW_MERGE_FN, KW_METADATA, KW_NOT, KW_NULL, KW_NULLS, KW_OFFSET,
-  KW_ON, KW_OR, KW_ORDER, KW_OUTER, KW_OVER, KW_OVERWRITE, KW_PARQUET, KW_PARQUETFILE,
-  KW_PARTITION, KW_PARTITIONED, KW_PARTITIONS, KW_PRECEDING,
-  KW_PREPARE_FN, KW_PRODUCED, KW_RANGE, KW_RCFILE, KW_REFRESH, KW_REGEXP, KW_RENAME,
+  KW_ON, KW_OR, KW_ORDER, KW_OUTER, KW_OVER, KW_OVERLAPS, KW_OVERWRITE, KW_PARQUET, KW_PARQUETFILE,
+  KW_PARTITION, KW_PARTITIONED, KW_PARTITIONS, KW_POINT, KW_POINTS, KW_PRECEDING,
+  KW_PREPARE_FN, KW_PRODUCED, KW_RANGE, KW_RCFILE, KW_RECTANGLE, KW_REFRESH, KW_REGEXP, KW_RENAME,
   KW_REPLACE, KW_RETURNS, KW_REVOKE, KW_RIGHT, KW_RLIKE, KW_ROLE, KW_ROLES, KW_ROW,
   KW_ROWS, KW_SCHEMA, KW_SCHEMAS, KW_SELECT, KW_SEMI, KW_SEQUENCEFILE, KW_SERDEPROPERTIES,
   KW_SERIALIZE_FN, KW_SET, KW_SHOW, KW_SMALLINT, KW_STORED, KW_STRAIGHT_JOIN,
   KW_STRING, KW_STRUCT, KW_SYMBOL, KW_TABLE, KW_TABLES, KW_TBLPROPERTIES, KW_TERMINATED,
   KW_TEXTFILE, KW_THEN,
   KW_TIMESTAMP, KW_TINYINT, KW_STATS, KW_TO, KW_TRUE, KW_UNBOUNDED, KW_UNCACHED,
   KW_UNION, KW_UPDATE_FN, KW_USE, KW_USING,
-  KW_VALUES, KW_VARCHAR, KW_VIEW, KW_WHEN, KW_WHERE, KW_WITH;
+  KW_VALUES, KW_VARCHAR, KW_VIEW, KW_WHEN, KW_WHERE, KW_WITH, KW_WITHDIST, KW_WITHK;
 
 terminal COLON, COMMA, DOT, DOTDOTDOT, STAR, LPAREN, RPAREN, LBRACKET, RBRACKET,
   DIVIDE, MOD, ADD, SUBTRACT;
@@ -438,6 +443,14 @@ nonterminal TFunctionCategory opt_function_category;
 nonterminal HashMap create_function_args_map;
 nonterminal CreateFunctionStmtBase.OptArg create_function_arg_key;
 
+// For Spatial related queries
+nonterminal SpatialPointInclusionStmt spatial_point_inclusion_stmt;
+nonterminal Rectangle rectangle; 
+nonterminal NumericLiteral rectangle_arg;
+nonterminal SpatialKnnStmt spatial_knn_stmt;
+nonterminal Point point;
+nonterminal NumericLiteral point_arg;
+
 precedence left KW_OR;
 precedence left KW_AND;
 precedence left KW_NOT, NOT;
@@ -549,6 +562,46 @@ stmt ::=
   {: RESULT = grant_privilege; :}
   | revoke_privilege_stmt:revoke_privilege
   {: RESULT = revoke_privilege; :}
+  | spatial_point_inclusion_stmt:spatial_point_inclusion
+  {: RESULT = spatial_point_inclusion; :}
+  | spatial_knn_stmt:spatial_knn
+  {: RESULT = spatial_knn; :}
+  ;
+
+spatial_point_inclusion_stmt ::=
+  KW_LOAD KW_POINTS KW_FROM KW_TABLE table_name:tbl KW_OVERLAPS rectangle:rect
+  {: RESULT = new SpatialPointInclusionStmt(tbl, rect); :}
+  ;
+
+spatial_knn_stmt ::=
+  KW_LOAD KW_POINTS KW_FROM KW_TABLE table_name:tbl KW_KNN point:p KW_WITHK expr:k KW_WITHDIST order_by_elements:dist
+  {: RESULT = new SpatialKnnStmt(tbl,p,new LimitElement(k, null), dist); :}
+  ;
+
+rectangle ::=
+  KW_RECTANGLE LPAREN rectangle_arg:x1 COMMA rectangle_arg:y1 COMMA
+  rectangle_arg:x2 COMMA rectangle_arg:y2 RPAREN
+  {: RESULT = new Rectangle(x1.getDoubleValue(), y1.getDoubleValue(),
+   x2.getDoubleValue(), y2.getDoubleValue()); :}
+  ;
+
+point ::=
+  KW_POINT LPAREN point_arg:x COMMA point_arg:y RPAREN
+  {: RESULT = new Point(x.getDoubleValue(), y.getDoubleValue()); :}
+  ;
+
+rectangle_arg ::=
+  INTEGER_LITERAL:l
+  {: RESULT = new NumericLiteral(l); :}
+  | DECIMAL_LITERAL:l
+  {: RESULT = new NumericLiteral(l); :}
+  ;
+
+point_arg ::=
+  INTEGER_LITERAL:l
+  {: RESULT = new NumericLiteral(l); :}
+  | DECIMAL_LITERAL:l
+  {: RESULT = new NumericLiteral(l); :}
   ;
 
 load_stmt ::=

fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java

@@ -27,6 +27,8 @@
 import com.cloudera.impala.thrift.TQueryCtx;
 import com.google.common.base.Preconditions;
 
+import org.gistic.spatialImpala.analysis.*;
+
 /**
  * Wrapper class for parser and analyzer.
  */
@@ -54,7 +56,11 @@ static public class AnalysisResult {
     public boolean isAlterTableStmt() { return stmt_ instanceof AlterTableStmt; }
     public boolean isAlterViewStmt() { return stmt_ instanceof AlterViewStmt; }
     public boolean isComputeStatsStmt() { return stmt_ instanceof ComputeStatsStmt; }
-    public boolean isQueryStmt() { return stmt_ instanceof QueryStmt; }
+    public boolean isQueryStmt() {
+      return (stmt_ instanceof QueryStmt
+        || stmt_ instanceof SpatialPointInclusionStmt
+        || stmt_ instanceof SpatialKnnStmt);
+    }
     public boolean isInsertStmt() { return stmt_ instanceof InsertStmt; }
     public boolean isDropDbStmt() { return stmt_ instanceof DropDbStmt; }
     public boolean isDropTableOrViewStmt() {
@@ -203,7 +209,14 @@ public LoadDataStmt getLoadDataStmt() {
 
     public QueryStmt getQueryStmt() {
       Preconditions.checkState(isQueryStmt());
-      return (QueryStmt) stmt_;
+      if (stmt_ instanceof QueryStmt)
+    	  return (QueryStmt) stmt_;
+      else if (stmt_ instanceof SpatialPointInclusionStmt)
+    	  return ((SpatialPointInclusionStmt) stmt_).getSelectStmtIfAny();
+      else if (stmt_ instanceof SpatialKnnStmt)
+        return ((SpatialKnnStmt) stmt_).getSelectStmtIfAny();
+      else
+    	  return null;
     }
 
     public InsertStmt getInsertStmt() {

fe/src/main/java/com/cloudera/impala/analysis/CreateTableStmt.java

@@ -37,6 +37,8 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
+import org.gistic.spatialImpala.catalog.GlobalIndex;
+
 /**
  * Represents a CREATE TABLE statement.
  */
@@ -205,7 +207,15 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
     }
 
     if (location_ != null) location_.analyze(analyzer, Privilege.ALL);
-
+
+    // If the table is Spatial, ensure that the global index path exists
+    // and is valid.
+    if (tblProperties_ != null) {
+      String globalIndexPathIfAny = tblProperties_.get(GlobalIndex.GLOBAL_INDEX_TABLE_PARAM);
+      if (globalIndexPathIfAny != null)
+        (new HdfsUri(globalIndexPathIfAny)).analyze(analyzer, Privilege.ALL);
+    }
+
     analyzeRowFormatValue(rowFormat_.getFieldDelimiter());
     analyzeRowFormatValue(rowFormat_.getLineDelimiter());
     analyzeRowFormatValue(rowFormat_.getEscapeChar());

fe/src/main/java/com/cloudera/impala/analysis/LimitElement.java

@@ -23,7 +23,7 @@
 /**
  * Combination of limit and offset expressions.
  */
-class LimitElement {
+public class LimitElement {
   private final Expr limitExpr_;
   private final Expr offsetExpr_;
   private long limit_;

fe/src/main/java/com/cloudera/impala/analysis/SelectListItem.java

@@ -16,7 +16,7 @@
 
 import com.google.common.base.Preconditions;
 
-class SelectListItem {
+public class SelectListItem {
   private final Expr expr_;
   private String alias_;
 

fe/src/main/java/com/cloudera/impala/analysis/SelectStmt.java

@@ -65,7 +65,7 @@ public class SelectStmt extends QueryStmt {
   // directly
   private ExprSubstitutionMap baseTblSmap_ = new ExprSubstitutionMap();
 
-  SelectStmt(SelectList selectList,
+  public SelectStmt(SelectList selectList,
              List<TableRef> tableRefList,
              Expr wherePredicate, ArrayList<Expr> groupingExprs,
              Expr havingPredicate, ArrayList<OrderByElement> orderByElements,

fe/src/main/java/com/cloudera/impala/analysis/StatementBase.java

@@ -19,7 +19,7 @@
 /**
  * Base class for all Impala SQL statements.
  */
-abstract class StatementBase implements ParseNode {
+public abstract class StatementBase implements ParseNode {
   // True if this Stmt is the top level of an explain stmt.
   protected boolean isExplain_ = false;
 

fe/src/main/java/com/cloudera/impala/catalog/Table.java

@@ -38,6 +38,8 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
+import org.gistic.spatialImpala.catalog.SpatialHdfsTable;
+
 /**
  * Base class for table metadata.
  *
@@ -108,7 +110,7 @@ protected Table(TableId id, org.apache.hadoop.hive.metastore.api.Table msTable,
    */
   public abstract void load(Table oldValue, HiveMetaStoreClient client,
       org.apache.hadoop.hive.metastore.api.Table msTbl) throws TableLoadingException;
-
+  
   public void addColumn(Column col) {
     colsByPos_.add(col);
     colsByName_.put(col.getName().toLowerCase(), col);
@@ -195,7 +197,11 @@ public static Table fromMetastoreTable(TableId id, Db db,
       // data source.
       table = new DataSourceTable(id, msTbl, db, msTbl.getTableName(), msTbl.getOwner());
     } else if (HdfsFileFormat.isHdfsFormatClass(msTbl.getSd().getInputFormat())) {
-      table = new HdfsTable(id, msTbl, db, msTbl.getTableName(), msTbl.getOwner());
+    	// Checking if the table is Spatial or not.
+    	if (SpatialHdfsTable.isSpatial(msTbl))
+    		table = new SpatialHdfsTable(id, msTbl, db, msTbl.getTableName(), msTbl.getOwner());
+    	else
+    		table = new HdfsTable(id, msTbl, db, msTbl.getTableName(), msTbl.getOwner());
     }
     return table;
   }