SDAP-13

.gitignore

@@ -17,3 +17,9 @@ service/.classpath
 web/.classpath
 web/.externalToolBuilders/
 web/maven-eclipse.xml
+storage/lib
+storage/.classpath
+storage/.externalToolBuilders/Maven_Ant_Builder.launch
+storage/.gitignore
+storage/maven-eclipse.xml
+storage/target

core/src/main/java/org/apache/sdap/mudrod/discoveryengine/MetadataDiscoveryEngine.java

@@ -35,7 +35,7 @@ public class MetadataDiscoveryEngine extends DiscoveryEngineAbstract implements
   private static final long serialVersionUID = 1L;
   private static final Logger LOG = LoggerFactory.getLogger(MetadataDiscoveryEngine.class);
 
-  public MetadataDiscoveryEngine(Properties props, ESDriver es, SparkDriver spark) {
+  public MetadataDiscoveryEngine(Properties props, StorageDriver es, SparkDriver spark) {
     super(props, es, spark);
   }
 

core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java

@@ -17,9 +17,9 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.sdap.mudrod.driver.ESDriver;
 import org.apache.sdap.mudrod.driver.SparkDriver;
 import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.storage.StorageDriver;
 import org.apache.sdap.mudrod.weblog.pre.*;
 import org.apache.sdap.mudrod.weblog.process.ClickStreamAnalyzer;
 import org.apache.sdap.mudrod.weblog.process.UserHistoryAnalyzer;
@@ -46,7 +46,7 @@ public class WeblogDiscoveryEngine extends DiscoveryEngineAbstract {
   private static final Logger LOG = LoggerFactory.getLogger(WeblogDiscoveryEngine.class);
   public String timeSuffix = null;
 
-  public WeblogDiscoveryEngine(Properties props, ESDriver es, SparkDriver spark) {
+  public WeblogDiscoveryEngine(Properties props, StorageDriver sd, SparkDriver spark) {
     super(props, es, spark);
     LOG.info("Started Mudrod Weblog Discovery Engine.");
   }

core/src/main/java/org/apache/sdap/mudrod/driver/StorageDriver.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.storage;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.sdap.mudrod.storage.elasticsearch.ElasticSearchDriver;
+import org.apache.sdap.mudrod.storage.solr.SolrDriver;
+
+/**
+ * Core storage datum from which all concrete storage-related 
+ * implementations should extend.
+ * @since v0.0.1-SNAPSHOT
+ */
+public interface StorageDriver {
+
+  /**
+   * 
+   * @return an initialized {@link org.apache.sdap.mudrod.storage.StorageDriver} implementation.
+   */
+  default StorageDriver initialize(Properties props) {
+    StorageDriver sDriver = null;
+    if (props != null) {
+      switch (props.getProperty("mudrod.storage.driver", "elasticsearch")) {
+        case "elasticsearch":
+        sDriver = new ElasticSearchDriver(props);
+        break;
+        default:
+        sDriver = new SolrDriver(props);
+        break;
+      }
+    } else {
+
+    }
+    return sDriver;
+  }
+
+  abstract void createBulkProcessor();
+
+  abstract void destroyBulkProcessor();
+
+  abstract void putMapping(String indexName, String settingsJson, String mappingJson) throws IOException;
+
+  abstract String customAnalyzing(String indexName, String str) throws InterruptedException, ExecutionException;
+
+  abstract String customAnalyzing(String indexName, String analyzer, String str) throws InterruptedException, ExecutionException;
+
+  abstract List<String> customAnalyzing(String indexName, List<String> list) throws InterruptedException, ExecutionException;
+
+  //abstract void deleteAllByQuery(String index, String type, QueryBuilder query);
+
+  abstract void deleteType(String index, String type);
+
+  abstract List<String> getTypeListWithPrefix(Object object, Object object2);
+
+  abstract  List<String> getIndexListWithPrefix(Object object);
+
+  abstract String searchByQuery(String index, String type, String query);
+
+  abstract String searchByQuery(String index, String type, String query, Boolean bDetail);
+
+  abstract List<List<String>> buildMeasurementHierarchies(
+          List<String> topics, List<String> terms, List<String> variables, 
+          List<String> variableDetails);
+
+  abstract List<String> autoComplete(String index, String term);
+
+  abstract void close();
+
+  abstract void refreshIndex();
+
+  //abstract Client makeClient(Properties props);
+
+  //abstract UpdateRequest generateUpdateRequest(String index, 
+  // String type, String id, String field1, Object value1);
+
+  //UpdateRequest generateUpdateRequest(String index, String type, String id,
+  // Map<String, Object> filedValueMap);
+
+  abstract int getDocCount(String index, String... type);
+
+  abstract int getDocCount(String[] index, String[] type);
+
+  //public int getDocCount(String[] index, String[] type, QueryBuilder filterSearch);
+
+}

core/src/main/java/org/apache/sdap/mudrod/driver/StorageDriverFactory.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.driver;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.apache.sdap.mudrod.utils.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class StorageDriverFactory {
+
+  public static final Logger log = LoggerFactory.getLogger(StorageDriverFactory.class);
+
+  public static final String MUDROD_DEFAULT_PROPERTIES_FILE = "config.properties";
+
+  public static final String MUDROD_DEFAULT_DATASTORE_KEY = "mudrod.datastore.default";
+
+  private StorageDriverFactory() { }
+
+  /**
+   * Creates a new {@link Properties}. It adds the default gora configuration
+   * resources. This properties object can be modified and used to instantiate
+   * store instances. It is recommended to use a properties object for a single
+   * store, because the properties object is passed on to store initialization
+   * methods that are able to store the properties as a field.   
+   * @return The new properties object.
+   */
+  public static Properties createProps() {
+    try {
+      Properties properties = new Properties();
+      InputStream stream = StorageDriverFactory.class.getClassLoader()
+              .getResourceAsStream(MUDROD_DEFAULT_PROPERTIES_FILE);
+      if(stream != null) {
+        try {
+          properties.load(stream);
+          return properties;
+        } finally {
+          stream.close();
+        }
+      } else {
+        log.warn(MUDROD_DEFAULT_PROPERTIES_FILE + " not found, properties will be empty.");
+      }
+      return properties;
+    } catch(Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static void initializeStorageDriver(
+          StorageDriver storageDriver, Properties properties) throws IOException {
+    storageDriver.initialize(properties);
+  }
+
+  /**
+   * Instantiate a new {@link DataStore}.
+   * 
+   * @param storageDriverClass The datastore implementation class.
+   * @param properties The properties to be used in the store.
+   * @return A new {@link org.apache.sdap.mudrod.storage.StorageDriver} instance.
+   * @throws Exception
+   */
+  public static StorageDriver createDataStore(Class<?> storageDriverClass, Properties properties) throws Exception{
+    try {
+      StorageDriver storageDriver =
+              (StorageDriver) ReflectionUtils.newInstance(storageDriverClass);
+      initializeStorageDriver(storageDriver, properties);
+      return storageDriver;
+    } catch(Exception ex) {
+      throw new Exception(ex);
+    }
+  }
+}

core/src/main/java/org/apache/sdap/mudrod/main/MudrodConstants.java

@@ -137,6 +137,8 @@ public interface MudrodConstants {
   public static final String SPARK_APP_NAME = "mudrod.spark.app.name";
 
   public static final String SPARK_MASTER = "mudrod.spark.master";
+
+  public static final String STORAGE_DRIVER = "mudrod.storage.driver";
 
   public static final String RANKING_MODEL = "mudrod.ranking.model";