Metadata server

.travis.yml

@@ -5,6 +5,9 @@ dist: trusty
 language: python
 python: "3.4"
 
+services:
+    - postgresql
+
 env:
     global:
         - LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/dependencies/libcsv/.libs
@@ -24,6 +27,7 @@ install:
     - sudo apt-get update -q
     - sudo apt-get install g++-4.9 -y
     - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 60
+    - sudo apt-get install libdbi-dev libdbd-pgsql -y
     - sudo apt-get -y install openjdk-8-jdk icedtea-plugin
     - jdk_switcher use openjdk8
     - echo $JAVA_HOME
@@ -33,6 +37,14 @@ install:
     #Copy protobuf binaries
     - wget https://github.com/Intel-HLS/GenomicsDB/releases/download/0.4.0/protobuf-3.0.2-trusty.tar.gz -O protobuf-3.0.2-trusty.tar.gz
     - tar xzf protobuf-3.0.2-trusty.tar.gz && sudo rsync -a protobuf-3.0.2-trusty/ /usr/
+    #install gtest
+    #- sudo apt-get -y install libgtest0 libgtest-dev
+    # above package (libgtest0) not built for 14.04
+    - sudo apt-get -y install libgtest-dev
+    - cd /usr/src/gtest
+    - sudo cmake .
+    - sudo make
+    - sudo mv libgtest* /usr/lib
     #Install libcsv
     - cd $TRAVIS_BUILD_DIR
     - wget -O dependencies/libcsv.tar.gz http://downloads.sourceforge.net/project/libcsv/libcsv/libcsv-3.0.3/libcsv-3.0.3.tar.gz
@@ -44,6 +56,9 @@ install:
 
 before_script:
     - cd $GENOMICSDB_BUILD_DIR && lcov --directory . --zerocounters
+    - psql -U postgres -c 'create database gendb'
+    - psql -U postgres gendb < $TRAVIS_BUILD_DIR/src/resources/gendb_state_for_tests.txt
+    - psql gendb -U postgres -c 'select count(*) from reference'
 
 script:
     - pylint $TRAVIS_BUILD_DIR/docker/vcf_combiner/usr/bin/combine_vcf.py

CMakeLists.txt

@@ -31,6 +31,7 @@ set(LIBCSV_DIR "" CACHE PATH "Path to libcsv header and library")
 set(DO_PROFILING False CACHE BOOL "Collect some stats during execution - doesn't add much overhead")
 set(DO_MEMORY_PROFILING False CACHE BOOL "Collect memory consumption in parts of the combine gVCF program - high overhead")
 set(GENOMICSDB_MAVEN_BUILD_DIR ${CMAKE_BINARY_DIR}/target CACHE PATH "Path to maven build directory")
+set(LIBDBI_DIR "/usr" CACHE PATH "Path to libdbi install directory")
 set(MAVEN_QUIET False CACHE BOOL "Do not print mvn messages")
 
 #Platform check modules
@@ -104,6 +105,13 @@ include_directories(${TILEDB_INCLUDE_DIR})
 #librt
 find_library(LIBRT_LIBRARY rt)
 
+#pgsql driver and dbi libs
+find_package(libdbi)
+if(LIBDBI_FOUND)
+    include_directories(${LIBDBI_INCLUDE_DIR})
+    add_definitions(-DLIBDBI)
+endif()
+
 #Protobuf library
 find_package(ProtobufWrapper REQUIRED)
 include_directories(${PROTOBUF_INCLUDE_DIRS})
@@ -170,10 +178,11 @@ include_directories (
     src/main/cpp/include/query_operations
     src/main/cpp/include/utils
     src/main/cpp/include/vcf
+    src/test/cpp/include
 )
 
-function(build_GenomicsDB_executable target)
-    add_executable(${target} src/${target}.cc)
+function(build_GenomicsDB_executable_common target)
+    #add_executable(${target} src/${target}.cc)
     if(TILEDB_SOURCE_DIR)
         add_dependencies(${target} TileDB)
     endif()
@@ -184,13 +193,21 @@ function(build_GenomicsDB_executable target)
     if(LIBCSV_FOUND)
         target_link_libraries(${target} ${LIBCSV_LIBRARY})
     endif()
+    if(LIBDBI_FOUND)
+        target_link_libraries(${target} ${LIBDBI_DEV_LIBRARY} ${LIBPGSQL_DRIVER_LIBRARY})
+    endif()
     target_link_libraries(${target} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PROTOBUF_LIBRARIES})
     if(LIBRT_LIBRARY)
         target_link_libraries(${target} ${LIBRT_LIBRARY})
     endif()
     install(TARGETS ${target} RUNTIME DESTINATION bin)
 endfunction()
 
+function(build_GenomicsDB_executable target)
+    add_executable(${target} src/${target}.cc)
+    build_GenomicsDB_executable_common(${target})
+endfunction()
+
 add_subdirectory(src)
 include_directories(${PROTOBUF_GENERATED_CXX_HDRS_INCLUDE_DIRS})
 add_subdirectory(tools)
@@ -275,4 +292,6 @@ if(BUILD_JAVA)
         COMMAND /usr/bin/python ${CMAKE_SOURCE_DIR}/tests/run.py
         ${CMAKE_BINARY_DIR}
         ${CMAKE_INSTALL_PREFIX})
+
+    add_test(NAME All_GTests COMMAND runAllGTests)
 endif()

cmake/Modules/Findlibcsv.cmake

@@ -2,7 +2,7 @@
 # Once done this will define
 # LIBCSV_FOUND - libcsv found
 
-find_path(LIBCSV_INCLUDE_DIR NAMES csv.h HINTS "${LIBCSV_DIR}")
-find_library(LIBCSV_LIBRARY NAMES csv HINTS "${LIBCSV_DIR}" "${LIBCSV_DIR}/.libs")
+find_path(LIBCSV_INCLUDE_DIR NAMES csv.h HINTS "${LIBCSV_DIR}/include" "${LIBCSV_DIR}")
+find_library(LIBCSV_LIBRARY NAMES csv HINTS "${LIBCSV_DIR}/lib" "${LIBCSV_DIR}/.libs" "${LIBCSV_DIR}")
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(libcsv "Could not find libcsv headers and/or libraries ${DEFAULT_MSG}" LIBCSV_INCLUDE_DIR LIBCSV_LIBRARY)

cmake/Modules/Findlibdbi.cmake

@@ -0,0 +1,13 @@
+# Determine compiler flags for libdbi
+# Once done this will define
+# LIBDBI_FOUND - libdbi found
+
+find_path(LIBDBI_INCLUDE_DIR NAMES dbi.h HINTS "${LIBDBI_DIR}/include/dbi")
+
+find_library(LIBPGSQL_DRIVER_LIBRARY NAMES dbdpgsql HINTS "${LIBDBI_DIR}/lib/${CMAKE_C_LIBRARY_ARCHITECTURE}/dbd")
+
+find_library(LIBDBI_DEV_LIBRARY NAMES dbi HINTS "${LIBDBI_DIR}/lib/${CMAKE_C_LIBRARY_ARCHITECTURE}")
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(libdbi "Could not find libdbi headers and/or libraries ${DEFAULT_MSG}" LIBDBI_INCLUDE_DIR LIBDBI_DEV_LIBRARY LIBPGSQL_DRIVER_LIBRARY)

src/CMakeLists.txt

@@ -1,2 +1,3 @@
 add_subdirectory(resources)
 add_subdirectory(main)
+add_subdirectory(test)

src/main/CMakeLists.txt

@@ -23,6 +23,7 @@ set(GenomicsDB_library_sources
     cpp/src/utils/lut.cc
     cpp/src/utils/known_field_info.cc
     cpp/src/utils/vid_mapper.cc
+    cpp/src/utils/vid_mapper_sql.cc
     cpp/src/utils/timer.cc
     cpp/src/vcf/vcf_adapter.cc
     cpp/src/vcf/genomicsdb_bcf_generator.cc

src/main/cpp/include/utils/vid_mapper_sql.h

@@ -0,0 +1,174 @@
+/*
+ * The MIT License (MIT)
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+ * the Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifdef LIBDBI
+#ifndef GENOMICSDB_VID_MAPPER_SQL_H
+#define GENOMICSDB_VID_MAPPER_SQL_H
+
+#include "vid_mapper.h"
+#include <string>
+#include <vector>
+#include <map>
+#include <dbi/dbi.h>
+
+const std::string PGSQL_DRIVER = "pgsql";
+const std::string DBCONN_HOST = "host";
+const std::string DBCONN_USERNAME = "username";
+const std::string DBCONN_PASSWORD = "password";
+const std::string DBCONN_DBNAME = "dbname";
+const std::string DBCONN_ENCODING = "encoding";
+const std::string UTF8_ENCODING = "UTF-8";
+
+const std::string DBCONN_NEW_CONNECTION_FAILED = "FAILED to Create new Connection";
+const std::string DBCONN_CONNECT_To_DB_FAILED = "FAILED to Connect to DB";
+const std::string DBQUERY_FAILED = "DB Query Failed";
+
+const std::string DBTABLE_REFERENCE_COLUMN_NAME = "name";
+const std::string DBTABLE_REFERENCE_COLUMN_OFFSET = "tiledb_column_offset";
+const std::string DBTABLE_REFERENCE_COLUMN_LENGTH = "length";
+
+const std::string DBTABLE_CALLSET_COLUMN_NAME = "name";
+const std::string DBTABLE_CALLSET_COLUMN_ID = "id";
+
+const std::string DBTABLE_FIELD_COLUMN_ID = "id";
+const std::string DBTABLE_FIELD_COLUMN_NAME = "field";
+const std::string DBTABLE_FIELD_COLUMN_FSID = "field_set_id";
+const std::string DBTABLE_FIELD_COLUMN_TYPE = "type";
+const std::string DBTABLE_FIELD_COLUMN_FILTER = "is_filter";
+const std::string DBTABLE_FIELD_COLUMN_FORMAT = "is_format";
+const std::string DBTABLE_FIELD_COLUMN_INFO = "is_info";
+const std::string DBTABLE_FIELD_COLUMN_LENTYPE = "length_type";
+const std::string DBTABLE_FIELD_COLUMN_LENVAL = "length_intval";
+const std::string DBTABLE_FIELD_COLUMN_COMBOP = "vcf_field_combine_operation";
+
+const std::string COMBINE_OPERATION_ERROR_PREFIX = ("VCF field combined operation 'concatenate' can only be used with fields whose length descriptors are 'VAR'; field ");
+const std::string COMBINE_OPERATION_ERROR_SUFFIX = (" does not have 'VAR' as its length descriptor");
+
+enum {
+  GENOMICSDB_VID_MAPPER_SUCCESS = 0x0,
+  GENOMICSDB_VID_MAPPER_FAILURE = 0x1
+};
+
+/**
+ * This class encapsulates parameters needed to establish a DB connection and
+ * also identify the workspace and db_array.
+ */
+class SQLVidMapperRequest {
+  public:
+    std::string host_name;
+    std::string user_name;
+    std::string pass_word;
+    std::string db_name;
+    std::string work_space;
+    std::string array_name;
+};
+
+/**
+ * This class will be used to load VID Mapper information from DB. In future it
+ * may also be used to modify DB info.
+ */
+class SQLBasedVidMapper : public VidMapper {
+  public:
+    SQLBasedVidMapper(const SQLVidMapperRequest&);
+
+    /* following two declarations disable copy semantics since not needed */
+    SQLBasedVidMapper(const SQLBasedVidMapper&) = delete;
+
+    void operator=(const SQLBasedVidMapper&) = delete;
+
+    /* following two declarations disable move semantics since not needed */
+    SQLBasedVidMapper(SQLBasedVidMapper&&) = delete;
+
+    SQLBasedVidMapper&& operator=(SQLBasedVidMapper&&) = delete;
+
+    /* public method to load all mapping data from DB */
+    int load_mapping_data_from_db();
+
+    /* following three methods return contig mappings */
+    std::vector<ContigInfo>& get_contigs() { return(m_contig_idx_to_info); }
+
+    std::vector<std::pair<int64_t, int>>& get_contig_begin_offsets() {
+      return(m_contig_begin_2_idx);
+    }
+
+    std::vector<std::pair<int64_t, int>>& get_contig_end_offsets() {
+      return(m_contig_end_2_idx);
+    }
+
+    /* following two methods return callset mappings */
+    std::vector<CallSetInfo>& get_callsets() { return(m_row_idx_to_info); }
+
+    std::unordered_map<std::string, int64_t>& get_callset_name_to_idx_map() {
+      return(m_callset_name_to_row_idx);
+    }
+
+    /* following two methods return field mappings */
+    std::vector<FieldInfo>& get_fields() { return(m_field_idx_to_info); }
+
+    std::unordered_map<std::string, int>& get_field_name_to_idx_map() {
+      return(m_field_name_to_idx);
+    }
+
+    ~SQLBasedVidMapper();
+  protected:
+    dbi_conn m_conn;
+    dbi_inst m_instance;
+    std::string m_work_space;
+    std::string m_array_name;
+
+    /* method to retrieve contig info from DB and load into VID mapper */
+    int load_contig_info();
+
+    /* method to retrieve callset info from DB and load into VID mapper */
+    int load_callset_info();
+
+    /* method to retrieve field info from DB and load into VID mapper */
+    int load_field_info();
+};
+
+class SQLBasedVidMapperException : public std::exception {
+  public:
+    /**
+     * Default constructor
+     */
+    SQLBasedVidMapperException(const std::string m="");
+
+    /*
+     * Destructor: must be called before end of scope of a
+     * VidMapper exception object
+     */
+    ~SQLBasedVidMapperException();
+
+    /*
+     * ACCESSORS
+     */
+
+    /**
+     * Returns the exception message.
+     */
+    const char* what() const noexcept { return msg_.c_str(); }
+
+  private:
+    std::string msg_;
+};
+
+#endif  // GENOMICSDB_VID_MAPPER_SQL_H
+#endif