From 7df7757e00244b9af021218e5fcea33505919efb Mon Sep 17 00:00:00 2001
From: Kane Baccigalupi <kane.baccigalupi@gsa.gov>
Date: Mon, 22 Feb 2016 09:44:52 -0800
Subject: [PATCH 1/7] Extracts DataLoader object to take on some of Config
 responsibility

---
 lib/data_magic/config.rb | 61 +++++++++++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 14 deletions(-)

diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb
index d7b69e5b..3a4878b1 100644
--- a/lib/data_magic/config.rb
+++ b/lib/data_magic/config.rb
@@ -1,11 +1,12 @@
 require_relative '../data_magic.rb'
+require_relative 'example.rb'
+require_relative 'category.rb'
 
 module DataMagic
-  require_relative 'example.rb'
-  require_relative 'category.rb'
   class Config
     attr_reader :data_path, :data, :dictionary, :files, :s3, :api_endpoints,
                 :null_value, :file_config
+
     attr_accessor :page_size
 
     def initialize(options = {})
@@ -16,10 +17,8 @@ def initialize(options = {})
       @extensions = DataMagic::DEFAULT_EXTENSIONS
       @s3 = options[:s3]
 
-      @data_path = options[:data_path] || ENV['DATA_PATH']
-      if @data_path.nil? or @data_path.empty?
-        @data_path = DEFAULT_PATH
-      end
+      @data_path = DataLoader.new(options).path
+
       if options[:load_datayaml] == false
         @data = {}
       else
@@ -27,6 +26,41 @@ def initialize(options = {})
       end
     end
 
+    def data_loader(opts)
+      DataLoader.new(opts)
+    end
+
+    class DataLoader
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      def path
+        options_path || env_path || default_path
+      end
+
+      private
+
+      def options_path
+        assure_non_empty(options[:data_path])
+      end
+
+      def env_path
+        assure_non_empty(ENV['DATA_PATH'])
+      end
+
+      def default_path
+        DataMagic::DEFAULT_PATH
+      end
+
+      def assure_non_empty(value)
+        return nil unless value && !value.empty?
+        value
+      end
+    end
+
     def options
       @data['options']
     end
@@ -42,14 +76,13 @@ def valid_types
     end
 
     def examples
-      if @examples.nil?
-        api = api_endpoint_names[0]
-        data['examples'] ||= []
-        @examples = data['examples'].map do |i|
-          Example.new(i.merge(endpoint: api))
-        end
+      return @examples unless @examples.nil?
+
+      api = api_endpoint_names[0]
+      data['examples'] ||= []
+      @examples = data['examples'].map do |i|
+        Example.new(i.merge(endpoint: api))
       end
-      @examples
     end
 
     def categories
@@ -83,7 +116,7 @@ def self.logger
     end
 
     def logger
-      Config.logger
+      self.class.logger
     end
 
     def csv_column_type(column_name)

From f35ff493cecd33b21d2d527c3d328d5077320ee9 Mon Sep 17 00:00:00 2001
From: Kane Baccigalupi <kane.baccigalupi@gsa.gov>
Date: Mon, 22 Feb 2016 12:00:50 -0800
Subject: [PATCH 2/7] Extracts s3 and local adapters for reading yaml files

---
 lib/data_magic/config.rb             | 105 +++++----------------------
 lib/data_magic/load/local_adapter.rb |  30 ++++++++
 lib/data_magic/load/s3_adapter.rb    |  38 ++++++++++
 lib/data_magic/load/yaml_data.rb     |  34 +++++++++
 4 files changed, 119 insertions(+), 88 deletions(-)
 create mode 100644 lib/data_magic/load/local_adapter.rb
 create mode 100644 lib/data_magic/load/s3_adapter.rb
 create mode 100644 lib/data_magic/load/yaml_data.rb

diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb
index 3a4878b1..390c1fd1 100644
--- a/lib/data_magic/config.rb
+++ b/lib/data_magic/config.rb
@@ -2,6 +2,10 @@
 require_relative 'example.rb'
 require_relative 'category.rb'
 
+require_relative 'load/yaml_data'
+require_relative 'load/s3_adapter'
+require_relative 'load/local_adapter'
+
 module DataMagic
   class Config
     attr_reader :data_path, :data, :dictionary, :files, :s3, :api_endpoints,
@@ -17,7 +21,7 @@ def initialize(options = {})
       @extensions = DataMagic::DEFAULT_EXTENSIONS
       @s3 = options[:s3]
 
-      @data_path = DataLoader.new(options).path
+      @data_path = Load::YamlData.new(options).path
 
       if options[:load_datayaml] == false
         @data = {}
@@ -26,41 +30,6 @@ def initialize(options = {})
       end
     end
 
-    def data_loader(opts)
-      DataLoader.new(opts)
-    end
-
-    class DataLoader
-      attr_reader :options
-
-      def initialize(options)
-        @options = options
-      end
-
-      def path
-        options_path || env_path || default_path
-      end
-
-      private
-
-      def options_path
-        assure_non_empty(options[:data_path])
-      end
-
-      def env_path
-        assure_non_empty(ENV['DATA_PATH'])
-      end
-
-      def default_path
-        DataMagic::DEFAULT_PATH
-      end
-
-      def assure_non_empty(value)
-        return nil unless value && !value.empty?
-        value
-      end
-    end
-
     def options
       @data['options']
     end
@@ -356,64 +325,24 @@ def index_needs_update?(index_name=nil)
       old_config.nil? || old_config["version"] != @data["version"]
     end
 
-    # read from the key,
-    # return contents or...
-    # nil if not found, otherwise raise exception
-    def read_from_s3(bucket, key)
-      result = nil
-      begin
-        response = @s3.get_object(bucket: bucket, key: key)
-        result = response.body.read
-      rescue Aws::S3::Errors::NoSuchKey
-        # we don't want to raise this one, might be expected
-        result = nil
-      rescue => e
-        logger.debug "read_from_s3 failed: #{bucket} #{key} with #{e.class}:#{e.message}"
-        raise e
+    def read_path(path)
+      adapter = get_adapter(path)
+      raise ArgumentError, "unexpected scheme: #{scheme}" if !adapter
+
+      if adapter.is_a?(Load::S3Adapter)
+        adapter.read
+      else
+        adapter.read
       end
-      result
     end
 
-    # read local file and return content
-    # if not found, return nil
-    # any other failure, raise exception
-    def read_path_local(path)
-      result = nil
-      begin
-        result = File.read(path)
-      rescue => e
-        if e.message.include? "No such file or directory"
-          result = nil
-        else
-          logger.error "read_path_local failed: #{path} with #{e.class}:#{e.message}"
-          raise e
-        end
-      end
-      result
+    def get_adapter(path)
+      adapters(path).detect(&:can_handle?)
     end
 
-    # reads a file or s3 object, returns a string
-    # path follows URI pattern
-    # could be
-    #   s3://username:password@bucket_name/path
-    #   s3://bucket_name/path
-    #   s3://bucket_name
-    #   a local path like: './data'
-    def read_path(path)
-      result = nil
+    def adapters(path)
       uri = URI(path)
-      scheme = uri.scheme
-      case scheme
-        when nil
-          result = read_path_local(uri.path)
-        when "s3"
-          key = uri.path
-          key[0] = ''  # remove initial /
-          result = read_from_s3(uri.hostname, key)
-        else
-          raise ArgumentError, "unexpected scheme: #{scheme}"
-      end
-      result
+      [Load::S3Adapter.new(uri, s3), Load::LocalAdapter.new(uri)]
     end
 
     def load_yaml(path = nil)
diff --git a/lib/data_magic/load/local_adapter.rb b/lib/data_magic/load/local_adapter.rb
new file mode 100644
index 00000000..a7c12a41
--- /dev/null
+++ b/lib/data_magic/load/local_adapter.rb
@@ -0,0 +1,30 @@
+module DataMagic
+  module Load
+    class LocalAdapter
+      attr_reader :uri
+
+      def initialize(uri)
+        @uri = uri
+      end
+
+      def can_handle?
+        uri.scheme == nil
+      end
+
+      def read
+        File.read(path)
+      rescue => e
+        if e.message.include? "No such file or directory"
+          nil
+        else
+          logger.error "read_path_local failed: #{path} with #{e.class}:#{e.message}"
+          raise e
+        end
+      end
+
+      def path
+        uri.path
+      end
+    end
+  end
+end
diff --git a/lib/data_magic/load/s3_adapter.rb b/lib/data_magic/load/s3_adapter.rb
new file mode 100644
index 00000000..9617941d
--- /dev/null
+++ b/lib/data_magic/load/s3_adapter.rb
@@ -0,0 +1,38 @@
+module DataMagic
+  module Load
+    class S3Adapter
+      attr_reader :s3, :uri
+
+      def initialize(uri, s3)
+        @uri = uri
+        @s3 = s3
+      end
+
+      def can_handle?
+        uri.scheme == 's3'
+      end
+
+      def read
+        response = s3.get_object(bucket: bucket, key: key)
+        response.body.read
+      rescue Aws::S3::Errors::NoSuchKey
+        # we don't want to raise this one, might be expected
+        nil
+      rescue => e
+        logger.debug "read_from_s3 failed: #{bucket} #{key} with #{e.class}:#{e.message}"
+        raise e
+      end
+
+      def key
+        key = uri.path
+        key[0] = ''  # remove initial /
+        key
+      end
+
+      def bucket
+        uri.hostname
+      end
+    end
+  end
+end
+
diff --git a/lib/data_magic/load/yaml_data.rb b/lib/data_magic/load/yaml_data.rb
new file mode 100644
index 00000000..c5ed539c
--- /dev/null
+++ b/lib/data_magic/load/yaml_data.rb
@@ -0,0 +1,34 @@
+module DataMagic
+  module Load
+    class YamlData
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      def path
+        options_path || env_path || default_path
+      end
+
+      private
+
+      def options_path
+        assure_non_empty(options[:data_path])
+      end
+
+      def env_path
+        assure_non_empty(ENV['DATA_PATH'])
+      end
+
+      def default_path
+        DataMagic::DEFAULT_PATH
+      end
+
+      def assure_non_empty(value)
+        return nil unless value && !value.empty?
+        value
+      end
+    end
+  end
+end

From fe17e653d93f4437f2bf70317ab535fe5c78276a Mon Sep 17 00:00:00 2001
From: Kane Baccigalupi <kane.baccigalupi@gsa.gov>
Date: Mon, 22 Feb 2016 13:44:56 -0800
Subject: [PATCH 3/7] pushes more logic into Load::YamlData

---
 lib/data_magic/config.rb         | 23 ++++----------------
 lib/data_magic/load/yaml_data.rb | 36 ++++++++++++++++++++++++++++----
 2 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb
index 390c1fd1..1e311759 100644
--- a/lib/data_magic/config.rb
+++ b/lib/data_magic/config.rb
@@ -9,7 +9,7 @@
 module DataMagic
   class Config
     attr_reader :data_path, :data, :dictionary, :files, :s3, :api_endpoints,
-                :null_value, :file_config
+                :null_value, :file_config, :yaml_data
 
     attr_accessor :page_size
 
@@ -21,7 +21,8 @@ def initialize(options = {})
       @extensions = DataMagic::DEFAULT_EXTENSIONS
       @s3 = options[:s3]
 
-      @data_path = Load::YamlData.new(options).path
+      @yaml_data = Load::YamlData.new(options)
+      @data_path = yaml_data.path
 
       if options[:load_datayaml] == false
         @data = {}
@@ -326,23 +327,7 @@ def index_needs_update?(index_name=nil)
     end
 
     def read_path(path)
-      adapter = get_adapter(path)
-      raise ArgumentError, "unexpected scheme: #{scheme}" if !adapter
-
-      if adapter.is_a?(Load::S3Adapter)
-        adapter.read
-      else
-        adapter.read
-      end
-    end
-
-    def get_adapter(path)
-      adapters(path).detect(&:can_handle?)
-    end
-
-    def adapters(path)
-      uri = URI(path)
-      [Load::S3Adapter.new(uri, s3), Load::LocalAdapter.new(uri)]
+      yaml_data.read(path)
     end
 
     def load_yaml(path = nil)
diff --git a/lib/data_magic/load/yaml_data.rb b/lib/data_magic/load/yaml_data.rb
index c5ed539c..011fc426 100644
--- a/lib/data_magic/load/yaml_data.rb
+++ b/lib/data_magic/load/yaml_data.rb
@@ -1,18 +1,46 @@
 module DataMagic
   module Load
     class YamlData
-      attr_reader :options
+      attr_reader :options, :s3_client
 
       def initialize(options)
         @options = options
+        @s3_client = options[:s3]
       end
 
-      def path
-        options_path || env_path || default_path
+      def path(alt_path=nil)
+        @path || assure_non_empty(alt_path) || default_path
+      end
+
+      def read(alt_path=nil)
+        set_path(alt_path)
+        raise ArgumentError, "unexpected scheme: #{scheme}" if !adapter
+        adapter.read
       end
 
       private
 
+      def adapter
+        adapters.detect(&:can_handle?)
+      end
+
+      def adapters
+        uri = URI(path)
+        [Load::S3Adapter.new(uri, s3_client), Load::LocalAdapter.new(uri)]
+      end
+
+      def scheme
+        URI(path).scheme
+      end
+
+      def set_path(path)
+        @path = assure_non_empty(path) || default_path
+      end
+
+      def default_path
+        options_path || env_path || global_default_path
+      end
+
       def options_path
         assure_non_empty(options[:data_path])
       end
@@ -21,7 +49,7 @@ def env_path
         assure_non_empty(ENV['DATA_PATH'])
       end
 
-      def default_path
+      def global_default_path
         DataMagic::DEFAULT_PATH
       end
 

From d0bf37ee1f03151941c6eb489f1175afb2d0fce2 Mon Sep 17 00:00:00 2001
From: Kane Baccigalupi <kane.baccigalupi@gsa.gov>
Date: Fri, 26 Feb 2016 09:42:15 -0800
Subject: [PATCH 4/7] Moves loading of yaml data into YamlData

---
 lib/data_magic/config.rb         | 10 +---------
 lib/data_magic/load/yaml_data.rb | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb
index 1e311759..f7d5fc83 100644
--- a/lib/data_magic/config.rb
+++ b/lib/data_magic/config.rb
@@ -331,15 +331,7 @@ def read_path(path)
     end
 
     def load_yaml(path = nil)
-      logger.info "load_yaml: #{path}"
-      raw = read_path(File.join(path, "data.yaml"))
-      raw ||= read_path(File.join(path, "data.yml"))
-      raw ||= '{}' if ENV['ALLOW_MISSING_YML']
-      if raw.nil?
-        raise IOError, "No data.y?ml found at #{path}. Did you mean to define ALLOW_MISSING_YML environment variable?"
-      end
-
-      YAML.load(raw)
+      yaml_data.read_yaml(path)
     end
 
     def list_files(path)
diff --git a/lib/data_magic/load/yaml_data.rb b/lib/data_magic/load/yaml_data.rb
index 011fc426..900b746c 100644
--- a/lib/data_magic/load/yaml_data.rb
+++ b/lib/data_magic/load/yaml_data.rb
@@ -18,8 +18,24 @@ def read(alt_path=nil)
         adapter.read
       end
 
+      def read_yaml(path)
+        raw = find_and_read(path)
+        YAML.load(raw)
+      end
+
       private
 
+      def find_and_read(path)
+        read(File.join(path, "data.yaml")) ||
+          read(File.join(path, "data.yml")) ||
+          handle_nil_data(path)
+      end
+
+      def handle_nil_data(path)
+        return '{}' if ENV['ALLOW_MISSING_YML']
+        raise IOError, "No data.y?ml found at #{path}. Did you mean to define ALLOW_MISSING_YML environment variable?"
+      end
+
       def adapter
         adapters.detect(&:can_handle?)
       end

From 80e21336ea6b2e861639dd5d92570ff0582c54a0 Mon Sep 17 00:00:00 2001
From: Kane Baccigalupi <kane.baccigalupi@gsa.gov>
Date: Fri, 26 Feb 2016 14:52:16 -0800
Subject: [PATCH 5/7] Category mapping methods for possible extraction

---
 lib/data_magic/config.rb | 123 ++++++++++++++++++++-------------------
 1 file changed, 64 insertions(+), 59 deletions(-)

diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb
index f7d5fc83..06fd3534 100644
--- a/lib/data_magic/config.rb
+++ b/lib/data_magic/config.rb
@@ -31,6 +31,70 @@ def initialize(options = {})
       end
     end
 
+    def self.init(s3 = nil)
+      logger.info "Config.init #{s3.inspect}"
+      @s3 = s3
+      Config.load
+    end
+
+    # ----------
+    # Logger stuff
+    def self.logger=(new_logger)
+      @logger = new_logger
+    end
+
+    def self.logger
+      @logger ||= Logger.new("log/#{ENV['RACK_ENV'] || 'development'}.log")
+    end
+
+    def logger
+      self.class.logger
+    end
+    # ---------
+
+
+    # ----------
+    # File manipulation and stuff
+    def read_path(path)
+      yaml_data.read(path)
+    end
+
+    def load_yaml(path = nil)
+      yaml_data.read_yaml(path)
+    end
+
+    def load_datayaml(directory_path = nil)
+      logger.debug "---- Config.load -----"
+      if directory_path.nil? or directory_path.empty?
+        directory_path = data_path
+      end
+
+      if @data and @data['data_path'] == directory_path
+        logger.debug "already loaded, nothing to do!"
+      else
+        logger.debug "load config #{directory_path.inspect}"
+        @data = load_yaml(directory_path)
+        @data['unique'] ||= []
+        logger.debug "config: #{@data.inspect[0..600]}"
+        @data['index'] ||= clean_index(@data_path)
+        endpoint = @data['api'] || clean_index(@data_path)
+        @dictionary = @data['dictionary'] || {}
+        @data['options'] ||= {}
+        Hashie.symbolize_keys! @data['options']
+        @api_endpoints[endpoint] = {index: @data['index']}
+        @files, @data['files'] = parse_files(directory_path, @data['files'], @data['options'])
+
+        logger.debug "file_config: #{@data['files'].inspect}"
+        logger.debug "no files found" if @data['files'].empty?
+
+        # keep track of where we loaded our data, so we can avoid loading again
+        @data['data_path'] = directory_path
+        @data_path = directory_path  # make sure this is set, in case it changed
+      end
+      scoped_index_name
+    end
+    # ---------
+
     def options
       @data['options']
     end
@@ -63,12 +127,6 @@ def category_by_id id
       Category.new(id).assemble
     end
 
-    def self.init(s3 = nil)
-      logger.info "Config.init #{s3.inspect}"
-      @s3 = s3
-      Config.load
-    end
-
     def clear_all
       unless @data.nil? or @data.empty?
         logger.info "Config.clear_all: deleting index '#{scoped_index_name}'"
@@ -77,18 +135,6 @@ def clear_all
       end
     end
 
-    def self.logger=(new_logger)
-      @logger = new_logger
-    end
-
-    def self.logger
-      @logger ||= Logger.new("log/#{ENV['RACK_ENV'] || 'development'}.log")
-    end
-
-    def logger
-      self.class.logger
-    end
-
     def csv_column_type(column_name)
       extract_csv_column_types[column_name.to_s]
     end
@@ -118,7 +164,6 @@ def api_endpoint_names
       @api_endpoints.keys
     end
 
-
     def find_index_for(api)
       api_info = @api_endpoints[api] || {}
       api_info[:index]
@@ -326,14 +371,6 @@ def index_needs_update?(index_name=nil)
       old_config.nil? || old_config["version"] != @data["version"]
     end
 
-    def read_path(path)
-      yaml_data.read(path)
-    end
-
-    def load_yaml(path = nil)
-      yaml_data.read_yaml(path)
-    end
-
     def list_files(path)
       Dir["#{path}/*"].select { |file|
         @extensions.include? File.extname(file)
@@ -383,37 +420,5 @@ def clean_index(path)
     def null_value
       @data['null_value'] || 'NULL'
     end
-
-    def load_datayaml(directory_path = nil)
-      logger.debug "---- Config.load -----"
-      if directory_path.nil? or directory_path.empty?
-        directory_path = data_path
-      end
-
-      if @data and @data['data_path'] == directory_path
-        logger.debug "already loaded, nothing to do!"
-      else
-        logger.debug "load config #{directory_path.inspect}"
-        @data = load_yaml(directory_path)
-        @data['unique'] ||= []
-        logger.debug "config: #{@data.inspect[0..600]}"
-        @data['index'] ||= clean_index(@data_path)
-        endpoint = @data['api'] || clean_index(@data_path)
-        @dictionary = @data['dictionary'] || {}
-        @data['options'] ||= {}
-        Hashie.symbolize_keys! @data['options']
-        @api_endpoints[endpoint] = {index: @data['index']}
-        @files, @data['files'] = parse_files(directory_path, @data['files'], @data['options'])
-
-        logger.debug "file_config: #{@data['files'].inspect}"
-        logger.debug "no files found" if @data['files'].empty?
-
-        # keep track of where we loaded our data, so we can avoid loading again
-        @data['data_path'] = directory_path
-        @data_path = directory_path  # make sure this is set, in case it changed
-      end
-      scoped_index_name
-    end
-
   end # class Config
 end # module DataMagic

From 2c8eb089e80affa9e1aeb12f59a63c7e40318db1 Mon Sep 17 00:00:00 2001
From: Kane Baccigalupi <kane.baccigalupi@gsa.gov>
Date: Fri, 26 Feb 2016 15:03:38 -0800
Subject: [PATCH 6/7] Extracts configuration values into own value object

        * renames the Config#load_datayaml to #load_yaml_data
        * moves over values to values, ha ha!
---
 .rspec                                 |  1 +
 lib/data_magic/config.rb               | 86 +++++++++++++-------------
 lib/data_magic/configuration/values.rb | 32 ++++++++++
 lib/data_magic/index.rb                |  2 +-
 spec/lib/data_magic/config_spec.rb     |  4 +-
 5 files changed, 79 insertions(+), 46 deletions(-)
 create mode 100644 .rspec
 create mode 100644 lib/data_magic/configuration/values.rb

diff --git a/.rspec b/.rspec
new file mode 100644
index 00000000..4e1e0d2f
--- /dev/null
+++ b/.rspec
@@ -0,0 +1 @@
+--color
diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb
index 06fd3534..09e18fcc 100644
--- a/lib/data_magic/config.rb
+++ b/lib/data_magic/config.rb
@@ -6,28 +6,25 @@
 require_relative 'load/s3_adapter'
 require_relative 'load/local_adapter'
 
+require_relative 'configuration/values'
+
+require 'forwardable'
+
 module DataMagic
   class Config
-    attr_reader :data_path, :data, :dictionary, :files, :s3, :api_endpoints,
-                :null_value, :file_config, :yaml_data
-
-    attr_accessor :page_size
+    attr_reader :data_path, :s3, :null_value, :file_config, :yaml_data, :values, :data_path
 
     def initialize(options = {})
-      @api_endpoints = {}
-      @files = []
-      @dictionary = {}
-      @page_size = DataMagic::DEFAULT_PAGE_SIZE
-      @extensions = DataMagic::DEFAULT_EXTENSIONS
+      @values = Configuration::Values.new
       @s3 = options[:s3]
 
       @yaml_data = Load::YamlData.new(options)
       @data_path = yaml_data.path
 
       if options[:load_datayaml] == false
-        @data = {}
+        values.data = {}
       else
-        load_datayaml
+        load_yaml_data
       end
     end
 
@@ -37,6 +34,9 @@ def self.init(s3 = nil)
       Config.load
     end
 
+    extend Forwardable
+    def_delegators :values, :api_endpoints, :dictionary, :files, :page_size, :extensions, :data
+
     # ----------
     # Logger stuff
     def self.logger=(new_logger)
@@ -59,36 +59,36 @@ def read_path(path)
       yaml_data.read(path)
     end
 
-    def load_yaml(path = nil)
+    def read_yaml(path = nil)
       yaml_data.read_yaml(path)
     end
 
-    def load_datayaml(directory_path = nil)
+    def load_yaml_data(directory_path = nil)
       logger.debug "---- Config.load -----"
       if directory_path.nil? or directory_path.empty?
         directory_path = data_path
       end
 
-      if @data and @data['data_path'] == directory_path
+      if data and data['data_path'] == directory_path
         logger.debug "already loaded, nothing to do!"
       else
         logger.debug "load config #{directory_path.inspect}"
-        @data = load_yaml(directory_path)
-        @data['unique'] ||= []
-        logger.debug "config: #{@data.inspect[0..600]}"
-        @data['index'] ||= clean_index(@data_path)
-        endpoint = @data['api'] || clean_index(@data_path)
-        @dictionary = @data['dictionary'] || {}
-        @data['options'] ||= {}
-        Hashie.symbolize_keys! @data['options']
-        @api_endpoints[endpoint] = {index: @data['index']}
-        @files, @data['files'] = parse_files(directory_path, @data['files'], @data['options'])
-
-        logger.debug "file_config: #{@data['files'].inspect}"
-        logger.debug "no files found" if @data['files'].empty?
+        values.data = read_yaml(directory_path)
+        data['unique'] ||= []
+        logger.debug "config: #{data.inspect[0..600]}"
+        data['index'] ||= clean_index(data_path)
+        endpoint = data['api'] || clean_index(data_path)
+        values.dictionary = data['dictionary'] || {}
+        data['options'] ||= {}
+        Hashie.symbolize_keys! data['options']
+        api_endpoints[endpoint] = {index: data['index']}
+        values.files, data['files'] = parse_files(directory_path, data['files'], data['options'])
+
+        logger.debug "file_config: #{data['files'].inspect}"
+        logger.debug "no files found" if data['files'].empty?
 
         # keep track of where we loaded our data, so we can avoid loading again
-        @data['data_path'] = directory_path
+        data['data_path'] = directory_path
         @data_path = directory_path  # make sure this is set, in case it changed
       end
       scoped_index_name
@@ -96,7 +96,7 @@ def load_datayaml(directory_path = nil)
     # ---------
 
     def options
-      @data['options']
+      data['options']
     end
 
     def dictionary_only_search?
@@ -128,7 +128,7 @@ def category_by_id id
     end
 
     def clear_all
-      unless @data.nil? or @data.empty?
+      unless data.nil? or data.empty?
         logger.info "Config.clear_all: deleting index '#{scoped_index_name}'"
         Stretchy.delete scoped_index_name
         DataMagic.client.indices.clear_cache
@@ -142,18 +142,18 @@ def csv_column_type(column_name)
     # fetches file configuration
     # add: whatever
     def additional_data_for_file(index)
-      @data.fetch('files', []).fetch(index, {}).fetch('add', nil)
+      data.fetch('files', []).fetch(index, {}).fetch('add', nil)
     end
 
     def info_for_file(index, field)
       field = field.to_s
-      result = @data.fetch('files', []).fetch(index, {}).fetch(field, nil)
+      result = data.fetch('files', []).fetch(index, {}).fetch(field, nil)
       result = IndifferentHash.new(result) if result.is_a? Hash
       result
     end
 
     def scoped_index_name(index_name = nil)
-      index_name ||= @data['index']
+      index_name ||= data['index']
       env = ENV['RACK_ENV']
       "#{env}-#{index_name}"
     end
@@ -161,19 +161,19 @@ def scoped_index_name(index_name = nil)
     # returns an array of api_endpoints
     # list of strings
     def api_endpoint_names
-      @api_endpoints.keys
+      api_endpoints.keys
     end
 
     def find_index_for(api)
-      api_info = @api_endpoints[api] || {}
+      api_info = api_endpoints[api] || {}
       api_info[:index]
     end
 
     def dictionary=(yaml_hash = {})
-      @dictionary = IndifferentHash.new(yaml_hash)
-      @dictionary.each do |key, info|
+      values.dictionary = IndifferentHash.new(yaml_hash)
+      dictionary.each do |key, info|
         if info === String
-          @dictionary[key] = {source: info}
+          dictionary[key] = {source: info}
         end
       end
     end
@@ -212,7 +212,7 @@ def make_nested(nest_config, all_fields)
     end
 
     def file_config
-      @data.fetch('files', [])
+      data.fetch('files', [])
     end
 
     # returns a hash that lets us know the types of what we read from csv
@@ -366,14 +366,14 @@ def index_needs_update?(index_name=nil)
         end
       end
       logger.debug "old config version (from es): #{(old_config.nil? ? old_config : old_config['version']).inspect}"
-      logger.debug "new config version (from data.yaml): #{@data['version'].inspect}"
+      logger.debug "new config version (from data.yaml): #{data['version'].inspect}"
 
-      old_config.nil? || old_config["version"] != @data["version"]
+      old_config.nil? || old_config["version"] != data["version"]
     end
 
     def list_files(path)
       Dir["#{path}/*"].select { |file|
-        @extensions.include? File.extname(file)
+        extensions.include? File.extname(file)
       }.map { |file|
         File.basename file
       }
@@ -418,7 +418,7 @@ def clean_index(path)
     end
 
     def null_value
-      @data['null_value'] || 'NULL'
+      data['null_value'] || 'NULL'
     end
   end # class Config
 end # module DataMagic
diff --git a/lib/data_magic/configuration/values.rb b/lib/data_magic/configuration/values.rb
new file mode 100644
index 00000000..f785f8f3
--- /dev/null
+++ b/lib/data_magic/configuration/values.rb
@@ -0,0 +1,32 @@
+module DataMagic
+  module Configuration
+    class Values
+      attr_writer :api_endpoints, :files, :dictionary, :page_size, :extensions, :data
+
+      def api_endpoints
+        @api_endpoints ||= {}
+      end
+
+      def dictionary
+        @dictionary ||= {}
+      end
+
+      def data
+        @data ||= {}
+      end
+
+      def files
+        @files ||= []
+      end
+
+      def page_size
+        @page_size ||= DataMagic::DEFAULT_PAGE_SIZE
+      end
+
+      def extensions
+        @extensions ||= DataMagic::DEFAULT_EXTENSIONS
+      end
+    end
+  end
+end
+
diff --git a/lib/data_magic/index.rb b/lib/data_magic/index.rb
index 0bd6dfa6..f1437b09 100644
--- a/lib/data_magic/index.rb
+++ b/lib/data_magic/index.rb
@@ -27,7 +27,7 @@ def self.import_with_dictionary(options = {})
     options[:mapping] = config.field_mapping
     options = options.merge(config.options)
 
-    es_index_name = self.config.load_datayaml(options[:data_path])
+    es_index_name = self.config.load_yaml_data(options[:data_path])
     unless config.index_exists?(es_index_name)
       logger.info "creating #{es_index_name}"   # TO DO: fix #14
       create_index es_index_name, config.field_types
diff --git a/spec/lib/data_magic/config_spec.rb b/spec/lib/data_magic/config_spec.rb
index 0903e6e8..440caf90 100644
--- a/spec/lib/data_magic/config_spec.rb
+++ b/spec/lib/data_magic/config_spec.rb
@@ -110,7 +110,7 @@
       context "after loading config" do
         let(:fixture_path) { "./spec/fixtures/import_with_dictionary" }
         before do
-          config.load_datayaml(fixture_path)
+          config.load_yaml_data(fixture_path)
         end
         it "should be true" do
           expect(config.update_indexed_config).to be true
@@ -132,7 +132,7 @@
       end
 
       it 'should set null value field' do
-        config.load_datayaml("./spec/fixtures/import_with_null_value")
+        config.load_yaml_data("./spec/fixtures/import_with_null_value")
         expect(config.null_value).to eq('abc123')
       end
     end

From 7639c4ed2d3635fc743ff3b92d6bd45a0aa87984 Mon Sep 17 00:00:00 2001
From: Kane Baccigalupi <kane.baccigalupi@gsa.gov>
Date: Mon, 29 Feb 2016 09:15:40 -0800
Subject: [PATCH 7/7] fixes git merge fail

---
 lib/data_magic/config.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/data_magic/config.rb b/lib/data_magic/config.rb
index 09e18fcc..80b68f5c 100644
--- a/lib/data_magic/config.rb
+++ b/lib/data_magic/config.rb
@@ -324,7 +324,7 @@ def recreate_indexed_config
       # index when we update the configuration document
       DataMagic.client.indices.delete index: scoped_index_name if index_exists?
       DataMagic.create_index(scoped_index_name, field_types)  ## DataMagic::Index.create ?
-      DataMagic.client.index index: scoped_index_name, type: 'config', id: 1, body: @data
+      DataMagic.client.index index: scoped_index_name, type: 'config', id: 1, body: data
     end
 
     def delete_index_and_reload_config
@@ -345,7 +345,6 @@ def update_indexed_config
       updated
     end
 
-
     def index_exists?(index_name=nil)
       index_name ||= scoped_index_name
       logger.debug "looking for: #{index_name}"