From cc6dfd0993f8b6a5f700355f6c5fb181506440a8 Mon Sep 17 00:00:00 2001 From: Brian Jinwright Date: Thu, 7 Sep 2017 14:55:26 -0400 Subject: [PATCH 1/8] Notice that DynamoDB and Cloudant moved to DocB --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b13458..432bc4d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ ![alt text](https://s3.amazonaws.com/capless/images/kev-small.png "KEV - Keys, Extra Stuff, and Values") - # kev K.E.V. (Keys, Extra Stuff, and Values) is a Python ORM for key-value stores and document databases based on [**Valley**](https://www.github.com/capless/valley). Currently supported backends are Redis, S3, DynamoDB and a S3/Redis hybrid backend. +## PROJECT UPDATE +The DynamoDB and Cloudant backends were moved to [DocB](https://github.com/capless/docb). The Redis, S3, and S3/Redis backend will continue supported here in Kev. We felt the need to split up the key-value stores and document databases. + [![Build Status](https://travis-ci.org/capless/kev.svg?branch=master)](https://travis-ci.org/capless/kev) ## Python Versions From cec7bb77b38a201722d51b49d08d65686980350a Mon Sep 17 00:00:00 2001 From: Brian Jinwright Date: Wed, 22 Aug 2018 14:43:17 -0400 Subject: [PATCH 2/8] Update README.md --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 432bc4d..228bd73 100644 --- a/README.md +++ b/README.md @@ -51,12 +51,6 @@ kev_handler = KevHandler({ 'host': 'your-redis-host.com', 'port': 6379, } - }, - 'dynamodb': { - 'backend': 'kev.backends.dynamodb.db.DynamoDB', - 'connection': { - 'table': 'your-dynamodb-table', - } } }) ``` From 902f4d5d89482b2eedcf9a396d57be7083357024 Mon Sep 17 00:00:00 2001 From: Brian Jinwright Date: Wed, 22 Aug 2018 14:57:55 -0400 Subject: [PATCH 3/8] Update README.md --- README.md | 87 ------------------------------------------------------- 1 file changed, 87 deletions(-) diff --git a/README.md b/README.md index 228bd73..3ca77b5 100644 --- a/README.md +++ b/README.md @@ -169,93 +169,6 @@ Prefix filters currently only work with the S3 backend. Use wildcard filters wit >>>TestDocument.objects().filter({'state':'N'}) [] ``` -### DynamoDB setup -#### Create a table -* **Table name** should be between 3 and 255 characters long. (A-Z,a-z,0-9,_,-,.) -* **Primary key** (partition key) should be equal to `_id` - -#### Filter Documents -If you want to make `filter()` queries, you should create an index for every attribute that you want to filter by. -* **Primary key** should be equal to attribute name. -* **Index name** should be equal to attribute name postfixed by *"-index"*. (It will be filled by AWS automatically). -For example, for attribute *"city"*: *Primary key* = *"city"* and index name = *"city-index"*. -* **Index name** can be directly specified by `index_name` argument: -```python - name = CharProperty(required=True,unique=True,min_length=5,max_length=20,index_name='name_index') -``` -- **IMPORTANT: In other words, if your indexed attribute is named city, then your index name should be city-index, -if you didn't specify `index_name` argument.** -* **Projected attributes**: *All*. - -### Use DynamoDB locally -#### Run DynamoDB -* with persistent storage `docker run -d -p 8000:8000 -v /tmp/data:/data/ dwmkerr/dynamodb -dbPath /data/` - -#### Configuration -**Example:** loading.py -```python -from kev.loading import KevHandler - - -kev_handler = KevHandler({ - 'dynamodb': { - 'backend': 'kev.backends.dynamodb.db.DynamoDB', - 'connection': { - 'table': 'your-dynamodb-table', - 'endpoint_url': 'http://127.0.0.1:8000' - } - } -}) -``` - -#### Testing -##### Run DynamoDB -* in memory (best performance) `docker run -d -p 8000:8000 dwmkerr/dynamodb -inMemory` - -##### Create a table for testing. - -```python -import boto3 - - -table_wcu = 2000 -table_rcu = 2000 -index_wcu = 3000 -index_rcu = 2000 -table_name = 'localtable' - -dynamodb = boto3.resource('dynamodb', endpoint_url="http://127.0.0.1:8000") -dynamodb.create_table(TableName=table_name, KeySchema=[{'AttributeName': '_id', 'KeyType': 'HASH'}], - ProvisionedThroughput={'ReadCapacityUnits': table_rcu, - 'WriteCapacityUnits': table_wcu}, - AttributeDefinitions=[{'AttributeName': '_id', 'AttributeType': 'S'}, - {u'AttributeName': u'city', u'AttributeType': u'S'}, - {u'AttributeName': u'email', u'AttributeType': u'S'}, - {u'AttributeName': u'name', u'AttributeType': u'S'}, - {u'AttributeName': u'slug', u'AttributeType': u'S'}], - GlobalSecondaryIndexes=[ - {'IndexName': 'city-index', 'Projection': {'ProjectionType': 'ALL'}, - 'ProvisionedThroughput': {'WriteCapacityUnits': index_wcu, - 'ReadCapacityUnits': index_rcu}, - 'KeySchema': [{'KeyType': 'HASH', 'AttributeName': 'city'}]}, - {'IndexName': 'name-index', 'Projection': {'ProjectionType': 'ALL'}, - 'ProvisionedThroughput': {'WriteCapacityUnits': index_wcu, - 'ReadCapacityUnits': index_rcu}, - 'KeySchema': [{'KeyType': 'HASH', 'AttributeName': 'name'}]}, - {'IndexName': 'slug-index', 'Projection': {'ProjectionType': 'ALL'}, - 'ProvisionedThroughput': {'WriteCapacityUnits': index_wcu, - 'ReadCapacityUnits': index_rcu}, - 'KeySchema': [{'KeyType': 'HASH', 'AttributeName': 'slug'}]}, - {'IndexName': 'email-index', 'Projection': {'ProjectionType': 'ALL'}, - 'ProvisionedThroughput': {'WriteCapacityUnits': index_wcu, - 'ReadCapacityUnits': index_rcu}, - 'KeySchema': [{'KeyType': 'HASH', 'AttributeName': 'email'}]}]) -``` -##### Setup environment variables. -```bash -export DYNAMO_TABLE_TEST='localtable' -export DYNAMO_ENDPOINT_URL_TEST='http://127.0.0.1:8000' -``` ### Backup and Restore From e569a122304d41e5f50f1ff7873c9fa552420b1e Mon Sep 17 00:00:00 2001 From: Benoit Brayer Date: Fri, 20 Dec 2019 12:17:22 +0100 Subject: [PATCH 4/8] Add proper support to custom endpoint_url --- kev/backends/s3/db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kev/backends/s3/db.py b/kev/backends/s3/db.py index 586826f..e9e50ae 100644 --- a/kev/backends/s3/db.py +++ b/kev/backends/s3/db.py @@ -21,10 +21,14 @@ def __init__(self,**kwargs): # session_kwargs = {k: v for k, v in kwargs.items() if k in self.session_kwargs} + endpoint_url = None + if 'endpoint_url' in session_kwargs.keys(): + endpoint_url = session_kwargs['endpoint_url'] + del(session_kwargs['endpoint_url']) if len(session_kwargs.keys()) > 0: boto3.Session(**session_kwargs) - self._db = boto3.resource('s3') + self._db = boto3.resource('s3', endpoint_url=endpoint_url) self.bucket = kwargs['bucket'] self._indexer = self._db.Bucket(self.bucket) From 613e2c71c04704aacc62b7dbfdc19f48e9377ca0 Mon Sep 17 00:00:00 2001 From: Benoit Brayer Date: Fri, 20 Dec 2019 14:03:27 +0100 Subject: [PATCH 5/8] Fix parse_requirements import for pip >= 10 --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90ff614..57b8987 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,9 @@ from setuptools import setup, find_packages -from pip.req import parse_requirements + +try: # Fix for pip >= 10 + from pip._internal.req import parse_requirements +except ImportError: # for pip <= 9.0.3 + from pip.req import parse_requirements install_reqs = parse_requirements('requirements.txt', session=False) From e1f0ba8141dac922e5546cf29b9b04d858e54427 Mon Sep 17 00:00:00 2001 From: Benoit Brayer Date: Fri, 20 Dec 2019 14:22:48 +0100 Subject: [PATCH 6/8] Add support to region_name Session attribute. ?! --- kev/backends/s3/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kev/backends/s3/db.py b/kev/backends/s3/db.py index e9e50ae..0e421be 100644 --- a/kev/backends/s3/db.py +++ b/kev/backends/s3/db.py @@ -15,7 +15,7 @@ class S3DB(DocDB): '[-\W\w\s]+)/(?P[-\w]+):id:' \ '(?P[-\w]+):(?P[-\w]+)$' session_kwargs = ['aws_secret_access_key', 'aws_access_key_id', - 'endpoint_url'] + 'region_name', 'endpoint_url'] def __init__(self,**kwargs): # From 36ce4319c5399e8ce32cdb8b59c4139a0faf137d Mon Sep 17 00:00:00 2001 From: Benoit Brayer Date: Fri, 20 Dec 2019 15:22:53 +0100 Subject: [PATCH 7/8] Add support to signature_version. --- kev/backends/s3/db.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/kev/backends/s3/db.py b/kev/backends/s3/db.py index 0e421be..0300a07 100644 --- a/kev/backends/s3/db.py +++ b/kev/backends/s3/db.py @@ -15,7 +15,8 @@ class S3DB(DocDB): '[-\W\w\s]+)/(?P[-\w]+):id:' \ '(?P[-\w]+):(?P[-\w]+)$' session_kwargs = ['aws_secret_access_key', 'aws_access_key_id', - 'region_name', 'endpoint_url'] + 'region_name', 'endpoint_url', + 'signature_version'] def __init__(self,**kwargs): # @@ -25,10 +26,23 @@ def __init__(self,**kwargs): if 'endpoint_url' in session_kwargs.keys(): endpoint_url = session_kwargs['endpoint_url'] del(session_kwargs['endpoint_url']) + signature_version = None + if 'signature_version' in session_kwargs.keys(): + signature_version = session_kwargs['signature_version'] + del(session_kwargs['signature_version']) if len(session_kwargs.keys()) > 0: boto3.Session(**session_kwargs) - self._db = boto3.resource('s3', endpoint_url=endpoint_url) + if signature_version is not None: + self._db = boto3.resource( + 's3', + endpoint_url=endpoint_url, + config=boto3.session.Config( + signature_version=signature_version + ) + ) + else: + self._db = boto3.resource('s3', endpoint_url=endpoint_url) self.bucket = kwargs['bucket'] self._indexer = self._db.Bucket(self.bucket) From 2924799c0d38e7024746c51a668ff8e323294dae Mon Sep 17 00:00:00 2001 From: Benoit Brayer Date: Fri, 20 Dec 2019 15:47:25 +0100 Subject: [PATCH 8/8] Do not bother with Session... --- kev/backends/s3/db.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/kev/backends/s3/db.py b/kev/backends/s3/db.py index 0300a07..fb62df9 100644 --- a/kev/backends/s3/db.py +++ b/kev/backends/s3/db.py @@ -20,29 +20,21 @@ class S3DB(DocDB): def __init__(self,**kwargs): # - session_kwargs = {k: v for k, v in kwargs.items() if k in - self.session_kwargs} - endpoint_url = None - if 'endpoint_url' in session_kwargs.keys(): - endpoint_url = session_kwargs['endpoint_url'] - del(session_kwargs['endpoint_url']) - signature_version = None + session_kwargs = { + k: v for k, v in kwargs.items() if k in self.session_kwargs + } if 'signature_version' in session_kwargs.keys(): signature_version = session_kwargs['signature_version'] del(session_kwargs['signature_version']) - if len(session_kwargs.keys()) > 0: - boto3.Session(**session_kwargs) - - if signature_version is not None: self._db = boto3.resource( 's3', - endpoint_url=endpoint_url, config=boto3.session.Config( - signature_version=signature_version - ) + signature_version=signature_version + ), + **session_kwargs ) else: - self._db = boto3.resource('s3', endpoint_url=endpoint_url) + self._db = boto3.resource('s3', **session_kwargs) self.bucket = kwargs['bucket'] self._indexer = self._db.Bucket(self.bucket)