From 309c53dee6bec6a3b9447ae9cf9461cfec9ad477 Mon Sep 17 00:00:00 2001 From: ChangJoo Park Date: Fri, 23 Aug 2019 13:19:45 +0900 Subject: [PATCH] Support MongoDB replica set connections / clients https://github.com/slacy/minimongo/pull/31#issue-3607082 --- .gitignore | 1 + README.rst | 3 +- minimongo/collection.py | 1 + minimongo/model.py | 78 ++++++++++++++++++----------------- minimongo/options.py | 4 ++ minimongo/tests/test_model.py | 75 ++++++++++++++------------------- setup.py | 16 +++---- 7 files changed, 88 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index a2b31ea..4b93e5a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ minimongo.egg-info/ minimongo/app_config.py pymongo-*/ +venv diff --git a/README.rst b/README.rst index cd39778..7e33f6a 100644 --- a/README.rst +++ b/README.rst @@ -28,10 +28,11 @@ you can use ``easy_install -U minimongo``. Otherwise, you can download the source from `GitHub `_ and run ``python setup.py install``. +Instead of installing as above, you should use `pip install -e .`. Dependencies ============ -- pymongo_ 1.9+ +- pymongo_ 2.8+ - `sphinx `_ (optional -- for documentation generation) diff --git a/minimongo/collection.py b/minimongo/collection.py index c9022ae..646698a 100644 --- a/minimongo/collection.py +++ b/minimongo/collection.py @@ -7,6 +7,7 @@ class Cursor(PyMongoCursor): def __init__(self, *args, **kwargs): self._wrapper_class = kwargs.pop('wrap') + kwargs['projection'] = kwargs.pop('fields', None) super(Cursor, self).__init__(*args, **kwargs) def next(self): diff --git a/minimongo/model.py b/minimongo/model.py index 523e0cf..27a892e 100644 --- a/minimongo/model.py +++ b/minimongo/model.py @@ -1,21 +1,26 @@ # -*- coding: utf-8 -*- -import copy +from __future__ import absolute_import +import copy +import logging import re + +import six from bson import DBRef, ObjectId -from minimongo.collection import DummyCollection -from minimongo.options import _Options -from pymongo import Connection +from pymongo import MongoClient, MongoReplicaSetClient +from pymongo.read_preferences import ReadPreference + +from .collection import DummyCollection +from .options import _Options class ModelBase(type): """Metaclass for all models. - .. todo:: add Meta inheritance -- so that missing attributes are populated from the parrent's Meta if any. """ - # A very rudimentary connection pool. + # A very rudimentary connection pool, keyed by replicaSet name. _connections = {} def __new__(mcs, name, bases, attrs): @@ -32,8 +37,8 @@ def __new__(mcs, name, bases, attrs): except AttributeError: meta = None else: - delattr(new_class, 'Meta') # Won't need the original metadata - # container anymore. + # Won't need the original metadata container anymore. + delattr(new_class, 'Meta') options = _Options(meta) options.collection = options.collection or to_underscore(name) @@ -49,21 +54,30 @@ def __new__(mcs, name, bases, attrs): 'Model %r improperly configured: %s %s %s' % ( name, options.host, options.port, options.database)) - # Checking connection pool for an existing connection. - hostport = options.host, options.port - if hostport in mcs._connections: - connection = mcs._connections[hostport] + # Checking connection / client pool for an existing connection / client. + pool_key = options.host,options.port + if options.replica_set_name: + logging.debug("Using replica_set_name=%s as database pool key." % options.replica_set_name) + pool_key = options.replica_set_name + + if pool_key in mcs._connections: + client = mcs._connections[pool_key] + logging.debug("Got database client from pool for pool_key=%s" % (pool_key,)) else: - # _connect=False option - # creates :class:`pymongo.connection.Connection` object without - # establishing connection. It's required if there is no running - # mongodb at this time but we want to create :class:`Model`. - connection = Connection(*hostport, _connect=False) - mcs._connections[hostport] = connection + logging.debug("Creating new database client for pool_key=%s" % (pool_key,)) + if options.replica_set_name: + logging.debug("Setting up a replica set client...") + client = MongoReplicaSetClient(options.replica_set_uri, replicaSet=options.replica_set_name) + client.read_preference = ReadPreference.SECONDARY_PREFERRED + else: + logging.debug("Setting up a normal client...") + client = MongoClient(options.host, options.port) + + mcs._connections[pool_key] = client new_class._meta = options - new_class.connection = connection - new_class.database = connection[options.database] + new_class.connection = client + new_class.database = client[options.database] if options.username and options.password: new_class.database.authenticate(options.username, options.password) new_class.collection = options.collection_class( @@ -76,13 +90,11 @@ def __new__(mcs, name, bases, attrs): def auto_index(mcs): """Builds all indices, listed in model's Meta class. - >>> class SomeModel(Model) ... class Meta: ... indices = ( ... Index('foo'), ... ) - .. note:: this will result in calls to :meth:`pymongo.collection.Collection.ensure_index` method at import time, so import all your models up @@ -98,13 +110,13 @@ def __init__(self, initial=None, **kwargs): # AttrDict. Maybe this could be better done with the builtin # defaultdict? if initial: - for key, value in initial.iteritems(): + for key, value in six.iteritems(initial): # Can't just say self[k] = v here b/c of recursion. self.__setitem__(key, value) # Process the other arguments (assume they are also default values). # This is the same behavior as the regular dict constructor. - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): self.__setitem__(key, value) super(AttrDict, self).__init__() @@ -141,9 +153,10 @@ def __setitem__(self, key, value): return super(AttrDict, self).__setitem__(key, new_value) +@six.python_2_unicode_compatible +@six.add_metaclass(ModelBase) class Model(AttrDict): """Base class for all Minimongo objects. - >>> class Foo(Model): ... class Meta: ... database = 'somewhere' @@ -157,16 +170,10 @@ class Model(AttrDict): >>> foo.bar == 42 True """ - - __metaclass__ = ModelBase - def __str__(self): return '%s(%s)' % (self.__class__.__name__, super(Model, self).__str__()) - def __unicode__(self): - return str(self).decode('utf-8') - def __setitem__(self, key, value): # Go through the defined list of field mappers. If the fild # matches, then modify the field value by calling the function in @@ -186,10 +193,8 @@ def __setitem__(self, key, value): def dbref(self, with_database=True, **kwargs): """Returns a DBRef for the current object. - If `with_database` is False, the resulting :class:`pymongo.dbref.DBRef` won't have a :attr:`database` field. - Any other parameters will be passed to the DBRef constructor, as per the mongo specs. """ @@ -217,15 +222,14 @@ def mongo_update(self, values=None, **kwargs): def save(self, *args, **kwargs): """Save this object to it's mongo collection.""" + kwargs.pop('safe', None) self.collection.save(self, *args, **kwargs) return self def load(self, fields=None, **kwargs): """Allow partial loading of a document. :attr:fields is a dictionary as per the pymongo specs - self.collection.find_one( self._id, fields={'name': 1} ) - """ values = self.collection.find_one({'_id': self._id}, fields=fields, **kwargs) @@ -238,10 +242,8 @@ def load(self, fields=None, **kwargs): def to_underscore(string): """Converts a given string from CamelCase to under_score. - >>> to_underscore('FooBar') 'foo_bar' """ new_string = re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', string) - new_string = re.sub(r'([a-z\d])([A-Z])', r'\1_\2', new_string) - return new_string.lower() + new_string = re.sub(r'([a-z\d])([A-Z])', r'\1_\2', new_string) \ No newline at end of file diff --git a/minimongo/options.py b/minimongo/options.py index 96b497c..1bd6e37 100644 --- a/minimongo/options.py +++ b/minimongo/options.py @@ -51,6 +51,10 @@ class _Options(object): username = None password = None + # Replica set details. + replica_set_name = None + replica_set_uri = None + # Should indices be created at startup? auto_index = True diff --git a/minimongo/tests/test_model.py b/minimongo/tests/test_model.py index 91e7b37..4c2f9a8 100644 --- a/minimongo/tests/test_model.py +++ b/minimongo/tests/test_model.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement -import operator +from __future__ import absolute_import, unicode_literals import pytest - from bson import DBRef -from minimongo import Collection, Index, Model from pymongo.errors import DuplicateKeyError +from .. import Collection, Index, Model + class TestCollection(Collection): def custom(self): @@ -166,7 +165,8 @@ def test_save_with_arguments(): model = TestModel(foo=0) model.save(manipulate=False) with pytest.raises(AttributeError): - _value = model._id + model._id + # but the object was actually saved model = TestModel.collection.find_one({'foo': 0}) assert model.foo == 0 @@ -205,12 +205,13 @@ def test_load(): object_a = TestModel(x=0, y=1).save() object_b = TestModel(_id=object_a._id) with pytest.raises(AttributeError): - _value = object_b.x + object_b.x + # Partial load. only the x value object_b.load(fields={'x': 1}) assert object_b.x == object_a.x with pytest.raises(AttributeError): - _value = object_b.y + object_b.y # Complete load. change the value first object_a.x = 2 @@ -231,7 +232,7 @@ def test_load_and_field_mapper(): object_b.load(fields={'x': 1}) assert object_b.x == 16.0 with pytest.raises(AttributeError): - _value = object_b.y # object_b does not have the 'y' field + object_b.y # object_b does not have the 'y' field object_b.load() assert object_b.y == 1 @@ -240,15 +241,19 @@ def test_load_and_field_mapper(): def test_index_existance(): '''Test that indexes were created properly.''' indices = TestModel.collection.index_information() - assert (indices['x_1'] == {'key': [('x', 1)]})\ - or (indices['x_1'] == {u'key': [(u'x', 1)], u'v': 1}) + # Even though PyMongo documents that indices should not contain + # "ns", the seem to do in practice. + assert "x_1" in indices + assert indices["x_1"]["key"] == [("x", 1)] +@pytest.mark.xfail(reason="drop_dups is unsupported since MongoDB 2.7.5") def test_unique_index(): '''Test behavior of indices with unique=True''' # This will work (y is undefined) TestModelUnique({'x': 1}).save() TestModelUnique({'x': 1}).save() + # Assert that there's only one object in the collection, even though # we inserted two. The uniqueness constraint on the index has dropped # one of the inserts (silently, I guess). @@ -308,8 +313,7 @@ def test_deletion(): object_b = TestModel.collection.find({'x': 100}) assert object_b.count() == 1 - - map(operator.methodcaller('remove'), object_b) + object_b[0].remove() object_a = TestModel.collection.find({'x': 100}) assert object_a.count() == 0 @@ -323,7 +327,7 @@ def test_complex_types(): object_a.y = {'m': 'n', 'o': 'p'} object_a['z'] = {'q': 'r', - 's': {'t': 'u'}} + 's': {'t': ''}} object_a.save() @@ -355,11 +359,8 @@ def test_complex_types(): def test_type_from_cursor(): - object_a = TestModel({'x':1}).save() - object_b = TestModel({'x':2}).save() - object_c = TestModel({'x':3}).save() - object_d = TestModel({'x':4}).save() - object_e = TestModel({'x':5}).save() + for i in range(6): + TestModel({'x': i}).save() objects = TestModel.collection.find() for single_object in objects: @@ -368,7 +369,7 @@ def test_type_from_cursor(): assert isinstance(single_object, dict) assert isinstance(single_object, object) assert isinstance(single_object, TestModel) - assert type(single_object['x']) == int + assert isinstance(single_object['x'], int) def test_delete_field(): @@ -378,8 +379,9 @@ def test_delete_field(): del object_a.x object_a.save() - assert TestModel.collection.find_one({'y': 2}) == \ - {'y': 2, '_id': object_a._id} + assert TestModel.collection.find_one({'y': 2}) == { + 'y': 2, '_id': object_a._id + } def test_count_and_fetch(): @@ -476,19 +478,9 @@ def test_collection_class(): assert model.collection.custom() == 'It works!' -def test_str_and_unicode(): +def test_str(): assert str(TestModel()) == 'TestModel({})' - assert str(TestModel({'foo': 'bar'})) == 'TestModel({\'foo\': \'bar\'})' - - assert unicode(TestModel({'foo': 'bar'})) == \ - u'TestModel({\'foo\': \'bar\'})' - - # __unicode__() doesn't decode any bytestring values to unicode, - # leaving it up to the user. - assert unicode(TestModel({'foo': '←'})) == \ - u'TestModel({\'foo\': \'\\xe2\\x86\\x90\'})' - assert unicode(TestModel({'foo': u'←'})) == \ - u'TestModel({\'foo\': u\'\\u2190\'})' + assert str(TestModel({'foo': 'bar'})) == 'TestModel({u\'foo\': u\'bar\'})' def test_auto_collection_name(): @@ -505,19 +497,14 @@ class Meta: def test_no_auto_index(): TestNoAutoIndexModel({'x': 1}).save() - assert (TestNoAutoIndexModel.collection.index_information() == \ - {u'_id_': {u'key': [(u'_id', 1)]}})\ - or (TestNoAutoIndexModel.collection.index_information() ==\ - {u'_id_': {u'key': [(u'_id', 1)], u'v': 1}}) + indices = TestNoAutoIndexModel.collection.index_information() + assert indices["_id_"]["key"] == [("_id", 1)] TestNoAutoIndexModel.auto_index() - assert (TestNoAutoIndexModel.collection.index_information() == \ - {u'_id_': {u'key': [(u'_id', 1)], u'v': 1}, - u'x_1': {u'key': [(u'x', 1)], u'v': 1}})\ - or (TestNoAutoIndexModel.collection.index_information() == \ - {u'_id_': {u'key': [(u'_id', 1)]}, - u'x_1': {u'key': [(u'x', 1)]}}) + indices = TestNoAutoIndexModel.collection.index_information() + assert indices["_id_"]["key"] == [("_id", 1)] + assert indices["x_1"]["key"] == [("x", 1)] def test_interface_models(): @@ -584,4 +571,4 @@ def test_slicing(): assert obj_list == [object_c, object_d, object_e] assert type(obj_list[0] == TestModel) assert type(obj_list[1] == TestModel) - assert type(obj_list[2] == TestModel) + assert type(obj_list[2] == TestModel) \ No newline at end of file diff --git a/setup.py b/setup.py index 094cb85..f116f68 100644 --- a/setup.py +++ b/setup.py @@ -13,19 +13,21 @@ here = os.path.abspath(os.path.dirname(__file__)) DESCRIPTION = "Minimal database Model management for MongoDB" -LONG_DESCRIPTION = DESCRIPTION try: LONG_DESCRIPTION = open(os.path.join(here, "README.rst")).read() except IOError: - pass + print("Warning: IOError raised: cannot open README.rst.") + LONG_DESCRIPTION = DESCRIPTION CLASSIFIERS = ( "Development Status :: 3 - Alpha", + "License :: OSI Approved :: BSD License" "Intended Audience :: Developers", - "Programming Language :: Python", "Topic :: Database", + "Programming Language :: Python", + "Programming Language :: Python :: 2", ) @@ -44,12 +46,12 @@ def run(self): requires = ["pymongo"] setup(name="minimongo", - version="0.2.6", + version="0.3.4", packages=find_packages(), cmdclass={"test": PyTest}, platforms=["any"], - install_requires = ["pymongo>=1.9"], + install_requires = ["pymongo>=2.9"], zip_safe=False, include_package_data=True, @@ -59,5 +61,5 @@ def run(self): long_description=LONG_DESCRIPTION, classifiers=CLASSIFIERS, keywords=["mongo", "mongodb", "pymongo", "orm"], - url="http://github.com/slacy/minimongo", -) + url="https://github.com/HappierApp/minimongo", +) \ No newline at end of file