diff --git a/.gitignore b/.gitignore index 4518fac54..006519bff 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ dump.rdb /.pyflymakercc /.emacs.desktop /redisco/.ropeproject/config.py +.Python +lib +bin +include diff --git a/redisco/models/attributes.py b/redisco/models/attributes.py index aa36d5d3b..a2e342de4 100644 --- a/redisco/models/attributes.py +++ b/redisco/models/attributes.py @@ -12,7 +12,7 @@ __all__ = ['Attribute', 'CharField', 'ListField', 'DateTimeField', 'DateField', 'ReferenceField', 'Collection', 'IntegerField', - 'FloatField', 'BooleanField', 'Counter', 'ZINDEXABLE'] + 'FloatField', 'BooleanField', 'Counter'] class Attribute(object): @@ -46,6 +46,7 @@ def __init__(self, self.validator = validator self.default = default self.unique = unique + self.zindexable = False def __get__(self, instance, owner): try: @@ -119,11 +120,8 @@ def __init__(self, max_length=255, **kwargs): def validate(self, instance): errors = [] - try: - super(CharField, self).validate(instance) - except FieldValidationError as err: - errors.extend(err.errors) - + super(CharField, self).validate(instance) + val = getattr(instance, self.name) if val and len(val) > self.max_length: @@ -150,6 +148,10 @@ def acceptable_types(self): class IntegerField(Attribute): + def __init__(self, **kwargs): + super(IntegerField, self).__init__(**kwargs) + self.zindexable = True + def typecast_for_read(self, value): return int(value) @@ -166,6 +168,10 @@ def acceptable_types(self): class FloatField(Attribute): + def __init__(self, **kwargs): + super(FloatField, self).__init__(**kwargs) + self.zindexable = True + def typecast_for_read(self, value): return float(value) @@ -187,6 +193,7 @@ def __init__(self, auto_now=False, auto_now_add=False, **kwargs): super(DateTimeField, self).__init__(**kwargs) self.auto_now = auto_now self.auto_now_add = auto_now_add + self.zindexable = True def typecast_for_read(self, value): try: @@ -221,6 +228,7 @@ def __init__(self, auto_now=False, auto_now_add=False, **kwargs): super(DateField, self).__init__(**kwargs) self.auto_now = auto_now self.auto_now_add = auto_now_add + self.zindexable = True def typecast_for_read(self, value): try: @@ -280,7 +288,7 @@ def __get__(self, instance, owner): if instance.is_new(): val = self.default else: - key = instance.key()[self.name] + key = instance.key(att=self.name) val = List(key).members if val is not None: klass = self.value_type() @@ -446,6 +454,7 @@ def __init__(self, **kwargs): super(Counter, self).__init__(**kwargs) if not kwargs.has_key('default') or self.default is None: self.default = 0 + self.zindexable = True def __set__(self, instance, value): raise AttributeError("can't set a counter.") @@ -458,6 +467,3 @@ def __get__(self, instance, owner): return int(v) else: return 0 - - -ZINDEXABLE = (IntegerField, DateTimeField, DateField, FloatField, Counter) diff --git a/redisco/models/base.py b/redisco/models/base.py index 3cf70b989..4cb425d66 100644 --- a/redisco/models/base.py +++ b/redisco/models/base.py @@ -4,15 +4,12 @@ import redisco from redisco.containers import Set, List, SortedSet, NonPersistentList from attributes import * -from key import Key from managers import ManagerDescriptor, Manager from exceptions import FieldValidationError, MissingID, BadKeyError, WatchError from attributes import Counter __all__ = ['Model', 'from_key'] -ZINDEXABLE = (IntegerField, DateTimeField, DateField, FloatField) - ############################## # Model Class Initialization # ############################## @@ -150,12 +147,13 @@ def _initialize_counters(model_class, name, bases, attrs): model_class._counters.append(k) -def _initialize_key(model_class, name): +def _initialize_keys(model_class, name): """ Initializes the key of the model. """ - model_class._key = Key(model_class._meta['key'] or name) - + model_class._key = model_class._meta['key'] or name + model_class._all_key = u"%s:all" % model_class._key + model_class._id_gen_key = u"%s:id" % model_class._key def _initialize_manager(model_class): """ @@ -208,7 +206,7 @@ def __init__(cls, name, bases, attrs): _initialize_counters(cls, name, bases, attrs) _initialize_lists(cls, name, bases, attrs) _initialize_indices(cls, name, bases, attrs) - _initialize_key(cls, name) + _initialize_keys(cls, name) _initialize_manager(cls) # if targeted by a reference field using a string, # override for next try @@ -298,10 +296,10 @@ def update_attributes(self, **kwargs): if att.name in kwargs: att.__set__(self, kwargs[att.name]) - def save(self): + def save(self, validate_fields=True): """ Saves the instance to the datastore with the following steps: - 1. Validate all the fields + 1. Validate all the fields (DEFAULT/OPTIONAL. Validation can be turned off with validate_fields=False) 2. Assign an ID if the object is new 3. Save to the datastore. @@ -315,13 +313,12 @@ def save(self): True >>> f.delete() """ - if not self.is_valid(): - return self._errors + if validate_fields and not self.is_valid(): + return False _new = self.is_new() if _new: self._initialize_id() - with Mutex(self): - self._write(_new) + self._write(_new) return True def key(self, att=None): @@ -340,9 +337,9 @@ def key(self, att=None): True """ if att is not None: - return self._key[self.id][att] + return u"%s:%s" % (self._instance_key, att) else: - return self._key[self.id] + return self._instance_key def delete(self): """Deletes the object from the datastore.""" @@ -443,18 +440,6 @@ def id(self): raise MissingID return self._id - @id.setter - def id(self, val): - """ - Setting the id for the object will fetch it from the datastorage. - """ - self._id = str(val) - stored_attrs = self.db.hgetall(self.key()) - attrs = self.attributes.values() - for att in attrs: - if att.name in stored_attrs and not isinstance(att, Counter): - att.__set__(self, att.typecast_for_read(stored_attrs[att.name])) - @property def attributes(self): """Return the attributes of the model. @@ -510,6 +495,12 @@ def counters(self): """Returns the mapping of the counters.""" return self._counters + def load_attributes_from_redis_raw_data(self, stored_attrs): + attrs = self.attributes.values() + for att in attrs: + if att.name in stored_attrs and not isinstance(att, Counter): + att.__set__(self, att.typecast_for_read(stored_attrs[att.name])) + ################# # Class Methods # ################# @@ -517,8 +508,31 @@ def counters(self): @classmethod def exists(cls, id): """Checks if the model with id exists.""" - return bool((cls._meta['db'] or redisco.get_client()).exists(cls._key[str(id)]) or - (cls._meta['db'] or redisco.get_client()).sismember(cls._key['all'], str(id))) + _conn = cls._meta['db'] or redisco.get_client() + return bool(_conn.exists(cls.instance_key(id)) or _conn.sismember(cls._all_key, str(id))) + + @classmethod + def instance_key(cls, id): + return u"%s:%s" % (cls._key, str(id)) + + @classmethod + def get_by_id(cls, id): + instance = cls() + instance._id = str(id) + instance._set_instance_keys() + stored_attrs = instance.db.hgetall(instance._instance_key) + if not stored_attrs: # object does not exist + return None + instance.load_attributes_from_redis_raw_data(stored_attrs) + return instance + + @classmethod + def load_from_raw_data(cls, id, stored_attrs): + instance = cls() + instance._id = str(id) + instance._set_instance_keys() + instance.load_attributes_from_redis_raw_data(stored_attrs) + return instance ################### # Private methods # @@ -526,7 +540,8 @@ def exists(cls, id): def _initialize_id(self): """Initializes the id of the instance.""" - self._id = str(self.db.incr(self._key['id'])) + self._id = str(self.db.incr(self._id_gen_key)) + self._set_instance_keys() def _write(self, _new=False): """Writes the values of the attributes to the datastore. @@ -537,7 +552,7 @@ def _write(self, _new=False): pipeline = self.db.pipeline() self._create_membership(pipeline) self._update_indices(pipeline) - h = {} + h = {"id": self._instance_key} # attributes for k, v in self.attributes.iteritems(): if isinstance(v, DateTimeField): @@ -564,13 +579,12 @@ def _write(self, _new=False): h[index] = unicode(v) except UnicodeError: h[index] = unicode(v.decode('utf-8')) - pipeline.delete(self.key()) - if h: - pipeline.hmset(self.key(), h) + pipeline.delete(self._instance_key) + pipeline.hmset(self._instance_key, h) # lists for k, v in self.lists.iteritems(): - l = List(self.key()[k], pipeline=pipeline) + l = List(self.key(att=k), pipeline=pipeline) l.clear() values = getattr(self, k) if values: @@ -580,6 +594,11 @@ def _write(self, _new=False): l.extend(values) pipeline.execute() + def _set_instance_keys(self): + self._instance_key = self.instance_key(self._id) + self._indices_key = u"%s:%s:_indices" % (self._key, self._id) + self._zindices_key = u"%s:%s:_zindices" % (self._key, self._id) + ############## # Membership # ############## @@ -588,13 +607,13 @@ def _create_membership(self, pipeline=None): """Adds the id of the object to the set of all objects of the same class. """ - Set(self._key['all'], pipeline=pipeline).add(self.id) + Set(self._all_key, pipeline=pipeline).add(self.id) def _delete_membership(self, pipeline=None): """Removes the id of the object to the set of all objects of the same class. """ - Set(self._key['all'], pipeline=pipeline).remove(self.id) + Set(self._all_key, pipeline=pipeline).remove(self.id) ############ # INDICES! # @@ -622,26 +641,26 @@ def _add_to_index(self, att, val=None, pipeline=None): t, index = index if t == 'attribute': pipeline.sadd(index, self.id) - pipeline.sadd(self.key()['_indices'], index) + pipeline.sadd(self._indices_key, index) elif t == 'list': for i in index: pipeline.sadd(i, self.id) - pipeline.sadd(self.key()['_indices'], i) + pipeline.sadd(self._indices_key, i) elif t == 'sortedset': zindex, index = index pipeline.sadd(index, self.id) - pipeline.sadd(self.key()['_indices'], index) + pipeline.sadd(self._indices_key, index) descriptor = self.attributes[att] score = descriptor.typecast_for_storage(getattr(self, att)) pipeline.zadd(zindex, self.id, score) - pipeline.sadd(self.key()['_zindices'], zindex) + pipeline.sadd(self._zindices_key, zindex) def _delete_from_indices(self, pipeline): """Deletes the object's id from the sets(indices) it has been added to and removes its list of indices (used for housekeeping). """ - s = Set(self.key()['_indices'], pipeline=self.db) - z = Set(self.key()['_zindices'], pipeline=self.db) + s = Set(self._indices_key, pipeline=self.db) + z = Set(self._zindices_key, pipeline=self.db) for index in s.members: pipeline.srem(index, self.id) for index in z.members: @@ -667,7 +686,7 @@ def _index_key_for(self, att, value=None): def _get_index_key_for_non_list_attr(self, att, value): descriptor = self.attributes.get(att) - if descriptor and isinstance(descriptor, ZINDEXABLE): + if descriptor and descriptor.zindexable: sval = descriptor.typecast_for_storage(value) return self._tuple_for_index_key_attr_zset(att, value, sval) elif descriptor: @@ -684,11 +703,12 @@ def _tuple_for_index_key_attr_list(self, att, val): return ('list', [self._index_key_for_attr_val(att, e) for e in val]) def _tuple_for_index_key_attr_zset(self, att, val, sval): + key = "%s:%s" % (self._key, att) return ('sortedset', - (self._key[att], self._index_key_for_attr_val(att, sval))) + (key, self._index_key_for_attr_val(att, sval))) def _index_key_for_attr_val(self, att, val): - return self._key[att][val] + return "%s:%s:%s" % (self._key, att, val) ################## # Python methods # diff --git a/redisco/models/basetests.py b/redisco/models/basetests.py index a2125eb8e..e1f055fbb 100644 --- a/redisco/models/basetests.py +++ b/redisco/models/basetests.py @@ -12,6 +12,8 @@ class Person(models.Model): first_name = models.CharField() last_name = models.CharField() + year_of_birth = models.IntegerField(indexed=True) + month_of_birth = models.Attribute(indexed=True) def full_name(self): return "%s %s" % (self.first_name, self.last_name,) @@ -23,7 +25,7 @@ class Meta: class RediscoTestCase(unittest.TestCase): def setUp(self): self.client = redisco.get_client() - self.client.flushdb() + self.client.flushall() def tearDown(self): self.client.flushdb() @@ -45,9 +47,12 @@ def test_CharFields(self): def test_save(self): person1 = Person(first_name="Granny", last_name="Goose") - person1.save() + self.assertTrue(person1.save()) person2 = Person(first_name="Jejomar") - person2.save() + self.assertTrue(person2.save()) + person3 = Person(first_name=2, last_name=3) + self.assertFalse(person3.save()) # Validation error + self.assertTrue(person3.save(validate_fields=False)) # Validation error self.assertEqual('1', person1.id) self.assertEqual('2', person2.id) @@ -68,11 +73,11 @@ def test_unicode(self): def test_repr(self): person1 = Person(first_name="Granny", last_name="Goose") - self.assertEqual("", + self.assertEqual("", repr(person1)) self.assert_(person1.save()) - self.assertEqual("", + self.assertEqual("", repr(person1)) def test_update(self): @@ -130,11 +135,11 @@ def test_manager_create(self): def test_indices(self): person = Person.objects.create(first_name="Granny", last_name="Goose") db = person.db - key = person.key() + key = person.key(att='_indices') ckey = Person._key index = 'Person:first_name:%s' % "Granny" - self.assertTrue(index in db.smembers(key['_indices'])) + self.assertTrue(index in db.smembers(key)) self.assertTrue("1" in db.smembers(index)) def test_delete(self): @@ -171,6 +176,7 @@ def test_filter(self): Person.objects.create(first_name="Granny", last_name="Kent") persons = Person.objects.filter(first_name="Granny") + self.assertEqual(len(persons), 3) self.assertEqual('1', persons[0].id) self.assertEqual(3, len(persons)) @@ -182,6 +188,34 @@ def test_filter(self): self.assertEqual(1, len(persons)) self.assertEqual("Granny Mommy", persons[0].full_name()) + def test_filter_different_db(self): + class DifferentPerson(models.Model): + first_name = models.CharField() + last_name = models.CharField() + def full_name(self): + return "%s %s" % (self.first_name, self.last_name,) + class Meta: + indices = ['full_name'] + db = redis.Redis(db=8) + + DifferentPerson.objects.create(first_name="Granny", last_name="Goose") + DifferentPerson.objects.create(first_name="Clark", last_name="Kent") + DifferentPerson.objects.create(first_name="Granny", last_name="Mommy") + DifferentPerson.objects.create(first_name="Granny", last_name="Kent") + persons = DifferentPerson.objects.filter(first_name="Granny") + + self.assertEqual(len(persons), 3) + self.assertEqual('1', persons[0].id) + self.assertEqual(3, len(persons)) + + persons = DifferentPerson.objects.filter(first_name="Clark") + self.assertEqual(1, len(persons)) + + # by index + persons = DifferentPerson.objects.filter(full_name="Granny Mommy") + self.assertEqual(1, len(persons)) + self.assertEqual("Granny Mommy", persons[0].full_name()) + def test_exclude(self): Person.objects.create(first_name="Granny", last_name="Goose") Person.objects.create(first_name="Clark", last_name="Kent") @@ -518,6 +552,64 @@ def test_get_or_create(self): last_name="Weiss") self.assertEqual('7', p.id) + def test_values(self): + Person.objects.create(first_name="Granny", last_name="Goose") + Person.objects.create(first_name="Clark", last_name="Kent") + Person.objects.create(first_name="Granny", last_name="Mommy") + Person.objects.create(first_name="Lois", last_name="Kent") + Person.objects.create(first_name="Jonathan", last_name="Kent") + Person.objects.create(first_name="Martha", last_name="Kent") + Person.objects.create(first_name="Lex", last_name="Luthor") + Person.objects.create(first_name="Lionel", last_name="Luthor") + + persons_values = Person.objects.values().all() + persons = Person.objects.all() + for p, v in zip(persons, persons_values): + self.assertTrue(isinstance(p, Person)) + self.assertTrue(isinstance(v, dict)) + self.assertEqual(p.first_name, v["first_name"]) + self.assertEqual(p.last_name, v["last_name"]) + + persons_values = Person.objects.values().all()[2:5] + persons = Person.objects.all()[2:5] + for p, v in zip(persons, persons_values): + self.assertTrue(isinstance(p, Person)) + self.assertTrue(isinstance(v, dict)) + self.assertEqual(p.first_name, v["first_name"]) + self.assertEqual(p.last_name, v["last_name"]) + + p = Person.objects.get_by_id(1) + v = Person.objects.values().get_by_id(1) + self.assertTrue(isinstance(p, Person)) + self.assertTrue(isinstance(v, dict)) + self.assertEqual(p.first_name, v["first_name"]) + self.assertEqual(p.last_name, v["last_name"]) + + p = Person.objects.all().first() + v = Person.objects.values().all().first() + self.assertTrue(isinstance(p, Person)) + self.assertTrue(isinstance(v, dict)) + self.assertEqual(p.first_name, v["first_name"]) + self.assertEqual(p.last_name, v["last_name"]) + + p = Person.objects.all()[3] + v = Person.objects.values().all()[3] + self.assertTrue(isinstance(p, Person)) + self.assertTrue(isinstance(v, dict)) + self.assertEqual(p.first_name, v["first_name"]) + self.assertEqual(p.last_name, v["last_name"]) + + def test_count(self): + self.assertEqual(Person.objects.count(), 0) + Person.objects.create(first_name="Granny", last_name="Goose") + Person.objects.create(first_name="Clark", last_name="Kent") + Person.objects.create(first_name="Granny", last_name="Mommy") + Person.objects.create(first_name="Lois", last_name="Kent") + Person.objects.create(first_name="Jonathan", last_name="Kent") + Person.objects.create(first_name="Martha", last_name="Kent") + Person.objects.create(first_name="Lex", last_name="Luthor") + Person.objects.create(first_name="Lionel", last_name="Luthor") + self.assertEqual(Person.objects.count(), 8) def test_customizable_key(self): class Person(models.Model): @@ -532,6 +624,33 @@ class Meta: self.assert_('1' in self.client.smembers('People:all')) + def test_indexed_values(self): + Person.objects.create(first_name="Granny", last_name="Goose", year_of_birth=1980) + Person.objects.create(first_name="Clark", last_name="Kent", year_of_birth=1980) + Person.objects.create(first_name="Granny", last_name="Mommy", year_of_birth=1979) + Person.objects.create(first_name="Lois", last_name="Kent", year_of_birth=1960) + Person.objects.create(first_name="Jonathan", last_name="Kent", year_of_birth=1944) + Person.objects.create(first_name="Martha", last_name="Kent", year_of_birth=1961) + Person.objects.create(first_name="Lex", last_name="Luthor", year_of_birth=1960) + Person.objects.create(first_name="Lionel", last_name="Luthor", year_of_birth=1980) + + years = Person.objects.get_indexed_values("month_of_birth") + self.assertEqual(type(years), list) + self.assertEqual(len(years), 0) + + years = Person.objects.get_indexed_values("year_of_birth") + self.assertEqual(type(years), list) + self.assertEqual(len(years), 5) + for year in [1980, 1979, 1960, 1944, 1961]: + self.assertTrue(year in years) + + try: + years = Person.objects.get_indexed_values("first_name") + assert False + except: + assert True + + class Event(models.Model): name = models.CharField(required=True) @@ -561,7 +680,7 @@ def test_invalid_date(self): def test_indexes(self): d = date.today() Event.objects.create(name="Event #1", date=d) - self.assertTrue('1' in self.client.smembers(Event._key['all'])) + self.assertTrue('1' in self.client.smembers(Event._all_key)) # zfilter index self.assertTrue(self.client.exists("Event:date")) # other field indices @@ -782,6 +901,7 @@ class Character(models.Model): Word.objects.create() word = Word.objects.all()[0] + print "KEYS", self.client.keys("*") Character.objects.create(n=32, m='a', word=word) Character.objects.create(n=33, m='b', word=word) Character.objects.create(n=34, m='c', word=word) diff --git a/redisco/models/key.py b/redisco/models/key.py deleted file mode 100644 index 5157a0fbe..000000000 --- a/redisco/models/key.py +++ /dev/null @@ -1,3 +0,0 @@ -class Key(unicode): - def __getitem__(self, key): - return Key(u"%s:%s" % (self, key)) diff --git a/redisco/models/managers.py b/redisco/models/managers.py index aab1605dd..071625b24 100644 --- a/redisco/models/managers.py +++ b/redisco/models/managers.py @@ -45,4 +45,12 @@ def order(self, field): def zfilter(self, **kwargs): return self.get_model_set().zfilter(**kwargs) + def values(self): + return self.get_model_set().values() + + def count(self): + return self.get_model_set().count() + + def get_indexed_values(self, attribute): + return self.get_model_set().get_indexed_values(attribute) diff --git a/redisco/models/modelset.py b/redisco/models/modelset.py index b3d854945..f800ff5c9 100644 --- a/redisco/models/modelset.py +++ b/redisco/models/modelset.py @@ -5,13 +5,13 @@ import redisco from redisco.containers import SortedSet, Set, List, NonPersistentList from exceptions import AttributeNotIndexed -from attributes import ZINDEXABLE + # Model Set class ModelSet(Set): def __init__(self, model_class): self.model_class = model_class - self.key = model_class._key['all'] + self.key = model_class._all_key # We access directly _meta as .db is a property and should be # access from an instance, not a Class self._db = model_class._meta['db'] or redisco.get_client() @@ -21,6 +21,7 @@ def __init__(self, model_class): self._ordering = [] self._limit = None self._offset = None + self._return_values = False ################# # MAGIC METHODS # @@ -31,7 +32,17 @@ def __getitem__(self, index): Will look in _set to get the id and simply return the instance of the model. """ if isinstance(index, slice): - return map(lambda id: self._get_item_with_id(id), self._set[index]) + pipeline = self._db.pipeline() + for id in self._set[index]: + pipeline.hgetall(self.model_class.instance_key(id)) + rawdata = pipeline.execute() + if self._return_values: + return rawdata + else: + instances = [] + for id, rd in zip(self._set[index], rawdata): + instances.append(self.model_class.load_from_raw_data(id, rd)) + return instances else: id = self._set[index] if id: @@ -60,7 +71,6 @@ def __contains__(self, val): ########################################## # METHODS THAT RETURN A SET OF INSTANCES # ########################################## - def get_by_id(self, id): """ Returns the object definied by ``id``. @@ -82,8 +92,10 @@ def get_by_id(self, id): """ if (self._filters or self._exclusions or self._zfilters) and str(id) not in self._set: return - if self.model_class.exists(id): - return self._get_item_with_id(id) + return self._get_item_with_id(id) + + def get(self, **kwargs): + return self.filter(**kwargs).first() def first(self): """ @@ -109,11 +121,20 @@ def first(self): except IndexError: return None + count = __len__ ##################################### # METHODS THAT MODIFY THE MODEL SET # ##################################### + def values(self): + """ + Returns dictionaries rather than model-instance objects. + """ + clone = self._clone() + clone._return_values = True + return clone + def filter(self, **kwargs): """ Filter a collection on criteria @@ -198,7 +219,7 @@ def order(self, field): alpha = True if fname in self.model_class._attributes: v = self.model_class._attributes[fname] - alpha = not isinstance(v, ZINDEXABLE) + alpha = not v.zindexable clone = self._clone() if not clone._ordering: clone._ordering = [] @@ -266,7 +287,31 @@ def get_or_create(self, **kwargs): else: return self.create(**kwargs) - # + def get_indexed_values(self, attribute_name): + """ + Return indexed values for a model attribute. + + >>> from redisco import models + >>> class Foo(models.Model): + ... name = models.Attribute(indexed=True) + ... + >>> Foo.objects.create(name="Obama") + >>> Foo.objects.create(name="Hollande") + >>> Foo.objects.create(name="Merkel") + >>> Foo.objects.create(name="Merkel") + >>> Foo.objects.get_indexed_values("name) + ["Hollande", "Merkel", "Obama"] + """ + if attribute_name not in self.model_class._indices: + raise AttributeNotIndexed( + "Attribute %s is not indexed in %s class." % + (k, self.model_class.__name__)) + wildcard_search_key = "%s:%s:*" % (self.model_class._key, attribute_name) + keys = self._db.keys(wildcard_search_key) + attribute = self.model_class._attributes[attribute_name] + values = [attribute.typecast_for_read(k.split(":")[2]) for k in keys] + return values + @property def db(self): @@ -286,7 +331,7 @@ def _set(self): # For performance reasons, only one zfilter is allowed. if hasattr(self, '_cached_set'): return self._cached_set - s = Set(self.key) + s = Set(self.key, db=self.db) if self._zfilters: s = self._add_zfilters(s) if self._filters: @@ -359,7 +404,7 @@ def _add_zfilters(self, s): att, op = k.split('__') except ValueError: raise ValueError("zfilter should have an operator.") - index = self.model_class._key[att] + index = "%s:%s" % (self.model_class._key, att) desc = self.model_class._attributes[att] zset = SortedSet(index, db=self.db) limit, offset = self._get_limit_and_offset() @@ -385,13 +430,13 @@ def _add_zfilters(self, s): elif op == 'in': members = zset.between(min, max, limit, offset) - temp_set = Set(new_set_key_temp) + temp_set = Set(new_set_key_temp, db=self.db) if members: temp_set.add(*members) temp_set.set_expire() s.intersection(new_set_key, temp_set) - new_set = Set(new_set_key) + new_set = Set(new_set_key, db=self.db) new_set.set_expire() return new_set @@ -421,7 +466,7 @@ def _set_with_ordering(self, skey): else: desc = False new_set_key = "%s#%s.%s" % (old_set_key, ordering, id(self)) - by = "%s->%s" % (self.model_class._key['*'], ordering) + by = "%s:*->%s" % (self.model_class._key, ordering) self.db.sort(old_set_key, by=by, store=new_set_key, @@ -471,13 +516,14 @@ def _get_limit_and_offset(self): def _get_item_with_id(self, id): """ - Fetch an object and return the instance. The real fetching is - done by assigning the id to the Instance. See ``Model`` class. + Fetch an object and return the instance. """ - instance = self.model_class() - instance.id = str(id) - return instance - + if self._return_values: + instance_key = self.model_class.instance_key(id) + return self.db.hgetall(instance_key) + else: + return self.model_class.get_by_id(id) + def _build_key_from_filter_item(self, index, value): """ Build the keys from the filter so we can fetch the good keys @@ -489,7 +535,8 @@ def _build_key_from_filter_item(self, index, value): desc = self.model_class._attributes.get(index) if desc: value = desc.typecast_for_storage(value) - return self.model_class._key[index][value] + _k = u"%s:%s:%s" % (self.model_class._key, index, value) + return _k def _clone(self): """ @@ -511,4 +558,5 @@ def _clone(self): c._ordering = self._ordering c._limit = self._limit c._offset = self._offset + c._return_values = self._return_values return c diff --git a/redisco/tests/__init__.py b/redisco/tests/__init__.py index 38f2990df..65bd07272 100644 --- a/redisco/tests/__init__.py +++ b/redisco/tests/__init__.py @@ -28,7 +28,7 @@ def all_tests(): suite.addTest(unittest.makeSuite(ReferenceFieldTestCase)) suite.addTest(unittest.makeSuite(DateTimeFieldTestCase)) suite.addTest(unittest.makeSuite(CounterFieldTestCase)) - suite.addTest(unittest.makeSuite(MutexTestCase)) + #suite.addTest(unittest.makeSuite(MutexTestCase)) suite.addTest(unittest.makeSuite(HashTestCase)) suite.addTest(unittest.makeSuite(CharFieldTestCase)) return suite diff --git a/redisco/tests/__main__.py b/redisco/tests/__main__.py new file mode 100644 index 000000000..d2bd451ed --- /dev/null +++ b/redisco/tests/__main__.py @@ -0,0 +1,6 @@ +import unittest + +if __name__ == "__main__": + from . import all_tests + suite = all_tests() + unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file diff --git a/redisco/tests/performance.py b/redisco/tests/performance.py new file mode 100644 index 000000000..db75a4141 --- /dev/null +++ b/redisco/tests/performance.py @@ -0,0 +1,134 @@ + +#!/usr/bin/python +# -*- coding: utf-8 -* +import redis +import time +from redisco import models + +REDIS = redis.Redis() + +class RediscoPost(models.Model): + uid = models.Attribute(indexed=False) # ID unico interno + postid = models.Attribute(indexed=False) # Id do provider + text = models.Attribute(indexed=False) # Texto / caption + url = models.Attribute(indexed=False) # URL post original + media_url = models.Attribute(default=None, indexed=False) # Link de eventual midia (video, imagem) + provider = models.Attribute() # Provider (Facebook, Twitter, Vimeo, Youtube, Instagram) + post_type = models.Attribute() # Tipo de post (video, texto, imagem) + locale = models.Attribute(indexed=False) # Lingua do post + brand = models.Attribute() # Marca + userid = models.Attribute() # Userid + user_name = models.Attribute(indexed=False) # User name + user_avatar = models.Attribute(indexed=False) # User Avatar + country = models.Attribute() # Pais de origem + is_own_post = models.BooleanField(indexed=False) # Post gerado pelo dono da marca? + interactions = models.IntegerField() # Numero de interacoes do post + ts = models.IntegerField(indexed=False) # Timestamp de criacao do post + +from datetime import datetime + +# REDIS.flushall() +# import cPickle + +# start = datetime.now() +# for i in range(0, 10000): +# post = RediscoPost(uid = str(i), +# postid = str(i), +# text = "text", +# url = "http", +# provider = 2, +# post_type = 1, +# media_url = "", +# locale = "pt", +# brand = 1, +# country = None, +# userid = str(i), +# user_name = "bla", +# user_avatar = "", +# is_own_post = True, +# interactions = 10, +# ts = i +# ) +# REDIS.set(i, cPickle.dumps(post)) +# delta = datetime.now()-start +# print "PICKLE", delta + +# start = datetime.now() +# for i in range(0, 10000): +# cPickle.loads(REDIS.get(i)) +# delta = datetime.now()-start +# print "UNPICKLE", delta + +# start = datetime.now() +# pickleds = REDIS.mget(range(0, 10000)) +# for pickled in pickleds: +# cPickle.loads(pickled) +# delta = datetime.now()-start +# print "MULTI UNPICKLE", delta + + +# print +# print + +REDIS.flushall() + +start = datetime.now() +for i in range(0, 10000): + post = RediscoPost(uid = str(i), + postid = str(i), + text = "text", + url = "http", + provider = 2, + post_type = 1, + media_url = "", + locale = "pt", + brand = 1, + country = None, + userid = str(i), + user_name = "bla", + user_avatar = "", + is_own_post = True, + interactions = 10, + ts = i + ) + post.save(validate_fields=False) + +delta = datetime.now()-start +print "SAVE REDISCO", delta + +start = datetime.now() +for i in range(0, 10000): + RediscoPost.objects.get_by_id(i) +delta = datetime.now()-start +print "GET REDISCO", delta + +start = datetime.now() +for i in range(0, 10000): + RediscoPost.objects.values().get_by_id(i) +delta = datetime.now()-start +print "GET REDISCO VALUES", delta + +start = datetime.now() +posts = RediscoPost.objects.filter(provider=2) +for p in posts: + p.uid +delta = datetime.now()-start +print "QUERY REDISCO", delta, len(posts) + +start = datetime.now() +posts = RediscoPost.objects.values().filter(provider=2) +for p in posts: + p["uid"] +delta = datetime.now()-start +print "QUERY REDISCO VALUES", delta, len(posts) + +start = datetime.now() +posts = RediscoPost.objects.filter(provider=2)[:] +delta = datetime.now()-start +print "QUERY REDISCO SLICE", delta, len(posts) + + +start = datetime.now() +posts = RediscoPost.objects.values().filter(provider=2)[:] +delta = datetime.now()-start +print "QUERY REDISCO SLICE VALUES", delta, len(posts) diff --git a/setup.py b/setup.py index 4daeb5dbb..d41c5482c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os -version = '0.2.4' +version = '0.3.0' try: from setuptools import setup @@ -14,14 +14,14 @@ def read(fname): setup(name='redisco', version=version, description='Python Containers and Simple Models for Redis', - url='http://kiddouk.github.com/redisco', + url='', download_url='', long_description=read('README.rst'), install_requires=read('requirements.txt').splitlines(), author='Tim Medina', author_email='iamteem@gmail.com', - maintainer='Sebastien Requiem', - maintainer_email='sebastien.requiem@gmail.com', + maintainer='Miguel Galves', + maintainer_email='mgalves@gmail.com', keywords=['Redis', 'model', 'container'], license='MIT', packages=['redisco', 'redisco.models'],