Performance tuning and some other stuff

.gitignore

@@ -10,3 +10,7 @@ dump.rdb
 /.pyflymakercc
 /.emacs.desktop
 /redisco/.ropeproject/config.py
+.Python
+lib
+bin
+include

redisco/models/attributes.py

@@ -12,7 +12,7 @@
 
 __all__ = ['Attribute', 'CharField', 'ListField', 'DateTimeField',
         'DateField', 'ReferenceField', 'Collection', 'IntegerField',
-        'FloatField', 'BooleanField', 'Counter', 'ZINDEXABLE']
+        'FloatField', 'BooleanField', 'Counter']
 
 
 class Attribute(object):
@@ -46,6 +46,7 @@ def __init__(self,
         self.validator = validator
         self.default = default
         self.unique = unique
+        self.zindexable = False
 
     def __get__(self, instance, owner):
         try:
@@ -119,11 +120,8 @@ def __init__(self, max_length=255, **kwargs):
 
     def validate(self, instance):
         errors = []
-        try:
-            super(CharField, self).validate(instance)
-        except FieldValidationError as err:
-            errors.extend(err.errors)
-
+        super(CharField, self).validate(instance)
+
         val = getattr(instance, self.name)
 
         if val and len(val) > self.max_length:
@@ -150,6 +148,10 @@ def acceptable_types(self):
 
 
 class IntegerField(Attribute):
+    def __init__(self, **kwargs):
+        super(IntegerField, self).__init__(**kwargs)
+        self.zindexable = True
+
     def typecast_for_read(self, value):
         return int(value)
 
@@ -166,6 +168,10 @@ def acceptable_types(self):
 
 
 class FloatField(Attribute):
+    def __init__(self, **kwargs):
+        super(FloatField, self).__init__(**kwargs)
+        self.zindexable = True
+
     def typecast_for_read(self, value):
         return float(value)
 
@@ -187,6 +193,7 @@ def __init__(self, auto_now=False, auto_now_add=False, **kwargs):
         super(DateTimeField, self).__init__(**kwargs)
         self.auto_now = auto_now
         self.auto_now_add = auto_now_add
+        self.zindexable = True
 
     def typecast_for_read(self, value):
         try:
@@ -221,6 +228,7 @@ def __init__(self, auto_now=False, auto_now_add=False, **kwargs):
         super(DateField, self).__init__(**kwargs)
         self.auto_now = auto_now
         self.auto_now_add = auto_now_add
+        self.zindexable = True
 
     def typecast_for_read(self, value):
         try:
@@ -280,7 +288,7 @@ def __get__(self, instance, owner):
             if instance.is_new():
                 val = self.default
             else:
-                key = instance.key()[self.name]
+                key = instance.key(att=self.name)
                 val = List(key).members
             if val is not None:
                 klass = self.value_type()
@@ -446,6 +454,7 @@ def __init__(self, **kwargs):
         super(Counter, self).__init__(**kwargs)
         if not kwargs.has_key('default') or self.default is None:
             self.default = 0
+        self.zindexable = True
 
     def __set__(self, instance, value):
         raise AttributeError("can't set a counter.")
@@ -458,6 +467,3 @@ def __get__(self, instance, owner):
             return int(v)
         else:
             return 0
-
-
-ZINDEXABLE = (IntegerField, DateTimeField, DateField, FloatField, Counter)

redisco/models/base.py

@@ -4,15 +4,12 @@
 import redisco
 from redisco.containers import Set, List, SortedSet, NonPersistentList
 from attributes import *
-from key import Key
 from managers import ManagerDescriptor, Manager
 from exceptions import FieldValidationError, MissingID, BadKeyError, WatchError
 from attributes import Counter
 
 __all__ = ['Model', 'from_key']
 
-ZINDEXABLE = (IntegerField, DateTimeField, DateField, FloatField)
-
 ##############################
 # Model Class Initialization #
 ##############################
@@ -150,12 +147,13 @@ def _initialize_counters(model_class, name, bases, attrs):
             model_class._counters.append(k)
 
 
-def _initialize_key(model_class, name):
+def _initialize_keys(model_class, name):
     """
     Initializes the key of the model.
     """
-    model_class._key = Key(model_class._meta['key'] or name)
-
+    model_class._key = model_class._meta['key'] or name
+    model_class._all_key = u"%s:all" % model_class._key
+    model_class._id_gen_key = u"%s:id" % model_class._key
 
 def _initialize_manager(model_class):
     """
@@ -208,7 +206,7 @@ def __init__(cls, name, bases, attrs):
         _initialize_counters(cls, name, bases, attrs)
         _initialize_lists(cls, name, bases, attrs)
         _initialize_indices(cls, name, bases, attrs)
-        _initialize_key(cls, name)
+        _initialize_keys(cls, name)
         _initialize_manager(cls)
         # if targeted by a reference field using a string,
         # override for next try
@@ -298,10 +296,10 @@ def update_attributes(self, **kwargs):
             if att.name in kwargs:
                 att.__set__(self, kwargs[att.name])
 
-    def save(self):
+    def save(self, validate_fields=True):
         """
         Saves the instance to the datastore with the following steps:
-        1. Validate all the fields
+        1. Validate all the fields (DEFAULT/OPTIONAL. Validation can be turned off with validate_fields=False)
         2. Assign an ID if the object is new
         3. Save to the datastore.
 
@@ -315,13 +313,12 @@ def save(self):
         True
         >>> f.delete()
         """
-        if not self.is_valid():
-            return self._errors
+        if validate_fields and not self.is_valid():
+            return False
         _new = self.is_new()
         if _new:
             self._initialize_id()
-        with Mutex(self):
-            self._write(_new)
+        self._write(_new)
         return True
 
     def key(self, att=None):
@@ -340,9 +337,9 @@ def key(self, att=None):
         True
         """
         if att is not None:
-            return self._key[self.id][att]
+            return u"%s:%s" % (self._instance_key, att)
         else:
-            return self._key[self.id]
+            return self._instance_key
 
     def delete(self):
         """Deletes the object from the datastore."""
@@ -443,18 +440,6 @@ def id(self):
             raise MissingID
         return self._id
 
-    @id.setter
-    def id(self, val):
-        """
-        Setting the id for the object will fetch it from the datastorage.
-        """
-        self._id = str(val)
-        stored_attrs = self.db.hgetall(self.key())
-        attrs = self.attributes.values()
-        for att in attrs:
-            if att.name in stored_attrs and not isinstance(att, Counter):
-                att.__set__(self, att.typecast_for_read(stored_attrs[att.name]))
-
     @property
     def attributes(self):
         """Return the attributes of the model.
@@ -510,23 +495,53 @@ def counters(self):
         """Returns the mapping of the counters."""
         return self._counters
 
+    def load_attributes_from_redis_raw_data(self, stored_attrs):
+        attrs = self.attributes.values()
+        for att in attrs:
+            if att.name in stored_attrs and not isinstance(att, Counter):
+                att.__set__(self, att.typecast_for_read(stored_attrs[att.name]))
+
     #################
     # Class Methods #
     #################
 
     @classmethod
     def exists(cls, id):
         """Checks if the model with id exists."""
-        return bool((cls._meta['db'] or redisco.get_client()).exists(cls._key[str(id)]) or
-                    (cls._meta['db'] or redisco.get_client()).sismember(cls._key['all'], str(id)))
+        _conn = cls._meta['db'] or redisco.get_client()
+        return bool(_conn.exists(cls.instance_key(id)) or _conn.sismember(cls._all_key, str(id)))
+
+    @classmethod
+    def instance_key(cls, id):
+        return u"%s:%s" % (cls._key, str(id))
+
+    @classmethod
+    def get_by_id(cls, id):
+        instance = cls()
+        instance._id = str(id)
+        instance._set_instance_keys()
+        stored_attrs = instance.db.hgetall(instance._instance_key)
+        if not stored_attrs: # object does not exist
+            return None
+        instance.load_attributes_from_redis_raw_data(stored_attrs)
+        return instance
+
+    @classmethod
+    def load_from_raw_data(cls, id, stored_attrs):
+        instance = cls()
+        instance._id = str(id)
+        instance._set_instance_keys()
+        instance.load_attributes_from_redis_raw_data(stored_attrs)
+        return instance
 
     ###################
     # Private methods #
     ###################
 
     def _initialize_id(self):
         """Initializes the id of the instance."""
-        self._id = str(self.db.incr(self._key['id']))
+        self._id = str(self.db.incr(self._id_gen_key))
+        self._set_instance_keys()
 
     def _write(self, _new=False):
         """Writes the values of the attributes to the datastore.
@@ -537,7 +552,7 @@ def _write(self, _new=False):
         pipeline = self.db.pipeline()
         self._create_membership(pipeline)
         self._update_indices(pipeline)
-        h = {}
+        h = {"id": self._instance_key}
         # attributes
         for k, v in self.attributes.iteritems():
             if isinstance(v, DateTimeField):
@@ -564,13 +579,12 @@ def _write(self, _new=False):
                         h[index] = unicode(v)
                     except UnicodeError:
                         h[index] = unicode(v.decode('utf-8'))
-        pipeline.delete(self.key())
-        if h:
-            pipeline.hmset(self.key(), h)
+        pipeline.delete(self._instance_key)
+        pipeline.hmset(self._instance_key, h)
 
         # lists
         for k, v in self.lists.iteritems():
-            l = List(self.key()[k], pipeline=pipeline)
+            l = List(self.key(att=k), pipeline=pipeline)
             l.clear()
             values = getattr(self, k)
             if values:
@@ -580,6 +594,11 @@ def _write(self, _new=False):
                     l.extend(values)
         pipeline.execute()
 
+    def _set_instance_keys(self):
+        self._instance_key = self.instance_key(self._id)
+        self._indices_key = u"%s:%s:_indices" % (self._key, self._id)
+        self._zindices_key = u"%s:%s:_zindices" % (self._key, self._id)
+
     ##############
     # Membership #
     ##############
@@ -588,13 +607,13 @@ def _create_membership(self, pipeline=None):
         """Adds the id of the object to the set of all objects of the same
         class.
         """
-        Set(self._key['all'], pipeline=pipeline).add(self.id)
+        Set(self._all_key, pipeline=pipeline).add(self.id)
 
     def _delete_membership(self, pipeline=None):
         """Removes the id of the object to the set of all objects of the
         same class.
         """
-        Set(self._key['all'], pipeline=pipeline).remove(self.id)
+        Set(self._all_key, pipeline=pipeline).remove(self.id)
 
     ############
     # INDICES! #
@@ -622,26 +641,26 @@ def _add_to_index(self, att, val=None, pipeline=None):
         t, index = index
         if t == 'attribute':
             pipeline.sadd(index, self.id)
-            pipeline.sadd(self.key()['_indices'], index)
+            pipeline.sadd(self._indices_key, index)
         elif t == 'list':
             for i in index:
                 pipeline.sadd(i, self.id)
-                pipeline.sadd(self.key()['_indices'], i)
+                pipeline.sadd(self._indices_key, i)
         elif t == 'sortedset':
             zindex, index = index
             pipeline.sadd(index, self.id)
-            pipeline.sadd(self.key()['_indices'], index)
+            pipeline.sadd(self._indices_key, index)
             descriptor = self.attributes[att]
             score = descriptor.typecast_for_storage(getattr(self, att))
             pipeline.zadd(zindex, self.id, score)
-            pipeline.sadd(self.key()['_zindices'], zindex)
+            pipeline.sadd(self._zindices_key, zindex)
 
     def _delete_from_indices(self, pipeline):
         """Deletes the object's id from the sets(indices) it has been added
         to and removes its list of indices (used for housekeeping).
         """
-        s = Set(self.key()['_indices'], pipeline=self.db)
-        z = Set(self.key()['_zindices'], pipeline=self.db)
+        s = Set(self._indices_key, pipeline=self.db)
+        z = Set(self._zindices_key, pipeline=self.db)
         for index in s.members:
             pipeline.srem(index, self.id)
         for index in z.members:
@@ -667,7 +686,7 @@ def _index_key_for(self, att, value=None):
 
     def _get_index_key_for_non_list_attr(self, att, value):
         descriptor = self.attributes.get(att)
-        if descriptor and isinstance(descriptor, ZINDEXABLE):
+        if descriptor and descriptor.zindexable:
             sval = descriptor.typecast_for_storage(value)
             return self._tuple_for_index_key_attr_zset(att, value, sval)
         elif descriptor:
@@ -684,11 +703,12 @@ def _tuple_for_index_key_attr_list(self, att, val):
         return ('list', [self._index_key_for_attr_val(att, e) for e in val])
 
     def _tuple_for_index_key_attr_zset(self, att, val, sval):
+        key = "%s:%s" % (self._key, att)
         return ('sortedset',
-                (self._key[att], self._index_key_for_attr_val(att, sval)))
+                (key, self._index_key_for_attr_val(att, sval)))
 
     def _index_key_for_attr_val(self, att, val):
-        return self._key[att][val]
+        return "%s:%s:%s" % (self._key, att, val)
 
     ##################
     # Python methods #