diff --git a/firefly/__init__.py b/firefly/__init__.py index d9b396b..984ce77 100644 --- a/firefly/__init__.py +++ b/firefly/__init__.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-10-21 @@ -7,4 +7,4 @@ from firefly._version import version -__version__ = version.short() \ No newline at end of file +__version__ = version.short() diff --git a/firefly/_version.py b/firefly/_version.py index 5d75144..4330a7d 100644 --- a/firefly/_version.py +++ b/firefly/_version.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-10-21 @@ -6,4 +6,4 @@ ''' from twisted.python import versions -version = versions.Version('firefly', 1, 3, 3) \ No newline at end of file +version = versions.Version('firefly', 1, 3, 3) diff --git a/firefly/dbentrust/dbpool.py b/firefly/dbentrust/dbpool.py index 97307d3..e55bc6d 100644 --- a/firefly/dbentrust/dbpool.py +++ b/firefly/dbentrust/dbpool.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-8 @@ -7,20 +7,22 @@ from DBUtils.PooledDB import PooledDB import MySQLdb -DBCS = {'mysql':MySQLdb,} +DBCS = {'mysql': MySQLdb, } + class DBPool(object): + ''' ''' - def initPool(self,**kw): + + def initPool(self, **kw): ''' ''' self.config = kw - creator = DBCS.get(kw.get('engine','mysql'),MySQLdb) - self.pool = PooledDB(creator,5,**kw) - + creator = DBCS.get(kw.get('engine', 'mysql'), MySQLdb) + self.pool = PooledDB(creator, 5, **kw) + def connection(self): return self.pool.connection() dbpool = DBPool() - diff --git a/firefly/dbentrust/dbutils.py b/firefly/dbentrust/dbutils.py index 470158f..cc71784 100644 --- a/firefly/dbentrust/dbutils.py +++ b/firefly/dbentrust/dbutils.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-21 @@ -11,7 +11,7 @@ def safeunicode(obj, encoding='utf-8'): r""" Converts any given object to unicode string. - + >>> safeunicode('hello') u'hello' >>> safeunicode(2) @@ -30,11 +30,12 @@ def safeunicode(obj, encoding='utf-8'): return unicode(obj) else: return str(obj).decode(encoding) - + + def safestr(obj, encoding='utf-8'): r""" - Converts any given object to utf-8 encoded string. - + Converts any given object to utf-8 encoded string. + >>> safestr('hello') 'hello' >>> safestr(u'\u1234') @@ -46,12 +47,13 @@ def safestr(obj, encoding='utf-8'): return obj.encode(encoding) elif isinstance(obj, str): return obj - elif hasattr(obj, 'next'): # iterator + elif hasattr(obj, 'next'): # iterator return itertools.imap(safestr, obj) else: return str(obj) - -def sqlify(obj): + + +def sqlify(obj): """ converts `obj` to its proper SQL version @@ -74,13 +76,15 @@ def sqlify(obj): elif datetime and isinstance(obj, datetime.datetime): return repr(obj.isoformat()) else: - if isinstance(obj, unicode): obj = obj.encode('utf8') + if isinstance(obj, unicode): + obj = obj.encode('utf8') return repr(obj) - -def sqllist(lst): + + +def sqllist(lst): """ Converts the arguments for use in something like a WHERE clause. - + >>> sqllist(['a', 'b']) 'a, b' >>> sqllist('a') @@ -88,11 +92,12 @@ def sqllist(lst): >>> sqllist(u'abc') u'abc' """ - if isinstance(lst, basestring): + if isinstance(lst, basestring): return lst else: return ', '.join(lst) - + + def _sqllist(values): """ >>> _sqllist([1, 2, 3]) @@ -106,8 +111,9 @@ def _sqllist(values): items.append(sqlparam(v)) items.append(')') return SQLQuery(items) - -def sqlquote(a): + + +def sqlquote(a): """ Ensures `a` is quoted properly for use in a SQL query. @@ -120,8 +126,9 @@ def sqlquote(a): return _sqllist(a) else: return sqlparam(a).sqlquery() - -def _interpolate(sformat): + + +def _interpolate(sformat): """ Takes a format string and returns a list of 2-tuples of the form (boolean, string) where boolean says whether string should be evaled @@ -130,7 +137,7 @@ def _interpolate(sformat): from (public domain, Ka-Ping Yee) """ from tokenize import tokenprog - + tokenprog = tokenprog def matchorfail(text, pos): @@ -140,13 +147,13 @@ def matchorfail(text, pos): return match, match.end() namechars = "abcdefghijklmnopqrstuvwxyz" \ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" chunks = [] pos = 0 - while 1: + while True: dollar = sformat.find("$", pos) - if dollar < 0: + if dollar < 0: break nextchar = sformat[dollar + 1] @@ -157,9 +164,9 @@ def matchorfail(text, pos): match, pos = matchorfail(sformat, pos) tstart, tend = match.regs[3] token = sformat[tstart:tend] - if token == "{": + if token == "{": level = level + 1 - elif token == "}": + elif token == "}": level = level - 1 chunks.append((1, sformat[dollar + 2:pos - 1])) @@ -168,7 +175,7 @@ def matchorfail(text, pos): match, pos = matchorfail(sformat, dollar + 1) while pos < len(sformat): if sformat[pos] == "." and \ - pos + 1 < len(sformat) and sformat[pos + 1] in namechars: + pos + 1 < len(sformat) and sformat[pos + 1] in namechars: match, pos = matchorfail(sformat, pos + 1) elif sformat[pos] in "([": pos, level = pos + 1, 1 @@ -176,25 +183,26 @@ def matchorfail(text, pos): match, pos = matchorfail(sformat, pos) tstart, tend = match.regs[3] token = sformat[tstart:tend] - if token[0] in "([": + if token[0] in "([": level = level + 1 - elif token[0] in ")]": + elif token[0] in ")]": level = level - 1 - else: + else: break chunks.append((1, sformat[dollar + 1:pos])) else: chunks.append((0, sformat[pos:dollar + 1])) pos = dollar + 1 + (nextchar == "$") - if pos < len(sformat): + if pos < len(sformat): chunks.append((0, sformat[pos:])) return chunks -def sqlwhere(dictionary, grouping=' AND '): + +def sqlwhere(dictionary, grouping=' AND '): """ Converts a `dictionary` to an SQL WHERE clause `SQLQuery`. - + >>> sqlwhere({'cust_id': 2, 'order_id':3}) >>> sqlwhere({'cust_id': 2, 'order_id':3}, grouping=', ') @@ -202,9 +210,11 @@ def sqlwhere(dictionary, grouping=' AND '): >>> sqlwhere({'a': 'a', 'b': 'b'}).query() 'a = %s AND b = %s' """ - return SQLQuery.join([k + ' = ' + sqlparam(v) for k, v in dictionary.items()], grouping) - -def reparam(string_, dictionary): + return SQLQuery.join([k + ' = ' + sqlparam(v) + for k, v in dictionary.items()], grouping) + + +def reparam(string_, dictionary): """ Takes a string and a dictionary and interpolates the string using values from the dictionary. Returns an `SQLQuery` for the result. @@ -214,17 +224,19 @@ def reparam(string_, dictionary): >>> reparam("s IN $s", dict(s=[1, 2])) """ - dictionary = dictionary.copy() # eval mucks with it + dictionary = dictionary.copy() # eval mucks with it result = [] for live, chunk in _interpolate(string_): if live: v = eval(chunk, dictionary) result.append(sqlquote(v)) - else: + else: result.append(chunk) return SQLQuery.join(result, '') -class UnknownParamstyle(Exception): + +class UnknownParamstyle(Exception): + """ raised for unsupported db paramstyles @@ -232,19 +244,24 @@ class UnknownParamstyle(Exception): """ pass -class _ItplError(ValueError): + +class _ItplError(ValueError): + def __init__(self, text, pos): ValueError.__init__(self) self.text = text self.pos = pos + def __str__(self): return "unfinished expression in %s at char %d" % ( repr(self.text), self.pos) + class SQLParam(object): + """ Parameter in SQLQuery. - + >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam("joe")]) >>> q @@ -257,7 +274,7 @@ class SQLParam(object): def __init__(self, value): self.value = value - + def get_marker(self, paramstyle='pyformat'): if paramstyle == 'qmark': return '?' @@ -265,26 +282,28 @@ def get_marker(self, paramstyle='pyformat'): return ':1' elif paramstyle is None or paramstyle in ['format', 'pyformat']: return '%s' - raise UnknownParamstyle, paramstyle - - def sqlquery(self): + raise UnknownParamstyle(paramstyle) + + def sqlquery(self): return SQLQuery([self]) - + def __add__(self, other): return self.sqlquery() + other - + def __radd__(self, other): - return other + self.sqlquery() - - def __str__(self): + return other + self.sqlquery() + + def __str__(self): return str(self.value) - + def __repr__(self): return '' % repr(self.value) -sqlparam = SQLParam +sqlparam = SQLParam + class SQLQuery(object): + """ You can pass this sort of thing as a clause in any db function. Otherwise, you can pass a dictionary to the keyword argument `vars` @@ -298,7 +317,7 @@ class SQLQuery(object): # tested in sqlquote's docstring def __init__(self, items=None): r"""Creates a new SQLQuery. - + >>> SQLQuery("x") >>> q = SQLQuery(['SELECT * FROM ', 'test', ' WHERE x=', SQLParam(1)]) @@ -319,10 +338,11 @@ def __init__(self, items=None): self.items = list(items.items) else: self.items = [items] - + # Take care of SQLLiterals for i, item in enumerate(self.items): - if isinstance(item, SQLParam) and isinstance(item.value, SQLLiteral): + if isinstance(item, SQLParam) and isinstance( + item.value, SQLLiteral): self.items[i] = item.value.v def append(self, value): @@ -342,7 +362,7 @@ def __radd__(self, other): items = [other] else: return NotImplemented - + return SQLQuery(items + self.items) def __iadd__(self, other): @@ -356,7 +376,7 @@ def __iadd__(self, other): def __len__(self): return len(self.query()) - + def query(self, paramstyle=None): """ Returns the query part of the sql query. @@ -374,13 +394,14 @@ def query(self, paramstyle=None): else: x = safestr(x) # automatically escape % characters in the query - # For backward compatability, ignore escaping when the query looks already escaped + # For backward compatability, ignore escaping when the query + # looks already escaped if paramstyle in ['format', 'pyformat']: if '%' in x and '%%' not in x: x = x.replace('%', '%%') s.append(x) return "".join(s) - + def values(self): """ Returns the values of the parameters used in the sql query. @@ -389,11 +410,11 @@ def values(self): ['joe'] """ return [i.value for i in self.items if isinstance(i, SQLParam)] - + def join(items, sep=' ', prefix=None, suffix=None, target=None): """ Joins multiple queries. - + >>> SQLQuery.join(['a', 'b'], ', ') @@ -423,25 +444,27 @@ def join(items, sep=' ', prefix=None, suffix=None, target=None): if suffix: target_items.append(suffix) return target - + join = staticmethod(join) - + def _str(self): try: - return self.query() % tuple([sqlify(x) for x in self.values()]) + return self.query() % tuple([sqlify(x) for x in self.values()]) except (ValueError, TypeError): return self.query() - + def __str__(self): return safestr(self._str()) - + def __unicode__(self): return safeunicode(self._str()) def __repr__(self): return '' % repr(str(self)) - -class SQLLiteral: + + +class SQLLiteral: + """ Protects a string from `sqlquote`. @@ -450,25 +473,29 @@ class SQLLiteral: >>> sqlquote(SQLLiteral('NOW()')) """ - def __init__(self, v): + + def __init__(self, v): self.v = v - def __repr__(self): + def __repr__(self): return self.v -class SQLProducer: + +class SQLProducer: + """Database""" + def __init__(self): """Creates a database. """ pass - - def query(self, sql_query,processed=False, svars=None): + + def query(self, sql_query, processed=False, svars=None): """ Execute SQL query `sql_query` using dictionary `vars` to interpolate it. - If `processed=True`, `vars` is a `reparam`-style list to use + If `processed=True`, `vars` is a `reparam`-style list to use instead of interpolating. - + >>> db = DB(None, {}) >>> db.query("SELECT * FROM foo", _test=True) @@ -479,13 +506,13 @@ def query(self, sql_query,processed=False, svars=None): """ if svars is None: svars = {} - + if not processed and not isinstance(sql_query, SQLQuery): sql_query = reparam(sql_query, svars) - + return sql_query - - def sql_clauses(self, what, tables, where, group, order, limit, offset): + + def sql_clauses(self, what, tables, where, group, order, limit, offset): return ( ('SELECT', what), ('FROM', sqllist(tables)), @@ -494,28 +521,30 @@ def sql_clauses(self, what, tables, where, group, order, limit, offset): ('ORDER BY', order), ('LIMIT', limit), ('OFFSET', offset)) - - def gen_clause(self, sql, val, svars): + + def gen_clause(self, sql, val, svars): if isinstance(val, (int, long)): if sql == 'WHERE': nout = 'id = ' + sqlquote(val) else: nout = SQLQuery(val) - + elif isinstance(val, (list, tuple)) and len(val) == 2: - nout = SQLQuery(val[0], val[1]) # backwards-compatibility + nout = SQLQuery(val[0], val[1]) # backwards-compatibility elif isinstance(val, SQLQuery): nout = val else: nout = reparam(val, svars) def xjoin(a, b): - if a and b: return a + ' ' + b - else: return a or b + if a and b: + return a + ' ' + b + else: + return a or b return xjoin(sql, nout) - - def _where(self, where, svars): + + def _where(self, where, svars): if isinstance(where, (int, long)): where = "id = " + sqlparam(where) elif isinstance(where, (list, tuple)) and len(where) == 2: @@ -523,35 +552,49 @@ def _where(self, where, svars): elif isinstance(where, SQLQuery): pass else: - where = reparam(where, svars) + where = reparam(where, svars) return where - - def select(self, tables, svars=None, what='*', where=None, order=None, group=None, - limit=None, offset=None, _test=False): + + def select(self, tables, svars=None, what='*', where=None, order=None, group=None, + limit=None, offset=None, _test=False): """ - Selects `what` from `tables` with clauses `where`, `order`, - `group`, `limit`, and `offset`. Uses vars to interpolate. + Selects `what` from `tables` with clauses `where`, `order`, + `group`, `limit`, and `offset`. Uses vars to interpolate. Otherwise, each clause can be a SQLQuery. - + >>> db = DB(None, {}) >>> db.select('foo', _test=True) >>> db.select(['foo', 'bar'], where="foo.bar_id = bar.id", limit=5, _test=True) """ - if svars is None: svars = {} - sql_clauses = self.sql_clauses(what, tables, where, group, order, limit, offset) - clauses = [self.gen_clause(sql, val, svars) for sql, val in sql_clauses if val is not None] + if svars is None: + svars = {} + sql_clauses = self.sql_clauses( + what, + tables, + where, + group, + order, + limit, + offset) + clauses = [ + self.gen_clause( + sql, + val, + svars) for sql, + val in sql_clauses if val is not None] qout = SQLQuery.join(clauses) - if _test: return qout + if _test: + return qout return self.query(qout, processed=True) - def insert(self, tablename, seqname=None, _test=False, **values): + def insert(self, tablename, seqname=None, _test=False, **values): """ Inserts `values` into `tablename`. Returns current sequence ID. Set `seqname` to the ID if it's not the default, or to `False` if there isn't one. - + >>> db = DB(None, {}) >>> q = db.insert('foo', name='bob', age=2, created=SQLLiteral('NOW()'), _test=True) >>> q @@ -561,67 +604,86 @@ def insert(self, tablename, seqname=None, _test=False, **values): >>> q.values() [2, 'bob'] """ - def q(x): return "(" + x + ")" - + def q(x): + return "(" + x + ")" + if values: _keys = SQLQuery.join(values.keys(), ', ') - _values = SQLQuery.join([sqlparam(v) for v in values.values()], ', ') - sql_query = "INSERT INTO %s " % tablename + q(_keys) + ' VALUES ' + q(_values) + _values = SQLQuery.join([sqlparam(v) + for v in values.values()], ', ') + sql_query = "INSERT INTO %s " % tablename + \ + q(_keys) + ' VALUES ' + q(_values) else: - sql_query = SQLQuery(self._get_insert_default_values_query(tablename)) + sql_query = SQLQuery( + self._get_insert_default_values_query(tablename)) return sql_query - + def _get_insert_default_values_query(self, table): return "INSERT INTO %s DEFAULT VALUES" % table def multiple_insert(self, tablename, values, seqname=None, _test=False): """ - Inserts multiple rows into `tablename`. The `values` must be a list of dictioanries, + Inserts multiple rows into `tablename`. The `values` must be a list of dictioanries, one for each row to be inserted, each with the same set of keys. - Returns the list of ids of the inserted rows. + Returns the list of ids of the inserted rows. Set `seqname` to the ID if it's not the default, or to `False` if there isn't one. - + >>> db = DB(None, {}) >>> db.supports_multiple_insert = True >>> values = [{"name": "foo", "email": "foo@example.com"}, {"name": "bar", "email": "bar@example.com"}] >>> db.multiple_insert('person', values=values, _test=True) - """ + """ if not values: return [] - + if not self.supports_multiple_insert: - out = [self.insert(tablename, seqname=seqname, _test=_test, **v) for v in values] + out = [ + self.insert( + tablename, + seqname=seqname, + _test=_test, + **v) for v in values] if seqname is False: return None else: return out - + keys = values[0].keys() #@@ make sure all keys are valid # make sure all rows have same keys. for v in values: if v.keys() != keys: - raise ValueError, 'Bad data' + raise ValueError('Bad data') - sql_query = SQLQuery('INSERT INTO %s (%s) VALUES ' % (tablename, ', '.join(keys))) + sql_query = SQLQuery( + 'INSERT INTO %s (%s) VALUES ' % + (tablename, ', '.join(keys))) for i, row in enumerate(values): if i != 0: sql_query.append(", ") - SQLQuery.join([SQLParam(row[k]) for k in keys], sep=", ", target=sql_query, prefix="(", suffix=")") - - if _test: return sql_query + SQLQuery.join([SQLParam(row[k]) for k in keys], + sep=", ", + target=sql_query, + prefix="(", + suffix=")") + + if _test: + return sql_query db_cursor = self._db_cursor() - if seqname is not False: - sql_query = self._process_insert_query(sql_query, tablename, seqname) + if seqname is not False: + sql_query = self._process_insert_query( + sql_query, + tablename, + seqname) if isinstance(sql_query, tuple): - # for some databases, a separate query has to be made to find + # for some databases, a separate query has to be made to find # the id of the inserted row. q1, q2 = sql_query self._db_execute(db_cursor, q1) @@ -629,18 +691,17 @@ def multiple_insert(self, tablename, values, seqname=None, _test=False): else: self._db_execute(db_cursor, sql_query) - try: + try: out = db_cursor.fetchone()[0] - out = range(out-len(values)+1, out+1) - except Exception: + out = range(out - len(values) + 1, out + 1) + except Exception: out = None - if not self.ctx.transactions: + if not self.ctx.transactions: self.ctx.commit() return out - - def update(self, tables, where, svars=None, _test=False, **values): + def update(self, tables, where, svars=None, _test=False, **values): """ Update `tables` with clause `where` (interpolated using `vars`) and setting `values`. @@ -656,23 +717,25 @@ def update(self, tables, where, svars=None, _test=False, **values): >>> q.values() [2, 'bob', 'Joseph'] """ - if svars is None: svars = {} + if svars is None: + svars = {} where = self._where(where, svars) query = ( - "UPDATE " + sqllist(tables) + - " SET " + sqlwhere(values, ', ') + - " WHERE " + where) + "UPDATE " + sqllist(tables) + + " SET " + sqlwhere(values, ', ') + + " WHERE " + where) + + if _test: + return query - if _test: return query - db_cursor = self._db_cursor() self._db_execute(db_cursor, query) - if not self.ctx.transactions: + if not self.ctx.transactions: self.ctx.commit() return db_cursor.rowcount - - def delete(self, table, where, using=None, svars=None, _test=False): + + def delete(self, table, where, using=None, svars=None, _test=False): """ Deletes from `table` with clauses `where` and `using`. @@ -692,9 +755,8 @@ def delete(self, table, where, using=None, svars=None, _test=False): q += ' WHERE ' + where return q - + sqlproducer = SQLProducer() print sqlproducer.delete("tb_item", where="id=123") - diff --git a/firefly/dbentrust/madminanager.py b/firefly/dbentrust/madminanager.py index 014a517..e97935d 100644 --- a/firefly/dbentrust/madminanager.py +++ b/firefly/dbentrust/madminanager.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-22 @@ -6,36 +6,33 @@ ''' from firefly.utils.singleton import Singleton + class MAdminManager: __metaclass__ = Singleton - + def __init__(self): """ """ self.admins = {} - - def registe(self,admin): + + def registe(self, admin): """ """ self.admins[admin._name] = admin - - def dropAdmin(self,adminname): + + def dropAdmin(self, adminname): """ """ - if self.admins.has_key(adminname): + if adminname in self.admins: del self.admins[adminname] - - def getAdmin(self,adminname): + + def getAdmin(self, adminname): """ """ return self.admins.get(adminname) - + def checkAdmins(self): """ """ for admin in self.admins.values(): admin.checkAll() - - - - \ No newline at end of file diff --git a/firefly/dbentrust/memclient.py b/firefly/dbentrust/memclient.py index 4289939..a3a3ab4 100644 --- a/firefly/dbentrust/memclient.py +++ b/firefly/dbentrust/memclient.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-7-10 memcached client @@ -6,99 +6,102 @@ ''' import memcache -class MemConnError(Exception): + +class MemConnError(Exception): + """ """ + def __str__(self): return "memcache connect error" + class MemClient: + '''memcached ''' - - def __init__(self,timeout = 0): + + def __init__(self, timeout=0): ''' ''' self._hostname = "" self._urls = [] self.connection = None - - def connect(self,urls,hostname): + + def connect(self, urls, hostname): '''memcached connect ''' self._hostname = hostname self._urls = urls - self.connection = memcache.Client(self._urls,debug=0) - if not self.connection.set("__testkey__",1): + self.connection = memcache.Client(self._urls, debug=0) + if not self.connection.set("__testkey__", 1): raise MemConnError() - - def produceKey(self,keyname): + + def produceKey(self, keyname): ''' ''' if isinstance(keyname, basestring): - return ''.join([self._hostname,':',keyname]) + return ''.join([self._hostname, ':', keyname]) else: raise "type error" - - def get(self,key): + + def get(self, key): ''' ''' key = self.produceKey(key) return self.connection.get(key) - - def get_multi(self,keys): + + def get_multi(self, keys): ''' ''' keynamelist = [self.produceKey(keyname) for keyname in keys] olddict = self.connection.get_multi(keynamelist) newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], - olddict.values())) + olddict.values())) return newdict - - def set(self,keyname,value): + + def set(self, keyname, value): ''' ''' key = self.produceKey(keyname) - result = self.connection.set(key,value) - if not result:#如果写入失败 - self.connect(self._urls,self._hostname)#重新连接 - return self.connection.set(key,value) + result = self.connection.set(key, value) + if not result: # 如果写入失败 + self.connect(self._urls, self._hostname) # 重新连接 + return self.connection.set(key, value) return result - - def set_multi(self,mapping): + + def set_multi(self, mapping): ''' ''' newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], mapping.values())) result = self.connection.set_multi(newmapping) - if result:#如果写入失败 - self.connect(self._urls,self._hostname)#重新连接 + if result: # 如果写入失败 + self.connect(self._urls, self._hostname) # 重新连接 return self.connection.set_multi(newmapping) return result - - def incr(self,key,delta): + + def incr(self, key, delta): ''' ''' key = self.produceKey(key) return self.connection.incr(key, delta) - - def delete(self,key): + + def delete(self, key): ''' ''' key = self.produceKey(key) return self.connection.delete(key) - - def delete_multi(self,keys): + + def delete_multi(self, keys): """ """ keys = [self.produceKey(key) for key in keys] return self.connection.delete_multi(keys) - + def flush_all(self): ''' ''' self.connection.flush_all() - -mclient = MemClient() - +mclient = MemClient() diff --git a/firefly/dbentrust/memobject.py b/firefly/dbentrust/memobject.py index bc8505e..b578f71 100644 --- a/firefly/dbentrust/memobject.py +++ b/firefly/dbentrust/memobject.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2012-7-10 memcached 关系对象 @@ -7,11 +7,13 @@ @author: lan (www.9miao.com) ''' + class MemObject: + '''memcached 关系对象 ''' - - def __init__(self,name,mc): + + def __init__(self, name, mc): ''' @param name: str 对象的名称 @param _lock: int 对象锁 为1时表示对象被锁定无法进行修改 @@ -19,58 +21,58 @@ def __init__(self,name,mc): self._client = mc self._name = name self._lock = 0 - - def produceKey(self,keyname): + + def produceKey(self, keyname): '''重新生成key ''' if isinstance(keyname, basestring): - return ''.join([self._name,':',keyname]) + return ''.join([self._name, ':', keyname]) else: raise "type error" - + def locked(self): '''检测对象是否被锁定 ''' key = self.produceKey('_lock') return self._client.get(key) - + def lock(self): '''锁定对象 ''' key = self.produceKey('_lock') self._client.set(key, 1) - + def release(self): '''释放锁 ''' key = self.produceKey('_lock') self._client.set(key, 0) - - def get(self,key): + + def get(self, key): '''获取对象值 ''' key = self.produceKey(key) return self._client.get(key) - - def get_multi(self,keys): + + def get_multi(self, keys): '''一次获取多个key的值 @param keys: list(str) key的列表 ''' keynamelist = [self.produceKey(keyname) for keyname in keys] olddict = self._client.get_multi(keynamelist) newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], - olddict.values())) + olddict.values())) return newdict - def update(self,key,values): + def update(self, key, values): '''修改对象的值 ''' if self.locked(): return False key = self.produceKey(key) - return self._client.set(key,values) - - def update_multi(self,mapping): + return self._client.set(key, values) + + def update_multi(self, mapping): '''同时修改多个key值 ''' if self.locked(): @@ -78,7 +80,7 @@ def update_multi(self,mapping): newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], mapping.values())) return self._client.set_multi(newmapping) - + def mdelete(self): '''删除memcache中的数据 ''' @@ -87,13 +89,13 @@ def mdelete(self): keys = nowdict.keys() keys = [self.produceKey(key) for key in keys] self._client.delete_multi(keys) - + def incr(self, key, delta): '''自增 ''' key = self.produceKey(key) - return self._client.incr( key, delta) - + return self._client.incr(key, delta) + def insert(self): '''插入对象记录 ''' @@ -102,7 +104,3 @@ def insert(self): newmapping = dict(zip([self.produceKey(keyname) for keyname in nowdict.keys()], nowdict.values())) self._client.set_multi(newmapping) - - - - diff --git a/firefly/dbentrust/mmode.py b/firefly/dbentrust/mmode.py index f1e6ed6..313b8a1 100644 --- a/firefly/dbentrust/mmode.py +++ b/firefly/dbentrust/mmode.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-8 @@ -9,149 +9,165 @@ import util import time -MMODE_STATE_ORI = 0 #未变更 -MMODE_STATE_NEW = 1 #创建 -MMODE_STATE_UPDATE = 2 #更新 -MMODE_STATE_DEL = 3 #删除 - +MMODE_STATE_ORI = 0 # 未变更 +MMODE_STATE_NEW = 1 # 创建 +MMODE_STATE_UPDATE = 2 # 更新 +MMODE_STATE_DEL = 3 # 删除 TIMEOUT = 1800 + def _insert(args): - record,pkname,mmname,cls = args + record, pkname, mmname, cls = args pk = record[pkname] - mm = cls(mmname+':%s'%pk,pkname,data=record) + mm = cls(mmname + ':%s' % pk, pkname, data=record) mm.insert() return pk -class PKValueError(ValueError): + +class PKValueError(ValueError): + """ """ + def __init__(self, data): ValueError.__init__(self) self.data = data + def __str__(self): return "new record has no 'PK': %s" % (self.data) + class MMode(MemObject): + """内存数据模型 """ - def __init__(self, name,pk,data={}): + + def __init__(self, name, pk, data={}): """ """ MemObject.__init__(self, name, mclient) - self._state = MMODE_STATE_ORI#对象的状态 0未变更 1新建 2更新 3删除 + self._state = MMODE_STATE_ORI # 对象的状态 0未变更 1新建 2更新 3删除 self._pk = pk self.data = data self._time = time.time() - + def update(self, key, values): - data = self.get_multi(['data','_state']) + data = self.get_multi(['data', '_state']) ntime = time.time() - data['data'].update({key:values}) - if data.get('_state')==MMODE_STATE_NEW: - props = {'data':data.get('data'),'_time':ntime} + data['data'].update({key: values}) + if data.get('_state') == MMODE_STATE_NEW: + props = {'data': data.get('data'), '_time': ntime} else: - props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} + props = { + '_state': MMODE_STATE_UPDATE, + 'data': data.get('data'), + '_time': ntime} return MemObject.update_multi(self, props) - + def update_multi(self, mapping): ntime = time.time() - data = self.get_multi(['data','_state']) + data = self.get_multi(['data', '_state']) data['data'].update(mapping) - if data.get('_state')==MMODE_STATE_NEW: - props = {'data':data.get('data'),'_time':ntime} + if data.get('_state') == MMODE_STATE_NEW: + props = {'data': data.get('data'), '_time': ntime} else: - props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} + props = { + '_state': MMODE_STATE_UPDATE, + 'data': data.get('data'), + '_time': ntime} return MemObject.update_multi(self, props) - + def get(self, key): ntime = time.time() MemObject.update(self, "_time", ntime) return MemObject.get(self, key) - + def get_multi(self, keys): ntime = time.time() MemObject.update(self, "_time", ntime) return MemObject.get_multi(self, keys) - + def delete(self): '''删除对象 ''' - return MemObject.update(self,'_state',MMODE_STATE_DEL) - + return MemObject.update(self, '_state', MMODE_STATE_DEL) + def mdelete(self): """清理对象 """ self.syncDB() MemObject.mdelete(self) - + def IsEffective(self): '''检测对象是否有效 ''' - if self.get('_state')==MMODE_STATE_DEL: + if self.get('_state') == MMODE_STATE_DEL: return False return True - + def syncDB(self): """同步到数据库 """ state = self.get('_state') tablename = self._name.split(':')[0] - if state==MMODE_STATE_ORI: + if state == MMODE_STATE_ORI: return - elif state==MMODE_STATE_NEW: + elif state == MMODE_STATE_NEW: props = self.get('data') pk = self.get('_pk') result = util.InsertIntoDB(tablename, props) - elif state==MMODE_STATE_UPDATE: + elif state == MMODE_STATE_UPDATE: props = self.get('data') pk = self.get('_pk') - prere = {pk:props.get(pk)} + prere = {pk: props.get(pk)} util.UpdateWithDict(tablename, props, prere) result = True else: pk = self.get('_pk') props = self.get('data') - prere = {pk:props.get(pk)} - result = util.DeleteFromDB(tablename,prere) + prere = {pk: props.get(pk)} + result = util.DeleteFromDB(tablename, prere) if result: - MemObject.update(self,'_state', MMODE_STATE_ORI) - - def checkSync(self,timeout=TIMEOUT): + MemObject.update(self, '_state', MMODE_STATE_ORI) + + def checkSync(self, timeout=TIMEOUT): """检测同步 """ ntime = time.time() objtime = MemObject.get(self, '_time') - if ntime -objtime>=timeout and timeout: + if ntime - objtime >= timeout and timeout: self.mdelete() else: self.syncDB() - - + + class MFKMode(MemObject): + """内存数据模型 """ - def __init__(self, name,pklist = []): + + def __init__(self, name, pklist=[]): MemObject.__init__(self, name, mclient) self.pklist = pklist - + + class MAdmin(MemObject): - - def __init__(self, name,pk,timeout=TIMEOUT,**kw): + + def __init__(self, name, pk, timeout=TIMEOUT, **kw): MemObject.__init__(self, name, mclient) self._pk = pk - self._fk = kw.get('fk','') - self._incrkey = kw.get('incrkey','') - self._incrvalue = kw.get('incrvalue',0) + self._fk = kw.get('fk', '') + self._incrkey = kw.get('incrkey', '') + self._incrvalue = kw.get('incrvalue', 0) self._timeout = timeout - + def insert(self): if self._incrkey and not self.get("_incrvalue"): self._incrvalue = util.GetTableIncrValue(self._name) MemObject.insert(self) - + def load(self): '''读取数据到数据库中 ''' @@ -159,71 +175,70 @@ def load(self): recordlist = util.ReadDataFromDB(mmname) for record in recordlist: pk = record[self._pk] - mm = MMode(self._name+':%s'%pk,self._pk,data=record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() - + @property def madmininfo(self): keys = self.__dict__.keys() info = self.get_multi(keys) return info - - def getAllPkByFk(self,fk): + + def getAllPkByFk(self, fk): '''根据外键获取主键列表 ''' - name = '%s_fk:%s'%(self._name,fk) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) pklist = fkmm.get('pklist') if pklist is not None: return pklist - props = {self._fk:fk} + props = {self._fk: fk} dbkeylist = util.getAllPkByFkInDB(self._name, self._pk, props) - name = '%s_fk:%s'%(self._name,fk) - fkmm = MFKMode(name, pklist = dbkeylist) + name = '%s_fk:%s' % (self._name, fk) + fkmm = MFKMode(name, pklist=dbkeylist) fkmm.insert() return dbkeylist - - def getObj(self,pk): + + def getObj(self, pk): ''' ''' - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): return None if mm.get('data'): return mm - props = {self._pk:pk} - record = util.GetOneRecordInfo(self._name,props) + props = {self._pk: pk} + record = util.GetOneRecordInfo(self._name, props) if not record: return None - mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() return mm - - def getObjData(self,pk): + + def getObjData(self, pk): ''' ''' - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): return None data = mm.get('data') if mm.get('data'): return data - props = {self._pk:pk} - record = util.GetOneRecordInfo(self._name,props) + props = {self._pk: pk} + record = util.GetOneRecordInfo(self._name, props) if not record: return None - mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() return record - - - def getObjList(self,pklist): + + def getObjList(self, pklist): ''' ''' _pklist = [] objlist = [] for pk in pklist: - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): continue if mm.get('data'): @@ -231,15 +246,15 @@ def getObjList(self,pklist): else: _pklist.append(pk) if _pklist: - recordlist = util.GetRecordList(self._name, self._pk,_pklist) + recordlist = util.GetRecordList(self._name, self._pk, _pklist) for record in recordlist: pk = record[self._pk] - mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() objlist.append(mm) return objlist - - def deleteMode(self,pk): + + def deleteMode(self, pk): ''' ''' mm = self.getObj(pk) @@ -247,8 +262,8 @@ def deleteMode(self,pk): if self._fk: data = mm.get('data') if data: - fk = data.get(self._fk,0) - name = '%s_fk:%s'%(self._name,fk) + fk = data.get(self._fk, 0) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) pklist = fkmm.get('pklist') if pklist and pk in pklist: @@ -256,12 +271,12 @@ def deleteMode(self,pk): fkmm.update('pklist', pklist) mm.delete() return True - + def checkAll(self): - key = '%s:%s:'%(mclient._hostname,self._name) + key = '%s:%s:' % (mclient._hostname, self._name) _pklist = util.getallkeys(key, mclient.connection) for pk in _pklist: - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): mm.mdelete() continue @@ -269,42 +284,41 @@ def checkAll(self): continue mm.checkSync(timeout=self._timeout) self.deleteAllFk() - + def deleteAllFk(self): """删除所有的外键 """ - key = '%s:%s_fk:'%(mclient._hostname,self._name) + key = '%s:%s_fk:' % (mclient._hostname, self._name) _fklist = util.getallkeys(key, mclient.connection) for fk in _fklist: - name = '%s_fk:%s'%(self._name,fk) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) fkmm.mdelete() - - def new(self,data): + + def new(self, data): """创建一个新的对象 """ incrkey = self._incrkey if incrkey: incrvalue = self.incr('_incrvalue', 1) - data[incrkey] = incrvalue - 1 + data[incrkey] = incrvalue - 1 pk = data.get(self._pk) if pk is None: raise PKValueError(data) - mm = MMode(self._name+':%s'%pk,self._pk,data=data) - setattr(mm,incrkey,pk) + mm = MMode(self._name + ':%s' % pk, self._pk, data=data) + setattr(mm, incrkey, pk) else: pk = data.get(self._pk) - mm = MMode(self._name+':%s'%pk,self._pk,data=data) + mm = MMode(self._name + ':%s' % pk, self._pk, data=data) if self._fk: - fk = data.get(self._fk,0) - name = '%s_fk:%s'%(self._name,fk) + fk = data.get(self._fk, 0) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) pklist = fkmm.get('pklist') if pklist is None: pklist = self.getAllPkByFk(fk) pklist.append(pk) fkmm.update('pklist', pklist) - setattr(mm,'_state',MMODE_STATE_NEW) + setattr(mm, '_state', MMODE_STATE_NEW) mm.insert() return mm - diff --git a/firefly/dbentrust/util.py b/firefly/dbentrust/util.py index 570e862..0e65f56 100644 --- a/firefly/dbentrust/util.py +++ b/firefly/dbentrust/util.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-8 @@ -11,71 +11,79 @@ from twisted.python import log -def forEachPlusInsertProps(tablename,props): - assert type(props) == dict - pkeysstr = str(tuple(props.keys())).replace('\'','`') - pvaluesstr = ["%s,"%val if isinstance(val,Number) else - "'%s',"%str(val).replace("'", "\\'") for val in props.values()] +def forEachPlusInsertProps(tablename, props): + assert isinstance(props, dict) + pkeysstr = str(tuple(props.keys())).replace('\'', '`') + pvaluesstr = ["%s," % val if isinstance(val, Number) else + "'%s'," % str(val).replace("'", "\\'") for val in props.values()] pvaluesstr = ''.join(pvaluesstr)[:-1] - sqlstr = """INSERT INTO `%s` %s values (%s);"""%(tablename,pkeysstr,pvaluesstr) + sqlstr = """INSERT INTO `%s` %s values (%s);""" % ( + tablename, pkeysstr, pvaluesstr) return sqlstr + def FormatCondition(props): """生成查询条件字符串 """ items = props.items() itemstrlist = [] for _item in items: - if isinstance(_item[1],Number): - sqlstr = " `%s`=%s AND"%_item + if isinstance(_item[1], Number): + sqlstr = " `%s`=%s AND" % _item else: - sqlstr = " `%s`='%s' AND "%(_item[0],str(_item[1]).replace("'", "\\'")) + sqlstr = " `%s`='%s' AND " % ( + _item[0], str(_item[1]).replace("'", "\\'")) itemstrlist.append(sqlstr) sqlstr = ''.join(itemstrlist) return sqlstr[:-4] + def FormatUpdateStr(props): """生成更新语句 """ items = props.items() itemstrlist = [] for _item in items: - if isinstance(_item[1],Number): - sqlstr = " `%s`=%s,"%_item + if isinstance(_item[1], Number): + sqlstr = " `%s`=%s," % _item else: - sqlstr = " `%s`='%s',"%(_item[0],str(_item[1]).replace("'", "\\'")) + sqlstr = " `%s`='%s'," % ( + _item[0], str(_item[1]).replace("'", "\\'")) itemstrlist.append(sqlstr) sqlstr = ''.join(itemstrlist) return sqlstr[:-1] - -def forEachUpdateProps(tablename,props,prere): + + +def forEachUpdateProps(tablename, props, prere): '''遍历所要修改的属性，以生成sql语句''' - assert type(props) == dict + assert isinstance(props, dict) pro = FormatUpdateStr(props) pre = FormatCondition(prere) - sqlstr = """UPDATE `%s` SET %s WHERE %s;"""%(tablename,pro,pre) + sqlstr = """UPDATE `%s` SET %s WHERE %s;""" % (tablename, pro, pre) return sqlstr + def EachQueryProps(props): '''遍历字段列表生成sql语句 ''' sqlstr = "" if props == '*': return '*' - elif type(props) == type([0]): + elif isinstance(props, type([0])): for prop in props: - sqlstr = sqlstr + prop +',' + sqlstr = sqlstr + prop + ',' sqlstr = sqlstr[:-1] return sqlstr else: raise Exception('props to query must be dict') return + def forEachQueryProps(sqlstr, props): '''遍历所要查询属性，以生成sql语句''' if props == '*': sqlstr += ' *' - elif type(props) == type([0]): + elif isinstance(props, type([0])): i = 0 for prop in props: if(i == 0): @@ -88,12 +96,13 @@ def forEachQueryProps(sqlstr, props): return return sqlstr + def GetTableIncrValue(tablename): """ """ database = dbpool.config.get('db') sql = """SELECT AUTO_INCREMENT FROM information_schema.`TABLES` \ - WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s';"""%(database,tablename) + WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s';""" % (database, tablename) conn = dbpool.connection() cursor = conn.cursor() cursor.execute(sql) @@ -104,54 +113,58 @@ def GetTableIncrValue(tablename): return result[0] return result + def ReadDataFromDB(tablename): """ """ - sql = """select * from %s"""%tablename + sql = """select * from %s""" % tablename conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) - result=cursor.fetchall() + result = cursor.fetchall() cursor.close() conn.close() return result -def DeleteFromDB(tablename,props): + +def DeleteFromDB(tablename, props): '''从数据库中删除 ''' prers = FormatCondition(props) - sql = """DELETE FROM %s WHERE %s ;"""%(tablename,prers) + sql = """DELETE FROM %s WHERE %s ;""" % (tablename, prers) conn = dbpool.connection() cursor = conn.cursor() count = 0 try: count = cursor.execute(sql) conn.commit() - except Exception,e: + except Exception as e: log.err(e) log.err(sql) cursor.close() conn.close() return bool(count) -def InsertIntoDB(tablename,data): + +def InsertIntoDB(tablename, data): """写入数据库 """ - sql = forEachPlusInsertProps(tablename,data) + sql = forEachPlusInsertProps(tablename, data) conn = dbpool.connection() cursor = conn.cursor() count = 0 try: count = cursor.execute(sql) conn.commit() - except Exception,e: + except Exception as e: log.err(e) log.err(sql) cursor.close() conn.close() return bool(count) -def UpdateWithDict(tablename,props,prere): + +def UpdateWithDict(tablename, props, prere): """更新记录 """ sql = forEachUpdateProps(tablename, props, prere) @@ -161,7 +174,7 @@ def UpdateWithDict(tablename,props,prere): try: count = cursor.execute(sql) conn.commit() - except Exception,e: + except Exception as e: log.err(e) log.err(sql) cursor.close() @@ -170,11 +183,12 @@ def UpdateWithDict(tablename,props,prere): return True return False -def getAllPkByFkInDB(tablename,pkname,props): + +def getAllPkByFkInDB(tablename, pkname, props): """根据所有的外键获取主键ID """ props = FormatCondition(props) - sql = """Select `%s` from `%s` where %s"""%(pkname,tablename,props) + sql = """Select `%s` from `%s` where %s""" % (pkname, tablename, props) conn = dbpool.connection() cursor = conn.cursor() cursor.execute(sql) @@ -183,52 +197,57 @@ def getAllPkByFkInDB(tablename,pkname,props): conn.close() return [key[0] for key in result] -def GetOneRecordInfo(tablename,props): + +def GetOneRecordInfo(tablename, props): '''获取单条数据的信息 ''' props = FormatCondition(props) - sql = """Select * from `%s` where %s"""%(tablename,props) + sql = """Select * from `%s` where %s""" % (tablename, props) conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) result = cursor.fetchone() cursor.close() conn.close() return result -def GetRecordList(tablename,pkname,pklist): + +def GetRecordList(tablename, pkname, pklist): """ """ pkliststr = "" for pkid in pklist: - pkliststr+="%s,"%pkid - pkliststr = "(%s)"%pkliststr[:-1] - sql = """SELECT * FROM `%s` WHERE `%s` IN %s;"""%(tablename,pkname,pkliststr) + pkliststr += "%s," % pkid + pkliststr = "(%s)" % pkliststr[:-1] + sql = """SELECT * FROM `%s` WHERE `%s` IN %s;""" % ( + tablename, pkname, pkliststr) conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) result = cursor.fetchall() cursor.close() conn.close() return result + def DBTest(): sql = """SELECT * FROM tb_item WHERE characterId=1000001;""" conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) - result=cursor.fetchall() + result = cursor.fetchall() cursor.close() conn.close() return result -def getallkeys(key,mem): + +def getallkeys(key, mem): itemsinfo = mem.get_stats('items') itemindex = [] for items in itemsinfo: - itemindex += [ _key.split(':')[1] for _key in items[1].keys()] - s = set(itemindex) - itemss = [mem.get_stats('cachedump %s 0'%i) for i in s] + itemindex += [_key.split(':')[1] for _key in items[1].keys()] + s = set(itemindex) + itemss = [mem.get_stats('cachedump %s 0' % i) for i in s] allkeys = set([]) for item in itemss: for _item in item: @@ -244,6 +263,7 @@ def getallkeys(key,mem): allkeys = allkeys.union(nowlist) return allkeys -def getAllPkByFkInMEM(key,fk,mem): - + +def getAllPkByFkInMEM(key, fk, mem): + pass diff --git a/firefly/distributed/child.py b/firefly/distributed/child.py index d26e01b..ccc3943 100644 --- a/firefly/distributed/child.py +++ b/firefly/distributed/child.py @@ -1,35 +1,33 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @author: lan (www.9miao.com) ''' + + class Child(object): + '''子节点对象''' - - def __init__(self,cid,name): + + def __init__(self, cid, name): '''初始化子节点对象 ''' self._id = cid self._name = name self._transport = None - + def getName(self): '''获取子节点的名称''' return self._name - - def setTransport(self,transport): + + def setTransport(self, transport): '''设置子节点的通道''' self._transport = transport - - def callbackChild(self,*args,**kw): + + def callbackChild(self, *args, **kw): '''回调子节点的接口 return a Defered Object (recvdata) ''' - recvdata = self._transport.callRemote('callChild',*args,**kw) + recvdata = self._transport.callRemote('callChild', *args, **kw) return recvdata - - - - - \ No newline at end of file diff --git a/firefly/distributed/manager.py b/firefly/distributed/manager.py index 266d479..4b2c558 100644 --- a/firefly/distributed/manager.py +++ b/firefly/distributed/manager.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @@ -9,111 +9,115 @@ from zope.interface import Interface from zope.interface import implements + class _ChildsManager(Interface): + '''节点管理器接口''' - + def __init__(self): '''初始化接口''' - - def getChildById(self,childId): + + def getChildById(self, childId): '''根据节点id获取节点实例''' - - def getChildByName(self,childname): + + def getChildByName(self, childname): '''根据节点的名称获取节点实例''' - - def addChild(self,child): + + def addChild(self, child): '''添加一个child节点 @param child: Child object ''' - - def dropChild(self,*arg,**kw): + + def dropChild(self, *arg, **kw): '''删除一个节点''' - - def callChild(self,*args,**kw): + + def callChild(self, *args, **kw): '''调用子节点的接口''' - - def callChildByName(self,*args,**kw): + + def callChildByName(self, *args, **kw): '''调用子节点的接口 @param childname: str 子节点的名称 ''' - - def dropChildByID(self,childId): + + def dropChildByID(self, childId): '''删除一个child 节点 - @param childId: Child ID + @param childId: Child ID ''' - + def dropChildSessionId(self, session_id): """根据session_id删除child节点 """ + class ChildsManager(object): + '''子节点管理器''' - + implements(_ChildsManager) - + def __init__(self): '''初始化子节点管理器''' self._childs = {} - - def getChildById(self,childId): + + def getChildById(self, childId): '''根据节点的ID获取节点实例''' return self._childs.get(childId) - - def getChildByName(self,childname): + + def getChildByName(self, childname): '''根据节点的名称获取节点实例''' - for key,child in self._childs.items(): + for key, child in self._childs.items(): if child.getName() == childname: return self._childs[key] return None - - def addChild(self,child): + + def addChild(self, child): '''添加一个child节点 @param child: Child object ''' key = child._id - if self._childs.has_key(key): - raise "child node %s exists"% key + if key in self._childs: + raise "child node %s exists" % key self._childs[key] = child - - def dropChild(self,child): + + def dropChild(self, child): '''删除一个child 节点 - @param child: Child Object + @param child: Child Object ''' key = child._id try: del self._childs[key] - except Exception,e: + except Exception as e: log.msg(str(e)) - - def dropChildByID(self,childId): + + def dropChildByID(self, childId): '''删除一个child 节点 - @param childId: Child ID + @param childId: Child ID ''' try: del self._childs[childId] - except Exception,e: + except Exception as e: log.msg(str(e)) - - def callChild(self,childId,*args,**kw): + + def callChild(self, childId, *args, **kw): '''调用子节点的接口 @param childId: int 子节点的id ''' - child = self._childs.get(childId,None) + child = self._childs.get(childId, None) if not child: - log.err("child %s doesn't exists"%childId) + log.err("child %s doesn't exists" % childId) return - return child.callbackChild(*args,**kw) - - def callChildByName(self,childname,*args,**kw): + return child.callbackChild(*args, **kw) + + def callChildByName(self, childname, *args, **kw): '''调用子节点的接口 @param childname: str 子节点的名称 ''' child = self.getChildByName(childname) if not child: - log.err("child %s doesn't exists"%childname) + log.err("child %s doesn't exists" % childname) return - return child.callbackChild(*args,**kw) - + return child.callbackChild(*args, **kw) + def getChildBYSessionId(self, session_id): """根据sessionID获取child节点信息 """ @@ -121,5 +125,3 @@ def getChildBYSessionId(self, session_id): if child._transport.broker.transport.sessionno == session_id: return child return None - - \ No newline at end of file diff --git a/firefly/distributed/node.py b/firefly/distributed/node.py index 992902a..87c85c2 100644 --- a/firefly/distributed/node.py +++ b/firefly/distributed/node.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @@ -9,17 +9,19 @@ reactor = reactor from reference import ProxyReference -def callRemote(obj,funcName,*args,**kw): + +def callRemote(obj, funcName, *args, **kw): '''远程调用 @param funcName: str 远程方法 ''' - return obj.callRemote(funcName, *args,**kw) - - + return obj.callRemote(funcName, *args, **kw) + + class RemoteObject(object): + '''远程调用对象''' - - def __init__(self,name): + + def __init__(self, name): '''初始化远程调用对象 @param port: int 远程分布服的端口号 @param rootaddr: 根节点服务器地址 @@ -28,40 +30,41 @@ def __init__(self,name): self._factory = pb.PBClientFactory() self._reference = ProxyReference() self._addr = None - - def setName(self,name): + + def setName(self, name): '''设置节点的名称''' self._name = name - + def getName(self): '''获取节点的名称''' return self._name - - def connect(self,addr): + + def connect(self, addr): '''初始化远程调用对象''' self._addr = addr reactor.connectTCP(addr[0], addr[1], self._factory) self.takeProxy() - + def reconnect(self): '''重新连接''' self.connect(self._addr) - - def addServiceChannel(self,service): + + def addServiceChannel(self, service): '''设置引用对象''' self._reference.addService(service) - + def takeProxy(self): '''像远程服务端发送代理通道对象 ''' deferedRemote = self._factory.getRootObject() - deferedRemote.addCallback(callRemote,'takeProxy',self._name,self._reference) - - def callRemote(self,commandId,*args,**kw): + deferedRemote.addCallback( + callRemote, + 'takeProxy', + self._name, + self._reference) + + def callRemote(self, commandId, *args, **kw): '''远程调用''' deferedRemote = self._factory.getRootObject() - return deferedRemote.addCallback(callRemote,'callTarget',commandId,*args,**kw) - - - - \ No newline at end of file + return deferedRemote.addCallback( + callRemote, 'callTarget', commandId, *args, **kw) diff --git a/firefly/distributed/reference.py b/firefly/distributed/reference.py index 4c0a8bf..f511ebd 100644 --- a/firefly/distributed/reference.py +++ b/firefly/distributed/reference.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @@ -9,21 +9,18 @@ class ProxyReference(pb.Referenceable): + '''代理通道''' - + def __init__(self): '''初始化''' self._service = Service('proxy') - - def addService(self,service): + + def addService(self, service): '''添加一条服务通道''' self._service = service - - def remote_callChild(self, command,*arg,**kw): + + def remote_callChild(self, command, *arg, **kw): '''代理发送数据 ''' - return self._service.callTarget(command,*arg,**kw) - - - - \ No newline at end of file + return self._service.callTarget(command, *arg, **kw) diff --git a/firefly/distributed/root.py b/firefly/distributed/root.py index f420213..e2ea8b1 100644 --- a/firefly/distributed/root.py +++ b/firefly/distributed/root.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 分布式根节点 @@ -9,69 +9,69 @@ from manager import ChildsManager from child import Child + class BilateralBroker(pb.Broker): - + def connectionLost(self, reason): clientID = self.transport.sessionno - log.msg("node [%d] lose"%clientID) + log.msg("node [%d] lose" % clientID) self.factory.root.dropChildSessionId(clientID) pb.Broker.connectionLost(self, reason) - - - + + class BilateralFactory(pb.PBServerFactory): - + protocol = BilateralBroker - - + class PBRoot(pb.Root): + '''PB 协议''' - - def __init__(self,dnsmanager = ChildsManager()): + + def __init__(self, dnsmanager=ChildsManager()): '''初始化根节点 ''' self.service = None self.childsmanager = dnsmanager - - def addServiceChannel(self,service): + + def addServiceChannel(self, service): '''添加服务通道 @param service: Service Object(In bilateral.services) ''' self.service = service - - def remote_takeProxy(self,name,transport): + + def remote_takeProxy(self, name, transport): '''设置代理通道 @param addr: (hostname,port)hostname 根节点的主机名,根节点的端口 ''' - log.msg('node [%s] takeProxy ready'%name) - child = Child(name,name) + log.msg('node [%s] takeProxy ready' % name) + child = Child(name, name) self.childsmanager.addChild(child) child.setTransport(transport) self.doChildConnect(name, transport) - - def doChildConnect(self,name,transport): + + def doChildConnect(self, name, transport): """当node节点连接时的处理 """ pass - - def remote_callTarget(self,command,*args,**kw): + + def remote_callTarget(self, command, *args, **kw): '''远程调用方法 @param commandId: int 指令号 @param data: str 调用参数 ''' - data = self.service.callTarget(command,*args,**kw) + data = self.service.callTarget(command, *args, **kw) return data - - def dropChild(self,*args,**kw): + + def dropChild(self, *args, **kw): '''删除子节点记录''' - self.childsmanager.dropChild(*args,**kw) - - def dropChildByID(self,childId): + self.childsmanager.dropChild(*args, **kw) + + def dropChildByID(self, childId): '''删除子节点记录''' self.doChildLostConnect(childId) self.childsmanager.dropChildByID(childId) - + def dropChildSessionId(self, session_id): '''删除子节点记录''' child = self.childsmanager.getChildBYSessionId(session_id) @@ -80,23 +80,22 @@ def dropChildSessionId(self, session_id): child_id = child._id self.doChildLostConnect(child_id) self.childsmanager.dropChildByID(child_id) - - def doChildLostConnect(self,childId): + + def doChildLostConnect(self, childId): """当node节点连接时的处理 """ pass - - def callChild(self,key,*args,**kw): + + def callChild(self, key, *args, **kw): '''调用子节点的接口 @param childId: int 子节点的id return Defered Object ''' - return self.childsmanager.callChild(key,*args,**kw) - - def callChildByName(self,childname,*args,**kw): + return self.childsmanager.callChild(key, *args, **kw) + + def callChildByName(self, childname, *args, **kw): '''调用子节点的接口 @param childId: int 子节点的id return Defered Object ''' - return self.childsmanager.callChildByName(childname,*args,**kw) - + return self.childsmanager.callChildByName(childname, *args, **kw) diff --git a/firefly/management/__init__.py b/firefly/management/__init__.py index dc34009..dc8cf8c 100644 --- a/firefly/management/__init__.py +++ b/firefly/management/__init__.py @@ -1,41 +1,42 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-8 @author: lan (www.9miao.com) ''' -from firefly.management import commands +from firefly.management import commands import sys -class CommandError(Exception): + +class CommandError(Exception): + """ """ + def __str__(self): return "command error" + class Command: - - def __init__(self,subcommond,*args): + + def __init__(self, subcommond, *args): '''工具类指令 ''' self.subcommond = subcommond self.args = args - + def execute(self): ''' ''' - commmd = getattr(commands,self.subcommond ) + commmd = getattr(commands, self.subcommond) commmd.execute(*self.args) - - + + def execute_commands(*args): ''' ''' - if len(args)<2: + if len(args) < 2: raise CommandError() subcommand = args[1] - comm = Command(subcommand,*tuple(args[2:])) + comm = Command(subcommand, *tuple(args[2:])) comm.execute() - - - \ No newline at end of file diff --git a/firefly/management/commands/__init__.py b/firefly/management/commands/__init__.py index 7a36be4..da25b2b 100644 --- a/firefly/management/commands/__init__.py +++ b/firefly/management/commands/__init__.py @@ -1,3 +1,3 @@ import createproject import stopservice -import reloadmodule \ No newline at end of file +import reloadmodule diff --git a/firefly/management/commands/createproject.py b/firefly/management/commands/createproject.py index c5ef42d..5ab121f 100644 --- a/firefly/management/commands/createproject.py +++ b/firefly/management/commands/createproject.py @@ -1,35 +1,135 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-8 @author: lan (www.9miao.com) ''' -import sys,os -startmasterfile =['#coding:utf8\n', '\n', 'import os\n', "if os.name!='nt' and os.name!='posix':\n", ' from twisted.internet import epollreactor\n', ' epollreactor.install()\n', '\n', 'if __name__=="__main__":\n', ' from firefly.master.master import Master\n', ' master = Master()\n', " master.config('config.json','appmain.py')\n", ' master.start()\n', ' \n', ' '] -configfile = ['{\n', +import sys +import os +startmasterfile = [ + '#coding:utf8\n', + '\n', + 'import os\n', + "if os.name!='nt' and os.name!='posix':\n", + ' from twisted.internet import epollreactor\n', + ' epollreactor.install()\n', + '\n', + 'if __name__=="__main__":\n', + ' from firefly.master.master import Master\n', + ' master = Master()\n', + " master.config('config.json','appmain.py')\n", + ' master.start()\n', + ' \n', + ' '] +configfile = ['{\n', '"master":{"roothost":"localhost","rootport":9999,"webport":9998},\n', - '"servers":{\n', '"net":{"netport":1000,"name":"gate","remoteport":[{"rootport":20001,"rootname":"gate"}],"app":"app.apptest"},\n', - '"gate":{"rootport":20001,"name":"gate"}\n', - '},\n', '"db":{\n', '"host":"localhost",\n', - '"user":"root",\n', '"passwd":"111",\n', - '"port":3306,\n', - '"db":"test",\n', - '"charset":"utf8"\n', - '},\n', - '"memcached":{\n', - '"urls":["127.0.0.1:11211"],\n', - '"hostname":"test"\n', - '}\n', - '}\n'] -appmainfile = ['#coding:utf8\n', '\n','import os\n', "if os.name!='nt' and os.name!='posix':\n", ' from twisted.internet import epollreactor\n', ' epollreactor.install()\n', '\n', 'import json,sys\n', 'from firefly.server.server import FFServer\n', '\n', 'if __name__=="__main__":\n', ' args = sys.argv\n', ' servername = None\n', ' config = None\n', ' if len(args)>2:\n', ' servername = args[1]\n', " config = json.load(open(args[2],'r'))\n", ' else:\n', ' raise ValueError\n', " dbconf = config.get('db')\n", " memconf = config.get('memcached')\n", " sersconf = config.get('servers',{})\n", " masterconf = config.get('master',{})\n", ' serconfig = sersconf.get(servername)\n', ' ser = FFServer()\n', ' ser.config(serconfig, servername=servername, dbconfig=dbconf, memconfig=memconf, masterconf=masterconf)\n', ' ser.start()\n', ' \n', ' '] -apptestfile = ['#coding:utf8\n', '\n', 'from firefly.server.globalobject import netserviceHandle\n', '\n', '@netserviceHandle\n', 'def echo_1(_conn,data):\n', ' return data\n', '\n', ' \n', '\n', '\n'] -clientfile = ['#coding:utf8\n', '\n', 'import time\n', '\n', 'from socket import AF_INET,SOCK_STREAM,socket\n', 'from thread import start_new\n', 'import struct\n', "HOST='localhost'\n", 'PORT=1000\n', 'BUFSIZE=1024\n', 'ADDR=(HOST , PORT)\n', 'client = socket(AF_INET,SOCK_STREAM)\n', 'client.connect(ADDR)\n', '\n', 'def sendData(sendstr,commandId):\n', ' HEAD_0 = chr(0)\n', ' HEAD_1 = chr(0)\n', ' HEAD_2 = chr(0)\n', ' HEAD_3 = chr(0)\n', ' ProtoVersion = chr(0)\n', ' ServerVersion = 0\n', ' sendstr = sendstr\n', " data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,\\\n", ' HEAD_3,ProtoVersion,ServerVersion,\\\n', ' len(sendstr)+4,commandId)\n', ' senddata = data+sendstr\n', ' return senddata\n', '\n', 'def resolveRecvdata(data):\n', " head = struct.unpack('!sssss3I',data[:17])\n", ' length = head[6]\n', ' data = data[17:17+length]\n', ' return data\n', '\n', 's1 = time.time()\n', '\n', 'def start():\n', ' for i in xrange(10):\n', " client.sendall(sendData('asdfe',1))\n", '\n', 'for i in range(10):\n', ' start_new(start,())\n', 'while True:\n', ' pass\n', '\n'] - - -def createfile(rootpath,path,filecontent): + '"servers":{\n', '"net":{"netport":1000,"name":"gate","remoteport":[{"rootport":20001,"rootname":"gate"}],"app":"app.apptest"},\n', + '"gate":{"rootport":20001,"name":"gate"}\n', + '},\n', '"db":{\n', '"host":"localhost",\n', + '"user":"root",\n', '"passwd":"111",\n', + '"port":3306,\n', + '"db":"test",\n', + '"charset":"utf8"\n', + '},\n', + '"memcached":{\n', + '"urls":["127.0.0.1:11211"],\n', + '"hostname":"test"\n', + '}\n', + '}\n'] +appmainfile = [ + '#coding:utf8\n', + '\n', + 'import os\n', + "if os.name!='nt' and os.name!='posix':\n", + ' from twisted.internet import epollreactor\n', + ' epollreactor.install()\n', + '\n', + 'import json,sys\n', + 'from firefly.server.server import FFServer\n', + '\n', + 'if __name__=="__main__":\n', + ' args = sys.argv\n', + ' servername = None\n', + ' config = None\n', + ' if len(args)>2:\n', + ' servername = args[1]\n', + " config = json.load(open(args[2],'r'))\n", + ' else:\n', + ' raise ValueError\n', + " dbconf = config.get('db')\n", + " memconf = config.get('memcached')\n", + " sersconf = config.get('servers',{})\n", + " masterconf = config.get('master',{})\n", + ' serconfig = sersconf.get(servername)\n', + ' ser = FFServer()\n', + ' ser.config(serconfig, servername=servername, dbconfig=dbconf, memconfig=memconf, masterconf=masterconf)\n', + ' ser.start()\n', + ' \n', + ' '] +apptestfile = [ + '#coding:utf8\n', + '\n', + 'from firefly.server.globalobject import netserviceHandle\n', + '\n', + '@netserviceHandle\n', + 'def echo_1(_conn,data):\n', + ' return data\n', + '\n', + ' \n', + '\n', + '\n'] +clientfile = [ + '#coding:utf8\n', + '\n', + 'import time\n', + '\n', + 'from socket import AF_INET,SOCK_STREAM,socket\n', + 'from thread import start_new\n', + 'import struct\n', + "HOST='localhost'\n", + 'PORT=1000\n', + 'BUFSIZE=1024\n', + 'ADDR=(HOST , PORT)\n', + 'client = socket(AF_INET,SOCK_STREAM)\n', + 'client.connect(ADDR)\n', + '\n', + 'def sendData(sendstr,commandId):\n', + ' HEAD_0 = chr(0)\n', + ' HEAD_1 = chr(0)\n', + ' HEAD_2 = chr(0)\n', + ' HEAD_3 = chr(0)\n', + ' ProtoVersion = chr(0)\n', + ' ServerVersion = 0\n', + ' sendstr = sendstr\n', + " data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,\\\n", + ' HEAD_3,ProtoVersion,ServerVersion,\\\n', + ' len(sendstr)+4,commandId)\n', + ' senddata = data+sendstr\n', + ' return senddata\n', + '\n', + 'def resolveRecvdata(data):\n', + " head = struct.unpack('!sssss3I',data[:17])\n", + ' length = head[6]\n', + ' data = data[17:17+length]\n', + ' return data\n', + '\n', + 's1 = time.time()\n', + '\n', + 'def start():\n', + ' for i in xrange(10):\n', + " client.sendall(sendData('asdfe',1))\n", + '\n', + 'for i in range(10):\n', + ' start_new(start,())\n', + 'while True:\n', + ' pass\n', + '\n'] + + +def createfile(rootpath, path, filecontent): ''' ''' - mfile = open(rootpath+'/'+path,'w') + mfile = open(rootpath + '/' + path, 'w') mfile.writelines(filecontent) mfile.close() @@ -38,23 +138,21 @@ def execute(*args): if not args: sys.stdout.write("command error \n") projectname = args[0] - sys.stdout.write("create dir %s \n"%projectname) + sys.stdout.write("create dir %s \n" % projectname) rootpath = projectname os.mkdir(rootpath) - createfile(rootpath,'startmaster.py',startmasterfile) - createfile(rootpath,'config.json',configfile) - createfile(rootpath,'appmain.py',appmainfile) - - rootpath = projectname+'/'+'app' + createfile(rootpath, 'startmaster.py', startmasterfile) + createfile(rootpath, 'config.json', configfile) + createfile(rootpath, 'appmain.py', appmainfile) + + rootpath = projectname + '/' + 'app' os.mkdir(rootpath) - createfile(rootpath,'__init__.py',[]) - createfile(rootpath,'apptest.py',apptestfile) - - rootpath = projectname+'/'+'tool' + createfile(rootpath, '__init__.py', []) + createfile(rootpath, 'apptest.py', apptestfile) + + rootpath = projectname + '/' + 'tool' os.mkdir(rootpath) - createfile(rootpath,'__init__.py',[]) - createfile(rootpath,'clienttest.py',clientfile) - + createfile(rootpath, '__init__.py', []) + createfile(rootpath, 'clienttest.py', clientfile) + sys.stdout.write("create success \n") - - \ No newline at end of file diff --git a/firefly/management/commands/reloadmodule.py b/firefly/management/commands/reloadmodule.py index 96715a0..80a2b5f 100644 --- a/firefly/management/commands/reloadmodule.py +++ b/firefly/management/commands/reloadmodule.py @@ -1,19 +1,21 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-12 @author: lan (www.9miao.com) ''' -import urllib,sys +import urllib +import sys + def execute(*args): """ """ if not args: - masterport =9998 + masterport = 9998 else: masterport = int(args[0]) - url = "http://localhost:%s/reloadmodule"%masterport + url = "http://localhost:%s/reloadmodule" % masterport try: response = urllib.urlopen(url) except: @@ -21,4 +23,4 @@ def execute(*args): if response: sys.stdout.write("reload module success \n") else: - sys.stdout.write("reload module failed \n") \ No newline at end of file + sys.stdout.write("reload module failed \n") diff --git a/firefly/management/commands/stopservice.py b/firefly/management/commands/stopservice.py index f2a1ee6..edc84f6 100644 --- a/firefly/management/commands/stopservice.py +++ b/firefly/management/commands/stopservice.py @@ -1,26 +1,28 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-11 @author: lan (www.9miao.com) ''' -import urllib,sys +import urllib +import sys + def execute(*args): """ """ if not args: - masterport =9998 + masterport = 9998 hostname = "localhost" else: - if len(args)>1: + if len(args) > 1: hostname = args[0] masterport = int(args[1]) else: hostname = "localhost" masterport = int(args[0]) - - url = "http://%s:%s/stop"%(hostname, masterport) + + url = "http://%s:%s/stop" % (hostname, masterport) try: response = urllib.urlopen(url) except: @@ -29,5 +31,3 @@ def execute(*args): sys.stdout.write("stop service success \n") else: sys.stdout.write("stop service failed \n") - - \ No newline at end of file diff --git a/firefly/master/master.py b/firefly/master/master.py index 46986c2..3597ed6 100644 --- a/firefly/master/master.py +++ b/firefly/master/master.py @@ -1,13 +1,15 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @author: lan (www.9miao.com) ''' -import subprocess,json,sys +import subprocess +import json +import sys from twisted.internet import reactor -from firefly.utils import services -from firefly.distributed.root import PBRoot,BilateralFactory +from firefly.utils import services +from firefly.distributed.root import PBRoot, BilateralFactory from firefly.server.globalobject import GlobalObject from twisted.web import vhost from firefly.web.delayrequest import DelaySite @@ -21,11 +23,11 @@ MASTER_SERVER_MODE = 3 - class Master: + """ """ - + def __init__(self): """ """ @@ -33,17 +35,17 @@ def __init__(self): self.mainpath = None self.root = None self.webroot = None - - def config(self,configpath,mainpath): + + def config(self, configpath, mainpath): """ """ self.configpath = configpath self.mainpath = mainpath - + def masterapp(self): """ """ - config = json.load(open(self.configpath,'r')) + config = json.load(open(self.configpath, 'r')) GlobalObject().json_config = config mastercnf = config.get('master') rootport = mastercnf.get('rootport') @@ -57,18 +59,18 @@ def masterapp(self): GlobalObject().root = self.root GlobalObject().webroot = self.webroot if masterlog: - log.addObserver(loogoo(masterlog))#日志处理 + log.addObserver(loogoo(masterlog)) # 日志处理 log.startLogging(sys.stdout) import webapp import rootapp reactor.listenTCP(webport, DelaySite(self.webroot)) reactor.listenTCP(rootport, BilateralFactory(self.root)) - + def start(self): ''' ''' sys_args = sys.argv - if len(sys_args)>2 and sys_args[1] == "single": + if len(sys_args) > 2 and sys_args[1] == "single": server_name = sys_args[2] if server_name == "master": mode = MASTER_SERVER_MODE @@ -77,22 +79,22 @@ def start(self): else: mode = MULTI_SERVER_MODE server_name = "" - + if mode == MULTI_SERVER_MODE: self.masterapp() - config = json.load(open(self.configpath,'r')) + config = json.load(open(self.configpath, 'r')) sersconf = config.get('servers') for sername in sersconf.keys(): - cmds = 'python %s %s %s'%(self.mainpath,sername,self.configpath) - subprocess.Popen(cmds,shell=True) + cmds = 'python %s %s %s' % ( + self.mainpath, sername, self.configpath) + subprocess.Popen(cmds, shell=True) reactor.run() elif mode == SINGLE_SERVER_MODE: - config = json.load(open(self.configpath,'r')) + config = json.load(open(self.configpath, 'r')) sername = server_name - cmds = 'python %s %s %s'%(self.mainpath,sername,self.configpath) - subprocess.Popen(cmds,shell=True) + cmds = 'python %s %s %s' % ( + self.mainpath, sername, self.configpath) + subprocess.Popen(cmds, shell=True) else: self.masterapp() reactor.run() - - diff --git a/firefly/master/rootapp.py b/firefly/master/rootapp.py index 560985e..5aae8e5 100644 --- a/firefly/master/rootapp.py +++ b/firefly/master/rootapp.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-7 @@ -8,34 +8,45 @@ from twisted.python import log -def _doChildConnect(name,transport): +def _doChildConnect(name, transport): """当server节点连接到master的处理 """ - server_config = GlobalObject().json_config.get('servers',{}).get(name,{}) - remoteport = server_config.get('remoteport',[]) + server_config = GlobalObject().json_config.get('servers', {}).get(name, {}) + remoteport = server_config.get('remoteport', []) child_host = transport.broker.transport.client[0] root_list = [rootport.get('rootname') for rootport in remoteport] - GlobalObject().remote_map[name] = {"host":child_host,"root_list":root_list} - #通知有需要连的node节点连接到此root节点 - for servername,remote_list in GlobalObject().remote_map.items(): - remote_host = remote_list.get("host","") - remote_name_host = remote_list.get("root_list","") + GlobalObject().remote_map[name] = { + "host": child_host, + "root_list": root_list} + # 通知有需要连的node节点连接到此root节点 + for servername, remote_list in GlobalObject().remote_map.items(): + remote_host = remote_list.get("host", "") + remote_name_host = remote_list.get("root_list", "") if name in remote_name_host: - GlobalObject().root.callChild(servername,"remote_connect",name,remote_host) - #查看当前是否有可供连接的root节点 + GlobalObject().root.callChild( + servername, + "remote_connect", + name, + remote_host) + # 查看当前是否有可供连接的root节点 master_node_list = GlobalObject().remote_map.keys() for root_name in root_list: if root_name in master_node_list: root_host = GlobalObject().remote_map[root_name]['host'] - GlobalObject().root.callChild(name,"remote_connect",root_name,root_host) - + GlobalObject().root.callChild( + name, + "remote_connect", + root_name, + root_host) + + def _doChildLostConnect(childId): """ """ try: del GlobalObject().remote_map[childId] - except Exception,e: + except Exception as e: log.msg(str(e)) GlobalObject().root.doChildConnect = _doChildConnect -GlobalObject().root.doChildLostConnect = _doChildLostConnect \ No newline at end of file +GlobalObject().root.doChildLostConnect = _doChildLostConnect diff --git a/firefly/master/webapp.py b/firefly/master/webapp.py index 973a0a3..17fa4b6 100644 --- a/firefly/master/webapp.py +++ b/firefly/master/webapp.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-7 @@ -9,31 +9,38 @@ from firefly.server.globalobject import GlobalObject root = GlobalObject().webroot reactor = reactor + + def ErrorBack(reason): pass + def masterwebHandle(cls): ''' ''' root.putChild(cls.__name__, cls()) + @masterwebHandle class stop(resource.Resource): + '''stop service''' - + def render(self, request): ''' ''' for child in GlobalObject().root.childsmanager._childs.values(): d = child.callbackChild('serverStop') d.addCallback(ErrorBack) - reactor.callLater(0.5,reactor.stop) + reactor.callLater(0.5, reactor.stop) return "stop" + @masterwebHandle class reloadmodule(resource.Resource): + '''reload module''' - + def render(self, request): ''' ''' @@ -41,7 +48,3 @@ def render(self, request): d = child.callbackChild('sreload') d.addCallback(ErrorBack) return "reload" - - - - diff --git a/firefly/netconnect/connection.py b/firefly/netconnect/connection.py index d27d93d..0771dbc 100644 --- a/firefly/netconnect/connection.py +++ b/firefly/netconnect/connection.py @@ -1,13 +1,16 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2010-12-31 @author: sean_lan ''' + class Connection: + ''' ''' + def __init__(self, _conn): ''' id 连接的ID @@ -15,15 +18,13 @@ def __init__(self, _conn): ''' self.id = _conn.transport.sessionno self.instance = _conn - + def loseConnection(self): '''断开与客户端的连接 ''' self.instance.transport.loseConnection() - - def safeToWriteData(self,topicID,msg): + + def safeToWriteData(self, topicID, msg): """发送消息 """ - self.instance.safeToWriteData(msg,topicID) - - + self.instance.safeToWriteData(msg, topicID) diff --git a/firefly/netconnect/datapack.py b/firefly/netconnect/datapack.py index b9b0d0a..382792a 100644 --- a/firefly/netconnect/datapack.py +++ b/firefly/netconnect/datapack.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-1 @@ -7,7 +7,9 @@ from twisted.python import log import struct + class DataPackError(Exception): + """An error occurred binding to an interface""" def __str__(self): @@ -17,10 +19,14 @@ def __str__(self): s = '%s.' % s return s + class DataPackProtoc: + """数据包协议 """ - def __init__(self,HEAD_0 = 0,HEAD_1=0,HEAD_2=0,HEAD_3=0,protoVersion= 0,serverVersion=0): + + def __init__(self, HEAD_0=0, HEAD_1=0, HEAD_2=0, + HEAD_3=0, protoVersion=0, serverVersion=0): '''初始化 @param HEAD_0: int 协议头0 @param HEAD_1: int 协议头1 @@ -35,53 +41,53 @@ def __init__(self,HEAD_0 = 0,HEAD_1=0,HEAD_2=0,HEAD_3=0,protoVersion= 0,serverVe self.HEAD_3 = HEAD_3 self.protoVersion = protoVersion self.serverVersion = serverVersion - + def setHEAD_0(self, HEAD_0): self.HEAD_0 = HEAD_0 - + def setHEAD_1(self, HEAD_1): self.HEAD_1 = HEAD_1 - + def setHEAD_2(self, HEAD_2): self.HEAD_2 = HEAD_2 - + def setHEAD_3(self, HEAD_3): self.HEAD_3 = HEAD_3 - + def setprotoVersion(self, protoVersion): self.protoVersion = protoVersion - + def setserverVersion(self, serverVersion): self.serverVersion = serverVersion - + def getHeadlength(self): """获取数据包的长度 """ return 17 - - def unpack(self,dpack): + + def unpack(self, dpack): '''解包 ''' try: - ud = struct.unpack('!sssss3I',dpack) - except DataPackError,de: + ud = struct.unpack('!sssss3I', dpack) + except DataPackError as de: log.err(de) - return {'result':False,'command':0,'length':0} + return {'result': False, 'command': 0, 'length': 0} HEAD_0 = ord(ud[0]) HEAD_1 = ord(ud[1]) HEAD_2 = ord(ud[2]) HEAD_3 = ord(ud[3]) protoVersion = ord(ud[4]) serverVersion = ud[5] - length = ud[6]-4 + length = ud[6] - 4 command = ud[7] - if HEAD_0 <>self.HEAD_0 or HEAD_1<>self.HEAD_1 or\ - HEAD_2<>self.HEAD_2 or HEAD_3<>self.HEAD_3 or\ - protoVersion<>self.protoVersion or serverVersion<>self.serverVersion: - return {'result':False,'command':0,'length':0} - return {'result':True,'command':command,'length':length} - - def pack(self,response,command): + if HEAD_0 != self.HEAD_0 or HEAD_1 != self.HEAD_1 or\ + HEAD_2 != self.HEAD_2 or HEAD_3 != self.HEAD_3 or\ + protoVersion != self.protoVersion or serverVersion != self.serverVersion: + return {'result': False, 'command': 0, 'length': 0} + return {'result': True, 'command': command, 'length': length} + + def pack(self, response, command): '''打包数据包 ''' HEAD_0 = chr(self.HEAD_0) @@ -90,12 +96,9 @@ def pack(self,response,command): HEAD_3 = chr(self.HEAD_3) protoVersion = chr(self.protoVersion) serverVersion = self.serverVersion - length = response.__len__()+4 + length = response.__len__() + 4 commandID = command - data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,HEAD_3,\ - protoVersion,serverVersion,length,commandID) + data = struct.pack('!sssss3I', HEAD_0, HEAD_1, HEAD_2, HEAD_3, + protoVersion, serverVersion, length, commandID) data = data + response return data - - - \ No newline at end of file diff --git a/firefly/netconnect/manager.py b/firefly/netconnect/manager.py index 0d4bf80..74b0717 100644 --- a/firefly/netconnect/manager.py +++ b/firefly/netconnect/manager.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2010-12-31 连接管理器 @@ -8,30 +8,32 @@ from twisted.python import log from connection import Connection + class ConnectionManager: + ''' 连接管理器 @param _connections: dict {connID:conn Object}管理的所有连接 ''' - + def __init__(self): '''初始化 @param _connections: dict {connID:conn Object} ''' self._connections = {} - + def getNowConnCnt(self): '''获取当前连接数量''' return len(self._connections.items()) - + def addConnection(self, conn): '''加入一条连接 @param _conn: Conn object ''' _conn = Connection(conn) - if self._connections.has_key(_conn.id): + if _conn.id in self._connections: raise Exception("系统记录冲突") self._connections[_conn.id] = _conn - + def dropConnectionByID(self, connID): '''更加连接的id删除连接实例 @param connID: int 连接的id @@ -40,29 +42,27 @@ def dropConnectionByID(self, connID): del self._connections[connID] except Exception as e: log.msg(str(e)) - + def getConnectionByID(self, connID): """根据ID获取一条连接 @param connID: int 连接的id """ - return self._connections.get(connID,None) - - def loseConnection(self,connID): + return self._connections.get(connID, None) + + def loseConnection(self, connID): """根据连接ID主动端口与客户端的连接 """ conn = self.getConnectionByID(connID) if conn: conn.loseConnection() - - def pushObject(self,topicID , msg, sendList): + + def pushObject(self, topicID, msg, sendList): """主动推送消息 """ for target in sendList: try: conn = self.getConnectionByID(target) if conn: - conn.safeToWriteData(topicID,msg) - except Exception,e: + conn.safeToWriteData(topicID, msg) + except Exception as e: log.err(str(e)) - - diff --git a/firefly/netconnect/protoc.py b/firefly/netconnect/protoc.py index 4ad3ecd..57babdd 100644 --- a/firefly/netconnect/protoc.py +++ b/firefly/netconnect/protoc.py @@ -1,135 +1,139 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-9-20 登陆服务器协议 @author: lan (www.9miao.com) ''' -from twisted.internet import protocol,reactor +from twisted.internet import protocol, reactor from twisted.python import log from manager import ConnectionManager from datapack import DataPackProtoc reactor = reactor + def DefferedErrorHandle(e): '''延迟对象的错误处理''' log.err(str(e)) return + class LiberateProtocol(protocol.Protocol): + '''协议''' - + buff = "" - + def connectionMade(self): '''连接建立处理 ''' - log.msg('Client %d login in.[%s,%d]'%(self.transport.sessionno,\ - self.transport.client[0],self.transport.client[1])) + log.msg('Client %d login in.[%s,%d]' % (self.transport.sessionno, + self.transport.client[0], self.transport.client[1])) self.factory.connmanager.addConnection(self) self.factory.doConnectionMade(self) - self.datahandler=self.dataHandleCoroutine() + self.datahandler = self.dataHandleCoroutine() self.datahandler.next() - - def connectionLost(self,reason): + + def connectionLost(self, reason): '''连接断开处理 ''' - log.msg('Client %d login out.'%(self.transport.sessionno)) + log.msg('Client %d login out.' % (self.transport.sessionno)) self.factory.doConnectionLost(self) self.factory.connmanager.dropConnectionByID(self.transport.sessionno) - - def safeToWriteData(self,data,command): + + def safeToWriteData(self, data, command): '''线程安全的向客户端发送数据 @param data: str 要向客户端写的数据 ''' if not self.transport.connected or data is None: return - senddata = self.factory.produceResult(data,command) - reactor.callFromThread(self.transport.write,senddata) - + senddata = self.factory.produceResult(data, command) + reactor.callFromThread(self.transport.write, senddata) + def dataHandleCoroutine(self): """ """ - length = self.factory.dataprotocl.getHeadlength()#获取协议头的长度 + length = self.factory.dataprotocl.getHeadlength() # 获取协议头的长度 while True: data = yield self.buff += data - while self.buff.__len__() >= length: - unpackdata = self.factory.dataprotocl.unpack(self.buff[:length]) + while self.buff.__len__() >= length: + unpackdata = self.factory.dataprotocl.unpack( + self.buff[ + :length]) if not unpackdata.get('result'): log.msg('illegal data package --') self.transport.loseConnection() break command = unpackdata.get('command') rlength = unpackdata.get('length') - request = self.buff[length:length+rlength] - if request.__len__()< rlength: + request = self.buff[length:length + rlength] + if request.__len__() < rlength: log.msg('some data lose') break - self.buff = self.buff[length+rlength:] - d = self.factory.doDataReceived(self,command,request) + self.buff = self.buff[length + rlength:] + d = self.factory.doDataReceived(self, command, request) if not d: continue - d.addCallback(self.safeToWriteData,command) + d.addCallback(self.safeToWriteData, command) d.addErrback(DefferedErrorHandle) - - def dataReceived(self, data): '''数据到达处理 @param data: str 客户端传送过来的数据 ''' self.datahandler.send(data) - + + class LiberateFactory(protocol.ServerFactory): + '''协议工厂''' - + protocol = LiberateProtocol - - def __init__(self,dataprotocl=DataPackProtoc()): + + def __init__(self, dataprotocl=DataPackProtoc()): '''初始化 ''' self.service = None self.connmanager = ConnectionManager() self.dataprotocl = dataprotocl - - def setDataProtocl(self,dataprotocl): + + def setDataProtocl(self, dataprotocl): ''' ''' self.dataprotocl = dataprotocl - - def doConnectionMade(self,conn): + + def doConnectionMade(self, conn): '''当连接建立时的处理''' pass - - def doConnectionLost(self,conn): + + def doConnectionLost(self, conn): '''连接断开时的处理''' pass - - def addServiceChannel(self,service): + + def addServiceChannel(self, service): '''添加服务通道''' self.service = service - - def doDataReceived(self,conn,commandID,data): + + def doDataReceived(self, conn, commandID, data): '''数据到达时的处理''' - defer_tool = self.service.callTarget(commandID,conn,data) + defer_tool = self.service.callTarget(commandID, conn, data) return defer_tool - - def produceResult(self,command,response): + + def produceResult(self, command, response): '''产生客户端需要的最终结果 @param response: str 分布式客户端获取的结果 ''' - return self.dataprotocl.pack(command,response) - - def loseConnection(self,connID): + return self.dataprotocl.pack(command, response) + + def loseConnection(self, connID): """主动端口与客户端的连接 """ self.connmanager.loseConnection(connID) - - def pushObject(self,topicID , msg, sendList): + + def pushObject(self, topicID, msg, sendList): '''服务端向客户端推消息 @param topicID: int 消息的主题id号 @param msg: 消息的类容，protobuf结构类型 @param sendList: 推向的目标列表(客户端id 列表) ''' self.connmanager.pushObject(topicID, msg, sendList) - diff --git a/firefly/script/firefly-admin.py b/firefly/script/firefly-admin.py index 2ce4576..ca563cb 100644 --- a/firefly/script/firefly-admin.py +++ b/firefly/script/firefly-admin.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-8 @@ -9,4 +9,4 @@ if __name__ == "__main__": args = sys.argv - management.execute_commands(*args) \ No newline at end of file + management.execute_commands(*args) diff --git a/firefly/server/admin.py b/firefly/server/admin.py index 5cb8188..d7aefd7 100644 --- a/firefly/server/admin.py +++ b/firefly/server/admin.py @@ -1,10 +1,10 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-12 @author: lan (www.9miao.com) ''' -from globalobject import GlobalObject,masterserviceHandle +from globalobject import GlobalObject, masterserviceHandle from twisted.internet import reactor from twisted.python import log @@ -18,9 +18,10 @@ def serverStop(): log.msg('stop') if GlobalObject().stophandler: GlobalObject().stophandler() - reactor.callLater(0.5,reactor.stop) + reactor.callLater(0.5, reactor.stop) return True + @masterserviceHandle def sreload(): """ @@ -30,9 +31,9 @@ def sreload(): reload(GlobalObject().reloadmodule) return True + @masterserviceHandle def remote_connect(rname, rhost): """ """ GlobalObject().remote_connect(rname, rhost) - diff --git a/firefly/server/globalobject.py b/firefly/server/globalobject.py index f4c5e98..b805da9 100644 --- a/firefly/server/globalobject.py +++ b/firefly/server/globalobject.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @@ -6,14 +6,15 @@ ''' from firefly.utils.singleton import Singleton + class GlobalObject: - + __metaclass__ = Singleton - + def __init__(self): - self.netfactory = None#net前端 - self.root = None#分布式root节点 - self.remote = {}#remote节点 + self.netfactory = None # net前端 + self.root = None # 分布式root节点 + self.remote = {} # remote节点 self.db = None self.stophandler = None self.webroot = None @@ -22,39 +23,44 @@ def __init__(self): self.remote_connect = None self.json_config = {} self.remote_map = {} - - def config(self,netfactory=None,root = None,remote=None,db=None): + + def config(self, netfactory=None, root=None, remote=None, db=None): self.netfactory = netfactory self.root = root self.remote = remote self.db = db - + + def masterserviceHandle(target): """ """ GlobalObject().masterremote._reference._service.mapTarget(target) - + + def netserviceHandle(target): """ """ GlobalObject().netfactory.service.mapTarget(target) - + + def rootserviceHandle(target): """ """ GlobalObject().root.service.mapTarget(target) - + + class webserviceHandle: + """这是一个修饰符对象 """ - - def __init__(self,url=None): + + def __init__(self, url=None): """ @param url: str http 访问的路径 """ self._url = url - - def __call__(self,cls): + + def __call__(self, cls): """ """ from twisted.web.resource import Resource @@ -66,10 +72,10 @@ def __call__(self,cls): temp_res = None path_list = [path for path in path_list if path] patn_len = len(path_list) - for index,path in enumerate(path_list): - if index==0: + for index, path in enumerate(path_list): + if index == 0: temp_res = GlobalObject().webroot - if index==patn_len-1: + if index == patn_len - 1: res = cls() temp_res.putChild(path, res) return @@ -78,20 +84,21 @@ def __call__(self,cls): if not res: res = Resource() temp_res.putChild(path, res) - temp_res=res - + temp_res = res + - class remoteserviceHandle: + """ """ - def __init__(self,remotename): + + def __init__(self, remotename): """ """ self.remotename = remotename - - def __call__(self,target): + + def __call__(self, target): """ """ - GlobalObject().remote[self.remotename]._reference._service.mapTarget(target) - + GlobalObject().remote[ + self.remotename]._reference._service.mapTarget(target) diff --git a/firefly/server/logobj.py b/firefly/server/logobj.py index b0c9459..f3e6513 100644 --- a/firefly/server/logobj.py +++ b/firefly/server/logobj.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-6 @@ -10,15 +10,16 @@ class loogoo: + '''日志处理 ''' implements(log.ILogObserver) - - def __init__(self,logpath): + + def __init__(self, logpath): '''配置日志路径 ''' self.file = file(logpath, 'w') - + def __call__(self, eventDict): '''日志处理 ''' @@ -32,6 +33,12 @@ def __call__(self, eventDict): if text is None or level != 'ERROR': return nowdate = datetime.datetime.now() - self.file.write('['+str(nowdate)+']\n'+str(level)+ '\n\t' + text + '\r\n') + self.file.write( + '[' + + str(nowdate) + + ']\n' + + str(level) + + '\n\t' + + text + + '\r\n') self.file.flush() - diff --git a/firefly/server/server.py b/firefly/server/server.py index 8548848..cb6e557 100644 --- a/firefly/server/server.py +++ b/firefly/server/server.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @@ -7,7 +7,7 @@ from firefly.netconnect.protoc import LiberateFactory from twisted.web import vhost from firefly.web.delayrequest import DelaySite -from firefly.distributed.root import PBRoot,BilateralFactory +from firefly.distributed.root import PBRoot, BilateralFactory from firefly.distributed.node import RemoteObject from firefly.dbentrust.dbpool import dbpool from firefly.dbentrust.memclient import mclient @@ -16,104 +16,115 @@ from twisted.python import log from twisted.internet import reactor from firefly.utils import services -import os,sys,affinity +import os +import sys +import affinity reactor = reactor + def serverStop(): log.msg('stop') if GlobalObject().stophandler: GlobalObject().stophandler() - reactor.callLater(0.5,reactor.stop) + reactor.callLater(0.5, reactor.stop) return True + class FFServer: - + def __init__(self): ''' ''' - self.netfactory = None#net前端 - self.root = None#分布式root节点 - self.webroot = None#http服务 - self.remote = {}#remote节点 + self.netfactory = None # net前端 + self.root = None # 分布式root节点 + self.webroot = None # http服务 + self.remote = {} # remote节点 self.master_remote = None self.db = None self.mem = None self.servername = None self.remoteportlist = [] - + def config(self, config, servername=None, dbconfig=None, - memconfig=None, masterconf=None): + memconfig=None, masterconf=None): '''配置服务器 ''' GlobalObject().json_config = config - netport = config.get('netport')#客户端连接 - webport = config.get('webport')#http连接 - rootport = config.get('rootport')#root节点配置 - self.remoteportlist = config.get('remoteport',[])#remote节点配置列表 + netport = config.get('netport') # 客户端连接 + webport = config.get('webport') # http连接 + rootport = config.get('rootport') # root节点配置 + self.remoteportlist = config.get('remoteport', []) # remote节点配置列表 if not servername: - servername = config.get('name')#服务器名称 - logpath = config.get('log')#日志 - hasdb = config.get('db')#数据库连接 - hasmem = config.get('mem')#memcached连接 - app = config.get('app')#入口模块名称 - cpuid = config.get('cpu')#绑定cpu - mreload = config.get('reload')#重新加载模块名称 + servername = config.get('name') # 服务器名称 + logpath = config.get('log') # 日志 + hasdb = config.get('db') # 数据库连接 + hasmem = config.get('mem') # memcached连接 + app = config.get('app') # 入口模块名称 + cpuid = config.get('cpu') # 绑定cpu + mreload = config.get('reload') # 重新加载模块名称 self.servername = servername - + if netport: self.netfactory = LiberateFactory() netservice = services.CommandService("netservice") self.netfactory.addServiceChannel(netservice) - reactor.listenTCP(netport,self.netfactory) - + reactor.listenTCP(netport, self.netfactory) + if webport: self.webroot = vhost.NameVirtualHost() GlobalObject().webroot = self.webroot reactor.listenTCP(webport, DelaySite(self.webroot)) - + if rootport: self.root = PBRoot() rootservice = services.Service("rootservice") self.root.addServiceChannel(rootservice) reactor.listenTCP(rootport, BilateralFactory(self.root)) - + for cnf in self.remoteportlist: rname = cnf.get('rootname') self.remote[rname] = RemoteObject(self.servername) - + if hasdb and dbconfig: log.msg(str(dbconfig)) dbpool.initPool(**dbconfig) - + if hasmem and memconfig: urls = memconfig.get('urls') hostname = str(memconfig.get('hostname')) mclient.connect(urls, hostname) - + if logpath: - log.addObserver(loogoo(logpath))#日志处理 + log.addObserver(loogoo(logpath)) # 日志处理 log.startLogging(sys.stdout) - + if cpuid: affinity.set_process_affinity_mask(os.getpid(), cpuid) - GlobalObject().config(netfactory = self.netfactory, root=self.root, - remote = self.remote) + GlobalObject().config(netfactory=self.netfactory, root=self.root, + remote=self.remote) if app: __import__(app) if mreload: _path_list = mreload.split(".") - GlobalObject().reloadmodule = __import__(mreload,fromlist=_path_list[:1]) + GlobalObject().reloadmodule = __import__( + mreload, + fromlist=_path_list[ + :1]) GlobalObject().remote_connect = self.remote_connect if masterconf: masterport = masterconf.get('rootport') masterhost = masterconf.get('roothost') self.master_remote = RemoteObject(servername) - addr = ('localhost',masterport) if not masterhost else (masterhost,masterport) + addr = ( + 'localhost', + masterport) if not masterhost else ( + masterhost, + masterport) self.master_remote.connect(addr) GlobalObject().masterremote = self.master_remote import admin - + def remote_connect(self, rname, rhost): """ """ @@ -122,17 +133,15 @@ def remote_connect(self, rname, rhost): if rname == _rname: rport = cnf.get('rootport') if not rhost: - addr = ('localhost',rport) + addr = ('localhost', rport) else: - addr = (rhost,rport) + addr = (rhost, rport) self.remote[rname].connect(addr) break - + def start(self): '''启动服务器 ''' - log.msg('%s start...'%self.servername) - log.msg('%s pid: %s'%(self.servername,os.getpid())) + log.msg('%s start...' % self.servername) + log.msg('%s pid: %s' % (self.servername, os.getpid())) reactor.run() - - diff --git a/firefly/test/test_dbentrust.py b/firefly/test/test_dbentrust.py index f2ac6c1..ab077ee 100644 --- a/firefly/test/test_dbentrust.py +++ b/firefly/test/test_dbentrust.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-7-31 @@ -6,40 +6,38 @@ ''' from firefly.dbentrust.dbpool import dbpool from firefly.dbentrust.madminanager import MAdminManager -from firefly.dbentrust import mmode +from firefly.dbentrust import mmode from firefly.dbentrust.memclient import mclient import time -if __name__=="__main__": +if __name__ == "__main__": - -# CREATE TABLE `tb_register` ( -# `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'id', -# `username` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '用户名', -# `password` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '用户密码', -# PRIMARY KEY (`id`,`username`) -# ) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 -# + # CREATE TABLE `tb_register` ( + # `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'id', + # `username` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '用户名', + # `password` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '用户密码', + # PRIMARY KEY (`id`,`username`) + # ) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 + # hostname = "localhost" username = 'root' password = '111' dbname = 'test' charset = 'utf8' - tablename = "test1"# - aa = {'host':"localhost", - 'user':'root', - 'passwd':'111', - 'db':'test', - 'port':3306, - 'charset':'utf8'} + tablename = "test1" + aa = {'host': "localhost", + 'user': 'root', + 'passwd': '111', + 'db': 'test', + 'port': 3306, + 'charset': 'utf8'} dbpool.initPool(**aa) mclient.connect(['127.0.0.1:11211'], "test") mmanager = MAdminManager() - m1 = mmode.MAdmin( 'test1', 'id', incrkey='id') + m1 = mmode.MAdmin('test1', 'id', incrkey='id') m1.insert() print m1.get('_incrvalue') - m2 = mmode.MAdmin( 'test1', 'id', incrkey='id') + m2 = mmode.MAdmin('test1', 'id', incrkey='id') print m2.get('_incrvalue') - diff --git a/firefly/test/test_distributed_node.py b/firefly/test/test_distributed_node.py index ba300e3..4f0d160 100644 --- a/firefly/test/test_distributed_node.py +++ b/firefly/test/test_distributed_node.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-17 @@ -7,17 +7,19 @@ from firefly.utils import services from firefly.distributed.node import RemoteObject from twisted.internet import reactor -from twisted.python import util,log +from twisted.python import util, log import sys log.startLogging(sys.stdout) reactor = reactor -addr = ('localhost',1000)#目标主机的地址 -remote = RemoteObject('test_node')#实例化远程调用对象 +addr = ('localhost', 1000) # 目标主机的地址 +remote = RemoteObject('test_node') # 实例化远程调用对象 -service = services.Service('reference',services.Service.SINGLE_STYLE)#实例化一条服务对象 +service = services.Service( + 'reference', + services.Service.SINGLE_STYLE) # 实例化一条服务对象 remote.addServiceChannel(service) @@ -27,22 +29,24 @@ def serviceHandle(target): ''' service.mapTarget(target) + @serviceHandle def printOK(data): print data print "############################" return "call printOK_01" - -def apptest(commandID,*args,**kw): - d = remote.callRemote(commandID,*args,**kw) - d.addCallback(lambda a:util.println(a)) + + +def apptest(commandID, *args, **kw): + d = remote.callRemote(commandID, *args, **kw) + d.addCallback(lambda a: util.println(a)) return d + def startClient(): - reactor.callLater(1,apptest,'printData1',u"node测试1",u"node测试2") - remote.connect(addr)#连接远程主机 + reactor.callLater(1, apptest, 'printData1', u"node测试1", u"node测试2") + remote.connect(addr) # 连接远程主机 reactor.run() -if __name__=='__main__': +if __name__ == '__main__': startClient() - diff --git a/firefly/test/test_distributed_root.py b/firefly/test/test_distributed_root.py index 0663cfc..9418687 100644 --- a/firefly/test/test_distributed_root.py +++ b/firefly/test/test_distributed_root.py @@ -1,17 +1,17 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-17 @author: lan (www.9miao.com) ''' -from firefly.utils import services -from firefly.distributed.root import PBRoot,BilateralFactory +from firefly.utils import services +from firefly.distributed.root import PBRoot, BilateralFactory from twisted.internet import reactor from twisted.python import log import sys reactor = reactor log.startLogging(sys.stdout) - + root = PBRoot() ser = services.Service('test') root.addServiceChannel(ser) @@ -23,21 +23,28 @@ def serviceHandle(target): ''' ser.mapTarget(target) + @serviceHandle -def printData1(data,data1): - print data,data1 +def printData1(data, data1): + print data, data1 print "############################" # d = root.callChildByName("test_node",1,u'Root测试') return data + @serviceHandle -def printData2(data,data1): - print data,data1 +def printData2(data, data1): + print data, data1 print "############################" # d = root.callChildByName("test_node",1,u'Root测试') return data -if __name__=='__main__': +if __name__ == '__main__': reactor.listenTCP(1000, BilateralFactory(root)) - reactor.callLater(5,root.callChildByName,'test_node','printOK','asdfawefasdf') + reactor.callLater( + 5, + root.callChildByName, + 'test_node', + 'printOK', + 'asdfawefasdf') reactor.run() diff --git a/firefly/test/test_netconnect_client.py b/firefly/test/test_netconnect_client.py index 514575b..82b661c 100644 --- a/firefly/test/test_netconnect_client.py +++ b/firefly/test/test_netconnect_client.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-12 @@ -6,17 +6,18 @@ ''' import time -from socket import AF_INET,SOCK_STREAM,socket +from socket import AF_INET, SOCK_STREAM, socket from thread import start_new import struct -HOST='localhost' -PORT=1000 -BUFSIZE=1024 -ADDR=(HOST , PORT) -client = socket(AF_INET,SOCK_STREAM) +HOST = 'localhost' +PORT = 1000 +BUFSIZE = 1024 +ADDR = (HOST, PORT) +client = socket(AF_INET, SOCK_STREAM) client.connect(ADDR) -def sendData(sendstr,commandId): + +def sendData(sendstr, commandId): HEAD_0 = chr(0) HEAD_1 = chr(0) HEAD_2 = chr(0) @@ -24,26 +25,27 @@ def sendData(sendstr,commandId): ProtoVersion = chr(0) ServerVersion = 0 sendstr = sendstr - data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,\ - HEAD_3,ProtoVersion,ServerVersion,\ - len(sendstr)+4,commandId) - senddata = data+sendstr + data = struct.pack('!sssss3I', HEAD_0, HEAD_1, HEAD_2, + HEAD_3, ProtoVersion, ServerVersion, + len(sendstr) + 4, commandId) + senddata = data + sendstr return senddata + def resolveRecvdata(data): - head = struct.unpack('!sssss3I',data[:17]) + head = struct.unpack('!sssss3I', data[:17]) length = head[6] - data = data[17:17+length] + data = data[17:17 + length] return data s1 = time.time() + def start(): for i in xrange(10): - client.sendall(sendData('asdfe',1)) + client.sendall(sendData('asdfe', 1)) for i in range(10): - start_new(start,()) + start_new(start, ()) while True: pass - diff --git a/firefly/test/test_netconnect_server.py b/firefly/test/test_netconnect_server.py index b192877..6d95ff3 100644 --- a/firefly/test/test_netconnect_server.py +++ b/firefly/test/test_netconnect_server.py @@ -1,53 +1,59 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-3 @author: lan (www.9miao.com) ''' -import sys,os +import sys +import os -if os.name!='nt':#对系统的类型的判断，如果不是NT系统的话使用epoll +if os.name != 'nt': # 对系统的类型的判断，如果不是NT系统的话使用epoll from twisted.internet import epollreactor epollreactor.install() - + from twisted.internet import reactor from twisted.python import log from firefly.utils import services from firefly.netconnect.protoc import LiberateFactory reactor = reactor -service = services.CommandService("loginService",runstyle= services.Service.PARALLEL_STYLE) +service = services.CommandService( + "loginService", + runstyle=services.Service.PARALLEL_STYLE) + def serviceHandle(target): '''服务处理 @param target: func Object ''' service.mapTarget(target) - + factory = LiberateFactory() + def doConnectionLost(conn): print conn.transport factory.doConnectionLost = doConnectionLost + def serverstart(): '''服务配置 ''' log.startLogging(sys.stdout) factory.addServiceChannel(service) - reactor.callLater(10,factory.pushObject,111,'asdfe',[0]) - reactor.callLater(15,factory.loseConnection,0) - reactor.listenTCP(1000,factory) + reactor.callLater(10, factory.pushObject, 111, 'asdfe', [0]) + reactor.callLater(15, factory.loseConnection, 0) + reactor.listenTCP(1000, factory) reactor.run() - + + @serviceHandle -def echo_1(_conn,data): +def echo_1(_conn, data): addr = _conn.transport.client print addr return "欢迎" if __name__ == "__main__": - - serverstart() + serverstart() diff --git a/firefly/test/test_server_apptest.py b/firefly/test/test_server_apptest.py index 87bd387..46b08b5 100644 --- a/firefly/test/test_server_apptest.py +++ b/firefly/test/test_server_apptest.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @@ -6,15 +6,14 @@ ''' from server.globalobject import GlobalObject + def netserviceHandle(target): '''服务处理 @param target: func Object ''' GlobalObject().netfactory.service.mapTarget(target) - -@netserviceHandle -def echo_111(_conn,data): - return data - +@netserviceHandle +def echo_111(_conn, data): + return data diff --git a/firefly/test/test_server_main.py b/firefly/test/test_server_main.py index 1b41e89..60aa236 100644 --- a/firefly/test/test_server_main.py +++ b/firefly/test/test_server_main.py @@ -1,28 +1,31 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-6 @author: lan (www.9miao.com) ''' -import json,sys +import json +import sys from firefly.server.server import FFServer -if __name__=="__main__": +if __name__ == "__main__": args = sys.argv servername = None config = None - if len(args)>2: + if len(args) > 2: servername = args[1] - config = json.load(open(args[2],'r')) + config = json.load(open(args[2], 'r')) else: raise ValueError dbconf = config.get('db') memconf = config.get('memcached') - sersconf = config.get('servers',{}) - masterconf = config.get('master',{}) + sersconf = config.get('servers', {}) + masterconf = config.get('master', {}) serconfig = sersconf.get(servername) ser = FFServer() - ser.config(serconfig, dbconfig=dbconf, memconfig=memconf,masterconf=masterconf) + ser.config( + serconfig, + dbconfig=dbconf, + memconfig=memconf, + masterconf=masterconf) ser.start() - - \ No newline at end of file diff --git a/firefly/test/test_server_master.py b/firefly/test/test_server_master.py index 873d24c..d385f6d 100644 --- a/firefly/test/test_server_master.py +++ b/firefly/test/test_server_master.py @@ -1,21 +1,20 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @author: lan (www.9miao.com) ''' import os -if os.name!='nt': +if os.name != 'nt': from twisted.internet import epollreactor epollreactor.install() - + + def println(a): print a -if __name__=="__main__": +if __name__ == "__main__": from firefly.master.master import Master master = Master() - master.config('config.json','test_server_main.py') + master.config('config.json', 'test_server_main.py') master.start() - - \ No newline at end of file diff --git a/firefly/utils/interfaces.py b/firefly/utils/interfaces.py index eb05a1d..a305a42 100644 --- a/firefly/utils/interfaces.py +++ b/firefly/utils/interfaces.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-10-17 @@ -9,20 +9,16 @@ class IDataPackProtoc(Interface): - + def getHeadlength(): """获取数据包的长度 """ pass - + def unpack(): '''解包 ''' - + def pack(): '''打包数据包 ''' - - - - diff --git a/firefly/utils/services.py b/firefly/utils/services.py index 4d97eb2..0982e85 100644 --- a/firefly/utils/services.py +++ b/firefly/utils/services.py @@ -1,36 +1,37 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-1-3 服务类 @author: sean_lan ''' import threading -from twisted.internet import defer,threads +from twisted.internet import defer, threads from twisted.python import log class Service(object): - """A remoting service - + + """A remoting service + attributes: ============ * name - string, service name. - * runstyle + * runstyle """ SINGLE_STYLE = 1 PARALLEL_STYLE = 2 - def __init__(self, name,runstyle = SINGLE_STYLE): + def __init__(self, name, runstyle=SINGLE_STYLE): self._name = name self._runstyle = runstyle self.unDisplay = set() self._lock = threading.RLock() - self._targets = {} # Keeps track of targets internally + self._targets = {} # Keeps track of targets internally def __iter__(self): return self._targets.itervalues() - - def addUnDisplayTarget(self,command): + + def addUnDisplayTarget(self, command): '''Add a target unDisplay when client call it.''' self.unDisplay.add(command) @@ -39,10 +40,10 @@ def mapTarget(self, target): self._lock.acquire() try: key = target.__name__ - if self._targets.has_key(key): + if key in self._targets: exist_target = self._targets.get(key) raise "target [%d] Already exists,\ - Conflict between the %s and %s"%(key,exist_target.__name__,target.__name__) + Conflict between the %s and %s" % (key, exist_target.__name__, target.__name__) self._targets[key] = target finally: self._lock.release() @@ -56,15 +57,15 @@ def unMapTarget(self, target): del self._targets[key] finally: self._lock.release() - - def unMapTargetByKey(self,targetKey): + + def unMapTargetByKey(self, targetKey): """Remove a target from the service.""" self._lock.acquire() try: del self._targets[targetKey] finally: self._lock.release() - + def getTarget(self, targetKey): """Get a target from the service by name.""" self._lock.acquire() @@ -73,46 +74,49 @@ def getTarget(self, targetKey): finally: self._lock.release() return target - - def callTarget(self,targetKey,*args,**kw): + + def callTarget(self, targetKey, *args, **kw): '''call Target @param conn: client connection @param targetKey: target ID @param data: client data ''' if self._runstyle == self.SINGLE_STYLE: - result = self.callTargetSingle(targetKey,*args,**kw) + result = self.callTargetSingle(targetKey, *args, **kw) else: - result = self.callTargetParallel(targetKey,*args,**kw) + result = self.callTargetParallel(targetKey, *args, **kw) return result - - def callTargetSingle(self,targetKey,*args,**kw): + + def callTargetSingle(self, targetKey, *args, **kw): '''call Target by Single @param conn: client connection @param targetKey: target ID @param data: client data ''' target = self.getTarget(targetKey) - + self._lock.acquire() try: if not target: - log.err('the command '+str(targetKey)+' not Found on service') + log.err( + 'the command ' + + str(targetKey) + + ' not Found on service') return None if targetKey not in self.unDisplay: - log.msg("call method %s on service[single]"%target.__name__) - defer_data = target(*args,**kw) + log.msg("call method %s on service[single]" % target.__name__) + defer_data = target(*args, **kw) if not defer_data: return None - if isinstance(defer_data,defer.Deferred): + if isinstance(defer_data, defer.Deferred): return defer_data d = defer.Deferred() d.callback(defer_data) finally: self._lock.release() return d - - def callTargetParallel(self,targetKey,*args,**kw): + + def callTargetParallel(self, targetKey, *args, **kw): '''call Target by Single @param conn: client connection @param targetKey: target ID @@ -122,32 +126,37 @@ def callTargetParallel(self,targetKey,*args,**kw): try: target = self.getTarget(targetKey) if not target: - log.err('the command '+str(targetKey)+' not Found on service') + log.err( + 'the command ' + + str(targetKey) + + ' not Found on service') return None - log.msg("call method %s on service[parallel]"%target.__name__) - d = threads.deferToThread(target,*args,**kw) + log.msg("call method %s on service[parallel]" % target.__name__) + d = threads.deferToThread(target, *args, **kw) finally: self._lock.release() return d - + class CommandService(Service): - """A remoting service + + """A remoting service According to Command ID search target """ + def mapTarget(self, target): """Add a target to the service.""" self._lock.acquire() try: key = int((target.__name__).split('_')[-1]) - if self._targets.has_key(key): + if key in self._targets: exist_target = self._targets.get(key) raise "target [%d] Already exists,\ - Conflict between the %s and %s"%(key,exist_target.__name__,target.__name__) + Conflict between the %s and %s" % (key, exist_target.__name__, target.__name__) self._targets[key] = target finally: self._lock.release() - + def unMapTarget(self, target): """Remove a target from the service.""" self._lock.acquire() @@ -157,9 +166,3 @@ def unMapTarget(self, target): del self._targets[key] finally: self._lock.release() - - - - - - \ No newline at end of file diff --git a/firefly/utils/singleton.py b/firefly/utils/singleton.py index 8da5e56..8964509 100644 --- a/firefly/utils/singleton.py +++ b/firefly/utils/singleton.py @@ -1,14 +1,15 @@ # coding: utf-8 + class Singleton(type): + """Singleton Metaclass""" - + def __init__(self, name, bases, dic): super(Singleton, self).__init__(name, bases, dic) self.instance = None - + def __call__(self, *args, **kwargs): if self.instance is None: self.instance = super(Singleton, self).__call__(*args, **kwargs) return self.instance - \ No newline at end of file diff --git a/firefly/web/delayrequest.py b/firefly/web/delayrequest.py index 4e65103..e96746b 100644 --- a/firefly/web/delayrequest.py +++ b/firefly/web/delayrequest.py @@ -1,18 +1,19 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-9-3 @author: lan ''' -from twisted.web.server import Request,Site +from twisted.web.server import Request, Site from twisted.internet import defer -from twisted.web import http,html +from twisted.web import http, html from twisted.python import log, reflect from twisted.web import resource from twisted.web.error import UnsupportedMethod from twisted.web.microdom import escape -import string,types +import string +import types NOT_DONE_YET = 1 @@ -25,10 +26,10 @@ class DelayRequest(Request): - + def __init__(self, *args, **kw): - Request.__init__(self,*args, **kw) - + Request.__init__(self, *args, **kw) + def render(self, resrc): """ Ask a resource to render itself. @@ -37,7 +38,7 @@ def render(self, resrc): """ try: body = resrc.render(self) - except UnsupportedMethod, e: + except UnsupportedMethod as e: allowedMethods = e.allowedMethods if (self.method == "HEAD") and ("GET" in allowedMethods): # We must support HEAD (RFC 2616, 5.1.1). If the @@ -70,11 +71,11 @@ def render(self, resrc): s = ('''Your browser approached me (at %(URI)s) with''' ''' the method "%(method)s". I only allow''' ''' the method%(plural)s %(allowed)s here.''' % { - 'URI': escape(self.uri), - 'method': self.method, - 'plural': ((len(allowedMethods) > 1) and 's') or '', - 'allowed': string.join(allowedMethods, ', ') - }) + 'URI': escape(self.uri), + 'method': self.method, + 'plural': ((len(allowedMethods) > 1) and 's') or '', + 'allowed': string.join(allowedMethods, ', ') + }) epage = resource.ErrorPage(http.NOT_ALLOWED, "Method Not Allowed", s) body = epage.render(self) @@ -88,7 +89,8 @@ def render(self, resrc): if body == NOT_DONE_YET: return - if not isinstance(body, defer.Deferred) and type(body) is not types.StringType: + if not isinstance(body, defer.Deferred) and not isinstance( + body, types.StringType): body = resource.ErrorPage( http.INTERNAL_SERVER_ERROR, "Request did not return a string", @@ -113,20 +115,17 @@ def render(self, resrc): self.setHeader('content-length', str(len(body))) self.write(body) self.finish() - - def _deferwrite(self,body): + + def _deferwrite(self, body): '''延迟等待数据返回 ''' self.setHeader('content-length', str(len(body))) self.write(body) self.finish() - - + + class DelaySite(Site): - - def __init__(self, resource, logPath=None, timeout=60*60*12): + + def __init__(self, resource, logPath=None, timeout=60 * 60 * 12): Site.__init__(self, resource, logPath=logPath, timeout=timeout) self.requestFactory = DelayRequest - - - \ No newline at end of file diff --git a/gfirefly/__init__.py b/gfirefly/__init__.py index 6f96d64..affcd0a 100644 --- a/gfirefly/__init__.py +++ b/gfirefly/__init__.py @@ -1,5 +1,6 @@ VERSION = (0, 1, 5, 'alpha', 0) + def get_version(*args, **kwargs): # Only import if it's actually called. from gfirefly.utils.version import get_version diff --git a/gfirefly/dbentrust/dbpool.py b/gfirefly/dbentrust/dbpool.py index 21119cc..11df5e0 100644 --- a/gfirefly/dbentrust/dbpool.py +++ b/gfirefly/dbentrust/dbpool.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-8 @@ -7,25 +7,26 @@ from DBUtils.PooledDB import PooledDB import MySQLdb -DBCS = {'mysql':MySQLdb,} +DBCS = {'mysql': MySQLdb, } + class DBPool(object): + '''数据库连接池 ''' - - def initPool(self,**kw): + + def initPool(self, **kw): '''根据连接配置初始化连接池配置信息. >>> aa = {'host':"localhost",'user':'root','passwd':'111','db':'test','port':3306,'charset':'utf8'} >>> dbpool.initPool(**aa) ''' self.config = kw - creator = DBCS.get(kw.get('engine','mysql'),MySQLdb) - self.pool = PooledDB(creator,5,**kw) - + creator = DBCS.get(kw.get('engine', 'mysql'), MySQLdb) + self.pool = PooledDB(creator, 5, **kw) + def connection(self): return self.pool.connection() -#数据库连接池对象 +# 数据库连接池对象 dbpool = DBPool() - diff --git a/gfirefly/dbentrust/dbutils.py b/gfirefly/dbentrust/dbutils.py index 8a4c6c7..c3ca620 100644 --- a/gfirefly/dbentrust/dbutils.py +++ b/gfirefly/dbentrust/dbutils.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-21 @@ -11,7 +11,7 @@ def safeunicode(obj, encoding='utf-8'): r""" Converts any given object to unicode string. - + >>> safeunicode('hello') u'hello' >>> safeunicode(2) @@ -30,11 +30,12 @@ def safeunicode(obj, encoding='utf-8'): return unicode(obj) else: return str(obj).decode(encoding) - + + def safestr(obj, encoding='utf-8'): r""" - Converts any given object to utf-8 encoded string. - + Converts any given object to utf-8 encoded string. + >>> safestr('hello') 'hello' >>> safestr(u'\u1234') @@ -46,12 +47,13 @@ def safestr(obj, encoding='utf-8'): return obj.encode(encoding) elif isinstance(obj, str): return obj - elif hasattr(obj, 'next'): # iterator + elif hasattr(obj, 'next'): # iterator return itertools.imap(safestr, obj) else: return str(obj) - -def sqlify(obj): + + +def sqlify(obj): """ converts `obj` to its proper SQL version @@ -74,13 +76,15 @@ def sqlify(obj): elif datetime and isinstance(obj, datetime.datetime): return repr(obj.isoformat()) else: - if isinstance(obj, unicode): obj = obj.encode('utf8') + if isinstance(obj, unicode): + obj = obj.encode('utf8') return repr(obj) - -def sqllist(lst): + + +def sqllist(lst): """ Converts the arguments for use in something like a WHERE clause. - + >>> sqllist(['a', 'b']) 'a, b' >>> sqllist('a') @@ -88,11 +92,12 @@ def sqllist(lst): >>> sqllist(u'abc') u'abc' """ - if isinstance(lst, basestring): + if isinstance(lst, basestring): return lst else: return ', '.join(lst) - + + def _sqllist(values): """ >>> _sqllist([1, 2, 3]) @@ -106,8 +111,9 @@ def _sqllist(values): items.append(sqlparam(v)) items.append(')') return SQLQuery(items) - -def sqlquote(a): + + +def sqlquote(a): """ Ensures `a` is quoted properly for use in a SQL query. @@ -120,8 +126,9 @@ def sqlquote(a): return _sqllist(a) else: return sqlparam(a).sqlquery() - -def _interpolate(sformat): + + +def _interpolate(sformat): """ Takes a format string and returns a list of 2-tuples of the form (boolean, string) where boolean says whether string should be evaled @@ -130,7 +137,7 @@ def _interpolate(sformat): from (public domain, Ka-Ping Yee) """ from tokenize import tokenprog - + tokenprog = tokenprog def matchorfail(text, pos): @@ -140,13 +147,13 @@ def matchorfail(text, pos): return match, match.end() namechars = "abcdefghijklmnopqrstuvwxyz" \ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" chunks = [] pos = 0 - while 1: + while True: dollar = sformat.find("$", pos) - if dollar < 0: + if dollar < 0: break nextchar = sformat[dollar + 1] @@ -157,9 +164,9 @@ def matchorfail(text, pos): match, pos = matchorfail(sformat, pos) tstart, tend = match.regs[3] token = sformat[tstart:tend] - if token == "{": + if token == "{": level = level + 1 - elif token == "}": + elif token == "}": level = level - 1 chunks.append((1, sformat[dollar + 2:pos - 1])) @@ -168,7 +175,7 @@ def matchorfail(text, pos): match, pos = matchorfail(sformat, dollar + 1) while pos < len(sformat): if sformat[pos] == "." and \ - pos + 1 < len(sformat) and sformat[pos + 1] in namechars: + pos + 1 < len(sformat) and sformat[pos + 1] in namechars: match, pos = matchorfail(sformat, pos + 1) elif sformat[pos] in "([": pos, level = pos + 1, 1 @@ -176,25 +183,26 @@ def matchorfail(text, pos): match, pos = matchorfail(sformat, pos) tstart, tend = match.regs[3] token = sformat[tstart:tend] - if token[0] in "([": + if token[0] in "([": level = level + 1 - elif token[0] in ")]": + elif token[0] in ")]": level = level - 1 - else: + else: break chunks.append((1, sformat[dollar + 1:pos])) else: chunks.append((0, sformat[pos:dollar + 1])) pos = dollar + 1 + (nextchar == "$") - if pos < len(sformat): + if pos < len(sformat): chunks.append((0, sformat[pos:])) return chunks -def sqlwhere(dictionary, grouping=' AND '): + +def sqlwhere(dictionary, grouping=' AND '): """ Converts a `dictionary` to an SQL WHERE clause `SQLQuery`. - + >>> sqlwhere({'cust_id': 2, 'order_id':3}) >>> sqlwhere({'cust_id': 2, 'order_id':3}, grouping=', ') @@ -202,9 +210,11 @@ def sqlwhere(dictionary, grouping=' AND '): >>> sqlwhere({'a': 'a', 'b': 'b'}).query() 'a = %s AND b = %s' """ - return SQLQuery.join([k + ' = ' + sqlparam(v) for k, v in dictionary.items()], grouping) - -def reparam(string_, dictionary): + return SQLQuery.join([k + ' = ' + sqlparam(v) + for k, v in dictionary.items()], grouping) + + +def reparam(string_, dictionary): """ Takes a string and a dictionary and interpolates the string using values from the dictionary. Returns an `SQLQuery` for the result. @@ -214,17 +224,19 @@ def reparam(string_, dictionary): >>> reparam("s IN $s", dict(s=[1, 2])) """ - dictionary = dictionary.copy() # eval mucks with it + dictionary = dictionary.copy() # eval mucks with it result = [] for live, chunk in _interpolate(string_): if live: v = eval(chunk, dictionary) result.append(sqlquote(v)) - else: + else: result.append(chunk) return SQLQuery.join(result, '') -class UnknownParamstyle(Exception): + +class UnknownParamstyle(Exception): + """ raised for unsupported db paramstyles @@ -232,19 +244,24 @@ class UnknownParamstyle(Exception): """ pass -class _ItplError(ValueError): + +class _ItplError(ValueError): + def __init__(self, text, pos): ValueError.__init__(self) self.text = text self.pos = pos + def __str__(self): return "unfinished expression in %s at char %d" % ( repr(self.text), self.pos) + class SQLParam(object): + """ Parameter in SQLQuery. - + >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam("joe")]) >>> q @@ -257,7 +274,7 @@ class SQLParam(object): def __init__(self, value): self.value = value - + def get_marker(self, paramstyle='pyformat'): if paramstyle == 'qmark': return '?' @@ -265,26 +282,28 @@ def get_marker(self, paramstyle='pyformat'): return ':1' elif paramstyle is None or paramstyle in ['format', 'pyformat']: return '%s' - raise UnknownParamstyle, paramstyle - - def sqlquery(self): + raise UnknownParamstyle(paramstyle) + + def sqlquery(self): return SQLQuery([self]) - + def __add__(self, other): return self.sqlquery() + other - + def __radd__(self, other): - return other + self.sqlquery() - - def __str__(self): + return other + self.sqlquery() + + def __str__(self): return str(self.value) - + def __repr__(self): return '' % repr(self.value) -sqlparam = SQLParam +sqlparam = SQLParam + class SQLQuery(object): + """ You can pass this sort of thing as a clause in any db function. Otherwise, you can pass a dictionary to the keyword argument `vars` @@ -298,7 +317,7 @@ class SQLQuery(object): # tested in sqlquote's docstring def __init__(self, items=None): r"""Creates a new SQLQuery. - + >>> SQLQuery("x") >>> q = SQLQuery(['SELECT * FROM ', 'test', ' WHERE x=', SQLParam(1)]) @@ -319,10 +338,11 @@ def __init__(self, items=None): self.items = list(items.items) else: self.items = [items] - + # Take care of SQLLiterals for i, item in enumerate(self.items): - if isinstance(item, SQLParam) and isinstance(item.value, SQLLiteral): + if isinstance(item, SQLParam) and isinstance( + item.value, SQLLiteral): self.items[i] = item.value.v def append(self, value): @@ -342,7 +362,7 @@ def __radd__(self, other): items = [other] else: return NotImplemented - + return SQLQuery(items + self.items) def __iadd__(self, other): @@ -356,7 +376,7 @@ def __iadd__(self, other): def __len__(self): return len(self.query()) - + def query(self, paramstyle=None): """ Returns the query part of the sql query. @@ -374,13 +394,14 @@ def query(self, paramstyle=None): else: x = safestr(x) # automatically escape % characters in the query - # For backward compatability, ignore escaping when the query looks already escaped + # For backward compatability, ignore escaping when the query + # looks already escaped if paramstyle in ['format', 'pyformat']: if '%' in x and '%%' not in x: x = x.replace('%', '%%') s.append(x) return "".join(s) - + def values(self): """ Returns the values of the parameters used in the sql query. @@ -389,11 +410,11 @@ def values(self): ['joe'] """ return [i.value for i in self.items if isinstance(i, SQLParam)] - + def join(items, sep=' ', prefix=None, suffix=None, target=None): """ Joins multiple queries. - + >>> SQLQuery.join(['a', 'b'], ', ') @@ -423,25 +444,27 @@ def join(items, sep=' ', prefix=None, suffix=None, target=None): if suffix: target_items.append(suffix) return target - + join = staticmethod(join) - + def _str(self): try: - return self.query() % tuple([sqlify(x) for x in self.values()]) + return self.query() % tuple([sqlify(x) for x in self.values()]) except (ValueError, TypeError): return self.query() - + def __str__(self): return safestr(self._str()) - + def __unicode__(self): return safeunicode(self._str()) def __repr__(self): return '' % repr(str(self)) - -class SQLLiteral: + + +class SQLLiteral: + """ Protects a string from `sqlquote`. @@ -450,25 +473,29 @@ class SQLLiteral: >>> sqlquote(SQLLiteral('NOW()')) """ - def __init__(self, v): + + def __init__(self, v): self.v = v - def __repr__(self): + def __repr__(self): return self.v -class SQLProducer: + +class SQLProducer: + """Database""" + def __init__(self): """Creates a database. """ pass - - def query(self, sql_query,processed=False, svars=None): + + def query(self, sql_query, processed=False, svars=None): """ Execute SQL query `sql_query` using dictionary `vars` to interpolate it. - If `processed=True`, `vars` is a `reparam`-style list to use + If `processed=True`, `vars` is a `reparam`-style list to use instead of interpolating. - + >>> db = DB(None, {}) >>> db.query("SELECT * FROM foo", _test=True) @@ -479,13 +506,13 @@ def query(self, sql_query,processed=False, svars=None): """ if svars is None: svars = {} - + if not processed and not isinstance(sql_query, SQLQuery): sql_query = reparam(sql_query, svars) - + return sql_query - - def sql_clauses(self, what, tables, where, group, order, limit, offset): + + def sql_clauses(self, what, tables, where, group, order, limit, offset): return ( ('SELECT', what), ('FROM', sqllist(tables)), @@ -494,28 +521,30 @@ def sql_clauses(self, what, tables, where, group, order, limit, offset): ('ORDER BY', order), ('LIMIT', limit), ('OFFSET', offset)) - - def gen_clause(self, sql, val, svars): + + def gen_clause(self, sql, val, svars): if isinstance(val, (int, long)): if sql == 'WHERE': nout = 'id = ' + sqlquote(val) else: nout = SQLQuery(val) - + elif isinstance(val, (list, tuple)) and len(val) == 2: - nout = SQLQuery(val[0], val[1]) # backwards-compatibility + nout = SQLQuery(val[0], val[1]) # backwards-compatibility elif isinstance(val, SQLQuery): nout = val else: nout = reparam(val, svars) def xjoin(a, b): - if a and b: return a + ' ' + b - else: return a or b + if a and b: + return a + ' ' + b + else: + return a or b return xjoin(sql, nout) - - def _where(self, where, svars): + + def _where(self, where, svars): if isinstance(where, (int, long)): where = "id = " + sqlparam(where) elif isinstance(where, (list, tuple)) and len(where) == 2: @@ -523,35 +552,49 @@ def _where(self, where, svars): elif isinstance(where, SQLQuery): pass else: - where = reparam(where, svars) + where = reparam(where, svars) return where - - def select(self, tables, svars=None, what='*', where=None, order=None, group=None, - limit=None, offset=None, _test=False): + + def select(self, tables, svars=None, what='*', where=None, order=None, group=None, + limit=None, offset=None, _test=False): """ - Selects `what` from `tables` with clauses `where`, `order`, - `group`, `limit`, and `offset`. Uses vars to interpolate. + Selects `what` from `tables` with clauses `where`, `order`, + `group`, `limit`, and `offset`. Uses vars to interpolate. Otherwise, each clause can be a SQLQuery. - + >>> db = DB(None, {}) >>> db.select('foo', _test=True) >>> db.select(['foo', 'bar'], where="foo.bar_id = bar.id", limit=5, _test=True) """ - if svars is None: svars = {} - sql_clauses = self.sql_clauses(what, tables, where, group, order, limit, offset) - clauses = [self.gen_clause(sql, val, svars) for sql, val in sql_clauses if val is not None] + if svars is None: + svars = {} + sql_clauses = self.sql_clauses( + what, + tables, + where, + group, + order, + limit, + offset) + clauses = [ + self.gen_clause( + sql, + val, + svars) for sql, + val in sql_clauses if val is not None] qout = SQLQuery.join(clauses) - if _test: return qout + if _test: + return qout return self.query(qout, processed=True) - def insert(self, tablename, seqname=None, _test=False, **values): + def insert(self, tablename, seqname=None, _test=False, **values): """ Inserts `values` into `tablename`. Returns current sequence ID. Set `seqname` to the ID if it's not the default, or to `False` if there isn't one. - + >>> db = DB(None, {}) >>> q = db.insert('foo', name='bob', age=2, created=SQLLiteral('NOW()'), _test=True) >>> q @@ -561,67 +604,86 @@ def insert(self, tablename, seqname=None, _test=False, **values): >>> q.values() [2, 'bob'] """ - def q(x): return "(" + x + ")" - + def q(x): + return "(" + x + ")" + if values: _keys = SQLQuery.join(values.keys(), ', ') - _values = SQLQuery.join([sqlparam(v) for v in values.values()], ', ') - sql_query = "INSERT INTO %s " % tablename + q(_keys) + ' VALUES ' + q(_values) + _values = SQLQuery.join([sqlparam(v) + for v in values.values()], ', ') + sql_query = "INSERT INTO %s " % tablename + \ + q(_keys) + ' VALUES ' + q(_values) else: - sql_query = SQLQuery(self._get_insert_default_values_query(tablename)) + sql_query = SQLQuery( + self._get_insert_default_values_query(tablename)) return sql_query - + def _get_insert_default_values_query(self, table): return "INSERT INTO %s DEFAULT VALUES" % table def multiple_insert(self, tablename, values, seqname=None, _test=False): """ - Inserts multiple rows into `tablename`. The `values` must be a list of dictioanries, + Inserts multiple rows into `tablename`. The `values` must be a list of dictioanries, one for each row to be inserted, each with the same set of keys. - Returns the list of ids of the inserted rows. + Returns the list of ids of the inserted rows. Set `seqname` to the ID if it's not the default, or to `False` if there isn't one. - + >>> db = DB(None, {}) >>> db.supports_multiple_insert = True >>> values = [{"name": "foo", "email": "foo@example.com"}, {"name": "bar", "email": "bar@example.com"}] >>> db.multiple_insert('person', values=values, _test=True) - """ + """ if not values: return [] - + if not self.supports_multiple_insert: - out = [self.insert(tablename, seqname=seqname, _test=_test, **v) for v in values] + out = [ + self.insert( + tablename, + seqname=seqname, + _test=_test, + **v) for v in values] if seqname is False: return None else: return out - + keys = values[0].keys() #@@ make sure all keys are valid # make sure all rows have same keys. for v in values: if v.keys() != keys: - raise ValueError, 'Bad data' + raise ValueError('Bad data') - sql_query = SQLQuery('INSERT INTO %s (%s) VALUES ' % (tablename, ', '.join(keys))) + sql_query = SQLQuery( + 'INSERT INTO %s (%s) VALUES ' % + (tablename, ', '.join(keys))) for i, row in enumerate(values): if i != 0: sql_query.append(", ") - SQLQuery.join([SQLParam(row[k]) for k in keys], sep=", ", target=sql_query, prefix="(", suffix=")") - - if _test: return sql_query + SQLQuery.join([SQLParam(row[k]) for k in keys], + sep=", ", + target=sql_query, + prefix="(", + suffix=")") + + if _test: + return sql_query db_cursor = self._db_cursor() - if seqname is not False: - sql_query = self._process_insert_query(sql_query, tablename, seqname) + if seqname is not False: + sql_query = self._process_insert_query( + sql_query, + tablename, + seqname) if isinstance(sql_query, tuple): - # for some databases, a separate query has to be made to find + # for some databases, a separate query has to be made to find # the id of the inserted row. q1, q2 = sql_query self._db_execute(db_cursor, q1) @@ -629,18 +691,17 @@ def multiple_insert(self, tablename, values, seqname=None, _test=False): else: self._db_execute(db_cursor, sql_query) - try: + try: out = db_cursor.fetchone()[0] - out = range(out-len(values)+1, out+1) - except Exception: + out = range(out - len(values) + 1, out + 1) + except Exception: out = None - if not self.ctx.transactions: + if not self.ctx.transactions: self.ctx.commit() return out - - def update(self, tables, where, svars=None, _test=False, **values): + def update(self, tables, where, svars=None, _test=False, **values): """ Update `tables` with clause `where` (interpolated using `vars`) and setting `values`. @@ -656,23 +717,25 @@ def update(self, tables, where, svars=None, _test=False, **values): >>> q.values() [2, 'bob', 'Joseph'] """ - if svars is None: svars = {} + if svars is None: + svars = {} where = self._where(where, svars) query = ( - "UPDATE " + sqllist(tables) + - " SET " + sqlwhere(values, ', ') + - " WHERE " + where) + "UPDATE " + sqllist(tables) + + " SET " + sqlwhere(values, ', ') + + " WHERE " + where) + + if _test: + return query - if _test: return query - db_cursor = self._db_cursor() self._db_execute(db_cursor, query) - if not self.ctx.transactions: + if not self.ctx.transactions: self.ctx.commit() return db_cursor.rowcount - - def delete(self, table, where, using=None, svars=None, _test=False): + + def delete(self, table, where, using=None, svars=None, _test=False): """ Deletes from `table` with clauses `where` and `using`. @@ -692,8 +755,5 @@ def delete(self, table, where, using=None, svars=None, _test=False): q += ' WHERE ' + where return q - -sqlproducer = SQLProducer() - - \ No newline at end of file +sqlproducer = SQLProducer() diff --git a/gfirefly/dbentrust/madminanager.py b/gfirefly/dbentrust/madminanager.py index 077a50a..5ff683b 100644 --- a/gfirefly/dbentrust/madminanager.py +++ b/gfirefly/dbentrust/madminanager.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-22 @@ -6,17 +6,19 @@ ''' from gfirefly.utils.singleton import Singleton + class MAdminManager: + """一个单例管理器。作为所有madmin的管理者 """ __metaclass__ = Singleton - + def __init__(self): """初始化所有管理的的madmin的集合，放在self.admins中 """ self.admins = {} - - def registe(self,admin): + + def registe(self, admin): """注册一个madmin对象到管理中. >>> madmin = MAdmin('tb_registe','characterId',incrkey='id') @@ -24,22 +26,22 @@ def registe(self,admin): """ self.admins[admin._name] = admin - - def dropAdmin(self,adminname): + + def dropAdmin(self, adminname): """移除一个madmin对象. >>> MAdminManager().dropAdmin(madmin) """ - if self.admins.has_key(adminname): + if adminname in self.admins: del self.admins[adminname] - - def getAdmin(self,adminname): + + def getAdmin(self, adminname): """根据madmin的名字获取madmin对象 >>> madmin = MAdminManager().getAdmin('tb_registe') """ return self.admins.get(adminname) - + def checkAdmins(self): """遍历所有的madmin，与数据库进行同步。 @@ -47,7 +49,3 @@ def checkAdmins(self): """ for admin in self.admins.values(): admin.checkAll() - - - - \ No newline at end of file diff --git a/gfirefly/dbentrust/memclient.py b/gfirefly/dbentrust/memclient.py index 966bbd5..ca3bbc3 100644 --- a/gfirefly/dbentrust/memclient.py +++ b/gfirefly/dbentrust/memclient.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-7-10 memcached client @@ -6,109 +6,111 @@ ''' import memcache -class MemConnError(Exception): + +class MemConnError(Exception): + """memcached 连接错误 """ - + def __str__(self): return "memcache connect error" + class MemClient: + '''memcached 连接类，对通过它存储到memcached中的key，定义了新的key的生成规则，避免key的冲突。\n @param _hostname: str 这个连接的命名空间。新生成的key的规则会是 _hostname:key。\n @param _urls: []list memcached的连接的配置\n @param connection: memcached的连接实例。\n >>> mclient = MemClient() ''' - - def __init__(self,timeout = 0): + + def __init__(self, timeout=0): ''' ''' self._hostname = "" self._urls = [] self.connection = None - - def connect(self,urls,hostname): + + def connect(self, urls, hostname): '''memcached 建立连接，配置连接信息 >>> mclient.connect(['127.0.0.1:11211'], "test") ''' self._hostname = hostname self._urls = urls - self.connection = memcache.Client(self._urls,debug=0) - if not self.connection.set("__testkey__",1): + self.connection = memcache.Client(self._urls, debug=0) + if not self.connection.set("__testkey__", 1): raise MemConnError() - - def produceKey(self,keyname): + + def produceKey(self, keyname): '''重新生成新的key，规则是 _hostname:key >>> mclient.produceKey('name') test:name ''' if isinstance(keyname, basestring): - return str(''.join([self._hostname,':',keyname])) + return str(''.join([self._hostname, ':', keyname])) else: raise "type error" - - def get(self,key): + + def get(self, key): ''' ''' key = self.produceKey(key) return self.connection.get(key) - - def get_multi(self,keys): + + def get_multi(self, keys): ''' ''' keynamelist = [self.produceKey(keyname) for keyname in keys] olddict = self.connection.get_multi(keynamelist) newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], - olddict.values())) + olddict.values())) return newdict - - def set(self,keyname,value): + + def set(self, keyname, value): ''' ''' key = self.produceKey(keyname) - result = self.connection.set(key,value) - if not result:#如果写入失败 - self.connect(self._urls,self._hostname)#重新连接 - return self.connection.set(key,value) + result = self.connection.set(key, value) + if not result: # 如果写入失败 + self.connect(self._urls, self._hostname) # 重新连接 + return self.connection.set(key, value) return result - - def set_multi(self,mapping): + + def set_multi(self, mapping): ''' ''' newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], mapping.values())) result = self.connection.set_multi(newmapping) - if result:#如果写入失败 - self.connect(self._urls,self._hostname)#重新连接 + if result: # 如果写入失败 + self.connect(self._urls, self._hostname) # 重新连接 return self.connection.set_multi(newmapping) return result - - def incr(self,key,delta): + + def incr(self, key, delta): ''' ''' key = self.produceKey(key) return self.connection.incr(key, delta) - - def delete(self,key): + + def delete(self, key): ''' ''' key = self.produceKey(key) return self.connection.delete(key) - - def delete_multi(self,keys): + + def delete_multi(self, keys): """ """ keys = [self.produceKey(key) for key in keys] return self.connection.delete_multi(keys) - + def flush_all(self): ''' ''' self.connection.flush_all() - -mclient = MemClient() - +mclient = MemClient() diff --git a/gfirefly/dbentrust/memobject.py b/gfirefly/dbentrust/memobject.py index e5cc3c0..6649886 100644 --- a/gfirefly/dbentrust/memobject.py +++ b/gfirefly/dbentrust/memobject.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2012-7-10 memcached 关系对象\n @@ -7,9 +7,11 @@ @author: lan (www.9miao.com) ''' + class MemObject: + '''memcached 关系对象,可以将一个对象的属性值记录到memcached中。 - + >>> class Mcharacter(MemObject): >>> def __init__(self,pid,name,mc): >>> MemObject.__init__(self, name, mc) @@ -24,8 +26,8 @@ class MemObject: >>> mcharacter.get('nickname') lan ''' - - def __init__(self,name,mc): + + def __init__(self, name, mc): ''' @param name: str 对象的名称\n @param _lock: int 对象锁 为1时表示对象被锁定无法进行修改\n @@ -33,58 +35,58 @@ def __init__(self,name,mc): self._client = mc self._name = name self._lock = 0 - - def produceKey(self,keyname): + + def produceKey(self, keyname): '''重新生成key ''' if isinstance(keyname, basestring): - return ''.join([self._name,':',keyname]) + return ''.join([self._name, ':', keyname]) else: raise "type error" - + def locked(self): '''检测对象是否被锁定 ''' key = self.produceKey('_lock') return self._client.get(key) - + def lock(self): '''锁定对象 ''' key = self.produceKey('_lock') self._client.set(key, 1) - + def release(self): '''释放锁 ''' key = self.produceKey('_lock') self._client.set(key, 0) - - def get(self,key): + + def get(self, key): '''获取对象值 ''' key = self.produceKey(key) return self._client.get(key) - - def get_multi(self,keys): + + def get_multi(self, keys): '''一次获取多个key的值 @param keys: list(str) key的列表 ''' keynamelist = [self.produceKey(keyname) for keyname in keys] olddict = self._client.get_multi(keynamelist) newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], - olddict.values())) + olddict.values())) return newdict - def update(self,key,values): + def update(self, key, values): '''修改对象的值 ''' if self.locked(): return False key = self.produceKey(key) - return self._client.set(key,values) - - def update_multi(self,mapping): + return self._client.set(key, values) + + def update_multi(self, mapping): '''同时修改多个key值 ''' if self.locked(): @@ -92,7 +94,7 @@ def update_multi(self,mapping): newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], mapping.values())) return self._client.set_multi(newmapping) - + def mdelete(self): '''删除memcache中的数据 ''' @@ -101,13 +103,13 @@ def mdelete(self): keys = nowdict.keys() keys = [self.produceKey(key) for key in keys] self._client.delete_multi(keys) - + def incr(self, key, delta): '''自增 ''' key = self.produceKey(key) - return self._client.incr( key, delta) - + return self._client.incr(key, delta) + def insert(self): '''插入对象记录 ''' @@ -116,7 +118,3 @@ def insert(self): newmapping = dict(zip([self.produceKey(keyname) for keyname in nowdict.keys()], nowdict.values())) self._client.set_multi(newmapping) - - - - diff --git a/gfirefly/dbentrust/mmode.py b/gfirefly/dbentrust/mmode.py index eb23510..311d1df 100644 --- a/gfirefly/dbentrust/mmode.py +++ b/gfirefly/dbentrust/mmode.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-8 @@ -9,153 +9,170 @@ import util import time -MMODE_STATE_ORI = 0 #未变更 -MMODE_STATE_NEW = 1 #创建 -MMODE_STATE_UPDATE = 2 #更新 -MMODE_STATE_DEL = 3 #删除 - +MMODE_STATE_ORI = 0 # 未变更 +MMODE_STATE_NEW = 1 # 创建 +MMODE_STATE_UPDATE = 2 # 更新 +MMODE_STATE_DEL = 3 # 删除 TIMEOUT = 1800 + def _insert(args): - record,pkname,mmname,cls = args + record, pkname, mmname, cls = args pk = record[pkname] - mm = cls(mmname+':%s'%pk,pkname,data=record) + mm = cls(mmname + ':%s' % pk, pkname, data=record) mm.insert() return pk -class PKValueError(ValueError): + +class PKValueError(ValueError): + """ """ + def __init__(self, data): ValueError.__init__(self) self.data = data + def __str__(self): return "new record has no 'PK': %s" % (self.data) + class MMode(MemObject): + """内存数据模型，最终对应到的是表中的一条记录 """ - def __init__(self, name,pk,data={}): + + def __init__(self, name, pk, data={}): """ """ MemObject.__init__(self, name, mclient) - self._state = MMODE_STATE_ORI#对象的状态 0未变更 1新建 2更新 3删除 + self._state = MMODE_STATE_ORI # 对象的状态 0未变更 1新建 2更新 3删除 self._pk = pk self.data = data self._time = time.time() - + def update(self, key, values): - data = self.get_multi(['data','_state']) + data = self.get_multi(['data', '_state']) ntime = time.time() - data['data'].update({key:values}) - if data.get('_state')==MMODE_STATE_NEW: - props = {'data':data.get('data'),'_time':ntime} + data['data'].update({key: values}) + if data.get('_state') == MMODE_STATE_NEW: + props = {'data': data.get('data'), '_time': ntime} else: - props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} + props = { + '_state': MMODE_STATE_UPDATE, + 'data': data.get('data'), + '_time': ntime} return MemObject.update_multi(self, props) - + def update_multi(self, mapping): ntime = time.time() - data = self.get_multi(['data','_state']) + data = self.get_multi(['data', '_state']) data['data'].update(mapping) - if data.get('_state')==MMODE_STATE_NEW: - props = {'data':data.get('data'),'_time':ntime} + if data.get('_state') == MMODE_STATE_NEW: + props = {'data': data.get('data'), '_time': ntime} else: - props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} + props = { + '_state': MMODE_STATE_UPDATE, + 'data': data.get('data'), + '_time': ntime} return MemObject.update_multi(self, props) - + def get(self, key): ntime = time.time() MemObject.update(self, "_time", ntime) return MemObject.get(self, key) - + def get_multi(self, keys): ntime = time.time() MemObject.update(self, "_time", ntime) return MemObject.get_multi(self, keys) - + def delete(self): '''删除对象 ''' - return MemObject.update(self,'_state',MMODE_STATE_DEL) - + return MemObject.update(self, '_state', MMODE_STATE_DEL) + def mdelete(self): """清理对象 """ self.syncDB() MemObject.mdelete(self) - + def IsEffective(self): '''检测对象是否有效 ''' - if self.get('_state')==MMODE_STATE_DEL: + if self.get('_state') == MMODE_STATE_DEL: return False return True - + def syncDB(self): """同步到数据库 """ state = self.get('_state') tablename = self._name.split(':')[0] - if state==MMODE_STATE_ORI: + if state == MMODE_STATE_ORI: return - elif state==MMODE_STATE_NEW: + elif state == MMODE_STATE_NEW: props = self.get('data') pk = self.get('_pk') result = util.InsertIntoDB(tablename, props) - elif state==MMODE_STATE_UPDATE: + elif state == MMODE_STATE_UPDATE: props = self.get('data') pk = self.get('_pk') - prere = {pk:props.get(pk)} + prere = {pk: props.get(pk)} util.UpdateWithDict(tablename, props, prere) result = True else: pk = self.get('_pk') props = self.get('data') - prere = {pk:props.get(pk)} - result = util.DeleteFromDB(tablename,prere) + prere = {pk: props.get(pk)} + result = util.DeleteFromDB(tablename, prere) if result: - MemObject.update(self,'_state', MMODE_STATE_ORI) - - def checkSync(self,timeout=TIMEOUT): + MemObject.update(self, '_state', MMODE_STATE_ORI) + + def checkSync(self, timeout=TIMEOUT): """检测同步 """ ntime = time.time() objtime = MemObject.get(self, '_time') - if ntime -objtime>=timeout and timeout: + if ntime - objtime >= timeout and timeout: self.mdelete() else: self.syncDB() - - + + class MFKMode(MemObject): + """外键内存数据模型 """ - def __init__(self, name,pklist = []): + + def __init__(self, name, pklist=[]): MemObject.__init__(self, name, mclient) self.pklist = pklist - + + class MAdmin(MemObject): + """MMode对象管理，同一个MAdmin管理同一类的MMode，对应的是数据库中的某一种表 """ - - def __init__(self, name,pk,timeout=TIMEOUT,**kw): + + def __init__(self, name, pk, timeout=TIMEOUT, **kw): MemObject.__init__(self, name, mclient) self._pk = pk - self._fk = kw.get('fk','') - self._incrkey = kw.get('incrkey','') - self._incrvalue = kw.get('incrvalue',0) + self._fk = kw.get('fk', '') + self._incrkey = kw.get('incrkey', '') + self._incrvalue = kw.get('incrvalue', 0) self._timeout = timeout - + def insert(self): """将MAdmin配置的信息写入memcached中保存。\n当在其他的进程中实例化相同的配置的MAdmin，可以使得数据同步。 """ if self._incrkey and not self.get("_incrvalue"): self._incrvalue = util.GetTableIncrValue(self._name) MemObject.insert(self) - + def load(self): '''读取数据到数据库中 ''' @@ -163,9 +180,9 @@ def load(self): recordlist = util.ReadDataFromDB(mmname) for record in recordlist: pk = record[self._pk] - mm = MMode(self._name+':%s'%pk,self._pk,data=record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() - + @property def madmininfo(self): """作为一个特性属性。可以获取这个madmin的相关信息 @@ -173,66 +190,65 @@ def madmininfo(self): keys = self.__dict__.keys() info = self.get_multi(keys) return info - - def getAllPkByFk(self,fk): + + def getAllPkByFk(self, fk): '''根据外键获取主键列表 ''' - name = '%s_fk:%s'%(self._name,fk) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) pklist = fkmm.get('pklist') if pklist is not None: return pklist - props = {self._fk:fk} + props = {self._fk: fk} dbkeylist = util.getAllPkByFkInDB(self._name, self._pk, props) - name = '%s_fk:%s'%(self._name,fk) - fkmm = MFKMode(name, pklist = dbkeylist) + name = '%s_fk:%s' % (self._name, fk) + fkmm = MFKMode(name, pklist=dbkeylist) fkmm.insert() return dbkeylist - - def getObj(self,pk): + + def getObj(self, pk): '''根据主键，可以获得mmode对象的实例.\n >>> m = madmin.getObj(1) ''' - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): return None if mm.get('data'): return mm - props = {self._pk:pk} - record = util.GetOneRecordInfo(self._name,props) + props = {self._pk: pk} + record = util.GetOneRecordInfo(self._name, props) if not record: return None - mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() return mm - - def getObjData(self,pk): + + def getObjData(self, pk): '''根据主键，可以获得mmode对象的实例的数据.\n >>> m = madmin.getObjData(1) ''' - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): return None data = mm.get('data') if mm.get('data'): return data - props = {self._pk:pk} - record = util.GetOneRecordInfo(self._name,props) + props = {self._pk: pk} + record = util.GetOneRecordInfo(self._name, props) if not record: return None - mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() return record - - - def getObjList(self,pklist): + + def getObjList(self, pklist): '''根据主键列表获取mmode对象的列表.\n >>> m = madmin.getObjList([1,2,3,4,5]) ''' _pklist = [] objlist = [] for pk in pklist: - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): continue if mm.get('data'): @@ -240,15 +256,15 @@ def getObjList(self,pklist): else: _pklist.append(pk) if _pklist: - recordlist = util.GetRecordList(self._name, self._pk,_pklist) + recordlist = util.GetRecordList(self._name, self._pk, _pklist) for record in recordlist: pk = record[self._pk] - mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm = MMode(self._name + ':%s' % pk, self._pk, data=record) mm.insert() objlist.append(mm) return objlist - - def deleteMode(self,pk): + + def deleteMode(self, pk): '''根据主键删除内存中的某条记录信息，\n这里只是修改内存中的记录状态_state为删除状态.\n >>> m = madmin.deleteMode(1) ''' @@ -257,8 +273,8 @@ def deleteMode(self,pk): if self._fk: data = mm.get('data') if data: - fk = data.get(self._fk,0) - name = '%s_fk:%s'%(self._name,fk) + fk = data.get(self._fk, 0) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) pklist = fkmm.get('pklist') if pklist and pk in pklist: @@ -266,15 +282,15 @@ def deleteMode(self,pk): fkmm.update('pklist', pklist) mm.delete() return True - + def checkAll(self): """同步内存中的数据到对应的数据表中。\n >>> m = madmin.checkAll() """ - key = '%s:%s:'%(mclient._hostname,self._name) + key = '%s:%s:' % (mclient._hostname, self._name) _pklist = util.getallkeys(key, mclient.connection) for pk in _pklist: - mm = MMode(self._name+':%s'%pk,self._pk) + mm = MMode(self._name + ':%s' % pk, self._pk) if not mm.IsEffective(): mm.mdelete() continue @@ -282,42 +298,41 @@ def checkAll(self): continue mm.checkSync(timeout=self._timeout) self.deleteAllFk() - + def deleteAllFk(self): """删除所有的外键 """ - key = '%s:%s_fk:'%(mclient._hostname,self._name) + key = '%s:%s_fk:' % (mclient._hostname, self._name) _fklist = util.getallkeys(key, mclient.connection) for fk in _fklist: - name = '%s_fk:%s'%(self._name,fk) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) fkmm.mdelete() - - def new(self,data): + + def new(self, data): """创建一个新的对象 """ incrkey = self._incrkey if incrkey: incrvalue = self.incr('_incrvalue', 1) - data[incrkey] = incrvalue - 1 + data[incrkey] = incrvalue - 1 pk = data.get(self._pk) if pk is None: raise PKValueError(data) - mm = MMode(self._name+':%s'%pk,self._pk,data=data) - setattr(mm,incrkey,pk) + mm = MMode(self._name + ':%s' % pk, self._pk, data=data) + setattr(mm, incrkey, pk) else: pk = data.get(self._pk) - mm = MMode(self._name+':%s'%pk,self._pk,data=data) + mm = MMode(self._name + ':%s' % pk, self._pk, data=data) if self._fk: - fk = data.get(self._fk,0) - name = '%s_fk:%s'%(self._name,fk) + fk = data.get(self._fk, 0) + name = '%s_fk:%s' % (self._name, fk) fkmm = MFKMode(name) pklist = fkmm.get('pklist') if pklist is None: pklist = self.getAllPkByFk(fk) pklist.append(pk) fkmm.update('pklist', pklist) - setattr(mm,'_state',MMODE_STATE_NEW) + setattr(mm, '_state', MMODE_STATE_NEW) mm.insert() return mm - diff --git a/gfirefly/dbentrust/util.py b/gfirefly/dbentrust/util.py index f317a27..e0d26d0 100644 --- a/gfirefly/dbentrust/util.py +++ b/gfirefly/dbentrust/util.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-5-8 @@ -11,71 +11,79 @@ from gtwisted.utils import log -def forEachPlusInsertProps(tablename,props): - assert type(props) == dict - pkeysstr = str(tuple(props.keys())).replace('\'','`') - pvaluesstr = ["%s,"%val if isinstance(val,Number) else - "'%s',"%str(val).replace("'", "\\'") for val in props.values()] +def forEachPlusInsertProps(tablename, props): + assert isinstance(props, dict) + pkeysstr = str(tuple(props.keys())).replace('\'', '`') + pvaluesstr = ["%s," % val if isinstance(val, Number) else + "'%s'," % str(val).replace("'", "\\'") for val in props.values()] pvaluesstr = ''.join(pvaluesstr)[:-1] - sqlstr = """INSERT INTO `%s` %s values (%s);"""%(tablename,pkeysstr,pvaluesstr) + sqlstr = """INSERT INTO `%s` %s values (%s);""" % ( + tablename, pkeysstr, pvaluesstr) return sqlstr + def FormatCondition(props): """生成查询条件字符串 """ items = props.items() itemstrlist = [] for _item in items: - if isinstance(_item[1],Number): - sqlstr = " `%s`=%s AND"%_item + if isinstance(_item[1], Number): + sqlstr = " `%s`=%s AND" % _item else: - sqlstr = " `%s`='%s' AND "%(_item[0],str(_item[1]).replace("'", "\\'")) + sqlstr = " `%s`='%s' AND " % ( + _item[0], str(_item[1]).replace("'", "\\'")) itemstrlist.append(sqlstr) sqlstr = ''.join(itemstrlist) return sqlstr[:-4] + def FormatUpdateStr(props): """生成更新语句 """ items = props.items() itemstrlist = [] for _item in items: - if isinstance(_item[1],Number): - sqlstr = " `%s`=%s,"%_item + if isinstance(_item[1], Number): + sqlstr = " `%s`=%s," % _item else: - sqlstr = " `%s`='%s',"%(_item[0],str(_item[1]).replace("'", "\\'")) + sqlstr = " `%s`='%s'," % ( + _item[0], str(_item[1]).replace("'", "\\'")) itemstrlist.append(sqlstr) sqlstr = ''.join(itemstrlist) return sqlstr[:-1] - -def forEachUpdateProps(tablename,props,prere): + + +def forEachUpdateProps(tablename, props, prere): '''遍历所要修改的属性，以生成sql语句''' - assert type(props) == dict + assert isinstance(props, dict) pro = FormatUpdateStr(props) pre = FormatCondition(prere) - sqlstr = """UPDATE `%s` SET %s WHERE %s;"""%(tablename,pro,pre) + sqlstr = """UPDATE `%s` SET %s WHERE %s;""" % (tablename, pro, pre) return sqlstr + def EachQueryProps(props): '''遍历字段列表生成sql语句 ''' sqlstr = "" if props == '*': return '*' - elif type(props) == type([0]): + elif isinstance(props, type([0])): for prop in props: - sqlstr = sqlstr + prop +',' + sqlstr = sqlstr + prop + ',' sqlstr = sqlstr[:-1] return sqlstr else: raise Exception('props to query must be dict') return + def forEachQueryProps(sqlstr, props): '''遍历所要查询属性，以生成sql语句''' if props == '*': sqlstr += ' *' - elif type(props) == type([0]): + elif isinstance(props, type([0])): i = 0 for prop in props: if(i == 0): @@ -88,12 +96,13 @@ def forEachQueryProps(sqlstr, props): return return sqlstr + def GetTableIncrValue(tablename): """ """ database = dbpool.config.get('db') sql = """SELECT AUTO_INCREMENT FROM information_schema.`TABLES` \ - WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s';"""%(database,tablename) + WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s';""" % (database, tablename) conn = dbpool.connection() cursor = conn.cursor() cursor.execute(sql) @@ -104,54 +113,58 @@ def GetTableIncrValue(tablename): return result[0] return result + def ReadDataFromDB(tablename): """ """ - sql = """select * from %s"""%tablename + sql = """select * from %s""" % tablename conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) - result=cursor.fetchall() + result = cursor.fetchall() cursor.close() conn.close() return result -def DeleteFromDB(tablename,props): + +def DeleteFromDB(tablename, props): '''从数据库中删除 ''' prers = FormatCondition(props) - sql = """DELETE FROM %s WHERE %s ;"""%(tablename,prers) + sql = """DELETE FROM %s WHERE %s ;""" % (tablename, prers) conn = dbpool.connection() cursor = conn.cursor() count = 0 try: count = cursor.execute(sql) conn.commit() - except Exception,e: + except Exception as e: log.err(e) log.err(sql) cursor.close() conn.close() return bool(count) -def InsertIntoDB(tablename,data): + +def InsertIntoDB(tablename, data): """写入数据库 """ - sql = forEachPlusInsertProps(tablename,data) + sql = forEachPlusInsertProps(tablename, data) conn = dbpool.connection() cursor = conn.cursor() count = 0 try: count = cursor.execute(sql) conn.commit() - except Exception,e: + except Exception as e: log.err(e) log.err(sql) cursor.close() conn.close() return bool(count) -def UpdateWithDict(tablename,props,prere): + +def UpdateWithDict(tablename, props, prere): """更新记录 """ sql = forEachUpdateProps(tablename, props, prere) @@ -161,7 +174,7 @@ def UpdateWithDict(tablename,props,prere): try: count = cursor.execute(sql) conn.commit() - except Exception,e: + except Exception as e: log.err(e) log.err(sql) cursor.close() @@ -170,11 +183,12 @@ def UpdateWithDict(tablename,props,prere): return True return False -def getAllPkByFkInDB(tablename,pkname,props): + +def getAllPkByFkInDB(tablename, pkname, props): """根据所有的外键获取主键ID """ props = FormatCondition(props) - sql = """Select `%s` from `%s` where %s"""%(pkname,tablename,props) + sql = """Select `%s` from `%s` where %s""" % (pkname, tablename, props) conn = dbpool.connection() cursor = conn.cursor() cursor.execute(sql) @@ -183,52 +197,57 @@ def getAllPkByFkInDB(tablename,pkname,props): conn.close() return [key[0] for key in result] -def GetOneRecordInfo(tablename,props): + +def GetOneRecordInfo(tablename, props): '''获取单条数据的信息 ''' props = FormatCondition(props) - sql = """Select * from `%s` where %s"""%(tablename,props) + sql = """Select * from `%s` where %s""" % (tablename, props) conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) result = cursor.fetchone() cursor.close() conn.close() return result -def GetRecordList(tablename,pkname,pklist): + +def GetRecordList(tablename, pkname, pklist): """ """ pkliststr = "" for pkid in pklist: - pkliststr+="%s,"%pkid - pkliststr = "(%s)"%pkliststr[:-1] - sql = """SELECT * FROM `%s` WHERE `%s` IN %s;"""%(tablename,pkname,pkliststr) + pkliststr += "%s," % pkid + pkliststr = "(%s)" % pkliststr[:-1] + sql = """SELECT * FROM `%s` WHERE `%s` IN %s;""" % ( + tablename, pkname, pkliststr) conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) result = cursor.fetchall() cursor.close() conn.close() return result + def DBTest(): sql = """SELECT * FROM tb_item WHERE characterId=1000001;""" conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) + cursor = conn.cursor(cursorclass=DictCursor) cursor.execute(sql) - result=cursor.fetchall() + result = cursor.fetchall() cursor.close() conn.close() return result -def getallkeys(key,mem): + +def getallkeys(key, mem): itemsinfo = mem.get_stats('items') itemindex = [] for items in itemsinfo: - itemindex += [ _key.split(':')[1] for _key in items[1].keys()] - s = set(itemindex) - itemss = [mem.get_stats('cachedump %s 0'%i) for i in s] + itemindex += [_key.split(':')[1] for _key in items[1].keys()] + s = set(itemindex) + itemss = [mem.get_stats('cachedump %s 0' % i) for i in s] allkeys = set([]) for item in itemss: for _item in item: @@ -244,6 +263,7 @@ def getallkeys(key,mem): allkeys = allkeys.union(nowlist) return allkeys -def getAllPkByFkInMEM(key,fk,mem): - + +def getAllPkByFkInMEM(key, fk, mem): + pass diff --git a/gfirefly/distributed/child.py b/gfirefly/distributed/child.py index 8b0959b..9d26401 100644 --- a/gfirefly/distributed/child.py +++ b/gfirefly/distributed/child.py @@ -1,48 +1,46 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @author: lan (www.9miao.com) ''' + + class Child(object): + '''子节点对象''' - - def __init__(self,name): + + def __init__(self, name): '''初始化子节点对象 ''' self._name = name self._transport = None - + def getName(self): '''获取子节点的名称''' return self._name - - def setTransport(self,transport): + + def setTransport(self, transport): '''设置子节点的通道''' self._transport = transport - - def callbackChild(self,*args,**kw): + + def callbackChild(self, *args, **kw): '''回调子节点的接口\n return a Defered Object (recvdata) ''' - return self.callbackChildForResult(*args,**kw) - - def callbackChildNotForResult(self,*args,**kw): + return self.callbackChildForResult(*args, **kw) + + def callbackChildNotForResult(self, *args, **kw): '''回调子节点的接口\n return a Defered Object (recvdata) ''' remote = self._transport.getRootObject() - remote.callRemoteNotForResult('callChild',*args,**kw) - - def callbackChildForResult(self,*args,**kw): + remote.callRemoteNotForResult('callChild', *args, **kw) + + def callbackChildForResult(self, *args, **kw): '''回调子节点的接口\n return a Defered Object (recvdata) ''' remote = self._transport.getRootObject() - recvdata = remote.callRemoteForResult('callChild',*args,**kw) + recvdata = remote.callRemoteForResult('callChild', *args, **kw) return recvdata - - - - - \ No newline at end of file diff --git a/gfirefly/distributed/manager.py b/gfirefly/distributed/manager.py index ab242f5..0a03a08 100644 --- a/gfirefly/distributed/manager.py +++ b/gfirefly/distributed/manager.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @@ -6,68 +6,70 @@ ''' from gtwisted.utils import log + class ChildsManager(object): + '''子节点管理器''' - + def __init__(self): '''初始化子节点管理器''' self._childs = {} - - def getChild(self,childname): + + def getChild(self, childname): '''根据节点的名称获取节点实例''' - for key,child in self._childs.items(): + for key, child in self._childs.items(): if child.getName() == childname: return self._childs[key] return None - - def addChild(self,child): + + def addChild(self, child): '''添加一个child节点\n @param child: Child object ''' key = child.getName() - if self._childs.has_key(key): - raise "child node %s exists"% key + if key in self._childs: + raise "child node %s exists" % key self._childs[key] = child - - def dropChild(self,child): + + def dropChild(self, child): '''删除一个child 节点\n - @param child: Child Object + @param child: Child Object ''' key = child.getName() try: del self._childs[key] - except Exception,e: + except Exception as e: log.msg(str(e)) - - def dropChildByID(self,childId): + + def dropChildByID(self, childId): '''删除一个child 节点\n - @param childId: Child ID + @param childId: Child ID ''' try: del self._childs[childId] - except Exception,e: + except Exception as e: log.msg(str(e)) - - def callChild(self,childname,*args,**kw): + + def callChild(self, childname, *args, **kw): '''调用子节点的接口\n @param childname: str 子节点的名称 ''' child = self.getChild(childname) if not child: - log.err("child %s doesn't exists"%childname) + log.err("child %s doesn't exists" % childname) return - return child.callbackChild(*args,**kw) - - def callChildNotForResult(self,childname,*args,**kw): + return child.callbackChild(*args, **kw) + + def callChildNotForResult(self, childname, *args, **kw): '''调用子节点的接口\n @param childname: str 子节点的名称 ''' - child = self._childs.get(childname,None) + child = self._childs.get(childname, None) if not child: - log.err("child %s doesn't exists"%childname) + log.err("child %s doesn't exists" % childname) return - child.callbackChildNotForResult(*args,**kw) - + child.callbackChildNotForResult(*args, **kw) + def getChildBYSessionId(self, session_id): """根据sessionID获取child节点信息 """ @@ -75,5 +77,3 @@ def getChildBYSessionId(self, session_id): if child._transport.transport.sessionno == session_id: return child return None - - \ No newline at end of file diff --git a/gfirefly/distributed/node.py b/gfirefly/distributed/node.py index fd61e80..6371ece 100644 --- a/gfirefly/distributed/node.py +++ b/gfirefly/distributed/node.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @@ -11,46 +11,46 @@ class BilateralClientProtocol(rpc.PBClientProtocl): - + def __init__(self, transport, factory): rpc.PBClientProtocl.__init__(self, transport, factory) - - def setProxyReference(self,pr): + + def setProxyReference(self, pr): """设置代理接口提供对象 """ self.reference = pr - + def getRemoteMethod(self, _name): """重写获取接口对象的方法，从代理接口提供对象中获取 """ - method = getattr(self.reference, "remote_%s"%_name) + method = getattr(self.reference, "remote_%s" % _name) return method - + def connectionLost(self, reason): rpc.PBClientProtocl.connectionLost(self, reason) self.factory - - + + class BilateralClientFactory(rpc.PBClientFactory): - + protocol = BilateralClientProtocol - - def __init__(self,ro=None): + + def __init__(self, ro=None): self.ro = ro rpc.PBClientFactory.__init__(self) - + def doconnectionLost(self): """node节点端开后的处理 """ if self.ro: self.ro.reconnect() - - - + + class RemoteObject(object): + '''远程调用对象''' - - def __init__(self,name,timeout=600): + + def __init__(self, name, timeout=600): '''初始化远程调用对象 @param port: int 远程分布服的端口号 @param rootaddr: 根节点服务器地址 @@ -60,55 +60,56 @@ def __init__(self,name,timeout=600): self._reference = ProxyReference() self._addr = None self._timeout = timeout - - def setName(self,name): + + def setName(self, name): '''设置节点的名称''' self._name = name - + def getName(self): '''获取节点的名称''' return self._name - - def connect(self,addr): + + def connect(self, addr): '''初始化远程调用对象''' self._addr = addr reactor.connectTCP(addr[0], addr[1], self._factory) self.takeProxy() - - def reconnect(self,addr=()): + + def reconnect(self, addr=()): '''重新连接''' if addr: self.connect(addr) else: self.connect(self._addr) - - def addServiceChannel(self,service): + + def addServiceChannel(self, service): '''设置引用对象''' self._reference.addService(service) - + def takeProxy(self): '''像远程服务端发送代理通道对象 ''' self._factory._protocol.setProxyReference(self._reference) deferedRemote = self._factory.getRootObject(timeout=self._timeout) - deferedRemote.callRemoteNotForResult('takeProxy',self._name) - - def callRemote(self,commandId,*args,**kw): + deferedRemote.callRemoteNotForResult('takeProxy', self._name) + + def callRemote(self, commandId, *args, **kw): """默认远程调用，等待结果放回 """ deferedRemote = self._factory.getRootObject(timeout=self._timeout) - return deferedRemote.callRemoteForResult('callTarget',commandId,*args,**kw) - - def callRemoteForResult(self,commandId,*args,**kw): + return deferedRemote.callRemoteForResult( + 'callTarget', commandId, *args, **kw) + + def callRemoteForResult(self, commandId, *args, **kw): '''远程调用，并等待结果放回 ''' deferedRemote = self._factory.getRootObject(timeout=self._timeout) - return deferedRemote.callRemoteForResult('callTarget',commandId,*args,**kw) - - def callRemoteNotForResult(self,commandId,*args,**kw): + return deferedRemote.callRemoteForResult( + 'callTarget', commandId, *args, **kw) + + def callRemoteNotForResult(self, commandId, *args, **kw): '''远程调用,不需要结果放回 ''' deferedRemote = self._factory.getRootObject(timeout=self._timeout) - return deferedRemote.callRemoteNotForResult('callTarget',commandId,*args,**kw) - - \ No newline at end of file + return deferedRemote.callRemoteNotForResult( + 'callTarget', commandId, *args, **kw) diff --git a/gfirefly/distributed/reference.py b/gfirefly/distributed/reference.py index 59d29e1..4c704d6 100644 --- a/gfirefly/distributed/reference.py +++ b/gfirefly/distributed/reference.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 @@ -8,21 +8,18 @@ class ProxyReference: + '''代理通道''' - + def __init__(self): '''初始化''' self._service = Service('proxy') - - def addService(self,service): + + def addService(self, service): '''添加一条服务通道''' self._service = service - - def remote_callChild(self, command,*arg,**kw): + + def remote_callChild(self, command, *arg, **kw): '''代理发送数据 ''' - return self._service.callTarget(command,*arg,**kw) - - - - \ No newline at end of file + return self._service.callTarget(command, *arg, **kw) diff --git a/gfirefly/distributed/root.py b/gfirefly/distributed/root.py index 7364e26..aeb29c6 100644 --- a/gfirefly/distributed/root.py +++ b/gfirefly/distributed/root.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-14 分布式根节点 @@ -9,85 +9,88 @@ from manager import ChildsManager from child import Child + class BilateralBroker(rpc.PBServerProtocl): - + def connectionLost(self, reason): clientID = self.transport.sessionno - log.msg("node [%d] lose"%clientID) + log.msg("node [%d] lose" % clientID) self.factory.root.dropChildSessionId(clientID) - - def remote_takeProxy(self,name): + + def remote_takeProxy(self, name): '''设置代理通道 @param name: 根节点的名称 ''' - self.factory.root.remote_takeProxy(name,self) - - def remote_callTarget(self,command,*args,**kw): + self.factory.root.remote_takeProxy(name, self) + + def remote_callTarget(self, command, *args, **kw): '''远程调用方法 @param commandId: int 指令号 @param data: str 调用参数 ''' - data = self.factory.root.remote_callTarget(command,*args,**kw) + data = self.factory.root.remote_callTarget(command, *args, **kw) return data - - + + class BilateralFactory(rpc.PBServerFactory): - + protocol = BilateralBroker - - def __init__(self,root): + + def __init__(self, root): rpc.PBServerFactory.__init__(self) self.root = root + class PBRoot: + '''PB 协议 ''' - - def __init__(self,dnsmanager = ChildsManager()): + + def __init__(self, dnsmanager=ChildsManager()): '''初始化根节点 ''' self.service = None self.childsmanager = dnsmanager - - def addServiceChannel(self,service): + + def addServiceChannel(self, service): '''添加服务通道 @param service: Service Object(In bilateral.services) ''' self.service = service - - def remote_takeProxy(self,name,transport): + + def remote_takeProxy(self, name, transport): '''设置代理通道 @param name: 根节点的名称 ''' - log.msg('node [%s] takeProxy ready'%name) + log.msg('node [%s] takeProxy ready' % name) child = Child(name) self.childsmanager.addChild(child) child.setTransport(transport) self.doChildConnect(name, transport) - - def doChildConnect(self,name,transport): + + def doChildConnect(self, name, transport): """当node节点连接时的处理 """ pass - - def remote_callTarget(self,command,*args,**kw): + + def remote_callTarget(self, command, *args, **kw): '''远程调用方法 @param commandId: int 指令号 @param data: str 调用参数 ''' - data = self.service.callTarget(command,*args,**kw) + data = self.service.callTarget(command, *args, **kw) return data - - def dropChild(self,*args,**kw): + + def dropChild(self, *args, **kw): '''删除子节点记录''' - self.childsmanager.dropChild(*args,**kw) - - def dropChildByID(self,childId): + self.childsmanager.dropChild(*args, **kw) + + def dropChildByID(self, childId): '''删除子节点记录 ''' self.doChildLostConnect(childId) self.childsmanager.dropChildByID(childId) - + def dropChildSessionId(self, session_id): '''删除子节点记录''' child = self.childsmanager.getChildBYSessionId(session_id) @@ -96,22 +99,22 @@ def dropChildSessionId(self, session_id): childname = child.getName() self.doChildLostConnect(childname) self.childsmanager.dropChildByID(childname) - - def doChildLostConnect(self,childname): + + def doChildLostConnect(self, childname): """当node节点连接时的处理 """ pass - - def callChild(self,key,*args,**kw): + + def callChild(self, key, *args, **kw): '''调用子节点的接口 @param childId: int 子节点的id return Defered Object ''' - return self.childsmanager.callChild(key,*args,**kw) - - def callChildNotForResult(self,childname,*args,**kw): + return self.childsmanager.callChild(key, *args, **kw) + + def callChildNotForResult(self, childname, *args, **kw): '''调用子节点的接口 @param childId: int 子节点的id return Defered Object ''' - self.childsmanager.callChildNotForResult(childname,*args,**kw) + self.childsmanager.callChildNotForResult(childname, *args, **kw) diff --git a/gfirefly/doc/source/conf.py b/gfirefly/doc/source/conf.py index 7ca1a9e..1317102 100644 --- a/gfirefly/doc/source/conf.py +++ b/gfirefly/doc/source/conf.py @@ -184,22 +184,22 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'gfirefly.tex', u'gfirefly Documentation', - u'lan', 'manual'), + ('index', 'gfirefly.tex', u'gfirefly Documentation', + u'lan', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -242,9 +242,9 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'gfirefly', u'gfirefly Documentation', - u'lan', 'gfirefly', 'One line description of project.', - 'Miscellaneous'), + ('index', 'gfirefly', u'gfirefly Documentation', + u'lan', 'gfirefly', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/gfirefly/management/__init__.py b/gfirefly/management/__init__.py index 38ac05d..e428e4e 100644 --- a/gfirefly/management/__init__.py +++ b/gfirefly/management/__init__.py @@ -1,47 +1,48 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-8 @author: lan (www.9miao.com) ''' -from gfirefly.management import commands +from gfirefly.management import commands import sys -class CommandError(Exception): + +class CommandError(Exception): + """ """ + def __str__(self): return "command error" + class Command: - - def __init__(self,subcommond,*args): + + def __init__(self, subcommond, *args): '''工具类指令 ''' self.subcommond = subcommond self.args = args - + def execute(self): ''' ''' try: - commmd = getattr(commands,self.subcommond ) + commmd = getattr(commands, self.subcommond) commmd.execute(*self.args) except: modes = dir(commands) - lines = ["\t"+m+"\n" for m in modes if not m.startswith("_")] + lines = ["\t" + m + "\n" for m in modes if not m.startswith("_")] firstline = "args error\nargs:\n" - print "".join([firstline,]+lines) - - + print "".join([firstline, ] + lines) + + def execute_commands(*args): ''' ''' - if len(args)<2: + if len(args) < 2: raise CommandError() subcommand = args[1] - comm = Command(subcommand,*tuple(args[2:])) + comm = Command(subcommand, *tuple(args[2:])) comm.execute() - - - \ No newline at end of file diff --git a/gfirefly/management/commands/__init__.py b/gfirefly/management/commands/__init__.py index 7a36be4..da25b2b 100644 --- a/gfirefly/management/commands/__init__.py +++ b/gfirefly/management/commands/__init__.py @@ -1,3 +1,3 @@ import createproject import stopservice -import reloadmodule \ No newline at end of file +import reloadmodule diff --git a/gfirefly/management/commands/createproject.py b/gfirefly/management/commands/createproject.py index a24687a..5c9fefc 100644 --- a/gfirefly/management/commands/createproject.py +++ b/gfirefly/management/commands/createproject.py @@ -1,35 +1,125 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-8 @author: lan (www.9miao.com) ''' -import sys,os -startmasterfile =['#coding:utf8\n', '\n', 'from gevent import monkey; monkey.patch_os()\n', '\n', 'if __name__=="__main__":\n', ' from gfirefly.master.master import Master\n', ' master = Master()\n', " master.config('config.json','appmain.py')\n", ' master.start()\n', ' \n', ' '] -configfile = ['{\n', +import sys +import os +startmasterfile = [ + '#coding:utf8\n', + '\n', + 'from gevent import monkey; monkey.patch_os()\n', + '\n', + 'if __name__=="__main__":\n', + ' from gfirefly.master.master import Master\n', + ' master = Master()\n', + " master.config('config.json','appmain.py')\n", + ' master.start()\n', + ' \n', + ' '] +configfile = ['{\n', '"master":{"roothost":"localhost","rootport":9999,"webport":9998},\n', - '"servers":{\n', '"net":{"netport":1000,"name":"gate","remoteport":[{"rootport":20001,"rootname":"gate"}],"app":"app.apptest"},\n', - '"gate":{"rootport":20001,"name":"gate"}\n', - '},\n', '"db":{\n', '"host":"localhost",\n', - '"user":"root",\n', '"passwd":"111",\n', - '"port":3306,\n', - '"db":"test",\n', - '"charset":"utf8"\n', - '},\n', - '"memcached":{\n', - '"urls":["127.0.0.1:11211"],\n', - '"hostname":"test"\n', - '}\n', - '}\n'] -appmainfile = ['#coding:utf8\n', '\n','from gevent import monkey; monkey.patch_os()\n', "import json,sys\n", 'from gfirefly.server.server import FFServer\n', '\n', 'if __name__=="__main__":\n', ' args = sys.argv\n', ' servername = None\n', ' config = None\n', ' if len(args)>2:\n', ' servername = args[1]\n', " config = json.load(open(args[2],'r'))\n", ' else:\n', ' raise ValueError\n', " dbconf = config.get('db')\n", " memconf = config.get('memcached')\n", " sersconf = config.get('servers',{})\n", " masterconf = config.get('master',{})\n", ' serconfig = sersconf.get(servername)\n', ' ser = FFServer()\n', ' ser.config(serconfig, servername=servername, dbconfig=dbconf, memconfig=memconf, masterconf=masterconf)\n', ' ser.start()\n', ' \n', ' '] -apptestfile = ['#coding:utf8\n', '\n', 'from gfirefly.server.globalobject import netserviceHandle\n', '\n', '@netserviceHandle\n', 'def echo_1(_conn,data):\n', ' return data\n', '\n', ' \n', '\n', '\n'] -clientfile = ['#coding:utf8\n', '\n', 'import time\n', '\n', 'from socket import AF_INET,SOCK_STREAM,socket\n', 'from thread import start_new\n', 'import struct\n', "HOST='localhost'\n", 'PORT=1000\n', 'BUFSIZE=1024\n', 'ADDR=(HOST , PORT)\n', 'client = socket(AF_INET,SOCK_STREAM)\n', 'client.connect(ADDR)\n', '\n', 'def sendData(sendstr,commandId):\n', ' HEAD_0 = chr(0)\n', ' HEAD_1 = chr(0)\n', ' HEAD_2 = chr(0)\n', ' HEAD_3 = chr(0)\n', ' ProtoVersion = chr(0)\n', ' ServerVersion = 0\n', ' sendstr = sendstr\n', " data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,\\\n", ' HEAD_3,ProtoVersion,ServerVersion,\\\n', ' len(sendstr)+4,commandId)\n', ' senddata = data+sendstr\n', ' return senddata\n', '\n', 'def resolveRecvdata(data):\n', " head = struct.unpack('!sssss3I',data[:17])\n", ' length = head[6]\n', ' data = data[17:17+length]\n', ' return data\n', '\n', 's1 = time.time()\n', '\n', 'def start():\n', ' while True:\n', ' time.sleep(1)\n', " client.sendall(sendData('asdfe',1))\n", 'start()','\n'] - - -def createfile(rootpath,path,filecontent): + '"servers":{\n', '"net":{"netport":1000,"name":"gate","remoteport":[{"rootport":20001,"rootname":"gate"}],"app":"app.apptest"},\n', + '"gate":{"rootport":20001,"name":"gate"}\n', + '},\n', '"db":{\n', '"host":"localhost",\n', + '"user":"root",\n', '"passwd":"111",\n', + '"port":3306,\n', + '"db":"test",\n', + '"charset":"utf8"\n', + '},\n', + '"memcached":{\n', + '"urls":["127.0.0.1:11211"],\n', + '"hostname":"test"\n', + '}\n', + '}\n'] +appmainfile = [ + '#coding:utf8\n', + '\n', + 'from gevent import monkey; monkey.patch_os()\n', + "import json,sys\n", + 'from gfirefly.server.server import FFServer\n', + '\n', + 'if __name__=="__main__":\n', + ' args = sys.argv\n', + ' servername = None\n', + ' config = None\n', + ' if len(args)>2:\n', + ' servername = args[1]\n', + " config = json.load(open(args[2],'r'))\n", + ' else:\n', + ' raise ValueError\n', + " dbconf = config.get('db')\n", + " memconf = config.get('memcached')\n", + " sersconf = config.get('servers',{})\n", + " masterconf = config.get('master',{})\n", + ' serconfig = sersconf.get(servername)\n', + ' ser = FFServer()\n', + ' ser.config(serconfig, servername=servername, dbconfig=dbconf, memconfig=memconf, masterconf=masterconf)\n', + ' ser.start()\n', + ' \n', + ' '] +apptestfile = [ + '#coding:utf8\n', + '\n', + 'from gfirefly.server.globalobject import netserviceHandle\n', + '\n', + '@netserviceHandle\n', + 'def echo_1(_conn,data):\n', + ' return data\n', + '\n', + ' \n', + '\n', + '\n'] +clientfile = [ + '#coding:utf8\n', + '\n', + 'import time\n', + '\n', + 'from socket import AF_INET,SOCK_STREAM,socket\n', + 'from thread import start_new\n', + 'import struct\n', + "HOST='localhost'\n", + 'PORT=1000\n', + 'BUFSIZE=1024\n', + 'ADDR=(HOST , PORT)\n', + 'client = socket(AF_INET,SOCK_STREAM)\n', + 'client.connect(ADDR)\n', + '\n', + 'def sendData(sendstr,commandId):\n', + ' HEAD_0 = chr(0)\n', + ' HEAD_1 = chr(0)\n', + ' HEAD_2 = chr(0)\n', + ' HEAD_3 = chr(0)\n', + ' ProtoVersion = chr(0)\n', + ' ServerVersion = 0\n', + ' sendstr = sendstr\n', + " data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,\\\n", + ' HEAD_3,ProtoVersion,ServerVersion,\\\n', + ' len(sendstr)+4,commandId)\n', + ' senddata = data+sendstr\n', + ' return senddata\n', + '\n', + 'def resolveRecvdata(data):\n', + " head = struct.unpack('!sssss3I',data[:17])\n", + ' length = head[6]\n', + ' data = data[17:17+length]\n', + ' return data\n', + '\n', + 's1 = time.time()\n', + '\n', + 'def start():\n', + ' while True:\n', + ' time.sleep(1)\n', + " client.sendall(sendData('asdfe',1))\n", + 'start()', + '\n'] + + +def createfile(rootpath, path, filecontent): ''' ''' - mfile = open(rootpath+'/'+path,'w') + mfile = open(rootpath + '/' + path, 'w') mfile.writelines(filecontent) mfile.close() @@ -38,23 +128,21 @@ def execute(*args): if not args: sys.stdout.write("command error \n") projectname = args[0] - sys.stdout.write("create dir %s \n"%projectname) + sys.stdout.write("create dir %s \n" % projectname) rootpath = projectname os.mkdir(rootpath) - createfile(rootpath,'startmaster.py',startmasterfile) - createfile(rootpath,'config.json',configfile) - createfile(rootpath,'appmain.py',appmainfile) - - rootpath = projectname+'/'+'app' + createfile(rootpath, 'startmaster.py', startmasterfile) + createfile(rootpath, 'config.json', configfile) + createfile(rootpath, 'appmain.py', appmainfile) + + rootpath = projectname + '/' + 'app' os.mkdir(rootpath) - createfile(rootpath,'__init__.py',[]) - createfile(rootpath,'apptest.py',apptestfile) - - rootpath = projectname+'/'+'tool' + createfile(rootpath, '__init__.py', []) + createfile(rootpath, 'apptest.py', apptestfile) + + rootpath = projectname + '/' + 'tool' os.mkdir(rootpath) - createfile(rootpath,'__init__.py',[]) - createfile(rootpath,'clienttest.py',clientfile) - + createfile(rootpath, '__init__.py', []) + createfile(rootpath, 'clienttest.py', clientfile) + sys.stdout.write("create success \n") - - \ No newline at end of file diff --git a/gfirefly/management/commands/reloadmodule.py b/gfirefly/management/commands/reloadmodule.py index 96715a0..80a2b5f 100644 --- a/gfirefly/management/commands/reloadmodule.py +++ b/gfirefly/management/commands/reloadmodule.py @@ -1,19 +1,21 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-12 @author: lan (www.9miao.com) ''' -import urllib,sys +import urllib +import sys + def execute(*args): """ """ if not args: - masterport =9998 + masterport = 9998 else: masterport = int(args[0]) - url = "http://localhost:%s/reloadmodule"%masterport + url = "http://localhost:%s/reloadmodule" % masterport try: response = urllib.urlopen(url) except: @@ -21,4 +23,4 @@ def execute(*args): if response: sys.stdout.write("reload module success \n") else: - sys.stdout.write("reload module failed \n") \ No newline at end of file + sys.stdout.write("reload module failed \n") diff --git a/gfirefly/management/commands/stopservice.py b/gfirefly/management/commands/stopservice.py index f2a1ee6..edc84f6 100644 --- a/gfirefly/management/commands/stopservice.py +++ b/gfirefly/management/commands/stopservice.py @@ -1,26 +1,28 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-11 @author: lan (www.9miao.com) ''' -import urllib,sys +import urllib +import sys + def execute(*args): """ """ if not args: - masterport =9998 + masterport = 9998 hostname = "localhost" else: - if len(args)>1: + if len(args) > 1: hostname = args[0] masterport = int(args[1]) else: hostname = "localhost" masterport = int(args[0]) - - url = "http://%s:%s/stop"%(hostname, masterport) + + url = "http://%s:%s/stop" % (hostname, masterport) try: response = urllib.urlopen(url) except: @@ -29,5 +31,3 @@ def execute(*args): sys.stdout.write("stop service success \n") else: sys.stdout.write("stop service failed \n") - - \ No newline at end of file diff --git a/gfirefly/master/master.py b/gfirefly/master/master.py index 8d677a1..484f817 100644 --- a/gfirefly/master/master.py +++ b/gfirefly/master/master.py @@ -1,13 +1,15 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @author: lan (www.9miao.com) ''' -import subprocess,json,sys +import subprocess +import json +import sys from gtwisted.core import reactor -from gfirefly.utils import services -from gfirefly.distributed.root import PBRoot,BilateralFactory +from gfirefly.utils import services +from gfirefly.distributed.root import PBRoot, BilateralFactory from gfirefly.server.globalobject import GlobalObject from gtwisted.utils import log from gfirefly.server.logobj import loogoo @@ -20,11 +22,11 @@ MASTER_SERVER_MODE = 3 - class Master: + """ """ - + def __init__(self): """ """ @@ -32,17 +34,17 @@ def __init__(self): self.mainpath = None self.root = None self.webroot = None - - def config(self,configpath,mainpath): + + def config(self, configpath, mainpath): """ """ self.configpath = configpath self.mainpath = mainpath - + def masterapp(self): """ """ - config = json.load(open(self.configpath,'r')) + config = json.load(open(self.configpath, 'r')) GlobalObject().json_config = config mastercnf = config.get('master') rootport = mastercnf.get('rootport') @@ -55,18 +57,18 @@ def masterapp(self): GlobalObject().root = self.root GlobalObject().webroot = self.webroot if masterlog: - log.addObserver(loogoo(masterlog))#日志处理 + log.addObserver(loogoo(masterlog)) # 日志处理 log.startLogging(sys.stdout) import webapp import rootapp reactor.listenWSGI(webport, self.webroot) reactor.listenTCP(rootport, BilateralFactory(self.root)) - + def start(self): ''' ''' sys_args = sys.argv - if len(sys_args)>2 and sys_args[1] == "single": + if len(sys_args) > 2 and sys_args[1] == "single": server_name = sys_args[2] if server_name == "master": mode = MASTER_SERVER_MODE @@ -75,22 +77,22 @@ def start(self): else: mode = MULTI_SERVER_MODE server_name = "" - + if mode == MULTI_SERVER_MODE: self.masterapp() - config = json.load(open(self.configpath,'r')) + config = json.load(open(self.configpath, 'r')) sersconf = config.get('servers') for sername in sersconf.keys(): - cmds = 'python %s %s %s'%(self.mainpath,sername,self.configpath) - subprocess.Popen(cmds,shell=True) + cmds = 'python %s %s %s' % ( + self.mainpath, sername, self.configpath) + subprocess.Popen(cmds, shell=True) reactor.run() elif mode == SINGLE_SERVER_MODE: - config = json.load(open(self.configpath,'r')) + config = json.load(open(self.configpath, 'r')) sername = server_name - cmds = 'python %s %s %s'%(self.mainpath,sername,self.configpath) - subprocess.Popen(cmds,shell=True) + cmds = 'python %s %s %s' % ( + self.mainpath, sername, self.configpath) + subprocess.Popen(cmds, shell=True) else: self.masterapp() reactor.run() - - diff --git a/gfirefly/master/rootapp.py b/gfirefly/master/rootapp.py index 9845147..5144820 100644 --- a/gfirefly/master/rootapp.py +++ b/gfirefly/master/rootapp.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-7 @@ -8,34 +8,45 @@ from gtwisted.utils import log -def _doChildConnect(name,transport): +def _doChildConnect(name, transport): """当server节点连接到master的处理 """ - server_config = GlobalObject().json_config.get('servers',{}).get(name,{}) - remoteport = server_config.get('remoteport',[]) + server_config = GlobalObject().json_config.get('servers', {}).get(name, {}) + remoteport = server_config.get('remoteport', []) child_host = transport.transport.address[0] root_list = [rootport.get('rootname') for rootport in remoteport] - GlobalObject().remote_map[name] = {"host":child_host,"root_list":root_list} - #通知有需要连的node节点连接到此root节点 - for servername,remote_list in GlobalObject().remote_map.items(): - remote_host = remote_list.get("host","") - remote_name_host = remote_list.get("root_list","") + GlobalObject().remote_map[name] = { + "host": child_host, + "root_list": root_list} + # 通知有需要连的node节点连接到此root节点 + for servername, remote_list in GlobalObject().remote_map.items(): + remote_host = remote_list.get("host", "") + remote_name_host = remote_list.get("root_list", "") if name in remote_name_host: - GlobalObject().root.callChild(servername,"remote_connect",name,remote_host) - #查看当前是否有可供连接的root节点 + GlobalObject().root.callChild( + servername, + "remote_connect", + name, + remote_host) + # 查看当前是否有可供连接的root节点 master_node_list = GlobalObject().remote_map.keys() for root_name in root_list: if root_name in master_node_list: root_host = GlobalObject().remote_map[root_name]['host'] - GlobalObject().root.callChild(name,"remote_connect",root_name,root_host) - + GlobalObject().root.callChild( + name, + "remote_connect", + root_name, + root_host) + + def _doChildLostConnect(childId): """ """ try: del GlobalObject().remote_map[childId] - except Exception,e: + except Exception as e: log.msg(str(e)) GlobalObject().root.doChildConnect = _doChildConnect -GlobalObject().root.doChildLostConnect = _doChildLostConnect \ No newline at end of file +GlobalObject().root.doChildLostConnect = _doChildLostConnect diff --git a/gfirefly/master/webapp.py b/gfirefly/master/webapp.py index d4fec57..ad0c493 100644 --- a/gfirefly/master/webapp.py +++ b/gfirefly/master/webapp.py @@ -1,22 +1,24 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-7 @author: lan (www.9miao.com) ''' from gtwisted.core import reactor -from gfirefly.server.globalobject import webserviceHandle,GlobalObject +from gfirefly.server.globalobject import webserviceHandle, GlobalObject reactor = reactor + @webserviceHandle('/stop') def stop(): '''stop service ''' for child in GlobalObject().root.childsmanager._childs.values(): child.callbackChildNotForResult('serverStop') - reactor.callLater(0.5,reactor.stop) + reactor.callLater(0.5, reactor.stop) return "stop" + @webserviceHandle('/reloadmodule') def reloadmodule(): '''reload module @@ -24,7 +26,3 @@ def reloadmodule(): for child in GlobalObject().root.childsmanager._childs.values(): child.callbackChildNotForResult('sreload') return "reload" - - - - diff --git a/gfirefly/netconnect/connection.py b/gfirefly/netconnect/connection.py index 440704e..888ce21 100644 --- a/gfirefly/netconnect/connection.py +++ b/gfirefly/netconnect/connection.py @@ -1,13 +1,16 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2014-2-23 @author: lan (www.9miao.com) ''' + class Connection: + ''' ''' + def __init__(self, _conn): ''' id 连接的ID @@ -15,16 +18,13 @@ def __init__(self, _conn): ''' self.id = _conn.transport.sessionno self.instance = _conn - + def loseConnection(self): '''断开与客户端的连接 ''' self.instance.transport.close() - - def safeToWriteData(self,topicID,msg): + + def safeToWriteData(self, topicID, msg): """发送消息 """ - self.instance.safeToWriteData(msg,topicID) - - - \ No newline at end of file + self.instance.safeToWriteData(msg, topicID) diff --git a/gfirefly/netconnect/datapack.py b/gfirefly/netconnect/datapack.py index 7cbdcef..7393cee 100644 --- a/gfirefly/netconnect/datapack.py +++ b/gfirefly/netconnect/datapack.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2014-2-23 @@ -7,7 +7,9 @@ from gtwisted.utils import log import struct + class DataPackError(Exception): + """An error occurred binding to an interface""" def __str__(self): @@ -17,10 +19,14 @@ def __str__(self): s = '%s.' % s return s + class DataPackProtoc: + """数据包协议 """ - def __init__(self,HEAD_0 = 0,HEAD_1=0,HEAD_2=0,HEAD_3=0,protoVersion= 0,serverVersion=0): + + def __init__(self, HEAD_0=0, HEAD_1=0, HEAD_2=0, + HEAD_3=0, protoVersion=0, serverVersion=0): '''初始化 @param HEAD_0: int 协议头0 @param HEAD_1: int 协议头1 @@ -35,53 +41,53 @@ def __init__(self,HEAD_0 = 0,HEAD_1=0,HEAD_2=0,HEAD_3=0,protoVersion= 0,serverVe self.HEAD_3 = HEAD_3 self.protoVersion = protoVersion self.serverVersion = serverVersion - + def setHEAD_0(self, HEAD_0): self.HEAD_0 = HEAD_0 - + def setHEAD_1(self, HEAD_1): self.HEAD_1 = HEAD_1 - + def setHEAD_2(self, HEAD_2): self.HEAD_2 = HEAD_2 - + def setHEAD_3(self, HEAD_3): self.HEAD_3 = HEAD_3 - + def setprotoVersion(self, protoVersion): self.protoVersion = protoVersion - + def setserverVersion(self, serverVersion): self.serverVersion = serverVersion - + def getHeadlength(self): """获取数据包的长度 """ return 17 - - def unpack(self,dpack): + + def unpack(self, dpack): '''解包 ''' try: - ud = struct.unpack('!sssss3I',dpack) - except struct.error,de: + ud = struct.unpack('!sssss3I', dpack) + except struct.error as de: log.err(de) - return {'result':False,'command':0,'length':0} + return {'result': False, 'command': 0, 'length': 0} HEAD_0 = ord(ud[0]) HEAD_1 = ord(ud[1]) HEAD_2 = ord(ud[2]) HEAD_3 = ord(ud[3]) protoVersion = ord(ud[4]) serverVersion = ud[5] - length = ud[6]-4 + length = ud[6] - 4 command = ud[7] - if HEAD_0 <>self.HEAD_0 or HEAD_1<>self.HEAD_1 or\ - HEAD_2<>self.HEAD_2 or HEAD_3<>self.HEAD_3 or\ - protoVersion<>self.protoVersion or serverVersion<>self.serverVersion: - return {'result':False,'command':0,'length':0} - return {'result':True,'command':command,'length':length} - - def pack(self,response,command): + if HEAD_0 != self.HEAD_0 or HEAD_1 != self.HEAD_1 or\ + HEAD_2 != self.HEAD_2 or HEAD_3 != self.HEAD_3 or\ + protoVersion != self.protoVersion or serverVersion != self.serverVersion: + return {'result': False, 'command': 0, 'length': 0} + return {'result': True, 'command': command, 'length': length} + + def pack(self, response, command): '''打包数据包 ''' HEAD_0 = chr(self.HEAD_0) @@ -90,12 +96,9 @@ def pack(self,response,command): HEAD_3 = chr(self.HEAD_3) protoVersion = chr(self.protoVersion) serverVersion = self.serverVersion - length = response.__len__()+4 + length = response.__len__() + 4 commandID = command - data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,HEAD_3,\ - protoVersion,serverVersion,length,commandID) + data = struct.pack('!sssss3I', HEAD_0, HEAD_1, HEAD_2, HEAD_3, + protoVersion, serverVersion, length, commandID) data = data + response return data - - - \ No newline at end of file diff --git a/gfirefly/netconnect/manager.py b/gfirefly/netconnect/manager.py index 9122804..00bb92a 100644 --- a/gfirefly/netconnect/manager.py +++ b/gfirefly/netconnect/manager.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2014-2-23 连接管理器 @@ -8,30 +8,32 @@ from gtwisted.utils import log from connection import Connection + class ConnectionManager: + ''' 连接管理器 @param _connections: dict {connID:conn Object}管理的所有连接 ''' - + def __init__(self): '''初始化 @param _connections: dict {connID:conn Object} ''' self._connections = {} - + def getNowConnCnt(self): '''获取当前连接数量''' return len(self._connections.items()) - + def addConnection(self, conn): '''加入一条连接 @param _conn: Conn object ''' _conn = Connection(conn) - if self._connections.has_key(_conn.id): + if _conn.id in self._connections: raise Exception("系统记录冲突") self._connections[_conn.id] = _conn - + def dropConnectionByID(self, connID): '''更加连接的id删除连接实例 @param connID: int 连接的id @@ -40,29 +42,27 @@ def dropConnectionByID(self, connID): del self._connections[connID] except Exception as e: log.msg(str(e)) - + def getConnectionByID(self, connID): """根据ID获取一条连接 @param connID: int 连接的id """ - return self._connections.get(connID,None) - - def loseConnection(self,connID): + return self._connections.get(connID, None) + + def loseConnection(self, connID): """根据连接ID主动端口与客户端的连接 """ conn = self.getConnectionByID(connID) if conn: conn.loseConnection() - - def pushObject(self,topicID , msg, sendList): + + def pushObject(self, topicID, msg, sendList): """主动推送消息 """ for target in sendList: try: conn = self.getConnectionByID(target) if conn: - conn.safeToWriteData(topicID,msg) - except Exception,e: + conn.safeToWriteData(topicID, msg) + except Exception as e: log.err(str(e)) - - diff --git a/gfirefly/netconnect/protoc.py b/gfirefly/netconnect/protoc.py index 76d96ea..2db54b3 100644 --- a/gfirefly/netconnect/protoc.py +++ b/gfirefly/netconnect/protoc.py @@ -1,122 +1,124 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2014-2-23 登陆服务器协议 @author: lan (www.9miao.com) ''' -from gtwisted.core import protocols,reactor +from gtwisted.core import protocols, reactor from gtwisted.utils import log from gfirefly.netconnect.manager import ConnectionManager from gfirefly.netconnect.datapack import DataPackProtoc reactor = reactor + class LiberateProtocol(protocols.BaseProtocol): + '''协议''' - + buff = "" - + def connectionMade(self): '''连接建立处理 ''' address = self.transport.getAddress() - log.msg('Client %d login in.[%s,%d]'%(self.transport.sessionno,\ - address[0],address[1])) + log.msg('Client %d login in.[%s,%d]' % (self.transport.sessionno, + address[0], address[1])) self.factory.connmanager.addConnection(self) self.factory.doConnectionMade(self) - - def connectionLost(self,reason): + + def connectionLost(self, reason): '''连接断开处理 ''' - log.msg('Client %d login out.'%(self.transport.sessionno)) + log.msg('Client %d login out.' % (self.transport.sessionno)) self.factory.doConnectionLost(self) self.factory.connmanager.dropConnectionByID(self.transport.sessionno) - - def safeToWriteData(self,data,command): + + def safeToWriteData(self, data, command): '''线程安全的向客户端发送数据 @param data: str 要向客户端写的数据 ''' if data is None: return - senddata = self.factory.produceResult(data,command) + senddata = self.factory.produceResult(data, command) self.transport.sendall(senddata) - + def dataReceived(self, data): - '''数据到达处理 @param data: str 客户端传送过来的数据 ''' - length = self.factory.dataprotocl.getHeadlength()#获取协议头的长度 + length = self.factory.dataprotocl.getHeadlength() # 获取协议头的长度 self.buff += data - while self.buff.__len__() >= length: + while self.buff.__len__() >= length: unpackdata = self.factory.dataprotocl.unpack(self.buff[:length]) if not unpackdata.get('result'): log.msg('illegal data package --') - self.factory.connmanager.loseConnection(self.transport.sessionno) + self.factory.connmanager.loseConnection( + self.transport.sessionno) break command = unpackdata.get('command') rlength = unpackdata.get('length') - request = self.buff[length:length+rlength] - if request.__len__()< rlength: + request = self.buff[length:length + rlength] + if request.__len__() < rlength: log.msg('some data lose') break - self.buff = self.buff[length+rlength:] - response = self.factory.doDataReceived(self,command,request) + self.buff = self.buff[length + rlength:] + response = self.factory.doDataReceived(self, command, request) if not response: continue self.safeToWriteData(response, command) - + + class LiberateFactory(protocols.ServerFactory): + '''协议工厂''' - + protocol = LiberateProtocol - - def __init__(self,dataprotocl=DataPackProtoc()): + + def __init__(self, dataprotocl=DataPackProtoc()): '''初始化 ''' protocols.ServerFactory.__init__(self) self.service = None self.connmanager = ConnectionManager() self.dataprotocl = dataprotocl - - def setDataProtocl(self,dataprotocl): + + def setDataProtocl(self, dataprotocl): ''' ''' self.dataprotocl = dataprotocl - - def doConnectionMade(self,conn): + + def doConnectionMade(self, conn): '''当连接建立时的处理''' pass - - def doConnectionLost(self,conn): + + def doConnectionLost(self, conn): '''连接断开时的处理''' pass - - def addServiceChannel(self,service): + + def addServiceChannel(self, service): '''添加服务通道''' self.service = service - - def doDataReceived(self,conn,commandID,data): + + def doDataReceived(self, conn, commandID, data): '''数据到达时的处理''' - response = self.service.callTarget(commandID,conn,data) + response = self.service.callTarget(commandID, conn, data) return response - - def produceResult(self,command,response): + + def produceResult(self, command, response): '''产生客户端需要的最终结果 @param response: str 分布式客户端获取的结果 ''' - return self.dataprotocl.pack(command,response) - - def loseConnection(self,connID): + return self.dataprotocl.pack(command, response) + + def loseConnection(self, connID): """主动端口与客户端的连接 """ self.connmanager.loseConnection(connID) - - def pushObject(self,topicID , msg, sendList): + + def pushObject(self, topicID, msg, sendList): '''服务端向客户端推消息 @param topicID: int 消息的主题id号 @param msg: 消息的类容，protobuf结构类型 @param sendList: 推向的目标列表(客户端id 列表) ''' self.connmanager.pushObject(topicID, msg, sendList) - - diff --git a/gfirefly/script/gfirefly-admin.py b/gfirefly/script/gfirefly-admin.py index 96087b6..24eef79 100644 --- a/gfirefly/script/gfirefly-admin.py +++ b/gfirefly/script/gfirefly-admin.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-8 @@ -9,4 +9,4 @@ if __name__ == "__main__": args = sys.argv - management.execute_commands(*args) \ No newline at end of file + management.execute_commands(*args) diff --git a/gfirefly/server/admin.py b/gfirefly/server/admin.py index 52ff36a..5e67e98 100644 --- a/gfirefly/server/admin.py +++ b/gfirefly/server/admin.py @@ -1,10 +1,10 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2014-2-24 @author: lan (www.9miao.com) ''' -from gfirefly.server.globalobject import GlobalObject,masterserviceHandle +from gfirefly.server.globalobject import GlobalObject, masterserviceHandle from gtwisted.core import reactor from gtwisted.utils import log @@ -18,9 +18,10 @@ def serverStop(): log.msg('stop') if GlobalObject().stophandler: GlobalObject().stophandler() - reactor.callLater(0.5,reactor.stop) + reactor.callLater(0.5, reactor.stop) return True + @masterserviceHandle def sreload(): """供master调用的接口：热更新模块 @@ -30,9 +31,9 @@ def sreload(): reload(GlobalObject().reloadmodule) return True + @masterserviceHandle def remote_connect(rname, rhost): """供master调用的接口：进行远程的rpc连接 """ GlobalObject().remote_connect(rname, rhost) - diff --git a/gfirefly/server/globalobject.py b/gfirefly/server/globalobject.py index ca1f363..cbf71bd 100644 --- a/gfirefly/server/globalobject.py +++ b/gfirefly/server/globalobject.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @@ -6,16 +6,18 @@ ''' from gfirefly.utils.singleton import Singleton + class GlobalObject: + """单例，存放了服务中的netfactory，root节点，所有的remote节点的句柄。 """ - + __metaclass__ = Singleton - + def __init__(self): - self.netfactory = None#net前端 - self.root = None#分布式root节点 - self.remote = {}#remote节点 + self.netfactory = None # net前端 + self.root = None # 分布式root节点 + self.remote = {} # remote节点 self.db = None self.stophandler = None self.webroot = None @@ -24,62 +26,68 @@ def __init__(self): self.remote_connect = None self.json_config = {} self.remote_map = {} - - def config(self,netfactory=None,root = None,remote=None,db=None): + + def config(self, netfactory=None, root=None, remote=None, db=None): """配置存放的对象实例 """ self.netfactory = netfactory self.root = root self.remote = remote self.db = db - + + def masterserviceHandle(target): """供master调用的接口描述符 """ GlobalObject().masterremote._reference._service.mapTarget(target) - + + def netserviceHandle(target): """供客户端连接调用的接口描述符 """ GlobalObject().netfactory.service.mapTarget(target) - + + def rootserviceHandle(target): """作为root节点，供remote节点调用的接口描述符 """ GlobalObject().root.service.mapTarget(target) - + + class webserviceHandle: + """这是一个修饰符对象,修饰http接口的连接地址。 """ - - def __init__(self,url,**kw): + + def __init__(self, url, **kw): """ @param url: str http 访问的路径 """ self._url = url self.kw = kw - - def __call__(self,cls): + + def __call__(self, cls): """ """ if self._url: child_name = self._url else: child_name = cls.__name__ - return GlobalObject().webroot.route(child_name,**self.kw)(cls) - + return GlobalObject().webroot.route(child_name, **self.kw)(cls) + - class remoteserviceHandle: + """作为remote节点，供某个root节点调用的接口描述符 """ - def __init__(self,remotename): + + def __init__(self, remotename): """ """ self.remotename = remotename - - def __call__(self,target): + + def __call__(self, target): """ """ - GlobalObject().remote[self.remotename]._reference._service.mapTarget(target) - + GlobalObject().remote[ + self.remotename]._reference._service.mapTarget(target) diff --git a/gfirefly/server/logobj.py b/gfirefly/server/logobj.py index 5002731..2beb78f 100644 --- a/gfirefly/server/logobj.py +++ b/gfirefly/server/logobj.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-6 @@ -10,15 +10,16 @@ class loogoo: + '''日志处理 ''' implements(log.ILogObserver) - - def __init__(self,logpath): + + def __init__(self, logpath): '''配置日志路径 ''' self.file = file(logpath, 'w') - + def __call__(self, eventDict): '''日志处理 ''' @@ -32,6 +33,12 @@ def __call__(self, eventDict): if text is None or level != 'ERROR': return nowdate = datetime.datetime.now() - self.file.write('['+str(nowdate)+']\n'+str(level)+ '\n\t' + text + '\r\n') + self.file.write( + '[' + + str(nowdate) + + ']\n' + + str(level) + + '\n\t' + + text + + '\r\n') self.file.flush() - diff --git a/gfirefly/server/server.py b/gfirefly/server/server.py index cb9de22..9b1f071 100644 --- a/gfirefly/server/server.py +++ b/gfirefly/server/server.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @@ -6,7 +6,7 @@ ''' from gfirefly.netconnect.protoc import LiberateFactory from flask import Flask -from gfirefly.distributed.root import PBRoot,BilateralFactory +from gfirefly.distributed.root import PBRoot, BilateralFactory from gfirefly.distributed.node import RemoteObject from gfirefly.dbentrust.dbpool import dbpool from gfirefly.dbentrust.memclient import mclient @@ -15,110 +15,122 @@ from gtwisted.utils import log from gtwisted.core import reactor from gfirefly.utils import services -import os,sys,affinity +import os +import sys +import affinity reactor = reactor + def serverStop(): """停止服务进程 """ log.msg('stop') if GlobalObject().stophandler: GlobalObject().stophandler() - reactor.callLater(0.5,reactor.stop) + reactor.callLater(0.5, reactor.stop) return True + class FFServer: + """抽象出的一个服务进程 """ - + def __init__(self): ''' ''' - self.netfactory = None#net前端 - self.root = None#分布式root节点 - self.webroot = None#http服务 - self.remote = {}#remote节点 + self.netfactory = None # net前端 + self.root = None # 分布式root节点 + self.webroot = None # http服务 + self.remote = {} # remote节点 self.master_remote = None self.db = None self.mem = None self.servername = None self.remoteportlist = [] - + def config(self, config, servername=None, dbconfig=None, - memconfig=None, masterconf=None): + memconfig=None, masterconf=None): '''配置服务器 ''' GlobalObject().json_config = config - netport = config.get('netport')#客户端连接 - webport = config.get('webport')#http连接 - rootport = config.get('rootport')#root节点配置 - self.remoteportlist = config.get('remoteport',[])#remote节点配置列表 + netport = config.get('netport') # 客户端连接 + webport = config.get('webport') # http连接 + rootport = config.get('rootport') # root节点配置 + self.remoteportlist = config.get('remoteport', []) # remote节点配置列表 if not servername: - servername = config.get('name')#服务器名称 - logpath = config.get('log')#日志 - hasdb = config.get('db')#数据库连接 - hasmem = config.get('mem')#memcached连接 - app = config.get('app')#入口模块名称 - cpuid = config.get('cpu')#绑定cpu - mreload = config.get('reload')#重新加载模块名称 + servername = config.get('name') # 服务器名称 + logpath = config.get('log') # 日志 + hasdb = config.get('db') # 数据库连接 + hasmem = config.get('mem') # memcached连接 + app = config.get('app') # 入口模块名称 + cpuid = config.get('cpu') # 绑定cpu + mreload = config.get('reload') # 重新加载模块名称 self.servername = servername - + if netport: self.netfactory = LiberateFactory() netservice = services.CommandService("netservice") self.netfactory.addServiceChannel(netservice) - reactor.listenTCP(netport,self.netfactory) - + reactor.listenTCP(netport, self.netfactory) + if webport: self.webroot = Flask("servername") GlobalObject().webroot = self.webroot reactor.listenWSGI(webport, self.webroot) - + if rootport: self.root = PBRoot() rootservice = services.Service("rootservice") self.root.addServiceChannel(rootservice) reactor.listenTCP(rootport, BilateralFactory(self.root)) - + for cnf in self.remoteportlist: rname = cnf.get('rootname') self.remote[rname] = RemoteObject(self.servername) - + if hasdb and dbconfig: log.msg(str(dbconfig)) dbpool.initPool(**dbconfig) - + if hasmem and memconfig: urls = memconfig.get('urls') hostname = str(memconfig.get('hostname')) mclient.connect(urls, hostname) - + if logpath: - log.addObserver(loogoo(logpath))#日志处理 + log.addObserver(loogoo(logpath)) # 日志处理 log.startLogging(sys.stdout) - + if cpuid: affinity.set_process_affinity_mask(os.getpid(), cpuid) - GlobalObject().config(netfactory = self.netfactory, root=self.root, - remote = self.remote) - + GlobalObject().config(netfactory=self.netfactory, root=self.root, + remote=self.remote) + if masterconf: masterport = masterconf.get('rootport') masterhost = masterconf.get('roothost') self.master_remote = RemoteObject(servername) - addr = ('localhost',masterport) if not masterhost else (masterhost,masterport) + addr = ( + 'localhost', + masterport) if not masterhost else ( + masterhost, + masterport) self.master_remote.connect(addr) GlobalObject().masterremote = self.master_remote import admin - + if app: __import__(app) if mreload: _path_list = mreload.split(".") - GlobalObject().reloadmodule = __import__(mreload,fromlist=_path_list[:1]) + GlobalObject().reloadmodule = __import__( + mreload, + fromlist=_path_list[ + :1]) GlobalObject().remote_connect = self.remote_connect - + def remote_connect(self, rname, rhost): """进行rpc的连接 """ @@ -127,17 +139,15 @@ def remote_connect(self, rname, rhost): if rname == _rname: rport = cnf.get('rootport') if not rhost: - addr = ('localhost',rport) + addr = ('localhost', rport) else: - addr = (rhost,rport) + addr = (rhost, rport) self.remote[rname].connect(addr) break - + def start(self): '''启动服务器 ''' - log.msg('[%s] started...'%self.servername) - log.msg('[%s] pid: %s'%(self.servername,os.getpid())) + log.msg('[%s] started...' % self.servername) + log.msg('[%s] pid: %s' % (self.servername, os.getpid())) reactor.run() - - diff --git a/gfirefly/test/test_distributed_node.py b/gfirefly/test/test_distributed_node.py index f68a8fd..c1e19bd 100644 --- a/gfirefly/test/test_distributed_node.py +++ b/gfirefly/test/test_distributed_node.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-17 @@ -14,10 +14,10 @@ reactor = reactor -addr = ('localhost',9090)#目标主机的地址 -remote = RemoteObject('test_node')#实例化远程调用对象 +addr = ('localhost', 9090) # 目标主机的地址 +remote = RemoteObject('test_node') # 实例化远程调用对象 -service = services.CommandService('reference')#实例化一条服务对象 +service = services.CommandService('reference') # 实例化一条服务对象 remote.addServiceChannel(service) @@ -27,22 +27,24 @@ def serviceHandle(target): ''' service.mapTarget(target) + @serviceHandle def printOK_1(data): print data print "############################" return "call printOK_01" - -def apptest(commandID,*args,**kw): - d = remote.callRemote(commandID,*args,**kw) - print "apptest result:",d + + +def apptest(commandID, *args, **kw): + d = remote.callRemote(commandID, *args, **kw) + print "apptest result:", d return d + def startClient(): - reactor.callLater(3,apptest,'printData1',u"node测试1",u"node测试2") - remote.connect(addr)#连接远程主机 + reactor.callLater(3, apptest, 'printData1', u"node测试1", u"node测试2") + remote.connect(addr) # 连接远程主机 reactor.run() -if __name__=='__main__': +if __name__ == '__main__': startClient() - diff --git a/gfirefly/test/test_distributed_root.py b/gfirefly/test/test_distributed_root.py index bc5619a..ee6a30f 100644 --- a/gfirefly/test/test_distributed_root.py +++ b/gfirefly/test/test_distributed_root.py @@ -1,17 +1,17 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-17 @author: lan (www.9miao.com) ''' -from gfirefly.utils import services -from gfirefly.distributed.root import PBRoot,BilateralFactory +from gfirefly.utils import services +from gfirefly.distributed.root import PBRoot, BilateralFactory from gtwisted.core import reactor from gtwisted.utils import log import sys reactor = reactor log.startLogging(sys.stdout) - + root = PBRoot() ser = services.Service('test') root.addServiceChannel(ser) @@ -23,20 +23,22 @@ def serviceHandle(target): ''' ser.mapTarget(target) + @serviceHandle -def printData1(data,data1): - print data,data1 +def printData1(data, data1): + print data, data1 print "############################" - d = root.callChild("test_node",1,u'Root测试') + d = root.callChild("test_node", 1, u'Root测试') return data + @serviceHandle -def printData2(data,data1): - print data,data1 +def printData2(data, data1): + print data, data1 print "############################" - d = root.callChild("test_node",1,u'Root测试') + d = root.callChild("test_node", 1, u'Root测试') return data -if __name__=='__main__': +if __name__ == '__main__': reactor.listenTCP(9090, BilateralFactory(root)) reactor.run() diff --git a/gfirefly/test/test_netconnect_client.py b/gfirefly/test/test_netconnect_client.py index 1790e09..224dc40 100644 --- a/gfirefly/test/test_netconnect_client.py +++ b/gfirefly/test/test_netconnect_client.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-12 @@ -6,17 +6,18 @@ ''' import time -from socket import AF_INET,SOCK_STREAM,socket +from socket import AF_INET, SOCK_STREAM, socket from thread import start_new import struct -HOST='localhost' -PORT=1000 -BUFSIZE=1024 -ADDR=(HOST , PORT) -client = socket(AF_INET,SOCK_STREAM) +HOST = 'localhost' +PORT = 1000 +BUFSIZE = 1024 +ADDR = (HOST, PORT) +client = socket(AF_INET, SOCK_STREAM) client.connect(ADDR) -def sendData(sendstr,commandId): + +def sendData(sendstr, commandId): HEAD_0 = chr(0) HEAD_1 = chr(0) HEAD_2 = chr(0) @@ -24,27 +25,28 @@ def sendData(sendstr,commandId): ProtoVersion = chr(0) ServerVersion = 0 sendstr = sendstr - data = struct.pack('!sssss3I',HEAD_0,HEAD_1,HEAD_2,\ - HEAD_3,ProtoVersion,ServerVersion,\ - len(sendstr)+4,commandId) - senddata = data+sendstr + data = struct.pack('!sssss3I', HEAD_0, HEAD_1, HEAD_2, + HEAD_3, ProtoVersion, ServerVersion, + len(sendstr) + 4, commandId) + senddata = data + sendstr return senddata + def resolveRecvdata(data): - head = struct.unpack('!sssss3I',data[:17]) + head = struct.unpack('!sssss3I', data[:17]) length = head[6] - data = data[17:17+length] + data = data[17:17 + length] return data s1 = time.time() + def start(): while True: -# time.sleep(1) - client.sendall(sendData('0',1)) + # time.sleep(1) + client.sendall(sendData('0', 1)) # data = client.recv(1024) # print data - #print resolveRecvdata(data) + # print resolveRecvdata(data) start() - diff --git a/gfirefly/test/test_netconnect_server.py b/gfirefly/test/test_netconnect_server.py index fd3b313..8254e72 100644 --- a/gfirefly/test/test_netconnect_server.py +++ b/gfirefly/test/test_netconnect_server.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-10-3 @@ -6,7 +6,7 @@ ''' import sys from gtwisted.core import reactor - + from gtwisted.utils import log from gfirefly.utils import services from gfirefly.netconnect.protoc import LiberateFactory @@ -14,39 +14,42 @@ # reactor = reactor service = services.CommandService("loginService") + def serviceHandle(target): '''服务处理 @param target: func Object ''' service.mapTarget(target) - + factory = LiberateFactory() + def doConnectionLost(conn): print conn.transport factory.doConnectionLost = doConnectionLost + def serverstart(): '''服务配置 ''' log.startLogging(sys.stdout) factory.addServiceChannel(service) - reactor.callLater(10,factory.pushObject,111,'asdfe',[0,1]) + reactor.callLater(10, factory.pushObject, 111, 'asdfe', [0, 1]) # reactor.callLater(15,factory.loseConnection,0) - reactor.listenTCP(1000,factory) + reactor.listenTCP(1000, factory) reactor.run() - + a = 0 - + + @serviceHandle -def echo_1(_conn,data): +def echo_1(_conn, data): global a - a+=1 + a += 1 print a return "0" if __name__ == "__main__": - - serverstart() + serverstart() diff --git a/gfirefly/test/test_server_apptest.py b/gfirefly/test/test_server_apptest.py index 716710c..3b7a32f 100644 --- a/gfirefly/test/test_server_apptest.py +++ b/gfirefly/test/test_server_apptest.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @@ -6,15 +6,14 @@ ''' from gfirefly.server.globalobject import GlobalObject + def netserviceHandle(target): '''服务处理 @param target: func Object ''' GlobalObject().netfactory.service.mapTarget(target) - -@netserviceHandle -def echo_111(_conn,data): - return data - +@netserviceHandle +def echo_111(_conn, data): + return data diff --git a/gfirefly/test/test_server_main.py b/gfirefly/test/test_server_main.py index e65bd96..581cdc0 100644 --- a/gfirefly/test/test_server_main.py +++ b/gfirefly/test/test_server_main.py @@ -1,28 +1,31 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-6 @author: lan (www.9miao.com) ''' -import json,sys +import json +import sys from gfirefly.server.server import FFServer -if __name__=="__main__": +if __name__ == "__main__": args = sys.argv servername = None config = None - if len(args)>2: + if len(args) > 2: servername = args[1] - config = json.load(open(args[2],'r')) + config = json.load(open(args[2], 'r')) else: raise ValueError dbconf = config.get('db') memconf = config.get('memcached') - sersconf = config.get('servers',{}) - masterconf = config.get('master',{}) + sersconf = config.get('servers', {}) + masterconf = config.get('master', {}) serconfig = sersconf.get(servername) ser = FFServer() - ser.config(serconfig, dbconfig=dbconf, memconfig=memconf,masterconf=masterconf) + ser.config( + serconfig, + dbconfig=dbconf, + memconfig=memconf, + masterconf=masterconf) ser.start() - - \ No newline at end of file diff --git a/gfirefly/test/test_server_master.py b/gfirefly/test/test_server_master.py index c79b27b..7069639 100644 --- a/gfirefly/test/test_server_master.py +++ b/gfirefly/test/test_server_master.py @@ -1,17 +1,16 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-8-2 @author: lan (www.9miao.com) ''' - + + def println(a): print a -if __name__=="__main__": +if __name__ == "__main__": from gfirefly.master.master import Master master = Master() - master.config('config.json','test_server_main.py') + master.config('config.json', 'test_server_main.py') master.start() - - \ No newline at end of file diff --git a/gfirefly/utils/interfaces.py b/gfirefly/utils/interfaces.py index eb05a1d..a305a42 100644 --- a/gfirefly/utils/interfaces.py +++ b/gfirefly/utils/interfaces.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2013-10-17 @@ -9,20 +9,16 @@ class IDataPackProtoc(Interface): - + def getHeadlength(): """获取数据包的长度 """ pass - + def unpack(): '''解包 ''' - + def pack(): '''打包数据包 ''' - - - - diff --git a/gfirefly/utils/services.py b/gfirefly/utils/services.py index c4b8a3b..1e7bd54 100644 --- a/gfirefly/utils/services.py +++ b/gfirefly/utils/services.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2011-1-3 服务类 @@ -8,33 +8,34 @@ class Service(object): - """A remoting service - + + """A remoting service + attributes: ============ * name - string, service name. - * runstyle + * runstyle """ def __init__(self, name): self._name = name self.unDisplay = set() - self._targets = {} # Keeps track of targets internally + self._targets = {} # Keeps track of targets internally def __iter__(self): return self._targets.itervalues() - - def addUnDisplayTarget(self,command): + + def addUnDisplayTarget(self, command): '''Add a target unDisplay when client call it.''' self.unDisplay.add(command) def mapTarget(self, target): """Add a target to the service.""" key = target.__name__ - if self._targets.has_key(key): + if key in self._targets: exist_target = self._targets.get(key) raise "target [%d] Already exists,\ - Conflict between the %s and %s"%(key,exist_target.__name__,target.__name__) + Conflict between the %s and %s" % (key, exist_target.__name__, target.__name__) self._targets[key] = target def unMapTarget(self, target): @@ -42,17 +43,17 @@ def unMapTarget(self, target): key = target.__name__ if key in self._targets: del self._targets[key] - - def unMapTargetByKey(self,targetKey): + + def unMapTargetByKey(self, targetKey): """Remove a target from the service.""" del self._targets[targetKey] - + def getTarget(self, targetKey): """Get a target from the service by name.""" target = self._targets.get(targetKey, None) return target - - def callTarget(self,targetKey,*args,**kw): + + def callTarget(self, targetKey, *args, **kw): '''call Target @param conn: client connection @param targetKey: target ID @@ -60,37 +61,33 @@ def callTarget(self,targetKey,*args,**kw): ''' target = self.getTarget(targetKey) if not target: - log.err('the command '+str(targetKey)+' not Found on service') + log.err('the command ' + str(targetKey) + ' not Found on service') return None if targetKey not in self.unDisplay: - log.msg("call method %s on service[single]"%target.__name__) - response = target(*args,**kw) + log.msg("call method %s on service[single]" % target.__name__) + response = target(*args, **kw) return response class CommandService(Service): - """A remoting service + + """A remoting service According to Command ID search target """ + def mapTarget(self, target): """Add a target to the service. """ key = int((target.__name__).split('_')[-1]) - if self._targets.has_key(key): + if key in self._targets: exist_target = self._targets.get(key) raise "target [%d] Already exists,\ - Conflict between the %s and %s"%(key,exist_target.__name__,target.__name__) + Conflict between the %s and %s" % (key, exist_target.__name__, target.__name__) self._targets[key] = target - + def unMapTarget(self, target): """Remove a target from the service. """ key = int((target.__name__).split('_')[-1]) if key in self._targets: del self._targets[key] - - - - - - \ No newline at end of file diff --git a/gfirefly/utils/singleton.py b/gfirefly/utils/singleton.py index 8da5e56..8964509 100644 --- a/gfirefly/utils/singleton.py +++ b/gfirefly/utils/singleton.py @@ -1,14 +1,15 @@ # coding: utf-8 + class Singleton(type): + """Singleton Metaclass""" - + def __init__(self, name, bases, dic): super(Singleton, self).__init__(name, bases, dic) self.instance = None - + def __call__(self, *args, **kwargs): if self.instance is None: self.instance = super(Singleton, self).__call__(*args, **kwargs) return self.instance - \ No newline at end of file diff --git a/gfirefly/utils/version.py b/gfirefly/utils/version.py index add699c..734d0c5 100644 --- a/gfirefly/utils/version.py +++ b/gfirefly/utils/version.py @@ -1,4 +1,4 @@ -#coding:utf8 +# coding:utf8 ''' Created on 2014年2月17日 @@ -10,6 +10,7 @@ import os import subprocess + def get_version(version=None): "Returns a PEP 386-compliant version number from VERSION." if version is None: @@ -43,8 +44,8 @@ def get_git_changeset(): """ repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) git_log = subprocess.Popen('git log --pretty=format:%ct --quiet -1 HEAD', - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True, cwd=repo_dir, universal_newlines=True) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, cwd=repo_dir, universal_newlines=True) timestamp = git_log.communicate()[0] try: timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))