python 内存NoSQL数据库
来自于网络，经过修改，秉承Open Source精神，回馈网络!
#!/usr/bin/python
#-*- coding: UTF-8 -*-
#
# memdb.py
# python memory db
#
# 2015-12
########################################################################
# The MIT License (MIT)
# http://opensource.org/licenses/MIT
#
# Copyright (c) 2015 copyright cheungmine
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject
# to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
########################################################################
from multiprocessing import RLock
# sync for threading
class ObjectSync:
def __init__(self, name):
self.refcount = 0
self.synclock = RLock()
self.keyname = name
def Lock(self):
self.synclock.acquire()
self.refcount = self.refcount + 1
def Unlock(self):
self.refcount = self.refcount - 1
self.synclock.release()
class ObjectSyncFactory:
def __init__(self):
self.globalLock = ObjectSync("")
self.rowlocks = {}
def __RemoveLock(self, sem, keyname):
self.globalLock.Lock()
self.rowlocks[keyname] = None
self.globalLock.Unlock()
def GetLock(self, tablename, key):
keyname = tablename + "," + str(key)
self.globalLock.Lock()
l = None
try:
l = self.rowlocks[keyname]
if l == None:
self.rowlocks[keyname] = ObjectSync(keyname)
l = self.rowlocks[keyname]
except:
self.rowlocks[keyname] = ObjectSync(keyname)
l = self.rowlocks[keyname]
self.globalLock.Unlock()
return l
class PairGuard:
def __init__(self, factory, sem):
self.syncfactory = factory
self.host = sem
self.host.Lock()
def __del__(self):
self.host.Unlock()
if self.host.refcount == 0 :
self.syncfactory._ObjectSyncFactory__RemoveLock(self.host, self.host.keyname)
########################################
# Database table
class MemTable:
def __init__(self):
self.rows = {}
self.tableLock = ObjectSync("")
def GetRowCount(self):
return len(self.rows)
def DeleteAll(self):
self.tableLock.Lock()
self.rows = {}
self.tableLock.Unlock()
def __DeleteAll(self):
self.rows = {}
def GetAllValue(self):
return self.rows
# throw KeyError if key not found.
def GetValue(self, key):
return self.rows[key]
# not exist: Add
# exist: Update
def AddValue(self, key, value):
self.tableLock.Lock()
self.rows[key] = value
self.tableLock.Unlock()
def __AddValue(self, key, value):
self.rows[key] = value
def DelValue(self, key):
self.AddValue(key,None)
def __DelValue(self, key):
self._MemTable__AddValue(key, None)
########################################
# MemDB
class MemDB:
def __init__(self):
self.tables = {}
self.syncFactory = ObjectSyncFactory()
# is not thread safed
def CreateTable(self, tablename):
self.tables[tablename] = MemTable()
# is not thread safed
def DropTable(self, tablename):
self.tables[tablename] = None
def GetValue(self, tablename, key):
mt = self.tables[tablename]
PairGuard(self.syncFactory, self.syncFactory.GetLock(tablename, key))
return mt.GetValue(key)
def AddValue(self, tablename, key, value):
mt = self.tables[tablename]
PairGuard(self.syncFactory, self.syncFactory.GetLock(tablename, key))
mt.AddValue(key, value)
def DelValue(self, tablename, key):
mt = self.tables[tablename]
PairGuard(self.syncFactory, self.syncFactory.GetLock(tablename, key))
mt.DelValue(key)
def __GetValue(self, tablename, key):
mt = self.tables[tablename]
return mt.GetValue(key)
def __AddValue(self, tablename, key, value):
mt = self.tables[tablename]
mt._MemTable__AddValue(key, value)
def __DelValue(self, tablename, key):
mt = self.tables[tablename]
mt._MemTable__DelValue(key)
class Transaction:
def __init__(self, conn):
self.dbconn = conn
self.logs = []
def Commit(self):
syncs = []
tables = {}
for p in self.logs:
tables[p[0]] = True
for name in tables:
syncTable = self.dbconn.memdb.syncFactory.GetLock(name, 'table')
syncs.append( (syncTable.keyname, syncTable) )
syncs.sort()
#lock
guards = []
for sync in syncs:
guards.append(PairGuard(self.dbconn.memdb.syncFactory, sync[1]))
#commit
self.logs.reverse()
while True:
if len(self.logs) == 0:
break
p = self.logs.pop()
self.dbconn.memdb._MemDB__AddValue(p[0], p[1], p[2])
#unlock
guards.reverse()
while True:
if len(guards) == 0:
break
guards.pop()
self.dbconn._MemDBConnect__EndTransaction()
def Rollback(self):
self.dbconn._MemDBConnect__EndTransaction()
def LogPoint(self, tablename, key, value):
self.logs.append((tablename, key, value))
class MemDBConnect:
def __init__(self, db):
self.memdb = db
self.bTransaction = False
self.trans = None
def BeginTransaction(self):
self.bTransaction = True
self.trans = Transaction(self)
return self.trans
def __EndTransaction(self):
self.bTransaction = False
self.trans = None
def CommitTransaction(self):
if self.bTransaction:
self.bTransaction = False
ts = self.trans
self.trans = None
if ts:
ts.Commit()
def RollbackTransaction(self):
if self.bTransaction:
self.bTransaction = False
ts = self.trans
self.trans = None
if ts:
ts.Rollback()
# not thread safe
def CreateTable(self, tablename):
self.memdb.CreateTable(tablename)
# not thread safe
def DropTable(self, tablename):
self.memdb.DropTable(tablename)
# not thread safe
def HasTable(self, tablename):
if self.memdb.tables.get(tablename):
return True
else:
return False
def GetValue(self, tablename, key):
if self.bTransaction:
return self.memdb._MemDB__GetValue(tablename, key)
else:
return self.memdb.GetValue(tablename, key)
def AddValue(self, tablename, key, value):
if self.bTransaction:
self.trans.LogPoint(tablename, key, value)
else:
self.memdb.AddValue(tablename, key, value)
def DelValue(self, tablename, key):
if self.bTransaction:
self.trans.LogPoint(tablename, key, None)
else:
self.memdb.DelValue(tablename, key)
def QueryTablesNothrow(self):
tables = []
try:
self.BeginTransaction()
for tablename,_ in self.memdb.tables.items():
tables.append(tablename)
self.CommitTransaction()
except:
tables = []
self.RollbackTransaction()
finally:
return tables
def QueryTableKeysNothrow(self, tablename):
keys = []
try:
self.BeginTransaction()
if self.HasTable(tablename):
rows_dict = self.memdb.tables[tablename].rows
for key, _ in rows_dict.items():
keys.append(key)
self.CommitTransaction()
except:
keys = []
self.RollbackTransaction()
finally:
return keys
def CreateTableNothrow(self, tablename):
try:
self.BeginTransaction()
if not self.HasTable(tablename):
self.memdb.CreateTable(tablename)
self.CommitTransaction()
except:
self.RollbackTransaction()
finally:
pass
def DropTableNothrow(self, tablename):
try:
self.BeginTransaction()
if self.HasTable(tablename):
self.memdb.DropTable(tablename)
self.CommitTransaction()
except:
self.RollbackTransaction()
finally:
pass
def GetValueNothrow(self, tablename, key, defaultvalue):
result = defaultvalue
try:
self.BeginTransaction()
result = self.GetValue(tablename, key)
self.CommitTransaction()
except:
self.RollbackTransaction()
finally:
return result
def AddValueNothrow(self, tablename, key, value):
result = False
try:
self.BeginTransaction()
self.AddValue(tablename, key, value)
self.CommitTransaction()
result = True
except:
self.RollbackTransaction()
finally:
return result
def DelValueNothrow(self, tablename, key):
result = False
try:
self.BeginTransaction()
self.DelValue(tablename, key)
self.CommitTransaction()
result = True
except:
self.RollbackTransaction()
finally:
return result
def AppendValueListNothrow(self, tablename, key, value, non_repeated_value):
try:
self.BeginTransaction()
if self.HasTable(tablename):
try:
values = self.GetValue(tablename, key)
if non_repeated_value:
if value not in values:
values.append(value)
self.AddValue(tablename, key, values)
else:
values.append(value)
self.AddValue(tablename, key, values)
except KeyError:
self.AddValue(tablename, key, [value])
finally:
self.CommitTransaction()
except:
self.RollbackTransaction()
finally:
pass
def AppendValueListMultiNothrow(self, tablenames, keys, values, non_repeated_values):
try:
self.BeginTransaction()
for i in range(0, len(tablenames)):
t, k, v, nrv = tablenames[i], keys[i], values[i], non_repeated_values[i]
if self.HasTable(t):
try:
vals = self.GetValue(t, k)
if nrv:
if v not in vals:
vals.append(v)
self.AddValue(t, k, vals)
else:
vals.append(v)
self.AddValue(t, k, vals)
except KeyError:
self.AddValue(t, k, [v])
self.CommitTransaction()
except:
self.RollbackTransaction()
finally:
pass
用法1：
可以直接嵌入到python中：
def test():
db = MemDB()
tname = "table1"
db.CreateTable(tname)
#for i in range(100000):
# db.AddValue(tname,i,"sdfsd")
db.AddValue(tname,11,"sdfsd")
print db.GetValue(tname, 11)
db.AddValue(tname,11,"dddddd")
print db.GetValue(tname,11)
db.AddValue(tname,12,"dsfdsfd")
print db.GetValue(tname,12)
conn = MemDBConnect(db)
t = conn.BeginTransaction()
for i in range(100000):
conn.AddValue(tname,i,"sdfsd")
conn.AddValue(tname,12,"sfdas")
conn.AddValue(tname,12,"ddddd")
t.Commit()
print db.GetValue(tname,12)
