fix: 全局连接数限制+空闲连接回收
- 添加全局信号量限制总连接数(400),防止超过MariaDB上限(500) - 实现空闲连接自动回收(IDLE_TIMEOUT=300秒) - 修复_del_sqlor中连接关闭时释放全局信号量
This commit is contained in:
parent
3a752f3719
commit
f6c6b4d913
@ -13,7 +13,7 @@ from appPublic.Singleton import SingletonDecorator
|
|||||||
from appPublic.myjson import loadf
|
from appPublic.myjson import loadf
|
||||||
from appPublic.jsonConfig import getConfig
|
from appPublic.jsonConfig import getConfig
|
||||||
from appPublic.rc4 import unpassword
|
from appPublic.rc4 import unpassword
|
||||||
from appPublic.log import exception
|
from appPublic.log import exception, debug
|
||||||
from appPublic.event_dispatcher import EventDispatcher
|
from appPublic.event_dispatcher import EventDispatcher
|
||||||
|
|
||||||
import threading
|
import threading
|
||||||
@ -25,6 +25,16 @@ from .aiosqliteor import Aiosqliteor
|
|||||||
from .mysqlor import MySqlor
|
from .mysqlor import MySqlor
|
||||||
from .aiopostgresqlor import AioPostgresqlor
|
from .aiopostgresqlor import AioPostgresqlor
|
||||||
|
|
||||||
|
# Global semaphore to limit total connections across all pools
|
||||||
|
_GLOBAL_CONN_LIMIT = 400 # Leave headroom below MariaDB max_connections=500
|
||||||
|
_global_conn_sema = None
|
||||||
|
|
||||||
|
def _get_global_sema():
|
||||||
|
global _global_conn_sema
|
||||||
|
if _global_conn_sema is None:
|
||||||
|
_global_conn_sema = asyncio.Semaphore(_GLOBAL_CONN_LIMIT)
|
||||||
|
return _global_conn_sema
|
||||||
|
|
||||||
def sqlorFactory(dbdesc):
|
def sqlorFactory(dbdesc):
|
||||||
driver = dbdesc.get('driver',dbdesc)
|
driver = dbdesc.get('driver',dbdesc)
|
||||||
def findSubclass(name,klass):
|
def findSubclass(name,klass):
|
||||||
@ -41,23 +51,33 @@ def sqlorFactory(dbdesc):
|
|||||||
return k(dbdesc=dbdesc.kwargs)
|
return k(dbdesc=dbdesc.kwargs)
|
||||||
|
|
||||||
class SqlorPool:
|
class SqlorPool:
|
||||||
|
# Idle timeout: connections unused for this long will be closed
|
||||||
|
IDLE_TIMEOUT = 300 # 5 minutes
|
||||||
|
|
||||||
def __init__(self, create_func, maxconn=100):
|
def __init__(self, create_func, maxconn=100):
|
||||||
self.sema = asyncio.Semaphore(maxconn)
|
self.sema = asyncio.Semaphore(maxconn)
|
||||||
self.create_func = create_func
|
self.create_func = create_func
|
||||||
self.sqlors = []
|
self.sqlors = []
|
||||||
|
|
||||||
async def _new_sqlor(self):
|
async def _new_sqlor(self):
|
||||||
sqlor = await self.create_func()
|
global_sema = _get_global_sema()
|
||||||
await sqlor.connect()
|
await global_sema.acquire()
|
||||||
x = DictObject(**{
|
try:
|
||||||
'used': True,
|
sqlor = await self.create_func()
|
||||||
'use_at': time.time(),
|
await sqlor.connect()
|
||||||
'sqlor':sqlor
|
x = DictObject(**{
|
||||||
})
|
'used': True,
|
||||||
self.sqlors.append(x)
|
'use_at': time.time(),
|
||||||
return x
|
'sqlor':sqlor
|
||||||
|
})
|
||||||
|
self.sqlors.append(x)
|
||||||
|
return x
|
||||||
|
except Exception:
|
||||||
|
global_sema.release()
|
||||||
|
raise
|
||||||
|
|
||||||
async def _del_sqlor(self, sor):
|
async def _del_sqlor(self, sor):
|
||||||
|
global_sema = _get_global_sema()
|
||||||
try:
|
try:
|
||||||
await sor.exit()
|
await sor.exit()
|
||||||
except:
|
except:
|
||||||
@ -66,6 +86,19 @@ class SqlorPool:
|
|||||||
await sor.close()
|
await sor.close()
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
global_sema.release()
|
||||||
|
|
||||||
|
def _cleanup_idle(self):
|
||||||
|
"""Remove idle connections that exceed IDLE_TIMEOUT."""
|
||||||
|
now = time.time()
|
||||||
|
to_remove = []
|
||||||
|
for s in self.sqlors:
|
||||||
|
if not s.used and (now - s.use_at) > self.IDLE_TIMEOUT:
|
||||||
|
to_remove.append(s)
|
||||||
|
for s in to_remove:
|
||||||
|
self.sqlors = [x for x in self.sqlors if x != s]
|
||||||
|
# Fire-and-forget close (best effort)
|
||||||
|
asyncio.ensure_future(self._del_sqlor(s.sqlor))
|
||||||
|
|
||||||
async def test_sqlor(self, sor):
|
async def test_sqlor(self, sor):
|
||||||
try:
|
try:
|
||||||
@ -79,6 +112,7 @@ class SqlorPool:
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def context(self):
|
async def context(self):
|
||||||
|
self._cleanup_idle()
|
||||||
async with self.sema:
|
async with self.sema:
|
||||||
sqlors = [s for s in self.sqlors]
|
sqlors = [s for s in self.sqlors]
|
||||||
yielded_sqlor = None
|
yielded_sqlor = None
|
||||||
@ -97,9 +131,11 @@ class SqlorPool:
|
|||||||
try:
|
try:
|
||||||
yield yielded_sqlor.sqlor
|
yield yielded_sqlor.sqlor
|
||||||
yielded_sqlor.used = False
|
yielded_sqlor.used = False
|
||||||
|
yielded_sqlor.use_at = time.time()
|
||||||
return
|
return
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
yielded_sqlor.used = False
|
yielded_sqlor.used = False
|
||||||
|
yielded_sqlor.use_at = time.time()
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user