fix: discard MySQL dead connections (OperationalError/InterfaceError) from pool
Previously sqlorContext only discarded connections on RuntimeError/OSError
('closed'/'handler is closed'). MySQL OperationalError (2014 Command Out of
Sync) and InterfaceError (0 Not connected) were not caught, causing dead
connections to be returned to the pool and reused — cascading to multiple
500 errors on /llmage/v1/chat/completions.
Now catches any Exception with known MySQL dead-connection messages
(Not connected, Command Out of Sync, Lost connection, Connection reset,
Server disconnected) and discards the connection.
Added tests: mysql_error_during_use (5 variants), business_error_not_discarded.
This commit is contained in:
parent
a9a02eb45b
commit
e117b2acaa
@ -153,6 +153,16 @@ class SqlorPool:
|
|||||||
self._discard_sqlor(yielded_sqlor)
|
self._discard_sqlor(yielded_sqlor)
|
||||||
yielded_sqlor = None # prevent finally from resetting
|
yielded_sqlor = None # prevent finally from resetting
|
||||||
raise
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
err_msg = str(e)
|
||||||
|
if any(x in err_msg for x in (
|
||||||
|
'Not connected', 'Command Out of Sync',
|
||||||
|
'Lost connection', 'Connection reset',
|
||||||
|
'Server disconnected',
|
||||||
|
)):
|
||||||
|
self._discard_sqlor(yielded_sqlor)
|
||||||
|
yielded_sqlor = None
|
||||||
|
raise
|
||||||
finally:
|
finally:
|
||||||
# Always reset state, even on CancelledError (BaseException in Python 3.8+)
|
# Always reset state, even on CancelledError (BaseException in Python 3.8+)
|
||||||
if yielded_sqlor is not None:
|
if yielded_sqlor is not None:
|
||||||
|
|||||||
@ -209,10 +209,59 @@ async def test_normal_usage_no_discard():
|
|||||||
print("PASS: Normal usage - connection stays in pool")
|
print("PASS: Normal usage - connection stays in pool")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_mysql_error_during_use():
|
||||||
|
"""MySQL OperationalError/InterfaceError during use should discard connection."""
|
||||||
|
global call_count
|
||||||
|
call_count = 0
|
||||||
|
|
||||||
|
factory = make_fake_sor([True])
|
||||||
|
pool = SqlorPool(factory)
|
||||||
|
|
||||||
|
mysql_errors = [
|
||||||
|
Exception("(0, 'Not connected')"),
|
||||||
|
Exception("(2014, 'Command Out of Sync')"),
|
||||||
|
Exception("Lost connection to MySQL server"),
|
||||||
|
Exception("Connection reset by peer"),
|
||||||
|
Exception("Server disconnected"),
|
||||||
|
]
|
||||||
|
|
||||||
|
for err in mysql_errors:
|
||||||
|
try:
|
||||||
|
async with pool.context() as sor:
|
||||||
|
raise err
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
assert len(pool.sqlors) == 0, f"{err} should be discarded"
|
||||||
|
pool = SqlorPool(make_fake_sor([True]))
|
||||||
|
|
||||||
|
print("PASS: MySQL errors during use -> connection discarded")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_business_error_not_discarded():
|
||||||
|
"""Normal business logic errors should NOT discard connection."""
|
||||||
|
global call_count
|
||||||
|
call_count = 0
|
||||||
|
|
||||||
|
factory = make_fake_sor([True])
|
||||||
|
pool = SqlorPool(factory)
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with pool.context() as sor:
|
||||||
|
raise ValueError("business logic error")
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
assert len(pool.sqlors) == 1, "Business error should not discard connection"
|
||||||
|
assert pool.sqlors[0].used == False
|
||||||
|
print("PASS: Business error - connection stays in pool")
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
await test_normal_usage_no_discard()
|
await test_normal_usage_no_discard()
|
||||||
await test_discard_dead_connection()
|
await test_discard_dead_connection()
|
||||||
await test_runtime_error_during_use()
|
await test_runtime_error_during_use()
|
||||||
|
await test_mysql_error_during_use()
|
||||||
|
await test_business_error_not_discarded()
|
||||||
print("\n=== ALL TESTS PASSED ===")
|
print("\n=== ALL TESTS PASSED ===")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user