yumoqing/sqlor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfix

Browse Source
This commit is contained in:
yumoqing 2025-10-23 15:17:24 +08:00
parent b4112adde7
commit fb8153cc3a
4 changed files with 154 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
README.md
View File

@ -23,7 +23,7 @@ SQLOR is a database api for python3, it is base on the python's DBAPI2
* clickhouse(clickhouse-connect)
* Other driver can be easy integreated
## Support Database Types
## Supported Database Types
* oracle
* mysql, mariadb
* TiDB
@ -83,87 +83,120 @@ password in json data is encrypted by aes.
## Using
### sqlor setup
First, Specified a server_path folder, under the server_path folder, need a named by "conf" subfolder.
and a config.json file must be in the conf.
the config.json need "password_key" and "databases" attributes, like:
```
import asyncio
from sqlor.dbpools import DBPools, sqlorContext
dbs={
{
.......
"password_key":"tfyugihjo245g7g642yubv24g534",
"databases":{
.......
"mydb":{
"driver":"mysql",
"kwargs":{
"user":"test",
"db":"cfae",
"password":"test123",
"host":"localhost"
}
},
"stock":{
"driver":"aiopg",
"async_mode":True,
"codeing":"utf-8",
"dbname":"stock",
"kwargs":{
"dbname":"stock",
"user":"test",
"password":"test123",
"host":"127.0.0.1"
}
},
"cfae":{
"driver":"mysql.connector",
"coding":"utf8",
"dbname":"cfae",
"kwargs":{
"user":"test",
"db":"cfae",
"password":"test123",
"db":"database_name_in_your_database_engine",
"password":"encoded_password_string",
"host":"localhost"
}
}
}
}
```
### generates encoded password string
sqlor has a dbpassword script let you generate the "encoded_password_string" in config.json
dbpassword usage:
```
dbpassword server_path database_password_of_user
```
loop = asyncio.get_event_loop()
pool = DBPools(dbs,loop=loop)
### script to use sqlor
```
import asyncio
from appPublic.worker import get_event_loop
from appPublic.jsonConfig import getConfig
from sqlor.dbpools import DBPools, sqlorContext
loop = get_event_loop()
config = getConfig(server_path)
pool = DBPools(config.databases, loop=loop)
async def testfunc():
async with sqlorContext('stock') as sor:
# start a transaction
# if exception happended, all change to database will rollback
# else will commit
sql = "select * from stock where stock_num = ${stock_num}"
recs = await sor.sqlExe(sql, {'stock_num':'688888'})
# return a list of DictObject instance
sql1 = "select * from stock"
recs = await sor.sqlPaging(sql, {'pagerows':50, 'page':1, 'sort':'stock_id'})
# return a dictionary {
# 'total': return all reocords count.
# 'rows': list of DictObject instance
# }
ns = {'id':'666667'} # filters dictionary, find all records which id = '666667'
recs = await sor.R('stock', ns)
# recs is a list of data in DictObject instance
ns = {'pagerows': 50, 'page':1, 'sort':stock_id'}
# find all record in table 'stock', return page 1 data which 50 max records
dic = await sor.R('stock', ns)
# return a dictionary {
# 'total': return all reocords count.
# 'rows': list of DictObject instance
# }
await sor.C('stock', {
'stock_id': '111111',
...
})
# add a record to 'stock' table
await sor.D('stock', {'stock_id':'111111'})
# delete a record in 'stock' table which stock_id = '111111'
await sor.U('stock', {'stock_id':'111111', 'name':'new_name'})
# update name field's value to 'new_name' which stock_id = '111111'
dbname = 'mydb'
db = DBPools()
async with sqlorContext('stock') as sor:
# start a transaction
# if exception happended, all change to database will rollback
# else will commit
sql = "select * from stock where stock_num = ${stock_num}"
recs = await sor.sqlExe(sql, {'stock_num':'688888'})
# return a list of DictObject instance
sql1 = "select * from stock"
recs = await sor.sqlPaging(sql, {'pagerows':50, 'page':1, 'sort':'stock_id'})
# return a dictionary {
# 'total': return all reocords count.
# 'rows': list of DictObject instance
# }
ns = {'id':'666667'} # filters dictionary, find all records which id = '666667'
recs = await sor.R('stock', ns)
# recs is a list of data in DictObject instance
ns = {'pagerows': 50, 'page':1, 'sort':stock_id'}
# find all record in table 'stock', return page 1 data which 50 max records
dic = await sor.R('stock', ns)
# return a dictionary {
# 'total': return all reocords count.
# 'rows': list of DictObject instance
# }
await sor.C('stock', {
'stock_id': '111111',
...
})
# add a record to 'stock' table
await sor.D('stock', {'stock_id':'111111'})
# delete a record in 'stock' table which stock_id = '111111'
await sor.U('stock', {'stock_id':'111111', 'name':'new_name'})
# update name field's value to 'new_name' which stock_id = '111111'
loop.run_until_complete(testfunc())
```
## scripts
### dbpassword
generate a encoded password for the password of the user of database
* Syntax
dbpassword server_path password_text
1 server_path is folder where server_path/conf/config.json file have specifies password_key and databases
2 password_text is the password of the specified user of database
* description
dbpassword encodes second argument password to a base64 based cyber, this cyber need to write into the password attribute under kwargs, and print the cyber to stdout
### dbloader
load data in xlsx file to your database
* Syntax
dbloader server_path dbname xlsxfile
1 server_path is folder where server_path/conf/config.json file have specifies password_key and databases
2 dbname is database where the data will insert into
3 xlsxfile is a data file, can contains many sheets, sheet name is tablename, first rows of sheet contains the field name of the table, fields without data not need to appear in the sheet.
* dbloader get data from each named sheet in xlsxfile, and insert them to database parellally,
## API
@ -175,25 +208,23 @@ how many databases and what database will using, and them connection parameters
dbdesc data is a dict data, format of the dbdesc as follow:
```
{
"aiocfae":{ # name to identify a database connect
"driver":"aiomysql", # database dbapi2 driver package name
"mydb1":{ # name to identify a database connect
"driver":"mysql", # database dbapi2 driver package name
"async_mode":True, # indicte this connection is asynchronous mode
"coding":"utf8", # charset coding
"dbname":"cfae", # database real name
"kwargs":{ # connection parameters
"user":"test",
"db":"cfae",
"password":"test123",
"password":"encoded_password",
"host":"localhost"
}
},
"cfae":{
"driver":"mysql.connector",
"coding":"utf8",
"dbname":"cfae",
"mydb2":{
"driver":"postgresql",
"kwargs":{
"user":"test",
"db":"cfae",
"dbname":"cfae",
"password":"test123",
"host":"localhost"
}

2
setup.cfg
View File

@ -1,6 +1,6 @@
[metadata]
name=sqlor
version = 2.0.1
version = 2.0.2
description = a new version of sqlor, each db's sor need to plugin to sqlor, and dbdriver now a isolated module
authors = yu moqing
author_email = yumoqing@gmail.com

51
sqlor/dbloader.py
View File

@ -1,7 +1,50 @@
from appPublic.worker import get_event_loop
import os
import sys
import asyncio
async def main():
print('to be programming')
from appPublic.worker import get_event_loop
from appPublic.jsonConfig import getConfig
from appPublic.worker import AsyncWorker
from sqlor.dbpools import DBPools
from appPublic.dictObject import DictObject
from xls2ddl.xlsxData import XLSXData
def chunked(lst, n):
for i in range(0, len(lst), n):
yield lst[i:i+n]
async def load_tabledata(dbname, tblname, data):
db = DBPools()
async with db.sqlorContext(dbname) as sor:
for r in data:
try:
await sor.D(tblname, {'id': r.id})
except:
pass
await sor.C(tblname, r.copy())
async def load_data():
if len(sys.argv) < 4:
print(f'{sys.argv[0]} server_path dbname datafile')
return 1
runpath = sys.argv[1]
dbname = sys.argv[2]
datafile = sys.argv[3]
config = getConfig(runpath)
db = DBPools(config.databases)
xlsx = XLSXData(datafile)
worker = AsyncWorker(maxtask=100)
tasks = []
for i,s in enumerate(xlsx.book.worksheets):
tblname = xlsx.book.sheetnames[i]
dic = xlsx.readRecords(tblname, s)
for chunk in chunked(dic[tblname], 100):
t = asyncio.create_task(load_tabledata(dbname, tblname, chunk))
tasks.append(t)
await asyncio.wait(tasks)
return 0
def main():
get_event_loop().run_until_complete(load_data())
if __name__ == '__main__':
get_event_loop().run_until_complete(main())
main()

4
sqlor/oracleor.py
View File

@ -66,8 +66,8 @@ from (
)
where row_id >=$[from_line]$ and row_id < $[end_line]$"""
def test_sqlstr(self):
return "select 1 from dual"
def test_sqlstr(self):
return "select 1 from dual"
def tablesSQL(self):
sqlcmd = """select