146 lines
6.5 KiB
Python
146 lines
6.5 KiB
Python
#!/usr/bin/env python3
|
||
"""统一初始化数据导入脚本（模块 init/data.json -> pipeline 库）。
|
||
|
||
读取各模块 init/data.json 的字典数据（appcodes / appcodes_kv / organization），
|
||
导入 pipeline 库对应表。幂等：已存在则跳过。
|
||
|
||
用法：
|
||
py3/bin/python scripts/import_init.py
|
||
"""
|
||
import sys, os, json, asyncio
|
||
|
||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
ROOT_DIR = os.path.dirname(SCRIPT_DIR)
|
||
sys.path.insert(0, os.path.join(ROOT_DIR, 'py3', 'lib', 'python3.10', 'site-packages'))
|
||
sys.path.insert(0, ROOT_DIR)
|
||
|
||
from sqlor.dbpools import DBPools
|
||
from appPublic.jsonConfig import getConfig
|
||
from appPublic.folderUtils import ProgramPath
|
||
from appPublic.uniqueID import getID
|
||
from ahserver.serverenv import ServerEnv
|
||
from ahserver.globalEnv import initEnv
|
||
|
||
# 各模块的 init 数据（模块名 -> 相对 ROOT_DIR 的 init/data.json 路径）
|
||
INIT_MODULES = [
|
||
('product_management', 'pkgs/product_management/init/data.json'),
|
||
('discount', 'pkgs/discount/init/data.json'),
|
||
('pipeline_core', 'pkgs/pipeline_core/init/data.json'),
|
||
('pipeline-sdlc', 'pkgs/pipeline-sdlc/init/data.json'),
|
||
('app_audit', 'pkgs/app_audit/init/data.json'),
|
||
('rbac', 'pkgs/rbac/init/data.json'),
|
||
('accounting', 'pkgs/accounting/init/data.json'),
|
||
('pipeline-bidding', 'pkgs/pipeline-bidding/init/data.json'),
|
||
('pipeline-opportunity', 'pkgs/pipeline-opportunity/init/data.json'),
|
||
('pipeline-llm', 'pkgs/pipeline-llm/init/data.json'),
|
||
]
|
||
|
||
|
||
def normalize(data):
|
||
"""兼容两种 data.json 格式，返回 (appcodes, appcodes_kv, organization)。
|
||
|
||
扁平格式（product_management/discount）：
|
||
{"appcodes": [{"id","name","hierarchy_flg"}],
|
||
"appcodes_kv": [{"id","parentid","k","v"}]}
|
||
嵌套格式（pipeline_core/pipeline-sdlc）：
|
||
{"appcodes": [{"parentid","parentname","items":[{"k","v"}]}]}
|
||
嵌套格式展开为扁平：appcodes.id=parentid、appcodes_kv.id=f"{parentid}_{k}"（稳定组合键，幂等）。
|
||
"""
|
||
raw_ac = data.get('appcodes', [])
|
||
if raw_ac and isinstance(raw_ac[0], dict) and 'parentid' in raw_ac[0]:
|
||
appcodes = []
|
||
appcodes_kv = []
|
||
for ac in raw_ac:
|
||
pid = ac['parentid']
|
||
appcodes.append({'id': pid, 'name': ac.get('parentname', pid), 'hierarchy_flg': '0'})
|
||
for item in ac.get('items', []):
|
||
appcodes_kv.append({
|
||
'id': f"{pid}_{item['k']}",
|
||
'parentid': pid,
|
||
'k': item['k'],
|
||
'v': item['v'],
|
||
})
|
||
return appcodes, appcodes_kv, data.get('organization', [])
|
||
return raw_ac, data.get('appcodes_kv', []), data.get('organization', [])
|
||
|
||
|
||
async def main():
|
||
config = getConfig(ROOT_DIR, NS={'workdir': ROOT_DIR, 'ProgramPath': ProgramPath()})
|
||
DBPools(config.databases)
|
||
initEnv()
|
||
env = ServerEnv()
|
||
env.get_module_dbname = lambda m: 'pipeline' if 'pipeline' in m else 'sage'
|
||
|
||
total = {'appcodes': 0, 'appcodes_kv': 0, 'organization': 0, 'roles': 0}
|
||
async with DBPools().sqlorContext('pipeline') as sor:
|
||
for mod, rel in INIT_MODULES:
|
||
f = os.path.join(ROOT_DIR, rel)
|
||
if not os.path.exists(f):
|
||
print(f' skip {mod}: {rel} not found')
|
||
continue
|
||
data = json.load(open(f, encoding='utf-8'))
|
||
appcodes, appcodes_kv, organization = normalize(data)
|
||
for ac in appcodes:
|
||
recs = await sor.R('appcodes', {'id': ac['id']})
|
||
if not recs:
|
||
await sor.C('appcodes', {
|
||
'id': ac['id'],
|
||
'name': ac.get('name', ''),
|
||
'hierarchy_flg': ac.get('hierarchy_flg', '0'),
|
||
})
|
||
total['appcodes'] += 1
|
||
for kv in appcodes_kv:
|
||
# 幂等检查用 (parentid, k) 组合（appcodes_kv 有 parentid+k 唯一索引，
|
||
# 且历史数据 id 是 UUID 非 parentid_k 组合键，用 id 检查会漏判 → 唯一索引冲突）
|
||
recs = await sor.sqlExe(
|
||
"SELECT id FROM appcodes_kv WHERE parentid=${p}$ AND k=${k}$",
|
||
{"p": kv.get('parentid', ''), "k": kv.get('k', '')})
|
||
if not recs:
|
||
await sor.C('appcodes_kv', {
|
||
'id': kv['id'],
|
||
'parentid': kv.get('parentid', ''),
|
||
'k': kv.get('k', ''),
|
||
'v': kv.get('v', ''),
|
||
})
|
||
total['appcodes_kv'] += 1
|
||
for org in organization:
|
||
recs = await sor.R('organization', {'id': org['id']})
|
||
if not recs:
|
||
await sor.C('organization', {
|
||
'id': org['id'],
|
||
'orgname': org.get('orgname', ''),
|
||
})
|
||
total['organization'] += 1
|
||
# RBAC 角色（如 app_audit 的 owner.audit 审计独立角色），幂等按主键 id 检查
|
||
for r in data.get('roles', []):
|
||
recs = await sor.R('role', {'id': r['id']})
|
||
if not recs:
|
||
await sor.C('role', {
|
||
'id': r['id'],
|
||
'orgtypeid': r.get('orgtypeid', '0'),
|
||
'name': r.get('name', ''),
|
||
})
|
||
total['roles'] += 1
|
||
# 记账配置表（accounting 模块：subject/account_config/accounting_config/currency/exchange_rate）
|
||
# 幂等按主键 id 检查，字段名与表结构一致
|
||
for tbl, recs_data in [
|
||
('subject', data.get('subject', [])),
|
||
('account_config', data.get('account_config', [])),
|
||
('accounting_config', data.get('accounting_config', [])),
|
||
('currency', data.get('currency', [])),
|
||
('exchange_rate', data.get('exchange_rate', [])),
|
||
]:
|
||
for rec in recs_data:
|
||
existing = await sor.R(tbl, {'id': rec.get('id')})
|
||
if not existing:
|
||
# 去掉 None 值字段（DB 用 NULL 而非 None）
|
||
clean = {k: v for k, v in rec.items() if v is not None}
|
||
await sor.C(tbl, clean)
|
||
total[tbl] = total.get(tbl, 0) + 1
|
||
summary = ' '.join(f'{k} {v}' for k, v in total.items())
|
||
print(f'import_init: {summary}')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
asyncio.run(main())
|