pipeline-app/scripts/import_init.py

#!/usr/bin/env python3
"""统一初始化数据导入脚本（模块 init/data.json -> pipeline 库）。

读取各模块 init/data.json 的字典数据（appcodes / appcodes_kv / organization），
导入 pipeline 库对应表。幂等：已存在则跳过。

用法：
  py3/bin/python scripts/import_init.py
"""
import sys, os, json, asyncio

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = os.path.dirname(SCRIPT_DIR)
sys.path.insert(0, os.path.join(ROOT_DIR, 'py3', 'lib', 'python3.10', 'site-packages'))
sys.path.insert(0, ROOT_DIR)

from sqlor.dbpools import DBPools
from appPublic.jsonConfig import getConfig
from appPublic.folderUtils import ProgramPath
from appPublic.uniqueID import getID
from ahserver.serverenv import ServerEnv
from ahserver.globalEnv import initEnv

# 各模块的 init 数据（模块名 -> 相对 ROOT_DIR 的 init/data.json 路径）
INIT_MODULES = [
    ('product_management', 'pkgs/product_management/init/data.json'),
    ('discount', 'pkgs/discount/init/data.json'),
    ('pipeline_core', 'pkgs/pipeline_core/init/data.json'),
    ('pipeline-sdlc', 'pkgs/pipeline-sdlc/init/data.json'),
    ('app_audit', 'pkgs/app_audit/init/data.json'),
    ('rbac', 'pkgs/rbac/init/data.json'),
    ('accounting', 'pkgs/accounting/init/data.json'),
    ('pipeline-bidding', 'pkgs/pipeline-bidding/init/data.json'),
]


def normalize(data):
    """兼容两种 data.json 格式，返回 (appcodes, appcodes_kv, organization)。

    扁平格式（product_management/discount）：
      {"appcodes": [{"id","name","hierarchy_flg"}],
       "appcodes_kv": [{"id","parentid","k","v"}]}
    嵌套格式（pipeline_core/pipeline-sdlc）：
      {"appcodes": [{"parentid","parentname","items":[{"k","v"}]}]}
    嵌套格式展开为扁平：appcodes.id=parentid、appcodes_kv.id=f"{parentid}_{k}"（稳定组合键，幂等）。
    """
    raw_ac = data.get('appcodes', [])
    if raw_ac and isinstance(raw_ac[0], dict) and 'parentid' in raw_ac[0]:
        appcodes = []
        appcodes_kv = []
        for ac in raw_ac:
            pid = ac['parentid']
            appcodes.append({'id': pid, 'name': ac.get('parentname', pid), 'hierarchy_flg': '0'})
            for item in ac.get('items', []):
                appcodes_kv.append({
                    'id': f"{pid}_{item['k']}",
                    'parentid': pid,
                    'k': item['k'],
                    'v': item['v'],
                })
        return appcodes, appcodes_kv, data.get('organization', [])
    return raw_ac, data.get('appcodes_kv', []), data.get('organization', [])


async def main():
    config = getConfig(ROOT_DIR, NS={'workdir': ROOT_DIR, 'ProgramPath': ProgramPath()})
    DBPools(config.databases)
    initEnv()
    env = ServerEnv()
    env.get_module_dbname = lambda m: 'pipeline' if 'pipeline' in m else 'sage'

    total = {'appcodes': 0, 'appcodes_kv': 0, 'organization': 0, 'roles': 0}
    async with DBPools().sqlorContext('pipeline') as sor:
        for mod, rel in INIT_MODULES:
            f = os.path.join(ROOT_DIR, rel)
            if not os.path.exists(f):
                print(f'  skip {mod}: {rel} not found')
                continue
            data = json.load(open(f, encoding='utf-8'))
            appcodes, appcodes_kv, organization = normalize(data)
            for ac in appcodes:
                recs = await sor.R('appcodes', {'id': ac['id']})
                if not recs:
                    await sor.C('appcodes', {
                        'id': ac['id'],
                        'name': ac.get('name', ''),
                        'hierarchy_flg': ac.get('hierarchy_flg', '0'),
                    })
                    total['appcodes'] += 1
            for kv in appcodes_kv:
                # 幂等检查用 (parentid, k) 组合（appcodes_kv 有 parentid+k 唯一索引，
                # 且历史数据 id 是 UUID 非 parentid_k 组合键，用 id 检查会漏判 → 唯一索引冲突）
                recs = await sor.sqlExe(
                    "SELECT id FROM appcodes_kv WHERE parentid=${p}$ AND k=${k}$",
                    {"p": kv.get('parentid', ''), "k": kv.get('k', '')})
                if not recs:
                    await sor.C('appcodes_kv', {
                        'id': kv['id'],
                        'parentid': kv.get('parentid', ''),
                        'k': kv.get('k', ''),
                        'v': kv.get('v', ''),
                    })
                    total['appcodes_kv'] += 1
            for org in organization:
                recs = await sor.R('organization', {'id': org['id']})
                if not recs:
                    await sor.C('organization', {
                        'id': org['id'],
                        'orgname': org.get('orgname', ''),
                    })
                    total['organization'] += 1
            # RBAC 角色（如 app_audit 的 owner.audit 审计独立角色），幂等按主键 id 检查
            for r in data.get('roles', []):
                recs = await sor.R('role', {'id': r['id']})
                if not recs:
                    await sor.C('role', {
                        'id': r['id'],
                        'orgtypeid': r.get('orgtypeid', '0'),
                        'name': r.get('name', ''),
                    })
                    total['roles'] += 1
            # 记账配置表（accounting 模块：subject/account_config/accounting_config/currency/exchange_rate）
            # 幂等按主键 id 检查，字段名与表结构一致
            for tbl, recs_data in [
                ('subject', data.get('subject', [])),
                ('account_config', data.get('account_config', [])),
                ('accounting_config', data.get('accounting_config', [])),
                ('currency', data.get('currency', [])),
                ('exchange_rate', data.get('exchange_rate', [])),
            ]:
                for rec in recs_data:
                    existing = await sor.R(tbl, {'id': rec.get('id')})
                    if not existing:
                        # 去掉 None 值字段（DB 用 NULL 而非 None）
                        clean = {k: v for k, v in rec.items() if v is not None}
                        await sor.C(tbl, clean)
                        total[tbl] = total.get(tbl, 0) + 1
    summary = ' '.join(f'{k} {v}' for k, v in total.items())
    print(f'import_init: {summary}')


if __name__ == '__main__':
    asyncio.run(main())