import os from traceback import format_exc from ahserver.serverenv import get_serverenv from ahserver.auth_api import get_session_userinfo from sqlor.dbpools import DBPools from appPublic.uniqueID import getID from appPublic.log import debug, error, exception from appPublic.dictObject import DictObject from appPublic.registerfunction import RegisterFunction from appPublic.worker import awaitify from ahserver.filestorage import FileStorage import hashlib def get_file_hash(filepath, algo='sha256', chunk_size=8192): h = hashlib.new(algo) with open(filepath, 'rb') as f: while chunk := f.read(chunk_size): h.update(chunk) return h.hexdigest() def get_dbname(): f = get_serverenv('get_module_dbname') dbname = f('filemgr') return dbname class FileMgrResult(DictObject): def __init__(self, status, message, **kw): super().__init__(status=status, message=message, **kw) class FileMgr: def __init__(self, fiid): self.fiid = fiid async def get_folder_ownerid(self, sor): pass async def file_uploaded(self, request, ns, userid): pass async def file_deleted(self, request, ns, userid): pass async def get_organization_quota(self, sor, orgid): return 0, '9999-12-31' async def is_folder_ownerid(self, sor, orgid): folder_owner = await self.get_folder_ownerid(sor) return folder_owner == orgid async def get_quota_used(self, sor, orgid): sql = "select sum(filesize) as quota_used from file where ownerid=${orgid}$" recs = await sor.sqlExe(sql, {'orgid': orgid}) if len(recs) == 0: return 0 r = recs[0] if r.quota_used is None: r.quota_used = 0 return r.quota_used / 1000000 async def add_file(self, request, params_kw): fs = FileStorage() webpath = params_kw.upfile realpath = fs.realPath(webpath) filesize = os.path.getsize(realpath) hashvalue = get_file_hash(realpath) dbname = get_dbname() u = await get_session_userinfo(request) if u is None: return FileMgrResult('error', 'user need login') db = DBPools() async with db.sqlorContext(dbname) as sor: bool = await self.is_folder_ownerid(sor, u.userorgid) if not bool: os.unlink(realpath) e = Exception(f'owner mismatch({u.userorgid=})') exception(f'{e=}\n{format_exc()}') raise e quota, expired_date = await self.get_organization_quota(sor, u.userorgid) if quota is None: os.unlink(realpath) e = Exception(f'{u.userorgid} has not quota') exception(f'{e}\n{format_exc()}') raise e quota_used = await self.get_quota_used(sor, u.userorgid) if quota_used + filesize / 1000000 >= quota: os.unlink(realpath) self.message = f'{quota=}M, {quota_used=}M {filesize=} overused' e = Exception(f'{u.userorgid} quota overflow') exception(f'{e}\n{format_exc()}') raise e recs = await sor.R('file',{'hashvalue': hashvalue, 'ownerid': u.userorgid}) if len(recs) > 0: os.unlink(realpath) e = Exception(f'File exists') exception(f'{e=}\n{format_exc()}') return FileMgrResult('failed', 'file exists') recs = await sor.R('file',{'hashvalue': hashvalue}) if len(recs) > 0: os.unlink(realpath) r = recs[0] webpath = r.webpath realpath = r.realpath ns = { "id": getID(), "folderid": params_kw.folderid, "fiid":params_kw.fiid, "name":os.path.basename(params_kw.upfile), "webpath": webpath, "realpath": realpath, "filetype":params_kw.upfile.split('.')[-1].lower(), "filesize":filesize, "ownerid": u.userorgid, "hashvalue": hashvalue } await sor.C('file', ns) if len(recs) == 0: ret = await self.file_uploaded(request, ns, u.userid) return ret return FileMgrResult('success', 'file added') return FileMgrResult('error', str(db.e_except)) async def del_folder(self, request, id): db = DBPools() dbname = get_dbname() delrecs = [] async with db.sqlorContext(dbname) as sor: u = await get_session_userinfo(request) recs = await self._del_folder(sor, id, u.userorgid) delrecs += delrecs self.file_deleted(request, delrecs, u.userid) async def _del_folder(self, sor, id, ownerid): debug(f'_del_folder("{id}"):here') folders = await sor.R('folder', {'id': id}) if len(folders) == 0: e = Exception(f'folder({id=}) not found') exception(f'{e=}\n{format_exc()}') raise e folder = folders[0] if folder.ownerid != ownerid: e = Exception(f'Wrong owner for folder({id=}, {ownerid=}') exception(f'{e=}\n{format_exc()}') raise e delrecs = [] debug(f'_del_folder("{id}"):here') files = await self.sor_get_subfile(sor, id) for f in files: rec = await self._del_file(sor, f.id, ownerid) delrecs += files debug(f'_del_folder("{id}"):here') folders = await self.sor_get_subfolder(sor, id) for f in folders: recs = await self._del_folder(sor, f.id, ownerid) delrecs += recs debug(f'_del_folder("{id}"):here') await sor.D('folder', {'id': id}) debug(f'_del_folder("{id}"):here') return delrecs async def del_file(self, request, fid): db = DBPools() dbname = get_dbname() delrecs = [] async with db.sqlorContext(dbname) as sor: u = await get_session_userinfo(request) rec = await self._del_file(sor, fid, u.userorgid) if rec: delrecs.append(rec) ret = await self.file_deleted(request, delrecs, u.userid) return ret return FileMgrResult('success', 'file deleted') async def _del_file(self, sor, fid, ownerid): recs = await sor.R('file', {'id': fid}) if recs: delrec = recs[0] if delrec.ownerid != ownerid: e = Exception(f'wrong owners:{ownerid}') exception(f'{e=},{format_exc()}') raise e await sor.D('file', {'id':fid}) remain = await sor.R('file', {'hashvalue':delrec.hashvalue}) if not remain: os.unlink(delrec.realpath) return delrec return None async def has_sub(self, sor, folderid): sql = """select unique a.* from folder a left join folder b on a.id = b.parentid where a.id=${folderid}$ and b.id is not null union select unique a.* from folder a left join file c on a.id=c.folderid where a.id=${folderid}$ and c.id is not null """ recs = await sor.sqlExe(sql, {'folderid':folderid}) if recs: return True return False async def folder_files(self, id): db = DBPools() dbanme = get_dbname() async with db.sqlorContext(dbname) as sor: for r in self._folder_files(sor, id): yield r async def _folder_files(self, sor, id): recs = await self.sor_get_subfile(sor, id) for r in recs: yield r folders = await self.sor_get_subfolder(sor, id) for f in folders: for r in self._folder_files(f.id): yield r async def get_subs(self, id): dbname = get_dbname() db = DBPools() async with db.sqlorContext(dbname) as sor: ret = await self.sor_get_subfolder(sor, id) ret += await self.sor_get_subfile(sor, id) return ret return [] async def sor_get_subfolder(self, sor, fid): sql = """select x.*, 'folder' as filetype, case when y.id is null then 1 else 0 end as is_left from folder as x left join ( select unique a.* from folder a left join folder b on a.id = b.parentid left join file c on a.id=c.folderid where b.id is not null or c.id is not null ) as y on x.id = y.id where x.parentid = ${fid}$ """ ns = { 'fid':fid } recs = await sor.sqlExe(sql, ns) return recs async def sor_get_subfile(self, sor, fid): sql = """select *, 1 as is_leaf from file where folderid=${fid}$""" ns = { 'fid':fid } recs = await sor.sqlExe(sql, ns) return recs