From 0f910cc605d603a1dfdcce19aeb2869f44cb3263 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Mon, 27 Jul 2026 16:03:01 +0800 Subject: [PATCH] init: VibeVoice ASR service --- .gitignore | 7 +++ app/vibevoice_asr_app.py | 57 +++++++++++++++++++++ conf/config.json | 43 ++++++++++++++++ vv/__init__.py | 9 ++++ vv/engine.py | 106 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 222 insertions(+) create mode 100644 .gitignore create mode 100644 app/vibevoice_asr_app.py create mode 100644 conf/config.json create mode 100644 vv/__init__.py create mode 100644 vv/engine.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..205990d --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +venv/ +logs/ +files/ +__pycache__/ +*.pyc +*.pid +*.log diff --git a/app/vibevoice_asr_app.py b/app/vibevoice_asr_app.py new file mode 100644 index 0000000..6634523 --- /dev/null +++ b/app/vibevoice_asr_app.py @@ -0,0 +1,57 @@ +"""VibeVoice ASR — ahserver app with same API as fastwhisper.""" +import asyncio +import json +from appPublic.worker import get_event_loop +from appPublic.log import debug +from vv import load_vibevoice_asr +from ahserver.webapp import webapp +from ahserver.configuredServer import add_startup +from ahserver.serverenv import ServerEnv +from appPublic.registerfunction import RegisterFunction + + +async def transcribe(request, *args, **kw): + env = request._run_ns + payload = dict(env.params_kw) + ret = await env.vibevoice.submit_task(payload) + return ret + + +async def get_status(request, *args, **kw): + env = request._run_ns + data = await env.vibevoice.get_status(env.params_kw.task_id) + return data + + +async def asr(request, *args, **kw): + env = request._run_ns + payload = json.loads(json.dumps(dict(env.params_kw))) + debug(f'asr(): payload={payload}') + ret = await env.vibevoice.submit_task(payload) + while True: + data = await env.vibevoice.get_status(ret['task_id']) + if data['status'] in ['SUCCEEDED', 'FAILED']: + break + await asyncio.sleep(0.2) + return data + + +async def start_engine(*args, **kw): + debug('starting VibeVoice ASR engine...') + env = ServerEnv() + asyncio.create_task(env.vibevoice.run()) + debug('VibeVoice ASR engine started') + + +def init(): + rf = RegisterFunction() + rf.register('asr', asr) + rf.register('transcribe', transcribe) + rf.register('get_status', get_status) + load_vibevoice_asr() + add_startup(start_engine) + + +if __name__ == '__main__': + loop = get_event_loop() + webapp(init) diff --git a/conf/config.json b/conf/config.json new file mode 100644 index 0000000..6f753c6 --- /dev/null +++ b/conf/config.json @@ -0,0 +1,43 @@ +{ + "filesroot": "$[workdir]$/files", + "model_path": "/share/models/VibeVoice-ASR-7B", + "redis_url": "redis://127.0.0.1:6379", + "worker_cnt": 2, + "logger": { + "name": "vibevoice-asr", + "levelname": "info", + "logfile": "$[workdir]$/logs/vibevoice.log" + }, + "website": { + "paths": [ + ["$[workdir]$/wwwroot", ""] + ], + "client_max_size": 10000, + "host": "0.0.0.0", + "port": 9926, + "coding": "utf-8", + "indexes": ["index.html", "index.dspy", "index.ui"], + "startswiths": [ + { + "leading": "/api/asr", + "registerfunction": "asr" + }, + { + "leading": "/api/transcribe", + "registerfunction": "transcribe" + }, + { + "leading": "/api/status", + "registerfunction": "get_status" + } + ], + "processors": [ + [".tmpl", "tmpl"], + [".app", "app"], + [".ui", "bui"], + [".dspy", "dspy"] + ], + "session_max_time": 3000, + "session_issue_time": 2500 + } +} diff --git a/vv/__init__.py b/vv/__init__.py new file mode 100644 index 0000000..a4212e1 --- /dev/null +++ b/vv/__init__.py @@ -0,0 +1,9 @@ +"""VibeVoice ASR init — register engine on ServerEnv.""" +from vv.engine import VibeVoiceASREngine +from ahserver.serverenv import ServerEnv + + +def load_vibevoice_asr(): + env = ServerEnv() + engine = VibeVoiceASREngine() + env.vibevoice = engine diff --git a/vv/engine.py b/vv/engine.py new file mode 100644 index 0000000..0ff7ca5 --- /dev/null +++ b/vv/engine.py @@ -0,0 +1,106 @@ +""" +VibeVoice ASR engine v4 — Pure VibeVoice source (model + processor). +Patched AutoModel.register(..., exist_ok=True) for Transformers 5.x compat. +""" +import torch +import sys +sys.path.insert(0, "/share/ymq/VibeVoice") + +from longtasks.longtasks import LongTasks +from appPublic.worker import awaitify +from appPublic.jsonConfig import getConfig +from appPublic.log import debug +from ahserver.filestorage import FileStorage + +from vibevoice.modular.modeling_vibevoice_asr import VibeVoiceASRForConditionalGeneration +from vibevoice.processor.vibevoice_asr_processor import VibeVoiceASRProcessor + + +class VibeVoiceASREngine(LongTasks): + def __init__(self): + self.config = getConfig() + super().__init__(self.config.redis_url, 'vibevoice-asr', worker_cnt=self.config.worker_cnt) + self.load_models() + + def load_models(self): + model_path = self.config.model_path + debug('loading VibeVoice ASR on cuda:0...') + self.model = VibeVoiceASRForConditionalGeneration.from_pretrained( + model_path, + dtype=torch.bfloat16, + device_map="cuda:0", + attn_implementation="sdpa", + trust_remote_code=True, + ignore_mismatched_sizes=True, + ) + self.model.eval() + self.processor = VibeVoiceASRProcessor.from_pretrained( + model_path, + language_model_pretrained_name="/share/models/Qwen2.5-7B", + ) + debug('VibeVoice ASR loaded') + + async def process_task(self, payload, workerid=None): + webpath = payload.get('audio_file') + if not webpath: + return {"task_status": "error", "message": "no audio_file"} + fs = FileStorage() + fpath = fs.realPath(webpath) + f = awaitify(self._transcribe) + return await f(fpath) + + def _transcribe(self, fpath): + inputs = self.processor( + audio=fpath, sampling_rate=None, return_tensors="pt", + padding=True, add_generation_prompt=True) + device = torch.device("cuda:0") + inputs = {k: v.to(device) if isinstance(v, torch.Tensor) else v + for k, v in inputs.items()} + with torch.no_grad(): + gen = self.model.generate(**inputs, max_new_tokens=4096, + temperature=0.0, do_sample=False) + text = self.processor.decode(gen[0], skip_special_tokens=True) + try: + segs = self.processor.post_process_transcription(text) + except Exception as e: + debug(f'parse failed: {e}') + segs = [] + + # Fallback: direct JSON extraction from model output + if not segs and text: + import re, json as _json + # Try full JSON first + m = re.search(r'assistant\n(.+?)(?:<\|im_end\|>)', text, re.DOTALL) + json_str = m.group(1).strip() if m else '' + if not json_str and 'assistant\n[' in text: + json_str = text[text.find('assistant\n[')+10:] + if json_str: + # Handle truncated JSON: find last complete object + try: + raw_segs = _json.loads(json_str) + except Exception: + last_good = json_str.rfind('"}') + if last_good > 0: + try: + raw_segs = _json.loads(json_str[:last_good+2] + ']') + except Exception: + raw_segs = [] + else: + raw_segs = [] + for s in raw_segs: + segs.append({ + 'start_time': s.get('Start', 0), + 'end_time': s.get('End', 0), + 'speaker_id': s.get('Speaker', 0), + 'text': s.get('Content', ''), + }) + + ws, ft = [], [] + for s in segs: + ws.append([s.get('start_time',0), s.get('end_time',0), s.get('text',''), []]) + ft.append(s.get('text','')) + return { + 'task_status': 'SUCCEEDED', 'language': 'auto', + 'language_probability': 1.0, 'content': ' '.join(ft), + 'segments': ws, 'raw_output': text, 'vibevoice_segments': segs, + }