fix: replace MTAudioProcessingTap with CoreAudio device capture for HLS audio recording
MTAudioProcessingTap with AVPlayerItem.audioMix doesn't fire its process callback for HLS streams on macOS (known limitation). This caused recorded videos to have no audio track when recording HLS content. Replace with CoreAudio AudioDeviceCreateIOProcID approach that captures PCM audio directly from the system output device. This works for all audio sources (HLS, local files, etc). Key changes: - Remove MTAudioProcessingTap and all tap callbacks - Add CoreAudio device IOProc for audio capture - Pre-allocated buffer to avoid malloc in real-time audio thread - Serial dispatch queue for CMSampleBuffer processing off audio thread - AudioTimeStamp → CMTime conversion for proper timestamps
This commit is contained in:
parent
f0e468122b
commit
f1ecbf2e91
@ -12,155 +12,129 @@ class AudioCaptureContext {
|
|||||||
var writerInput: AVAssetWriterInput?
|
var writerInput: AVAssetWriterInput?
|
||||||
nonisolated(unsafe) var isRunning: Bool = true
|
nonisolated(unsafe) var isRunning: Bool = true
|
||||||
nonisolated(unsafe) var formatDescription: CMAudioFormatDescription?
|
nonisolated(unsafe) var formatDescription: CMAudioFormatDescription?
|
||||||
|
nonisolated(unsafe) var sampleRate: Double = 44100.0
|
||||||
|
nonisolated(unsafe) var channelsPerFrame: UInt32 = 2
|
||||||
nonisolated(unsafe) var appendCount: Int = 0
|
nonisolated(unsafe) var appendCount: Int = 0
|
||||||
nonisolated(unsafe) var firstAppendLogged: Bool = false
|
nonisolated(unsafe) var firstAppendLogged: Bool = false
|
||||||
nonisolated(unsafe) var processCallCount: Int = 0
|
nonisolated(unsafe) var processCallCount: Int = 0
|
||||||
nonisolated(unsafe) var sampleRate: Double = 44100.0
|
|
||||||
nonisolated(unsafe) var channelsPerFrame: UInt32 = 2
|
/// Serial queue for processing audio buffers off the real-time thread
|
||||||
|
let audioQueue = DispatchQueue(label: "miniplayer.audioCapture")
|
||||||
|
/// Pre-allocated data buffer (avoids malloc in real-time audio thread)
|
||||||
|
var dataBuffer: UnsafeMutableRawPointer?
|
||||||
|
var dataBufferSize: UInt32 = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - MTAudioProcessingTap C 回调
|
// MARK: - CoreAudio 输出设备 IOProc 回调
|
||||||
|
|
||||||
private func tapInitCallback(
|
/// 从系统音频输出设备捕获 PCM 数据(适用于所有音频源:HLS、本地文件等)
|
||||||
_ tap: MTAudioProcessingTap,
|
/// 运行在高优先级音频线程,最小化工作
|
||||||
_ clientInfo: UnsafeMutableRawPointer?,
|
private func audioDeviceIOProc(
|
||||||
_ tapStorageOut: UnsafeMutablePointer<UnsafeMutableRawPointer?>
|
_ inDevice: AudioDeviceID,
|
||||||
) {
|
_ inNow: UnsafePointer<AudioTimeStamp>,
|
||||||
// tapStorage 不需要额外分配,context 通过全局变量访问
|
_ inInputData: UnsafePointer<AudioBufferList>,
|
||||||
tapStorageOut.pointee = clientInfo
|
_ inInputTime: UnsafePointer<AudioTimeStamp>,
|
||||||
}
|
_ inOutputData: UnsafeMutablePointer<AudioBufferList>,
|
||||||
|
_ inOutputTime: UnsafePointer<AudioTimeStamp>,
|
||||||
private func tapFinalizeCallback(_ tap: MTAudioProcessingTap) {
|
_ inClientData: UnsafeMutableRawPointer?
|
||||||
// 清理由 stopRecording 负责
|
) -> OSStatus {
|
||||||
}
|
guard let clientData = inClientData else { return noErr }
|
||||||
|
let ctx = Unmanaged<AudioCaptureContext>.fromOpaque(clientData).takeUnretainedValue()
|
||||||
private func tapPrepareCallback(
|
guard ctx.isRunning,
|
||||||
_ tap: MTAudioProcessingTap,
|
|
||||||
_ maxFrames: CMItemCount,
|
|
||||||
_ processingFormat: UnsafePointer<AudioStreamBasicDescription>
|
|
||||||
) {
|
|
||||||
guard let ctx = gAudioContext else { return }
|
|
||||||
var asbd = processingFormat.pointee
|
|
||||||
ctx.sampleRate = Double(asbd.mSampleRate)
|
|
||||||
ctx.channelsPerFrame = asbd.mChannelsPerFrame
|
|
||||||
var fmtDesc: CMAudioFormatDescription?
|
|
||||||
CMAudioFormatDescriptionCreate(
|
|
||||||
allocator: kCFAllocatorDefault,
|
|
||||||
asbd: &asbd,
|
|
||||||
layoutSize: 0, layout: nil,
|
|
||||||
magicCookieSize: 0, magicCookie: nil,
|
|
||||||
extensions: nil,
|
|
||||||
formatDescriptionOut: &fmtDesc
|
|
||||||
)
|
|
||||||
ctx.formatDescription = fmtDesc
|
|
||||||
NSLog("[Recorder] tapPrepare: %.1fHz, %uch, mFormatID=%u", asbd.mSampleRate, asbd.mChannelsPerFrame, asbd.mFormatID)
|
|
||||||
}
|
|
||||||
|
|
||||||
private func tapUnprepareCallback(_ tap: MTAudioProcessingTap) {
|
|
||||||
// Nothing to clean up
|
|
||||||
}
|
|
||||||
|
|
||||||
private func tapProcessCallback(
|
|
||||||
_ tap: MTAudioProcessingTap,
|
|
||||||
_ numberFrames: CMItemCount,
|
|
||||||
_ flags: MTAudioProcessingTapFlags,
|
|
||||||
_ bufferListInOut: UnsafeMutablePointer<AudioBufferList>,
|
|
||||||
_ numberFramesOut: UnsafeMutablePointer<CMItemCount>,
|
|
||||||
_ flagsOut: UnsafeMutablePointer<MTAudioProcessingTapFlags>
|
|
||||||
) {
|
|
||||||
// 始终先获取源音频(保证音频管道正常运行)
|
|
||||||
var timeRange = CMTimeRange()
|
|
||||||
var srcFlags: MTAudioProcessingTapFlags = 0
|
|
||||||
var actualFrames: CMItemCount = 0
|
|
||||||
|
|
||||||
let status = MTAudioProcessingTapGetSourceAudio(
|
|
||||||
tap, numberFrames, bufferListInOut,
|
|
||||||
&srcFlags, &timeRange, &actualFrames
|
|
||||||
)
|
|
||||||
|
|
||||||
// 必须始终设置输出帧数和标志(否则音频引擎可能阻塞)
|
|
||||||
numberFramesOut.pointee = actualFrames
|
|
||||||
flagsOut.pointee = srcFlags
|
|
||||||
|
|
||||||
guard status == noErr, actualFrames > 0 else { return }
|
|
||||||
|
|
||||||
// 通过全局变量获取 context — 未就绪时仅跳过录制,不影响播放
|
|
||||||
guard let ctx = gAudioContext, ctx.isRunning,
|
|
||||||
let writerInput = ctx.writerInput,
|
let writerInput = ctx.writerInput,
|
||||||
writerInput.isReadyForMoreMediaData else { return }
|
writerInput.isReadyForMoreMediaData else { return noErr }
|
||||||
|
|
||||||
ctx.processCallCount += 1
|
ctx.processCallCount += 1
|
||||||
|
|
||||||
// 定期记录回调活跃状态(每500次调用打一次日志)
|
let bufferList = inInputData.pointee
|
||||||
if ctx.processCallCount == 1 || ctx.processCallCount % 500 == 0 {
|
guard bufferList.mNumberBuffers > 0 else { return noErr }
|
||||||
NSLog("[Recorder] tapProcess #%d: frames=%ld, timescale=%d",
|
let buffer = bufferList.mBuffers
|
||||||
ctx.processCallCount, actualFrames, timeRange.duration.timescale)
|
let dataSize = buffer.mDataByteSize
|
||||||
|
guard dataSize > 0, let srcData = buffer.mData else { return noErr }
|
||||||
|
|
||||||
|
// Copy to pre-allocated buffer (avoid malloc in real-time thread)
|
||||||
|
if ctx.dataBufferSize < dataSize {
|
||||||
|
if let existing = ctx.dataBuffer { free(existing) }
|
||||||
|
ctx.dataBuffer = malloc(Int(dataSize))
|
||||||
|
ctx.dataBufferSize = dataSize
|
||||||
}
|
}
|
||||||
|
guard let dest = ctx.dataBuffer else { return noErr }
|
||||||
|
memcpy(dest, srcData, Int(dataSize))
|
||||||
|
|
||||||
let bufferList = bufferListInOut.pointee
|
let pts = inInputTime.pointee
|
||||||
guard bufferList.mNumberBuffers > 0 else { return }
|
let callNum = ctx.processCallCount
|
||||||
|
|
||||||
let buf = bufferList.mBuffers
|
// Process on serial queue (off the real-time audio thread)
|
||||||
let dataSize = Int(buf.mDataByteSize)
|
ctx.audioQueue.async {
|
||||||
guard dataSize > 0, let srcData = buf.mData else { return }
|
guard ctx.isRunning, let fd = ctx.formatDescription else { return }
|
||||||
|
|
||||||
// 创建 CMBlockBuffer
|
var blockBuffer: CMBlockBuffer?
|
||||||
var blockBuffer: CMBlockBuffer?
|
let blockStatus = CMBlockBufferCreateWithMemoryBlock(
|
||||||
let blockStatus = CMBlockBufferCreateWithMemoryBlock(
|
allocator: kCFAllocatorDefault, memoryBlock: nil,
|
||||||
allocator: kCFAllocatorDefault, memoryBlock: nil,
|
blockLength: Int(dataSize), blockAllocator: kCFAllocatorDefault,
|
||||||
blockLength: dataSize, blockAllocator: kCFAllocatorDefault,
|
customBlockSource: nil, offsetToData: 0,
|
||||||
customBlockSource: nil, offsetToData: 0,
|
dataLength: Int(dataSize),
|
||||||
dataLength: dataSize,
|
flags: kCMBlockBufferAssureMemoryNowFlag,
|
||||||
flags: kCMBlockBufferAssureMemoryNowFlag,
|
blockBufferOut: &blockBuffer
|
||||||
blockBufferOut: &blockBuffer
|
)
|
||||||
)
|
guard blockStatus == kCMBlockBufferNoErr, let bb = blockBuffer else { return }
|
||||||
guard blockStatus == kCMBlockBufferNoErr, let bb = blockBuffer else { return }
|
|
||||||
|
|
||||||
let replaceStatus = CMBlockBufferReplaceDataBytes(
|
let replaceStatus = CMBlockBufferReplaceDataBytes(
|
||||||
with: srcData, blockBuffer: bb,
|
with: dest, blockBuffer: bb,
|
||||||
offsetIntoDestination: 0, dataLength: dataSize
|
offsetIntoDestination: 0, dataLength: Int(dataSize)
|
||||||
)
|
)
|
||||||
guard replaceStatus == kCMBlockBufferNoErr else { return }
|
guard replaceStatus == kCMBlockBufferNoErr else { return }
|
||||||
|
|
||||||
// 获取 format description
|
var timingInfo = CMSampleTimingInfo()
|
||||||
guard let fd = ctx.formatDescription else {
|
// Convert AudioTimeStamp to CMTime
|
||||||
if ctx.processCallCount == 1 {
|
if pts.mFlags.contains(.sampleTimeValid) {
|
||||||
NSLog("[Recorder] ⚠️ formatDescription nil on first process call")
|
let sampleRate = ctx.sampleRate
|
||||||
|
timingInfo.presentationTimeStamp = CMTime(
|
||||||
|
value: Int64(pts.mSampleTime),
|
||||||
|
timescale: Int32(sampleRate)
|
||||||
|
)
|
||||||
|
} else if pts.mFlags.contains(.hostTimeValid) {
|
||||||
|
var timebase = mach_timebase_info_data_t()
|
||||||
|
mach_timebase_info(&timebase)
|
||||||
|
let nanos = pts.mHostTime * UInt64(timebase.numer) / UInt64(timebase.denom)
|
||||||
|
timingInfo.presentationTimeStamp = CMTime(value: CMTimeValue(nanos), timescale: 1_000_000_000)
|
||||||
|
} else {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
return
|
timingInfo.duration = .invalid
|
||||||
}
|
timingInfo.decodeTimeStamp = .invalid
|
||||||
|
|
||||||
// PTS 基于累积帧数(与视频一样从0开始)
|
var sampleBuffer: CMSampleBuffer?
|
||||||
let sampleRate = ctx.sampleRate
|
let bytesPerFrame = Int(ctx.channelsPerFrame) * MemoryLayout<Float32>.size
|
||||||
let ptsValue = Double(ctx.appendCount) / sampleRate
|
let sampleCount = Int(dataSize) / max(bytesPerFrame, 1)
|
||||||
let pts = CMTime(seconds: ptsValue, preferredTimescale: Int32(sampleRate))
|
let createStatus = CMSampleBufferCreateReady(
|
||||||
|
allocator: kCFAllocatorDefault,
|
||||||
var sampleBuffer: CMSampleBuffer?
|
dataBuffer: bb,
|
||||||
let createStatus = CMAudioSampleBufferCreateReadyWithPacketDescriptions(
|
formatDescription: fd,
|
||||||
allocator: kCFAllocatorDefault,
|
sampleCount: sampleCount,
|
||||||
dataBuffer: bb,
|
sampleTimingEntryCount: 1,
|
||||||
formatDescription: fd,
|
sampleTimingArray: &timingInfo,
|
||||||
sampleCount: actualFrames,
|
sampleSizeEntryCount: 0,
|
||||||
presentationTimeStamp: pts,
|
sampleSizeArray: nil,
|
||||||
packetDescriptions: nil,
|
sampleBufferOut: &sampleBuffer
|
||||||
sampleBufferOut: &sampleBuffer
|
)
|
||||||
)
|
guard createStatus == noErr, let sb = sampleBuffer else {
|
||||||
guard createStatus == noErr, let sb = sampleBuffer else {
|
if callNum <= 3 {
|
||||||
if ctx.processCallCount <= 3 {
|
NSLog("[Recorder] \u{26a0}\u{fe0f} Audio sample buffer create failed: %d", createStatus)
|
||||||
NSLog("[Recorder] ⚠️ CMAudioSampleBufferCreate failed: %d", createStatus)
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if writerInput.isReadyForMoreMediaData && writerInput.append(sb) {
|
||||||
|
ctx.appendCount += 1
|
||||||
|
if !ctx.firstAppendLogged {
|
||||||
|
ctx.firstAppendLogged = true
|
||||||
|
NSLog("[Recorder] ✓ First audio captured: pts=%.2fs, size=%u", pts.mSampleTime / ctx.sampleRate, dataSize)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if writerInput.append(sb) {
|
return noErr
|
||||||
ctx.appendCount += actualFrames
|
|
||||||
if !ctx.firstAppendLogged {
|
|
||||||
ctx.firstAppendLogged = true
|
|
||||||
NSLog("[Recorder] ✓ First audio appended: pts=%.2fs, frames=%ld", pts.seconds, actualFrames)
|
|
||||||
}
|
|
||||||
} else if ctx.processCallCount <= 3 {
|
|
||||||
NSLog("[Recorder] ⚠️ writerInput.append failed (input ready=%d)", writerInput.isReadyForMoreMediaData)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - PlayerRecorder
|
// MARK: - PlayerRecorder
|
||||||
@ -171,7 +145,7 @@ enum RecorderState {
|
|||||||
static nonisolated(unsafe) var pendingTerminate: Bool = false
|
static nonisolated(unsafe) var pendingTerminate: Bool = false
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 视频源录制器:AVPlayerItemVideoOutput 抓帧 + MTAudioProcessingTap 捕获音频
|
/// 视频源录制器:AVPlayerItemVideoOutput 抓帧 + CoreAudio 输出设备捕获音频
|
||||||
/// 边录边写文件,录制结束后仅改名和移动
|
/// 边录边写文件,录制结束后仅改名和移动
|
||||||
@MainActor
|
@MainActor
|
||||||
final class PlayerRecorder: NSObject {
|
final class PlayerRecorder: NSObject {
|
||||||
@ -189,9 +163,10 @@ final class PlayerRecorder: NSObject {
|
|||||||
private var lastPixelBuffer: CVPixelBuffer?
|
private var lastPixelBuffer: CVPixelBuffer?
|
||||||
private var captureFrameCount: Int = 0
|
private var captureFrameCount: Int = 0
|
||||||
|
|
||||||
// 音频 (MTAudioProcessingTap)
|
// 音频 (CoreAudio 输出设备)
|
||||||
private var audioTap: MTAudioProcessingTap?
|
private var audioDeviceID: AudioDeviceID = 0
|
||||||
private var audioContext: AudioCaptureContext?
|
private var audioCaptureContext: AudioCaptureContext?
|
||||||
|
private var audioIOProc: AudioDeviceIOProc?
|
||||||
private weak var currentPlayerItem: AVPlayerItem?
|
private weak var currentPlayerItem: AVPlayerItem?
|
||||||
private var tracksObservation: NSKeyValueObservation?
|
private var tracksObservation: NSKeyValueObservation?
|
||||||
|
|
||||||
@ -286,8 +261,8 @@ final class PlayerRecorder: NSObject {
|
|||||||
startCaptureLoop()
|
startCaptureLoop()
|
||||||
startDurationTimer()
|
startDurationTimer()
|
||||||
|
|
||||||
// 启动音频捕获(MTAudioProcessingTap)
|
// 启动音频捕获 (CoreAudio)
|
||||||
setupAudioTap(playerItem: playerItem, audioWriterInput: aInput)
|
startAudioCapture(audioWriterInput: aInput)
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - 停止录制
|
// MARK: - 停止录制
|
||||||
@ -301,23 +276,14 @@ final class PlayerRecorder: NSObject {
|
|||||||
captureTimer = nil
|
captureTimer = nil
|
||||||
stopDurationTimer()
|
stopDurationTimer()
|
||||||
|
|
||||||
// 停止音频 tap
|
// 保存音频统计并停止捕获
|
||||||
audioContext?.isRunning = false
|
let audioCalls = audioCaptureContext?.processCallCount ?? 0
|
||||||
tracksObservation = nil
|
let audioAppended = audioCaptureContext?.appendCount ?? 0
|
||||||
let totalAudioFrames = audioContext?.appendCount ?? 0
|
|
||||||
let processCalls = audioContext?.processCallCount ?? 0
|
|
||||||
NSLog("[Recorder] Stopping: videoFrames=%d, audioProcessCalls=%d, audioAppended=%d",
|
NSLog("[Recorder] Stopping: videoFrames=%d, audioProcessCalls=%d, audioAppended=%d",
|
||||||
captureFrameCount, processCalls, totalAudioFrames)
|
captureFrameCount, audioCalls, audioAppended)
|
||||||
|
stopAudioCapture()
|
||||||
// 移除 audioMix
|
|
||||||
currentPlayerItem?.audioMix = nil
|
|
||||||
currentPlayerItem = nil
|
currentPlayerItem = nil
|
||||||
|
|
||||||
// 释放 tap 和 context
|
|
||||||
audioTap = nil
|
|
||||||
audioContext = nil
|
|
||||||
gAudioContext = nil
|
|
||||||
|
|
||||||
videoInput?.markAsFinished()
|
videoInput?.markAsFinished()
|
||||||
audioInput?.markAsFinished()
|
audioInput?.markAsFinished()
|
||||||
|
|
||||||
@ -338,7 +304,7 @@ final class PlayerRecorder: NSObject {
|
|||||||
NSLog("[Recorder] finishWriting: status=%d, error=%@, videoFrames=%d, audioFrames=%d",
|
NSLog("[Recorder] finishWriting: status=%d, error=%@, videoFrames=%d, audioFrames=%d",
|
||||||
w.status.rawValue,
|
w.status.rawValue,
|
||||||
w.error?.localizedDescription ?? "none",
|
w.error?.localizedDescription ?? "none",
|
||||||
vFrames, totalAudioFrames)
|
vFrames, audioAppended)
|
||||||
|
|
||||||
if w.status == .completed, let tempURL = self.tempURL {
|
if w.status == .completed, let tempURL = self.tempURL {
|
||||||
// 检查文件是否存在且有内容
|
// 检查文件是否存在且有内容
|
||||||
@ -366,205 +332,97 @@ final class PlayerRecorder: NSObject {
|
|||||||
lastPixelBuffer = nil
|
lastPixelBuffer = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - 音频 Tap 设置
|
// MARK: - 音频捕获 (CoreAudio 输出设备)
|
||||||
|
|
||||||
private func setupAudioTap(playerItem: AVPlayerItem, audioWriterInput: AVAssetWriterInput) {
|
/// 从系统音频输出设备捕获音频(适用于 HLS、本地文件等所有音频源)
|
||||||
// HLS 流的 tracks 是异步加载的
|
private func startAudioCapture(audioWriterInput: AVAssetWriterInput) {
|
||||||
// 策略:先同步检查 → KVO 观察 → 定时轮询保底(含 async load 备选)
|
// 获取默认音频输出设备
|
||||||
|
var deviceID: AudioDeviceID = 0
|
||||||
// 先打印当前 track 状态(诊断)
|
var size = UInt32(MemoryLayout<AudioDeviceID>.size)
|
||||||
let tracks = playerItem.tracks
|
var address = AudioObjectPropertyAddress(
|
||||||
NSLog("[Recorder] setupAudioTap: playerItem.tracks.count=%d", tracks.count)
|
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
|
||||||
for (i, track) in tracks.enumerated() {
|
mScope: kAudioObjectPropertyScopeGlobal,
|
||||||
if let at = track.assetTrack {
|
mElement: kAudioObjectPropertyElementMain
|
||||||
NSLog("[Recorder] track[%d]: mediaType=%@, trackID=%d, enabled=%d",
|
)
|
||||||
i, at.mediaType.rawValue as NSString, at.trackID, track.isEnabled)
|
let status = AudioObjectGetPropertyData(
|
||||||
} else {
|
AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &deviceID
|
||||||
NSLog("[Recorder] track[%d]: assetTrack=NIL (HLS not loaded yet?)", i)
|
)
|
||||||
}
|
guard status == noErr, deviceID != 0 else {
|
||||||
|
NSLog("[Recorder] \u{274c} Cannot get default audio output device: %d", status)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
self.audioDeviceID = deviceID
|
||||||
|
|
||||||
// 先尝试直接获取(本地文件场景)
|
// 获取输出音频流格式
|
||||||
if let audioTrack = self.findAudioTrackID(in: playerItem) {
|
var streamFormatAddress = AudioObjectPropertyAddress(
|
||||||
NSLog("[Recorder] ✓ Audio track immediately available: trackID=%d", audioTrack)
|
mSelector: kAudioDevicePropertyStreamFormat,
|
||||||
self.installTap(on: playerItem, trackID: audioTrack, audioWriterInput: audioWriterInput)
|
mScope: kAudioDevicePropertyScopeOutput,
|
||||||
|
mElement: kAudioObjectPropertyElementMain
|
||||||
|
)
|
||||||
|
var asbd = AudioStreamBasicDescription()
|
||||||
|
size = UInt32(MemoryLayout<AudioStreamBasicDescription>.size)
|
||||||
|
let fmtStatus = AudioObjectGetPropertyData(deviceID, &streamFormatAddress, 0, nil, &size, &asbd)
|
||||||
|
guard fmtStatus == noErr else {
|
||||||
|
NSLog("[Recorder] \u{274c} Cannot get audio stream format: %d", fmtStatus)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// tracks 尚未加载 — 用 KVO 观察变化
|
NSLog("[Recorder] Audio output format: %.1fHz, %uch, %ubit, formatID=%u",
|
||||||
NSLog("[Recorder] ⏳ Audio tracks not yet loaded, setting up KVO + polling...")
|
asbd.mSampleRate, asbd.mChannelsPerFrame, asbd.mBitsPerChannel, asbd.mFormatID)
|
||||||
|
|
||||||
let observation = playerItem.observe(\.tracks, options: [.new]) { [weak self] item, _ in
|
// 创建 context
|
||||||
guard let self, self.isRunning, self.audioTap == nil else { return }
|
|
||||||
|
|
||||||
let tCount = item.tracks.count
|
|
||||||
NSLog("[Recorder] KVO tracks changed: count=%d", tCount)
|
|
||||||
|
|
||||||
if let trackID = self.findAudioTrackID(in: item) {
|
|
||||||
NSLog("[Recorder] ✓ Audio track via KVO: trackID=%d", trackID)
|
|
||||||
Task { @MainActor in
|
|
||||||
guard self.isRunning, self.audioTap == nil else { return }
|
|
||||||
self.installTap(on: item, trackID: trackID, audioWriterInput: audioWriterInput)
|
|
||||||
}
|
|
||||||
self.tracksObservation = nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
self.tracksObservation = observation
|
|
||||||
|
|
||||||
// 保底轮询:含 async asset.load(.tracks) 多次重试
|
|
||||||
// 超时保护:20秒后放弃
|
|
||||||
Task {
|
|
||||||
var attempts = 0
|
|
||||||
var lastAsyncAttempt = -100 // 确保首次就能尝试
|
|
||||||
while attempts < 200 && self.isRunning && self.audioTap == nil { // 20秒
|
|
||||||
try await Task.sleep(nanoseconds: 100_000_000) // 0.1秒
|
|
||||||
attempts += 1
|
|
||||||
|
|
||||||
// 方式1: 从 playerItem.tracks 查找
|
|
||||||
if let trackID = self.findAudioTrackID(in: playerItem) {
|
|
||||||
NSLog("[Recorder] ✓ Audio track via polling (attempt %d): trackID=%d", attempts, trackID)
|
|
||||||
await MainActor.run {
|
|
||||||
guard self.isRunning, self.audioTap == nil else { return }
|
|
||||||
self.installTap(on: playerItem, trackID: trackID, audioWriterInput: audioWriterInput)
|
|
||||||
}
|
|
||||||
self.tracksObservation = nil
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// 方式2: 每3秒重试一次 async load asset tracks(HLS 关键路径)
|
|
||||||
if attempts - lastAsyncAttempt >= 30 {
|
|
||||||
lastAsyncAttempt = attempts
|
|
||||||
NSLog("[Recorder] Trying async asset.load(.tracks) (attempt at %d)...", attempts)
|
|
||||||
do {
|
|
||||||
let assetTracks = try await playerItem.asset.load(.tracks)
|
|
||||||
NSLog("[Recorder] async load returned %d tracks", assetTracks.count)
|
|
||||||
for (i, t) in assetTracks.enumerated() {
|
|
||||||
NSLog("[Recorder] assetTrack[%d]: mediaType=%@, trackID=%d",
|
|
||||||
i, t.mediaType.rawValue as NSString, t.trackID)
|
|
||||||
}
|
|
||||||
// 找音频 track
|
|
||||||
if let audioAssetTrack = assetTracks.first(where: { $0.mediaType == .audio }) {
|
|
||||||
let trackID = audioAssetTrack.trackID
|
|
||||||
NSLog("[Recorder] ✓ Audio track via async load: trackID=%d", trackID)
|
|
||||||
await MainActor.run {
|
|
||||||
guard self.isRunning, self.audioTap == nil else { return }
|
|
||||||
self.installTap(on: playerItem, trackID: trackID, audioWriterInput: audioWriterInput)
|
|
||||||
}
|
|
||||||
self.tracksObservation = nil
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// 没找到音频,尝试非视频非音频 track(muxed)
|
|
||||||
if let muxedTrack = assetTracks.first(where: { $0.mediaType != .video && $0.mediaType != .audio }) {
|
|
||||||
let trackID = muxedTrack.trackID
|
|
||||||
NSLog("[Recorder] ✓ Muxed track via async load: trackID=%d, trying as audio", trackID)
|
|
||||||
await MainActor.run {
|
|
||||||
guard self.isRunning, self.audioTap == nil else { return }
|
|
||||||
self.installTap(on: playerItem, trackID: trackID, audioWriterInput: audioWriterInput)
|
|
||||||
}
|
|
||||||
self.tracksObservation = nil
|
|
||||||
return
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
NSLog("[Recorder] async load failed: %@", error.localizedDescription)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 每5秒打一次状态日志
|
|
||||||
if attempts % 50 == 0 {
|
|
||||||
let tc = playerItem.tracks.count
|
|
||||||
NSLog("[Recorder] polling attempt %d: playerItem.tracks.count=%d, tapInstalled=%d",
|
|
||||||
attempts, tc, self.audioTap != nil ? 1 : 0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if self.audioTap == nil {
|
|
||||||
NSLog("[Recorder] ❌ No audio track found after %d polling attempts (20s timeout)", attempts)
|
|
||||||
self.tracksObservation = nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// 从 playerItem.tracks 查找音频 trackID
|
|
||||||
/// 对 HLS 流,多路 fallback:playerItem.tracks → asset.tracks → 非视频推断
|
|
||||||
private func findAudioTrackID(in playerItem: AVPlayerItem) -> CMPersistentTrackID? {
|
|
||||||
// 方式1: playerItem.tracks + assetTrack(本地文件 / 已加载的HLS)
|
|
||||||
let itemTracks = playerItem.tracks
|
|
||||||
for track in itemTracks {
|
|
||||||
if let assetTrack = track.assetTrack, assetTrack.mediaType == .audio {
|
|
||||||
NSLog("[Recorder] findTrack: audio via playerItem.tracks, trackID=%d", assetTrack.trackID)
|
|
||||||
return assetTrack.trackID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 方式2: asset.tracks(HLS 流可能通过这个路径能拿到)
|
|
||||||
let assetTracks = playerItem.asset.tracks
|
|
||||||
for track in assetTracks {
|
|
||||||
if track.mediaType == .audio {
|
|
||||||
NSLog("[Recorder] findTrack: audio via asset.tracks, trackID=%d", track.trackID)
|
|
||||||
return track.trackID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 方式3: 如果有 track 但不是音频,找非视频的 track(可能是 muxed 或音频)
|
|
||||||
for track in itemTracks {
|
|
||||||
if let assetTrack = track.assetTrack,
|
|
||||||
assetTrack.mediaType != .video, assetTrack.mediaType != .audio {
|
|
||||||
NSLog("[Recorder] findTrack: non-video track (mediaType=%@, trackID=%d), trying as audio",
|
|
||||||
assetTrack.mediaType.rawValue as NSString, assetTrack.trackID)
|
|
||||||
return assetTrack.trackID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
private func installTap(on playerItem: AVPlayerItem, trackID: CMPersistentTrackID, audioWriterInput: AVAssetWriterInput) {
|
|
||||||
// 创建音频捕获上下文
|
|
||||||
let context = AudioCaptureContext()
|
let context = AudioCaptureContext()
|
||||||
context.writerInput = audioWriterInput
|
context.writerInput = audioWriterInput
|
||||||
|
context.sampleRate = Double(asbd.mSampleRate)
|
||||||
|
context.channelsPerFrame = asbd.mChannelsPerFrame
|
||||||
context.isRunning = true
|
context.isRunning = true
|
||||||
self.audioContext = context
|
self.audioCaptureContext = context
|
||||||
|
|
||||||
// 全局引用,供 C 回调访问
|
// 创建格式描述
|
||||||
gAudioContext = context
|
var fmtDesc: CMAudioFormatDescription?
|
||||||
|
var mutableASBD = asbd
|
||||||
// 创建 MTAudioProcessingTap 回调结构体
|
CMAudioFormatDescriptionCreate(
|
||||||
var callbacks = MTAudioProcessingTapCallbacks(
|
allocator: kCFAllocatorDefault,
|
||||||
version: kMTAudioProcessingTapCallbacksVersion_0,
|
asbd: &mutableASBD,
|
||||||
clientInfo: nil,
|
layoutSize: 0, layout: nil,
|
||||||
init: tapInitCallback,
|
magicCookieSize: 0, magicCookie: nil,
|
||||||
finalize: tapFinalizeCallback,
|
extensions: nil,
|
||||||
prepare: tapPrepareCallback,
|
formatDescriptionOut: &fmtDesc
|
||||||
unprepare: tapUnprepareCallback,
|
|
||||||
process: tapProcessCallback
|
|
||||||
)
|
)
|
||||||
|
context.formatDescription = fmtDesc
|
||||||
|
|
||||||
var tap: MTAudioProcessingTap?
|
// 创建 IOProc(使用 unmanaged pointer 传递 context)
|
||||||
let status = MTAudioProcessingTapCreate(
|
let contextPtr = Unmanaged.passUnretained(context).toOpaque()
|
||||||
kCFAllocatorDefault,
|
var ioProc: AudioDeviceIOProc?
|
||||||
&callbacks,
|
let createStatus = AudioDeviceCreateIOProcID(deviceID, audioDeviceIOProc, contextPtr, &ioProc)
|
||||||
kMTAudioProcessingTapCreationFlag_PostEffects,
|
guard createStatus == noErr, let proc = ioProc else {
|
||||||
&tap
|
NSLog("[Recorder] \u{274c} AudioDeviceCreateIOProcID failed: %d", createStatus)
|
||||||
)
|
|
||||||
|
|
||||||
guard status == noErr, let audioTap = tap else {
|
|
||||||
NSLog("[Recorder] ❌ MTAudioProcessingTapCreate failed: %d", status)
|
|
||||||
gAudioContext = nil
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
self.audioIOProc = proc
|
||||||
|
|
||||||
self.audioTap = audioTap
|
// 启动音频捕获
|
||||||
|
let startStatus = AudioDeviceStart(deviceID, proc)
|
||||||
// 关键:用 playerItem 的 trackID 创建 audioMix input parameters
|
guard startStatus == noErr else {
|
||||||
let params = AVMutableAudioMixInputParameters()
|
NSLog("[Recorder] \u{274c} AudioDeviceStart failed: %d", startStatus)
|
||||||
params.trackID = trackID
|
AudioDeviceDestroyIOProcID(deviceID, proc)
|
||||||
params.audioTapProcessor = audioTap
|
self.audioIOProc = nil
|
||||||
let audioMix = AVMutableAudioMix()
|
return
|
||||||
audioMix.inputParameters = [params]
|
}
|
||||||
|
NSLog("[Recorder] \u{2713} CoreAudio output device capture started (deviceID=%u)", deviceID)
|
||||||
playerItem.audioMix = audioMix
|
|
||||||
NSLog("[Recorder] ✓ Audio tap installed on playerItem (trackID=%d)", trackID)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func stopAudioCapture() {
|
||||||
|
audioCaptureContext?.isRunning = false
|
||||||
|
if let proc = audioIOProc, audioDeviceID != 0 {
|
||||||
|
AudioDeviceStop(audioDeviceID, proc)
|
||||||
|
AudioDeviceDestroyIOProcID(audioDeviceID, proc)
|
||||||
|
}
|
||||||
|
audioIOProc = nil
|
||||||
|
audioCaptureContext = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// MARK: - 视频抓帧 (30fps Timer)
|
// MARK: - 视频抓帧 (30fps Timer)
|
||||||
|
|
||||||
private func startCaptureLoop() {
|
private func startCaptureLoop() {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user