From 5ae4cc4e70e0bb368da66ad954d258e6ee3ff0e8 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Wed, 24 Jun 2026 23:14:47 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20HLS=E6=B5=81=E9=9F=B3=E9=A2=91=E5=BD=95?= =?UTF-8?q?=E5=88=B6=E2=80=94=E2=80=94MTAudioProcessingTap=E6=9B=BF?= =?UTF-8?q?=E4=BB=A3AVAssetReader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 问题: AVAssetReader不支持HLS流(m3u8)，录制HLS时音频丢失。 方案: 统一使用MTAudioProcessingTap捕获音频 - 从播放管道直接获取PCM音频数据，本地文件和HLS流通用 - tap回调通过MTAudioProcessingTapGetSourceAudio获取源音频+时间范围 - AudioCaptureContext通过Unmanaged传递给C回调 - init/finalize回调管理context生命周期 - 视频仍用AVPlayerItemVideoOutput(已验证稳定) API变更: - startRecording(from:playerItem:startTime:) 替代 sourceURL - PlayerBridge传入playerItem而非URL --- Sources/PlayerBridge.swift | 7 +- Sources/PlayerRecorder.swift | 394 ++++++++++++++++++++--------------- 2 files changed, 230 insertions(+), 171 deletions(-) diff --git a/Sources/PlayerBridge.swift b/Sources/PlayerBridge.swift index 613848c..a97d0cf 100644 --- a/Sources/PlayerBridge.swift +++ b/Sources/PlayerBridge.swift @@ -121,9 +121,12 @@ final class PlayerBridge: ObservableObject { return } guard !queue.isEmpty else { return } - let sourceURL = queue[currentIndex].url let startTime = player.currentTime() - screenRecorder.startRecording(from: output, sourceURL: sourceURL, startTime: startTime) + guard let playerItem = player.currentItem else { + showToast("No player item") + return + } + screenRecorder.startRecording(from: output, playerItem: playerItem, startTime: startTime) } #endif } diff --git a/Sources/PlayerRecorder.swift b/Sources/PlayerRecorder.swift index 7218272..0f9119b 100644 --- a/Sources/PlayerRecorder.swift +++ b/Sources/PlayerRecorder.swift @@ -1,8 +1,151 @@ import AVFoundation +import CoreMedia +import MediaToolbox #if os(macOS) import AppKit -/// 视频源录制器：从 AVPlayerItemVideoOutput 抓帧 + 源 URL 音频轨道，写入 mp4 +// MARK: - 音频捕获上下文（通过 tapStorage 传给 C 回调） +class AudioCaptureContext { + var writerInput: AVAssetWriterInput? + nonisolated(unsafe) var isRunning: Bool = true + nonisolated(unsafe) var formatDescription: CMAudioFormatDescription? + nonisolated(unsafe) var appendCount: Int = 0 +} + +// MARK: - MTAudioProcessingTap C 回调函数 + +private func tapInit(_ tap: MTAudioProcessingTap, + _ clientInfo: UnsafeMutableRawPointer?, + _ tapStorageOut: UnsafeMutablePointer) { + // 将 clientInfo 存入 tapStorage + tapStorageOut.pointee = clientInfo +} + +private func tapFinalize(_ tap: MTAudioProcessingTap) { + // 释放 context 的 Unmanaged 引用 + guard let storage = Optional(MTAudioProcessingTapGetStorage(tap)), + let ctxPtr = storage.assumingMemoryBound(to: UnsafeMutableRawPointer?.self).pointee else { return } + Unmanaged.fromOpaque(ctxPtr).release() +} + +private func tapPrepare(_ tap: MTAudioProcessingTap, + _ maxFrames: Int, + _ processingFormat: UnsafePointer) { + guard let storage = Optional(MTAudioProcessingTapGetStorage(tap)), + let ctxPtr = storage.assumingMemoryBound(to: UnsafeMutableRawPointer?.self).pointee else { return } + let ctx = Unmanaged.fromOpaque(ctxPtr).takeUnretainedValue() + + var asbd = processingFormat.pointee + var fmtDesc: CMAudioFormatDescription? + CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, + asbd: &asbd, + layoutSize: 0, layout: nil, + magicCookieSize: 0, magicCookie: nil, + extensions: nil, + formatDescriptionOut: &fmtDesc) + ctx.formatDescription = fmtDesc + print("[Recorder] Audio tap prepared: \(asbd.mSampleRate)Hz, \(asbd.mChannelsPerFrame)ch, format=\(asbd.mFormatID)") +} + +private func tapUnprepare(_ tap: MTAudioProcessingTap) { + // Nothing to clean up +} + +private func tapProcess(_ tap: MTAudioProcessingTap, + _ numberOfFrames: CMItemCount, + _ flags: MTAudioProcessingTapFlags, + _ bufferListInOut: UnsafeMutablePointer, + _ numberFramesOut: UnsafeMutablePointer, + _ flagsOut: UnsafeMutablePointer) { + numberFramesOut.pointee = 0 + flagsOut.pointee = 0 + + guard numberFrames > 0 else { return } + + // 获取源音频数据 + 时间范围 + var timeRange = CMTimeRange() + var srcFlags: UInt32 = 0 + var actualFrames: Int = 0 + + let status = MTAudioProcessingTapGetSourceAudio( + tap, + numberFrames, + bufferListInOut, + &srcFlags, + &timeRange, + &actualFrames + ) + + numberFramesOut.pointee = actualFrames + flagsOut.pointee = srcFlags + + guard status == noErr, actualFrames > 0 else { return } + + // 获取 context + guard let storage = Optional(MTAudioProcessingTapGetStorage(tap)), + let ctxPtr = storage.assumingMemoryBound(to: UnsafeMutableRawPointer?.self).pointee else { return } + let ctx = Unmanaged.fromOpaque(ctxPtr).takeUnretainedValue() + + guard ctx.isRunning, let writerInput = ctx.writerInput, + writerInput.isReadyForMoreMediaData else { return } + + let bufferList = bufferListInOut.pointee + guard bufferList.mNumberBuffers > 0 else { return } + + // 计算总字节数 + var totalBytes: UInt32 = 0 + for i in 0.. 0, let srcData = bufferList.mBuffers.mData else { return } + + // 创建 CMBlockBuffer + var blockBuffer: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: nil, + blockLength: Int(totalBytes), + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, + offsetToData: 0, + dataLength: Int(totalBytes), + flags: kCMBlockBufferAssureMemoryNowFlag, + blockBufferOut: &blockBuffer + ) == kCMBlockBufferNoErr, let bb = blockBuffer else { return } + + // 复制音频数据 + guard CMBlockBufferReplaceDataBytes( + with: srcData, + blockBuffer: bb, + offsetIntoDestination: 0, + dataLength: Int(totalBytes) + ) == kCMBlockBufferNoErr else { return } + + // 获取格式描述 + guard let fmtDesc = ctx.formatDescription else { return } + + // 创建 CMSampleBuffer（使用 timeRange 的 start 作为 PTS） + var sampleBuffer: CMSampleBuffer? + let createStatus = CMAudioSampleBufferCreateReadyWithPacketDescriptions( + allocator: kCFAllocatorDefault, + dataBuffer: bb, + formatDescription: fmtDesc, + sampleCount: actualFrames, + presentationTimeStamp: timeRange.start, + packetDescriptions: nil, + sampleBufferOut: &sampleBuffer + ) + + guard createStatus == noErr, let sb = sampleBuffer else { return } + + if writerInput.append(sb) { + ctx.appendCount += actualFrames + } +} + +/// 视频源录制器：AVPlayerItemVideoOutput 抓帧 + MTAudioProcessingTap 捕获音频 @MainActor final class PlayerRecorder: NSObject { @@ -14,19 +157,15 @@ final class PlayerRecorder: NSObject { private var weakOutput: AVPlayerItemVideoOutput? nonisolated(unsafe) private var isRunning = false - // 视频：缓存上一帧，确保静态场景也能持续写入 + // 视频：缓存上一帧 private var lastPixelBuffer: CVPixelBuffer? private var captureFrameCount: Int = 0 - // 音频 - private var audioReader: AVAssetReader? - private var audioReaderOutput: AVAssetReaderTrackOutput? - private nonisolated(unsafe) var audioFeedTimer: DispatchSourceTimer? + // 音频 (MTAudioProcessingTap) + private var audioTap: MTAudioProcessingTap? + private weak var currentPlayerItem: AVPlayerItem? - // 共享时间基准：player.currentTime() at record start nonisolated(unsafe) private var recordStartTime: CMTime = .zero - nonisolated(unsafe) private var audioStartHostTime: CFTimeInterval = 0 - nonisolated(unsafe) private var pendingAudioSample: CMSampleBuffer? @Published var isRecording = false @Published var durationText = "00:00" @@ -44,16 +183,15 @@ final class PlayerRecorder: NSObject { } // MARK: - 开始录制 - func startRecording(from output: AVPlayerItemVideoOutput, sourceURL: URL, startTime: CMTime) { + func startRecording(from output: AVPlayerItemVideoOutput, playerItem: AVPlayerItem, startTime: CMTime) { guard !isRecording else { return } weakOutput = output + currentPlayerItem = playerItem lastPixelBuffer = nil captureFrameCount = 0 isRunning = false - recordStartTime = startTime // 共享时间基准 - audioStartHostTime = 0 - pendingAudioSample = nil + recordStartTime = startTime // 临时文件 let formatter = DateFormatter() @@ -98,16 +236,14 @@ final class PlayerRecorder: NSObject { audioInput = aInput } - let success = writer.startWriting() - if !success { + guard writer.startWriting() else { print("[Recorder] startWriting failed: \(writer.error?.localizedDescription ?? "unknown")") onError?("Cannot start writer: \(writer.error?.localizedDescription ?? "unknown")") return } - // session 起点为 0，视频和音频都用相对时间戳 writer.startSession(atSourceTime: .zero) - print("[Recorder] Writer started, session at .zero") + print("[Recorder] Writer started") // 启动视频抓帧 isRecording = true @@ -115,8 +251,8 @@ final class PlayerRecorder: NSObject { startTimer() startCaptureLoop() - // 启动音频读取 - startAudioCapture(sourceURL: sourceURL, startTime: startTime, audioWriterInput: aInput) + // 启动音频捕获 + setupAudioTap(playerItem: playerItem, audioWriterInput: aInput) } // MARK: - 停止录制 @@ -128,15 +264,10 @@ final class PlayerRecorder: NSObject { captureTimer = nil stopTimer() - // 停止音频 timer - audioFeedTimer?.cancel() - audioFeedTimer = nil - - // 停止音频 reader - audioReader?.cancelReading() - audioReader = nil - audioReaderOutput = nil - pendingAudioSample = nil + // 停止音频 tap + currentPlayerItem?.audioMix = nil + currentPlayerItem = nil + audioTap = nil videoInput?.markAsFinished() audioInput?.markAsFinished() @@ -144,8 +275,8 @@ final class PlayerRecorder: NSObject { writer?.finishWriting { [weak self] in Task { @MainActor in guard let self = self, let w = self.writer else { return } - let count = self.captureFrameCount - print("[Recorder] finishWriting status: \(w.status.rawValue), error: \(w.error?.localizedDescription ?? "none"), frames: \(count)") + let vFrames = self.captureFrameCount + print("[Recorder] finishWriting status: \(w.status.rawValue), error: \(w.error?.localizedDescription ?? "none"), video frames: \(vFrames)") if w.status == .completed, let url = self.tempURL { self.showSaveDialog(tempURL: url) } else { @@ -160,6 +291,69 @@ final class PlayerRecorder: NSObject { lastPixelBuffer = nil } + // MARK: - 音频 Tap 设置 + private func setupAudioTap(playerItem: AVPlayerItem, audioWriterInput: AVAssetWriterInput) { + let asset = playerItem.asset + + Task { + do { + let audioTracks = try await asset.loadTracks(withMediaType: .audio) + guard let audioTrack = audioTracks.first else { + print("[Recorder] No audio track for tap") + return + } + guard self.isRunning else { return } + + // 创建 context，用 Unmanaged 传递给 C 回调 + let context = AudioCaptureContext() + context.writerInput = audioWriterInput + context.isRunning = true + + let contextPtr = Unmanaged.passRetained(context).toOpaque() + + // 创建回调结构体 + var callbacks = MTAudioProcessingTapCallbacks( + version: kMTAudioProcessingTapCallbacksVersion_0, + clientInfo: contextPtr, + init: tapInit, + finalize: tapFinalize, + prepare: tapPrepare, + unprepare: tapUnprepare, + process: tapProcess + ) + + var tap: MTAudioProcessingTap? + let status = withUnsafePointer(to: &callbacks) { cbPtr in + MTAudioProcessingTapCreate( + kCFAllocatorDefault, + cbPtr, + kMTAudioProcessingTapCreationFlag_PostEffects, + &tap + ) + } + + guard status == noErr, let audioTap = tap else { + print("[Recorder] MTAudioProcessingTapCreate failed: \(status)") + Unmanaged.fromOpaque(contextPtr).release() + return + } + + self.audioTap = audioTap + + // 创建 audioMix + let params = AVMutableAudioMixInputParameters(track: audioTrack) + params.audioTapProcessor = audioTap + let audioMix = AVMutableAudioMix() + audioMix.inputParameters = [params] + + playerItem.audioMix = audioMix + print("[Recorder] Audio tap installed") + } catch { + print("[Recorder] Audio tap setup failed: \(error)") + } + } + } + // MARK: - 视频抓帧 (30fps Timer) private var captureTimer: Timer? @@ -178,14 +372,12 @@ final class PlayerRecorder: NSObject { let hostTime = CACurrentMediaTime() let itemTime = output.itemTime(forHostTime: hostTime) - // 尝试获取新帧，否则复用上一帧 if let pb = output.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) { lastPixelBuffer = pb } guard let pb = lastPixelBuffer else { return } - // 相对时间戳：itemTime - recordStartTime let relativeTime = CMTimeSubtract(itemTime, recordStartTime) guard relativeTime.seconds >= 0 else { return } @@ -213,143 +405,7 @@ final class PlayerRecorder: NSObject { return sampleBuffer } - // MARK: - 音频读取（DispatchSourceTimer 限速到实时） - private func startAudioCapture(sourceURL: URL, startTime: CMTime, audioWriterInput: AVAssetWriterInput) { - let asset = AVURLAsset(url: sourceURL) - - Task { - do { - let audioTracks = try await asset.loadTracks(withMediaType: .audio) - guard let audioTrack = audioTracks.first else { - print("[Recorder] No audio track in source") - return - } - guard self.isRunning else { return } - - let reader = try AVAssetReader(asset: asset) - let readerOutput = AVAssetReaderTrackOutput(track: audioTrack, outputSettings: [ - AVFormatIDKey: kAudioFormatLinearPCM, - AVSampleRateKey: 44100, - AVNumberOfChannelsKey: 2, - AVLinearPCMBitDepthKey: 16, - AVLinearPCMIsFloatKey: false, - AVLinearPCMIsBigEndianKey: false, - AVLinearPCMIsNonInterleaved: false - ]) - readerOutput.alwaysCopiesSampleData = false - - // 从播放位置开始读取 - reader.timeRange = CMTimeRange(start: startTime, duration: .positiveInfinity) - - if reader.canAdd(readerOutput) { - reader.add(readerOutput) - reader.startReading() - - self.audioReader = reader - self.audioReaderOutput = readerOutput - - // 记录 wall-clock 起始时间，用于实时限速 - self.audioStartHostTime = CACurrentMediaTime() - - // 用 DispatchSourceTimer 每 20ms 喂一次音频，限速到实时 - let queue = DispatchQueue(label: "recorder.audio.feed") - let feedTimer = DispatchSource.makeTimerSource(queue: queue) - feedTimer.schedule(deadline: .now() + .milliseconds(10), repeating: .milliseconds(20)) - feedTimer.setEventHandler { [weak self] in - self?.feedAudioSamplesRealtime(writerInput: audioWriterInput, - readerOutput: readerOutput, - reader: reader) - } - self.audioFeedTimer = feedTimer - feedTimer.resume() - } - } catch { - print("[Recorder] Audio capture setup failed: \(error)") - } - } - } - - /// 每 20ms 调用，只写入不超过当前实时进度的音频 - private nonisolated func feedAudioSamplesRealtime(writerInput: AVAssetWriterInput, - readerOutput: AVAssetReaderTrackOutput, - reader: AVAssetReader) { - guard isRunning, reader.status == .reading else { - if isRunning && reader.status != .reading { - writerInput.markAsFinished() - audioFeedTimer?.cancel() - } - return - } - guard writerInput.isReadyForMoreMediaData else { return } - - // 实时进度：录制开始以来经过的秒数 - let elapsed = CACurrentMediaTime() - audioStartHostTime - let maxAudioTime = elapsed + 0.05 // 允许 50ms 预读 - - // 先处理上次缓存的超前 sample - if let pending = pendingAudioSample { - let pts = CMSampleBufferGetPresentationTimeStamp(pending) - let relative = CMTimeSubtract(pts, recordStartTime) - if relative.seconds > maxAudioTime { - return // 还没到时间，等下次 tick - } - pendingAudioSample = nil - if let nb = Self.remapAudioBuffer(pending, offset: recordStartTime) { - _ = writerInput.append(nb) - } - } - - // 继续读取，直到追上实时进度 - while isRunning, reader.status == .reading, writerInput.isReadyForMoreMediaData { - guard let sampleBuffer = readerOutput.copyNextSampleBuffer() else { - // 源文件音频读完 - if reader.status != .reading { - writerInput.markAsFinished() - audioFeedTimer?.cancel() - } - return - } - - let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - let relative = CMTimeSubtract(pts, recordStartTime) - - // 超出实时进度 → 缓存，等下次 tick - if relative.seconds > maxAudioTime { - pendingAudioSample = sampleBuffer - return - } - - // 跳过负时间戳（音频 pts 在 recordStartTime 之前） - guard relative.seconds >= 0 else { continue } - - if let nb = Self.remapAudioBuffer(sampleBuffer, offset: recordStartTime) { - if !writerInput.append(nb) { - return - } - } - } - } - - /// 重映射音频 sample buffer 的时间戳为相对时间 - private nonisolated static func remapAudioBuffer(_ sampleBuffer: CMSampleBuffer, offset: CMTime) -> CMSampleBuffer? { - var count: CMItemCount = 0 - CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, entryCount: 0, arrayToFill: nil, entriesNeededOut: &count) - guard count > 0 else { return sampleBuffer } - - var timingInfo = [CMSampleTimingInfo](repeating: CMSampleTimingInfo(), count: count) - CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, entryCount: count, arrayToFill: &timingInfo, entriesNeededOut: nil) - - for i in 0..