From 183267208110feb73934b1e86a62d6704b4f05b2 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Fri, 17 Jul 2026 21:47:01 +0800 Subject: [PATCH] fix(iOS): add MTAudioProcessingTap for audio capture, fix PTS timing - MTAudioProcessingTap captures audio samples from AVPlayer's audio track - Audio samples written to AVAssetWriter alongside video frames - First frame PTS used as session start (fixes duration stretch) - Cleaned up unused properties --- Sources/HLSRecorder.swift | 237 +++++++++++++++++++++++++++----------- 1 file changed, 167 insertions(+), 70 deletions(-) diff --git a/Sources/HLSRecorder.swift b/Sources/HLSRecorder.swift index d78fc55..8580178 100644 --- a/Sources/HLSRecorder.swift +++ b/Sources/HLSRecorder.swift @@ -2,10 +2,9 @@ import Foundation import AVFoundation import UIKit -import Photos +import MediaToolbox -/// iOS 录制器 — 从 AVPlayer 播放输出捕获视频帧和音频，写入 MP4 -/// 替代手动下载 HLS 片段的方案（对 CBS 等有 DRM/CDN 保护的流无效） +/// iOS 录制器 — 从 AVPlayer 播放输出捕获视频帧 + 音频，写入 MP4 @MainActor final class HLSRecorder: ObservableObject { @Published var isRecording = false @@ -22,18 +21,15 @@ final class HLSRecorder: ObservableObject { private var startTime: Date? private var timer: Timer? private var outputURL: URL? - private var lastVideoSampleTime: CMTime? - private var audioEngine: AVAudioEngine? - private var audioFile: AVAudioFile? - private var audioTapInstalled = false + private var firstPTS: CMTime? + private var audioTapID: MTAudioProcessingTapID? func startRecording(url: URL, player: AVPlayer) { guard !isRecording, let item = player.currentItem else { return } isRecording = true startTime = Date() - lastVideoSampleTime = nil + firstPTS = nil - // Timer for duration display timer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in Task { @MainActor in guard let self, let start = self.startTime else { return } @@ -42,13 +38,10 @@ final class HLSRecorder: ObservableObject { } } - // Output file let docs = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] outputURL = docs.appendingPathComponent("MiniPlayer_Recording_\(Int(Date().timeIntervalSince1970)).mp4") - Task { - await startCapture(item: item, player: player) - } + Task { await startCapture(item: item, player: player) } } func stopRecording() { @@ -56,8 +49,8 @@ final class HLSRecorder: ObservableObject { isRecording = false timer?.invalidate() displayLink?.invalidate() + videoOutput = nil - // Finalize the writer videoInput?.markAsFinished() audioInput?.markAsFinished() @@ -65,23 +58,23 @@ final class HLSRecorder: ObservableObject { DispatchQueue.main.async { guard let self, let url = self.outputURL else { return } let status = self.assetWriter?.status ?? .unknown - NSLog("[MiniPlayer] AVAssetWriter finished: %d, file: %@", status.rawValue, url.path) + NSLog("[MiniPlayer] AVAssetWriter finished: %d", status.rawValue) if status == .completed { let attrs = try? FileManager.default.attributesOfItem(atPath: url.path) let size = (attrs?[.size] as? Int64) ?? 0 - NSLog("[MiniPlayer] Recorded file size: %lld bytes", size) + NSLog("[MiniPlayer] Recorded: %lld bytes", size) self.onRecordingSaved?(url) } else { let err = self.assetWriter?.error?.localizedDescription ?? "unknown" - self.onError?("Export failed: \(err)") + self.onError?("Export: \(err)") try? FileManager.default.removeItem(at: url) } } } } - // MARK: - Capture + // MARK: - Capture setup private func startCapture(item: AVPlayerItem, player: AVPlayer) async { guard let outputURL else { return } @@ -89,12 +82,16 @@ final class HLSRecorder: ObservableObject { do { let writer = try AVAssetWriter(url: outputURL, fileType: .mp4) - // Video input: use source format from the asset when possible - let videoSettings: [String: Any] - let naturalSize = try? await item.asset.load(.tracks).first(where: { $0.mediaType == .video })?.naturalSize - let size = naturalSize ?? CGSize(width: 1280, height: 720) + // Natural size from asset + var size = CGSize(width: 1280, height: 720) + if let tracks = try? await item.asset.load(.tracks), + let videoTrack = tracks.first(where: { $0.mediaType == .video }) { + let ns = try? await videoTrack.load(.naturalSize) + if let ns, ns.width > 0 { size = ns } + } - videoSettings = [ + // Video input + let videoSettings: [String: Any] = [ AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: size.width, AVVideoHeightKey: size.height, @@ -103,12 +100,10 @@ final class HLSRecorder: ObservableObject { AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel ] ] - let vidInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings) vidInput.expectsMediaDataInRealTime = true - vidInput.transform = CGAffineTransform(translationX: 0, y: 0) guard writer.canAdd(vidInput) else { - throw NSError(domain: "Recorder", code: 1, userInfo: [NSLocalizedDescriptionKey: "Cannot add video input"]) + throw NSError(domain: "Rec", code: 1, userInfo: [NSLocalizedDescriptionKey: "Cannot add video input"]) } writer.add(vidInput) self.videoInput = vidInput @@ -123,32 +118,34 @@ final class HLSRecorder: ObservableObject { let audInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings) audInput.expectsMediaDataInRealTime = true guard writer.canAdd(audInput) else { - throw NSError(domain: "Recorder", code: 2, userInfo: [NSLocalizedDescriptionKey: "Cannot add audio input"]) + throw NSError(domain: "Rec", code: 2, userInfo: [NSLocalizedDescriptionKey: "Cannot add audio input"]) } writer.add(audInput) self.audioInput = audInput - // Start writing - writer.startWriting() - writer.startSession(atSourceTime: .zero) - self.assetWriter = writer + // MTAudioProcessingTap on the player item's audio track + installAudioTap(on: item) // Video output from player - let vo = AVPlayerItemVideoOutput(pixelBufferAttributes: [ + let pixAttrs: [String: Any] = [ kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, kCVPixelBufferWidthKey as String: size.width, kCVPixelBufferHeightKey as String: size.height - ]) + ] + let vo = AVPlayerItemVideoOutput(pixelBufferAttributes: pixAttrs) vo.suppressesPlayerRendering = false item.add(vo) self.videoOutput = vo - // Capture loop via CADisplayLink + writer.startWriting() + // session start will be set on first frame + self.assetWriter = writer + displayLink = CADisplayLink(target: self, selector: #selector(captureFrame)) - displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: 24, maximum: 30, preferred: 30) + displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: 15, maximum: 30, preferred: 30) displayLink?.add(to: .main, forMode: .common) - NSLog("[MiniPlayer] Capture started, output: %@", outputURL.path) + NSLog("[MiniPlayer] Capture started: %@", outputURL.path) } catch { await MainActor.run { @@ -159,50 +156,150 @@ final class HLSRecorder: ObservableObject { } } - @objc private func captureFrame() { - guard let writer = assetWriter, writer.status == .writing else { return } + // MARK: - Audio tap - // Video frame capture - if let vo = videoOutput, videoInput?.isReadyForMoreMediaData == true { - let itemTime = vo.itemTime(forHostTime: CACurrentMediaTime()) - if vo.hasNewPixelBuffer(forItemTime: itemTime), - let buf = vo.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) { - if lastVideoSampleTime == nil { - lastVideoSampleTime = itemTime - } - let sampleBuf = createSampleBuffer(from: buf, pts: itemTime) - if let sb = sampleBuf { - videoInput?.append(sb) + private func installAudioTap(on item: AVPlayerItem) { + guard let audioTrack = item.asset.tracks(withMediaType: .audio).first else { + NSLog("[MiniPlayer] No audio track found") + return + } + + var callbacks = MTAudioProcessingTapCallbacks( + version: kMTAudioProcessingTapCallbacksVersion_0, + clientInfo: Unmanaged.passUnretained(self).toOpaque(), + init: { tap, clientInfo, tapStorageOut in + tapStorageOut?.pointee = clientInfo + }, + finalize: { _ in }, + prepare: { _, _, _ in }, + unprepare: { _, _, _ in }, + process: { tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut in + guard let clientInfo = MTAudioProcessingTapGetStorage(tap) else { + numberFramesOut.pointee = 0 + return } + let recorder = Unmanaged.fromOpaque(clientInfo).takeUnretainedValue() + recorder.processAudio(tap: tap, numberFrames: numberFrames, flags: flags, + bufferListInOut: bufferListInOut, + numberFramesOut: numberFramesOut, flagsOut: flagsOut) } + ) + + var tap: Unmanaged? + let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, + kMTAudioProcessingTapCreationFlag_PreEffects, &tap) + guard status == noErr, let audioTap = tap?.takeRetainedValue() else { + NSLog("[MiniPlayer] MTAudioProcessingTapCreate failed: %d", status) + return + } + + let inputParams = AVMutableAudioMixInputParameters(track: audioTrack) + inputParams.audioTapProcessor = audioTap + + let audioMix = AVMutableAudioMix() + audioMix.inputParameters = [inputParams] + item.audioMix = audioMix + + NSLog("[MiniPlayer] Audio tap installed") + } + + private func processAudio(tap: MTAudioProcessingTap, numberFrames: CMItemCount, flags: MTAudioProcessingTapFlags, + bufferListInOut: UnsafeMutablePointer, + numberFramesOut: UnsafeMutablePointer, + flagsOut: UnsafeMutablePointer) { + // Get source audio + var sourceFlags = flags + var timeRange = CMTimeRange() + let status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, + flagsOut, &timeRange, numberFramesOut) + guard status == noErr, let input = audioInput, input.isReadyForMoreMediaData else { + flagsOut.pointee = flags + return + } + + // Create CMSampleBuffer from the audio buffer list + let pts = timeRange.start + var asbd = AudioStreamBasicDescription() + MTAudioProcessingTapGetStreamDescription(tap, &asbd) + + var formatDesc: CMAudioFormatDescription? + CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &asbd, + layoutSize: 0, layout: nil, 0, nil, nil, + formatDescriptionOut: &formatDesc) + guard let fd = formatDesc else { return } + + var sampleBuffer: CMSampleBuffer? + let numSamples = numberFramesOut.pointee + + var timing = CMSampleTimingInfo(duration: CMTime(value: CMTimeValue(numSamples), timescale: CMTimeScale(asbd.mSampleRate)), + presentationTimeStamp: pts, decodeTimeStamp: .invalid) + + CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, dataReady: false, + makeDataReadyCallback: nil, refcon: nil, formatDescription: fd, + sampleCount: numSamples, sampleTimingEntryCount: 1, + sampleTimingArray: &timing, sampleSizeEntryCount: 0, + sampleSizeArray: nil, sampleBufferOut: &sampleBuffer) + + guard var sb = sampleBuffer else { return } + + // Copy audio data into the sample buffer + let bufList = bufferListInOut.pointee + var blockBuffer: CMBlockBuffer? + CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: nil, + blockLength: Int(bufList.mBuffers.mDataByteSize), + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, + offsetToData: 0, + dataLength: Int(bufList.mBuffers.mDataByteSize), + flags: 0, + blockBufferOut: &blockBuffer + ) + if let bb = blockBuffer { + CMBlockBufferReplaceDataBytes(with: bufList.mBuffers.mData!, blockBuffer: bb, + offsetIntoDestination: 0, dataLength: Int(bufList.mBuffers.mDataByteSize)) + CMSampleBufferSetDataBuffer(&sb, newValue: bb) + input.append(sb) } } - /// Create a CMSampleBuffer from a CVPixelBuffer (needed because AVAssetWriterInput - /// expects CMSampleBuffer but AVPlayerItemVideoOutput gives CVPixelBuffer directly) - private func createSampleBuffer(from pixelBuffer: CVPixelBuffer, pts: CMTime) -> CMSampleBuffer? { - var sampleBuffer: CMSampleBuffer? - var timingInfo = CMSampleTimingInfo( + // MARK: - Video frame capture + + @objc private func captureFrame() { + guard let writer = assetWriter, let vo = videoOutput, let vidInput = videoInput else { return } + + let itemTime = vo.itemTime(forHostTime: CACurrentMediaTime()) + guard vo.hasNewPixelBuffer(forItemTime: itemTime), + let buf = vo.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) + else { return } + + // Start session on first frame + if writer.status == .unknown { + writer.startSession(atSourceTime: itemTime) + firstPTS = itemTime + NSLog("[MiniPlayer] Session started at PTS: %.3f", itemTime.seconds) + } + + guard vidInput.isReadyForMoreMediaData else { return } + + var timing = CMSampleTimingInfo( duration: CMTime(value: 1, timescale: 30), - presentationTimeStamp: pts, + presentationTimeStamp: itemTime, decodeTimeStamp: .invalid ) - var formatDescription: CMFormatDescription? - CMVideoFormatDescriptionCreateForImageBuffer( - allocator: kCFAllocatorDefault, - imageBuffer: pixelBuffer, - formatDescriptionOut: &formatDescription - ) - guard let fd = formatDescription else { return nil } + var formatDesc: CMFormatDescription? + CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, + imageBuffer: buf, formatDescriptionOut: &formatDesc) + guard let fd = formatDesc else { return } - CMSampleBufferCreateReadyWithImageBuffer( - allocator: kCFAllocatorDefault, - imageBuffer: pixelBuffer, - formatDescription: fd, - sampleTiming: &timingInfo, - sampleBufferOut: &sampleBuffer - ) - return sampleBuffer + var sampleBuffer: CMSampleBuffer? + CMSampleBufferCreateReadyWithImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: buf, + formatDescription: fd, sampleTiming: &timing, + sampleBufferOut: &sampleBuffer) + if let sb = sampleBuffer { + vidInput.append(sb) + } } } #endif