fix(iOS): add MTAudioProcessingTap for audio capture, fix PTS timing
- MTAudioProcessingTap captures audio samples from AVPlayer's audio track - Audio samples written to AVAssetWriter alongside video frames - First frame PTS used as session start (fixes duration stretch) - Cleaned up unused properties
This commit is contained in:
parent
bfe81cbc82
commit
1832672081
@ -2,10 +2,9 @@
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
import UIKit
|
||||
import Photos
|
||||
import MediaToolbox
|
||||
|
||||
/// iOS 录制器 — 从 AVPlayer 播放输出捕获视频帧和音频，写入 MP4
|
||||
/// 替代手动下载 HLS 片段的方案（对 CBS 等有 DRM/CDN 保护的流无效）
|
||||
/// iOS 录制器 — 从 AVPlayer 播放输出捕获视频帧 + 音频，写入 MP4
|
||||
@MainActor
|
||||
final class HLSRecorder: ObservableObject {
|
||||
@Published var isRecording = false
|
||||
@ -22,18 +21,15 @@ final class HLSRecorder: ObservableObject {
|
||||
private var startTime: Date?
|
||||
private var timer: Timer?
|
||||
private var outputURL: URL?
|
||||
private var lastVideoSampleTime: CMTime?
|
||||
private var audioEngine: AVAudioEngine?
|
||||
private var audioFile: AVAudioFile?
|
||||
private var audioTapInstalled = false
|
||||
private var firstPTS: CMTime?
|
||||
private var audioTapID: MTAudioProcessingTapID?
|
||||
|
||||
func startRecording(url: URL, player: AVPlayer) {
|
||||
guard !isRecording, let item = player.currentItem else { return }
|
||||
isRecording = true
|
||||
startTime = Date()
|
||||
lastVideoSampleTime = nil
|
||||
firstPTS = nil
|
||||
|
||||
// Timer for duration display
|
||||
timer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in
|
||||
Task { @MainActor in
|
||||
guard let self, let start = self.startTime else { return }
|
||||
@ -42,13 +38,10 @@ final class HLSRecorder: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
// Output file
|
||||
let docs = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
|
||||
outputURL = docs.appendingPathComponent("MiniPlayer_Recording_\(Int(Date().timeIntervalSince1970)).mp4")
|
||||
|
||||
Task {
|
||||
await startCapture(item: item, player: player)
|
||||
}
|
||||
Task { await startCapture(item: item, player: player) }
|
||||
}
|
||||
|
||||
func stopRecording() {
|
||||
@ -56,8 +49,8 @@ final class HLSRecorder: ObservableObject {
|
||||
isRecording = false
|
||||
timer?.invalidate()
|
||||
displayLink?.invalidate()
|
||||
videoOutput = nil
|
||||
|
||||
// Finalize the writer
|
||||
videoInput?.markAsFinished()
|
||||
audioInput?.markAsFinished()
|
||||
|
||||
@ -65,23 +58,23 @@ final class HLSRecorder: ObservableObject {
|
||||
DispatchQueue.main.async {
|
||||
guard let self, let url = self.outputURL else { return }
|
||||
let status = self.assetWriter?.status ?? .unknown
|
||||
NSLog("[MiniPlayer] AVAssetWriter finished: %d, file: %@", status.rawValue, url.path)
|
||||
NSLog("[MiniPlayer] AVAssetWriter finished: %d", status.rawValue)
|
||||
|
||||
if status == .completed {
|
||||
let attrs = try? FileManager.default.attributesOfItem(atPath: url.path)
|
||||
let size = (attrs?[.size] as? Int64) ?? 0
|
||||
NSLog("[MiniPlayer] Recorded file size: %lld bytes", size)
|
||||
NSLog("[MiniPlayer] Recorded: %lld bytes", size)
|
||||
self.onRecordingSaved?(url)
|
||||
} else {
|
||||
let err = self.assetWriter?.error?.localizedDescription ?? "unknown"
|
||||
self.onError?("Export failed: \(err)")
|
||||
self.onError?("Export: \(err)")
|
||||
try? FileManager.default.removeItem(at: url)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Capture
|
||||
// MARK: - Capture setup
|
||||
|
||||
private func startCapture(item: AVPlayerItem, player: AVPlayer) async {
|
||||
guard let outputURL else { return }
|
||||
@ -89,12 +82,16 @@ final class HLSRecorder: ObservableObject {
|
||||
do {
|
||||
let writer = try AVAssetWriter(url: outputURL, fileType: .mp4)
|
||||
|
||||
// Video input: use source format from the asset when possible
|
||||
let videoSettings: [String: Any]
|
||||
let naturalSize = try? await item.asset.load(.tracks).first(where: { $0.mediaType == .video })?.naturalSize
|
||||
let size = naturalSize ?? CGSize(width: 1280, height: 720)
|
||||
// Natural size from asset
|
||||
var size = CGSize(width: 1280, height: 720)
|
||||
if let tracks = try? await item.asset.load(.tracks),
|
||||
let videoTrack = tracks.first(where: { $0.mediaType == .video }) {
|
||||
let ns = try? await videoTrack.load(.naturalSize)
|
||||
if let ns, ns.width > 0 { size = ns }
|
||||
}
|
||||
|
||||
videoSettings = [
|
||||
// Video input
|
||||
let videoSettings: [String: Any] = [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||
AVVideoWidthKey: size.width,
|
||||
AVVideoHeightKey: size.height,
|
||||
@ -103,12 +100,10 @@ final class HLSRecorder: ObservableObject {
|
||||
AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel
|
||||
]
|
||||
]
|
||||
|
||||
let vidInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
|
||||
vidInput.expectsMediaDataInRealTime = true
|
||||
vidInput.transform = CGAffineTransform(translationX: 0, y: 0)
|
||||
guard writer.canAdd(vidInput) else {
|
||||
throw NSError(domain: "Recorder", code: 1, userInfo: [NSLocalizedDescriptionKey: "Cannot add video input"])
|
||||
throw NSError(domain: "Rec", code: 1, userInfo: [NSLocalizedDescriptionKey: "Cannot add video input"])
|
||||
}
|
||||
writer.add(vidInput)
|
||||
self.videoInput = vidInput
|
||||
@ -123,32 +118,34 @@ final class HLSRecorder: ObservableObject {
|
||||
let audInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings)
|
||||
audInput.expectsMediaDataInRealTime = true
|
||||
guard writer.canAdd(audInput) else {
|
||||
throw NSError(domain: "Recorder", code: 2, userInfo: [NSLocalizedDescriptionKey: "Cannot add audio input"])
|
||||
throw NSError(domain: "Rec", code: 2, userInfo: [NSLocalizedDescriptionKey: "Cannot add audio input"])
|
||||
}
|
||||
writer.add(audInput)
|
||||
self.audioInput = audInput
|
||||
|
||||
// Start writing
|
||||
writer.startWriting()
|
||||
writer.startSession(atSourceTime: .zero)
|
||||
self.assetWriter = writer
|
||||
// MTAudioProcessingTap on the player item's audio track
|
||||
installAudioTap(on: item)
|
||||
|
||||
// Video output from player
|
||||
let vo = AVPlayerItemVideoOutput(pixelBufferAttributes: [
|
||||
let pixAttrs: [String: Any] = [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey as String: size.width,
|
||||
kCVPixelBufferHeightKey as String: size.height
|
||||
])
|
||||
]
|
||||
let vo = AVPlayerItemVideoOutput(pixelBufferAttributes: pixAttrs)
|
||||
vo.suppressesPlayerRendering = false
|
||||
item.add(vo)
|
||||
self.videoOutput = vo
|
||||
|
||||
// Capture loop via CADisplayLink
|
||||
writer.startWriting()
|
||||
// session start will be set on first frame
|
||||
self.assetWriter = writer
|
||||
|
||||
displayLink = CADisplayLink(target: self, selector: #selector(captureFrame))
|
||||
displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: 24, maximum: 30, preferred: 30)
|
||||
displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: 15, maximum: 30, preferred: 30)
|
||||
displayLink?.add(to: .main, forMode: .common)
|
||||
|
||||
NSLog("[MiniPlayer] Capture started, output: %@", outputURL.path)
|
||||
NSLog("[MiniPlayer] Capture started: %@", outputURL.path)
|
||||
|
||||
} catch {
|
||||
await MainActor.run {
|
||||
@ -159,50 +156,150 @@ final class HLSRecorder: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
@objc private func captureFrame() {
|
||||
guard let writer = assetWriter, writer.status == .writing else { return }
|
||||
// MARK: - Audio tap
|
||||
|
||||
// Video frame capture
|
||||
if let vo = videoOutput, videoInput?.isReadyForMoreMediaData == true {
|
||||
let itemTime = vo.itemTime(forHostTime: CACurrentMediaTime())
|
||||
if vo.hasNewPixelBuffer(forItemTime: itemTime),
|
||||
let buf = vo.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) {
|
||||
if lastVideoSampleTime == nil {
|
||||
lastVideoSampleTime = itemTime
|
||||
}
|
||||
let sampleBuf = createSampleBuffer(from: buf, pts: itemTime)
|
||||
if let sb = sampleBuf {
|
||||
videoInput?.append(sb)
|
||||
}
|
||||
}
|
||||
}
|
||||
private func installAudioTap(on item: AVPlayerItem) {
|
||||
guard let audioTrack = item.asset.tracks(withMediaType: .audio).first else {
|
||||
NSLog("[MiniPlayer] No audio track found")
|
||||
return
|
||||
}
|
||||
|
||||
/// Create a CMSampleBuffer from a CVPixelBuffer (needed because AVAssetWriterInput
|
||||
/// expects CMSampleBuffer but AVPlayerItemVideoOutput gives CVPixelBuffer directly)
|
||||
private func createSampleBuffer(from pixelBuffer: CVPixelBuffer, pts: CMTime) -> CMSampleBuffer? {
|
||||
var callbacks = MTAudioProcessingTapCallbacks(
|
||||
version: kMTAudioProcessingTapCallbacksVersion_0,
|
||||
clientInfo: Unmanaged.passUnretained(self).toOpaque(),
|
||||
init: { tap, clientInfo, tapStorageOut in
|
||||
tapStorageOut?.pointee = clientInfo
|
||||
},
|
||||
finalize: { _ in },
|
||||
prepare: { _, _, _ in },
|
||||
unprepare: { _, _, _ in },
|
||||
process: { tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut in
|
||||
guard let clientInfo = MTAudioProcessingTapGetStorage(tap) else {
|
||||
numberFramesOut.pointee = 0
|
||||
return
|
||||
}
|
||||
let recorder = Unmanaged<HLSRecorder>.fromOpaque(clientInfo).takeUnretainedValue()
|
||||
recorder.processAudio(tap: tap, numberFrames: numberFrames, flags: flags,
|
||||
bufferListInOut: bufferListInOut,
|
||||
numberFramesOut: numberFramesOut, flagsOut: flagsOut)
|
||||
}
|
||||
)
|
||||
|
||||
var tap: Unmanaged<MTAudioProcessingTap>?
|
||||
let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks,
|
||||
kMTAudioProcessingTapCreationFlag_PreEffects, &tap)
|
||||
guard status == noErr, let audioTap = tap?.takeRetainedValue() else {
|
||||
NSLog("[MiniPlayer] MTAudioProcessingTapCreate failed: %d", status)
|
||||
return
|
||||
}
|
||||
|
||||
let inputParams = AVMutableAudioMixInputParameters(track: audioTrack)
|
||||
inputParams.audioTapProcessor = audioTap
|
||||
|
||||
let audioMix = AVMutableAudioMix()
|
||||
audioMix.inputParameters = [inputParams]
|
||||
item.audioMix = audioMix
|
||||
|
||||
NSLog("[MiniPlayer] Audio tap installed")
|
||||
}
|
||||
|
||||
private func processAudio(tap: MTAudioProcessingTap, numberFrames: CMItemCount, flags: MTAudioProcessingTapFlags,
|
||||
bufferListInOut: UnsafeMutablePointer<AudioBufferList>,
|
||||
numberFramesOut: UnsafeMutablePointer<CMItemCount>,
|
||||
flagsOut: UnsafeMutablePointer<MTAudioProcessingTapFlags>) {
|
||||
// Get source audio
|
||||
var sourceFlags = flags
|
||||
var timeRange = CMTimeRange()
|
||||
let status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut,
|
||||
flagsOut, &timeRange, numberFramesOut)
|
||||
guard status == noErr, let input = audioInput, input.isReadyForMoreMediaData else {
|
||||
flagsOut.pointee = flags
|
||||
return
|
||||
}
|
||||
|
||||
// Create CMSampleBuffer from the audio buffer list
|
||||
let pts = timeRange.start
|
||||
var asbd = AudioStreamBasicDescription()
|
||||
MTAudioProcessingTapGetStreamDescription(tap, &asbd)
|
||||
|
||||
var formatDesc: CMAudioFormatDescription?
|
||||
CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &asbd,
|
||||
layoutSize: 0, layout: nil, 0, nil, nil,
|
||||
formatDescriptionOut: &formatDesc)
|
||||
guard let fd = formatDesc else { return }
|
||||
|
||||
var sampleBuffer: CMSampleBuffer?
|
||||
var timingInfo = CMSampleTimingInfo(
|
||||
let numSamples = numberFramesOut.pointee
|
||||
|
||||
var timing = CMSampleTimingInfo(duration: CMTime(value: CMTimeValue(numSamples), timescale: CMTimeScale(asbd.mSampleRate)),
|
||||
presentationTimeStamp: pts, decodeTimeStamp: .invalid)
|
||||
|
||||
CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, dataReady: false,
|
||||
makeDataReadyCallback: nil, refcon: nil, formatDescription: fd,
|
||||
sampleCount: numSamples, sampleTimingEntryCount: 1,
|
||||
sampleTimingArray: &timing, sampleSizeEntryCount: 0,
|
||||
sampleSizeArray: nil, sampleBufferOut: &sampleBuffer)
|
||||
|
||||
guard var sb = sampleBuffer else { return }
|
||||
|
||||
// Copy audio data into the sample buffer
|
||||
let bufList = bufferListInOut.pointee
|
||||
var blockBuffer: CMBlockBuffer?
|
||||
CMBlockBufferCreateWithMemoryBlock(
|
||||
allocator: kCFAllocatorDefault,
|
||||
memoryBlock: nil,
|
||||
blockLength: Int(bufList.mBuffers.mDataByteSize),
|
||||
blockAllocator: kCFAllocatorDefault,
|
||||
customBlockSource: nil,
|
||||
offsetToData: 0,
|
||||
dataLength: Int(bufList.mBuffers.mDataByteSize),
|
||||
flags: 0,
|
||||
blockBufferOut: &blockBuffer
|
||||
)
|
||||
if let bb = blockBuffer {
|
||||
CMBlockBufferReplaceDataBytes(with: bufList.mBuffers.mData!, blockBuffer: bb,
|
||||
offsetIntoDestination: 0, dataLength: Int(bufList.mBuffers.mDataByteSize))
|
||||
CMSampleBufferSetDataBuffer(&sb, newValue: bb)
|
||||
input.append(sb)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Video frame capture
|
||||
|
||||
@objc private func captureFrame() {
|
||||
guard let writer = assetWriter, let vo = videoOutput, let vidInput = videoInput else { return }
|
||||
|
||||
let itemTime = vo.itemTime(forHostTime: CACurrentMediaTime())
|
||||
guard vo.hasNewPixelBuffer(forItemTime: itemTime),
|
||||
let buf = vo.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil)
|
||||
else { return }
|
||||
|
||||
// Start session on first frame
|
||||
if writer.status == .unknown {
|
||||
writer.startSession(atSourceTime: itemTime)
|
||||
firstPTS = itemTime
|
||||
NSLog("[MiniPlayer] Session started at PTS: %.3f", itemTime.seconds)
|
||||
}
|
||||
|
||||
guard vidInput.isReadyForMoreMediaData else { return }
|
||||
|
||||
var timing = CMSampleTimingInfo(
|
||||
duration: CMTime(value: 1, timescale: 30),
|
||||
presentationTimeStamp: pts,
|
||||
presentationTimeStamp: itemTime,
|
||||
decodeTimeStamp: .invalid
|
||||
)
|
||||
var formatDescription: CMFormatDescription?
|
||||
CMVideoFormatDescriptionCreateForImageBuffer(
|
||||
allocator: kCFAllocatorDefault,
|
||||
imageBuffer: pixelBuffer,
|
||||
formatDescriptionOut: &formatDescription
|
||||
)
|
||||
guard let fd = formatDescription else { return nil }
|
||||
var formatDesc: CMFormatDescription?
|
||||
CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault,
|
||||
imageBuffer: buf, formatDescriptionOut: &formatDesc)
|
||||
guard let fd = formatDesc else { return }
|
||||
|
||||
CMSampleBufferCreateReadyWithImageBuffer(
|
||||
allocator: kCFAllocatorDefault,
|
||||
imageBuffer: pixelBuffer,
|
||||
formatDescription: fd,
|
||||
sampleTiming: &timingInfo,
|
||||
sampleBufferOut: &sampleBuffer
|
||||
)
|
||||
return sampleBuffer
|
||||
var sampleBuffer: CMSampleBuffer?
|
||||
CMSampleBufferCreateReadyWithImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: buf,
|
||||
formatDescription: fd, sampleTiming: &timing,
|
||||
sampleBufferOut: &sampleBuffer)
|
||||
if let sb = sampleBuffer {
|
||||
vidInput.append(sb)
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user