MiniPlayer/Sources/PlayerRecorder.swift
yumoqing fb1bd32bbf fix: audio recording - double buffer + HLS variant switch support
- Replace single dataBuffer with double-buffer (dataBuffers tuple)
  to eliminate data race between real-time tapProcess thread and
  async dispatch queue reading the buffer
- Use kCMPersistentTrackID_Invalid for audioMix trackID so the tap
  survives HLS variant switches (track ID changes on quality change)
2026-06-27 15:44:55 +08:00

615 lines
23 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import AVFoundation
import CoreMedia
import MediaToolbox
import Foundation
#if os(macOS)
import AppKit
// MARK: - Tap C 访 Swift
private nonisolated(unsafe) var gTapContext: AudioTapContext?
class AudioTapContext {
nonisolated(unsafe) var writerInput: AVAssetWriterInput?
nonisolated(unsafe) var isRecording: Bool = false
nonisolated(unsafe) var formatDescription: CMAudioFormatDescription?
nonisolated(unsafe) var sampleRate: Double = 0
nonisolated(unsafe) var channelsPerFrame: UInt32 = 0
nonisolated(unsafe) var appendCount: Int = 0
nonisolated(unsafe) var processCallCount: Int = 0
nonisolated(unsafe) var firstAppendLogged: Bool = false
nonisolated(unsafe) var totalFramesWritten: Int64 = 0
/// AasyncB使
nonisolated(unsafe) var dataBuffers: (UnsafeMutableRawPointer?, UnsafeMutableRawPointer?) = (nil, nil)
nonisolated(unsafe) var dataBufferSizes: (Int, Int) = (0, 0)
nonisolated(unsafe) var writeIndex: Int = 0
let audioQueue = DispatchQueue(label: "miniplayer.audioTap")
}
// MARK: - MTAudioProcessingTap C
private func tapInit(
_ tap: MTAudioProcessingTap,
_ clientInfo: UnsafeMutableRawPointer?,
_ tapStorageOut: UnsafeMutablePointer<UnsafeMutableRawPointer?>
) {
tapStorageOut.pointee = clientInfo
}
private func tapFinalize(_ tap: MTAudioProcessingTap) {
// Context lifecycle managed elsewhere
}
private func tapPrepare(
_ tap: MTAudioProcessingTap,
_ maxFrames: CMItemCount,
_ processingFormat: UnsafePointer<AudioStreamBasicDescription>
) {
let asbd = processingFormat.pointee
NSLog("[Recorder] Tap prepare: rate=%.0f ch=%u formatFlags=0x%x",
asbd.mSampleRate, asbd.mChannelsPerFrame, asbd.mFormatFlags)
let storage = MTAudioProcessingTapGetStorage(tap)
let ctx = Unmanaged<AudioTapContext>.fromOpaque(storage).takeUnretainedValue()
ctx.sampleRate = asbd.mSampleRate
ctx.channelsPerFrame = asbd.mChannelsPerFrame
var fmtDesc: CMAudioFormatDescription?
var mutableASBD = asbd
CMAudioFormatDescriptionCreate(
allocator: kCFAllocatorDefault,
asbd: &mutableASBD,
layoutSize: 0, layout: nil,
magicCookieSize: 0, magicCookie: nil,
extensions: nil,
formatDescriptionOut: &fmtDesc
)
ctx.formatDescription = fmtDesc
NSLog("[Recorder] Tap format ready: %@", fmtDesc != nil ? "OK" : "FAILED")
}
private func tapUnprepare(_ tap: MTAudioProcessingTap) {
NSLog("[Recorder] Tap unprepare")
}
private func tapProcess(
_ tap: MTAudioProcessingTap,
_ numberFrames: Int,
_ flags: UInt32,
_ bufferListInOut: UnsafeMutablePointer<AudioBufferList>,
_ bufferListSizeOut: UnsafeMutablePointer<Int>,
_ flagsOut: UnsafeMutablePointer<UInt32>
) {
bufferListSizeOut.pointee = 1
flagsOut.pointee = flags
let storage = MTAudioProcessingTapGetStorage(tap)
let ctx = Unmanaged<AudioTapContext>.fromOpaque(storage).takeUnretainedValue()
ctx.processCallCount += 1
guard ctx.isRecording,
let _ = ctx.writerInput,
let _ = ctx.formatDescription else { return }
let ablPtr = UnsafeMutableAudioBufferListPointer(bufferListInOut)
//
var totalSize: Int = 0
for i in 0..<ablPtr.count {
if ablPtr[i].mData != nil && ablPtr[i].mDataByteSize > 0 {
totalSize += Int(ablPtr[i].mDataByteSize)
}
}
guard totalSize > 0 else { return }
// async
let wi = ctx.writeIndex
ctx.writeIndex = 1 - wi //
if ctx.dataBufferSizes.0 < totalSize || ctx.dataBufferSizes.1 < totalSize {
//
let newSize = max(totalSize, 65536) // 64KB
if wi == 0 {
if let existing = ctx.dataBuffers.0 { free(existing) }
ctx.dataBuffers.0 = malloc(newSize)
ctx.dataBufferSizes.0 = newSize
} else {
if let existing = ctx.dataBuffers.1 { free(existing) }
ctx.dataBuffers.1 = malloc(newSize)
ctx.dataBufferSizes.1 = newSize
}
}
let dest: UnsafeMutableRawPointer?
if wi == 0 { dest = ctx.dataBuffers.0 } else { dest = ctx.dataBuffers.1 }
guard let dest = dest else { return }
var offset = 0
for i in 0..<ablPtr.count {
if let src = ablPtr[i].mData, ablPtr[i].mDataByteSize > 0 {
let sz = Int(ablPtr[i].mDataByteSize)
memcpy(dest + offset, src, sz)
offset += sz
}
}
let frameCount = Int64(numberFrames)
let startFrame = ctx.totalFramesWritten
let readWi = wi // async
// CMSampleBuffer
ctx.audioQueue.async {
guard ctx.isRecording,
let writerInput = ctx.writerInput,
let fd = ctx.formatDescription else { return }
// 线
let readDest: UnsafeMutableRawPointer?
if readWi == 0 { readDest = ctx.dataBuffers.0 } else { readDest = ctx.dataBuffers.1 }
guard let readDest = readDest else { return }
var blockBuffer: CMBlockBuffer?
let blockStatus = CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault, memoryBlock: nil,
blockLength: totalSize, blockAllocator: kCFAllocatorDefault,
customBlockSource: nil, offsetToData: 0,
dataLength: totalSize,
flags: kCMBlockBufferAssureMemoryNowFlag,
blockBufferOut: &blockBuffer
)
guard blockStatus == kCMBlockBufferNoErr, let bb = blockBuffer else { return }
let replaceStatus = CMBlockBufferReplaceDataBytes(
with: readDest, blockBuffer: bb,
offsetIntoDestination: 0, dataLength: totalSize
)
guard replaceStatus == kCMBlockBufferNoErr else { return }
// PTS
let pts = CMTime(
value: CMTimeValue(startFrame),
timescale: Int32(ctx.sampleRate)
)
var timingInfo = CMSampleTimingInfo(
duration: CMTime(value: CMTimeValue(frameCount), timescale: Int32(ctx.sampleRate)),
presentationTimeStamp: pts,
decodeTimeStamp: .invalid
)
var sampleBuffer: CMSampleBuffer?
let createStatus = CMSampleBufferCreateReady(
allocator: kCFAllocatorDefault,
dataBuffer: bb,
formatDescription: fd,
sampleCount: Int(frameCount),
sampleTimingEntryCount: 1,
sampleTimingArray: &timingInfo,
sampleSizeEntryCount: 0,
sampleSizeArray: nil,
sampleBufferOut: &sampleBuffer
)
guard createStatus == noErr, let sb = sampleBuffer else {
NSLog("[Recorder] ⚠️ SampleBuffer create failed: %d", createStatus)
return
}
if writerInput.isReadyForMoreMediaData {
if writerInput.append(sb) {
ctx.appendCount += 1
ctx.totalFramesWritten += frameCount
if !ctx.firstAppendLogged {
ctx.firstAppendLogged = true
NSLog("[Recorder] ✓ First audio captured: pts=%.3fs, frames=%lld, size=%d",
pts.seconds, frameCount, totalSize)
}
}
}
}
}
// MARK: - RecorderState
enum RecorderState {
static nonisolated(unsafe) var isFinalizing: Bool = false
static nonisolated(unsafe) var pendingTerminate: Bool = false
}
// MARK: - PlayerRecorder
@MainActor
final class PlayerRecorder: NSObject {
private var writer: AVAssetWriter?
private var videoInput: AVAssetWriterInput?
private var audioInput: AVAssetWriterInput?
private var captureTimer: Timer?
private var durationTimer: Timer?
private var startDate: Date?
private weak var weakOutput: AVPlayerItemVideoOutput?
nonisolated(unsafe) private var isRunning = false
private var lastPixelBuffer: CVPixelBuffer?
private var captureFrameCount: Int = 0
// Audio Tap
private var tapContext: AudioTapContext?
private weak var currentPlayerItem: AVPlayerItem?
nonisolated(unsafe) private var recordStartTime: CMTime = .zero
@Published var isRecording = false
@Published var durationText = "00:00"
var onRecordingSaved: ((URL) -> Void)?
var onError: ((String) -> Void)?
private var tempURL: URL?
private var saveDirectory: URL {
let movies = FileManager.default.urls(for: .moviesDirectory, in: .userDomainMask).first!
let dir = movies.appendingPathComponent("MiniPlayer", isDirectory: true)
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
return dir
}
// MARK: - Audio Tap Setup
/// Tap playerItem
private func installAudioTap(on playerItem: AVPlayerItem) {
//
if gTapContext == nil {
gTapContext = AudioTapContext()
}
guard let ctx = gTapContext else { return }
// track asset
let audioTracks = playerItem.asset.tracks(withMediaType: .audio)
guard let audioTrack = audioTracks.first else {
NSLog("[Recorder] ⚠️ No audio track found (tried %d tracks)", audioTracks.count)
return
}
NSLog("[Recorder] Found audio track: ID=%d, formatDescriptions=%d",
audioTrack.trackID, audioTrack.formatDescriptions.count)
// MTAudioProcessingTap
var callbacks = MTAudioProcessingTapCallbacks(
version: kMTAudioProcessingTapCallbacksVersion_0,
clientInfo: Unmanaged.passUnretained(ctx).toOpaque(),
init: tapInit,
finalize: tapFinalize,
prepare: tapPrepare,
unprepare: tapUnprepare,
process: tapProcess
)
var tap: MTAudioProcessingTap?
let status = MTAudioProcessingTapCreate(
kCFAllocatorDefault, &callbacks,
kMTAudioProcessingTapCreationFlag_PreEffects,
&tap
)
guard status == noErr, let tapRef = tap else {
NSLog("[Recorder] ⚠️ MTAudioProcessingTapCreate failed: %d", status)
return
}
// AudioMix kCMPersistentTrackID_Invalid
// HLS variant audio track ID ID tap
let inputParams = AVMutableAudioMixInputParameters()
inputParams.trackID = kCMPersistentTrackID_Invalid
inputParams.audioTapProcessor = tapRef
let audioMix = AVMutableAudioMix()
audioMix.inputParameters = [inputParams]
playerItem.audioMix = audioMix
NSLog("[Recorder] ✓ Audio tap installed on playerItem, trackID=%d", audioTrack.trackID)
}
// MARK: -
func startRecording(from output: AVPlayerItemVideoOutput, playerItem: AVPlayerItem, startTime: CMTime) {
guard !isRecording else { return }
weakOutput = output
currentPlayerItem = playerItem
lastPixelBuffer = nil
captureFrameCount = 0
isRunning = false
recordStartTime = startTime
// Tap asset track
installAudioTap(on: playerItem)
let tempFilename = "temp_recording_\(Int(Date().timeIntervalSince1970)).mp4"
let url = saveDirectory.appendingPathComponent(tempFilename)
try? FileManager.default.removeItem(at: url)
tempURL = url
// tap context format
let ctx = gTapContext ?? AudioTapContext()
if gTapContext == nil { gTapContext = ctx }
self.tapContext = ctx
do {
writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
} catch {
onError?("Cannot create writer: \(error.localizedDescription)")
return
}
guard let writer = writer else { return }
// H264
let videoSettings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: 1920,
AVVideoHeightKey: 1080
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
vInput.expectsMediaDataInRealTime = true
if writer.canAdd(vInput) {
writer.add(vInput)
videoInput = vInput
}
// AAC startWriting
// 使 tap
let audioRate = ctx.sampleRate > 0 ? ctx.sampleRate : 48000.0
let audioCh = ctx.channelsPerFrame > 0 ? ctx.channelsPerFrame : 2
var sourceFmtDesc: CMAudioFormatDescription?
if let tapFmt = ctx.formatDescription {
sourceFmtDesc = tapFmt
} else {
// Tap prepare
var asbd = AudioStreamBasicDescription(
mSampleRate: audioRate,
mFormatID: kAudioFormatLinearPCM,
mFormatFlags: kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked | kAudioFormatFlagIsNonInterleaved,
mBytesPerPacket: 4,
mFramesPerPacket: 1,
mBytesPerFrame: 4,
mChannelsPerFrame: audioCh,
mBitsPerChannel: 32,
mReserved: 0
)
CMAudioFormatDescriptionCreate(
allocator: kCFAllocatorDefault,
asbd: &asbd, layoutSize: 0, layout: nil,
magicCookieSize: 0, magicCookie: nil,
extensions: nil, formatDescriptionOut: &sourceFmtDesc
)
}
if let fmtDesc = sourceFmtDesc {
let audioSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: audioRate,
AVNumberOfChannelsKey: audioCh,
AVEncoderBitRateKey: 128_000
]
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings, sourceFormatHint: fmtDesc)
aInput.expectsMediaDataInRealTime = true
if writer.canAdd(aInput) {
writer.add(aInput)
audioInput = aInput
NSLog("[Recorder] AudioInput added: AAC rate=%.0f ch=%u", audioRate, audioCh)
}
}
guard writer.startWriting() else {
NSLog("[Recorder] startWriting failed: %@", writer.error?.localizedDescription ?? "unknown")
onError?("Cannot start writer: \(writer.error?.localizedDescription ?? "unknown")")
return
}
writer.startSession(atSourceTime: .zero)
NSLog("[Recorder] Writer started, writing to: %@", url.lastPathComponent)
// tap context
ctx.isRecording = true
ctx.writerInput = audioInput
ctx.appendCount = 0
ctx.totalFramesWritten = 0
ctx.firstAppendLogged = false
ctx.processCallCount = 0
isRecording = true
isRunning = true
startCaptureLoop()
startDurationTimer()
}
// MARK: -
func stopRecording() {
guard isRecording else { return }
isRecording = false
isRunning = false
RecorderState.isFinalizing = true
captureTimer?.invalidate()
captureTimer = nil
stopDurationTimer()
// tap
tapContext?.isRecording = false
let audioAppended = tapContext?.appendCount ?? 0
let audioCalls = tapContext?.processCallCount ?? 0
NSLog("[Recorder] Stopping: videoFrames=%d, audioProcessCalls=%d, audioAppended=%d",
captureFrameCount, audioCalls, audioAppended)
currentPlayerItem = nil
videoInput?.markAsFinished()
audioInput?.markAsFinished()
writer?.finishWriting { [weak self] in
NSLog("[Recorder] finishWriting callback invoked")
DispatchQueue.main.async {
RecorderState.isFinalizing = false
guard let self = self, let w = self.writer else {
NSLog("[Recorder] ⚠️ self or writer is nil in finishWriting callback")
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSApp.reply(toApplicationShouldTerminate: true)
}
return
}
let vFrames = self.captureFrameCount
NSLog("[Recorder] finishWriting: status=%d, error=%@, videoFrames=%d, audioFrames=%d",
w.status.rawValue,
w.error?.localizedDescription ?? "none",
vFrames, audioAppended)
if w.status == .completed, let tempURL = self.tempURL {
if FileManager.default.fileExists(atPath: tempURL.path) {
let size = (try? FileManager.default.attributesOfItem(atPath: tempURL.path)[.size] as? UInt64) ?? 0
NSLog("[Recorder] ✓ Temp file exists: %@, size=%llu bytes", tempURL.lastPathComponent, size)
self.showSaveDialog(tempURL: tempURL)
} else {
NSLog("[Recorder] ⚠️ Temp file missing: %@", tempURL.path)
self.onError?("Recording file missing")
self.checkPendingTerminate()
}
} else {
NSLog("[Recorder] ✗ finishWriting failed: status=%d, error=%@",
w.status.rawValue, w.error?.localizedDescription ?? "unknown")
self.onError?("Recording failed: \(w.error?.localizedDescription ?? "unknown (status=\(w.status.rawValue))")")
if let url = self.tempURL {
try? FileManager.default.removeItem(at: url)
}
self.checkPendingTerminate()
}
}
}
lastPixelBuffer = nil
}
// MARK: - (30fps Timer)
private func startCaptureLoop() {
captureTimer = Timer.scheduledTimer(withTimeInterval: 1.0 / 30.0, repeats: true) { [weak self] _ in
Task { @MainActor in
self?.captureVideoFrame()
}
}
}
private func captureVideoFrame() {
guard isRunning, let output = weakOutput, let input = videoInput,
input.isReadyForMoreMediaData else { return }
let hostTime = CACurrentMediaTime()
let itemTime = output.itemTime(forHostTime: hostTime)
if let pb = output.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) {
lastPixelBuffer = pb
}
guard let pb = lastPixelBuffer else { return }
let relativeTime = CMTimeSubtract(itemTime, recordStartTime)
guard relativeTime.seconds >= 0 else { return }
if let sb = Self.createSampleBuffer(from: pb, time: relativeTime) {
if input.append(sb) {
captureFrameCount += 1
}
}
}
private static func createSampleBuffer(from pixelBuffer: CVPixelBuffer, time: CMTime) -> CMSampleBuffer? {
var info = CMSampleTimingInfo()
info.presentationTimeStamp = time
info.duration = CMTime(value: 1, timescale: 30)
info.decodeTimeStamp = .invalid
var formatDescription: CMFormatDescription?
CMVideoFormatDescriptionCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescriptionOut: &formatDescription
)
guard let fd = formatDescription else { return nil }
var sampleBuffer: CMSampleBuffer?
CMSampleBufferCreateReadyWithImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescription: fd,
sampleTiming: &info,
sampleBufferOut: &sampleBuffer
)
return sampleBuffer
}
// MARK: -
private func startDurationTimer() {
startDate = Date()
durationTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in
Task { @MainActor in
self?.updateDuration()
}
}
}
private func stopDurationTimer() {
durationTimer?.invalidate()
durationTimer = nil
}
private func updateDuration() {
guard let start = startDate else { return }
let elapsed = Int(Date().timeIntervalSince(start))
let min = elapsed / 60
let sec = elapsed % 60
durationText = String(format: "%02d:%02d", min, sec)
}
// MARK: -
private func showSaveDialog(tempURL: URL) {
let panel = NSSavePanel()
panel.allowedContentTypes = [.mpeg4Movie]
panel.nameFieldStringValue = tempURL.lastPathComponent
panel.directoryURL = saveDirectory
panel.begin { [weak self] response in
if response == .OK, let url = panel.url {
do {
if FileManager.default.fileExists(atPath: url.path) {
try FileManager.default.removeItem(at: url)
}
try FileManager.default.moveItem(at: tempURL, to: url)
NSLog("[Recorder] ✓ Saved: %@", url.path)
self?.onRecordingSaved?(url)
} catch {
NSLog("[Recorder] ✗ Save failed: %@", error.localizedDescription)
self?.onError?("Save failed: \(error.localizedDescription)")
}
} else {
try? FileManager.default.removeItem(at: tempURL)
}
self?.checkPendingTerminate()
}
}
private func checkPendingTerminate() {
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSApp.reply(toApplicationShouldTerminate: true)
}
}
}
#endif