MiniPlayer/Sources/PlayerRecorder.swift
yumoqing f1ecbf2e91 fix: replace MTAudioProcessingTap with CoreAudio device capture for HLS audio recording
MTAudioProcessingTap with AVPlayerItem.audioMix doesn't fire its process
callback for HLS streams on macOS (known limitation). This caused recorded
videos to have no audio track when recording HLS content.

Replace with CoreAudio AudioDeviceCreateIOProcID approach that captures
PCM audio directly from the system output device. This works for all
audio sources (HLS, local files, etc).

Key changes:
- Remove MTAudioProcessingTap and all tap callbacks
- Add CoreAudio device IOProc for audio capture
- Pre-allocated buffer to avoid malloc in real-time audio thread
- Serial dispatch queue for CMSampleBuffer processing off audio thread
- AudioTimeStamp → CMTime conversion for proper timestamps
2026-06-27 10:58:09 +08:00

579 lines
23 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import AVFoundation
import CoreMedia
import MediaToolbox
import Foundation
#if os(macOS)
import AppKit
// MARK: - C 访 Swift
private nonisolated(unsafe) var gAudioContext: AudioCaptureContext?
class AudioCaptureContext {
var writerInput: AVAssetWriterInput?
nonisolated(unsafe) var isRunning: Bool = true
nonisolated(unsafe) var formatDescription: CMAudioFormatDescription?
nonisolated(unsafe) var sampleRate: Double = 44100.0
nonisolated(unsafe) var channelsPerFrame: UInt32 = 2
nonisolated(unsafe) var appendCount: Int = 0
nonisolated(unsafe) var firstAppendLogged: Bool = false
nonisolated(unsafe) var processCallCount: Int = 0
/// Serial queue for processing audio buffers off the real-time thread
let audioQueue = DispatchQueue(label: "miniplayer.audioCapture")
/// Pre-allocated data buffer (avoids malloc in real-time audio thread)
var dataBuffer: UnsafeMutableRawPointer?
var dataBufferSize: UInt32 = 0
}
// MARK: - CoreAudio IOProc
/// PCM HLS
/// 线
private func audioDeviceIOProc(
_ inDevice: AudioDeviceID,
_ inNow: UnsafePointer<AudioTimeStamp>,
_ inInputData: UnsafePointer<AudioBufferList>,
_ inInputTime: UnsafePointer<AudioTimeStamp>,
_ inOutputData: UnsafeMutablePointer<AudioBufferList>,
_ inOutputTime: UnsafePointer<AudioTimeStamp>,
_ inClientData: UnsafeMutableRawPointer?
) -> OSStatus {
guard let clientData = inClientData else { return noErr }
let ctx = Unmanaged<AudioCaptureContext>.fromOpaque(clientData).takeUnretainedValue()
guard ctx.isRunning,
let writerInput = ctx.writerInput,
writerInput.isReadyForMoreMediaData else { return noErr }
ctx.processCallCount += 1
let bufferList = inInputData.pointee
guard bufferList.mNumberBuffers > 0 else { return noErr }
let buffer = bufferList.mBuffers
let dataSize = buffer.mDataByteSize
guard dataSize > 0, let srcData = buffer.mData else { return noErr }
// Copy to pre-allocated buffer (avoid malloc in real-time thread)
if ctx.dataBufferSize < dataSize {
if let existing = ctx.dataBuffer { free(existing) }
ctx.dataBuffer = malloc(Int(dataSize))
ctx.dataBufferSize = dataSize
}
guard let dest = ctx.dataBuffer else { return noErr }
memcpy(dest, srcData, Int(dataSize))
let pts = inInputTime.pointee
let callNum = ctx.processCallCount
// Process on serial queue (off the real-time audio thread)
ctx.audioQueue.async {
guard ctx.isRunning, let fd = ctx.formatDescription else { return }
var blockBuffer: CMBlockBuffer?
let blockStatus = CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault, memoryBlock: nil,
blockLength: Int(dataSize), blockAllocator: kCFAllocatorDefault,
customBlockSource: nil, offsetToData: 0,
dataLength: Int(dataSize),
flags: kCMBlockBufferAssureMemoryNowFlag,
blockBufferOut: &blockBuffer
)
guard blockStatus == kCMBlockBufferNoErr, let bb = blockBuffer else { return }
let replaceStatus = CMBlockBufferReplaceDataBytes(
with: dest, blockBuffer: bb,
offsetIntoDestination: 0, dataLength: Int(dataSize)
)
guard replaceStatus == kCMBlockBufferNoErr else { return }
var timingInfo = CMSampleTimingInfo()
// Convert AudioTimeStamp to CMTime
if pts.mFlags.contains(.sampleTimeValid) {
let sampleRate = ctx.sampleRate
timingInfo.presentationTimeStamp = CMTime(
value: Int64(pts.mSampleTime),
timescale: Int32(sampleRate)
)
} else if pts.mFlags.contains(.hostTimeValid) {
var timebase = mach_timebase_info_data_t()
mach_timebase_info(&timebase)
let nanos = pts.mHostTime * UInt64(timebase.numer) / UInt64(timebase.denom)
timingInfo.presentationTimeStamp = CMTime(value: CMTimeValue(nanos), timescale: 1_000_000_000)
} else {
return
}
timingInfo.duration = .invalid
timingInfo.decodeTimeStamp = .invalid
var sampleBuffer: CMSampleBuffer?
let bytesPerFrame = Int(ctx.channelsPerFrame) * MemoryLayout<Float32>.size
let sampleCount = Int(dataSize) / max(bytesPerFrame, 1)
let createStatus = CMSampleBufferCreateReady(
allocator: kCFAllocatorDefault,
dataBuffer: bb,
formatDescription: fd,
sampleCount: sampleCount,
sampleTimingEntryCount: 1,
sampleTimingArray: &timingInfo,
sampleSizeEntryCount: 0,
sampleSizeArray: nil,
sampleBufferOut: &sampleBuffer
)
guard createStatus == noErr, let sb = sampleBuffer else {
if callNum <= 3 {
NSLog("[Recorder] \u{26a0}\u{fe0f} Audio sample buffer create failed: %d", createStatus)
}
return
}
if writerInput.isReadyForMoreMediaData && writerInput.append(sb) {
ctx.appendCount += 1
if !ctx.firstAppendLogged {
ctx.firstAppendLogged = true
NSLog("[Recorder] ✓ First audio captured: pts=%.2fs, size=%u", pts.mSampleTime / ctx.sampleRate, dataSize)
}
}
}
return noErr
}
// MARK: - PlayerRecorder
/// finalize退
enum RecorderState {
static nonisolated(unsafe) var isFinalizing: Bool = false
static nonisolated(unsafe) var pendingTerminate: Bool = false
}
/// AVPlayerItemVideoOutput + CoreAudio
///
@MainActor
final class PlayerRecorder: NSObject {
private var writer: AVAssetWriter?
private var videoInput: AVAssetWriterInput?
private var audioInput: AVAssetWriterInput?
private var captureTimer: Timer?
private var durationTimer: Timer?
private var startDate: Date?
private weak var weakOutput: AVPlayerItemVideoOutput?
nonisolated(unsafe) private var isRunning = false
//
private var lastPixelBuffer: CVPixelBuffer?
private var captureFrameCount: Int = 0
// (CoreAudio )
private var audioDeviceID: AudioDeviceID = 0
private var audioCaptureContext: AudioCaptureContext?
private var audioIOProc: AudioDeviceIOProc?
private weak var currentPlayerItem: AVPlayerItem?
private var tracksObservation: NSKeyValueObservation?
//
nonisolated(unsafe) private var recordStartTime: CMTime = .zero
@Published var isRecording = false
@Published var durationText = "00:00"
var onRecordingSaved: ((URL) -> Void)?
var onError: ((String) -> Void)?
///
private var tempURL: URL?
///
private var saveDirectory: URL {
let movies = FileManager.default.urls(for: .moviesDirectory, in: .userDomainMask).first!
let dir = movies.appendingPathComponent("MiniPlayer", isDirectory: true)
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
return dir
}
// MARK: -
func startRecording(from output: AVPlayerItemVideoOutput, playerItem: AVPlayerItem, startTime: CMTime) {
guard !isRecording else { return }
weakOutput = output
currentPlayerItem = playerItem
lastPixelBuffer = nil
captureFrameCount = 0
isRunning = false
recordStartTime = startTime
//
let tempFilename = "temp_recording_\(Int(Date().timeIntervalSince1970)).mp4"
let url = saveDirectory.appendingPathComponent(tempFilename)
//
try? FileManager.default.removeItem(at: url)
tempURL = url
do {
writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
} catch {
onError?("Cannot create writer: \(error.localizedDescription)")
return
}
guard let writer = writer else { return }
// H264
let videoSettings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: 1920,
AVVideoHeightKey: 1080
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
vInput.expectsMediaDataInRealTime = true
if writer.canAdd(vInput) {
writer.add(vInput)
videoInput = vInput
}
// AAC
let audioSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44100,
AVNumberOfChannelsKey: 2,
AVEncoderBitRateKey: 128000
]
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings)
aInput.expectsMediaDataInRealTime = true
if writer.canAdd(aInput) {
writer.add(aInput)
audioInput = aInput
}
guard writer.startWriting() else {
NSLog("[Recorder] startWriting failed: %@", writer.error?.localizedDescription ?? "unknown")
onError?("Cannot start writer: \(writer.error?.localizedDescription ?? "unknown")")
return
}
// session 0
writer.startSession(atSourceTime: .zero)
NSLog("[Recorder] Writer started, writing to: %@", url.lastPathComponent)
// 30fps
isRecording = true
isRunning = true
startCaptureLoop()
startDurationTimer()
// (CoreAudio)
startAudioCapture(audioWriterInput: aInput)
}
// MARK: -
func stopRecording() {
guard isRecording else { return }
isRecording = false
isRunning = false
RecorderState.isFinalizing = true // 退 finishWriting
captureTimer?.invalidate()
captureTimer = nil
stopDurationTimer()
//
let audioCalls = audioCaptureContext?.processCallCount ?? 0
let audioAppended = audioCaptureContext?.appendCount ?? 0
NSLog("[Recorder] Stopping: videoFrames=%d, audioProcessCalls=%d, audioAppended=%d",
captureFrameCount, audioCalls, audioAppended)
stopAudioCapture()
currentPlayerItem = nil
videoInput?.markAsFinished()
audioInput?.markAsFinished()
writer?.finishWriting { [weak self] in
NSLog("[Recorder] finishWriting callback invoked")
DispatchQueue.main.async {
RecorderState.isFinalizing = false // 退
guard let self = self, let w = self.writer else {
NSLog("[Recorder] ⚠️ self or writer is nil in finishWriting callback")
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSApp.reply(toApplicationShouldTerminate: true)
}
return
}
let vFrames = self.captureFrameCount
NSLog("[Recorder] finishWriting: status=%d, error=%@, videoFrames=%d, audioFrames=%d",
w.status.rawValue,
w.error?.localizedDescription ?? "none",
vFrames, audioAppended)
if w.status == .completed, let tempURL = self.tempURL {
//
if FileManager.default.fileExists(atPath: tempURL.path) {
let size = (try? FileManager.default.attributesOfItem(atPath: tempURL.path)[.size] as? UInt64) ?? 0
NSLog("[Recorder] ✓ Temp file exists: %@, size=%llu bytes", tempURL.lastPathComponent, size)
self.showSaveDialog(tempURL: tempURL)
} else {
NSLog("[Recorder] ⚠️ Temp file missing: %@", tempURL.path)
self.onError?("Recording file missing")
self.checkPendingTerminate()
}
} else {
NSLog("[Recorder] ✗ finishWriting failed: status=%d, error=%@",
w.status.rawValue, w.error?.localizedDescription ?? "unknown")
self.onError?("Recording failed: \(w.error?.localizedDescription ?? "unknown (status=\(w.status.rawValue))")")
if let url = self.tempURL {
try? FileManager.default.removeItem(at: url)
}
self.checkPendingTerminate()
}
}
}
lastPixelBuffer = nil
}
// MARK: - (CoreAudio )
/// HLS
private func startAudioCapture(audioWriterInput: AVAssetWriterInput) {
//
var deviceID: AudioDeviceID = 0
var size = UInt32(MemoryLayout<AudioDeviceID>.size)
var address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
let status = AudioObjectGetPropertyData(
AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &deviceID
)
guard status == noErr, deviceID != 0 else {
NSLog("[Recorder] \u{274c} Cannot get default audio output device: %d", status)
return
}
self.audioDeviceID = deviceID
//
var streamFormatAddress = AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyStreamFormat,
mScope: kAudioDevicePropertyScopeOutput,
mElement: kAudioObjectPropertyElementMain
)
var asbd = AudioStreamBasicDescription()
size = UInt32(MemoryLayout<AudioStreamBasicDescription>.size)
let fmtStatus = AudioObjectGetPropertyData(deviceID, &streamFormatAddress, 0, nil, &size, &asbd)
guard fmtStatus == noErr else {
NSLog("[Recorder] \u{274c} Cannot get audio stream format: %d", fmtStatus)
return
}
NSLog("[Recorder] Audio output format: %.1fHz, %uch, %ubit, formatID=%u",
asbd.mSampleRate, asbd.mChannelsPerFrame, asbd.mBitsPerChannel, asbd.mFormatID)
// context
let context = AudioCaptureContext()
context.writerInput = audioWriterInput
context.sampleRate = Double(asbd.mSampleRate)
context.channelsPerFrame = asbd.mChannelsPerFrame
context.isRunning = true
self.audioCaptureContext = context
//
var fmtDesc: CMAudioFormatDescription?
var mutableASBD = asbd
CMAudioFormatDescriptionCreate(
allocator: kCFAllocatorDefault,
asbd: &mutableASBD,
layoutSize: 0, layout: nil,
magicCookieSize: 0, magicCookie: nil,
extensions: nil,
formatDescriptionOut: &fmtDesc
)
context.formatDescription = fmtDesc
// IOProc使 unmanaged pointer context
let contextPtr = Unmanaged.passUnretained(context).toOpaque()
var ioProc: AudioDeviceIOProc?
let createStatus = AudioDeviceCreateIOProcID(deviceID, audioDeviceIOProc, contextPtr, &ioProc)
guard createStatus == noErr, let proc = ioProc else {
NSLog("[Recorder] \u{274c} AudioDeviceCreateIOProcID failed: %d", createStatus)
return
}
self.audioIOProc = proc
//
let startStatus = AudioDeviceStart(deviceID, proc)
guard startStatus == noErr else {
NSLog("[Recorder] \u{274c} AudioDeviceStart failed: %d", startStatus)
AudioDeviceDestroyIOProcID(deviceID, proc)
self.audioIOProc = nil
return
}
NSLog("[Recorder] \u{2713} CoreAudio output device capture started (deviceID=%u)", deviceID)
}
private func stopAudioCapture() {
audioCaptureContext?.isRunning = false
if let proc = audioIOProc, audioDeviceID != 0 {
AudioDeviceStop(audioDeviceID, proc)
AudioDeviceDestroyIOProcID(audioDeviceID, proc)
}
audioIOProc = nil
audioCaptureContext = nil
}
// MARK: - (30fps Timer)
private func startCaptureLoop() {
captureTimer = Timer.scheduledTimer(withTimeInterval: 1.0 / 30.0, repeats: true) { [weak self] _ in
Task { @MainActor in
self?.captureVideoFrame()
}
}
}
private func captureVideoFrame() {
guard isRunning, let output = weakOutput, let input = videoInput,
input.isReadyForMoreMediaData else { return }
let hostTime = CACurrentMediaTime()
let itemTime = output.itemTime(forHostTime: hostTime)
//
if let pb = output.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) {
lastPixelBuffer = pb
}
// 使
guard let pb = lastPixelBuffer else { return }
//
let relativeTime = CMTimeSubtract(itemTime, recordStartTime)
guard relativeTime.seconds >= 0 else { return }
// sample buffer
if let sb = Self.createSampleBuffer(from: pb, time: relativeTime) {
if input.append(sb) {
captureFrameCount += 1
}
}
}
private static func createSampleBuffer(from pixelBuffer: CVPixelBuffer, time: CMTime) -> CMSampleBuffer? {
var info = CMSampleTimingInfo()
info.presentationTimeStamp = time
info.duration = CMTime(value: 1, timescale: 30)
info.decodeTimeStamp = .invalid
var formatDescription: CMFormatDescription?
CMVideoFormatDescriptionCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescriptionOut: &formatDescription
)
guard let fd = formatDescription else { return nil }
var sampleBuffer: CMSampleBuffer?
CMSampleBufferCreateReadyWithImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescription: fd,
sampleTiming: &info,
sampleBufferOut: &sampleBuffer
)
return sampleBuffer
}
// MARK: -
private func startDurationTimer() {
startDate = Date()
durationText = "00:00"
durationTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in
Task { @MainActor in
guard let self = self, let start = self.startDate else { return }
let elapsed = Int(Date().timeIntervalSince(start))
let m = elapsed / 60
let s = elapsed % 60
let h = m / 60
if h > 0 {
self.durationText = String(format: "%d:%02d:%02d", h, m % 60, s)
} else {
self.durationText = String(format: "%02d:%02d", m, s)
}
}
}
}
private func stopDurationTimer() {
durationTimer?.invalidate()
durationTimer = nil
startDate = nil
durationText = "00:00"
}
// MARK: -
private func generateFinalFilename() -> String {
let formatter = DateFormatter()
formatter.dateFormat = "yyyy-MM-dd_HH-mm-ss"
return "Recording_\(formatter.string(from: Date())).mp4"
}
// MARK: -
private func showSaveDialog(tempURL: URL) {
let panel = NSSavePanel()
panel.nameFieldStringValue = generateFinalFilename()
panel.allowedContentTypes = [.mpeg4Movie]
panel.directoryURL = saveDirectory
panel.title = "保存录制文件"
panel.prompt = "保存"
panel.begin { [weak self] response in
Task { @MainActor in
guard let self = self else {
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSApp.reply(toApplicationShouldTerminate: true)
}
return
}
if response == .OK, let finalURL = panel.url {
do {
//
if FileManager.default.fileExists(atPath: finalURL.path) {
try FileManager.default.removeItem(at: finalURL)
}
//
try FileManager.default.moveItem(at: tempURL, to: finalURL)
NSLog("[Recorder] ✓ Saved: %@", finalURL.path)
self.onRecordingSaved?(finalURL)
} catch {
self.onError?("Save failed: \(error.localizedDescription)")
try? FileManager.default.removeItem(at: tempURL)
}
} else {
//
NSLog("[Recorder] User cancelled save, deleting temp file")
try? FileManager.default.removeItem(at: tempURL)
}
self.checkPendingTerminate()
}
}
}
private func checkPendingTerminate() {
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSLog("[MiniPlayer] Pending terminate triggered, replying to terminate")
NSApp.reply(toApplicationShouldTerminate: true)
}
}
}
#endif