MiniPlayer/Sources/PlayerRecorder.swift
yumoqing 1733ecc7bc fix: 修复录制视频无音频问题
- audioInput 从 PCM pass-through 改为 AAC 编码 (MP4 不支持裸 Float32 PCM)
- IOProc 优先读 inOutputData (播放数据) 而非 inInputData (麦克风)
- 音频时间戳改为相对时间 (从首次回调开始计算), 与视频同步
- 复用 CMAudioFormatDescription 避免重复创建
2026-06-27 13:55:49 +08:00

591 lines
23 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import AVFoundation
import CoreMedia
import MediaToolbox
import Foundation
#if os(macOS)
import AppKit
// MARK: - C 访 Swift
private nonisolated(unsafe) var gAudioContext: AudioCaptureContext?
class AudioCaptureContext {
var writerInput: AVAssetWriterInput?
nonisolated(unsafe) var isRunning: Bool = true
nonisolated(unsafe) var formatDescription: CMAudioFormatDescription?
nonisolated(unsafe) var sampleRate: Double = 44100.0
nonisolated(unsafe) var channelsPerFrame: UInt32 = 2
nonisolated(unsafe) var appendCount: Int = 0
nonisolated(unsafe) var firstAppendLogged: Bool = false
nonisolated(unsafe) var processCallCount: Int = 0
nonisolated(unsafe) var firstAudioSampleTime: Int64 = -1
let audioQueue = DispatchQueue(label: "miniplayer.audioCapture")
var dataBuffer: UnsafeMutableRawPointer?
var dataBufferSize: UInt32 = 0
}
// MARK: - CoreAudio IOProc
/// PCM
/// inOutputData inInputData
private func audioDeviceIOProc(
_ inDevice: AudioDeviceID,
_ inNow: UnsafePointer<AudioTimeStamp>,
_ inInputData: UnsafePointer<AudioBufferList>,
_ inInputTime: UnsafePointer<AudioTimeStamp>,
_ inOutputData: UnsafeMutablePointer<AudioBufferList>,
_ inOutputTime: UnsafePointer<AudioTimeStamp>,
_ inClientData: UnsafeMutableRawPointer?
) -> OSStatus {
guard let clientData = inClientData else { return noErr }
let ctx = Unmanaged<AudioCaptureContext>.fromOpaque(clientData).takeUnretainedValue()
guard ctx.isRunning else { return noErr }
ctx.processCallCount += 1
// inOutputData
let outBufferList = inOutputData.pointee
let inBufferList = inInputData.pointee
let callNum = ctx.processCallCount
if callNum <= 3 {
print("[Recorder] IOProc#\(callNum): outBufs=\(outBufferList.mNumberBuffers), inBufs=\(inBufferList.mNumberBuffers)")
if outBufferList.mNumberBuffers > 0 {
let b = outBufferList.mBuffers
print("[Recorder] outBuf[0]: size=\(b.mDataByteSize), channels=\(b.mNumberChannels), data=\(b.mData != nil)")
}
if inBufferList.mNumberBuffers > 0 {
let b = inBufferList.mBuffers
print("[Recorder] inBuf[0]: size=\(b.mDataByteSize), channels=\(b.mNumberChannels), data=\(b.mData != nil)")
}
}
// inOutputData loopback
let bufferList: AudioBufferList
if outBufferList.mNumberBuffers > 0 {
let b = outBufferList.mBuffers
if b.mDataByteSize > 0 && b.mData != nil {
bufferList = outBufferList
} else if inBufferList.mNumberBuffers > 0 {
bufferList = inBufferList
} else {
return noErr
}
} else if inBufferList.mNumberBuffers > 0 {
bufferList = inBufferList
} else {
return noErr
}
let buffer = bufferList.mBuffers
let dataSize = buffer.mDataByteSize
guard dataSize > 0, let srcData = buffer.mData else {
if callNum <= 3 { print("[Recorder] IOProc#\(callNum): no data (size=\(buffer.mDataByteSize), data=\(buffer.mData != nil))") }
return noErr
}
// buffer
if ctx.dataBufferSize < dataSize {
if let existing = ctx.dataBuffer { free(existing) }
ctx.dataBuffer = malloc(Int(dataSize))
ctx.dataBufferSize = dataSize
}
guard let dest = ctx.dataBuffer else { return noErr }
memcpy(dest, srcData, Int(dataSize))
let pts = inOutputTime.pointee
//
ctx.audioQueue.async {
guard ctx.isRunning else { return }
guard let fd = ctx.formatDescription, let writerInput = ctx.writerInput else { return }
var blockBuffer: CMBlockBuffer?
let blockStatus = CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault, memoryBlock: nil,
blockLength: Int(dataSize), blockAllocator: kCFAllocatorDefault,
customBlockSource: nil, offsetToData: 0,
dataLength: Int(dataSize),
flags: kCMBlockBufferAssureMemoryNowFlag,
blockBufferOut: &blockBuffer
)
guard blockStatus == kCMBlockBufferNoErr, let bb = blockBuffer else {
if callNum <= 3 { print("[Recorder] CMBlockBufferCreate failed: \(blockStatus)") }
return
}
let replaceStatus = CMBlockBufferReplaceDataBytes(
with: dest, blockBuffer: bb,
offsetIntoDestination: 0, dataLength: Int(dataSize)
)
guard replaceStatus == kCMBlockBufferNoErr else { return }
var timingInfo = CMSampleTimingInfo()
if pts.mFlags.contains(.sampleTimeValid) {
// 使
if ctx.firstAudioSampleTime < 0 {
ctx.firstAudioSampleTime = Int64(pts.mSampleTime)
}
let relativeSampleTime = Int64(pts.mSampleTime) - ctx.firstAudioSampleTime
timingInfo.presentationTimeStamp = CMTime(
value: relativeSampleTime,
timescale: Int32(ctx.sampleRate)
)
} else if pts.mFlags.contains(.hostTimeValid) {
var timebase = mach_timebase_info_data_t()
mach_timebase_info(&timebase)
let nanos = pts.mHostTime * UInt64(timebase.numer) / UInt64(timebase.denom)
timingInfo.presentationTimeStamp = CMTime(value: CMTimeValue(nanos), timescale: 1_000_000_000)
} else {
return
}
timingInfo.duration = .invalid
timingInfo.decodeTimeStamp = .invalid
let bytesPerFrame = Int(ctx.channelsPerFrame) * MemoryLayout<Float32>.size
let sampleCount = Int(dataSize) / max(bytesPerFrame, 1)
var sampleBuffer: CMSampleBuffer?
let createStatus = CMSampleBufferCreateReady(
allocator: kCFAllocatorDefault,
dataBuffer: bb,
formatDescription: fd,
sampleCount: sampleCount,
sampleTimingEntryCount: 1,
sampleTimingArray: &timingInfo,
sampleSizeEntryCount: 0,
sampleSizeArray: nil,
sampleBufferOut: &sampleBuffer
)
guard createStatus == noErr, let sb = sampleBuffer else {
if callNum <= 3 {
print("[Recorder] ⚠️ SampleBuffer create failed: \(createStatus), dataSize=\(dataSize), sampleCount=\(sampleCount)")
}
return
}
if writerInput.isReadyForMoreMediaData {
let appended = writerInput.append(sb)
if appended {
ctx.appendCount += 1
if !ctx.firstAppendLogged {
ctx.firstAppendLogged = true
print("[Recorder] ✓ First audio captured: pts=\(timingInfo.presentationTimeStamp.seconds)s, size=\(dataSize)")
}
} else if callNum <= 3 {
print("[Recorder] ⚠️ writerInput.append returned false")
}
} else if callNum <= 3 {
print("[Recorder] ⚠️ writerInput not ready for more data")
}
}
return noErr
}
// MARK: - PlayerRecorder
/// finalize退
enum RecorderState {
static nonisolated(unsafe) var isFinalizing: Bool = false
static nonisolated(unsafe) var pendingTerminate: Bool = false
}
/// AVPlayerItemVideoOutput + CoreAudio
@MainActor
final class PlayerRecorder: NSObject {
private var writer: AVAssetWriter?
private var videoInput: AVAssetWriterInput?
private var audioInput: AVAssetWriterInput?
private var captureTimer: Timer?
private var durationTimer: Timer?
private var startDate: Date?
private weak var weakOutput: AVPlayerItemVideoOutput?
nonisolated(unsafe) private var isRunning = false
private var lastPixelBuffer: CVPixelBuffer?
private var captureFrameCount: Int = 0
// (CoreAudio)
private var audioDeviceID: AudioDeviceID = 0
private var audioCaptureContext: AudioCaptureContext?
private var audioIOProc: AudioDeviceIOProc?
private weak var currentPlayerItem: AVPlayerItem?
nonisolated(unsafe) private var recordStartTime: CMTime = .zero
@Published var isRecording = false
@Published var durationText = "00:00"
var onRecordingSaved: ((URL) -> Void)?
var onError: ((String) -> Void)?
private var tempURL: URL?
private var saveDirectory: URL {
let movies = FileManager.default.urls(for: .moviesDirectory, in: .userDomainMask).first!
let dir = movies.appendingPathComponent("MiniPlayer", isDirectory: true)
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
return dir
}
// MARK: -
func startRecording(from output: AVPlayerItemVideoOutput, playerItem: AVPlayerItem, startTime: CMTime) {
guard !isRecording else { return }
weakOutput = output
currentPlayerItem = playerItem
lastPixelBuffer = nil
captureFrameCount = 0
isRunning = false
recordStartTime = startTime
let tempFilename = "temp_recording_\(Int(Date().timeIntervalSince1970)).mp4"
let url = saveDirectory.appendingPathComponent(tempFilename)
try? FileManager.default.removeItem(at: url)
tempURL = url
do {
writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
} catch {
onError?("Cannot create writer: \(error.localizedDescription)")
return
}
guard let writer = writer else { return }
// H264
let videoSettings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: 1920,
AVVideoHeightKey: 1080
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
vInput.expectsMediaDataInRealTime = true
if writer.canAdd(vInput) {
writer.add(vInput)
videoInput = vInput
}
// startAudioCapture AAC
guard writer.startWriting() else {
NSLog("[Recorder] startWriting failed: %@", writer.error?.localizedDescription ?? "unknown")
onError?("Cannot start writer: \(writer.error?.localizedDescription ?? "unknown")")
return
}
writer.startSession(atSourceTime: .zero)
NSLog("[Recorder] Writer started, writing to: %@", url.lastPathComponent)
isRecording = true
isRunning = true
startCaptureLoop()
startDurationTimer()
// (CoreAudio audioInput)
startAudioCapture()
}
// MARK: -
func stopRecording() {
guard isRecording else { return }
isRecording = false
isRunning = false
RecorderState.isFinalizing = true
captureTimer?.invalidate()
captureTimer = nil
stopDurationTimer()
let audioCalls = audioCaptureContext?.processCallCount ?? 0
let audioAppended = audioCaptureContext?.appendCount ?? 0
NSLog("[Recorder] Stopping: videoFrames=%d, audioProcessCalls=%d, audioAppended=%d",
captureFrameCount, audioCalls, audioAppended)
stopAudioCapture()
currentPlayerItem = nil
videoInput?.markAsFinished()
audioInput?.markAsFinished()
writer?.finishWriting { [weak self] in
NSLog("[Recorder] finishWriting callback invoked")
DispatchQueue.main.async {
RecorderState.isFinalizing = false
guard let self = self, let w = self.writer else {
NSLog("[Recorder] ⚠️ self or writer is nil in finishWriting callback")
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSApp.reply(toApplicationShouldTerminate: true)
}
return
}
let vFrames = self.captureFrameCount
NSLog("[Recorder] finishWriting: status=%d, error=%@, videoFrames=%d, audioFrames=%d",
w.status.rawValue,
w.error?.localizedDescription ?? "none",
vFrames, audioAppended)
if w.status == .completed, let tempURL = self.tempURL {
if FileManager.default.fileExists(atPath: tempURL.path) {
let size = (try? FileManager.default.attributesOfItem(atPath: tempURL.path)[.size] as? UInt64) ?? 0
NSLog("[Recorder] ✓ Temp file exists: %@, size=%llu bytes", tempURL.lastPathComponent, size)
self.showSaveDialog(tempURL: tempURL)
} else {
NSLog("[Recorder] ⚠️ Temp file missing: %@", tempURL.path)
self.onError?("Recording file missing")
self.checkPendingTerminate()
}
} else {
NSLog("[Recorder] ✗ finishWriting failed: status=%d, error=%@",
w.status.rawValue, w.error?.localizedDescription ?? "unknown")
self.onError?("Recording failed: \(w.error?.localizedDescription ?? "unknown (status=\(w.status.rawValue))")")
if let url = self.tempURL {
try? FileManager.default.removeItem(at: url)
}
self.checkPendingTerminate()
}
}
}
lastPixelBuffer = nil
}
// MARK: - (CoreAudio )
private func startAudioCapture() {
print("[Recorder] startAudioCapture called")
var deviceID: AudioDeviceID = 0
var size = UInt32(MemoryLayout<AudioDeviceID>.size)
var address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
let status = AudioObjectGetPropertyData(
AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &deviceID
)
print("[Recorder] Get default device status: \(status), deviceID: \(deviceID)")
guard status == noErr, deviceID != 0 else {
print("[Recorder] ❌ Cannot get default audio output device")
return
}
self.audioDeviceID = deviceID
//
var streamFormatAddress = AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyStreamFormat,
mScope: kAudioDevicePropertyScopeOutput,
mElement: kAudioObjectPropertyElementMain
)
var asbd = AudioStreamBasicDescription()
size = UInt32(MemoryLayout<AudioStreamBasicDescription>.size)
let fmtStatus = AudioObjectGetPropertyData(deviceID, &streamFormatAddress, 0, nil, &size, &asbd)
print("[Recorder] Stream format: status=\(fmtStatus), rate=\(asbd.mSampleRate), ch=\(asbd.mChannelsPerFrame), bits=\(asbd.mBitsPerChannel), formatID=\(asbd.mFormatID)")
guard fmtStatus == noErr else {
print("[Recorder] ❌ Cannot get audio stream format")
return
}
// AAC MP4 PCM pass-through
// CMAudioFormatDescription
var fmtDesc: CMAudioFormatDescription?
var mutableASBD = asbd
CMAudioFormatDescriptionCreate(
allocator: kCFAllocatorDefault,
asbd: &mutableASBD,
layoutSize: 0, layout: nil,
magicCookieSize: 0, magicCookie: nil,
extensions: nil,
formatDescriptionOut: &fmtDesc
)
guard let sourceFmtDesc = fmtDesc else {
print("[Recorder] ❌ Cannot create CMAudioFormatDescription")
return
}
let audioSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: asbd.mSampleRate,
AVNumberOfChannelsKey: asbd.mChannelsPerFrame,
AVEncoderBitRateKey: 128_000
]
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings, sourceFormatHint: sourceFmtDesc)
aInput.expectsMediaDataInRealTime = true
if let w = writer, w.canAdd(aInput) {
w.add(aInput)
audioInput = aInput
print("[Recorder] ✓ AudioInput created with AAC encoding (rate=\(asbd.mSampleRate), ch=\(asbd.mChannelsPerFrame))")
} else {
print("[Recorder] ❌ Cannot add audio input to writer")
return
}
// context
let context = AudioCaptureContext()
context.writerInput = aInput
context.sampleRate = Double(asbd.mSampleRate)
context.channelsPerFrame = asbd.mChannelsPerFrame
context.isRunning = true
self.audioCaptureContext = context
// 使
context.formatDescription = sourceFmtDesc
print("[Recorder] formatDescription created: true")
// IOProc
let contextPtr = Unmanaged.passUnretained(context).toOpaque()
var ioProc: AudioDeviceIOProc?
let createStatus = AudioDeviceCreateIOProcID(deviceID, audioDeviceIOProc, contextPtr, &ioProc)
print("[Recorder] CreateIOProcID status: \(createStatus)")
guard createStatus == noErr, let proc = ioProc else {
print("[Recorder] ❌ AudioDeviceCreateIOProcID failed: \(createStatus)")
return
}
self.audioIOProc = proc
let startStatus = AudioDeviceStart(deviceID, proc)
print("[Recorder] AudioDeviceStart status: \(startStatus)")
guard startStatus == noErr else {
print("[Recorder] ❌ AudioDeviceStart failed: \(startStatus)")
AudioDeviceDestroyIOProcID(deviceID, proc)
self.audioIOProc = nil
return
}
NSLog("[Recorder] ✓ CoreAudio output device capture started (deviceID=%u)", deviceID)
}
private func stopAudioCapture() {
audioCaptureContext?.isRunning = false
if let proc = audioIOProc, audioDeviceID != 0 {
AudioDeviceStop(audioDeviceID, proc)
AudioDeviceDestroyIOProcID(audioDeviceID, proc)
}
audioIOProc = nil
audioCaptureContext = nil
}
// MARK: - (30fps Timer)
private func startCaptureLoop() {
captureTimer = Timer.scheduledTimer(withTimeInterval: 1.0 / 30.0, repeats: true) { [weak self] _ in
Task { @MainActor in
self?.captureVideoFrame()
}
}
}
private func captureVideoFrame() {
guard isRunning, let output = weakOutput, let input = videoInput,
input.isReadyForMoreMediaData else { return }
let hostTime = CACurrentMediaTime()
let itemTime = output.itemTime(forHostTime: hostTime)
if let pb = output.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) {
lastPixelBuffer = pb
}
guard let pb = lastPixelBuffer else { return }
let relativeTime = CMTimeSubtract(itemTime, recordStartTime)
guard relativeTime.seconds >= 0 else { return }
if let sb = Self.createSampleBuffer(from: pb, time: relativeTime) {
if input.append(sb) {
captureFrameCount += 1
}
}
}
private static func createSampleBuffer(from pixelBuffer: CVPixelBuffer, time: CMTime) -> CMSampleBuffer? {
var info = CMSampleTimingInfo()
info.presentationTimeStamp = time
info.duration = CMTime(value: 1, timescale: 30)
info.decodeTimeStamp = .invalid
var formatDescription: CMFormatDescription?
CMVideoFormatDescriptionCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescriptionOut: &formatDescription
)
guard let fd = formatDescription else { return nil }
var sampleBuffer: CMSampleBuffer?
CMSampleBufferCreateReadyWithImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescription: fd,
sampleTiming: &info,
sampleBufferOut: &sampleBuffer
)
return sampleBuffer
}
// MARK: -
private func startDurationTimer() {
startDate = Date()
durationTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in
Task { @MainActor in
self?.updateDuration()
}
}
}
private func stopDurationTimer() {
durationTimer?.invalidate()
durationTimer = nil
}
private func updateDuration() {
guard let start = startDate else { return }
let elapsed = Int(Date().timeIntervalSince(start))
let min = elapsed / 60
let sec = elapsed % 60
durationText = String(format: "%02d:%02d", min, sec)
}
// MARK: -
private func showSaveDialog(tempURL: URL) {
let panel = NSSavePanel()
panel.allowedContentTypes = [.mpeg4Movie]
panel.nameFieldStringValue = tempURL.lastPathComponent
panel.directoryURL = saveDirectory
panel.begin { [weak self] response in
if response == .OK, let url = panel.url {
do {
if FileManager.default.fileExists(atPath: url.path) {
try FileManager.default.removeItem(at: url)
}
try FileManager.default.moveItem(at: tempURL, to: url)
NSLog("[Recorder] ✓ Saved: %@", url.path)
self?.onRecordingSaved?(url)
} catch {
NSLog("[Recorder] ✗ Save failed: %@", error.localizedDescription)
self?.onError?("Save failed: \(error.localizedDescription)")
}
} else {
try? FileManager.default.removeItem(at: tempURL)
}
self?.checkPendingTerminate()
}
}
private func checkPendingTerminate() {
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSApp.reply(toApplicationShouldTerminate: true)
}
}
}
#endif