2 * An example showing how to play a stream sync'd to video, using ffmpeg.
7 #include <condition_variable>
32 _Pragma("GCC diagnostic push")
33 _Pragma("GCC diagnostic ignored \"-Wconversion\"")
34 _Pragma("GCC diagnostic ignored \"-Wold-style-cast\"")
37 #include "libavcodec/avcodec.h"
38 #include "libavformat/avformat.h"
39 #include "libavformat/avio.h"
40 #include "libavformat/version.h"
41 #include "libavutil/avutil.h"
42 #include "libavutil/error.h"
43 #include "libavutil/frame.h"
44 #include "libavutil/mem.h"
45 #include "libavutil/pixfmt.h"
46 #include "libavutil/rational.h"
47 #include "libavutil/samplefmt.h"
48 #include "libavutil/time.h"
49 #include "libavutil/version.h"
50 #include "libavutil/channel_layout.h"
51 #include "libswscale/swscale.h"
52 #include "libswresample/swresample.h"
54 constexpr auto AVNoPtsValue = AV_NOPTS_VALUE;
55 constexpr auto AVErrorEOF = AVERROR_EOF;
60 #define SDL_MAIN_HANDLED
63 _Pragma("GCC diagnostic pop")
70 #include "common/alhelpers.h"
75 inline constexpr int64_t operator "" _i64(unsigned long long int n) noexcept { return static_cast<int64_t>(n); }
78 #define M_PI (3.14159265358979323846)
81 using fixed32 = std::chrono::duration<int64_t,std::ratio<1,(1_i64<<32)>>;
82 using nanoseconds = std::chrono::nanoseconds;
83 using microseconds = std::chrono::microseconds;
84 using milliseconds = std::chrono::milliseconds;
85 using seconds = std::chrono::seconds;
86 using seconds_d64 = std::chrono::duration<double>;
87 using std::chrono::duration_cast;
89 const std::string AppName{"alffplay"};
91 ALenum DirectOutMode{AL_FALSE};
92 bool EnableWideStereo{false};
93 bool EnableUhj{false};
94 bool EnableSuperStereo{false};
95 bool DisableVideo{false};
96 LPALGETSOURCEI64VSOFT alGetSourcei64vSOFT;
97 LPALCGETINTEGER64VSOFT alcGetInteger64vSOFT;
98 LPALEVENTCONTROLSOFT alEventControlSOFT;
99 LPALEVENTCALLBACKSOFT alEventCallbackSOFT;
101 LPALBUFFERCALLBACKSOFT alBufferCallbackSOFT;
103 const seconds AVNoSyncThreshold{10};
105 #define VIDEO_PICTURE_QUEUE_SIZE 24
107 const seconds_d64 AudioSyncThreshold{0.03};
108 const milliseconds AudioSampleCorrectionMax{50};
109 /* Averaging filter coefficient for audio sync. */
110 #define AUDIO_DIFF_AVG_NB 20
111 const double AudioAvgFilterCoeff{std::pow(0.01, 1.0/AUDIO_DIFF_AVG_NB)};
112 /* Per-buffer size, in time */
113 constexpr milliseconds AudioBufferTime{20};
114 /* Buffer total size, in time (should be divisible by the buffer time) */
115 constexpr milliseconds AudioBufferTotalTime{800};
116 constexpr auto AudioBufferCount = AudioBufferTotalTime / AudioBufferTime;
119 FF_MOVIE_DONE_EVENT = SDL_USEREVENT
122 enum class SyncMaster {
131 inline microseconds get_avtime()
132 { return microseconds{av_gettime()}; }
134 /* Define unique_ptrs to auto-cleanup associated ffmpeg objects. */
135 struct AVIOContextDeleter {
136 void operator()(AVIOContext *ptr) { avio_closep(&ptr); }
138 using AVIOContextPtr = std::unique_ptr<AVIOContext,AVIOContextDeleter>;
140 struct AVFormatCtxDeleter {
141 void operator()(AVFormatContext *ptr) { avformat_close_input(&ptr); }
143 using AVFormatCtxPtr = std::unique_ptr<AVFormatContext,AVFormatCtxDeleter>;
145 struct AVCodecCtxDeleter {
146 void operator()(AVCodecContext *ptr) { avcodec_free_context(&ptr); }
148 using AVCodecCtxPtr = std::unique_ptr<AVCodecContext,AVCodecCtxDeleter>;
150 struct AVPacketDeleter {
151 void operator()(AVPacket *pkt) { av_packet_free(&pkt); }
153 using AVPacketPtr = std::unique_ptr<AVPacket,AVPacketDeleter>;
155 struct AVFrameDeleter {
156 void operator()(AVFrame *ptr) { av_frame_free(&ptr); }
158 using AVFramePtr = std::unique_ptr<AVFrame,AVFrameDeleter>;
160 struct SwrContextDeleter {
161 void operator()(SwrContext *ptr) { swr_free(&ptr); }
163 using SwrContextPtr = std::unique_ptr<SwrContext,SwrContextDeleter>;
165 struct SwsContextDeleter {
166 void operator()(SwsContext *ptr) { sws_freeContext(ptr); }
168 using SwsContextPtr = std::unique_ptr<SwsContext,SwsContextDeleter>;
171 struct ChannelLayout : public AVChannelLayout {
172 ChannelLayout() : AVChannelLayout{} { }
173 ~ChannelLayout() { av_channel_layout_uninit(this); }
177 template<size_t SizeLimit>
179 std::mutex mPacketMutex, mFrameMutex;
180 std::condition_variable mPacketCond;
181 std::condition_variable mInFrameCond, mOutFrameCond;
183 std::deque<AVPacketPtr> mPackets;
184 size_t mTotalSize{0};
185 bool mFinished{false};
187 AVPacketPtr getPacket()
189 std::unique_lock<std::mutex> plock{mPacketMutex};
190 while(mPackets.empty() && !mFinished)
191 mPacketCond.wait(plock);
195 auto ret = std::move(mPackets.front());
196 mPackets.pop_front();
197 mTotalSize -= static_cast<unsigned int>(ret->size);
202 int sendPacket(AVCodecContext *codecctx)
204 AVPacketPtr packet{getPacket()};
208 std::unique_lock<std::mutex> flock{mFrameMutex};
209 while((ret=avcodec_send_packet(codecctx, packet.get())) == AVERROR(EAGAIN))
210 mInFrameCond.wait_for(flock, milliseconds{50});
212 mOutFrameCond.notify_one();
216 if(!ret) return AVErrorEOF;
217 std::cerr<< "Failed to send flush packet: "<<ret <<std::endl;
221 std::cerr<< "Failed to send packet: "<<ret <<std::endl;
225 int receiveFrame(AVCodecContext *codecctx, AVFrame *frame)
229 std::unique_lock<std::mutex> flock{mFrameMutex};
230 while((ret=avcodec_receive_frame(codecctx, frame)) == AVERROR(EAGAIN))
231 mOutFrameCond.wait_for(flock, milliseconds{50});
233 mInFrameCond.notify_one();
240 std::lock_guard<std::mutex> _{mPacketMutex};
243 mPacketCond.notify_one();
249 std::lock_guard<std::mutex> _{mPacketMutex};
255 mPacketCond.notify_one();
258 bool put(const AVPacket *pkt)
261 std::unique_lock<std::mutex> lock{mPacketMutex};
262 if(mTotalSize >= SizeLimit || mFinished)
265 mPackets.push_back(AVPacketPtr{av_packet_alloc()});
266 if(av_packet_ref(mPackets.back().get(), pkt) != 0)
272 mTotalSize += static_cast<unsigned int>(mPackets.back()->size);
274 mPacketCond.notify_one();
285 AVStream *mStream{nullptr};
286 AVCodecCtxPtr mCodecCtx;
288 DataQueue<2*1024*1024> mQueue;
290 /* Used for clock difference average computation */
291 seconds_d64 mClockDiffAvg{0};
293 /* Time of the next sample to be buffered */
294 nanoseconds mCurrentPts{0};
296 /* Device clock time that the stream started at. */
297 nanoseconds mDeviceStartTime{nanoseconds::min()};
299 /* Decompressed sample frame, and swresample context for conversion */
300 AVFramePtr mDecodedFrame;
301 SwrContextPtr mSwresCtx;
303 /* Conversion format, for what gets fed to OpenAL */
304 uint64_t mDstChanLayout{0};
305 AVSampleFormat mDstSampleFmt{AV_SAMPLE_FMT_NONE};
307 /* Storage of converted samples */
308 uint8_t *mSamples{nullptr};
309 int mSamplesLen{0}; /* In samples */
313 std::unique_ptr<uint8_t[]> mBufferData;
314 size_t mBufferDataSize{0};
315 std::atomic<size_t> mReadPos{0};
316 std::atomic<size_t> mWritePos{0};
319 ALenum mFormat{AL_NONE};
320 ALuint mFrameSize{0};
322 std::mutex mSrcMutex;
323 std::condition_variable mSrcCond;
324 std::atomic_flag mConnected;
326 std::array<ALuint,AudioBufferCount> mBuffers{};
327 ALuint mBufferIdx{0};
329 AudioState(MovieState &movie) : mMovie(movie)
330 { mConnected.test_and_set(std::memory_order_relaxed); }
334 alDeleteSources(1, &mSource);
336 alDeleteBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
341 static void AL_APIENTRY eventCallbackC(ALenum eventType, ALuint object, ALuint param,
342 ALsizei length, const ALchar *message, void *userParam)
343 { static_cast<AudioState*>(userParam)->eventCallback(eventType, object, param, length, message); }
344 void eventCallback(ALenum eventType, ALuint object, ALuint param, ALsizei length,
345 const ALchar *message);
347 static ALsizei AL_APIENTRY bufferCallbackC(void *userptr, void *data, ALsizei size)
348 { return static_cast<AudioState*>(userptr)->bufferCallback(data, size); }
349 ALsizei bufferCallback(void *data, ALsizei size);
351 nanoseconds getClockNoLock();
352 nanoseconds getClock()
354 std::lock_guard<std::mutex> lock{mSrcMutex};
355 return getClockNoLock();
358 bool startPlayback();
362 bool readAudio(uint8_t *samples, unsigned int length, int &sample_skip);
363 bool readAudio(int sample_skip);
371 AVStream *mStream{nullptr};
372 AVCodecCtxPtr mCodecCtx;
374 DataQueue<14*1024*1024> mQueue;
376 /* The pts of the currently displayed frame, and the time (av_gettime) it
377 * was last updated - used to have running video pts
379 nanoseconds mDisplayPts{0};
380 microseconds mDisplayPtsTime{microseconds::min()};
381 std::mutex mDispPtsMutex;
383 /* Swscale context for format conversion */
384 SwsContextPtr mSwscaleCtx;
388 nanoseconds mPts{nanoseconds::min()};
390 std::array<Picture,VIDEO_PICTURE_QUEUE_SIZE> mPictQ;
391 std::atomic<size_t> mPictQRead{0u}, mPictQWrite{1u};
392 std::mutex mPictQMutex;
393 std::condition_variable mPictQCond;
395 SDL_Texture *mImage{nullptr};
396 int mWidth{0}, mHeight{0}; /* Full texture size */
397 bool mFirstUpdate{true};
399 std::atomic<bool> mEOS{false};
400 std::atomic<bool> mFinalUpdate{false};
402 VideoState(MovieState &movie) : mMovie(movie) { }
406 SDL_DestroyTexture(mImage);
410 nanoseconds getClock();
412 void display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame);
413 void updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw);
418 AVIOContextPtr mIOContext;
419 AVFormatCtxPtr mFormatCtx;
421 SyncMaster mAVSyncType{SyncMaster::Default};
423 microseconds mClockBase{microseconds::min()};
425 std::atomic<bool> mQuit{false};
430 std::mutex mStartupMutex;
431 std::condition_variable mStartupCond;
432 bool mStartupDone{false};
434 std::thread mParseThread;
435 std::thread mAudioThread;
436 std::thread mVideoThread;
438 std::string mFilename;
440 MovieState(std::string fname)
441 : mAudio(*this), mVideo(*this), mFilename(std::move(fname))
446 if(mParseThread.joinable())
450 static int decode_interrupt_cb(void *ctx);
452 void setTitle(SDL_Window *window);
455 nanoseconds getClock();
457 nanoseconds getMasterClock();
459 nanoseconds getDuration();
461 int streamComponentOpen(unsigned int stream_index);
466 nanoseconds AudioState::getClockNoLock()
468 // The audio clock is the timestamp of the sample currently being heard.
469 if(alcGetInteger64vSOFT)
471 // If device start time = min, we aren't playing yet.
472 if(mDeviceStartTime == nanoseconds::min())
473 return nanoseconds::zero();
475 // Get the current device clock time and latency.
476 auto device = alcGetContextsDevice(alcGetCurrentContext());
477 ALCint64SOFT devtimes[2]{0,0};
478 alcGetInteger64vSOFT(device, ALC_DEVICE_CLOCK_LATENCY_SOFT, 2, devtimes);
479 auto latency = nanoseconds{devtimes[1]};
480 auto device_time = nanoseconds{devtimes[0]};
482 // The clock is simply the current device time relative to the recorded
483 // start time. We can also subtract the latency to get more a accurate
484 // position of where the audio device actually is in the output stream.
485 return device_time - mDeviceStartTime - latency;
488 if(mBufferDataSize > 0)
490 if(mDeviceStartTime == nanoseconds::min())
491 return nanoseconds::zero();
493 /* With a callback buffer and no device clock, mDeviceStartTime is
494 * actually the timestamp of the first sample frame played. The audio
495 * clock, then, is that plus the current source offset.
497 ALint64SOFT offset[2];
498 if(alGetSourcei64vSOFT)
499 alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
503 alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
504 offset[0] = ALint64SOFT{ioffset} << 32;
507 /* NOTE: The source state must be checked last, in case an underrun
508 * occurs and the source stops between getting the state and retrieving
509 * the offset+latency.
512 alGetSourcei(mSource, AL_SOURCE_STATE, &status);
515 if(status == AL_PLAYING || status == AL_PAUSED)
516 pts = mDeviceStartTime - nanoseconds{offset[1]} +
517 duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
520 /* If the source is stopped, the pts of the next sample to be heard
521 * is the pts of the next sample to be buffered, minus the amount
522 * already in the buffer ready to play.
524 const size_t woffset{mWritePos.load(std::memory_order_acquire)};
525 const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
526 const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
529 pts = mCurrentPts - nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
535 /* The source-based clock is based on 4 components:
536 * 1 - The timestamp of the next sample to buffer (mCurrentPts)
537 * 2 - The length of the source's buffer queue
538 * (AudioBufferTime*AL_BUFFERS_QUEUED)
539 * 3 - The offset OpenAL is currently at in the source (the first value
540 * from AL_SAMPLE_OFFSET_LATENCY_SOFT)
541 * 4 - The latency between OpenAL and the DAC (the second value from
542 * AL_SAMPLE_OFFSET_LATENCY_SOFT)
544 * Subtracting the length of the source queue from the next sample's
545 * timestamp gives the timestamp of the sample at the start of the source
546 * queue. Adding the source offset to that results in the timestamp for the
547 * sample at OpenAL's current position, and subtracting the source latency
548 * from that gives the timestamp of the sample currently at the DAC.
550 nanoseconds pts{mCurrentPts};
553 ALint64SOFT offset[2];
554 if(alGetSourcei64vSOFT)
555 alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
559 alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
560 offset[0] = ALint64SOFT{ioffset} << 32;
563 ALint queued, status;
564 alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
565 alGetSourcei(mSource, AL_SOURCE_STATE, &status);
567 /* If the source is AL_STOPPED, then there was an underrun and all
568 * buffers are processed, so ignore the source queue. The audio thread
569 * will put the source into an AL_INITIAL state and clear the queue
570 * when it starts recovery.
572 if(status != AL_STOPPED)
574 pts -= AudioBufferTime*queued;
575 pts += duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
577 /* Don't offset by the latency if the source isn't playing. */
578 if(status == AL_PLAYING)
579 pts -= nanoseconds{offset[1]};
582 return std::max(pts, nanoseconds::zero());
585 bool AudioState::startPlayback()
587 const size_t woffset{mWritePos.load(std::memory_order_acquire)};
588 const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
589 const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
592 if(mBufferDataSize > 0)
596 if(!alcGetInteger64vSOFT)
597 mDeviceStartTime = mCurrentPts -
598 nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
603 alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
604 if(queued == 0) return false;
607 alSourcePlay(mSource);
608 if(alcGetInteger64vSOFT)
610 /* Subtract the total buffer queue time from the current pts to get the
611 * pts of the start of the queue.
613 int64_t srctimes[2]{0,0};
614 alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_CLOCK_SOFT, srctimes);
615 auto device_time = nanoseconds{srctimes[1]};
616 auto src_offset = duration_cast<nanoseconds>(fixed32{srctimes[0]}) /
617 mCodecCtx->sample_rate;
619 /* The mixer may have ticked and incremented the device time and sample
620 * offset, so subtract the source offset from the device time to get
621 * the device time the source started at. Also subtract startpts to get
622 * the device time the stream would have started at to reach where it
625 if(mBufferDataSize > 0)
627 nanoseconds startpts{mCurrentPts -
628 nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate};
629 mDeviceStartTime = device_time - src_offset - startpts;
633 nanoseconds startpts{mCurrentPts - AudioBufferTotalTime};
634 mDeviceStartTime = device_time - src_offset - startpts;
640 int AudioState::getSync()
642 if(mMovie.mAVSyncType == SyncMaster::Audio)
645 auto ref_clock = mMovie.getMasterClock();
646 auto diff = ref_clock - getClockNoLock();
648 if(!(diff < AVNoSyncThreshold && diff > -AVNoSyncThreshold))
650 /* Difference is TOO big; reset accumulated average */
651 mClockDiffAvg = seconds_d64::zero();
655 /* Accumulate the diffs */
656 mClockDiffAvg = mClockDiffAvg*AudioAvgFilterCoeff + diff;
657 auto avg_diff = mClockDiffAvg*(1.0 - AudioAvgFilterCoeff);
658 if(avg_diff < AudioSyncThreshold/2.0 && avg_diff > -AudioSyncThreshold)
661 /* Constrain the per-update difference to avoid exceedingly large skips */
662 diff = std::min<nanoseconds>(diff, AudioSampleCorrectionMax);
663 return static_cast<int>(duration_cast<seconds>(diff*mCodecCtx->sample_rate).count());
666 int AudioState::decodeFrame()
669 while(int ret{mQueue.receiveFrame(mCodecCtx.get(), mDecodedFrame.get())})
671 if(ret == AVErrorEOF) return 0;
672 std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
674 } while(mDecodedFrame->nb_samples <= 0);
676 /* If provided, update w/ pts */
677 if(mDecodedFrame->best_effort_timestamp != AVNoPtsValue)
678 mCurrentPts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
679 static_cast<double>(mDecodedFrame->best_effort_timestamp)});
681 if(mDecodedFrame->nb_samples > mSamplesMax)
684 av_samples_alloc(&mSamples, nullptr, mCodecCtx->ch_layout.nb_channels,
685 mDecodedFrame->nb_samples, mDstSampleFmt, 0);
686 mSamplesMax = mDecodedFrame->nb_samples;
688 /* Return the amount of sample frames converted */
689 int data_size{swr_convert(mSwresCtx.get(), &mSamples, mDecodedFrame->nb_samples,
690 const_cast<const uint8_t**>(mDecodedFrame->data), mDecodedFrame->nb_samples)};
692 av_frame_unref(mDecodedFrame.get());
696 /* Duplicates the sample at in to out, count times. The frame size is a
697 * multiple of the template type size.
700 static void sample_dup(uint8_t *out, const uint8_t *in, size_t count, size_t frame_size)
702 auto *sample = reinterpret_cast<const T*>(in);
703 auto *dst = reinterpret_cast<T*>(out);
705 /* NOTE: frame_size is a multiple of sizeof(T). */
706 size_t type_mult{frame_size / sizeof(T)};
708 std::fill_n(dst, count, *sample);
709 else for(size_t i{0};i < count;++i)
711 for(size_t j{0};j < type_mult;++j)
712 dst[i*type_mult + j] = sample[j];
716 static void sample_dup(uint8_t *out, const uint8_t *in, size_t count, size_t frame_size)
718 if((frame_size&7) == 0)
719 sample_dup<uint64_t>(out, in, count, frame_size);
720 else if((frame_size&3) == 0)
721 sample_dup<uint32_t>(out, in, count, frame_size);
722 else if((frame_size&1) == 0)
723 sample_dup<uint16_t>(out, in, count, frame_size);
725 sample_dup<uint8_t>(out, in, count, frame_size);
728 bool AudioState::readAudio(uint8_t *samples, unsigned int length, int &sample_skip)
730 unsigned int audio_size{0};
732 /* Read the next chunk of data, refill the buffer, and queue it
734 length /= mFrameSize;
735 while(mSamplesLen > 0 && audio_size < length)
737 unsigned int rem{length - audio_size};
740 const auto len = static_cast<unsigned int>(mSamplesLen - mSamplesPos);
741 if(rem > len) rem = len;
742 std::copy_n(mSamples + static_cast<unsigned int>(mSamplesPos)*mFrameSize,
743 rem*mFrameSize, samples);
747 rem = std::min(rem, static_cast<unsigned int>(-mSamplesPos));
749 /* Add samples by copying the first sample */
750 sample_dup(samples, mSamples, rem, mFrameSize);
754 mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
755 samples += rem*mFrameSize;
758 while(mSamplesPos >= mSamplesLen)
760 mSamplesLen = decodeFrame();
761 mSamplesPos = std::min(mSamplesLen, sample_skip);
762 if(mSamplesLen <= 0) break;
764 sample_skip -= mSamplesPos;
766 // Adjust the device start time and current pts by the amount we're
767 // skipping/duplicating, so that the clock remains correct for the
768 // current stream position.
769 auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
770 mDeviceStartTime -= skip;
777 if(audio_size < length)
779 const unsigned int rem{length - audio_size};
780 std::fill_n(samples, rem*mFrameSize,
781 (mDstSampleFmt == AV_SAMPLE_FMT_U8) ? 0x80 : 0x00);
782 mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
787 bool AudioState::readAudio(int sample_skip)
789 size_t woffset{mWritePos.load(std::memory_order_acquire)};
790 const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
791 while(mSamplesLen > 0)
793 const size_t nsamples{((roffset > woffset) ? roffset-woffset-1
794 : (roffset == 0) ? (mBufferDataSize-woffset-1)
795 : (mBufferDataSize-woffset)) / mFrameSize};
800 const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(-mSamplesPos))};
802 sample_dup(&mBufferData[woffset], mSamples, rem, mFrameSize);
803 woffset += rem * mFrameSize;
804 if(woffset == mBufferDataSize) woffset = 0;
805 mWritePos.store(woffset, std::memory_order_release);
807 mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
808 mSamplesPos += static_cast<int>(rem);
812 const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(mSamplesLen-mSamplesPos))};
813 const size_t boffset{static_cast<ALuint>(mSamplesPos) * size_t{mFrameSize}};
814 const size_t nbytes{rem * mFrameSize};
816 memcpy(&mBufferData[woffset], mSamples + boffset, nbytes);
818 if(woffset == mBufferDataSize) woffset = 0;
819 mWritePos.store(woffset, std::memory_order_release);
821 mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
822 mSamplesPos += static_cast<int>(rem);
824 while(mSamplesPos >= mSamplesLen)
826 mSamplesLen = decodeFrame();
827 mSamplesPos = std::min(mSamplesLen, sample_skip);
828 if(mSamplesLen <= 0) return false;
830 sample_skip -= mSamplesPos;
832 auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
833 mDeviceStartTime -= skip;
842 void AL_APIENTRY AudioState::eventCallback(ALenum eventType, ALuint object, ALuint param,
843 ALsizei length, const ALchar *message)
845 if(eventType == AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT)
847 /* Temporarily lock the source mutex to ensure it's not between
848 * checking the processed count and going to sleep.
850 std::unique_lock<std::mutex>{mSrcMutex}.unlock();
851 mSrcCond.notify_one();
855 std::cout<< "\n---- AL Event on AudioState "<<this<<" ----\nEvent: ";
858 case AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT: std::cout<< "Buffer completed"; break;
859 case AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT: std::cout<< "Source state changed"; break;
860 case AL_EVENT_TYPE_DISCONNECTED_SOFT: std::cout<< "Disconnected"; break;
862 std::cout<< "0x"<<std::hex<<std::setw(4)<<std::setfill('0')<<eventType<<std::dec<<
863 std::setw(0)<<std::setfill(' '); break;
866 "Object ID: "<<object<<"\n"
867 "Parameter: "<<param<<"\n"
868 "Message: "<<std::string{message, static_cast<ALuint>(length)}<<"\n----"<<
871 if(eventType == AL_EVENT_TYPE_DISCONNECTED_SOFT)
874 std::lock_guard<std::mutex> lock{mSrcMutex};
875 mConnected.clear(std::memory_order_release);
877 mSrcCond.notify_one();
881 ALsizei AudioState::bufferCallback(void *data, ALsizei size)
885 size_t roffset{mReadPos.load(std::memory_order_acquire)};
888 const size_t woffset{mWritePos.load(std::memory_order_relaxed)};
889 if(woffset == roffset) break;
891 size_t todo{((woffset < roffset) ? mBufferDataSize : woffset) - roffset};
892 todo = std::min<size_t>(todo, static_cast<ALuint>(size-got));
894 memcpy(data, &mBufferData[roffset], todo);
895 data = static_cast<ALbyte*>(data) + todo;
896 got += static_cast<ALsizei>(todo);
899 if(roffset == mBufferDataSize)
902 mReadPos.store(roffset, std::memory_order_release);
907 int AudioState::handler()
909 std::unique_lock<std::mutex> srclock{mSrcMutex, std::defer_lock};
910 milliseconds sleep_time{AudioBufferTime / 3};
912 struct EventControlManager {
913 const std::array<ALenum,3> evt_types{{
914 AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT, AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT,
915 AL_EVENT_TYPE_DISCONNECTED_SOFT}};
917 EventControlManager(milliseconds &sleep_time)
919 if(alEventControlSOFT)
921 alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
923 alEventCallbackSOFT(&AudioState::eventCallbackC, this);
924 sleep_time = AudioBufferTotalTime;
927 ~EventControlManager()
929 if(alEventControlSOFT)
931 alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
933 alEventCallbackSOFT(nullptr, nullptr);
937 EventControlManager event_controller{sleep_time};
939 std::unique_ptr<uint8_t[]> samples;
940 ALsizei buffer_len{0};
942 /* Find a suitable format for OpenAL. */
945 if((mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLT || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLTP
946 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBL
947 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBLP
948 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32
949 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32P
950 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64
951 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64P)
952 && alIsExtensionPresent("AL_EXT_FLOAT32"))
954 mDstSampleFmt = AV_SAMPLE_FMT_FLT;
956 if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
958 if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
960 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_7POINT1)
962 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
964 mFormat = alGetEnumValue("AL_FORMAT_71CHN32");
966 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1
967 || mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK)
969 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
971 mFormat = alGetEnumValue("AL_FORMAT_51CHN32");
973 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_QUAD)
975 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
977 mFormat = alGetEnumValue("AL_FORMAT_QUAD32");
980 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_MONO)
982 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
984 mFormat = AL_FORMAT_MONO_FLOAT32;
987 else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
988 && alIsExtensionPresent("AL_EXT_BFORMAT"))
990 /* Calculate what should be the ambisonic order from the number of
991 * channels, and confirm that's the number of channels. Opus allows
992 * an optional non-diegetic stereo stream with the B-Format stream,
993 * which we can ignore, so check for that too.
995 auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
996 int channels{(order+1) * (order+1)};
997 if(channels == mCodecCtx->ch_layout.nb_channels
998 || channels+2 == mCodecCtx->ch_layout.nb_channels)
1000 /* OpenAL only supports first-order with AL_EXT_BFORMAT, which
1001 * is 4 channels for 3D buffers.
1004 mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_FLOAT32");
1007 if(!mFormat || mFormat == -1)
1009 mDstChanLayout = AV_CH_LAYOUT_STEREO;
1011 mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN_FLOAT32_SOFT : AL_FORMAT_STEREO_FLOAT32;
1014 if(mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8P)
1016 mDstSampleFmt = AV_SAMPLE_FMT_U8;
1018 if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
1020 if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
1022 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_7POINT1)
1024 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1026 mFormat = alGetEnumValue("AL_FORMAT_71CHN8");
1028 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1
1029 || mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK)
1031 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1033 mFormat = alGetEnumValue("AL_FORMAT_51CHN8");
1035 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_QUAD)
1037 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1039 mFormat = alGetEnumValue("AL_FORMAT_QUAD8");
1042 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_MONO)
1044 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1046 mFormat = AL_FORMAT_MONO8;
1049 else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
1050 && alIsExtensionPresent("AL_EXT_BFORMAT"))
1052 auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
1053 int channels{(order+1) * (order+1)};
1054 if(channels == mCodecCtx->ch_layout.nb_channels
1055 || channels+2 == mCodecCtx->ch_layout.nb_channels)
1058 mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_8");
1061 if(!mFormat || mFormat == -1)
1063 mDstChanLayout = AV_CH_LAYOUT_STEREO;
1065 mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN8_SOFT : AL_FORMAT_STEREO8;
1068 if(!mFormat || mFormat == -1)
1070 mDstSampleFmt = AV_SAMPLE_FMT_S16;
1072 if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
1074 if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
1076 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_7POINT1)
1078 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1080 mFormat = alGetEnumValue("AL_FORMAT_71CHN16");
1082 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1
1083 || mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK)
1085 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1087 mFormat = alGetEnumValue("AL_FORMAT_51CHN16");
1089 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_QUAD)
1091 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1093 mFormat = alGetEnumValue("AL_FORMAT_QUAD16");
1096 if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_MONO)
1098 mDstChanLayout = mCodecCtx->ch_layout.u.mask;
1100 mFormat = AL_FORMAT_MONO16;
1103 else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
1104 && alIsExtensionPresent("AL_EXT_BFORMAT"))
1106 auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
1107 int channels{(order+1) * (order+1)};
1108 if(channels == mCodecCtx->ch_layout.nb_channels
1109 || channels+2 == mCodecCtx->ch_layout.nb_channels)
1112 mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_16");
1115 if(!mFormat || mFormat == -1)
1117 mDstChanLayout = AV_CH_LAYOUT_STEREO;
1119 mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN16_SOFT : AL_FORMAT_STEREO16;
1128 mDecodedFrame.reset(av_frame_alloc());
1131 std::cerr<< "Failed to allocate audio frame" <<std::endl;
1135 /* Note that ffmpeg assumes AmbiX (ACN layout, SN3D normalization). */
1136 const bool has_bfmt_ex{alIsExtensionPresent("AL_SOFT_bformat_ex") != AL_FALSE};
1137 const ALenum ambi_layout{AL_ACN_SOFT};
1138 const ALenum ambi_scale{AL_SN3D_SOFT};
1142 /* OpenAL only supports first-order ambisonics with AL_EXT_BFORMAT, so
1143 * we have to drop any extra channels.
1145 ChannelLayout layout{};
1146 av_channel_layout_from_string(&layout, "ambisonic 1");
1149 int err{swr_alloc_set_opts2(&ps, &layout, mDstSampleFmt, mCodecCtx->sample_rate,
1150 &mCodecCtx->ch_layout, mCodecCtx->sample_fmt, mCodecCtx->sample_rate, 0, nullptr)};
1151 mSwresCtx.reset(ps);
1154 char errstr[AV_ERROR_MAX_STRING_SIZE]{};
1155 std::cerr<< "Failed to allocate SwrContext: "
1156 <<av_make_error_string(errstr, AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
1161 std::cout<< "Found AL_SOFT_bformat_ex" <<std::endl;
1164 std::cout<< "Found AL_EXT_BFORMAT" <<std::endl;
1165 /* Without AL_SOFT_bformat_ex, OpenAL only supports FuMa channel
1166 * ordering and normalization, so a custom matrix is needed to
1167 * scale and reorder the source from AmbiX.
1169 std::vector<double> mtx(64*64, 0.0);
1170 mtx[0 + 0*64] = std::sqrt(0.5);
1171 mtx[3 + 1*64] = 1.0;
1172 mtx[1 + 2*64] = 1.0;
1173 mtx[2 + 3*64] = 1.0;
1174 swr_set_matrix(mSwresCtx.get(), mtx.data(), 64);
1179 ChannelLayout layout{};
1180 av_channel_layout_from_mask(&layout, mDstChanLayout);
1183 int err{swr_alloc_set_opts2(&ps, &layout, mDstSampleFmt, mCodecCtx->sample_rate,
1184 &mCodecCtx->ch_layout, mCodecCtx->sample_fmt, mCodecCtx->sample_rate, 0, nullptr)};
1185 mSwresCtx.reset(ps);
1188 char errstr[AV_ERROR_MAX_STRING_SIZE]{};
1189 std::cerr<< "Failed to allocate SwrContext: "
1190 <<av_make_error_string(errstr, AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
1194 if(int err{swr_init(mSwresCtx.get())})
1196 char errstr[AV_ERROR_MAX_STRING_SIZE]{};
1197 std::cerr<< "Failed to initialize audio converter: "
1198 <<av_make_error_string(errstr, AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
1202 alGenBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
1203 alGenSources(1, &mSource);
1206 alSourcei(mSource, AL_DIRECT_CHANNELS_SOFT, DirectOutMode);
1207 if(EnableWideStereo)
1209 const float angles[2]{static_cast<float>(M_PI / 3.0), static_cast<float>(-M_PI / 3.0)};
1210 alSourcefv(mSource, AL_STEREO_ANGLES, angles);
1214 for(ALuint bufid : mBuffers)
1216 alBufferi(bufid, AL_AMBISONIC_LAYOUT_SOFT, ambi_layout);
1217 alBufferi(bufid, AL_AMBISONIC_SCALING_SOFT, ambi_scale);
1221 if(EnableSuperStereo)
1222 alSourcei(mSource, AL_STEREO_MODE_SOFT, AL_SUPER_STEREO_SOFT);
1225 if(alGetError() != AL_NO_ERROR)
1228 bool callback_ok{false};
1229 if(alBufferCallbackSOFT)
1231 alBufferCallbackSOFT(mBuffers[0], mFormat, mCodecCtx->sample_rate, bufferCallbackC, this);
1232 alSourcei(mSource, AL_BUFFER, static_cast<ALint>(mBuffers[0]));
1233 if(alGetError() != AL_NO_ERROR)
1235 fprintf(stderr, "Failed to set buffer callback\n");
1236 alSourcei(mSource, AL_BUFFER, 0);
1240 mBufferDataSize = static_cast<size_t>(duration_cast<seconds>(mCodecCtx->sample_rate *
1241 AudioBufferTotalTime).count()) * mFrameSize;
1242 mBufferData = std::make_unique<uint8_t[]>(mBufferDataSize);
1243 std::fill_n(mBufferData.get(), mBufferDataSize, uint8_t{});
1245 mReadPos.store(0, std::memory_order_relaxed);
1246 mWritePos.store(mBufferDataSize/mFrameSize/2*mFrameSize, std::memory_order_relaxed);
1249 alcGetIntegerv(alcGetContextsDevice(alcGetCurrentContext()), ALC_REFRESH, 1, &refresh);
1250 sleep_time = milliseconds{seconds{1}} / refresh;
1255 buffer_len = static_cast<int>(duration_cast<seconds>(mCodecCtx->sample_rate *
1256 AudioBufferTime).count() * mFrameSize);
1258 samples = std::make_unique<uint8_t[]>(static_cast<ALuint>(buffer_len));
1260 /* Prefill the codec buffer. */
1261 auto packet_sender = [this]()
1265 const int ret{mQueue.sendPacket(mCodecCtx.get())};
1266 if(ret == AVErrorEOF) break;
1269 auto sender = std::async(std::launch::async, packet_sender);
1272 if(alcGetInteger64vSOFT)
1275 alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()), ALC_DEVICE_CLOCK_SOFT,
1277 mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
1280 mSamplesLen = decodeFrame();
1283 mSamplesPos = std::min(mSamplesLen, getSync());
1285 auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
1286 mDeviceStartTime -= skip;
1287 mCurrentPts += skip;
1292 if(mMovie.mQuit.load(std::memory_order_relaxed))
1294 /* If mQuit is set, drain frames until we can't get more audio,
1295 * indicating we've reached the flush packet and the packet sender
1299 mSamplesLen = decodeFrame();
1300 mSamplesPos = mSamplesLen;
1301 } while(mSamplesLen > 0);
1306 if(mBufferDataSize > 0)
1308 alGetSourcei(mSource, AL_SOURCE_STATE, &state);
1310 /* If mQuit is not set, don't quit even if there's no more audio,
1311 * so what's buffered has a chance to play to the real end.
1313 readAudio(getSync());
1317 ALint processed, queued;
1319 /* First remove any processed buffers. */
1320 alGetSourcei(mSource, AL_BUFFERS_PROCESSED, &processed);
1321 while(processed > 0)
1324 alSourceUnqueueBuffers(mSource, 1, &bid);
1328 /* Refill the buffer queue. */
1329 int sync_skip{getSync()};
1330 alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
1331 while(static_cast<ALuint>(queued) < mBuffers.size())
1333 /* Read the next chunk of data, filling the buffer, and queue
1336 if(!readAudio(samples.get(), static_cast<ALuint>(buffer_len), sync_skip))
1339 const ALuint bufid{mBuffers[mBufferIdx]};
1340 mBufferIdx = static_cast<ALuint>((mBufferIdx+1) % mBuffers.size());
1342 alBufferData(bufid, mFormat, samples.get(), buffer_len, mCodecCtx->sample_rate);
1343 alSourceQueueBuffers(mSource, 1, &bufid);
1347 /* Check that the source is playing. */
1348 alGetSourcei(mSource, AL_SOURCE_STATE, &state);
1349 if(state == AL_STOPPED)
1351 /* AL_STOPPED means there was an underrun. Clear the buffer
1352 * queue since this likely means we're late, and rewind the
1353 * source to get it back into an AL_INITIAL state.
1355 alSourceRewind(mSource);
1356 alSourcei(mSource, AL_BUFFER, 0);
1357 if(alcGetInteger64vSOFT)
1359 /* Also update the device start time with the current
1360 * device clock, so the decoder knows we're running behind.
1363 alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()),
1364 ALC_DEVICE_CLOCK_SOFT, 1, &devtime);
1365 mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
1371 /* (re)start the source if needed, and wait for a buffer to finish */
1372 if(state != AL_PLAYING && state != AL_PAUSED)
1374 if(!startPlayback())
1377 if(ALenum err{alGetError()})
1378 std::cerr<< "Got AL error: 0x"<<std::hex<<err<<std::dec
1379 << " ("<<alGetString(err)<<")" <<std::endl;
1381 mSrcCond.wait_for(srclock, sleep_time);
1385 alSourceRewind(mSource);
1386 alSourcei(mSource, AL_BUFFER, 0);
1393 nanoseconds VideoState::getClock()
1395 /* NOTE: This returns incorrect times while not playing. */
1396 std::lock_guard<std::mutex> _{mDispPtsMutex};
1397 if(mDisplayPtsTime == microseconds::min())
1398 return nanoseconds::zero();
1399 auto delta = get_avtime() - mDisplayPtsTime;
1400 return mDisplayPts + delta;
1403 /* Called by VideoState::updateVideo to display the next video frame. */
1404 void VideoState::display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame)
1409 double aspect_ratio;
1413 int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
1414 int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
1415 if(frame->sample_aspect_ratio.num == 0)
1419 aspect_ratio = av_q2d(frame->sample_aspect_ratio) * frame_width /
1422 if(aspect_ratio <= 0.0)
1423 aspect_ratio = static_cast<double>(frame_width) / frame_height;
1425 SDL_GetWindowSize(screen, &win_w, &win_h);
1427 w = (static_cast<int>(std::rint(h * aspect_ratio)) + 3) & ~3;
1431 h = (static_cast<int>(std::rint(w / aspect_ratio)) + 3) & ~3;
1433 x = (win_w - w) / 2;
1434 y = (win_h - h) / 2;
1436 SDL_Rect src_rect{ static_cast<int>(frame->crop_left), static_cast<int>(frame->crop_top),
1437 frame_width, frame_height };
1438 SDL_Rect dst_rect{ x, y, w, h };
1439 SDL_RenderCopy(renderer, mImage, &src_rect, &dst_rect);
1440 SDL_RenderPresent(renderer);
1443 /* Called regularly on the main thread where the SDL_Renderer was created. It
1444 * handles updating the textures of decoded frames and displaying the latest
1447 void VideoState::updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw)
1449 size_t read_idx{mPictQRead.load(std::memory_order_relaxed)};
1450 Picture *vp{&mPictQ[read_idx]};
1452 auto clocktime = mMovie.getMasterClock();
1453 bool updated{false};
1456 size_t next_idx{(read_idx+1)%mPictQ.size()};
1457 if(next_idx == mPictQWrite.load(std::memory_order_acquire))
1459 Picture *nextvp{&mPictQ[next_idx]};
1460 if(clocktime < nextvp->mPts && !mMovie.mQuit.load(std::memory_order_relaxed))
1462 /* For the first update, ensure the first frame gets shown. */
1463 if(!mFirstUpdate || updated)
1469 read_idx = next_idx;
1471 if(mMovie.mQuit.load(std::memory_order_relaxed))
1474 mFinalUpdate = true;
1475 mPictQRead.store(read_idx, std::memory_order_release);
1476 std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1477 mPictQCond.notify_one();
1481 AVFrame *frame{vp->mFrame.get()};
1484 mPictQRead.store(read_idx, std::memory_order_release);
1485 std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1486 mPictQCond.notify_one();
1488 /* allocate or resize the buffer! */
1489 bool fmt_updated{false};
1490 if(!mImage || mWidth != frame->width || mHeight != frame->height)
1494 SDL_DestroyTexture(mImage);
1495 mImage = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING,
1496 frame->width, frame->height);
1498 std::cerr<< "Failed to create YV12 texture!" <<std::endl;
1499 mWidth = frame->width;
1500 mHeight = frame->height;
1503 int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
1504 int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
1505 if(mFirstUpdate && frame_width > 0 && frame_height > 0)
1507 /* For the first update, set the window size to the video size. */
1508 mFirstUpdate = false;
1510 if(frame->sample_aspect_ratio.den != 0)
1512 double aspect_ratio = av_q2d(frame->sample_aspect_ratio);
1513 if(aspect_ratio >= 1.0)
1514 frame_width = static_cast<int>(frame_width*aspect_ratio + 0.5);
1515 else if(aspect_ratio > 0.0)
1516 frame_height = static_cast<int>(frame_height/aspect_ratio + 0.5);
1518 SDL_SetWindowSize(screen, frame_width, frame_height);
1523 void *pixels{nullptr};
1526 if(mCodecCtx->pix_fmt == AV_PIX_FMT_YUV420P)
1527 SDL_UpdateYUVTexture(mImage, nullptr,
1528 frame->data[0], frame->linesize[0],
1529 frame->data[1], frame->linesize[1],
1530 frame->data[2], frame->linesize[2]
1532 else if(SDL_LockTexture(mImage, nullptr, &pixels, &pitch) != 0)
1533 std::cerr<< "Failed to lock texture" <<std::endl;
1536 // Convert the image into YUV format that SDL uses
1537 int w{frame->width};
1538 int h{frame->height};
1539 if(!mSwscaleCtx || fmt_updated)
1541 mSwscaleCtx.reset(sws_getContext(
1542 w, h, mCodecCtx->pix_fmt,
1543 w, h, AV_PIX_FMT_YUV420P, 0,
1544 nullptr, nullptr, nullptr
1548 /* point pict at the queue */
1549 uint8_t *pict_data[3];
1550 pict_data[0] = static_cast<uint8_t*>(pixels);
1551 pict_data[1] = pict_data[0] + w*h;
1552 pict_data[2] = pict_data[1] + w*h/4;
1554 int pict_linesize[3];
1555 pict_linesize[0] = pitch;
1556 pict_linesize[1] = pitch / 2;
1557 pict_linesize[2] = pitch / 2;
1559 sws_scale(mSwscaleCtx.get(), reinterpret_cast<uint8_t**>(frame->data), frame->linesize,
1560 0, h, pict_data, pict_linesize);
1561 SDL_UnlockTexture(mImage);
1570 /* Show the picture! */
1571 display(screen, renderer, frame);
1576 auto disp_time = get_avtime();
1578 std::lock_guard<std::mutex> _{mDispPtsMutex};
1579 mDisplayPts = vp->mPts;
1580 mDisplayPtsTime = disp_time;
1582 if(mEOS.load(std::memory_order_acquire))
1584 if((read_idx+1)%mPictQ.size() == mPictQWrite.load(std::memory_order_acquire))
1586 mFinalUpdate = true;
1587 std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1588 mPictQCond.notify_one();
1593 int VideoState::handler()
1595 std::for_each(mPictQ.begin(), mPictQ.end(),
1596 [](Picture &pict) -> void
1597 { pict.mFrame = AVFramePtr{av_frame_alloc()}; });
1599 /* Prefill the codec buffer. */
1600 auto packet_sender = [this]()
1604 const int ret{mQueue.sendPacket(mCodecCtx.get())};
1605 if(ret == AVErrorEOF) break;
1608 auto sender = std::async(std::launch::async, packet_sender);
1611 std::lock_guard<std::mutex> _{mDispPtsMutex};
1612 mDisplayPtsTime = get_avtime();
1615 auto current_pts = nanoseconds::zero();
1618 size_t write_idx{mPictQWrite.load(std::memory_order_relaxed)};
1619 Picture *vp{&mPictQ[write_idx]};
1621 /* Retrieve video frame. */
1622 AVFrame *decoded_frame{vp->mFrame.get()};
1623 while(int ret{mQueue.receiveFrame(mCodecCtx.get(), decoded_frame)})
1625 if(ret == AVErrorEOF) goto finish;
1626 std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
1629 /* Get the PTS for this frame. */
1630 if(decoded_frame->best_effort_timestamp != AVNoPtsValue)
1631 current_pts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
1632 static_cast<double>(decoded_frame->best_effort_timestamp)});
1633 vp->mPts = current_pts;
1635 /* Update the video clock to the next expected PTS. */
1636 auto frame_delay = av_q2d(mCodecCtx->time_base);
1637 frame_delay += decoded_frame->repeat_pict * (frame_delay * 0.5);
1638 current_pts += duration_cast<nanoseconds>(seconds_d64{frame_delay});
1640 /* Put the frame in the queue to be loaded into a texture and displayed
1641 * by the rendering thread.
1643 write_idx = (write_idx+1)%mPictQ.size();
1644 mPictQWrite.store(write_idx, std::memory_order_release);
1646 if(write_idx == mPictQRead.load(std::memory_order_acquire))
1648 /* Wait until we have space for a new pic */
1649 std::unique_lock<std::mutex> lock{mPictQMutex};
1650 while(write_idx == mPictQRead.load(std::memory_order_acquire))
1651 mPictQCond.wait(lock);
1657 std::unique_lock<std::mutex> lock{mPictQMutex};
1658 while(!mFinalUpdate) mPictQCond.wait(lock);
1664 int MovieState::decode_interrupt_cb(void *ctx)
1666 return static_cast<MovieState*>(ctx)->mQuit.load(std::memory_order_relaxed);
1669 bool MovieState::prepare()
1671 AVIOContext *avioctx{nullptr};
1672 AVIOInterruptCB intcb{decode_interrupt_cb, this};
1673 if(avio_open2(&avioctx, mFilename.c_str(), AVIO_FLAG_READ, &intcb, nullptr))
1675 std::cerr<< "Failed to open "<<mFilename <<std::endl;
1678 mIOContext.reset(avioctx);
1680 /* Open movie file. If avformat_open_input fails it will automatically free
1681 * this context, so don't set it onto a smart pointer yet.
1683 AVFormatContext *fmtctx{avformat_alloc_context()};
1684 fmtctx->pb = mIOContext.get();
1685 fmtctx->interrupt_callback = intcb;
1686 if(avformat_open_input(&fmtctx, mFilename.c_str(), nullptr, nullptr) != 0)
1688 std::cerr<< "Failed to open "<<mFilename <<std::endl;
1691 mFormatCtx.reset(fmtctx);
1693 /* Retrieve stream information */
1694 if(avformat_find_stream_info(mFormatCtx.get(), nullptr) < 0)
1696 std::cerr<< mFilename<<": failed to find stream info" <<std::endl;
1700 /* Dump information about file onto standard error */
1701 av_dump_format(mFormatCtx.get(), 0, mFilename.c_str(), 0);
1703 mParseThread = std::thread{std::mem_fn(&MovieState::parse_handler), this};
1705 std::unique_lock<std::mutex> slock{mStartupMutex};
1706 while(!mStartupDone) mStartupCond.wait(slock);
1710 void MovieState::setTitle(SDL_Window *window)
1712 auto pos1 = mFilename.rfind('/');
1713 auto pos2 = mFilename.rfind('\\');
1714 auto fpos = ((pos1 == std::string::npos) ? pos2 :
1715 (pos2 == std::string::npos) ? pos1 :
1716 std::max(pos1, pos2)) + 1;
1717 SDL_SetWindowTitle(window, (mFilename.substr(fpos)+" - "+AppName).c_str());
1720 nanoseconds MovieState::getClock()
1722 if(mClockBase == microseconds::min())
1723 return nanoseconds::zero();
1724 return get_avtime() - mClockBase;
1727 nanoseconds MovieState::getMasterClock()
1729 if(mAVSyncType == SyncMaster::Video && mVideo.mStream)
1730 return mVideo.getClock();
1731 if(mAVSyncType == SyncMaster::Audio && mAudio.mStream)
1732 return mAudio.getClock();
1736 nanoseconds MovieState::getDuration()
1737 { return std::chrono::duration<int64_t,std::ratio<1,AV_TIME_BASE>>(mFormatCtx->duration); }
1739 int MovieState::streamComponentOpen(unsigned int stream_index)
1741 if(stream_index >= mFormatCtx->nb_streams)
1744 /* Get a pointer to the codec context for the stream, and open the
1747 AVCodecCtxPtr avctx{avcodec_alloc_context3(nullptr)};
1748 if(!avctx) return -1;
1750 if(avcodec_parameters_to_context(avctx.get(), mFormatCtx->streams[stream_index]->codecpar))
1753 const AVCodec *codec{avcodec_find_decoder(avctx->codec_id)};
1754 if(!codec || avcodec_open2(avctx.get(), codec, nullptr) < 0)
1756 std::cerr<< "Unsupported codec: "<<avcodec_get_name(avctx->codec_id)
1757 << " (0x"<<std::hex<<avctx->codec_id<<std::dec<<")" <<std::endl;
1761 /* Initialize and start the media type handler */
1762 switch(avctx->codec_type)
1764 case AVMEDIA_TYPE_AUDIO:
1765 mAudio.mStream = mFormatCtx->streams[stream_index];
1766 mAudio.mCodecCtx = std::move(avctx);
1769 case AVMEDIA_TYPE_VIDEO:
1770 mVideo.mStream = mFormatCtx->streams[stream_index];
1771 mVideo.mCodecCtx = std::move(avctx);
1778 return static_cast<int>(stream_index);
1781 int MovieState::parse_handler()
1783 auto &audio_queue = mAudio.mQueue;
1784 auto &video_queue = mVideo.mQueue;
1786 int video_index{-1};
1787 int audio_index{-1};
1789 /* Find the first video and audio streams */
1790 for(unsigned int i{0u};i < mFormatCtx->nb_streams;i++)
1792 auto codecpar = mFormatCtx->streams[i]->codecpar;
1793 if(codecpar->codec_type == AVMEDIA_TYPE_VIDEO && !DisableVideo && video_index < 0)
1794 video_index = streamComponentOpen(i);
1795 else if(codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_index < 0)
1796 audio_index = streamComponentOpen(i);
1800 std::unique_lock<std::mutex> slock{mStartupMutex};
1801 mStartupDone = true;
1803 mStartupCond.notify_all();
1805 if(video_index < 0 && audio_index < 0)
1807 std::cerr<< mFilename<<": could not open codecs" <<std::endl;
1811 /* Set the base time 750ms ahead of the current av time. */
1812 mClockBase = get_avtime() + milliseconds{750};
1814 if(audio_index >= 0)
1815 mAudioThread = std::thread{std::mem_fn(&AudioState::handler), &mAudio};
1816 if(video_index >= 0)
1817 mVideoThread = std::thread{std::mem_fn(&VideoState::handler), &mVideo};
1819 /* Main packet reading/dispatching loop */
1820 AVPacketPtr packet{av_packet_alloc()};
1821 while(!mQuit.load(std::memory_order_relaxed))
1823 if(av_read_frame(mFormatCtx.get(), packet.get()) < 0)
1826 /* Copy the packet into the queue it's meant for. */
1827 if(packet->stream_index == video_index)
1829 while(!mQuit.load(std::memory_order_acquire) && !video_queue.put(packet.get()))
1830 std::this_thread::sleep_for(milliseconds{100});
1832 else if(packet->stream_index == audio_index)
1834 while(!mQuit.load(std::memory_order_acquire) && !audio_queue.put(packet.get()))
1835 std::this_thread::sleep_for(milliseconds{100});
1838 av_packet_unref(packet.get());
1840 /* Finish the queues so the receivers know nothing more is coming. */
1841 video_queue.setFinished();
1842 audio_queue.setFinished();
1844 /* all done - wait for it */
1845 if(mVideoThread.joinable())
1846 mVideoThread.join();
1847 if(mAudioThread.joinable())
1848 mAudioThread.join();
1851 std::unique_lock<std::mutex> lock{mVideo.mPictQMutex};
1852 while(!mVideo.mFinalUpdate)
1853 mVideo.mPictQCond.wait(lock);
1857 evt.user.type = FF_MOVIE_DONE_EVENT;
1858 SDL_PushEvent(&evt);
1863 void MovieState::stop()
1866 mAudio.mQueue.flush();
1867 mVideo.mQueue.flush();
1871 // Helper class+method to print the time with human-readable formatting.
1875 std::ostream &operator<<(std::ostream &os, const PrettyTime &rhs)
1877 using hours = std::chrono::hours;
1878 using minutes = std::chrono::minutes;
1880 seconds t{rhs.mTime};
1887 // Only handle up to hour formatting
1889 os << duration_cast<hours>(t).count() << 'h' << std::setfill('0') << std::setw(2)
1890 << (duration_cast<minutes>(t).count() % 60) << 'm';
1892 os << duration_cast<minutes>(t).count() << 'm' << std::setfill('0');
1893 os << std::setw(2) << (duration_cast<seconds>(t).count() % 60) << 's' << std::setw(0)
1894 << std::setfill(' ');
1901 int main(int argc, char *argv[])
1905 std::unique_ptr<MovieState> movState;
1909 std::cerr<< "Usage: "<<argv[0]<<" [-device <device name>] [-direct] <files...>" <<std::endl;
1912 /* Register all formats and codecs */
1913 #if !(LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 9, 100))
1916 /* Initialize networking protocols */
1917 avformat_network_init();
1919 if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS))
1921 std::cerr<< "Could not initialize SDL - <<"<<SDL_GetError() <<std::endl;
1925 /* Make a window to put our video */
1926 SDL_Window *screen{SDL_CreateWindow(AppName.c_str(), 0, 0, 640, 480, SDL_WINDOW_RESIZABLE)};
1929 std::cerr<< "SDL: could not set video mode - exiting" <<std::endl;
1932 /* Make a renderer to handle the texture image surface and rendering. */
1933 Uint32 render_flags{SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC};
1934 SDL_Renderer *renderer{SDL_CreateRenderer(screen, -1, render_flags)};
1937 SDL_RendererInfo rinf{};
1940 /* Make sure the renderer supports IYUV textures. If not, fallback to a
1941 * software renderer. */
1942 if(SDL_GetRendererInfo(renderer, &rinf) == 0)
1944 for(Uint32 i{0u};!ok && i < rinf.num_texture_formats;i++)
1945 ok = (rinf.texture_formats[i] == SDL_PIXELFORMAT_IYUV);
1949 std::cerr<< "IYUV pixelformat textures not supported on renderer "<<rinf.name <<std::endl;
1950 SDL_DestroyRenderer(renderer);
1956 render_flags = SDL_RENDERER_SOFTWARE | SDL_RENDERER_PRESENTVSYNC;
1957 renderer = SDL_CreateRenderer(screen, -1, render_flags);
1961 std::cerr<< "SDL: could not create renderer - exiting" <<std::endl;
1964 SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
1965 SDL_RenderFillRect(renderer, nullptr);
1966 SDL_RenderPresent(renderer);
1968 /* Open an audio device */
1970 if(InitAL(&argv, &argc))
1972 std::cerr<< "Failed to set up audio device" <<std::endl;
1977 auto device = alcGetContextsDevice(alcGetCurrentContext());
1978 if(alcIsExtensionPresent(device, "ALC_SOFT_device_clock"))
1980 std::cout<< "Found ALC_SOFT_device_clock" <<std::endl;
1981 alcGetInteger64vSOFT = reinterpret_cast<LPALCGETINTEGER64VSOFT>(
1982 alcGetProcAddress(device, "alcGetInteger64vSOFT")
1987 if(alIsExtensionPresent("AL_SOFT_source_latency"))
1989 std::cout<< "Found AL_SOFT_source_latency" <<std::endl;
1990 alGetSourcei64vSOFT = reinterpret_cast<LPALGETSOURCEI64VSOFT>(
1991 alGetProcAddress("alGetSourcei64vSOFT")
1994 if(alIsExtensionPresent("AL_SOFT_events"))
1996 std::cout<< "Found AL_SOFT_events" <<std::endl;
1997 alEventControlSOFT = reinterpret_cast<LPALEVENTCONTROLSOFT>(
1998 alGetProcAddress("alEventControlSOFT"));
1999 alEventCallbackSOFT = reinterpret_cast<LPALEVENTCALLBACKSOFT>(
2000 alGetProcAddress("alEventCallbackSOFT"));
2002 if(alIsExtensionPresent("AL_SOFT_callback_buffer"))
2004 std::cout<< "Found AL_SOFT_callback_buffer" <<std::endl;
2005 alBufferCallbackSOFT = reinterpret_cast<LPALBUFFERCALLBACKSOFT>(
2006 alGetProcAddress("alBufferCallbackSOFT"));
2010 for(;fileidx < argc;++fileidx)
2012 if(strcmp(argv[fileidx], "-direct") == 0)
2014 if(alIsExtensionPresent("AL_SOFT_direct_channels_remix"))
2016 std::cout<< "Found AL_SOFT_direct_channels_remix" <<std::endl;
2017 DirectOutMode = AL_REMIX_UNMATCHED_SOFT;
2019 else if(alIsExtensionPresent("AL_SOFT_direct_channels"))
2021 std::cout<< "Found AL_SOFT_direct_channels" <<std::endl;
2022 DirectOutMode = AL_DROP_UNMATCHED_SOFT;
2025 std::cerr<< "AL_SOFT_direct_channels not supported for direct output" <<std::endl;
2027 else if(strcmp(argv[fileidx], "-wide") == 0)
2029 if(!alIsExtensionPresent("AL_EXT_STEREO_ANGLES"))
2030 std::cerr<< "AL_EXT_STEREO_ANGLES not supported for wide stereo" <<std::endl;
2033 std::cout<< "Found AL_EXT_STEREO_ANGLES" <<std::endl;
2034 EnableWideStereo = true;
2037 else if(strcmp(argv[fileidx], "-uhj") == 0)
2039 if(!alIsExtensionPresent("AL_SOFT_UHJ"))
2040 std::cerr<< "AL_SOFT_UHJ not supported for UHJ decoding" <<std::endl;
2043 std::cout<< "Found AL_SOFT_UHJ" <<std::endl;
2047 else if(strcmp(argv[fileidx], "-superstereo") == 0)
2049 if(!alIsExtensionPresent("AL_SOFT_UHJ"))
2050 std::cerr<< "AL_SOFT_UHJ not supported for Super Stereo decoding" <<std::endl;
2053 std::cout<< "Found AL_SOFT_UHJ (Super Stereo)" <<std::endl;
2054 EnableSuperStereo = true;
2057 else if(strcmp(argv[fileidx], "-novideo") == 0)
2058 DisableVideo = true;
2063 while(fileidx < argc && !movState)
2065 movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
2066 if(!movState->prepare()) movState = nullptr;
2070 std::cerr<< "Could not start a video" <<std::endl;
2073 movState->setTitle(screen);
2075 /* Default to going to the next movie at the end of one. */
2076 enum class EomAction {
2078 } eom_action{EomAction::Next};
2079 seconds last_time{seconds::min()};
2082 /* SDL_WaitEventTimeout is broken, just force a 10ms sleep. */
2083 std::this_thread::sleep_for(milliseconds{10});
2085 auto cur_time = std::chrono::duration_cast<seconds>(movState->getMasterClock());
2086 if(cur_time != last_time)
2088 auto end_time = std::chrono::duration_cast<seconds>(movState->getDuration());
2089 std::cout<< " \r "<<PrettyTime{cur_time}<<" / "<<PrettyTime{end_time} <<std::flush;
2090 last_time = cur_time;
2093 bool force_redraw{false};
2095 while(SDL_PollEvent(&event) != 0)
2100 switch(event.key.keysym.sym)
2104 eom_action = EomAction::Quit;
2109 eom_action = EomAction::Next;
2117 case SDL_WINDOWEVENT:
2118 switch(event.window.event)
2120 case SDL_WINDOWEVENT_RESIZED:
2121 SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
2122 SDL_RenderFillRect(renderer, nullptr);
2123 force_redraw = true;
2126 case SDL_WINDOWEVENT_EXPOSED:
2127 force_redraw = true;
2137 eom_action = EomAction::Quit;
2140 case FF_MOVIE_DONE_EVENT:
2142 last_time = seconds::min();
2143 if(eom_action != EomAction::Quit)
2146 while(fileidx < argc && !movState)
2148 movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
2149 if(!movState->prepare()) movState = nullptr;
2153 movState->setTitle(screen);
2158 /* Nothing more to play. Shut everything down and quit. */
2163 SDL_DestroyRenderer(renderer);
2165 SDL_DestroyWindow(screen);
2176 movState->mVideo.updateVideo(screen, renderer, force_redraw);
2179 std::cerr<< "SDL_WaitEvent error - "<<SDL_GetError() <<std::endl;