Feature: add visualizer with fft data

app/src/main/cpp/CMakeLists.txt

@@ -33,7 +33,6 @@ add_library( # Sets the name of the library.
         video_cutting.cpp
         visualizer/fft.cpp
         visualizer/fixed_fft.cpp
-        visualizer/block_queue.c
         visualizer/frank_visualizer.cpp
         visualizer/frank_visualizer_jni.cpp
         visualizer/window.cpp

app/src/main/cpp/audio_player_jni.cpp

@@ -10,6 +10,13 @@
 
 FFAudioPlayer *audioPlayer;
 
+void fftCallback(JNIEnv *env, jobject thiz, jmethodID fft_method, int8_t *data, int size) {
+    jbyteArray dataArray = env->NewByteArray(size);
+    env->SetByteArrayRegion(dataArray, 0, size, data);
+    env->CallVoidMethod(thiz, fft_method, dataArray);
+    env->DeleteLocalRef(dataArray);
+}
+
 AUDIO_PLAYER_FUNC(void, play, jstring path) {
     if (path == nullptr)
         return;
@@ -32,6 +39,8 @@ AUDIO_PLAYER_FUNC(void, play, jstring path) {
     // method if of write
     jmethodID write_method = env->GetMethodID(audio_track_class, "write", "([BII)I");
 
+    jmethodID fft_method = env->GetMethodID(audio_class, "fftCallbackFromJNI", "([B)V");
+
     // demux decode and play
     while (result >= 0) {
         result = audioPlayer->decodeAudio();
@@ -49,6 +58,10 @@ AUDIO_PLAYER_FUNC(void, play, jstring path) {
         env->CallIntMethod(audio_track, write_method, audio_array, 0, size);
         env->DeleteLocalRef(audio_array);
 
+        if (audioPlayer->enableVisualizer()) {
+            fftCallback(env, thiz, fft_method, audioPlayer->getFFTData(), audioPlayer->getFFTSize());
+        }
+
         // audio sync
         usleep(SLEEP_TIME);
     }

app/src/main/cpp/ff_audio_player.cpp

@@ -132,6 +132,9 @@ int FFAudioPlayer::open(const char *path) {
     filterFrame = av_frame_alloc();
     initFilter(FILTER_DESC, codecContext, &audioFilterGraph,
                    &audioSrcContext, &audioSinkContext);
+    // init visualizer
+    mVisualizer = new FrankVisualizer();
+    mVisualizer->init_visualizer();
 
     return 0;
 }
@@ -172,6 +175,13 @@ int FFAudioPlayer::decodeAudio() {
         }
     }
 
+    // visualizer: do fft
+    int nb_samples = inputFrame->nb_samples < MAX_FFT_SIZE ? inputFrame->nb_samples : MAX_FFT_SIZE;
+    if (m_enableVisualizer && nb_samples >= MIN_FFT_SIZE) {
+        mVisualizer->fft_run(inputFrame->data[0], nb_samples);
+    }
+
+    // change filter
     if (filterAgain) {
         filterAgain = false;
         avfilter_graph_free(&audioFilterGraph);
@@ -216,6 +226,26 @@ uint8_t *FFAudioPlayer::getDecodeFrame() const {
     return out_buffer;
 }
 
+void FFAudioPlayer::setEnableVisualizer(bool enable) {
+    m_enableVisualizer = enable;
+}
+
+bool FFAudioPlayer::enableVisualizer() const {
+    return m_enableVisualizer;
+}
+
+int8_t* FFAudioPlayer::getFFTData() const {
+    if (!mVisualizer)
+        return nullptr;
+    return mVisualizer->getFFTData();
+}
+
+int FFAudioPlayer::getFFTSize() const {
+    if (!mVisualizer)
+        return 0;
+    return mVisualizer->getOutputSample();
+}
+
 void FFAudioPlayer::setFilterAgain(bool again) {
     filterAgain = again;
 }
@@ -250,4 +280,7 @@ void FFAudioPlayer::close() {
         avfilter_graph_free(&audioFilterGraph);
     }
     delete[] out_buffer;
+    if (mVisualizer) {
+        mVisualizer->release_visualizer();
+    }
 }

app/src/main/cpp/ff_audio_player.h

@@ -7,6 +7,7 @@
 
 #include <stdatomic.h>
 #include "ffmpeg_jni_define.h"
+#include "visualizer/frank_visualizer.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -46,6 +47,9 @@ private:
     AVFilterContext *audioSrcContext;
     AVFilterContext *audioSinkContext;
 
+    bool m_enableVisualizer = false;
+    FrankVisualizer *mVisualizer;
+
 public:
     int open(const char* path);
 
@@ -61,6 +65,14 @@ public:
 
     void setFilterDesc(const char *filterDescription);
 
+    void setEnableVisualizer(bool enable);
+
+    bool enableVisualizer() const;
+
+    int8_t* getFFTData() const;
+
+    int getFFTSize() const;
+
     void setExit(bool exit);
 
     void close();

app/src/main/cpp/visualizer/block_queue.c

@@ -1,70 +0,0 @@
-//
-// Created by frank on 2021/08/08.
-//
-#include "block_queue.h"
-#include <stdlib.h>
-
-vlc_queue_t *vlc_queue_init(int size) {
-    vlc_queue_t *queue = (vlc_queue_t *)(malloc(sizeof(vlc_queue_t)));
-    queue->size = size;
-    queue->next_to_read = 0;
-    queue->next_to_write = 0;
-    int i;
-    queue->packets = (void **)(malloc(sizeof(*queue->packets) * size));
-    for (i = 0; i < size; i++) {
-        block_t *packet = malloc(sizeof(block_t));
-        packet->i_nb_samples = 0;
-        queue->packets[i] = packet;
-    }
-    pthread_mutex_init(&queue->mutex, NULL);
-    pthread_cond_init(&queue->cond, NULL);
-    return queue;
-}
-
-void vlc_queue_free(vlc_queue_t *queue) {
-    int i;
-    for (i = 0; i < queue->size; i++) {
-        free(queue->packets[i]);
-    }
-    free(queue->packets);
-    free(queue);
-    pthread_cond_destroy(&queue->cond);
-    pthread_mutex_destroy(&queue->mutex);
-}
-
-int vlc_queue_next(vlc_queue_t *queue, int current) {
-    return (current + 1) % queue->size;
-}
-
-void *vlc_queue_push(vlc_queue_t *queue, void *data) {
-    pthread_mutex_lock(&queue->mutex);//lock
-    int current = queue->next_to_write;
-    int next_to_write;
-    for (;;) {
-        next_to_write = vlc_queue_next(queue, current);
-        if (next_to_write != queue->next_to_read) {
-            break;
-        }
-        pthread_cond_wait(&queue->cond, &queue->mutex);
-    }
-    queue->next_to_write = next_to_write;
-    queue->packets[current] = data;//TODO
-    pthread_cond_broadcast(&queue->cond);
-    pthread_mutex_unlock(&queue->mutex);//unlock
-    return queue->packets[current];
-}
-
-void *vlc_queue_pop(vlc_queue_t *queue) {
-    pthread_mutex_lock(&queue->mutex);
-    int current = queue->next_to_read;
-    for (;;) {
-        if (queue->next_to_write != queue->next_to_read) {
-            break;
-        }
-        pthread_cond_wait(&queue->cond, &queue->mutex);
-    }
-    queue->next_to_read = vlc_queue_next(queue, current);
-    pthread_cond_broadcast(&queue->cond);
-    pthread_mutex_unlock(&queue->mutex);
-    return queue->packets[current];
-}

app/src/main/cpp/visualizer/block_queue.h

@@ -1,57 +0,0 @@
-//
-// Created by frank on 2021/08/08.
-//
-
-#ifndef BLOCK_QUEUE_H
-#define BLOCK_QUEUE_H
-
-#include <pthread.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef int64_t vlc_tick_t;
-
-typedef struct FFT_callback {
-    void (*callback)(void*);
-} FFT_callback;
-
-typedef struct block_t {
-    uint8_t    *p_buffer; /** Payload start */
-
-    unsigned    i_nb_samples; /** Used for audio */
-
-    vlc_tick_t  i_pts;
-    vlc_tick_t  i_length;
-
-    FFT_callback fft_callback;
-} block_t;
-
-typedef struct vlc_queue {
-    //the size of queue
-    int size;
-    //packet array
-    void **packets;
-    //the packet next to write
-    int next_to_write;
-    //the packet next to read
-    int next_to_read;
-
-    pthread_mutex_t mutex;
-    pthread_cond_t cond;
-} vlc_queue_t;
-
-vlc_queue_t *vlc_queue_init(int size);
-
-void vlc_queue_free(vlc_queue_t *queue);
-
-void *vlc_queue_push(vlc_queue_t *queue, void *data);
-
-void *vlc_queue_pop(vlc_queue_t *queue);
-
-#endif //BLOCK_QUEUE_H
-
-#ifdef __cplusplus
-}
-#endif

app/src/main/cpp/visualizer/frank_visualizer.cpp

@@ -160,6 +160,8 @@ FrankVisualizer::~FrankVisualizer() {
 
 int8_t* FrankVisualizer::fft_run(uint8_t *input_buffer, int nb_samples) {
     mFftLock.lock();
+    if (!fft_context)
+        return nullptr;
     fft_context->nb_samples = nb_samples;
     memcpy(fft_context->data, input_buffer, static_cast<size_t>(nb_samples));
     filter_sys_t *p_sys = fft_context;
@@ -179,11 +181,13 @@ int FrankVisualizer::getOutputSample() {
     return 0;
 }
 
+int8_t *FrankVisualizer::getFFTData() {
+    return fft_context ? fft_context->output : nullptr;
+}
+
 int FrankVisualizer::init_visualizer() {
     fft_context = new filter_sys_t();
     filter_sys_t *p_filter = fft_context;
-    if (!p_filter)
-        return -1;
 
     p_filter->convert_to_float = false;
     p_filter->i_channels = 1;

app/src/main/cpp/visualizer/frank_visualizer.h

@@ -47,6 +47,8 @@ public:
 
     int getOutputSample();
 
+    int8_t* getFFTData();
+
     int8_t* fft_run(uint8_t *input_buffer, int nb_samples);
 
     int init_visualizer();