codal_port/modaudio: Rework audio playing to play a long AudioFrame.

dpgeorge · dpgeorge · commit d55f725b39b6 · 2023-11-14T12:30:12.000+11:00
- Audio playback now supports AudioFrames with arbitrary size.
- Allows passing a single AudioFrame to audio.play.

Signed-off-by: Damien George &lt;damien@micropython.org&gt;
diff --git a/src/codal_app/microbithal.h b/src/codal_app/microbithal.h
@@ -173,9 +173,10 @@ bool microbit_hal_audio_is_expression_active(void);
 void microbit_hal_audio_play_expression(const char *expr);
 void microbit_hal_audio_stop_expression(void);
 
-void microbit_hal_audio_init(uint32_t sample_rate);
-void microbit_hal_audio_write_data(const uint8_t *buf, size_t num_samples);
-void microbit_hal_audio_ready_callback(void);
+void microbit_hal_audio_raw_init(uint32_t sample_rate);
+void microbit_hal_audio_raw_set_rate(uint32_t sample_rate);
+void microbit_hal_audio_raw_write_data(const uint8_t *buf, size_t num_samples);
+void microbit_hal_audio_raw_ready_callback(void);
 
 void microbit_hal_audio_speech_init(uint32_t sample_rate);
 void microbit_hal_audio_speech_write_data(const uint8_t *buf, size_t num_samples);
diff --git a/src/codal_app/microbithal_audio.cpp b/src/codal_app/microbithal_audio.cpp
@@ -33,6 +33,7 @@ class AudioSource : public DataSource {
     DataSink *sink;
     ManagedBuffer buf;
     void (*callback)(void);
+    MixerChannel *channel;
 
     AudioSource()
         : started(false) {
@@ -52,7 +53,7 @@ class AudioSource : public DataSource {
     }
 };
 
-static AudioSource data_source;
+static AudioSource raw_source;
 static AudioSource speech_source;
 
 extern "C" {
@@ -104,29 +105,35 @@ void microbit_hal_audio_stop_expression(void) {
     uBit.audio.soundExpressions.stop();
 }
 
-void microbit_hal_audio_init(uint32_t sample_rate) {
-    if (!data_source.started) {
+void microbit_hal_audio_raw_init(uint32_t sample_rate) {
+    if (!raw_source.started) {
         MicroBitAudio::requestActivation();
-        data_source.started = true;
-        data_source.callback = microbit_hal_audio_ready_callback;
-        uBit.audio.mixer.addChannel(data_source, sample_rate, 255);
+        raw_source.started = true;
+        raw_source.callback = microbit_hal_audio_raw_ready_callback;
+        raw_source.channel = uBit.audio.mixer.addChannel(raw_source, sample_rate, 255);
+    } else {
+        raw_source.channel->setSampleRate(sample_rate);
     }
 }
 
-void microbit_hal_audio_write_data(const uint8_t *buf, size_t num_samples) {
-    if ((size_t)data_source.buf.length() != num_samples) {
-        data_source.buf = ManagedBuffer(num_samples);
+void microbit_hal_audio_raw_set_rate(uint32_t sample_rate) {
+    raw_source.channel->setSampleRate(sample_rate);
+}
+
+void microbit_hal_audio_raw_write_data(const uint8_t *buf, size_t num_samples) {
+    if ((size_t)raw_source.buf.length() != num_samples) {
+        raw_source.buf = ManagedBuffer(num_samples);
     }
-    memcpy(data_source.buf.getBytes(), buf, num_samples);
-    data_source.sink->pullRequest();
+    memcpy(raw_source.buf.getBytes(), buf, num_samples);
+    raw_source.sink->pullRequest();
 }
 
 void microbit_hal_audio_speech_init(uint32_t sample_rate) {
     if (!speech_source.started) {
         MicroBitAudio::requestActivation();
         speech_source.started = true;
         speech_source.callback = microbit_hal_audio_speech_ready_callback;
-        uBit.audio.mixer.addChannel(speech_source, sample_rate, 255);
+        speech_source.channel = uBit.audio.mixer.addChannel(speech_source, sample_rate, 255);
     }
 }
 
diff --git a/src/codal_port/modaudio.c b/src/codal_port/modaudio.c
@@ -30,7 +30,9 @@
 #include "modaudio.h"
 #include "modmicrobit.h"
 
-#define audio_source_iter MP_STATE_PORT(audio_source)
+// Convenience macros to access the root-pointer state.
+#define audio_source_frame MP_STATE_PORT(audio_source_frame_state)
+#define audio_source_iter MP_STATE_PORT(audio_source_iter_state)
 
 #define LOG_AUDIO_CHUNK_SIZE (5)
 #define AUDIO_CHUNK_SIZE (1 << LOG_AUDIO_CHUNK_SIZE)
@@ -47,15 +49,18 @@ typedef enum {
 static uint8_t audio_output_buffer[OUT_CHUNK_SIZE];
 static volatile audio_output_state_t audio_output_state;
 static volatile bool audio_fetcher_scheduled;
+static size_t audio_raw_offset;
 
 microbit_audio_frame_obj_t *microbit_audio_frame_make_new(size_t size);
 
 static inline bool audio_is_running(void) {
-    return audio_source_iter != NULL;
+    return audio_source_frame != NULL || audio_source_iter != MP_OBJ_NULL;
 }
 
 void microbit_audio_stop(void) {
+    audio_source_frame = NULL;
     audio_source_iter = NULL;
+    audio_raw_offset = 0;
 }
 
 STATIC void audio_buffer_ready(void) {
@@ -64,50 +69,77 @@ STATIC void audio_buffer_ready(void) {
     audio_output_state = AUDIO_OUTPUT_STATE_DATA_READY;
     MICROPY_END_ATOMIC_SECTION(atomic_state);
     if (old_state == AUDIO_OUTPUT_STATE_IDLE) {
-        microbit_hal_audio_ready_callback();
+        microbit_hal_audio_raw_ready_callback();
     }
 }
 
 STATIC void audio_data_fetcher(void) {
     audio_fetcher_scheduled = false;
-    if (audio_source_iter == NULL) {
-        return;
-    }
-    mp_obj_t buffer_obj;
-    nlr_buf_t nlr;
-    if (nlr_push(&nlr) == 0) {
-        buffer_obj = mp_iternext_allow_raise(audio_source_iter);
-        nlr_pop();
-    } else {
-        if (!mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(((mp_obj_base_t*)nlr.ret_val)->type),
-            MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
-            mp_sched_exception(MP_OBJ_FROM_PTR(nlr.ret_val));
+
+    if (audio_source_frame != NULL) {
+        // An existing AudioFrame is being played, see if there's any data left.
+        if (audio_raw_offset >= audio_source_frame->size) {
+            // AudioFrame is exhausted.
+            audio_source_frame = NULL;
         }
-        buffer_obj = MP_OBJ_STOP_ITERATION;
     }
-    if (buffer_obj == MP_OBJ_STOP_ITERATION) {
-        // End of audio iterator
-        microbit_audio_stop();
-    } else if (mp_obj_get_type(buffer_obj) != &microbit_audio_frame_type) {
-        // Audio iterator did not return an AudioFrame
-        microbit_audio_stop();
-        mp_sched_exception(mp_obj_new_exception_msg(&mp_type_TypeError, MP_ERROR_TEXT("not an AudioFrame")));
-    } else {
-        microbit_audio_frame_obj_t *buffer = (microbit_audio_frame_obj_t *)buffer_obj;
-        uint8_t *dest = &audio_output_buffer[0];
-        uint32_t last = dest[BUFFER_EXPANSION * AUDIO_CHUNK_SIZE - 1];
-        for (int i = 0; i < AUDIO_CHUNK_SIZE; ++i) {
-            uint32_t cur = buffer->data[i];
-            for (int j = 0; j < BUFFER_EXPANSION; ++j) {
-                // Get next sample with linear interpolation.
-                uint32_t sample = ((BUFFER_EXPANSION - 1 - j) * last + (j + 1) * cur) / BUFFER_EXPANSION;
-                // Write sample to the buffer.
-                *dest++ = sample;
+
+    if (audio_source_frame == NULL) {
+        // There is no AudioFrame, so try to get one from the audio iterator.
+
+        if (audio_source_iter == MP_OBJ_NULL) {
+            // Audio iterator is already exhausted.
+            microbit_audio_stop();
+            return;
+        }
+
+        // Get the next item from the audio iterator.
+        nlr_buf_t nlr;
+        mp_obj_t frame_obj;
+        if (nlr_push(&nlr) == 0) {
+            frame_obj = mp_iternext_allow_raise(audio_source_iter);
+            nlr_pop();
+        } else {
+            if (!mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(((mp_obj_base_t*)nlr.ret_val)->type),
+                MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
+                mp_sched_exception(MP_OBJ_FROM_PTR(nlr.ret_val));
             }
-            last = cur;
+            frame_obj = MP_OBJ_STOP_ITERATION;
+        }
+        if (frame_obj == MP_OBJ_STOP_ITERATION) {
+            // End of audio iterator.
+            microbit_audio_stop();
+            return;
+        }
+        if (!mp_obj_is_type(frame_obj, &microbit_audio_frame_type)) {
+            // Audio iterator did not return an AudioFrame.
+            microbit_audio_stop();
+            mp_sched_exception(mp_obj_new_exception_msg(&mp_type_TypeError, MP_ERROR_TEXT("not an AudioFrame")));
+            return;
         }
-        audio_buffer_ready();
+
+        // We have the next AudioFrame.
+        audio_source_frame = MP_OBJ_TO_PTR(frame_obj);
+        audio_raw_offset = 0;
     }
+
+    const uint8_t *src = &audio_source_frame->data[audio_raw_offset];
+    audio_raw_offset += AUDIO_CHUNK_SIZE;
+
+    uint8_t *dest = &audio_output_buffer[0];
+    uint32_t last = dest[OUT_CHUNK_SIZE - 1];
+    for (int i = 0; i < AUDIO_CHUNK_SIZE; ++i) {
+        uint32_t cur = src[i];
+        for (int j = 0; j < BUFFER_EXPANSION; ++j) {
+            // Get next sample with linear interpolation.
+            uint32_t sample = ((BUFFER_EXPANSION - 1 - j) * last + (j + 1) * cur) / BUFFER_EXPANSION;
+            // Write sample to the buffer.
+            *dest++ = sample;
+        }
+        last = cur;
+    }
+
+    audio_buffer_ready();
 }
 
 STATIC mp_obj_t audio_data_fetcher_wrapper(mp_obj_t arg) {
@@ -116,10 +148,10 @@ STATIC mp_obj_t audio_data_fetcher_wrapper(mp_obj_t arg) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(audio_data_fetcher_wrapper_obj, audio_data_fetcher_wrapper);
 
-void microbit_hal_audio_ready_callback(void) {
+void microbit_hal_audio_raw_ready_callback(void) {
     if (audio_output_state == AUDIO_OUTPUT_STATE_DATA_READY) {
         // there is data ready to send out to the audio pipeline, so send it
-        microbit_hal_audio_write_data(&audio_output_buffer[0], OUT_CHUNK_SIZE);
+        microbit_hal_audio_raw_write_data(&audio_output_buffer[0], OUT_CHUNK_SIZE);
         audio_output_state = AUDIO_OUTPUT_STATE_DATA_WRITTEN;
     } else {
         // no data ready, need to call this function later when data is ready
@@ -134,7 +166,7 @@ void microbit_hal_audio_ready_callback(void) {
 static void audio_init(uint32_t sample_rate) {
     audio_fetcher_scheduled = false;
     audio_output_state = AUDIO_OUTPUT_STATE_IDLE;
-    microbit_hal_audio_init(BUFFER_EXPANSION * sample_rate);
+    microbit_hal_audio_raw_init(BUFFER_EXPANSION * sample_rate);
 }
 
 void microbit_audio_play_source(mp_obj_t src, mp_obj_t pin_select, bool wait, uint32_t sample_rate) {
@@ -150,6 +182,9 @@ void microbit_audio_play_source(mp_obj_t src, mp_obj_t pin_select, bool wait, ui
         sound_expr_data = sound->name;
     } else if (mp_obj_is_type(src, &microbit_soundeffect_type)) {
         sound_expr_data = microbit_soundeffect_get_sound_expr_data(src);
+    } else if (mp_obj_is_type(src, &microbit_audio_frame_type)) {
+        audio_source_frame = MP_OBJ_TO_PTR(src);
+        audio_raw_offset = 0;
     } else if (mp_obj_is_type(src, &mp_type_tuple) || mp_obj_is_type(src, &mp_type_list)) {
         // A tuple/list passed in.  Need to check if it contains SoundEffect instances.
         size_t len;
@@ -167,7 +202,13 @@ void microbit_audio_play_source(mp_obj_t src, mp_obj_t pin_select, bool wait, ui
             }
             // Replace last "," with a string null terminator.
             data[-1] = '\0';
+        } else {
+            // A tuple/list of AudioFrame instances.
+            audio_source_iter = mp_getiter(src, NULL);
         }
+    } else {
+        // An iterator of AudioFrame instances.
+        audio_source_iter = mp_getiter(src, NULL);
     }
 
     if (sound_expr_data != NULL) {
@@ -190,9 +231,8 @@ void microbit_audio_play_source(mp_obj_t src, mp_obj_t pin_select, bool wait, ui
         return;
     }
 
-    // Get the iterator and start the audio running.
+    // Start the audio running.
     // The scheduler must be locked because audio_data_fetcher() can also be called from the scheduler.
-    audio_source_iter = mp_getiter(src, NULL);
     mp_sched_lock();
     audio_data_fetcher();
     mp_sched_unlock();
@@ -440,4 +480,5 @@ microbit_audio_frame_obj_t *microbit_audio_frame_make_new(size_t size) {
     return res;
 }
 
-MP_REGISTER_ROOT_POINTER(void *audio_source);
+MP_REGISTER_ROOT_POINTER(struct _microbit_audio_frame_obj_t *audio_source_frame_state);
+MP_REGISTER_ROOT_POINTER(mp_obj_t audio_source_iter_state);
