diff --git a/include/SDL3/SDL_audio.h b/include/SDL3/SDL_audio.h index dd7b238293..d2692aba2a 100644 --- a/include/SDL3/SDL_audio.h +++ b/include/SDL3/SDL_audio.h @@ -1126,6 +1126,73 @@ extern DECLSPEC void SDLCALL SDL_DestroyAudioStream(SDL_AudioStream *stream); */ extern DECLSPEC SDL_AudioStream *SDLCALL SDL_OpenAudioDeviceStream(SDL_AudioDeviceID devid, const SDL_AudioSpec *spec, SDL_AudioStreamCallback callback, void *userdata); + +/** + * A callback that fires when data is about to be fed to an audio device. + * + * This is useful for accessing the final mix, perhaps for writing a + * visualizer or applying a final effect to the audio data before playback. + * + * \sa SDL_SetAudioDevicePostmixCallback + */ +typedef void (SDLCALL *SDL_AudioPostmixCallback)(void *userdata, const SDL_AudioSpec *spec, float *buffer, int buflen); + +/** + * Set a callback that fires when data is about to be fed to an audio device. + * + * This is useful for accessing the final mix, perhaps for writing a + * visualizer or applying a final effect to the audio data before playback. + * + * The buffer is the final mix of all bound audio streams on an opened + * device; this callback will fire regularly for any device that is both + * opened and unpaused. If there is no new data to mix, either because no + * streams are bound to the device or all the streams are empty, this + * callback will still fire with the entire buffer set to silence. + * + * This callback is allowed to make changes to the data; the contents of + * the buffer after this call is what is ultimately passed along to the + * hardware. + * + * The callback is always provided the data in float format (values from + * -1.0f to 1.0f), but the number of channels or sample rate may be + * different than the format the app requested when opening the device; SDL + * might have had to manage a conversion behind the scenes, or the playback + * might have jumped to new physical hardware when a system default changed, + * etc. These details may change between calls. Accordingly, the size of the + * buffer might change between calls as well. + * + * This callback can run at any time, and from any thread; if you need to + * serialize access to your app's data, you should provide and use a mutex or + * other synchronization device. + * + * All of this to say: there are specific needs this callback can fulfill, + * but it is not the simplest interface. Apps should generally provide audio + * in their preferred format through an SDL_AudioStream and let SDL handle + * the difference. + * + * This function is extremely time-sensitive; the callback should do the + * least amount of work possible and return as quickly as it can. The longer + * the callback runs, the higher the risk of audio dropouts or other problems. + * + * This function will block until the audio device is in between iterations, + * so any existing callback that might be running will finish before this + * function sets the new callback and returns. + * + * Setting a NULL callback function disables any previously-set callback. + * + * \param devid The ID of an opened audio device. + * \param callback A callback function to be called. Can be NULL. + * \param userdata App-controlled pointer passed to callback. Can be NULL. + * \returns zero on success, -1 on error; call SDL_GetError() for more + * information. + * + * \threadsafety It is safe to call this function from any thread. + * + * \since This function is available since SDL 3.0.0. + */ +extern DECLSPEC int SDLCALL SDL_SetAudioPostmixCallback(SDL_AudioDeviceID devid, SDL_AudioPostmixCallback callback, void *userdata); + + /** * Load the audio data of a WAVE file into memory. * diff --git a/src/audio/SDL_audio.c b/src/audio/SDL_audio.c index 11dd3e3126..102c66a8ac 100644 --- a/src/audio/SDL_audio.c +++ b/src/audio/SDL_audio.c @@ -715,6 +715,7 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device) // can we do a basic copy without silencing/mixing the buffer? This is an extremely likely scenario, so we special-case it. const SDL_bool simple_copy = device->logical_devices && // there's a logical device !device->logical_devices->next && // there's only _ONE_ logical device + !device->logical_devices->postmix && // there isn't a postmix callback !SDL_AtomicGet(&device->logical_devices->paused) && // it isn't paused device->logical_devices->bound_streams && // there's a bound stream !device->logical_devices->bound_streams->next_binding; // there's only _ONE_ bound stream. @@ -731,7 +732,7 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device) SDL_memset(device_buffer + br, device->silence_value, buffer_size - br); // silence whatever we didn't write to. } } else { // need to actually mix (or silence the buffer) - float *mix_buffer = (float *) ((device->spec.format == SDL_AUDIO_F32) ? device_buffer : device->mix_buffer); + float *final_mix_buffer = (float *) ((device->spec.format == SDL_AUDIO_F32) ? device_buffer : device->mix_buffer); const int needed_samples = buffer_size / SDL_AUDIO_BYTESIZE(device->spec.format); const int work_buffer_size = needed_samples * sizeof (float); SDL_AudioSpec outspec; @@ -742,13 +743,20 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device) outspec.channels = device->spec.channels; outspec.freq = device->spec.freq; - SDL_memset(mix_buffer, '\0', buffer_size); // start with silence. + SDL_memset(final_mix_buffer, '\0', work_buffer_size); // start with silence. for (SDL_LogicalAudioDevice *logdev = device->logical_devices; logdev != NULL; logdev = logdev->next) { if (SDL_AtomicGet(&logdev->paused)) { continue; // paused? Skip this logical device. } + const SDL_AudioPostmixCallback postmix = logdev->postmix; + float *mix_buffer = final_mix_buffer; + if (postmix) { + mix_buffer = device->postmix_buffer; + SDL_memset(mix_buffer, '\0', work_buffer_size); // start with silence. + } + for (SDL_AudioStream *stream = logdev->bound_streams; stream != NULL; stream = stream->next_binding) { SDL_SetAudioStreamFormat(stream, NULL, &outspec); @@ -764,12 +772,18 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device) MixFloat32Audio(mix_buffer, (float *) device->work_buffer, br); } } + + if (postmix) { + SDL_assert(mix_buffer == device->postmix_buffer); + postmix(logdev->postmix_userdata, &outspec, mix_buffer, work_buffer_size); + MixFloat32Audio(final_mix_buffer, mix_buffer, work_buffer_size); + } } - if (((Uint8 *) mix_buffer) != device_buffer) { + if (((Uint8 *) final_mix_buffer) != device_buffer) { // !!! FIXME: we can't promise the device buf is aligned/padded for SIMD. - //ConvertAudio(needed_samples * device->spec.channels, mix_buffer, SDL_AUDIO_F32, device->spec.channels, device_buffer, device->spec.format, device->spec.channels, device->work_buffer); - ConvertAudio(needed_samples / device->spec.channels, mix_buffer, SDL_AUDIO_F32, device->spec.channels, device->work_buffer, device->spec.format, device->spec.channels, NULL); + //ConvertAudio(needed_samples * device->spec.channels, final_mix_buffer, SDL_AUDIO_F32, device->spec.channels, device_buffer, device->spec.format, device->spec.channels, device->work_buffer); + ConvertAudio(needed_samples / device->spec.channels, final_mix_buffer, SDL_AUDIO_F32, device->spec.channels, device->work_buffer, device->spec.format, device->spec.channels, NULL); SDL_memcpy(device_buffer, device->work_buffer, buffer_size); } } @@ -837,21 +851,37 @@ SDL_bool SDL_CaptureAudioThreadIterate(SDL_AudioDevice *device) current_audio.impl.FlushCapture(device); // nothing wants data, dump anything pending. } else { // this SHOULD NOT BLOCK, as we are holding a lock right now. Block in WaitCaptureDevice! - const int rc = current_audio.impl.CaptureFromDevice(device, device->work_buffer, device->buffer_size); - if (rc < 0) { // uhoh, device failed for some reason! + int br = current_audio.impl.CaptureFromDevice(device, device->work_buffer, device->buffer_size); + if (br < 0) { // uhoh, device failed for some reason! retval = SDL_FALSE; - } else if (rc > 0) { // queue the new data to each bound stream. + } else if (br > 0) { // queue the new data to each bound stream. for (SDL_LogicalAudioDevice *logdev = device->logical_devices; logdev != NULL; logdev = logdev->next) { if (SDL_AtomicGet(&logdev->paused)) { continue; // paused? Skip this logical device. } + void *output_buffer = device->work_buffer; + SDL_AudioSpec outspec; + SDL_copyp(&outspec, &device->spec); + + // I don't know why someone would want a postmix on a capture device, but we offer it for API consistency. + if (logdev->postmix) { + // move to float format. + output_buffer = device->postmix_buffer; + outspec.format = SDL_AUDIO_F32; + const int frames = br / SDL_AUDIO_FRAMESIZE(device->spec); + br = frames * SDL_AUDIO_FRAMESIZE(outspec); + ConvertAudio(frames, device->work_buffer, device->spec.format, outspec.channels, device->postmix_buffer, SDL_AUDIO_F32, outspec.channels, NULL); + logdev->postmix(logdev->postmix_userdata, &outspec, device->postmix_buffer, br); + } + for (SDL_AudioStream *stream = logdev->bound_streams; stream != NULL; stream = stream->next_binding) { /* this will hold a lock on `stream` while putting. We don't explicitly lock the streams for iterating here because the binding linked list can only change while the device lock is held. (we _do_ lock the stream during binding/unbinding to make sure that two threads can't try to bind the same stream to different devices at the same time, though.) */ - if (SDL_PutAudioStreamData(stream, device->work_buffer, rc) < 0) { + SDL_SetAudioStreamFormat(stream, &outspec, NULL); + if (SDL_PutAudioStreamData(stream, output_buffer, br) < 0) { // oh crud, we probably ran out of memory. This is possibly an overreaction to kill the audio device, but it's likely the whole thing is going down in a moment anyhow. retval = SDL_FALSE; break; @@ -1138,6 +1168,9 @@ static void ClosePhysicalAudioDevice(SDL_AudioDevice *device) SDL_aligned_free(device->mix_buffer); device->mix_buffer = NULL; + SDL_aligned_free(device->postmix_buffer); + device->postmix_buffer = NULL; + SDL_memcpy(&device->spec, &device->default_spec, sizeof (SDL_AudioSpec)); device->sample_frames = 0; device->silence_value = SDL_GetSilenceValueForFormat(device->spec.format); @@ -1395,6 +1428,28 @@ SDL_bool SDL_IsAudioDevicePaused(SDL_AudioDeviceID devid) return retval; } +int SDL_SetAudioPostmixCallback(SDL_AudioDeviceID devid, SDL_AudioPostmixCallback callback, void *userdata) +{ + SDL_LogicalAudioDevice *logdev = ObtainLogicalAudioDevice(devid); + int retval = 0; + if (logdev) { + SDL_AudioDevice *device = logdev->physical_device; + if (!device->postmix_buffer) { + device->postmix_buffer = (float *)SDL_aligned_alloc(SDL_SIMDGetAlignment(), device->work_buffer_size); + if (device->mix_buffer == NULL) { + retval = SDL_OutOfMemory(); + } + } + + if (retval == 0) { + logdev->postmix = callback; + logdev->postmix_userdata = userdata; + } + + SDL_UnlockMutex(device->lock); + } + return retval; +} int SDL_BindAudioStreams(SDL_AudioDeviceID devid, SDL_AudioStream **streams, int num_streams) { @@ -1782,6 +1837,14 @@ int SDL_AudioDeviceFormatChangedAlreadyLocked(SDL_AudioDevice *device, const SDL kill_device = SDL_TRUE; } + if (device->postmix_buffer) { + SDL_aligned_free(device->postmix_buffer); + device->postmix_buffer = (float *)SDL_aligned_alloc(SDL_SIMDGetAlignment(), device->work_buffer_size); + if (!device->postmix_buffer) { + kill_device = SDL_TRUE; + } + } + SDL_aligned_free(device->mix_buffer); device->mix_buffer = NULL; if (device->spec.format != SDL_AUDIO_F32) { diff --git a/src/audio/SDL_sysaudio.h b/src/audio/SDL_sysaudio.h index 0709661972..6649000ec0 100644 --- a/src/audio/SDL_sysaudio.h +++ b/src/audio/SDL_sysaudio.h @@ -214,6 +214,12 @@ struct SDL_LogicalAudioDevice // SDL_TRUE if device was opened with SDL_OpenAudioDeviceStream (so it forbids binding changes, etc). SDL_bool simplified; + // If non-NULL, callback into the app that lets them access the final postmix buffer. + SDL_AudioPostmixCallback postmix; + + // App-supplied pointer for postmix callback. + void *postmix_userdata; + // double-linked list of opened devices on the same physical device. SDL_LogicalAudioDevice *next; SDL_LogicalAudioDevice *prev; @@ -264,6 +270,7 @@ struct SDL_AudioDevice // Scratch buffers used for mixing. Uint8 *work_buffer; Uint8 *mix_buffer; + float *postmix_buffer; // Size of work_buffer (and mix_buffer) in bytes. int work_buffer_size; diff --git a/src/dynapi/SDL_dynapi.sym b/src/dynapi/SDL_dynapi.sym index b122296f44..0a34ccdc91 100644 --- a/src/dynapi/SDL_dynapi.sym +++ b/src/dynapi/SDL_dynapi.sym @@ -904,6 +904,7 @@ SDL3_0.0.0 { SDL_SetWindowFocusable; SDL_GetAudioStreamFrequencyRatio; SDL_SetAudioStreamFrequencyRatio; + SDL_SetAudioPostmixCallback; # extra symbols go here (don't modify this line) local: *; }; diff --git a/src/dynapi/SDL_dynapi_overrides.h b/src/dynapi/SDL_dynapi_overrides.h index e78911a93d..a66a47a3d3 100644 --- a/src/dynapi/SDL_dynapi_overrides.h +++ b/src/dynapi/SDL_dynapi_overrides.h @@ -929,3 +929,4 @@ #define SDL_SetWindowFocusable SDL_SetWindowFocusable_REAL #define SDL_GetAudioStreamFrequencyRatio SDL_GetAudioStreamFrequencyRatio_REAL #define SDL_SetAudioStreamFrequencyRatio SDL_SetAudioStreamFrequencyRatio_REAL +#define SDL_SetAudioPostmixCallback SDL_SetAudioPostmixCallback_REAL diff --git a/src/dynapi/SDL_dynapi_procs.h b/src/dynapi/SDL_dynapi_procs.h index 5ddb0a24e2..f9e3521aa0 100644 --- a/src/dynapi/SDL_dynapi_procs.h +++ b/src/dynapi/SDL_dynapi_procs.h @@ -975,3 +975,4 @@ SDL_DYNAPI_PROC(int,SDL_GDKGetDefaultUser,(XUserHandle *a),(a),return) SDL_DYNAPI_PROC(int,SDL_SetWindowFocusable,(SDL_Window *a, SDL_bool b),(a,b),return) SDL_DYNAPI_PROC(float,SDL_GetAudioStreamFrequencyRatio,(SDL_AudioStream *a),(a),return) SDL_DYNAPI_PROC(int,SDL_SetAudioStreamFrequencyRatio,(SDL_AudioStream *a, float b),(a,b),return) +SDL_DYNAPI_PROC(int,SDL_SetAudioPostmixCallback,(SDL_AudioDeviceID a, SDL_AudioPostmixCallback b, void *c),(a,b,c),return) diff --git a/test/testaudio.c b/test/testaudio.c index 31659635bc..628c1ec9d5 100644 --- a/test/testaudio.c +++ b/test/testaudio.c @@ -10,6 +10,8 @@ #include "testutils.h" #define POOF_LIFETIME 250 +#define VISUALIZER_WIDTH 100 +#define VISUALIZER_HEIGHT 50 typedef struct Texture { @@ -50,6 +52,15 @@ struct Thing SDL_bool iscapture; SDL_AudioSpec spec; Thing *physdev; + SDL_bool visualizer_enabled; + SDL_bool visualizer_updated; + SDL_Texture *visualizer; + SDL_Mutex *postmix_lock; + float *postmix_buffer; + int postmix_buflen; + int postmix_allocated; + SDL_AudioSpec postmix_spec; + SDL_AtomicInt postmix_updated; } logdev; struct { SDL_AudioSpec spec; @@ -98,6 +109,7 @@ static SDLTest_CommonState *state = NULL; static Thing *things = NULL; static char *current_titlebar = NULL; +static Thing *mouseover_thing = NULL; static Thing *droppable_highlighted_thing = NULL; static Thing *dragging_thing = NULL; static int dragging_button = -1; @@ -279,21 +291,40 @@ static void DestroyThing(Thing *thing) return; } + if (mouseover_thing == thing) { + mouseover_thing = NULL; + } + + if (droppable_highlighted_thing == thing) { + droppable_highlighted_thing = NULL; + } + + if (dragging_thing == thing) { + dragging_thing = NULL; + } + switch (thing->what) { case THING_POOF: break; case THING_NULL: break; case THING_TRASHCAN: break; + case THING_LOGDEV: case THING_LOGDEV_CAPTURE: SDL_CloseAudioDevice(thing->data.logdev.devid); + SDL_DestroyTexture(thing->data.logdev.visualizer); + SDL_DestroyMutex(thing->data.logdev.postmix_lock); + SDL_free(thing->data.logdev.postmix_buffer); break; + case THING_PHYSDEV: case THING_PHYSDEV_CAPTURE: SDL_free(thing->data.physdev.name); break; + case THING_WAV: SDL_free(thing->data.wav.buf); break; + case THING_STREAM: SDL_DestroyAudioStream(thing->data.stream.stream); break; @@ -748,6 +779,135 @@ static void LogicalDeviceThing_ondrop(Thing *thing, int button, float x, float y } } +static void SDLCALL PostmixCallback(void *userdata, const SDL_AudioSpec *spec, float *buffer, int buflen) +{ + Thing *thing = (Thing *) userdata; + + SDL_LockMutex(thing->data.logdev.postmix_lock); + + if (thing->data.logdev.postmix_allocated < buflen) { + void *ptr = SDL_realloc(thing->data.logdev.postmix_buffer, buflen); + if (!ptr) { + SDL_UnlockMutex(thing->data.logdev.postmix_lock); + return; /* oh well. */ + } + thing->data.logdev.postmix_buffer = (float *) ptr; + thing->data.logdev.postmix_allocated = buflen; + } + + SDL_copyp(&thing->data.logdev.postmix_spec, spec); + SDL_memcpy(thing->data.logdev.postmix_buffer, buffer, buflen); + thing->data.logdev.postmix_buflen = buflen; + SDL_AtomicSet(&thing->data.logdev.postmix_updated, 1); + + SDL_UnlockMutex(thing->data.logdev.postmix_lock); +} + +static void UpdateVisualizer(SDL_Renderer *renderer, SDL_Texture *visualizer, const int channels, const float *buffer, const int buflen) +{ + static const SDL_Color channel_colors[8] = { + { 255, 255, 255, 255 }, + { 255, 0, 0, 255 }, + { 0, 255, 0, 255 }, + { 0, 0, 255, 255 }, + { 255, 255, 0, 255 }, + { 0, 255, 255, 255 }, + { 255, 0, 255, 255 }, + { 127, 127, 127, 255 } + }; + + SDL_SetRenderTarget(renderer, visualizer); + SDL_SetRenderDrawColor(renderer, 0, 0, 0, 0); + SDL_RenderClear(renderer); + + if (buffer && buflen) { + const int frames = (buflen / sizeof (float)) / channels; + const int skip = frames / (VISUALIZER_WIDTH * 2); + int i, j; + + for (i = channels - 1; i >= 0; i--) { + const SDL_Color *color = &channel_colors[i % SDL_arraysize(channel_colors)]; + SDL_FPoint points[VISUALIZER_WIDTH + 2]; + float prevx = 0.0f; + int pointidx = 1; + + points[0].x = 0.0f; + points[0].y = VISUALIZER_HEIGHT * 0.5f; + + for (j = 0; j < (SDL_arraysize(points)-1); j++) { + const float val = buffer[((j * skip) * channels) + i]; + const float x = prevx + 2; + const float y = (VISUALIZER_HEIGHT * 0.5f) - (VISUALIZER_HEIGHT * (val * 0.5f)); + SDL_assert(pointidx < SDL_arraysize(points)); + points[pointidx].x = x; + points[pointidx].y = y; + pointidx++; + prevx = x; + } + + SDL_SetRenderDrawColor(renderer, color->r, color->g, color->b, 255); + SDL_RenderLines(renderer, points, pointidx); + } + } + + SDL_SetRenderTarget(renderer, NULL); +} + +static void LogicalDeviceThing_ontick(Thing *thing, Uint64 now) +{ + const SDL_bool ismousedover = (thing == mouseover_thing) ? SDL_TRUE : SDL_FALSE; + + if (!thing->data.logdev.visualizer || !thing->data.logdev.postmix_lock) { /* need these to work, skip if they failed. */ + return; + } + + if (thing->data.logdev.visualizer_enabled != ismousedover) { + thing->data.logdev.visualizer_enabled = ismousedover; + if (!ismousedover) { + SDL_SetAudioPostmixCallback(thing->data.logdev.devid, NULL, NULL); + } else { + if (thing->data.logdev.postmix_buffer) { + SDL_memset(thing->data.logdev.postmix_buffer, '\0', thing->data.logdev.postmix_buflen); + } + SDL_AtomicSet(&thing->data.logdev.postmix_updated, 1); /* so this will at least clear the texture later. */ + SDL_SetAudioPostmixCallback(thing->data.logdev.devid, PostmixCallback, thing); + } + } +} + +static void LogicalDeviceThing_ondraw(Thing *thing, SDL_Renderer *renderer) +{ + if (thing->data.logdev.visualizer_enabled) { + SDL_FRect dst; + dst.w = thing->rect.w; + dst.h = thing->rect.h; + dst.x = thing->rect.x + ((thing->rect.w - dst.w) / 2); + dst.y = thing->rect.y + ((thing->rect.h - dst.h) / 2); + + if (SDL_AtomicGet(&thing->data.logdev.postmix_updated)) { + float *buffer; + int channels; + int buflen; + + SDL_LockMutex(thing->data.logdev.postmix_lock); + channels = thing->data.logdev.postmix_spec.channels; + buflen = thing->data.logdev.postmix_buflen; + buffer = (float *) SDL_malloc(thing->data.logdev.postmix_buflen); + if (buffer) { + SDL_memcpy(buffer, thing->data.logdev.postmix_buffer, thing->data.logdev.postmix_buflen); + SDL_AtomicSet(&thing->data.logdev.postmix_updated, 0); + } + SDL_UnlockMutex(thing->data.logdev.postmix_lock); + + UpdateVisualizer(renderer, thing->data.logdev.visualizer, channels, buffer, buflen); + SDL_free(buffer); + } + + SDL_SetRenderDrawColor(renderer, 255, 255, 255, 30); + SDL_RenderTexture(renderer, thing->data.logdev.visualizer, NULL, &dst); + } +} + static Thing *CreateLogicalDeviceThing(Thing *parent, const SDL_AudioDeviceID which, const float x, const float y) { static const ThingType can_be_dropped_onto[] = { THING_TRASHCAN, THING_NULL }; @@ -760,9 +920,16 @@ static Thing *CreateLogicalDeviceThing(Thing *parent, const SDL_AudioDeviceID wh thing->data.logdev.devid = which; thing->data.logdev.iscapture = iscapture; thing->data.logdev.physdev = physthing; + thing->data.logdev.visualizer = SDL_CreateTexture(state->renderers[0], SDL_PIXELFORMAT_RGBA8888, SDL_TEXTUREACCESS_TARGET, VISUALIZER_WIDTH, VISUALIZER_HEIGHT); + thing->data.logdev.postmix_lock = SDL_CreateMutex(); + if (thing->data.logdev.visualizer) { + SDL_SetTextureBlendMode(thing->data.logdev.visualizer, SDL_BLENDMODE_BLEND); + } thing->line_connected_to = physthing; + thing->ontick = LogicalDeviceThing_ontick; thing->ondrag = DeviceThing_ondrag; thing->ondrop = LogicalDeviceThing_ondrop; + thing->ondraw = LogicalDeviceThing_ondraw; thing->can_be_dropped_onto = can_be_dropped_onto; SetLogicalDeviceTitlebar(thing);