From 9d0f437975ba043f78b14f92e965835c56b56923 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Thu, 30 May 2019 17:20:22 +0100 Subject: [PATCH] (buf)RTNoveltySlice: wrappers, classes, helps, code --- .../Classes/FluidBufNoveltySlice.sc | 4 +- .../Classes/FluidBufRTNoveltySlice.sc | 21 +++ .../Classes/FluidRTNoveltySlice.sc | 17 ++ .../Classes/FluidBufNoveltySlice.schelp | 2 +- .../Classes/FluidBufRTNoveltySlice.schelp | 169 ++++++++++++++++++ .../Classes/FluidRTNoveltySlice.schelp | 81 +++++++++ src/FluidBufRTNoveltySlice/CMakeLists.txt | 20 +++ .../FluidBufRTNoveltySlice.cpp | 14 ++ src/FluidRTNoveltySlice/CMakeLists.txt | 20 +++ .../FluidRTNoveltySlice.cpp | 13 ++ 10 files changed, 358 insertions(+), 3 deletions(-) create mode 100644 release-packaging/Classes/FluidBufRTNoveltySlice.sc create mode 100644 release-packaging/Classes/FluidRTNoveltySlice.sc create mode 100644 release-packaging/HelpSource/Classes/FluidBufRTNoveltySlice.schelp create mode 100644 release-packaging/HelpSource/Classes/FluidRTNoveltySlice.schelp create mode 100644 src/FluidBufRTNoveltySlice/CMakeLists.txt create mode 100644 src/FluidBufRTNoveltySlice/FluidBufRTNoveltySlice.cpp create mode 100644 src/FluidRTNoveltySlice/CMakeLists.txt create mode 100644 src/FluidRTNoveltySlice/FluidRTNoveltySlice.cpp diff --git a/release-packaging/Classes/FluidBufNoveltySlice.sc b/release-packaging/Classes/FluidBufNoveltySlice.sc index b7b934e..c3832e2 100644 --- a/release-packaging/Classes/FluidBufNoveltySlice.sc +++ b/release-packaging/Classes/FluidBufNoveltySlice.sc @@ -1,5 +1,5 @@ FluidBufNoveltySlice{ - *process { arg server, source, startFrame = 0, numFrames = -1, startChan = 0, numChans = -1, indices, kernSize = 3, threshold = 0.8, filterSize = 1, winSize = 1024, hopSize = -1, fftSize = -1, action; + *process { arg server, source, startFrame = 0, numFrames = -1, startChan = 0, numChans = -1, indices, kernelSize = 3, threshold = 0.8, filterSize = 1, winSize = 1024, hopSize = -1, fftSize = -1, action; //var maxFFTSize = if (fftSize == -1) {winSize.nextPowerOfTwo} {fftSize}; //ready for when we need it from the RT wrapper @@ -12,7 +12,7 @@ FluidBufNoveltySlice{ server = server ? Server.default; forkIfNeeded{ - server.sendMsg(\cmd, \BufNoveltySlice, source, startFrame, numFrames, startChan, numChans, indices, kernSize, threshold, filterSize, winSize, hopSize, fftSize); + server.sendMsg(\cmd, \BufNoveltySlice, source, startFrame, numFrames, startChan, numChans, indices, kernelSize, threshold, filterSize, winSize, hopSize, fftSize); server.sync; indices = server.cachedBufferAt(indices); indices.updateInfo; server.sync; action.value(indices); diff --git a/release-packaging/Classes/FluidBufRTNoveltySlice.sc b/release-packaging/Classes/FluidBufRTNoveltySlice.sc new file mode 100644 index 0000000..104f034 --- /dev/null +++ b/release-packaging/Classes/FluidBufRTNoveltySlice.sc @@ -0,0 +1,21 @@ +FluidBufRTNoveltySlice{ + *process { arg server, source, startFrame = 0, numFrames = -1, startChan = 0, numChans = -1, indices, feature = 0, kernelSize = 3, threshold = 0.8, filterSize = 1, winSize = 1024, hopSize = -1, fftSize = -1, action; + + var maxFFTSize = if (fftSize == -1) {winSize.nextPowerOfTwo} {fftSize}; + + source = source.asUGenInput; + indices = indices.asUGenInput; + + source.isNil.if {"FluidBufNoveltySlice: Invalid source buffer".throw}; + indices.isNil.if {"FluidBufNoveltySlice: Invalid features buffer".throw}; + + server = server ? Server.default; + + forkIfNeeded{ + server.sendMsg(\cmd, \BufRTNoveltySlice, source, startFrame, numFrames, startChan, numChans, indices, feature, kernelSize, threshold, filterSize, winSize, hopSize, fftSize, maxFFTSize, kernelSize, filterSize); + server.sync; + indices = server.cachedBufferAt(indices); indices.updateInfo; server.sync; + action.value(indices); + }; + } +} diff --git a/release-packaging/Classes/FluidRTNoveltySlice.sc b/release-packaging/Classes/FluidRTNoveltySlice.sc new file mode 100644 index 0000000..b908004 --- /dev/null +++ b/release-packaging/Classes/FluidRTNoveltySlice.sc @@ -0,0 +1,17 @@ +FluidRTNoveltySlice : UGen { + *ar { arg in = 0, feature = 0, kernelSize = 3, threshold = 0.8, filterSize = 1, winSize = 1024, hopSize = -1, fftSize = -1, maxFFTSize = 16384, maxKernelSize = 101, maxFilterSize = 100; + ^this.multiNew('audio', in.asAudioRateInput(this), feature, kernelSize, threshold, filterSize, winSize, hopSize, fftSize, maxFFTSize, maxKernelSize, maxFilterSize) + } + checkInputs { + if(inputs.at(8).rate != 'scalar') { + ^(": maxFFTSize cannot be modulated."); + }; + if(inputs.at(9).rate != 'scalar') { + ^(": maxKernelSize cannot be modulated."); + }; + if(inputs.at(10).rate != 'scalar') { + ^(": maxFilterSize cannot be modulated."); + }; + ^this.checkValidInputs; + } +} diff --git a/release-packaging/HelpSource/Classes/FluidBufNoveltySlice.schelp b/release-packaging/HelpSource/Classes/FluidBufNoveltySlice.schelp index a0d7e6c..3c2220e 100644 --- a/release-packaging/HelpSource/Classes/FluidBufNoveltySlice.schelp +++ b/release-packaging/HelpSource/Classes/FluidBufNoveltySlice.schelp @@ -36,7 +36,7 @@ ARGUMENT:: numChans ARGUMENT:: indices The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis. -ARGUMENT:: kernSize +ARGUMENT:: kernelSize The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes. ARGUMENT:: threshold diff --git a/release-packaging/HelpSource/Classes/FluidBufRTNoveltySlice.schelp b/release-packaging/HelpSource/Classes/FluidBufRTNoveltySlice.schelp new file mode 100644 index 0000000..586e586 --- /dev/null +++ b/release-packaging/HelpSource/Classes/FluidBufRTNoveltySlice.schelp @@ -0,0 +1,169 @@ +TITLE:: FluidBufRTNoveltySlice +SUMMARY:: Buffer-Based Novelty-Based Slicer +CATEGORIES:: Libraries>FluidDecomposition, UGens>Buffer +RELATED:: Guides/FluCoMa, Guides/FluidDecomposition + + +DESCRIPTION:: +This class implements a non-real-time slicer using an algorithm assessing novelty in the signal to estimate the slicing points. A novelty curve is being derived from running a kernel across the diagonal of the similarity matrix, and looking for peak of changes. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. It is part of the Fluid Decomposition Toolkit of the FluCoMa project. footnote::This was made possible thanks to the FluCoMa project ( http://www.flucoma.org/ ) funded by the European Research Council ( https://erc.europa.eu/ ) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899).:: + + The process will return a buffer which contains indices (in sample) of estimated starting points of different slices. + + +CLASSMETHODS:: + +METHOD:: process +This is the method that calls for the slicing to be calculated on a given source buffer. + +ARGUMENT:: server + The server on which the buffers to be processed are allocated. + +ARGUMENT:: source + The index of the buffer to use as the source material to be sliced through novelty identification. The different channels of multichannel buffers will be summed. + +ARGUMENT:: startFrame + Where in the srcBuf should the slicing process start, in sample. + +ARGUMENT:: numFrames + How many frames should be processed. + +ARGUMENT:: startChan + For multichannel srcBuf, which channel should be processed. + +ARGUMENT:: numChans + For multichannel srcBuf, how many channel should be summed. + +ARGUMENT:: indices + The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis. + +ARGUMENT:: feature + The feature on which novelty is computed. + table:: + ##0 || Spectrum || todo + ##1 || MFCC || todo + ##2 || Pitch || todo + ##3 || Loudness || todo +:: +ARGUMENT:: kernelSize + The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes. + +ARGUMENT:: threshold + The normalised threshold, between 0 an 1, on the novelty curve to consider it a segmentation point. + +ARGUMENT:: filterSize + The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. + +ARGUMENT:: winSize + The window size. As novelty estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty + +ARGUMENT:: hopSize + The window hop size. As novelty estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. + +ARGUMENT:: fftSize + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. + +ARGUMENT:: action + A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument. + +RETURNS:: + Nothing, as the various destination buffers are declared in the function call. + + +EXAMPLES:: + +code:: +// load some buffers +( +b = Buffer.read(s,File.realpath(FluidBufRTNoveltySlice.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/Tremblay-AaS-AcousticStrums-M.wav"); +c = Buffer.new(s); +) + +( +// with basic params +Routine{ + t = Main.elapsedTime; + FluidBufRTNoveltySlice.process(s,b, indices: c, threshold:0.6); + (Main.elapsedTime - t).postln; +}.play +) + +//check the number of slices: it is the number of frames in the transBuf minus the boundary index. +c.query; + +//loops over a splice with the MouseX +( +{ + BufRd.ar(1, b, + Phasor.ar(0,1, + BufRd.kr(1, c, + MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1), + BufRd.kr(1, c, + MouseX.kr(1, BufFrames.kr(c)), 0, 1), + BufRd.kr(1,c, + MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1)), 0, 1); + }.play; +) + :: + +STRONG::Examples of the impact of the filterSize:: + + CODE:: +// load some buffers +( +b = Buffer.read(s,File.realpath(FluidBufRTNoveltySlice.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/Tremblay-AaS-AcousticStrums-M.wav"); +c = Buffer.new(s); +) + +// process with a given filterSize +FluidBufRTNoveltySlice.process(s,b, indices: c, kernSize:31, threshold:0.3, filterSize:0) + +//check the number of slices: it is the number of frames in the transBuf minus the boundary index. +c.query; + +//play slice number 2 +( +{ + BufRd.ar(1, b, + Line.ar( + BufRd.kr(1, c, DC.kr(2), 0, 1), + BufRd.kr(1, c, DC.kr(3), 0, 1), + (BufRd.kr(1, c, DC.kr(3)) - BufRd.kr(1, c, DC.kr(2), 0, 1) + 1) / s.sampleRate), + 0,1); +}.play; +) + +// change the filterSize in the code above to 4. Then to 8. Listen in between to the differences. + +// What's happening? In the first instance (filterSize = 1), the novelty line is jittery and therefore overtriggers on the arpegiated guitar. We also can hear attacks at the end of the segment. Setting the threshold higher (like in the 'Basic Example' pane) misses some more subtle variations. + +// So in the second settings (filterSize = 4), we smooth the novelty line a little, which allows us to catch small differences that are not jittery. It also corrects the ending cutting by the same trick: the averaging of the sharp pick is sliding up, crossing the threshold slightly earlier. + +// If we smooth too much, like the third settings (filterSize = 8), we start to loose precision. Have fun with different values of theshold then will allow you to find the perfect segment for your signal. +:: + +STRONG::A stereo buffer example.:: +CODE:: + +// make a stereo buffer +b = Buffer.alloc(s,88200,2); + +// add some stereo clicks and listen to them +((0..3)*22050+11025).do({|item,index| b.set(item+(index%2), 1.0)}); +b.play + +// create a new buffer as destinations +c = Buffer.new(s); + +//run the process on them +( +// with basic params +Routine{ + t = Main.elapsedTime; + FluidBufRTNoveltySlice.process(s,b, indices: c, threshold:0.6); + (Main.elapsedTime - t).postln; +}.play +) + +// list the indicies of detected attacks - the two input channels have been summed +c.getn(0,c.numFrames,{|item|item.postln;}) +:: diff --git a/release-packaging/HelpSource/Classes/FluidRTNoveltySlice.schelp b/release-packaging/HelpSource/Classes/FluidRTNoveltySlice.schelp new file mode 100644 index 0000000..58304bc --- /dev/null +++ b/release-packaging/HelpSource/Classes/FluidRTNoveltySlice.schelp @@ -0,0 +1,81 @@ +TITLE:: FluidRTNoveltySlice +SUMMARY:: Spectral Difference-Based Real-Time Audio Slicer +CATEGORIES:: Libraries>FluidDecomposition +RELATED:: Guides/FluCoMa, Guides/FluidDecomposition + +DESCRIPTION:: +This class implements many spectral based onset detection functions, most of them taken from the literature. (http://www.dafx.ca/proceedings/papers/p_133.pdf) Some are already available in SuperCollider's LINK::Classes/Onsets:: object. It is part of the Fluid Decomposition Toolkit of the FluCoMa project.footnote::This was made possible thanks to the FluCoMa project ( http://www.flucoma.org/ ) funded by the European Research Council ( https://erc.europa.eu/ ) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899).:: + +The process will return an audio steam with sample-long impulses at estimated starting points of the different slices. + + +CLASSMETHODS:: + +METHOD:: ar + The audio rate version of the object. + +ARGUMENT:: in + The audio to be processed. + +ARGUMENT:: feature + The feature on which novelty is computed. + table:: + ##0 || Spectrum || todo + ##1 || MFCC || todo + ##2 || Pitch || todo + ##3 || Loudness || todo +:: +ARGUMENT:: kernelSize + The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes. + +ARGUMENT:: threshold + The normalised threshold, between 0 an 1, on the novelty curve to consider it a segmentation point. + +ARGUMENT:: filterSize + The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. + +ARGUMENT:: winSize + The window size. As sinusoidal estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty + +ARGUMENT:: hopSize + The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of winSize (overlap of 2). + +ARGUMENT:: fftSize + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize. + +ARGUMENT:: maxFFTSize + How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated. + +ARGUMENT:: maxKernelSize + This cannot be modulated. + +ARGUMENT:: maxFilterSize + This cannot be modulated. + +RETURNS:: + An audio stream with impulses at detected transients. The latency between the input and the output is winSize at maximum. + +EXAMPLES:: + +code:: +//load some sounds +b = Buffer.read(s,File.realpath(FluidRTNoveltySlice.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/Nicol-LoopE-M.wav"); + +// basic param (the process add a latency of winSize samples +{var sig = PlayBuf.ar(1,b,loop:1); [FluidRTNoveltySlice.ar(sig,0,3,0.2) * 0.5, DelayN.ar(sig, 1, 1024/ s.sampleRate)]}.play + +// other parameters +{var sig = PlayBuf.ar(1,b,loop:1); [FluidRTNoveltySlice.ar(sig, 0, 31, 0.05, 4, 128, 64) * 0.5, DelayN.ar(sig, 1, (128)/ s.sampleRate)]}.play + +// more musical trans-trigged autopan +( +{ + var sig, trig, syncd, pan; + sig = PlayBuf.ar(1,b,loop:1); + trig = FluidRTNoveltySlice.ar(sig, 0, 0.2, 100, 8, 0, 128); + syncd = DelayN.ar(sig, 1, ( 128 / s.sampleRate)); + pan = TRand.ar(-1,1,trig); + Pan2.ar(syncd,pan); +}.play +) +:: \ No newline at end of file diff --git a/src/FluidBufRTNoveltySlice/CMakeLists.txt b/src/FluidBufRTNoveltySlice/CMakeLists.txt new file mode 100644 index 0000000..3693881 --- /dev/null +++ b/src/FluidBufRTNoveltySlice/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.3) +get_filename_component(PLUGIN ${CMAKE_CURRENT_LIST_DIR} NAME_WE) +message("Configuring ${PLUGIN}") +set(FILENAME ${PLUGIN}.cpp) + +add_library( + ${PLUGIN} + MODULE + ${FILENAME} +) + +target_include_directories( + ${PLUGIN} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/../../include +) + +target_link_libraries( + ${PLUGIN} PRIVATE FLUID_DECOMPOSITION +) + +include(${CMAKE_CURRENT_LIST_DIR}/../../scripts/target_post.cmake) diff --git a/src/FluidBufRTNoveltySlice/FluidBufRTNoveltySlice.cpp b/src/FluidBufRTNoveltySlice/FluidBufRTNoveltySlice.cpp new file mode 100644 index 0000000..c637663 --- /dev/null +++ b/src/FluidBufRTNoveltySlice/FluidBufRTNoveltySlice.cpp @@ -0,0 +1,14 @@ + +// A tool from the FluCoMa project, funded by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899) + +#include +#include +#include + +static InterfaceTable *ft; + +PluginLoad(OfflineFluidDecompositionUGens) { + ft = inTable; + using namespace fluid::client; + makeSCWrapper("BufRTNoveltySlice", ft); +} diff --git a/src/FluidRTNoveltySlice/CMakeLists.txt b/src/FluidRTNoveltySlice/CMakeLists.txt new file mode 100644 index 0000000..3693881 --- /dev/null +++ b/src/FluidRTNoveltySlice/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.3) +get_filename_component(PLUGIN ${CMAKE_CURRENT_LIST_DIR} NAME_WE) +message("Configuring ${PLUGIN}") +set(FILENAME ${PLUGIN}.cpp) + +add_library( + ${PLUGIN} + MODULE + ${FILENAME} +) + +target_include_directories( + ${PLUGIN} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/../../include +) + +target_link_libraries( + ${PLUGIN} PRIVATE FLUID_DECOMPOSITION +) + +include(${CMAKE_CURRENT_LIST_DIR}/../../scripts/target_post.cmake) diff --git a/src/FluidRTNoveltySlice/FluidRTNoveltySlice.cpp b/src/FluidRTNoveltySlice/FluidRTNoveltySlice.cpp new file mode 100644 index 0000000..8e23423 --- /dev/null +++ b/src/FluidRTNoveltySlice/FluidRTNoveltySlice.cpp @@ -0,0 +1,13 @@ + +// A tool from the FluCoMa project, funded by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899) + +#include +#include + +static InterfaceTable *ft; + +PluginLoad(FluidSTFTUGen) { + ft = inTable; + using namespace fluid::client; + makeSCWrapper("FluidRTNoveltySlice", ft); +}