From 050c80286f8bf18949816f0904b95ba7a1fcf748 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 12 Jun 2019 15:28:09 -0400 Subject: [PATCH 1/4] examples: folder iteration and 2 passes now working fine --- .../buffer_compositing/fileiterator-2passes.sc | 10 +++++----- .../ignore/Examples/buffer_compositing/fileiterator.sc | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/release-packaging/ignore/Examples/buffer_compositing/fileiterator-2passes.sc b/release-packaging/ignore/Examples/buffer_compositing/fileiterator-2passes.sc index 79c2ecb..4eaa92c 100644 --- a/release-packaging/ignore/Examples/buffer_compositing/fileiterator-2passes.sc +++ b/release-packaging/ignore/Examples/buffer_compositing/fileiterator-2passes.sc @@ -2,7 +2,7 @@ ( var fileNames; -c = [0]; +c = []; FileDialog.new({|selection| var total, totaldur = 0, maxchans = 0; @@ -16,14 +16,11 @@ FileDialog.new({|selection| SoundFile.use(fp.asAbsolutePath , { arg file; var dur = file.numFrames; - c = c.add(dur); + c = c.add(totaldur); totaldur = totaldur + dur; maxchans = maxchans.max(file.numChannels); }); }); - c.postln; - totaldur.postln; - maxchans.postln; Routine{ b = Buffer.alloc(s,totaldur,maxchans); s.sync; @@ -42,4 +39,7 @@ b.plot c.postln b.play + +{PlayBuf.ar(1,b.bufnum,startPos:c[740])}.play + Buffer.freeAll \ No newline at end of file diff --git a/release-packaging/ignore/Examples/buffer_compositing/fileiterator.sc b/release-packaging/ignore/Examples/buffer_compositing/fileiterator.sc index e8ef47f..9808f72 100644 --- a/release-packaging/ignore/Examples/buffer_compositing/fileiterator.sc +++ b/release-packaging/ignore/Examples/buffer_compositing/fileiterator.sc @@ -36,3 +36,5 @@ FileDialog.new({|selection| b.plot c.postln b.play + +{PlayBuf.ar(1,b.bufnum,startPos:c[740])}.play From b1c56d03254ab013fa6a258a23305a8f8692216a Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 12 Jun 2019 16:10:42 -0400 Subject: [PATCH 2/4] new MFCC example (and corrected MelBands) thanks to Sam Pluta --- .../HelpSource/Classes/FluidMFCC.schelp | 174 ++++++++++++++++-- .../HelpSource/Classes/FluidMelBands.schelp | 26 +-- 2 files changed, 178 insertions(+), 22 deletions(-) diff --git a/release-packaging/HelpSource/Classes/FluidMFCC.schelp b/release-packaging/HelpSource/Classes/FluidMFCC.schelp index 8bedc57..0f877ee 100644 --- a/release-packaging/HelpSource/Classes/FluidMFCC.schelp +++ b/release-packaging/HelpSource/Classes/FluidMFCC.schelp @@ -50,24 +50,27 @@ RETURNS:: EXAMPLES:: code:: -//create a monitoring bus for the descriptors -b = Bus.new(\control,0,13); - //create a monitoring window for the values ( +b = Bus.new(\control,0,13); w = Window("MFCCs Monitor", Rect(10, 10, 420, 320)).front; a = MultiSliderView(w,Rect(10, 10, 400, 300)).elasticMode_(1).isFilled_(1); +a.reference_(Array.fill(13,{0.5})); //make a center line to show 0 ) //run the window updating routine. ( + +~winRange = 500; + r = Routine { { b.get({ arg val; { if(w.isClosed.not) { - a.value = val; + //val.postln; + a.value = val.linlin(~winRange.neg,~winRange,0,1); } }.defer }); @@ -78,8 +81,8 @@ r = Routine { //play a simple sound to observe the values ( -x = {arg type = 0; - var source = Select.ar(type,[SinOsc.ar(220),Saw.ar(220),Pulse.ar(220)]) * LFTri.kr(0.1).exprange(0.01,0.1); +x = {arg type = 0; + var source = Select.ar(type,[SinOsc.ar(220),Saw.ar(220),Pulse.ar(220)]) * LFTri.kr(0.1).exprange(0.01,0.1); Out.kr(b,FluidMFCC.kr(source,maxNumCoeffs:13)); source.dup; }.play; @@ -87,19 +90,20 @@ x = {arg type = 0; // change the wave types, observe the amplitude invariance of the descriptors, apart from the leftmost coefficient x.set(\type, 1) +~winRange = 100; //adjust the range above and below 0 to zoom in or out on the MFCC x.set(\type, 2) x.set(\type, 0) // free this source x.free // load a more exciting one -c = Buffer.read(s,File.realpath(FluidMelBands.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/Tremblay-AaS-SynthTwoVoices-M.wav"); +c = Buffer.read(s,File.realpath(FluidMFCC.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/Tremblay-AaS-SynthTwoVoices-M.wav"); // analyse with parameters to be changed ( -x = {arg bands = 40, low = 20, high = 20000; +x = {arg bands = 40, low = 20, high = 20000; var source = PlayBuf.ar(1,c,loop:1); - Out.kr(b,FluidMelBands.kr(source, bands, low, high, 40) / 10); + Out.kr(b,FluidMFCC.kr(source, 13, bands, low, high, 13) / 10); source.dup; }.play; ) @@ -111,7 +115,7 @@ x.set(\bands,20) x.set(\bands,40) // focus all the bands on a mid range -x.set(\low,320, \high, 8000) +x.set(\low,320, \high, 800) // focusing on the low end shows the fft resolution issue. One could restart the analysis with a larger fft to show more precision x.set(\low,20, \high, 160) @@ -126,5 +130,153 @@ x.free;b.free;c.free;r.stop; STRONG::A musical example:: CODE:: -// todo: port the Max one +//program that freezes mfcc spectra, then looks for matches between two frozen spectra +( +SynthDef("MFCCJamz", {arg freq=220, source = 0, buffer, mfccBus, distBus, t_freeze0=0, t_freeze1=0, onsetsOn0=0, onsetsOn1=0; + var sound, mfcc, mfccFreeze0, mfccFreeze1, dist0, dist1, closest, slice; + + sound = SelectX.ar(source, [ + SinOsc.ar(freq, 0, 0.1), + LFTri.ar(freq, 0, 0.1), + LFSaw.ar(freq, 0, 0.1), + Pulse.ar(freq, 0.5, 0.1), + WhiteNoise.ar(0.1), + PinkNoise.ar(0.1), + PlayBuf.ar(1, buffer, 1, loop:1) + ]); + + slice = FluidOnsetSlice.ar(sound); //onset detection for mfcc freeze on onset + + mfcc = FluidMFCC.kr(sound,maxNumCoeffs:13); + mfccFreeze0 = Latch.kr(mfcc, t_freeze0+(slice*onsetsOn0)); + mfccFreeze1 = Latch.kr(mfcc, t_freeze1+(slice*onsetsOn1)); + + Out.kr(mfccBus,mfcc.addAll(mfccFreeze0).addAll(mfccFreeze1)); + + //distance calculations + + dist0 = Mix((mfcc.copyRange(1,12) - mfccFreeze0.copyRange(1,12)).squared).sqrt; + dist1 = Mix((mfcc.copyRange(1,12) - mfccFreeze1.copyRange(1,12)).squared).sqrt; + + Out.kr(distBus, [dist0, dist1]); + + //sends a trigger when the item with a closer euclidean distance changes + SendTrig.kr(Trig1.kr(dist1-dist0, 0.001)+Trig1.kr(dist0-dist1, 0.001), 0, dist1 Date: Thu, 13 Jun 2019 11:21:56 -0400 Subject: [PATCH 3/4] name change for Onset functions to metrics --- release-packaging/Classes/FluidBufOnsetSlice.sc | 4 ++-- release-packaging/Classes/FluidOnsetSlice.sc | 4 ++-- .../HelpSource/Classes/FluidBufOnsetSlice.schelp | 12 ++++++------ .../HelpSource/Classes/FluidOnsetSlice.schelp | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/release-packaging/Classes/FluidBufOnsetSlice.sc b/release-packaging/Classes/FluidBufOnsetSlice.sc index d7bbcac..4e3304c 100644 --- a/release-packaging/Classes/FluidBufOnsetSlice.sc +++ b/release-packaging/Classes/FluidBufOnsetSlice.sc @@ -1,5 +1,5 @@ FluidBufOnsetSlice{ - *process { arg server, source, startFrame = 0, numFrames = -1, startChan = 0, numChans = -1, indices, function = 0, threshold = 0.1, minSliceLength = 2, filterSize = 5, frameDelta = 0, windowSize = 1024, hopSize = -1, fftSize = -1, action; + *process { arg server, source, startFrame = 0, numFrames = -1, startChan = 0, numChans = -1, indices, metric = 0, threshold = 0.5, minSliceLength = 2, filterSize = 5, frameDelta = 0, windowSize = 1024, hopSize = -1, fftSize = -1, action; var maxFFTSize = if (fftSize == -1) {windowSize.nextPowerOfTwo} {fftSize}; @@ -15,7 +15,7 @@ FluidBufOnsetSlice{ //whatever has been passed in language-side (e.g maxFFTSize still exists as a parameter for the server plugin, but makes less sense here: it just needs to be set to a legal value) forkIfNeeded{ - server.sendMsg(\cmd, \BufOnsetSlice, source, startFrame, numFrames, startChan, numChans, indices, function, threshold, minSliceLength, filterSize, frameDelta, windowSize, hopSize, fftSize, maxFFTSize); + server.sendMsg(\cmd, \BufOnsetSlice, source, startFrame, numFrames, startChan, numChans, indices, metric, threshold, minSliceLength, filterSize, frameDelta, windowSize, hopSize, fftSize, maxFFTSize); server.sync; indices = server.cachedBufferAt(indices); indices.updateInfo; server.sync; action.value(indices); diff --git a/release-packaging/Classes/FluidOnsetSlice.sc b/release-packaging/Classes/FluidOnsetSlice.sc index 2bedb64..2ecbb51 100644 --- a/release-packaging/Classes/FluidOnsetSlice.sc +++ b/release-packaging/Classes/FluidOnsetSlice.sc @@ -1,6 +1,6 @@ FluidOnsetSlice : UGen { - *ar { arg in = 0, function = 0, threshold = 0.5, minSliceLength = 2, filterSize = 5, frameDelta = 0, windowSize = 1024, hopSize = -1, fftSize = -1, maxFFTSize = 16384; - ^this.multiNew('audio', in.asAudioRateInput(this), function, threshold, minSliceLength, filterSize, frameDelta, windowSize, hopSize, fftSize, maxFFTSize) + *ar { arg in = 0, metric = 0, threshold = 0.5, minSliceLength = 2, filterSize = 5, frameDelta = 0, windowSize = 1024, hopSize = -1, fftSize = -1, maxFFTSize = 16384; + ^this.multiNew('audio', in.asAudioRateInput(this), metric, threshold, minSliceLength, filterSize, frameDelta, windowSize, hopSize, fftSize, maxFFTSize) } checkInputs { if(inputs.at(9).rate != 'scalar') { diff --git a/release-packaging/HelpSource/Classes/FluidBufOnsetSlice.schelp b/release-packaging/HelpSource/Classes/FluidBufOnsetSlice.schelp index ecd692f..09389f7 100644 --- a/release-packaging/HelpSource/Classes/FluidBufOnsetSlice.schelp +++ b/release-packaging/HelpSource/Classes/FluidBufOnsetSlice.schelp @@ -4,7 +4,7 @@ CATEGORIES:: Libraries>FluidDecomposition RELATED:: Guides/FluCoMa, Guides/FluidDecomposition DESCRIPTION:: -This class implements many spectral-based onset detection functions, most of them taken from the literature. (http://www.dafx.ca/proceedings/papers/p_133.pdf) Some are already available in SuperCollider's LINK::Classes/Onsets:: object yet not as offline processes. It is part of the Fluid Decomposition Toolkit of the FluCoMa project.footnote::This was made possible thanks to the FluCoMa project ( http://www.flucoma.org/ ) funded by the European Research Council ( https://erc.europa.eu/ ) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899).:: +This class implements many spectral-based onset detection metrics, most of them taken from the literature. (http://www.dafx.ca/proceedings/papers/p_133.pdf) Some are already available in SuperCollider's LINK::Classes/Onsets:: object yet not as offline processes. It is part of the Fluid Decomposition Toolkit of the FluCoMa project.footnote::This was made possible thanks to the FluCoMa project ( http://www.flucoma.org/ ) funded by the European Research Council ( https://erc.europa.eu/ ) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899).:: The process will return a buffer which contains indices (in sample) of estimated starting points of different slices. @@ -34,8 +34,8 @@ ARGUMENT:: numChans ARGUMENT:: indices The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis. -ARGUMENT:: function - The function used to derive a difference curve between spectral frames. It can be any of the following: +ARGUMENT:: metric + The metric used to derive a difference curve between spectral frames. It can be any of the following: TABLE:: ##0 || Energy || thresholds on (sum of squares of magnitudes / nBins) (like Onsets \power) ##1 || HFC || thresholds on (sum of (squared magnitudes * binNum) / nBins) @@ -50,7 +50,7 @@ ARGUMENT:: function :: ARGUMENT:: threshold -The thresholding of a new slice. Value ranges are different for each function, from 0 upwards. +The thresholding of a new slice. Value ranges are different for each metric, from 0 upwards. ARGUMENT:: minSliceLength The minimum duration of a slice in number of hopSize. @@ -59,7 +59,7 @@ ARGUMENT:: filterSize The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. ARGUMENT:: frameDelta - For certain functions (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distane between the comparison window in samples. + For certain metrics (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distane between the comparison window in samples. ARGUMENT:: windowSize The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty @@ -71,7 +71,7 @@ ARGUMENT:: fftSize The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize. ARGUMENT:: action - A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument. + A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The metric will be passed indices as an argument. RETURNS:: Nothing, as the various destination buffers are declared in the function call. diff --git a/release-packaging/HelpSource/Classes/FluidOnsetSlice.schelp b/release-packaging/HelpSource/Classes/FluidOnsetSlice.schelp index 449c05f..1918be3 100644 --- a/release-packaging/HelpSource/Classes/FluidOnsetSlice.schelp +++ b/release-packaging/HelpSource/Classes/FluidOnsetSlice.schelp @@ -4,7 +4,7 @@ CATEGORIES:: Libraries>FluidDecomposition RELATED:: Guides/FluCoMa, Guides/FluidDecomposition DESCRIPTION:: -This class implements many spectral based onset detection functions, most of them taken from the literature. (http://www.dafx.ca/proceedings/papers/p_133.pdf) Some are already available in SuperCollider's LINK::Classes/Onsets:: object. It is part of the Fluid Decomposition Toolkit of the FluCoMa project.footnote::This was made possible thanks to the FluCoMa project ( http://www.flucoma.org/ ) funded by the European Research Council ( https://erc.europa.eu/ ) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899).:: +This class implements many spectral based onset detection metrics, most of them taken from the literature. (http://www.dafx.ca/proceedings/papers/p_133.pdf) Some are already available in SuperCollider's LINK::Classes/Onsets:: object. It is part of the Fluid Decomposition Toolkit of the FluCoMa project.footnote::This was made possible thanks to the FluCoMa project ( http://www.flucoma.org/ ) funded by the European Research Council ( https://erc.europa.eu/ ) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899).:: The process will return an audio steam with sample-long impulses at estimated starting points of the different slices. @@ -17,8 +17,8 @@ METHOD:: ar ARGUMENT:: in The audio to be processed. -ARGUMENT:: function - The function used to derive a difference curve between spectral frames. It can be any of the following: +ARGUMENT:: metric + The metric used to derive a difference curve between spectral frames. It can be any of the following: TABLE:: ##0 || Energy || thresholds on (sum of squares of magnitudes / nBins) (like Onsets \power) ##1 || HFC || thresholds on (sum of (squared magnitudes * binNum) / nBins) @@ -33,7 +33,7 @@ ARGUMENT:: function :: ARGUMENT:: threshold -The thresholding of a new slice. Value ranges are different for each function, from 0 upwards. +The thresholding of a new slice. Value ranges are different for each metric, from 0 upwards. ARGUMENT:: minSliceLength The minimum duration of a slice in number of hopSize. @@ -42,7 +42,7 @@ ARGUMENT:: filterSize The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. ARGUMENT:: frameDelta - For certain functions (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distane between the comparison window in samples. + For certain metrics (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distane between the comparison window in samples. ARGUMENT:: windowSize The window size. As sinusoidal estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty From f95cbe1b2c530612aae28fd228ae2482a1c2adfa Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Thu, 13 Jun 2019 12:12:28 -0400 Subject: [PATCH 4/4] (buf)sines: amended the description to make the fftSize capping behaviour clear. --- release-packaging/HelpSource/Classes/FluidBufSines.schelp | 4 ++-- release-packaging/HelpSource/Classes/FluidSines.schelp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/release-packaging/HelpSource/Classes/FluidBufSines.schelp b/release-packaging/HelpSource/Classes/FluidBufSines.schelp index c283211..e0e767c 100644 --- a/release-packaging/HelpSource/Classes/FluidBufSines.schelp +++ b/release-packaging/HelpSource/Classes/FluidBufSines.schelp @@ -43,7 +43,7 @@ ARGUMENT:: residual The index of the buffer where the residual of the sinusoidal component will be reconstructed. ARGUMENT:: bandwidth - The width in bins of the fragment of the fft window that is considered a normal deviation for a potential continuous sinusoidal track. It has an effect on CPU cost: the widest is more accurate but more computationally expensive. + The width in bins of the fragment of the fft window that is considered a normal deviation for a potential continuous sinusoidal track. It has an effect on CPU cost: the widest is more accurate but more computationally expensive. It is capped at (fftSize / 2) + 1. ARGUMENT:: threshold The normalised threshold, between 0 an 1, to consider a peak as a sinusoidal component from the normalized cross-correlation. @@ -64,7 +64,7 @@ ARGUMENT:: hopSize The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. ARGUMENT:: fftSize - The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to the highest of windowSize and (bandwidth - 1) * 2. ARGUMENT:: action A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [sines, residual] as an argument. diff --git a/release-packaging/HelpSource/Classes/FluidSines.schelp b/release-packaging/HelpSource/Classes/FluidSines.schelp index fd6152f..f4b9a41 100644 --- a/release-packaging/HelpSource/Classes/FluidSines.schelp +++ b/release-packaging/HelpSource/Classes/FluidSines.schelp @@ -22,7 +22,7 @@ ARGUMENT:: in The input to be processed ARGUMENT:: bandwidth - The width in bins of the fragment of the fft window that is considered a normal deviation for a potential continuous sinusoidal track. It has an effect on CPU cost: the widest is more accurate but more computationally expensive. + The width in bins of the fragment of the fft window that is considered a normal deviation for a potential continuous sinusoidal track. It has an effect on CPU cost: the widest is more accurate but more computationally expensive. It is capped at (fftSize / 2) + 1. ARGUMENT:: threshold The normalised threshold, between 0 an 1, to consider a peak as a sinusoidal component from the normalized cross-correlation. @@ -43,7 +43,7 @@ ARGUMENT:: hopSize The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2). ARGUMENT:: fftSize - The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize. + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to the highest of windowSize and (bandwidth - 1) * 2. ARGUMENT:: maxFFTSize How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.