tremblap
3 years ago
committed by
GitHub
commit
39860d14d7
@ -1,68 +0,0 @@
|
||||
TITLE:: FluidAudioTransport
|
||||
summary:: Interpolate between sounds
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidBufAudioTransport
|
||||
|
||||
DESCRIPTION::
|
||||
Interpolates between the spectra of two sounds using the Optimal Transport algorithm
|
||||
|
||||
See
|
||||
Henderson and Solomonm (2019) AUDIO TRANSPORT: A GENERALIZED PORTAMENTO VIA OPTIMAL TRANSPORT, DaFx
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
Process incoming audio signals
|
||||
|
||||
ARGUMENT:: in
|
||||
Source A
|
||||
|
||||
ARGUMENT:: in2
|
||||
Source B
|
||||
|
||||
ARGUMENT:: interpolation
|
||||
The amount to interpolate between A and B (0-1, 0 = A, 1 = B)
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size in samples. As AudioTransport relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size in samples. As AudioTransport relies on spectral frames, we need to move the window forward. It can be any size but low overlap may create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long; at least the size of the window; and a power of 2. Making it larger than the window size provides interpolation in frequency. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
RETURNS::
|
||||
An audio stream with the interpolated spectrum of the inputs.
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//didactic - the mouse X axis interpolates between the two sinewaves
|
||||
{FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1),SinOsc.ar(440,mul: 0.02),MouseX.kr())}.play;
|
||||
|
||||
//notice how the interpolation quantizes to the FFT bins. Like most spectral processes, it benefits from oversampling the fft... at the cost of CPU power, obviously.
|
||||
{FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1),SinOsc.ar(440,mul: 0.02),MouseX.kr(),fftSize: 8192)}.play;
|
||||
|
||||
// when the signal is steady, larger hopSize can be accommodated to save back on the CPU
|
||||
{FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1),SinOsc.ar(440,mul: 0.02),MouseX.kr(),windowSize: 8192)}.play; // here we get a default hop of half the window so 8 times less than above.
|
||||
|
||||
//if you CPU can cope, try this setting, almost smooth, but attacks would smear (the Y axis mixes some in to hear the effect)
|
||||
{var attacks = Impulse.ar(1,mul: MouseY.kr(-40,10).dbamp); FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1,add: attacks),SinOsc.ar(440,mul: 0.02,add: attacks),MouseX.kr(),windowSize: 16000)}.play;
|
||||
|
||||
//richer with complex spectra
|
||||
//load 2 files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-CEL-GlitchyMusicBoxMelo.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-CF-ChurchBells.wav"));
|
||||
)
|
||||
//listen to them
|
||||
b.play
|
||||
c.play
|
||||
//stereo cross!
|
||||
{FluidAudioTransport.ar(PlayBuf.ar(2,b,loop: 1),PlayBuf.ar(2,c,loop: 1),MouseX.kr())}.play;
|
||||
|
||||
::
|
||||
@ -1,344 +0,0 @@
|
||||
TITLE:: FluidBufAudioTransport
|
||||
SUMMARY:: Interpolate between buffers
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Classes/FluidAudioTransport
|
||||
DESCRIPTION::
|
||||
|
||||
|
||||
Interpolates between the spectra of two sounds using the Optimal Transport algorithm
|
||||
|
||||
See Henderson and Solomonm (2019) AUDIO TRANSPORT: A GENERALIZED PORTAMENTO VIA OPTIMAL TRANSPORT, DaFx
|
||||
|
||||
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
Processs the source LINK::Classes/Buffer:: on the LINK::Classes/Server::. CODE::processBlocking:: will execute directly in the server command FIFO, whereas CODE::process:: will delegate to a separate worker thread. The latter is generally only worthwhile for longer-running jobs where you don't wish to tie up the server.
|
||||
|
||||
ARGUMENT:: server
|
||||
The LINK::Classes/Server:: on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: sourceA
|
||||
|
||||
|
||||
The first source buffer
|
||||
|
||||
|
||||
ARGUMENT:: startFrameA
|
||||
|
||||
|
||||
offset into the first source buffer (samples)
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFramesA
|
||||
|
||||
|
||||
number of samples to use from first source buffer
|
||||
|
||||
|
||||
ARGUMENT:: startChanA
|
||||
|
||||
|
||||
starting channel of first source buffer
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChansA
|
||||
|
||||
|
||||
number of channels to process in first source buffer
|
||||
|
||||
|
||||
ARGUMENT:: sourceB
|
||||
|
||||
|
||||
the second source buffer
|
||||
|
||||
|
||||
ARGUMENT:: startFrameB
|
||||
|
||||
|
||||
offset into the second source buffer (samples)
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFramesB
|
||||
|
||||
|
||||
number of samples to process from second buffer
|
||||
|
||||
|
||||
ARGUMENT:: startChanB
|
||||
|
||||
|
||||
starting channel for second buffer
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChansB
|
||||
|
||||
|
||||
number of channels to process in second buffer
|
||||
|
||||
|
||||
ARGUMENT:: destination
|
||||
|
||||
|
||||
buffer for interpolated audio
|
||||
|
||||
|
||||
ARGUMENT:: interpolation
|
||||
|
||||
|
||||
The amount to interpolate between A and B (0-1, 0 = A, 1 = B)
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0.0
|
||||
|
||||
##
|
||||
Maximum: 1.0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the highest of windowSize and (bandwidth - 1) * 2.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default CODE::true::
|
||||
|
||||
ARGUMENT:: action
|
||||
A function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed CODE::[features]:: as an argument.
|
||||
|
||||
RETURNS:: An instance of the processor
|
||||
|
||||
METHOD:: kr
|
||||
Trigger the equivalent behaviour to CODE::processBlocking / process:: from a LINK::Classes/Synth::. Can be useful for expressing a sequence of buffer and data processing jobs to execute. Note that the work still executes on the server command FIFO (not the audio thread), and it is the caller's responsibility to manage the sequencing, using the CODE::done:: status of the various UGens.
|
||||
ARGUMENT:: sourceA
|
||||
|
||||
|
||||
The first source buffer
|
||||
|
||||
|
||||
ARGUMENT:: startFrameA
|
||||
|
||||
|
||||
offset into the first source buffer (samples)
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFramesA
|
||||
|
||||
|
||||
number of samples to use from first source buffer
|
||||
|
||||
|
||||
ARGUMENT:: startChanA
|
||||
|
||||
|
||||
starting channel of first source buffer
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChansA
|
||||
|
||||
|
||||
number of channels to process in first source buffer
|
||||
|
||||
|
||||
ARGUMENT:: sourceB
|
||||
|
||||
|
||||
the second source buffer
|
||||
|
||||
|
||||
ARGUMENT:: startFrameB
|
||||
|
||||
|
||||
offset into the second source buffer (samples)
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFramesB
|
||||
|
||||
|
||||
number of samples to process from second buffer
|
||||
|
||||
|
||||
ARGUMENT:: startChanB
|
||||
|
||||
|
||||
starting channel for second buffer
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChansB
|
||||
|
||||
|
||||
number of channels to process in second buffer
|
||||
|
||||
|
||||
ARGUMENT:: destination
|
||||
|
||||
|
||||
buffer for interpolated audio
|
||||
|
||||
|
||||
ARGUMENT:: interpolation
|
||||
|
||||
|
||||
The amount to interpolate between A and B (0-1, 0 = A, 1 = B)
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0.0
|
||||
|
||||
##
|
||||
Maximum: 1.0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the highest of windowSize and (bandwidth - 1) * 2.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: trig
|
||||
A CODE::kr:: signal that will trigger execution
|
||||
|
||||
ARGUMENT:: blocking
|
||||
Whether to execute this process directly on the server command FIFO or delegate to a worker thread. See CODE::processBlocking/process:: for caveats.
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
METHOD:: kr
|
||||
Returns a UGen that reports the progress of the running task when executing in a worker thread. Calling code::scope:: with this can be used for a convinient progress monitor
|
||||
|
||||
METHOD:: cancel
|
||||
Cancels non-blocking processing
|
||||
|
||||
METHOD:: wait
|
||||
When called in the context of a LINK::Classes/Routine:: (it won't work otherwise), will block execution until the processor has finished. This can be convinient for writing sequences of processes more linearly than using lots of nested actions.
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//Didactic:
|
||||
//Make 2 sinewave sources to be interpolated
|
||||
(
|
||||
b = Buffer.loadCollection(s, FloatArray.fill(44100, {|a|(a / pi).sin * 0.1}));
|
||||
c = Buffer.loadCollection(s, FloatArray.fill(44100, {|a|(a / pi / 2).sin * 0.02}));
|
||||
d = Buffer.new
|
||||
)
|
||||
|
||||
//make an sound interpolating their spectrum
|
||||
FluidBufAudioTransport.process(s,b,source2:c,destination:d,interpolation:0.5,action:{"Ding".postln})
|
||||
|
||||
// listen to the source and the result
|
||||
b.play
|
||||
c.play
|
||||
d.play
|
||||
|
||||
// note that the process is quantized by the spectral bins. For an example of the pros and cons of these settings on this given process, please see the real-time FluidAudioTransport helpfile.
|
||||
|
||||
// more interesting sources: two cardboard bowing gestures
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Green-Box641.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Green-Box639.wav"));
|
||||
d = Buffer.new
|
||||
)
|
||||
|
||||
// listen to the source
|
||||
b.play
|
||||
c.play
|
||||
|
||||
// process and listen
|
||||
FluidBufAudioTransport.process(s,b,source2:c,destination:d,interpolation:0.5,action:{"Ding".postln})
|
||||
d.play
|
||||
// try various interpolation factors (0.1 and 0.9 are quite good
|
||||
::
|
||||
@ -1,376 +0,0 @@
|
||||
TITLE:: FluidBufChroma
|
||||
SUMMARY:: A histogram of pitch classes on a Buffer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Classes/FluidChroma,Classes/FluidBufPitch,Classes/FluidBufLoudness,Classes/FluidBufMFCC,Classes/FluidBufSpectralShape,Classes/FluidBufStats,Guides/FluidCorpusManipulationToolkit,Classes/FluidBufMFCC
|
||||
DESCRIPTION::
|
||||
|
||||
|
||||
This class computes a histogram of the energy contained for each pitch class across the analysis frequency range.
|
||||
|
||||
|
||||
|
||||
Also known as a chromagram, this typically allows you to get a contour of how much each semitone is represented in the spectrum over time. The number of chroma bins (and, thus, pitch classes) and the central reference frequency can be adjusted.
|
||||
|
||||
The process will return a single multichannel buffer of STRONG::numChroma:: per input channel. Each frame represents a value, which is every hopSize.
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
Processs the source LINK::Classes/Buffer:: on the LINK::Classes/Server::. CODE::processBlocking:: will execute directly in the server command FIFO, whereas CODE::process:: will delegate to a separate worker thread. The latter is generally only worthwhile for longer-running jobs where you don't wish to tie up the server.
|
||||
|
||||
ARGUMENT:: server
|
||||
The LINK::Classes/Server:: on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
|
||||
|
||||
The index of the buffer to use as the source material to be analysed. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
|
||||
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
|
||||
|
||||
How many frames should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: startChan
|
||||
|
||||
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChans
|
||||
|
||||
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: features
|
||||
|
||||
|
||||
The destination buffer for the STRONG::numChroma:: to be written to.
|
||||
|
||||
|
||||
ARGUMENT:: numChroma
|
||||
|
||||
|
||||
The number of chroma bins per octave. It will determine how many channels are output per input channel.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 2
|
||||
|
||||
##
|
||||
Maximum: CODE::maxNumChroma::
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: ref
|
||||
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
##
|
||||
Maximum: 22000
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: normalize
|
||||
|
||||
|
||||
This flag enables the scaling of the output. It is off (0) by default. (1) will normalise each frame to sum to 1. (2) normalises each frame relative to the loudest chroma bin being 1.
|
||||
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
|
||||
|
||||
The lower frequency included in the analysis, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
|
||||
|
||||
The highest frequency included in the analysis, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: -1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As chroma description relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As chroma description relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts.
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision.
|
||||
|
||||
|
||||
ARGUMENT:: padding
|
||||
|
||||
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default CODE::true::
|
||||
|
||||
ARGUMENT:: action
|
||||
A function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed CODE::[features]:: as an argument.
|
||||
|
||||
RETURNS:: An instance of the processor
|
||||
|
||||
METHOD:: kr
|
||||
Trigger the equivalent behaviour to CODE::processBlocking / process:: from a LINK::Classes/Synth::. Can be useful for expressing a sequence of buffer and data processing jobs to execute. Note that the work still executes on the server command FIFO (not the audio thread), and it is the caller's responsibility to manage the sequencing, using the CODE::done:: status of the various UGens.
|
||||
ARGUMENT:: source
|
||||
|
||||
|
||||
The index of the buffer to use as the source material to be analysed. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
|
||||
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
|
||||
|
||||
How many frames should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: startChan
|
||||
|
||||
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChans
|
||||
|
||||
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: features
|
||||
|
||||
|
||||
The destination buffer for the STRONG::numChroma:: to be written to.
|
||||
|
||||
|
||||
ARGUMENT:: numChroma
|
||||
|
||||
|
||||
The number of chroma bins per octave. It will determine how many channels are output per input channel.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 2
|
||||
|
||||
##
|
||||
Maximum: CODE::maxNumChroma::
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: ref
|
||||
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
##
|
||||
Maximum: 22000
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: normalize
|
||||
|
||||
|
||||
This flag enables the scaling of the output. It is off (0) by default. (1) will normalise each frame to sum to 1. (2) normalises each frame relative to the loudest chroma bin being 1.
|
||||
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
|
||||
|
||||
The lower frequency included in the analysis, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
|
||||
|
||||
The highest frequency included in the analysis, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: -1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As chroma description relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As chroma description relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts.
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision.
|
||||
|
||||
|
||||
ARGUMENT:: padding
|
||||
|
||||
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: trig
|
||||
A CODE::kr:: signal that will trigger execution
|
||||
|
||||
ARGUMENT:: blocking
|
||||
Whether to execute this process directly on the server command FIFO or delegate to a worker thread. See CODE::processBlocking/process:: for caveats.
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
METHOD:: kr
|
||||
Returns a UGen that reports the progress of the running task when executing in a worker thread. Calling code::scope:: with this can be used for a convinient progress monitor
|
||||
|
||||
METHOD:: cancel
|
||||
Cancels non-blocking processing
|
||||
|
||||
METHOD:: wait
|
||||
When called in the context of a LINK::Classes/Routine:: (it won't work otherwise), will block execution until the processor has finished. This can be convinient for writing sequences of processes more linearly than using lots of nested actions.
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// create some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SlideChoirAdd-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// run the process with basic parameters
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufChroma.process(s, b, features: c, windowSize: 4096).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// listen to the source and look at the buffer
|
||||
b.play;
|
||||
c.plot
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create a buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufChroma.process(s, b, features: c, windowSize: 4096).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the buffer: 12 chroma bins for left, then 12 chroma bins for right
|
||||
c.plot(separately:true)
|
||||
::
|
||||
@ -1,112 +0,0 @@
|
||||
TITLE:: FluidBufCompose
|
||||
SUMMARY:: Buffer Compositing Utility
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation, UGens>Buffer
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading, Classes/Buffer
|
||||
|
||||
DESCRIPTION::
|
||||
A FluidBufCompose object provides a flexible utility for combining the contents of buffers on the server. It can be used for thing like mixing down multichannel buffers, or converting from left-right stereo to mid-side. It is used extensively in all the example code of LINK::Guides/FluidCorpusManipulation:: as part of the FluCoMa project. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
At its most simple, the object copies the content of a source buffer into a destination buffer. The flexibility comes from the various flags controlling which portions and channels of the source to use, and by applying gains (which can be positive or negative) to the source data and the portion of the destination that would be overwritten.
|
||||
|
||||
The algorithm takes a srcBuf, and writes the information at the provided dstBuf. These buffer arguments can all point to the same buffer, which gives great flexibility in transforming and reshaping.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This method triggers the compositing.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The bufNum of the source buffer.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
The starting point (in samples) from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
The duration (in samples) to copy from the source buffer. The default (-1) copies the full lenght of the buffer.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
The first channel from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
The number of channels from which to copy in the source buffer. This parameter will wrap around the number of channels in the source buffer. The default (-1) copies all of the buffer's channel.
|
||||
|
||||
ARGUMENT:: gain
|
||||
The gain applied to the samples to be copied from the source buffer.
|
||||
|
||||
ARGUMENT:: destination
|
||||
The bufNum of the destination buffer.
|
||||
|
||||
ARGUMENT:: destStartFrame
|
||||
The time offset (in samples) in the destination buffer to start writing the source at. The destination buffer will be resized if the portion to copy is overflowing.
|
||||
|
||||
ARGUMENT:: destStartChan
|
||||
The channel offest in the destination buffer to start writing the source at. The destination buffer will be resized if the number of channels to copy is overflowing.
|
||||
|
||||
ARGUMENT:: destGain
|
||||
The gain applied to the samples in the region of the destination buffer over which the source is to be copied. The default value (0) will overwrite that section of the destination buffer, and a value of 1.0 would sum the source to the material that was present.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and destination instance variables have been updated on the client side. The function will be passed destination as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
DISCUSSION::
|
||||
It is important to understand the rules used for determining the final desintinaiton buffer dimensions to get the most out of this object. If needs be, the destination buffer will be resized to the maxima of the requsted source numFrames and numChannels. Frames will be written up to the limit of actually available samples (meaning you can create zero padding); channels will be written modulo the available channels, taking into account the channel offsets, meaning you can have channels repeat or loop into the source buffer's channels. See the examples below.
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// load some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
d = Buffer.new(s);
|
||||
)
|
||||
|
||||
// with basic params (basic summing of each full buffer in all dimensions)
|
||||
(
|
||||
Routine{
|
||||
FluidBufCompose.process(s, source: b, destination: d);
|
||||
FluidBufCompose.process(s, source: c, destination: d, destGain: 1.0);
|
||||
s.sync;
|
||||
d.query;
|
||||
d.play;
|
||||
}.play;
|
||||
)
|
||||
//constructing a mono buffer, with a quiet punch from the synth, with a choked piano resonance from the left channel
|
||||
(
|
||||
Routine{
|
||||
d.free; d = Buffer.new(s);
|
||||
FluidBufCompose.process(s, source: b, numFrames: 9000, gain: 0.5, destination: d);
|
||||
FluidBufCompose.process(s, source: c, startFrame:30000, numFrames:44100, numChans:1, gain:0.9, destination: d, destGain: 1.0).wait;
|
||||
d.query;
|
||||
d.play;
|
||||
}.play
|
||||
)
|
||||
//constructing a stereo buffer, with the end of the mono synth in both channels, with a piano resonance in swapped stereo
|
||||
(
|
||||
Routine{
|
||||
d.free; d = Buffer.new(s);
|
||||
FluidBufCompose.process(s, source: b, startFrame: 441000, numChans: 2, gain: 0.6, destination: d);
|
||||
FluidBufCompose.process(s, source: c, numFrames: 78000, startChan: 1, numChans: 2, gain: 0.5, destStartFrame: 22050, destination: d, destGain: 1.0).wait;
|
||||
d.query;
|
||||
d.play;
|
||||
}.play
|
||||
)
|
||||
//constructing a one second buffer: the first second of each buffer, the mono synth on the right, the piano on the left
|
||||
(
|
||||
Routine{
|
||||
d.free; d = Buffer.new(s);
|
||||
FluidBufCompose.process(s, source: b, numFrames: 44100, numChans: 1, destStartChan: 1, destination: d);
|
||||
FluidBufCompose.process(s, source: c, numFrames:44100, numChans:1, destination: d, destGain: 1.0).wait;
|
||||
d.query;
|
||||
d.play;
|
||||
}.play
|
||||
)
|
||||
::
|
||||
@ -1,120 +0,0 @@
|
||||
TITLE:: FluidBufFlatten
|
||||
summary:: Flatten a multichannel buffer on the server
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/Buffer, Classes/FluidBufCompose, Classes/FluidBufSelect, Classes/FluidBufSelectEvery
|
||||
|
||||
DESCRIPTION::
|
||||
Flatten a multichannel link::Classes/Buffer:: to a single channel. This can be useful for constructing n-dimensional data points for use with link::Classes/FluidDataSet::
|
||||
|
||||
The code::axis:: determines how the flattening is arranged. The default value, 1, flattens channel-wise, such that (if we imagine channels are rows, time positions are columns):
|
||||
|
||||
table::
|
||||
## a 1 || a 2 || a 3
|
||||
## b 1 || b 2 || b 3
|
||||
## c 1 || c 2 || c 3
|
||||
::
|
||||
|
||||
becomes
|
||||
|
||||
table::
|
||||
## a 1 || b 1 || c 1 || a 2 || b 2 || c 2 || a 3 || b 3 || c 3
|
||||
::
|
||||
|
||||
whereas with code::axis = 0:: we get
|
||||
|
||||
table::
|
||||
## a 1 || a 2 || a 3 || b 1 || b 2 || b 3 || c 1 || c 2 || c 3
|
||||
::
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
private::new1
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
|
||||
Run the process on the given sever, and perfrom code::action:: when done
|
||||
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run
|
||||
|
||||
ARGUMENT:: source
|
||||
The link::Classes/Buffer:: to flatten
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the source should the flattening process start, in samples.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel source buffers, which channel to start processing at.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel source buffers, how many channels should be processed.
|
||||
|
||||
ARGUMENT:: destination
|
||||
The link::Classes/Buffer:: to write the flattened data to
|
||||
|
||||
ARGUMENT:: axis
|
||||
Whether to group points channel-wise or frame-wise
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
Runs when processing is complete
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//FluidBufPitch is useful to illustrate the effect of this, because the pitch and confidence values are easily distinguishable
|
||||
|
||||
(
|
||||
~path = FluidFilesPath();
|
||||
~randomsoundfile = SoundFile.collect(~path +/+ '*').choose;
|
||||
b = Buffer.read(s,~randomsoundfile.path,action:{"Sound Loaded".postln});
|
||||
~pitchdata = Buffer.new;
|
||||
~flatdata = Buffer.new;
|
||||
)
|
||||
|
||||
//Pitch analysis, writes pitches as frequencies to chan 0, confidences [0-1] to chan 1
|
||||
FluidBufPitch.process(s,b,numFrames:512 * 10,numChans:1,features:~pitchdata,action:{"Pitch Analysis Done".postln});
|
||||
|
||||
// Flatten and print the flat buffer. We expect to see larger numbers (20-2000) interleaved with smaller (0-1)
|
||||
(
|
||||
FluidBufFlatten.process(s,~pitchdata, destination: ~flatdata, axis:1, action:{
|
||||
~flatdata.loadToFloatArray(action:{ |a|
|
||||
a.postln;
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
//changing the axis, we see all large numbers first
|
||||
(
|
||||
FluidBufFlatten.process(s,~pitchdata, destination:~flatdata, axis:0, action:{
|
||||
~flatdata.loadToFloatArray(action:{ |a|
|
||||
a.postln;
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
//adding the source range make this processor very powerful, but can be quite confusing
|
||||
//here we take only one frame starting at the second one (0-counting)
|
||||
(
|
||||
FluidBufFlatten.process(s,~pitchdata,startFrame: 1,numFrames: 1, destination:~flatdata, action:{
|
||||
~flatdata.loadToFloatArray(action:{ |a|
|
||||
a.postln;
|
||||
})
|
||||
})
|
||||
)
|
||||
//and here we take only the confidences
|
||||
(
|
||||
FluidBufFlatten.process(s,~pitchdata, startChan: 1, destination:~flatdata, action:{
|
||||
~flatdata.loadToFloatArray(action:{ |a|
|
||||
a.postln;
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
::
|
||||
@ -1,124 +0,0 @@
|
||||
TITLE:: FluidBufLoudness
|
||||
SUMMARY:: A Loudness and True-Peak Descriptor on a Buffer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements two loudness descriptors, computing the true peak of the signal as well as applying the filters proposed by broadcasting standards to emulate the perception of amplitude. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a multichannel buffer with two channels per input channel, one for loudness and one for the true peak value of the frame, both in dBfs. More information on broadcasting standardisation of loudness measurement is available at the reference page FOOTNOTE::https://tech.ebu.ch/docs/tech/tech3341.pdf:: and in more musician-friendly explantions here FOOTNOTE::http://designingsound.org/2013/02/06/loudness-and-metering-part-1/::. Each sample represents a value, which is every hopSize. Its sampling rate is STRONG::sourceSR / hopSize::.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the loudness descriptor to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be described. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
ARGUMENT:: features
|
||||
The destination buffer for the loudness descriptors.
|
||||
|
||||
ARGUMENT:: kWeighting
|
||||
A flag to switch the perceptual model of loudness. On by default, removing it makes the algorithm more CPU efficient by reverting to a simple RMS of the frame.
|
||||
|
||||
ARGUMENT:: truePeak
|
||||
A flag to switch the computation of TruePeak. On by default, removing it makes the algorithm more CPU efficient by reverting to a simple absolute peak of the frame.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The size of the window on which the computation is done. By default 1024 to be similar with all other FluCoMa objects, the EBU specifies other values as per the examples below.
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
How much the buffered window moves forward, in samples. By default 512 to be similar with all other FluCoMa objects, the EBU specifies other values as per the examples below.
|
||||
|
||||
ARGUMENT:: padding
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [features] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// create a buffer with a short clicking sinusoidal burst (220Hz) starting at frame 8192 for 1024 frames
|
||||
(
|
||||
b = Buffer.sendCollection(s, (Array.fill(8192,{0}) ++ (Signal.sineFill(1203,[0,0,0,0,0,1],[0,0,0,0,0,0.5pi]).takeThese({|x,i|i>1023})) ++ Array.fill(8192,{0})));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// listen to the source and look at the buffer
|
||||
b.play; b.plot;
|
||||
|
||||
// run the process with basic parameters
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufLoudness.process(s, source:b, features: c).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the analysis
|
||||
c.plot(minval:-130, maxval:6)
|
||||
|
||||
// The values are interleaved [loudness,truepeak] in the buffer as they are on 2 channels: to get to the right frame, divide the SR of the input by the hopSize, then multiply by 2 because of the channel interleaving
|
||||
// here we are querying from one frame before (the signal starts at 8192, which is frame 16 (8192/512), therefore starting the query at frame 15, which is index 30.
|
||||
|
||||
c.getn(30,10,{|x|x.postln})
|
||||
|
||||
// observe that the first frame is silent, as expected. We can appreciate the overshoot of TruePeak of a full range sinewave starting on the second sample (fourth item in the list).
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create a buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them with EBU standard Instant Loudness of
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufLoudness.process(s, b, features: c, windowSize: 17640, hopSize:4410).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the buffer: [loudness,truepeak] for left then [loudness,truepeak] for right
|
||||
c.plot(minval:-40, maxval:0)
|
||||
::
|
||||
@ -1,126 +0,0 @@
|
||||
TITLE:: FluidBufMFCC
|
||||
SUMMARY:: Mel-Frequency Cepstral Coefficients as Spectral Descriptors on a Buffer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading, Classes/FluidBufMelBands
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a classic spectral descriptor, the Mel-Frequency Cepstral Coefficients (https://en.wikipedia.org/wiki/Mel-frequency_cepstrum). The input is first filtered in to STRONG::numBands:: perceptually-spaced bands, as in LINK::Classes/FluidMelBands::. It is then analysed into STRONG::numCoeffs:: number of cepstral coefficients. It has the advantage of being amplitude invariant, except for the first coefficient. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a single multichannel buffer of STRONG::numCoeffs:: per input channel. Each frame represents a value, which is every hopSize.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the spectral shape descriptors to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be described through the various descriptors. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
ARGUMENT:: features
|
||||
The destination buffer for the numCoeffs coefficients describing the spectral shape.
|
||||
|
||||
ARGUMENT:: numCoeffs
|
||||
The number of cepstral coefficients to be outputed. It will decide how many channels are produce per channel of the source.
|
||||
|
||||
ARGUMENT:: numBands
|
||||
The number of bands that will be perceptually equally distributed between STRONG::minFreq:: and STRONG::maxFreq::.
|
||||
|
||||
ARGUMENT:: startCoeff
|
||||
The lowest index of the output cepstral coefficient, zero-counting.
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
The lower boundary of the lowest band of the model, in Hz.
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
The highest boundary of the highest band of the model, in Hz.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As MFCC computation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As MFCC computation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: padding
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [features] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// create some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// run the process with basic parameters
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufMFCC.process(s, b, features: c).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// listen to the source and look at the buffer
|
||||
b.play;
|
||||
c.plot(separately:true)
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create a buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufMFCC.process(s, b, numCoeffs:5, features: c).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the buffer: 5 coefs for left, then 5 coefs for right (the first of each is linked to the loudness)
|
||||
c.plot(separately:true)
|
||||
::
|
||||
@ -1,127 +0,0 @@
|
||||
TITLE:: FluidBufMelBands
|
||||
SUMMARY:: A Perceptually Spread Spectral Contour Descriptor on a Buffer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading, Classes/FluidBufMFCC
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a spectral shape descriptor where the amplitude is given for a number of equally spread perceptual bands. The spread is based on the Mel scale (https://en.wikipedia.org/wiki/Mel_scale) which is one of the first attempt to mimic pitch perception scientifically. This implementation allows to select the range and number of bands dynamically. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a single multichannel buffer of STRONG::numBands:: per input channel. Each frame represents a value, which is every hopSize.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the spectral shape descriptors to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be described through the various descriptors. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
ARGUMENT:: features
|
||||
The destination buffer for the STRONG::numBands:: amplitudes describing the spectral shape.
|
||||
|
||||
ARGUMENT:: numBands
|
||||
The number of bands that will be perceptually equally distributed between STRONG::minFreq:: and STRONG::maxFreq::. It will decide how many channels are produce per channel of the source.
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
The lower boundary of the lowest band of the model, in Hz.
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
The highest boundary of the highest band of the model, in Hz.
|
||||
|
||||
ARGUMENT:: normalize
|
||||
This flag enables the scaling of the output to preserve the energy of the window. It is on (1) by default.
|
||||
|
||||
ARGUMENT:: scale
|
||||
This flag sets the scaling of the output value. It is either linear (0, by default) or in dB (1).
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As spectral description relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As spectral description relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: padding
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [features] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// create some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// run the process with basic parameters
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufMelBands.process(s, b, features: c, numBands:10).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// listen to the source and look at the buffer
|
||||
b.play;
|
||||
c.plot
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create a buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufMelBands.process(s, b, features: c, numBands:10).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the buffer: 10 bands for left, then 10 bands for right
|
||||
c.plot(separately:true)
|
||||
::
|
||||
@ -1,629 +0,0 @@
|
||||
TITLE:: FluidBufNMF
|
||||
SUMMARY:: Buffer-Based Non-Negative Matrix Factorisation on Spectral Frames
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Classes/FluidNMFMatch,Classes/FluidNMFFilter,Guides/FluidCorpusManipulationToolkit,Classes/FluidNMFMatch,Classes/FluidNMFFilter
|
||||
DESCRIPTION::
|
||||
|
||||
|
||||
Decomposes the spectrum of a sound into a number of components using Non-Negative Matrix Factorisation (NMF)
|
||||
|
||||
|
||||
|
||||
NMF has been a popular technique in signal processing research for things like source separation and transcription (see Smaragdis and Brown, Non-Negative Matrix Factorization for Polyphonic Music Transcription.), although its creative potential is so far relatively unexplored.
|
||||
|
||||
The algorithm takes a buffer in and divides it into a number of components, determined by the components argument. It works iteratively, by trying to find a combination of spectral templates ('bases') and envelopes ('activations') that yield the original magnitude spectrogram when added together. By and large, there is no unique answer to this question (i.e. there are different ways of accounting for an evolving spectrum in terms of some set of templates and envelopes). In its basic form, NMF is a form of unsupervised learning: it starts with some random data and then converges towards something that minimizes the distance between its generated data and the original:it tends to converge very quickly at first and then level out. Fewer iterations mean less processing, but also less predictable results.
|
||||
|
||||
DEFINITIONLIST::
|
||||
## The object can return either or all of the following:
|
||||
||
|
||||
LIST::
|
||||
##
|
||||
a spectral contour of each component in the form of a magnitude spectrogram (called a basis in NMF lingo);
|
||||
|
||||
##
|
||||
an amplitude envelope of each component in the form of gains for each consecutive frame of the underlying spectrogram (called an activation in NMF lingo);
|
||||
|
||||
##
|
||||
an audio reconstruction of each components in the time domain.
|
||||
|
||||
::
|
||||
::
|
||||
The bases and activations can be used to make a kind of vocoder based on what NMF has 'learned' from the original data. Alternatively, taking the matrix product of a basis and an activation will yield a synthetic magnitude spectrogram of a component (which could be reconsructed, given some phase informaiton from somewhere).
|
||||
|
||||
Some additional options and flexibility can be found through combinations of the basesMode and actMode arguments. If these flags are set to 1, the object expects to be supplied with pre-formed spectra (or envelopes) that will be used as seeds for the decomposition, providing more guided results. When set to 2, the supplied buffers won't be updated, so become templates to match against instead. Note that having both bases and activations set to 2 doesn't make sense, so the object will complain.
|
||||
|
||||
DEFINITIONLIST::
|
||||
## If supplying pre-formed data, it's up to the user to make sure that the supplied buffers are the right size:
|
||||
||
|
||||
LIST::
|
||||
##
|
||||
bases must be frames and channels
|
||||
|
||||
##
|
||||
activations must be frames and channels
|
||||
|
||||
::
|
||||
::
|
||||
In this implementation, the components are reconstructed by masking the original spectrum, such that they will sum to yield the original sound.
|
||||
|
||||
The whole process can be related to a channel vocoder where, instead of fixed bandpass filters, we get more complex filter shapes that are learned from the data, and the activations correspond to channel envelopes.
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
Processs the source LINK::Classes/Buffer:: on the LINK::Classes/Server::. CODE::processBlocking:: will execute directly in the server command FIFO, whereas CODE::process:: will delegate to a separate worker thread. The latter is generally only worthwhile for longer-running jobs where you don't wish to tie up the server.
|
||||
|
||||
ARGUMENT:: server
|
||||
The LINK::Classes/Server:: on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
|
||||
|
||||
The index of the buffer to use as the source material to be decomposed through the NMF process. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
|
||||
|
||||
Where in the srcBuf should the NMF process start, in sample.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
|
||||
|
||||
How many frames should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: startChan
|
||||
|
||||
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChans
|
||||
|
||||
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: resynth
|
||||
|
||||
|
||||
The index of the buffer where the different reconstructed components will be reconstructed. The buffer will be resized to channels and lenght. If is provided, the reconstruction will not happen.
|
||||
|
||||
|
||||
ARGUMENT:: bases
|
||||
|
||||
|
||||
The index of the buffer where the different bases will be written to and/or read from: the behaviour is set in the following argument. If is provided, no bases will be returned.
|
||||
|
||||
|
||||
ARGUMENT:: basesMode
|
||||
|
||||
|
||||
This flag decides of how the basis buffer passed as the previous argument is treated.
|
||||
|
||||
|
||||
ARGUMENT:: activations
|
||||
|
||||
|
||||
The index of the buffer where the different activations will be written to and/or read from: the behaviour is set in the following argument. If is provided, no activation will be returned.
|
||||
|
||||
|
||||
ARGUMENT:: actMode
|
||||
|
||||
|
||||
This flag decides of how the activation buffer passed as the previous argument is treated.
|
||||
|
||||
|
||||
ARGUMENT:: components
|
||||
|
||||
|
||||
The number of elements the NMF algorithm will try to divide the spectrogram of the source in.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: iterations
|
||||
|
||||
|
||||
The NMF process is iterative, trying to converge to the smallest error in its factorisation. The number of iterations will decide how many times it tries to adjust its estimates. Higher numbers here will be more CPU expensive, lower numbers will be more unpredictable in quality.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As NMF relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As NMF relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts.
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default CODE::true::
|
||||
|
||||
ARGUMENT:: action
|
||||
A function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed CODE::[features]:: as an argument.
|
||||
|
||||
RETURNS:: An instance of the processor
|
||||
|
||||
METHOD:: kr
|
||||
Trigger the equivalent behaviour to CODE::processBlocking / process:: from a LINK::Classes/Synth::. Can be useful for expressing a sequence of buffer and data processing jobs to execute. Note that the work still executes on the server command FIFO (not the audio thread), and it is the caller's responsibility to manage the sequencing, using the CODE::done:: status of the various UGens.
|
||||
ARGUMENT:: source
|
||||
|
||||
|
||||
The index of the buffer to use as the source material to be decomposed through the NMF process. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
|
||||
|
||||
Where in the srcBuf should the NMF process start, in sample.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
|
||||
|
||||
How many frames should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: startChan
|
||||
|
||||
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numChans
|
||||
|
||||
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
|
||||
ARGUMENT:: resynth
|
||||
|
||||
|
||||
The index of the buffer where the different reconstructed components will be reconstructed. The buffer will be resized to channels and lenght. If is provided, the reconstruction will not happen.
|
||||
|
||||
|
||||
ARGUMENT:: bases
|
||||
|
||||
|
||||
The index of the buffer where the different bases will be written to and/or read from: the behaviour is set in the following argument. If is provided, no bases will be returned.
|
||||
|
||||
|
||||
ARGUMENT:: basesMode
|
||||
|
||||
|
||||
This flag decides of how the basis buffer passed as the previous argument is treated.
|
||||
|
||||
|
||||
ARGUMENT:: activations
|
||||
|
||||
|
||||
The index of the buffer where the different activations will be written to and/or read from: the behaviour is set in the following argument. If is provided, no activation will be returned.
|
||||
|
||||
|
||||
ARGUMENT:: actMode
|
||||
|
||||
|
||||
This flag decides of how the activation buffer passed as the previous argument is treated.
|
||||
|
||||
|
||||
ARGUMENT:: components
|
||||
|
||||
|
||||
The number of elements the NMF algorithm will try to divide the spectrogram of the source in.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: iterations
|
||||
|
||||
|
||||
The NMF process is iterative, trying to converge to the smallest error in its factorisation. The number of iterations will decide how many times it tries to adjust its estimates. Higher numbers here will be more CPU expensive, lower numbers will be more unpredictable in quality.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As NMF relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As NMF relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts.
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: trig
|
||||
A CODE::kr:: signal that will trigger execution
|
||||
|
||||
ARGUMENT:: blocking
|
||||
Whether to execute this process directly on the server command FIFO or delegate to a worker thread. See CODE::processBlocking/process:: for caveats.
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
METHOD:: kr
|
||||
Returns a UGen that reports the progress of the running task when executing in a worker thread. Calling code::scope:: with this can be used for a convinient progress monitor
|
||||
|
||||
METHOD:: cancel
|
||||
Cancels non-blocking processing
|
||||
|
||||
METHOD:: wait
|
||||
When called in the context of a LINK::Classes/Routine:: (it won't work otherwise), will block execution until the processor has finished. This can be convinient for writing sequences of processes more linearly than using lots of nested actions.
|
||||
|
||||
EXAMPLES::
|
||||
STRONG::A didactic example::
|
||||
CODE::
|
||||
|
||||
// =============== decompose some sounds ===============
|
||||
|
||||
// let's decompose the drum loop that comes with the FluCoMa extension:
|
||||
~drums = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
|
||||
// hear the original mono sound file to know what we're working with
|
||||
~drums.play;
|
||||
|
||||
// an empty buffer for the decomposed components to be written into:
|
||||
~resynth = Buffer(s);
|
||||
|
||||
// how many components we want FluidBufNMF to try to decompose the buffer into:
|
||||
~n_components = 2;
|
||||
|
||||
// process it:
|
||||
FluidBufNMF.processBlocking(s,~drums,resynth:~resynth,components:~n_components,action:{"done".postln;});
|
||||
|
||||
// once it is done, play the separated components one by one (with a second of silence in between)
|
||||
(
|
||||
fork{
|
||||
~n_components.do{
|
||||
arg i;
|
||||
"decomposed part #%".format(i+1).postln;
|
||||
{
|
||||
PlayBuf.ar(~n_components,~resynth,BufRateScale.ir(~resynth),doneAction:2)[i].dup;
|
||||
}.play;
|
||||
(~drums.duration + 1).wait;
|
||||
}
|
||||
};
|
||||
)
|
||||
|
||||
// ======== now let's try it with three components. =========
|
||||
// make a guess as to what you think you'll hear
|
||||
|
||||
~n_components = 3;
|
||||
// process it:
|
||||
FluidBufNMF.processBlocking(s,~drums,resynth:~resynth,components:~n_components,action:{"done".postln;});
|
||||
|
||||
(
|
||||
fork{
|
||||
~n_components.do{
|
||||
arg i;
|
||||
"decomposed part #%".format(i+1).postln;
|
||||
{
|
||||
PlayBuf.ar(~n_components,~resynth,BufRateScale.ir(~resynth),doneAction:2)[i].dup;
|
||||
}.play;
|
||||
(~drums.duration + 1).wait;
|
||||
}
|
||||
};
|
||||
)
|
||||
|
||||
// you may have guessed that it would separate out the three components into: (1) snare, (2) hihat, and (3) kick
|
||||
// and it might have worked! but it may not have, and it won't provide the same result every time because it
|
||||
// starts each process from a stochastic state (you can seed this state if you want...see below).
|
||||
|
||||
// ====== bases and activations ========
|
||||
|
||||
// first, let's make two new buffers called...
|
||||
~bases = Buffer(s);
|
||||
~activations = Buffer(s);
|
||||
~n_components = 2; // return to 2 components for this example
|
||||
|
||||
// and we'll explicitly pass these into the process
|
||||
FluidBufNMF.processBlocking(s,~drums,bases:~bases,activations:~activations,resynth:~resynth,components:~n_components,action:{"done".postln;});
|
||||
|
||||
// now we can plot them (because this process starts from a stochastic state, your results may vary!):
|
||||
FluidWaveform(~drums,featureBuffer:~activations,bounds:Rect(0,0,1200,300));
|
||||
// the bases are a like a spectral template that FluidBufNMF has found in the source buffer
|
||||
// in one you should see one spectrum that resembles a snare spectrum (the resonant tone of the snare
|
||||
// in the mid range) and another that resembles the kick + hihat we heard earlier (a large peak in the very
|
||||
// low register and some shimmery higher stuff)
|
||||
|
||||
FluidWaveform(featureBuffer:~bases,bounds:Rect(0,0,1200,300));
|
||||
// the activations are the corresponding loudness envelope of each base above. It should like an amplitude
|
||||
// envelope follower of the drum hits in the corresponding bases.
|
||||
|
||||
// FluidBufNMF then uses the individual bases with their corresponding activations to resynthesize the sound of just
|
||||
// component.
|
||||
// the buffer passed to `resynth` will have one channel for each component you've requested
|
||||
|
||||
~resynth.numChannels
|
||||
~resynth.play;
|
||||
|
||||
// ======== to further understand NMF's bases and activations, consider one more object: FluidNMFFilter ==========
|
||||
// FluidNMFFilter will use the bases (spectral templates) of a FluidBufNMF analysis to filter (i.e., decompose) real-time audio
|
||||
|
||||
// for example, if we use the bases from the ~drums analysis above, it will separate the snare from the kick & hi hat like before
|
||||
// this time you'll hear one in each stereo channel (again, results may vary)
|
||||
|
||||
(
|
||||
{
|
||||
var src = PlayBuf.ar(1,~drums,BufRateScale.ir(~drums),doneAction:2);
|
||||
var sig = FluidNMFFilter.ar(src,~bases,2);
|
||||
sig;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// if we play a different source through FluidNMFFilter, it will try to decompose that real-time signal according to the bases
|
||||
// it is given (in our case the bases from the drum loop)
|
||||
~song = Buffer.readChannel(s,FluidFilesPath("Tremblay-BeatRemember.wav"),channels:[0]);
|
||||
|
||||
(
|
||||
{
|
||||
var src = PlayBuf.ar(1,~song,BufRateScale.ir(~song),doneAction:2);
|
||||
var sig = FluidNMFFilter.ar(src,~bases,2);
|
||||
sig;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// ========= the activations could also be used as an envelope through time ===========
|
||||
(
|
||||
{
|
||||
var activation = PlayBuf.ar(2,~activations,BufRateScale.ir(~activations),doneAction:2);
|
||||
var sig = WhiteNoise.ar(0.dbamp) * activation;
|
||||
sig;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// note that the samplerate of the ~activations buffer is not a usual one...
|
||||
~activations.sampleRate;
|
||||
// this is because each frame in this buffer doesn't correspond to one audio sample, but instead to one
|
||||
// hopSize, since these values are derived from an FFT analysis
|
||||
// so it is important to use BufRateScale (as seen above) in order to make sure they play back at the
|
||||
// correct rate
|
||||
|
||||
// if we control the amplitude of the white noise *and* send it through FluidNMFFilter, we'll get something
|
||||
// somewhat resembles both the spectral template and loudness envelope of the bases of the original
|
||||
// (of course it's also good to note that the combination of the *actual* bases and activations is how
|
||||
// FluidBufNMF creates the channels in the resynth buffer which will sound much better than this
|
||||
// filtered WhiteNoise version)
|
||||
(
|
||||
{
|
||||
var activation = PlayBuf.ar(2,~activations,BufRateScale.ir(~activations),doneAction:2);
|
||||
var sig = WhiteNoise.ar(0.dbamp);
|
||||
sig = FluidNMFFilter.ar(sig,~bases,2) * activation;
|
||||
sig;
|
||||
}.play;
|
||||
)
|
||||
|
||||
::
|
||||
STRONG::Fixed Bases: The process can be trained, and the learnt bases or activations can be used as templates.::
|
||||
|
||||
CODE::
|
||||
|
||||
//set some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
|
||||
~originalNMF = Buffer.new(s);
|
||||
~bases = Buffer.new(s);
|
||||
~trainedBases = Buffer.new(s);
|
||||
~activations = Buffer.new(s);
|
||||
~final = Buffer.new(s);
|
||||
~spectralshapes = Buffer.new(s);
|
||||
~stats = Buffer.new(s);
|
||||
~sortedNMF = Buffer.new(s);
|
||||
)
|
||||
|
||||
b.play
|
||||
|
||||
// train using the first 2 seconds of the sound file
|
||||
(
|
||||
Routine {
|
||||
FluidBufNMF.process(s,b,0,44100*5,0,1, ~originalNMF, ~bases, components:10).wait;
|
||||
~originalNMF.query;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// listen to the 10 components across the stereo image
|
||||
{Splay.ar(PlayBuf.ar(10, ~originalNMF))}.play
|
||||
|
||||
// plot the bases
|
||||
~bases.plot
|
||||
|
||||
// find the component that has the picking sound by checking the median spectral centroid
|
||||
(
|
||||
FluidBufSpectralShape.process(s, ~originalNMF, features: ~spectralshapes, action:{
|
||||
|shapes|FluidBufStats.process(s,shapes,stats:~stats, action:{
|
||||
|stats|stats.getn(0, (stats.numChannels * stats.numFrames) ,{
|
||||
|x| ~centroids = x.select({
|
||||
|item, index| (index.mod(7) == 0) && (index.div(70) == 5);
|
||||
})
|
||||
})
|
||||
})
|
||||
});
|
||||
)
|
||||
|
||||
//what is happening there? We run the spectralshapes on the buffer of 10 components from the nmf. See the structure of that buffer:
|
||||
~originalNMF.query
|
||||
//10 channel are therefore giving 70 channels: the 7 shapes of component0, then 7 shapes of compoenent1, etc
|
||||
~spectralshapes.query
|
||||
// we then run the bufstats on them. Each channel, which had a time series (an envelope) of each descriptor, is reduced to 7 frames
|
||||
~stats.query
|
||||
// we then need to retrieve the values that are where we want: the first of every 7 for the centroid, and the 6th frame of them as we want the median. Because we retrieve the values in an interleave format, the select function gets a bit tricky but we get the following values:
|
||||
~centroids.postln
|
||||
|
||||
// we then copy the basis with the highest median centroid to a channel, and all the other bases to the other channel, of a 2-channel bases for decomposition
|
||||
(
|
||||
z = (0..9);
|
||||
[z.removeAt(~centroids.maxIndex)].do{|chan|FluidBufCompose.process(s, ~bases, startChan: chan, numChans: 1, destination: ~trainedBases, destGain:1)};
|
||||
z.postln;
|
||||
z.do({|chan| FluidBufCompose.process(s, ~bases, startChan:chan, numChans: 1, destStartChan: 1, destination: ~trainedBases, destGain:1)});
|
||||
)
|
||||
~trainedBases.plot
|
||||
|
||||
//process the whole file, splitting it with the 2 trained bases
|
||||
(
|
||||
Routine{
|
||||
FluidBufNMF.process(s, b, resynth: ~sortedNMF, bases: ~trainedBases, basesMode: 2, components:2).wait;
|
||||
~originalNMF.query;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// play the result: pick on the left, sustain on the right!
|
||||
{PlayBuf.ar(2,~sortedNMF)}.play
|
||||
|
||||
// it even null-sums
|
||||
{(PlayBuf.ar(2,~sortedNMF,doneAction:2).sum)-(PlayBuf.ar(1,b,doneAction:2))}.play
|
||||
::
|
||||
|
||||
STRONG::Updating Bases: The process can update bases provided as seed.::
|
||||
|
||||
CODE::
|
||||
(
|
||||
// create buffers
|
||||
b = Buffer.alloc(s,44100);
|
||||
c = Buffer.alloc(s, 44100);
|
||||
d = Buffer.new(s);
|
||||
e = Buffer.alloc(s,513,3);
|
||||
f = Buffer.new(s);
|
||||
g = Buffer.new(s);
|
||||
)
|
||||
|
||||
(
|
||||
// fill them with 2 clearly segregated sine waves and composite a buffer where they are consecutive
|
||||
Routine {
|
||||
b.sine2([500],[1], false, false);
|
||||
c.sine2([5000],[1],false, false);
|
||||
s.sync;
|
||||
FluidBufCompose.process(s,b, destination:d);
|
||||
FluidBufCompose.process(s,c, destStartFrame:44100, destination:d, destGain:1);
|
||||
s.sync;
|
||||
d.query;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// check
|
||||
d.plot
|
||||
d.play //////(beware !!!! loud!!!)
|
||||
|
||||
(
|
||||
//make a seeding basis of 3 components:
|
||||
var highpass, lowpass, direct;
|
||||
highpass = Array.fill(513,{|i| (i < 50).asInteger});
|
||||
lowpass = 1 - highpass;
|
||||
direct = Array.fill(513,0.1);
|
||||
e.setn(0,[highpass, lowpass, direct].flop.flat);
|
||||
)
|
||||
|
||||
//check the basis: a steep lowpass, a steep highpass, and a small DC
|
||||
e.plot
|
||||
e.query
|
||||
|
||||
(
|
||||
// use the seeding basis, without updating
|
||||
Routine {
|
||||
FluidBufNMF.process(s, d, resynth:f, bases: e, basesMode: 2, activations:g, components:3).wait;
|
||||
e.query;
|
||||
f.query;
|
||||
g.query;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the resynthesised separated signal
|
||||
f.plot;
|
||||
|
||||
// look at the bases that have not changed
|
||||
e.plot;
|
||||
|
||||
// look at the activations
|
||||
g.plot;
|
||||
|
||||
(
|
||||
// use the seeding bases, with updating this time
|
||||
Routine {
|
||||
FluidBufNMF.process(s, d, resynth:f, bases: e, basesMode: 1, activations:g, components:3).wait;
|
||||
e.query;
|
||||
f.query;
|
||||
g.query;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the resynthesised separated signal
|
||||
f.plot;
|
||||
|
||||
// look at the bases that have now updated in place (with the 3rd channel being more focused
|
||||
e.plot;
|
||||
|
||||
// look at the activations (sharper 3rd component at transitions)
|
||||
g.plot;
|
||||
::
|
||||
@ -1,100 +0,0 @@
|
||||
TITLE:: FluidBufNMFSeed
|
||||
summary:: Non-Negative Double Singular Value Decomposition on a Buffer
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidBufNMF
|
||||
|
||||
DESCRIPTION::
|
||||
Find Initial Bases and Activations for FluidBufNMF via Non-Negative Double Singular Value Decomposition .
|
||||
|
||||
See http://nimfa.biolab.si/nimfa.methods.seeding.nndsvd.html
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the decomposition to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be decomposed through the NMF process. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: bases
|
||||
The index of the buffer where the different bases will be written to and/or read from: the behaviour is set in the following argument.
|
||||
|
||||
ARGUMENT:: activations
|
||||
The index of the buffer where the different activations will be written to and/or read from: the behaviour is set in the following argument.
|
||||
|
||||
ARGUMENT:: minComponents
|
||||
Minimum number of estimated components
|
||||
|
||||
ARGUMENT:: maxComponents
|
||||
Maximum number of estimated components
|
||||
|
||||
ARGUMENT:: coverage
|
||||
Fraction (0 to 1) of information preserved in the decomposition
|
||||
|
||||
ARGUMENT:: method
|
||||
The method used for the decomposition. Options are:
|
||||
|
||||
table::
|
||||
## 0 || NMF-SVD || faster
|
||||
## 1 || NNDSVDar || more accurate, fill in the zero elements with random values
|
||||
## 2 || NNDSVDa || fill in the zero elements with the average
|
||||
## 3 || NNDSVD || leave zero elements as zero, works better for sparse spectrograms
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the highest of windowSize and (bandwidth - 1) * 2.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [destination] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
private:: synth, server
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
~bases = Buffer.new(s);
|
||||
~activations = Buffer.new(s);
|
||||
~resynth = Buffer.new(s);
|
||||
)
|
||||
|
||||
//how many bases do I need to decompose the buffer with 90% accuracy
|
||||
(
|
||||
Routine{
|
||||
FluidBufNMFSeed.process(s, b, ~bases, ~activations, coverage: 0.9, method: 1).wait;
|
||||
"% bases".format(~bases.numChannels).postln;
|
||||
}.play;
|
||||
)
|
||||
//check how many bases we are returned:
|
||||
|
||||
|
||||
//try the same process with less accuracy
|
||||
(
|
||||
Routine{
|
||||
FluidBufNMFSeed.process(s, b, ~bases, ~activations, coverage: 0.5).wait;
|
||||
"% bases".format(~bases.numChannels).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
//use the bases to run NMF on
|
||||
FluidBufNMF.process(s, b, resynth: ~resynth, bases: ~bases, activations: ~activations,actMode: 2, components: ~bases.numChannels, action: {\done.postln;})
|
||||
{PlayBuf.ar(~resynth.numChannels, ~resynth)[2]}.play
|
||||
::
|
||||
@ -1,179 +0,0 @@
|
||||
TITLE:: FluidBufNoveltySlice
|
||||
SUMMARY:: Buffer-Based Novelty-Based Slicer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation, UGens>Buffer
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a non-real-time slicer using an algorithm assessing novelty in the signal to estimate the slicing points. A novelty curve is being derived from running a kernel across the diagonal of the similarity matrix, and looking for peak of changes. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a buffer which contains indices (in sample) of estimated starting points of different slices.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the slicing to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be sliced through novelty identification. The different channels of multichannel buffers will be summed.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the slicing process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be summed.
|
||||
|
||||
ARGUMENT:: indices
|
||||
The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis.
|
||||
|
||||
ARGUMENT:: feature
|
||||
The feature on which novelty is computed.
|
||||
table::
|
||||
##0 || Spectrum || The magnitude of the full spectrum.
|
||||
##1 || MFCC || 13 Mel-Frequency Cepstrum Coefficients.
|
||||
##2 || Chroma || The contour of a 12 band Chromagram.
|
||||
##3 || Pitch || The pitch and its confidence.
|
||||
##4 || Loudness || The TruePeak and Loudness.
|
||||
::
|
||||
ARGUMENT:: kernelSize
|
||||
The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes.
|
||||
|
||||
ARGUMENT:: threshold
|
||||
The normalised threshold, between 0 an 1, on the novelty curve to consider it a segmentation point.
|
||||
|
||||
ARGUMENT:: filterSize
|
||||
The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes.
|
||||
|
||||
ARGUMENT:: minSliceLength
|
||||
The minimum duration of a slice in number of hopSize.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As novelty estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As novelty estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// load some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
(
|
||||
// with basic params, with a minimum slight length to avoid over
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufNoveltySlice.process(s,b, indices: c, threshold:0.4,filterSize: 4, minSliceLength: 8).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
//check the number of slices: it is the number of frames in the transBuf minus the boundary index.
|
||||
c.query;
|
||||
|
||||
//loops over a splice with the MouseX
|
||||
(
|
||||
{
|
||||
BufRd.ar(1, b,
|
||||
Phasor.ar(0,1,
|
||||
BufRd.kr(1, c,
|
||||
MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1),
|
||||
BufRd.kr(1, c,
|
||||
MouseX.kr(1, BufFrames.kr(c)), 0, 1),
|
||||
BufRd.kr(1,c,
|
||||
MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1)), 0, 1);
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
|
||||
STRONG::Examples of the impact of the filterSize::
|
||||
|
||||
CODE::
|
||||
// load some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// process with a given filterSize
|
||||
(
|
||||
Routine{
|
||||
FluidBufNoveltySlice.process(s,b, indices: c, kernelSize:31, threshold:0.1, filterSize:1).wait;
|
||||
//check the number of slices: it is the number of frames in the transBuf minus the boundary index.
|
||||
c.query;
|
||||
}.play;
|
||||
)
|
||||
//play slice number 3
|
||||
(
|
||||
{
|
||||
BufRd.ar(1, b,
|
||||
Line.ar(
|
||||
BufRd.kr(1, c, DC.kr(3), 0, 1),
|
||||
BufRd.kr(1, c, DC.kr(4), 0, 1),
|
||||
(BufRd.kr(1, c, DC.kr(4)) - BufRd.kr(1, c, DC.kr(3), 0, 1) + 1) / s.sampleRate),
|
||||
0,1);
|
||||
}.play;
|
||||
)
|
||||
|
||||
// change the filterSize in the code above to 4. Then to 12. Listen in between to the differences.
|
||||
|
||||
// What's happening? In the first instance (filterSize = 1), the novelty line is jittery and therefore overtriggers on the arpegiated guitar. We also can hear attacks at the end of the segment. Setting the threshold higher (like in the 'Basic Example' pane) misses some more subtle variations.
|
||||
|
||||
// So in the second settings (filterSize = 4), we smooth the novelty line a little, which allows us to catch small differences that are not jittery. It also corrects the ending cutting by the same trick: the averaging of the sharp pick is sliding up, crossing the threshold slightly earlier.
|
||||
|
||||
// If we smooth too much, like the third settings (filterSize = 12), we start to loose precision and miss attacks. Have fun with different values of theshold then will allow you to find the perfect segment for your signal.
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// make a stereo buffer
|
||||
b = Buffer.alloc(s,88200,2);
|
||||
|
||||
// add some stereo clicks and listen to them
|
||||
((0..3)*22050+11025).do({|item,index| b.set(item+(index%2), 1.0)});
|
||||
b.play
|
||||
|
||||
// create a new buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
// with basic params
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufNoveltySlice.process(s,b, indices: c, threshold:0.3).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// list the indicies of detected attacks - the two input channels have been summed
|
||||
c.getn(0,c.numFrames,{|item|(item * 2).postln;})
|
||||
::
|
||||
@ -1,146 +0,0 @@
|
||||
TITLE:: FluidBufOnsetSlice
|
||||
SUMMARY:: Spectral Difference-Based Audio Buffer Slicer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements many spectral-based onset detection metrics, most of them taken from the literature. (http://www.dafx.ca/proceedings/papers/p_133.pdf) Some are already available in SuperCollider's LINK::Classes/Onsets:: object yet not as offline processes. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a buffer which contains indices (in samples) of estimated starting points of different slices. If a startFrame argument is supplied, the starting point indices are still specified in terms of their location in the original buffer; they are not offset by the number startFrames.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the slicing to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be sliced through novelty identification. The different channels of multichannel buffers will be summed.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the slicing process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel sources, which channel should be processed.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel sources, how many channel should be summed.
|
||||
|
||||
ARGUMENT:: indices
|
||||
The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis.
|
||||
|
||||
ARGUMENT:: metric
|
||||
The metric used to derive a difference curve between spectral frames. It can be any of the following:
|
||||
TABLE::
|
||||
##0 || Energy || thresholds on (sum of squares of magnitudes / nBins) (like Onsets \power)
|
||||
##1 || HFC || thresholds on (sum of (squared magnitudes * binNum) / nBins)
|
||||
##2 || SpectralFlux || thresholds on (diffence in magnitude between consecutive frames, half rectified)
|
||||
##3 || MKL || thresholds on (sum of log of magnitude ratio per bin) (or equivalent: sum of difference of the log magnitude per bin) (like Onsets \mkl)
|
||||
##4 || IS || (WILL PROBABLY BE REMOVED) Itakura - Saito divergence (see literature)
|
||||
##5 || Cosine || thresholds on (cosine distance between comparison frames)
|
||||
##6 || PhaseDev || takes the past 2 frames, projects to the current, as anticipated if it was a steady state, then compute the sum of the differences, on which it thresholds (like Onsets \phase)
|
||||
##7 || WPhaseDev || same as PhaseDev, but weighted by the magnitude in order to remove chaos noise floor (like Onsets \wphase)
|
||||
##8 || ComplexDev || same as PhaseDev, but in the complex domain - the anticipated amp is considered steady, and the phase is projected, then a complex subtraction is done with the actual present frame. The sum of magnitudes is used to threshold (like Onsets \complex)
|
||||
##9 || RComplexDev || same as above, but rectified (like Onsets \rcomplex)
|
||||
::
|
||||
|
||||
ARGUMENT:: threshold
|
||||
The thresholding of a new slice. Value ranges are different for each metric, from 0 upwards.
|
||||
|
||||
ARGUMENT:: minSliceLength
|
||||
The minimum duration of a slice in number of hopSize.
|
||||
|
||||
ARGUMENT:: filterSize
|
||||
The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes.
|
||||
|
||||
ARGUMENT:: frameDelta
|
||||
For certain metrics (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distance between the comparison window in samples.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As spectral differencing relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
CODE::
|
||||
(
|
||||
//prep some buffers
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
(
|
||||
// with basic params
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufOnsetSlice.process(s,b, indices: c, threshold:0.5).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
//check the number of slices: it is the number of frames in the transBuf minus the boundary index.
|
||||
c.query;
|
||||
|
||||
//loops over a splice with the MouseX
|
||||
(
|
||||
{
|
||||
BufRd.ar(1, b,
|
||||
Phasor.ar(0,1,
|
||||
BufRd.kr(1, c,
|
||||
MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1),
|
||||
BufRd.kr(1, c,
|
||||
MouseX.kr(1, BufFrames.kr(c)), 0, 1),
|
||||
BufRd.kr(1,c,
|
||||
MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1)), 0, 1);
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// make a stereo buffer
|
||||
b = Buffer.alloc(s,88200,2);
|
||||
|
||||
// add some stereo clicks and listen to them
|
||||
((0..3)*22050+11025).do({|item,index| b.set(item+(index%2), 1.0)})
|
||||
b.play
|
||||
|
||||
// create a new buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
// with basic params
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufOnsetSlice.process(s,b, indices: c, threshold:0.00001).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// list the indicies of detected attacks - the two input channels have been summed
|
||||
c.getn(0,c.numFrames,{|item|(item * 2).postln;})
|
||||
::
|
||||
@ -1,193 +0,0 @@
|
||||
TITLE:: FluidBufPitch
|
||||
SUMMARY:: A Selection of Pitch Descriptors on a Buffer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading, Classes/SpecCentroid, Classes/SpecFlatness, Classes/SpecCentroid, Classes/SpecPcile
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements three popular pitch descriptors, computed as frequency and the confidence in its value. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a multichannel buffer with two channels per input channel, one for pitch and one for the pitch tracking confidence. A pitch of 0 Hz is yield (or -999.0 when the unit is in MIDI note) when the algorithm cannot find a fundamental at all. Each sample represents a value, which is every hopSize. Its sampling rate is sourceSR / hopSize.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the pitch descriptor to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be pitch-tracked. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
ARGUMENT:: features
|
||||
The destination buffer for the pitch descriptors.
|
||||
|
||||
ARGUMENT:: algorithm
|
||||
The algorithm to estimate the pitch. The options are:
|
||||
TABLE::
|
||||
## 0 || Cepstrum: Returns a pitch estimate as the location of the second highest peak in the Cepstrum of the signal (after DC).
|
||||
## 1 || Harmonic Product Spectrum: Implements the Harmonic Product Spectrum algorithm for pitch detection . See e.g. FOOTNOTE:: A. Lerch, "An Introduction to Audio Content Analysis: Applications in Signal Processing and Music Informatics." John Wiley & Sons, 2012.https://onlinelibrary.wiley.com/doi/book/10.1002/9781118393550 ::
|
||||
## 2 || YinFFT: Implements the frequency domain version of the YIN algorithm, as described in FOOTNOTE::P. M. Brossier, "Automatic Annotation of Musical Audio for Interactive Applications." QMUL, London, UK, 2007. :: See also https://essentia.upf.edu/documentation/reference/streaming_PitchYinFFT.html
|
||||
::
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
The minimum frequency that the algorithm will search for an estimated fundamental. This sets the lowest value that will be generated.
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
The maximum frequency that the algorithm will search for an estimated fundamental. This sets the highest value that will be generated.
|
||||
|
||||
ARGUMENT:: unit
|
||||
The unit of the estimated value. The default of 0 is in Hz. A value of 1 will convert to MIDI note values.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As sinusoidal estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: padding
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [features] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// create a buffer with a short clicking sinusoidal burst (220Hz) starting at frame 8192 for 1024 frames
|
||||
(
|
||||
b = Buffer.sendCollection(s, (Array.fill(8192,{0}) ++ (Signal.sineFill(1203,[0,0,0,0,0,1],[0,0,0,0,0,0.5pi]).takeThese({|x,i|i>1023})) ++ Array.fill(8192,{0})));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// listen to the source and look at the buffer
|
||||
b.play; b.plot;
|
||||
|
||||
// run the process with basic parameters
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufPitch.process(s, b, features: c).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the analysis
|
||||
c.plot(separately:true)
|
||||
|
||||
// The values are interleaved [pitch,confidence] in the buffer as they are on 2 channels: to get to the right frame, divide the SR of the input by the hopSize, then multiply by 2 because of the channel interleaving
|
||||
// here we are querying from one frame before (the signal starts at 8192, which is frame 16 (8192/512), therefore starting the query at frame 15, which is index 30.
|
||||
c.getn(30,10,{|x|x.postln})
|
||||
|
||||
// observe that the first frame is silent, as expected. The next frame's confidence is low-ish, because the window is half full (window of 1024, overlap of 512). Then a full window is analysed, with strong confidence. Then another half full window, then silence, as expected.
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create a buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them, with limited bandwidth and units in MIDI notes
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufPitch.process(s, b, features: c, minFreq:200, maxFreq:2000, unit:1).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the buffer: [pitch,confidence] for left then [pitch,confidence] for right
|
||||
c.plot(separately:true)
|
||||
::
|
||||
|
||||
STRONG::A musical example.::
|
||||
code::
|
||||
// create some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// run the process with basic parameters and retrieve the array in the langage side
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufPitch.process(s, b, features: c,action:{c.loadToFloatArray(action: {|x| d = x.reshape((x.size()/2).asInteger, 2)})}).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
//look at the retrieved formatted array of [pitch,confidence] values
|
||||
d.postln
|
||||
|
||||
//iterate and make an array of the indices which are fitting the conditions
|
||||
(
|
||||
e = Array.new;
|
||||
d.do({
|
||||
arg val, i;
|
||||
if ((val[0] > 500) && (val[1] > 0.98)) {e = e.add(i)}; // if pitch is greater than 500Hz and confidence higher than 0.98, keep the index
|
||||
});
|
||||
)
|
||||
e.postln;
|
||||
|
||||
//granulate only the frames that are in our buffer
|
||||
// We need to convert our indices to frame start. Their position was (index * hopSize) - (hopSize) in FluidBufPitch
|
||||
f = e.collect({arg i; (i * 512) - 512});
|
||||
|
||||
// define a basic grain synth
|
||||
(
|
||||
SynthDef(\grain,
|
||||
{ arg out=0, buf =0 , ind = 0, pan = 0;
|
||||
var env;
|
||||
env = EnvGen.kr(Env.new([0,1,0],[512/s.sampleRate].dup,\sine), doneAction: Done.freeSelf);
|
||||
Out.ar(out, Pan2.ar(PlayBuf.ar(1,buf,startPos:ind),pan));
|
||||
}).add;
|
||||
)
|
||||
|
||||
// start the sequence
|
||||
(
|
||||
a = Pxrand(f, inf).asStream;
|
||||
Routine({
|
||||
loop({
|
||||
Synth(\grain, [\buf, b, \ind, a.next, \pan, (2.0.rand - 1)]);
|
||||
(256/s.sampleRate).wait;
|
||||
})
|
||||
}).play;
|
||||
)
|
||||
::
|
||||
@ -1,109 +0,0 @@
|
||||
TITLE:: FluidBufSTFT
|
||||
summary:: Perform a Short-Time Fourier Transform on one channel of a buffer
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/Buffer
|
||||
|
||||
DESCRIPTION::
|
||||
Performs either a forward or inverse Short-Time Fourier Transform (STFT) on a single channel source buffer~. In the forward case, resulting magnitudes and phases can be written to output buffers. In the inverse case, these buffers can be used to reconstruct the original source into a new buffer.
|
||||
|
||||
The magntude and phase buffers are laid out as (number of hops, number of bins). The number of hops is a function of the source length and the hop size. The number of bins is (1 + (fft size / 2)).
|
||||
|
||||
The object is restricted to analysing a single source channel, because the channel counts of the magntude and phase buffers would quickly get out of hand otherwise.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
private::new1
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
Run the process on the given sever, and perfrom code::action:: when done
|
||||
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run
|
||||
|
||||
ARGUMENT:: source
|
||||
The link::Classes/Buffer:: to use for the forward STFT
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
The starting point for analysis in the source (in samples)
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
The duration (in samples) to analyse
|
||||
|
||||
ARGUMENT:: startChan
|
||||
The channel to analyse
|
||||
|
||||
ARGUMENT:: magnitude
|
||||
The link::Classes/Buffer:: to write magnitudes to in the forward case, or read from in the inverse case. This is optional for the forward transform, mandatory for the inverse.
|
||||
|
||||
ARGUMENT:: phase
|
||||
The link::Classes/Buffer:: to write phases to in the forward case, or read from in the inverse case. This is optional for the forward transform, mandatory for the inverse.
|
||||
|
||||
ARGUMENT:: resynth
|
||||
The link::Classes/Buffer:: to write re-synthesised data to in the inverse case. Ignored for the forward transform. Mandatory in the inverse case.
|
||||
|
||||
ARGUMENT:: inverse
|
||||
When set to 1, an inverse STFT is performed, and the resynthesised data is written to the resynthesis buffer using overlap-add.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The number of source samples that are analysed at once.
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
How many samples there are in-between analysis windows. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize. For this object it is effectively capped at 65536.
|
||||
|
||||
ARGUMENT:: padding
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
Runs when processing is complete
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
s.reboot
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
m = Buffer.new;
|
||||
p = Buffer.new;
|
||||
r = Buffer.new;
|
||||
)
|
||||
|
||||
(
|
||||
fork{
|
||||
FluidBufSTFT.process(s,source:b,magnitude:m,phase:p).wait;
|
||||
FluidBufSTFT.process(s,magnitude:m,phase:p,resynth:r,inverse:1).wait;
|
||||
"Done".postln;
|
||||
}
|
||||
)
|
||||
|
||||
{ PlayBuf.ar(1,r); }.play
|
||||
|
||||
//nullsum
|
||||
{ PlayBuf.ar(1,r) - PlayBuf(1,b); }.play
|
||||
|
||||
//draw the magnitudes as a greyscale spectrogram
|
||||
// make the image
|
||||
i = Image.new(m.numFrames, m.numChannels)
|
||||
|
||||
//retreive the image and assign to pixels
|
||||
(
|
||||
m.loadToFloatArray(action: {|x|
|
||||
var mod = m.numChannels;
|
||||
{
|
||||
x.do{
|
||||
|val, index|
|
||||
i.setColor(Color.gray(val), index.div(mod), mod - 1 - index.mod(mod));
|
||||
};
|
||||
i.plot("spectrogram", showInfo: false);
|
||||
}.fork(AppClock)
|
||||
});
|
||||
)
|
||||
::
|
||||
@ -1,112 +0,0 @@
|
||||
TITLE:: FluidBufScale
|
||||
SUMMARY:: A Scaling Processor for Buffers
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading
|
||||
|
||||
This class implements a simple Buffer preprocessor, by scaling its values. It draws a simple translation from inputLow to outputLow, and from inputHigh to outputHigh. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a buffer with the same size and shape than the requested range.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the scaling to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffer to be processed is allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be processed.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
The starting point (in samples) from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
The duration (in samples) to copy from the source buffer. The default (-1) copies the full lenght of the buffer.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
The first channel from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
The number of channels from which to copy in the source buffer. This parameter will wrap around the number of channels in the source buffer. The default (-1) copies all of the buffer's channel.
|
||||
|
||||
ARGUMENT:: destination
|
||||
The index of the buffer to use as the destination for the processed material.
|
||||
|
||||
ARGUMENT:: inputLow
|
||||
The low reference point of the input. it will be scaled to yield outputLow at the output
|
||||
|
||||
ARGUMENT:: inputHigh
|
||||
The high reference point of the input. it will be scaled to yield outputHigh at the output
|
||||
|
||||
ARGUMENT:: outputLow
|
||||
The output value when the input is inputLow
|
||||
|
||||
ARGUMENT:: outputHigh
|
||||
The output value when the input is inputHigh
|
||||
|
||||
ARGUMENT:: clipping
|
||||
Optional clipping of the input (and therefore of the output). 0 is none. 1 caps the lowest input at inputLow. 2 caps the highest input at inputHigh, 3 caps both input low and high value within the described range.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The metric will be passed indices as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
(
|
||||
Routine{
|
||||
// make a buffer of known qualities
|
||||
b = Buffer.sendCollection(s,1.0.series(1.1,2.0));
|
||||
// and a destination buffer
|
||||
c = Buffer(s);
|
||||
// play with the scaling
|
||||
FluidBufScale.process(s, b, destination: c, inputLow: 0, inputHigh: 1, outputLow: 20, outputHigh:10).wait;
|
||||
// retrieve the buffer and enjoy the results.
|
||||
c.getn(0,10,{|x|x.round(0.000001).postln;})
|
||||
}.play
|
||||
)
|
||||
|
||||
// also works in multichannel - explore the following buffer
|
||||
|
||||
//process
|
||||
(
|
||||
Routine{
|
||||
b = Buffer.sendCollection(s,-10.0.series(-9,10.0).scramble,2);
|
||||
c = Buffer(s);
|
||||
s.sync;
|
||||
defer{b.plot(bounds:Rect(400,400,400,400)).plotMode_(\points).bounds};
|
||||
FluidBufScale.process(s, b, destination: c, inputLow: -20, inputHigh: 20, outputLow: 0, outputHigh:1).wait;
|
||||
//enjoy - same shape, different range
|
||||
defer{c.plot(bounds:Rect(800,400,400,400)).plotMode_(\points)};
|
||||
}.play;
|
||||
)
|
||||
|
||||
//also works with a subset of the input, resizing the output
|
||||
(
|
||||
Routine{
|
||||
b = Buffer.sendCollection(s,0.0.series(0.1,3.0).reshape(3,10).flop.flat,3);
|
||||
c = Buffer(s);
|
||||
s.sync;
|
||||
defer{b.plot(separately: true,bounds:Rect(400,400,400,400)).plotMode_(\points)};
|
||||
//process
|
||||
FluidBufScale.process(s, b, startFrame: 3,numFrames: 4,startChan: 1,numChans: 1, destination: c, inputLow: 0, inputHigh: 3, outputLow: 0, outputHigh:1).wait;
|
||||
//enjoy
|
||||
c.query;
|
||||
c.getn(0,4,{|x|x.postln;});
|
||||
defer{c.plot(separately: true,bounds:Rect(800,400,400,400)).plotMode_(\points)};
|
||||
}.play
|
||||
)
|
||||
::
|
||||
|
||||
|
||||
@ -1,62 +0,0 @@
|
||||
TITLE:: FluidBufSelect
|
||||
summary:: Cherry pick values from a buffer
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidBufSelectEvery
|
||||
|
||||
DESCRIPTION::
|
||||
Pick sets of values from a buffer, described in terms of a list of frame indices and channel numbers.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
private::new1
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
Run the process on the given sever, and perfrom code::action:: when done
|
||||
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run
|
||||
|
||||
ARGUMENT:: source
|
||||
The link::Classes/Buffer:: to select values from
|
||||
|
||||
ARGUMENT:: destination
|
||||
The link::Classes/Buffer:: to write the selected data to
|
||||
|
||||
ARGUMENT:: indices
|
||||
A 0-based list of frame indices to recover. Default is [-1], meaning all frames
|
||||
|
||||
ARGUMENT:: channels
|
||||
A 0-based list of channel numbers to recover. Default is [-1], meaning all frames
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
Runs when processing is complete
|
||||
|
||||
EXAMPLES::
|
||||
code::
|
||||
//send a known collection where the value of each frame in each channel is encoded
|
||||
//chan
|
||||
b = Buffer.sendCollection(s,30.collect{|x| x.mod(6) + (x.div(6) * 0.1)},6)
|
||||
//check the ranges (thus showing a plotter error...)
|
||||
b.plot(separately: true).plotMode_(\points)
|
||||
//you can also check the collection itself if in doubt
|
||||
b.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
|
||||
//let's make a destination buffer
|
||||
c = Buffer(s);
|
||||
|
||||
//using default values, we copy everything:
|
||||
FluidBufSelect.process(s,b,c,action: {c.query});
|
||||
c.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
|
||||
//more powerful copying, resizing the destination accordingly
|
||||
FluidBufSelect.process(s,b,c, indices: [1,3], channels: [2,4], action: {c.query});
|
||||
c.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
|
||||
//observe the order can be anything, and -1 (default) passes everything in that dimension
|
||||
FluidBufSelect.process(s,b,c, indices: [ -1 ] , channels: [3, 1, 4], action: {c.query});
|
||||
c.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
::
|
||||
|
||||
@ -1,74 +0,0 @@
|
||||
TITLE:: FluidBufSelectEvery
|
||||
summary:: Extract every N samples / channels from a buffer
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidBufSelect
|
||||
|
||||
DESCRIPTION::
|
||||
Pick every N frames and / or channels from a buffer, described in terms of independent hop sizes for frames and channels
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
private::new1
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
Run the process on the given sever, and perfrom code::action:: when done
|
||||
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run
|
||||
|
||||
ARGUMENT:: source
|
||||
The link::Classes/Buffer:: to select values from
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
The starting point (in samples) from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
The duration (in samples) to copy from the source buffer. The default (-1) copies the full lenght of the buffer.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
The first channel from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
The number of channels from which to copy in the source buffer. This parameter will wrap around the number of channels in the source buffer. The default (-1) copies all of the buffer's channel.
|
||||
|
||||
ARGUMENT:: destination
|
||||
The link::Classes/Buffer:: to write the selected data to
|
||||
|
||||
ARGUMENT:: frameHop
|
||||
Take every `frameHop` frames. Default = 1 = all frames (where 2 would be every other frame, etc.)
|
||||
|
||||
ARGUMENT:: chanHop
|
||||
Take every `chanHop` channels. Default = 1 = all channels (where 2 would be every other channel, etc.)
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
Runs when processing is complete
|
||||
|
||||
EXAMPLES::
|
||||
Didactic
|
||||
code::
|
||||
//send a known collection where the value of each frame in each channel is encoded
|
||||
//chan
|
||||
b = Buffer.sendCollection(s,30.collect{|x| x.mod(6) + (x.div(6) * 0.1)},6)
|
||||
//check the ranges (thus showing a plotter error...)
|
||||
b.plot(separately: true).plotMode_(\points)
|
||||
//you can also check the collection itself if in doubt
|
||||
b.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
|
||||
//let's make a destination buffer
|
||||
c = Buffer(s);
|
||||
|
||||
//using default values, we copy everything:
|
||||
FluidBufSelectEvery.process(s,b, destination: c, action: {c.query});
|
||||
c.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
|
||||
//more powerful copying, resizing the destination accordingly
|
||||
FluidBufSelectEvery.process(s,b, destination: c, frameHop: 2, chanHop: 3, action: {c.query});
|
||||
c.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
|
||||
//source buffer boundaries still apply before the hopping selection
|
||||
FluidBufSelectEvery.process(s,b, startFrame: 1, numFrames: 3, startChan: 2, numChans: 3, destination: c, frameHop: 1, chanHop: 2, action: {c.query});
|
||||
c.getToFloatArray(action: {|x|x.round(0.1).postln;});
|
||||
::
|
||||
@ -1,148 +0,0 @@
|
||||
TITLE:: FluidBufSines
|
||||
SUMMARY:: Buffer-Based Sinusoidal Modelling and Resynthesis
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation, UGens>Buffer
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading, Guides/FluidBufMultiThreading
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
This class triggers a Sinusoidal Modelling process on buffers on the non-real-time thread of the server. It implements a mix of algorithms taken from classic papers. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The algorithm will take a buffer in, and will divide it in two parts: LIST::
|
||||
## a reconstruction of what it detects as sinusoidal;
|
||||
## a residual derived from the previous buffer to allow null-summing::
|
||||
|
||||
The whole process is based on the assumption that signal is made of pitched steady components that have a long-enough duration and are periodic enough to be perceived as such, that can be tracked, resynthesised and removed from the original, leaving behind what is considered as non-pitched, noisy, and/or transient. It first tracks the peaks, then checks if they are the continuation of a peak in previous spectral frames, by assigning them a track.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the sinusoidal estimation to be calculated on a given source buffer and to be resynthesised.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be decomposed through the sinusoidal modelling process. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
ARGUMENT:: sines
|
||||
The index of the buffer where the extracted sinusoidal component will be reconstructed.
|
||||
|
||||
ARGUMENT:: residual
|
||||
The index of the buffer where the residual of the sinusoidal component will be reconstructed.
|
||||
|
||||
ARGUMENT:: bandwidth
|
||||
The number of bins used to resynthesises a peak. It has an effect on CPU cost: the widest is more accurate but more computationally expensive. It is capped at (fftSize / 2) + 1.
|
||||
|
||||
ARGUMENT:: detectionThreshold
|
||||
The threshold in dB above which a magnitude peak is considered to be a sinusoidal component.
|
||||
|
||||
ARGUMENT:: birthLowThreshold
|
||||
The threshold in dB above which to consider a peak to start a sinusoidal component tracking, for the low end of the spectrum. It is interpolated across the spectrum until birthHighThreshold at half-Nyquist.
|
||||
|
||||
ARGUMENT:: birthHighThreshold
|
||||
The threshold in dB above which to consider a peak to start a sinusoidal component tracking, for the high end of the spectrum. It is interpolated across the spectrum until birthLowThreshold at DC.
|
||||
|
||||
ARGUMENT:: minTrackLen
|
||||
The minimum duration, in spectral frames, for a sinusoidal track to be accepted as a partial. It allows to remove bubbly pitchy artefacts, but is more CPU intensive and might reject quick pitch material.
|
||||
|
||||
ARGUMENT:: trackingMethod
|
||||
The algorithm used to track the sinusoidal continuity between spectral frames. 0 is the default, "Greedy", and 1 is a more expensive "Hungarian" one. footnote::Neri, J., and Depalle, P., "Fast Partial Tracking of Audio with Real-Time Capability through Linear Programming". Proceedings of DAFx-2018.::
|
||||
|
||||
ARGUMENT:: trackMagRange
|
||||
The amplitude difference allowed for a track to diverge between frames, in dB.
|
||||
|
||||
ARGUMENT:: trackFreqRange
|
||||
The frequency difference allowed for a track to diverge between frames, in Hertz.
|
||||
|
||||
ARGUMENT:: trackProb
|
||||
The probability of the tracking algorithm to find a track.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the highest of windowSize and (bandwidth - 1) * 2.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [sines, residual] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// create some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
d = Buffer.new(s);
|
||||
)
|
||||
|
||||
// run the process with basic parameters
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufSines.process(s, b, sines: c, residual:d).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// listen to each component
|
||||
c.play;
|
||||
d.play;
|
||||
|
||||
//nullsumming tests
|
||||
{(PlayBuf.ar(1, c)) + (PlayBuf.ar(1,d)) - (PlayBuf.ar(1,b,doneAction:2))}.play
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create 2 new buffers as destinations
|
||||
d = Buffer.new(s); e = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufSines.process(s, b, sines: d, residual:e, windowSize: 2048, hopSize: 256, fftSize: 16384).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
//listen: stereo preserved!
|
||||
d.play
|
||||
e.play
|
||||
::
|
||||
@ -1,142 +0,0 @@
|
||||
TITLE:: FluidBufSpectralShape
|
||||
SUMMARY:: Seven Spectral Shape Descriptors on a Buffer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading, Classes/FluidSpectralShape, Classes/SpecCentroid, Classes/SpecFlatness, Classes/SpecCentroid, Classes/SpecPcile
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements seven of the most popular spectral shape descriptors, computed on a linear scale for both amplitude and frequency. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The descriptors are:
|
||||
LIST::
|
||||
##the four first statistical moments (https://en.wikipedia.org/wiki/Moment_(mathematics) ), more commonly known as:
|
||||
LIST::
|
||||
## the spectral centroid (1) in Hz. This is the point that splits the spectrum in 2 halves of equal energy. It is the weighted average of the magnitude spectrum.
|
||||
## the spectral spread (2) in Hz. This is the standard deviation of the spectrum envelop, or the average of the distance to the centroid.
|
||||
## the normalised skewness (3) as ratio. This indicates how tilted is the spectral curve in relation to the middle of the spectral frame, i.e. half of the Nyquist frequency. If it is below that frequency, i.e. the central bin of the magnitude spectrum, it is positive.
|
||||
## the normalised kurtosis (4) as ratio. This indicates how focused is the spectral curve. If it is peaky, it is high.
|
||||
::
|
||||
## the rolloff (5) in Hz. This indicates the frequency under which 95% of the energy is included.
|
||||
## the flatness (6) in dB. This is the ratio of geometric mean of the magnitude, over the arithmetic mean of the magnitudes. It yields a very approximate measure on how noisy a signal is.
|
||||
## the crest (7) in dB. This is the ratio of the loudest magnitude over the RMS of the whole frame. A high number is an indication of a loud peak poking out from the overal spectral curve.::
|
||||
|
||||
The drawings in Peeters 2003 (http://recherche.ircam.fr/anasyn/peeters/ARTICLES/Peeters_2003_cuidadoaudiofeatures.pdf) are useful, as are the commented examples below. For the mathematically-inclined reader, the tutorials and code offered here (https://www.audiocontentanalysis.org/) are interesting to further the understanding. For examples of the impact of computing the moments in power magnitudes, and/or in exponential frequency scale, please refer to the LINK::Classes/FluidSpectralShape:: helpfile.
|
||||
|
||||
The process will return a multichannel buffer with the seven channels per input channel, each containing the 7 shapes. Each sample represents a value, which is every hopSize.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the spectral shape descriptors to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be described through the various descriptors. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
ARGUMENT:: features
|
||||
The destination buffer for the 7 spectral features describing the spectral shape.
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
The minimum frequency that the algorithm will consider for computing the spectral shape. Frequencies below will be ignored. The default of 0 goes down to DC when possible.
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
The maximum frequency that the algorithm will consider for computing the spectral shape. Frequencies above will be ignored. The default of -1 goes up to Nyquist.
|
||||
|
||||
ARGUMENT:: rolloffPercent
|
||||
This sets the percentage of the frame's energy that will be reported as the rolloff frequency. The default is 95%.
|
||||
|
||||
ARGUMENT:: unit
|
||||
The frequency unit for the spectral shapes to be computed upon, and outputted at. The default (0) is in Hertz and computes the moments on a linear spectrum. The alternative is in MIDI note numbers(1), which compute the moments on an exponential spectrum.
|
||||
|
||||
ARGUMENT:: power
|
||||
This flag sets the scaling of the magnitudes in the moment calculation. It uses either its amplitude (0, by default) or its power (1).
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As spectral shape estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As spectral shape estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: padding
|
||||
Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [features] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// create some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// run the process with basic parameters
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufSpectralShape.process(s, b, features: c).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// listen to the source and look at the buffer
|
||||
b.play;
|
||||
c.plot(separately:true)
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create a buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufSpectralShape.process(s, b, features: c).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// look at the buffer: 7shapes for left, then 7 shapes for right
|
||||
c.plot(separately:true)
|
||||
::
|
||||
@ -1,71 +0,0 @@
|
||||
TITLE:: FluidBufThreadDemo
|
||||
SUMMARY:: A Tutorial Object to Experiment with Multithreading in FluidBuf* Objects
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a simple tutorial object to illustrate and experiment with the behaviour of the FluidBuf* objects. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
It simply starts a process that will, upon completion of its task, write a single value to the destination buffer. This is the general behaviour of much of the CPU consuming FluidBuf* objects. In this case, as a toy example, the task is simply just wait and do nothing, to simulate a task that would actually take that long in the background.
|
||||
|
||||
The process will, after waiting for STRONG::time:: millisecond, return its delay lenght as a Float writen at index [0] of the specified destination buffer.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the job to be done. In this case, simply waiting STRONG::time:: millisecond before writing a value in the destination buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the destination buffer is declared.
|
||||
|
||||
ARGUMENT:: result
|
||||
The destination buffer, where the value should be written at the end of the process.
|
||||
|
||||
ARGUMENT:: time
|
||||
The duration in milliseconds of the delay that the background thread will wait for before yielding the value to the destination buffer.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A function that will be executed upon completion. It is passed the destination buffer as argument.
|
||||
|
||||
returns::The instance of FluidNRTProcess which can be used to cancel the job.
|
||||
|
||||
|
||||
METHOD:: kr
|
||||
This is the UGen that is holding the link with the background thread. It is called by the 'process' method and can be used directly to monitor the progress of the job. For examples of such usage, please refer to the tutorial on link::Guides/FluidBufMultiThreading::.
|
||||
|
||||
ARGUMENT:: result
|
||||
The destination buffer, where the value should be written at the end of the process.
|
||||
|
||||
ARGUMENT:: time
|
||||
The duration of the delay that the background thread will wait for before yielding the value to the destination buffer.
|
||||
|
||||
ARGUMENT:: trig
|
||||
The trigger to start the job.
|
||||
|
||||
ARGUMENT:: blocking
|
||||
The thread on which the job is processed.
|
||||
|
||||
returns::It report the approximate job progress, from 0 to 1.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
For a thorough explanation, please refer to the tutorial on link::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CODE::
|
||||
// define a destination buffer
|
||||
b=Buffer.alloc(s,1);
|
||||
|
||||
// a simple call, where we query the destination buffer upon completion with the action message.
|
||||
FluidBufThreadDemo.process(s, b, 1000, action:{|x|x.get(0,{|y|y.postln});});
|
||||
|
||||
// as the 'process' returns its instance, we can cancel the process easily
|
||||
c = FluidBufThreadDemo.process(s, b, 100000, action: {|x|x.get(0,{|y|y.postln});});
|
||||
c.cancel;
|
||||
|
||||
// if a simple call to the UGen is used, the progress can be monitored. The usual cmd-period will cancel the job by freeing the synth.
|
||||
{c = FluidBufThreadDemo.kr(b,10000).poll; FreeSelfWhenDone.kr(c)}.scope;
|
||||
::
|
||||
@ -1,81 +0,0 @@
|
||||
TITLE:: FluidBufThresh
|
||||
SUMMARY:: A Gate Processor for Buffers
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a simple Buffer preprocessor, by replacing values under a threshold by 0s. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a buffer with the same size and shape than the requested range.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the thresholding to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffer to be processed is allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be processed.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
The starting point (in samples) from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
The duration (in samples) to copy from the source buffer. The default (-1) copies the full lenght of the buffer.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
The first channel from which to copy in the source buffer.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
The number of channels from which to copy in the source buffer. This parameter will wrap around the number of channels in the source buffer. The default (-1) copies all of the buffer's channel.
|
||||
|
||||
ARGUMENT:: destination
|
||||
The index of the buffer to use as the destination for the processed material.
|
||||
|
||||
ARGUMENT:: threshold
|
||||
The threshold under which values will be zeroed
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The metric will be passed indices as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// make a buffer of know qualities
|
||||
b = Buffer.sendCollection(s,0.0.series(0.1,1.0))
|
||||
// and a destination buffer
|
||||
c = Buffer(s)
|
||||
// play with the threshold
|
||||
FluidBufThresh.process(s, b, destination: c, threshold: 0.5)
|
||||
// retrieve the buffer and enjoy the results.
|
||||
c.getn(0,11,{|x|x.round(0.000001).postln;})
|
||||
|
||||
// also works in multichannel - explore the following buffer
|
||||
b = Buffer.sendCollection(s,0.0.series(0.1,2.0).scramble,2)
|
||||
b.plot.plotMode_(\points)
|
||||
//process and keep just the top values
|
||||
FluidBufThresh.process(s, b, destination: c, threshold: 1.6)
|
||||
//enjoy
|
||||
c.plot.plotMode_(\points)
|
||||
|
||||
//also works with a subset of the input, resizing the output
|
||||
b = Buffer.sendCollection(s,0.0.series(0.1,3.0).reshape(3,10).flop.flat,3)
|
||||
b.plot(separately: true).plotMode_(\points)
|
||||
//process and keep just the top values
|
||||
FluidBufThresh.process(s, b,startFrame: 3,numFrames: 4,startChan: 1,numChans: 1,destination: c, threshold: 1.6)
|
||||
//enjoy
|
||||
c.plot(separately: true).plotMode_(\points)
|
||||
c.query
|
||||
c.getn(0,4,{|x|x.round(0.000001).postln;})
|
||||
::
|
||||
@ -1,150 +0,0 @@
|
||||
TITLE:: FluidBufTransientSlice
|
||||
SUMMARY:: Buffer-Based Transient-Based Slicer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation, UGens>Buffer
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading, Classes/FluidBufTransients
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a non-real-time transient-based slice extractor relying on the same algorithm than Classes/FluidBufTransients using clicks/transients/derivation/anomaly in the signal to estimate the slicing points. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a buffer which contains indices (in sample) of estimated starting points of the different slices.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the slicing to be calculated on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be sliced through transient identification. The different channels of multichannel buffers will be summed.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the slicing process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be summed.
|
||||
|
||||
ARGUMENT:: indices
|
||||
The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis.
|
||||
|
||||
ARGUMENT:: order
|
||||
The order in samples of the impulse response filter used to model the estimated continuous signal. It is how many previous samples are used by the algorithm to predict the next one as reference for the model. The higher the order, the more accurate is its spectral definition, not unlike fft, improving low frequency resolution, but it differs in that it is not conected to its temporal resolution.
|
||||
|
||||
ARGUMENT:: blockSize
|
||||
The size in samples of frame on which it the algorithm is operating. High values are more cpu intensive, and also determines the maximum transient size, which will not be allowed to be more than half that lenght in size.
|
||||
|
||||
ARGUMENT:: padSize
|
||||
The size of the handles on each sides of the block simply used for analysis purpose and avoid boundary issues.
|
||||
|
||||
ARGUMENT:: skew
|
||||
The nervousness of the bespoke detection function with values from -10 to 10. It allows to decide how peaks are amplified or smoothed before the thresholding. High values increase the sensitivity to small variations.
|
||||
|
||||
ARGUMENT:: threshFwd
|
||||
The threshold of the onset of the smoothed error function. It allows tight start of the identification of the anomaly as it proceeds forward.
|
||||
|
||||
ARGUMENT:: threshBack
|
||||
The threshold of the offset of the smoothed error function. As it proceeds backwards in time, it allows tight ending of the identification of the anomaly.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The averaging window of the error detection function. It needs smoothing as it is very jittery. The longer the window, the less precise, but the less false positives.
|
||||
|
||||
ARGUMENT:: clumpLength
|
||||
The window size in sample within which positive detections will be clumped together to avoid overdetecting in time.
|
||||
|
||||
ARGUMENT:: minSliceLength
|
||||
The minimum duration of a slice in samples.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// load some buffers
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
)
|
||||
|
||||
// with basic parameters (wait for the computation time to appear)
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufTransientSlice.process(s,b, indices:c).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
//check the number of slices
|
||||
c.query;
|
||||
|
||||
//loops over a splice
|
||||
(
|
||||
{
|
||||
BufRd.ar(1, b,
|
||||
Phasor.ar(0,1,
|
||||
BufRd.kr(1, c,
|
||||
MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1),
|
||||
BufRd.kr(1, c,
|
||||
MouseX.kr(1, BufFrames.kr(c)), 0, 1),
|
||||
BufRd.kr(1,c,
|
||||
MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1)), 0, 1);
|
||||
}.play;
|
||||
)
|
||||
|
||||
// with everything changed to make it much better, at the cost of computation time (only 5 seconds are processed here, again wait for the (longer) computation time to appear)
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufTransientSlice.process(s,b, 0, 220500, 0, 1, c, 200, 2048, 1024, 1, 3, 1, 15, 30, 4410).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// play with the same player above to hear the segmentation difference
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// make a stereo buffer
|
||||
b = Buffer.alloc(s,88200,2);
|
||||
|
||||
// add some stereo clicks and listen to them
|
||||
((0..3)*22050+11025).do({|item,index| b.set(item+(index%2), 1.0)})
|
||||
|
||||
b.play
|
||||
|
||||
// create a new buffer as destinations
|
||||
c = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
// with basic params
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufTransientSlice.process(s,b, indices: c, threshFwd: 1.2).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// list the indicies of detected attacks - the two input channels have been summed
|
||||
c.getn(0,c.numFrames,{|item|(item*2).postln;})
|
||||
::
|
||||
@ -1,149 +0,0 @@
|
||||
TITLE:: FluidBufTransients
|
||||
SUMMARY:: Buffer-Based Transient Extractor
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation, UGens>Buffer
|
||||
RELATED:: Guides/FluidCorpusManipulation, Guides/FluidBufMultiThreading
|
||||
|
||||
DESCRIPTION::
|
||||
This class triggers a transient extractor on buffers on the non-real-time thread of the server. It implements declicking algorithm from chapter 5 of the classic Digital Audio Restoration by Godsill, Simon J., Rayner, Peter J.W. with some bespoke improvements on the detection function tracking. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The algorithm will take a buffer in, and will divide it in two outputs: LIST::
|
||||
## the transients, estimated from the signal and extracted from it;
|
||||
## the remainder of the material, as estimated by the algorithm.::
|
||||
|
||||
The whole process is based on the assumption that a transient is an element that is deviating from the surrounding material, as sort of click or anomaly. The algorithm then estimates what should have happened if the signal had followed its normal path, and resynthesises this estimate, removing the anomaly which is considered as the transient.
|
||||
|
||||
STRONG::Threading::
|
||||
|
||||
By default, this UGen spawns a new thread to avoid blocking the server command queue, so it is free to go about with its business. For a more detailed discussion of the available threading and monitoring options, including the two undocumented Class Methods below (.processBlocking and .kr) please read the guide LINK::Guides/FluidBufMultiThreading::.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: process, processBlocking
|
||||
This is the method that calls for the transient extraction to be performed on a given source buffer.
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which the buffers to be processed are allocated.
|
||||
|
||||
ARGUMENT:: source
|
||||
The index of the buffer to use as the source material to be decomposed through the NMF process. The different channels of multichannel buffers will be processing sequentially.
|
||||
|
||||
ARGUMENT:: startFrame
|
||||
Where in the srcBuf should the NMF process start, in sample.
|
||||
|
||||
ARGUMENT:: numFrames
|
||||
How many frames should be processed.
|
||||
|
||||
ARGUMENT:: startChan
|
||||
For multichannel srcBuf, which channel should be processed first.
|
||||
|
||||
ARGUMENT:: numChans
|
||||
For multichannel srcBuf, how many channel should be processed.
|
||||
|
||||
ARGUMENT:: transients
|
||||
The index of the buffer where the extracted transient component will be reconstructed.
|
||||
|
||||
ARGUMENT:: residual
|
||||
The index of the buffer where the estimated continuous component will be reconstructed.
|
||||
|
||||
ARGUMENT:: order
|
||||
The order in samples of the impulse response filter used to model the estimated continuous signal. It is how many previous samples are used by the algorithm to predict the next one as reference for the model. The higher the order, the more accurate is its spectral definition, not unlike fft, improving low frequency resolution, but it differs in that it is not conected to its temporal resolution.
|
||||
|
||||
ARGUMENT:: blockSize
|
||||
The size in samples of frame on which it the algorithm is operating. High values are more cpu intensive, and also determines the maximum transient size, which will not be allowed to be more than half that lenght in size.
|
||||
|
||||
ARGUMENT:: padSize
|
||||
The size of the handles on each sides of the block simply used for analysis purpose and avoid boundary issues.
|
||||
|
||||
ARGUMENT:: skew
|
||||
The nervousness of the bespoke detection function with values from -10 to 10. It allows to decide how peaks are amplified or smoothed before the thresholding. High values increase the sensitivity to small variations.
|
||||
|
||||
ARGUMENT:: threshFwd
|
||||
The threshold of the onset of the smoothed error function. It allows tight start of the identification of the anomaly as it proceeds forward.
|
||||
|
||||
ARGUMENT:: threshBack
|
||||
The threshold of the offset of the smoothed error function. As it proceeds backwards in time, it allows tight ending of the identification of the anomaly.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The averaging window of the error detection function. It needs smoothing as it is very jittery. The longer the window, the less precise, but the less false positive.
|
||||
|
||||
ARGUMENT:: clumpLength
|
||||
The window size in sample within which positive detections will be clumped together to avoid overdetecting in time.
|
||||
|
||||
ARGUMENT:: freeWhenDone
|
||||
Free the server instance when processing complete. Default true
|
||||
|
||||
ARGUMENT:: action
|
||||
A Function to be evaluated once the offline process has finished and all Buffer's instance variables have been updated on the client side. The function will be passed [transients, residual] as an argument.
|
||||
|
||||
returns:: an instance of the processor
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
c = Buffer.new(s);
|
||||
d = Buffer.new(s);
|
||||
)
|
||||
|
||||
// with basic params
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufTransients.process(s,b, transients:c, residual:d).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
);
|
||||
|
||||
// wait for the duration to appear in the post window as a cue that the computation is finished
|
||||
c.play;
|
||||
d.play;
|
||||
|
||||
//nullsumming tests
|
||||
{(PlayBuf.ar(1,c))+(PlayBuf.ar(1,d))+(-1*PlayBuf.ar(1,b,doneAction:2))}.play
|
||||
|
||||
|
||||
// with everything changed to make it much better, at the cost of computation time (only 5 seconds are processed here)
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufTransients.process(s,b, 0, 220500, 0, 1, c, d, 200, 2048, 1024, 1, 3, 1, 15, 30).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
// wait for the duration to appear in the post window as a cue that the computation is finished
|
||||
c.play;
|
||||
d.play;
|
||||
::
|
||||
|
||||
STRONG::A stereo buffer example.::
|
||||
CODE::
|
||||
|
||||
// load two very different files
|
||||
(
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav"));
|
||||
)
|
||||
|
||||
// composite one on left one on right as test signals
|
||||
FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000, destStartChan:1, destination:b)
|
||||
b.play
|
||||
|
||||
// create 2 new buffers as destinations
|
||||
d = Buffer.new(s); e = Buffer.new(s);
|
||||
|
||||
//run the process on them
|
||||
(
|
||||
Routine{
|
||||
t = Main.elapsedTime;
|
||||
FluidBufTransients.process(s, b, transients: d, residual:e, threshFwd:1.2, clumpLength:40).wait;
|
||||
(Main.elapsedTime - t).postln;
|
||||
}.play
|
||||
)
|
||||
|
||||
//listen: stereo preserved!
|
||||
d.play
|
||||
e.play
|
||||
::
|
||||
@ -1,227 +0,0 @@
|
||||
TITLE:: FluidChroma
|
||||
SUMMARY:: A histogram of pitch classes in Real-Time
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Classes/FluidBufChroma,Classes/FluidPitch,Classes/FluidLoudness,Classes/FluidMFCC,Classes/FluidSpectralShape,Guides/FluidCorpusManipulationToolkit,Classes/FluidMFCC
|
||||
DESCRIPTION::
|
||||
|
||||
|
||||
This class computes a histogram of the energy contained for each pitch class across the analysis frequency range.
|
||||
|
||||
|
||||
|
||||
Also known as a chromagram, this typically allows you to get a contour of how much each semitone is represented in the spectrum over time. The number of chroma bins (and, thus, pitch classes) and the central reference frequency can be adjusted.
|
||||
|
||||
The process will return a multichannel control steam of size maxNumChroma, which will be repeated if no change happens within the algorithm, i.e. when the hopSize is larger than the signal vector size.
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: kr
|
||||
|
||||
ARGUMENT:: in
|
||||
|
||||
Audio-rate signal to analyze
|
||||
|
||||
ARGUMENT:: numChroma
|
||||
|
||||
|
||||
The number of chroma bins per octave. It will determine how many channels are output per input channel.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 2
|
||||
|
||||
##
|
||||
Maximum: CODE::maxNumChroma::
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: ref
|
||||
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
##
|
||||
Maximum: 22000
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: normalize
|
||||
|
||||
|
||||
This flag enables the scaling of the output. It is off (0) by default. (1) will normalise each frame to sum to 1. (2) normalises each frame relative to the loudest chroma bin being 1.
|
||||
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
|
||||
|
||||
The lower frequency included in the analysis, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
|
||||
|
||||
The highest frequency included in the analysis, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: -1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As sinusoidal estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: maxNumChroma
|
||||
|
||||
|
||||
The maximum number of chroma bins. This sets the number of channels of the output stream, and therefore cannot be modulated.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 2
|
||||
|
||||
##
|
||||
Maximum: (max FFFT Size / 2) + 1`` (see maxFFTSize)
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
|
||||
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//create a monitoring bus for the descriptors
|
||||
b = Bus.new(\control,0,24);
|
||||
|
||||
//create a monitoring window for the values
|
||||
|
||||
(
|
||||
w = Window("Chroma Bins Monitor", Rect(10, 10, 620, 320)).front;
|
||||
a = MultiSliderView(w,Rect(10, 10, 600, 300)).elasticMode_(1).isFilled_(1);
|
||||
)
|
||||
|
||||
//run the window updating routine.
|
||||
(
|
||||
~winRange = 0.1;
|
||||
r = Routine {
|
||||
{
|
||||
b.get({ arg val;
|
||||
{
|
||||
if(w.isClosed.not) {
|
||||
a.value = val/~winRange;
|
||||
}
|
||||
}.defer
|
||||
});
|
||||
0.01.wait;
|
||||
}.loop
|
||||
}.play
|
||||
)
|
||||
|
||||
//play a simple sound to observe the values
|
||||
(
|
||||
x = {
|
||||
var source = SinOsc.ar(LFTri.kr(0.1).exprange(80,800),0,0.1);
|
||||
Out.kr(b,FluidChroma.kr(source, numChroma: 24, maxNumChroma:24));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
//we can observe that in the low end, the precision of the fft is not good enough to give a sharp pitch centre. We still can observe the octaviation quantized in quarter tones.
|
||||
|
||||
// free this source
|
||||
x.free
|
||||
|
||||
// load a more exciting one
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-SlideChoirAdd-M.wav"));
|
||||
|
||||
// analyse with parameters to be changed
|
||||
(
|
||||
x = {arg chroma = 24, low = 20, high = 20000, norm=1, t_cue, sel=0;
|
||||
var source = Select.ar(sel, [
|
||||
PlayBuf.ar(1,c,loop:1),
|
||||
Saw.ar(TIRand.kr(60.dup(3),96,t_cue).poll(t_cue).midicps,0.05).sum;
|
||||
]);
|
||||
Out.kr(b,FluidChroma.kr(source ,numChroma: chroma, minFreq: low, maxFreq: high, normalize: norm, maxNumChroma: 24, windowSize: 4096) / 10);
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
//set the winRange to a more informative value
|
||||
~winRange = 0.03;
|
||||
|
||||
//instead, let's normalise each frame independently
|
||||
~winRange = 0.12;
|
||||
x.set(\norm, 2);
|
||||
|
||||
// observe the number of chroma. The unused ones at the top are not updated
|
||||
x.set(\chroma,12)
|
||||
|
||||
// back to the full range
|
||||
x.set(\chroma,24)
|
||||
|
||||
// change the source to random three-note chords
|
||||
x.set(\sel, 1)
|
||||
|
||||
// trigger new chords and observe the chroma contour
|
||||
x.set(\t_cue, 1)
|
||||
|
||||
// focus all the chroma bin on a low mid range (there might be nothing!)
|
||||
x.set(\low,320, \high, 800)
|
||||
|
||||
// or on a specific octave
|
||||
x.set(\low, 60.midicps, \high, 72.midicps)
|
||||
|
||||
// back to full range
|
||||
x.set(\low,20, \high, 20000)
|
||||
|
||||
// free everything
|
||||
x.free;b.free;c.free;r.stop;
|
||||
::
|
||||
|
||||
STRONG::A musical example::
|
||||
|
||||
CODE::
|
||||
//something will happen here.
|
||||
::
|
||||
@ -1,185 +0,0 @@
|
||||
TITLE:: FluidDataSetQuery
|
||||
summary:: Query a FluidDataSet
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
A selection of columns and a set of conditions that match rows of a FluidDataSet.
|
||||
Use to filter and search in a database of descriptors.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Make a new instance
|
||||
ARGUMENT:: server
|
||||
The server on which to run this object
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init
|
||||
|
||||
METHOD:: addColumn
|
||||
Add a column to the query
|
||||
ARGUMENT:: column
|
||||
Column index
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: addRange
|
||||
Add a range of columns to the query
|
||||
ARGUMENT:: start
|
||||
First index
|
||||
ARGUMENT:: count
|
||||
Number of columns
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: filter
|
||||
Filter rows according to some condition.
|
||||
|
||||
Example: (3, ">", 0.5) filters rows where the value of the 4th column (starting at 0) is larger than 0.5.
|
||||
ARGUMENT:: column
|
||||
Column index
|
||||
ARGUMENT:: condition
|
||||
Condition string. Possible values: "==", "!=", "<", "<=", ">", ">="
|
||||
ARGUMENT:: value
|
||||
Condition value
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: and
|
||||
Add a condition to an existing filter with an "and" connector.
|
||||
ARGUMENT:: column
|
||||
Column index
|
||||
ARGUMENT:: condition
|
||||
Condition string. Possible values: "==", "!=", "<", "<=", ">", ">="
|
||||
ARGUMENT:: value
|
||||
Condition value
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
|
||||
METHOD:: or
|
||||
Add a condition to an existing filter with an "or" connector.
|
||||
ARGUMENT:: column
|
||||
Column index
|
||||
ARGUMENT:: condition
|
||||
Condition string. Possible values: "==", "!=", "<", "<=", ">", ">="
|
||||
ARGUMENT:: value
|
||||
Condition value
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: limit
|
||||
Limit the number of resulting rows.
|
||||
ARGUMENT:: rows
|
||||
Maximum number of rows
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: clear
|
||||
Clear the query, remove all columns, filters and limit.
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: transform
|
||||
Apply the query to a source link::Classes/FluidDataSet:: and write to a destination. Can be the same.
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: destDataSet
|
||||
Destination data, or the DataSet name
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: transformJoin
|
||||
Apply the query to a source link::Classes/FluidDataSet:: and join the resulting subset at the end of the items sharing the same identifiers in a second source. Items unique to a source dataset will be ignored. To add items at the end of a dataset instead, see the 'merge' method of link::Classes/FluidDataSet::
|
||||
ARGUMENT:: source1DataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: source2DataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: destDataSet
|
||||
Destination data, or the DataSet name
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
|
||||
s.reboot;
|
||||
// Create a DataSet with known data
|
||||
~dataSet= FluidDataSet(s);
|
||||
|
||||
(
|
||||
~points = 100.collect{|i|5.collect{|j|j+(i/100)}};
|
||||
~dataSet.clear;
|
||||
~tmpbuf = Buffer.alloc(s,5);
|
||||
|
||||
fork{
|
||||
s.sync;
|
||||
~points.do{|x,i|
|
||||
~tmpbuf.setn(0,x);
|
||||
~dataSet.addPoint(i,~tmpbuf);
|
||||
s.sync;
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
//check the source - the column is the integer part of the value, and the row is the fractional part. This will help us identify what we kept in our didactic query
|
||||
~dataSet.print;
|
||||
|
||||
// Prepare a FluidDataSetQuery object
|
||||
~query = FluidDataSetQuery.new;
|
||||
~out = FluidDataSet(s);
|
||||
|
||||
// prepare a simple query
|
||||
~query.filter(0,"<",0.04);
|
||||
~query.addColumn(2);
|
||||
~query.transform(~dataSet, ~out);
|
||||
|
||||
// check the result
|
||||
~out.print;
|
||||
|
||||
//prepare a more complex query
|
||||
~query.clear;
|
||||
~query.filter(0,">",0.03);
|
||||
~query.and(1,"<",1.08);
|
||||
~query.or(2,">",2.98);
|
||||
~query.addRange(2,2);
|
||||
~query.transform(~dataSet, ~out);
|
||||
|
||||
// Check the results
|
||||
~out.print;
|
||||
|
||||
::
|
||||
|
||||
STRONG:: Joining Datasets::
|
||||
|
||||
code::
|
||||
//this is how to join 2 datasets, adding columns to items with the same identifier
|
||||
//create 3 datasets
|
||||
(
|
||||
~dsA = FluidDataSet(s);
|
||||
~dsB = FluidDataSet(s);
|
||||
~dsC = FluidDataSet(s);
|
||||
)
|
||||
|
||||
//feed them items with almost overlaping identifier lists but with different dimensions
|
||||
~dsA.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\zero, [0,0], \one,[1,11],\two,[2,22], \three,[3,33],\four,[4,44]])]));
|
||||
~dsB.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\one,[111,1111],\two,[222,2222], \three,[333,3333],\four,[444,4444],\five,[555,5555]])]));
|
||||
~dsA.print;
|
||||
~dsB.print;
|
||||
|
||||
// no query/filter defined, copies all items with identifiers common to both, and all of the defined column of the first input
|
||||
~joiner = FluidDataSetQuery.new;
|
||||
~joiner.transformJoin(~dsA,~dsB,~dsC)
|
||||
~dsC.print
|
||||
|
||||
// all the sophisticated conditions applicable to 'transform' can be done on the first dataset too. Selected columns of the first source are appended to matching items in the second source.
|
||||
~joiner.filter(0,">",2.1)
|
||||
~joiner.and(1,"<", 40)
|
||||
~joiner.addColumn(0)
|
||||
~joiner.transformJoin(~dsA,~dsB,~dsC)
|
||||
~dsC.print
|
||||
::
|
||||
@ -1,38 +0,0 @@
|
||||
TITLE:: FluidGain
|
||||
SUMMARY:: Real-Time Buffered Gain Changer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation, UGens>Algebraic,UGens>Buffer
|
||||
RELATED:: Guides/FluidCorpusManipulation,Classes/UnaryOpFunction
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a sanity test for the FluCoMa Real-Time Client Wrapper. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
The audio rate version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The input to be processed
|
||||
|
||||
ARGUMENT:: gain
|
||||
Audio or control rate change of the gain.
|
||||
|
||||
RETURNS::
|
||||
Same as input, multiplied by the gain factor.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
Summing with the inverse (gain of -1) gives us CPU-expensive silence.
|
||||
CODE::
|
||||
{ var source = PinkNoise.ar(0.1); source + FluidGain.ar(source,-1); }.play
|
||||
::
|
||||
Varying the gain at audio rate.
|
||||
CODE::
|
||||
{ FluidGain.ar(PinkNoise.ar(0.1), gain:LFTri.ar(1)) }.play
|
||||
::
|
||||
Varying the gain at control rate, in beautiful stereo.
|
||||
CODE::
|
||||
{ FluidGain.ar(SinOsc.ar([222,333],mul:0.1), gain:LFTri.kr([0.5,0.7])) }.play
|
||||
::
|
||||
@ -1,201 +0,0 @@
|
||||
TITLE:: FluidGrid
|
||||
summary:: Constrain a 2D DataSet into a Grid.
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidMDS, Classes/FluidPCA, Classes/FluidUMAP, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
|
||||
Maps a set of 2D points in a link::Classes/FluidDataSet:: to a rectangular grid using a variant of the link::https://www.gwern.net/docs/statistics/decision/1987-jonker.pdf##Jonker-Volgenant algorithm::. It can be useful to obtain compact grid layouts from the output of dimensionality reduction algorithms such as link::Classes/FluidUMAP::, link::Classes/FluidPCA:: or link::Classes/FluidMDS::.
|
||||
|
||||
This is similar to projects like CloudToGrid (https://github.com/kylemcdonald/CloudToGrid/), RasterFairy (https://github.com/Quasimondo/RasterFairy) or IsoMatch (https://github.com/ohadf/isomatch).
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Make a new instance
|
||||
|
||||
ARGUMENT:: server
|
||||
The server on which to run this model
|
||||
|
||||
ARGUMENT:: oversample
|
||||
A factor to oversample the destination grid. The default is 1, so the grid has the same number of points as the input. Factors of 2 or more will allow a larger destination grid, which will respect the original shape a little more, but will also be sparser.
|
||||
|
||||
ARGUMENT:: extent
|
||||
The size to which the selected link::#axis:: will be constrained. The default is code::0::, which turns the constraints off.
|
||||
|
||||
ARGUMENT:: axis
|
||||
The axis to which the link::#extent:: constraint is applied. The default (code::0::) is horizontal, and (code::1::) is vertical.
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init
|
||||
|
||||
METHOD:: fitTransform
|
||||
Fit the model to a link::Classes/FluidDataSet:: and write the new projected data to a destination FluidDataSet.
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source FluidDataSet instance
|
||||
ARGUMENT:: destDataSet
|
||||
Destination FluidDataSet instance
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
STRONG::A didactic example::
|
||||
|
||||
code::
|
||||
|
||||
/// make a simple grid of numbers
|
||||
~simple = Dictionary.newFrom(36.collect{|i|[i.asSymbol, [i.mod(9), i.div(9)]]}.flatten(1));
|
||||
|
||||
//look at it
|
||||
(
|
||||
w = Window("the source", Rect(128, 64, 230, 100));
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
~simple.keysValuesDo{|key, val|
|
||||
Pen.stringCenteredIn(key, Rect((val[0] * 25), (val[1] * 25), 25, 25), Font( "Helvetica", 12 ), Color.black)
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
|
||||
//load it in a dataset
|
||||
~raw = FluidDataSet(s);
|
||||
~raw.load(Dictionary.newFrom([\cols, 2, \data, ~simple]));
|
||||
|
||||
// make a grid out of it
|
||||
~grid = FluidGrid(s);
|
||||
~gridified = FluidDataSet(s);
|
||||
~grid.fitTransform(~raw, ~gridified, action:{~gridified.dump{|x| ~gridifiedDict = x["data"]; \gridded.postln;}})
|
||||
|
||||
// watch the grid
|
||||
(
|
||||
w = Window("a perspective", Rect(358, 64, 350, 230));
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
~gridifiedDict.keysValuesDo{|key, val|
|
||||
Pen.stringCenteredIn(key, Rect((val[0] * 25), (val[1] * 25), 25, 25), Font( "Helvetica", 12 ), Color.black)
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
// change the constraints and draw again
|
||||
(
|
||||
~grid.axis_(0).extent_(4).fitTransform(~raw, ~gridified, action:{
|
||||
~gridified.dump{|x|
|
||||
~gridifiedDict = x["data"];\gridded.postln;
|
||||
{w.refresh;}.defer;
|
||||
}})
|
||||
)
|
||||
|
||||
// here we constrain in the other dimension
|
||||
(
|
||||
~grid.axis_(1).extent_(3).fitTransform(~raw, ~gridified, action:{
|
||||
~gridified.dump{|x|
|
||||
~gridifiedDict = x["data"];\gridded.postln;
|
||||
{w.refresh;}.defer;
|
||||
}})
|
||||
)
|
||||
|
||||
::
|
||||
|
||||
STRONG::A more colourful example exploring oversampling::
|
||||
|
||||
code::
|
||||
|
||||
// make all dependencies
|
||||
(
|
||||
~raw = FluidDataSet(s);
|
||||
~standardized = FluidDataSet(s);
|
||||
~reduced = FluidDataSet(s);
|
||||
~normalized = FluidDataSet(s);
|
||||
~standardizer = FluidStandardize(s);
|
||||
~normalizer = FluidNormalize(s, 0.05, 0.95);
|
||||
~umap = FluidUMAP(s).numDimensions_(2).numNeighbours_(5).minDist_(0.2).iterations_(50).learnRate_(0.2);
|
||||
~grid = FluidGrid(s);
|
||||
~gridified = FluidDataSet(s);
|
||||
)
|
||||
|
||||
// build a dataset of 400 points in 3D (colour in RGB)
|
||||
~colours = Dictionary.newFrom(400.collect{|i|[("entry"++i).asSymbol, 3.collect{1.0.rand}]}.flatten(1));
|
||||
~raw.load(Dictionary.newFrom([\cols, 3, \data, ~colours]));
|
||||
|
||||
//First standardize our DataSet, then apply the UMAP to get 2 dimensions, then normalise these 2 for drawing in the full window size
|
||||
(
|
||||
~standardizer.fitTransform(~raw,~standardized,action:{"Standardized".postln});
|
||||
~umap.fitTransform(~standardized,~reduced,action:{"Finished UMAP".postln});
|
||||
~normalizer.fitTransform(~reduced,~normalized,action:{"Normalized Output".postln});
|
||||
)
|
||||
//we recover the reduced dataset
|
||||
~normalized.dump{|x| ~normalizedDict = x["data"]};
|
||||
|
||||
//Visualise the 2D projection of our original 4D data
|
||||
(
|
||||
w = Window("a perspective", Rect(128, 64, 200, 200));
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
~normalizedDict.keysValuesDo{|key, val|
|
||||
Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]);
|
||||
Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5));
|
||||
~colours[key.asSymbol].flat;
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
//Force the UMAP-reduced dataset into a grid, normalise for viewing then print in another window
|
||||
(
|
||||
~grid.fitTransform(~reduced,~gridified,action:{"Gridded Output".postln;
|
||||
~normalizer.fitTransform(~gridified,~normalized,action:{"Normalized Output".postln;
|
||||
~normalized.dump{|x|
|
||||
~normalizedDict = x["data"];
|
||||
{
|
||||
y = Window("a grid", Rect(328, 64, 200, 200));
|
||||
y.drawFunc = {
|
||||
Pen.use {
|
||||
~normalizedDict.keysValuesDo{|key, val|
|
||||
Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]);
|
||||
Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5));
|
||||
~colours[key.asSymbol].flat;
|
||||
}
|
||||
}
|
||||
};
|
||||
y.refresh;
|
||||
y.front;
|
||||
}.defer;
|
||||
};
|
||||
});
|
||||
});
|
||||
)
|
||||
|
||||
// We can check the dimensions of the yielded grid by dumping the normalisation.The grid coordinates are zero-counting
|
||||
~normalizer.dump{|x|x["data_max"].postln}
|
||||
|
||||
// This looks ok, but let's improve it with oversampling
|
||||
(
|
||||
~grid.oversample_(3).fitTransform(~reduced,~gridified,action:{"Gridded Output".postln;
|
||||
~normalizer.fitTransform(~gridified,~normalized,action:{"Normalized Output".postln;
|
||||
~normalized.dump{|x|
|
||||
~normalizedDict = x["data"];
|
||||
{
|
||||
y.refresh;
|
||||
}.defer;
|
||||
};
|
||||
});
|
||||
});
|
||||
)
|
||||
|
||||
// Again, checking the normalisation dump to check the maxima of each dimension
|
||||
~normalizer.dump{|x|x["data_max"].postln}
|
||||
|
||||
::
|
||||
@ -1,177 +0,0 @@
|
||||
TITLE:: FluidKDTree
|
||||
summary:: KD Tree on the server
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
A server-side K-Dimensional tree for efficient neighbourhood searches of multi-dimensional data.
|
||||
|
||||
See https://scikit-learn.org/stable/modules/neighbors.html#nearest-neighbor-algorithms for more on KD Trees
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Make a new KDTree model for the given server.
|
||||
ARGUMENT:: server
|
||||
The server on which to make the model.
|
||||
ARGUMENT:: numNeighbours
|
||||
The number of neighbours to return. A 0 will return all points in order of distance. When a radius is defined, numNeighbours is the maximum of items returned.
|
||||
ARGUMENT:: radius
|
||||
The threshold of acceptable distance for a point to be returned. A 0 will bypass this function, returning numNeighbours points.
|
||||
ARGUMENT:: lookupDataSet
|
||||
An optional link::Classes/FluidDataSet:: from which data points will be returned for realtime queries. This does not need to be the same DataSet that the tree was fitted against, but does need to have matching labels. Using this mechanism, we have a way to, e.g. associate labels with segments of playback buffers, without needing pass strings around the server. warning::This parameter can not be changed after the instance of FluidKDTree has been created::
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
METHOD:: fit
|
||||
Build the tree by scanning the points of a LINK::Classes/FluidDataSet::
|
||||
ARGUMENT:: dataSet
|
||||
The LINK::Classes/FluidDataSet:: of interest. This can either be a data set object itself, or the name of one.
|
||||
ARGUMENT:: action
|
||||
A function to run when indexing is complete.
|
||||
|
||||
METHOD:: kNearest
|
||||
Returns the IDs of the CODE::k:: points nearest to the one passed.
|
||||
ARGUMENT:: buffer
|
||||
A LINK::Classes/Buffer:: containing a data point to match against. The number of frames in the buffer must match the dimensionality of the LINK::Classes/FluidDataSet:: the tree was fitted to.
|
||||
ARGUMENT:: action
|
||||
A function that will run when the query returns, whose argument is an array of point IDs from the tree's LINK::Classes/FluidDataSet::
|
||||
|
||||
METHOD:: kNearestDist
|
||||
Get the distances of the K nearest neighbours to a point.
|
||||
ARGUMENT:: buffer
|
||||
A LINK::Classes/Buffer:: containing a data point to match against. The number of frames in the buffer must match the dimensionality of the LINK::Classes/FluidDataSet:: the tree was fitted to.
|
||||
ARGUMENT:: action
|
||||
A function that will run when the query returns, whose argument is an array of distances.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
// Make a DataSet of random 2D points
|
||||
s.reboot;
|
||||
(
|
||||
fork{
|
||||
d = Dictionary.with(
|
||||
*[\cols -> 2,\data -> Dictionary.newFrom(
|
||||
100.collect{|i| [i, [ 1.0.linrand,1.0.linrand]]}.flatten)]);
|
||||
~ds = FluidDataSet(s);
|
||||
~ds.load(d, {~ds.print});
|
||||
}
|
||||
)
|
||||
|
||||
// Make a new tree, and fit it to the DataSet
|
||||
~tree = FluidKDTree(s,numNeighbours:5);
|
||||
|
||||
//Fit it to the DataSet
|
||||
~tree.fit(~ds);
|
||||
|
||||
// Should be 100 points, 2 columns
|
||||
~tree.size;
|
||||
~tree.cols;
|
||||
|
||||
//Return the labels of k nearest points to a new point
|
||||
(
|
||||
~p = [ 1.0.linrand,1.0.linrand ];
|
||||
~tmpbuf = Buffer.loadCollection(s, ~p, 1, {
|
||||
~tree.kNearest(~tmpbuf,{ |a|a.postln;~nearest = a;})
|
||||
});
|
||||
)
|
||||
|
||||
// Retrieve values from the DataSet by iterating through the returned labels
|
||||
(
|
||||
fork{
|
||||
~nearest.do{|n|
|
||||
~ds.getPoint(n, ~tmpbuf, {~tmpbuf.getn(0, 2, {|x|x.postln})});
|
||||
s.sync;
|
||||
}
|
||||
}
|
||||
)
|
||||
// Distances of the nearest points
|
||||
~tree.kNearestDist(~tmpbuf, { |a| a.postln });
|
||||
|
||||
// Explore changing the number of neighbourgs
|
||||
~tree.numNeighbours = 11; // note that this value needs to be sent to the server
|
||||
~tree.kNearest(~tmpbuf,{ |a|a.postln;});
|
||||
~tree.numNeighbours = 0; // 0 will return all items in order of distance
|
||||
~tree.kNearest(~tmpbuf,{ |a|a.postln;});
|
||||
|
||||
// Limit the search to an acceptable distance in a radius
|
||||
// Define a point, and observe typical distance values
|
||||
~p = [ 0.4,0.4];
|
||||
(
|
||||
~tmpbuf = Buffer.loadCollection(s, ~p, 1, {
|
||||
~tree.kNearest(~tmpbuf,{ |a|a.postln;});
|
||||
~tree.kNearestDist(~tmpbuf,{ |a|a.postln;});
|
||||
});
|
||||
)
|
||||
|
||||
// enter a valid radius.
|
||||
~tree.radius = 0.1;
|
||||
// FluidKDTree will return only values that are within that radius, up to numNeighbours values
|
||||
(
|
||||
~tmpbuf = Buffer.loadCollection(s, ~p, 1, {
|
||||
~tree.kNearest(~tmpbuf,{ |a|a.postln;});
|
||||
});
|
||||
)
|
||||
::
|
||||
|
||||
subsection:: Queries in a Synth
|
||||
|
||||
Input and output is done via buffers, which will need to be preallocated to the correct sizes:
|
||||
LIST::
|
||||
##Your input buffer should be sized to the input data dimension (2, in this example)
|
||||
##Your output buffer should be the number of neighbours * output dimensionality
|
||||
::
|
||||
|
||||
We can't simply return labels (i.e. strings) from a UGen, so the query returns the actual data points from a DataSet instead. By default, this is the FluidDataSet against which the tree was fitted. However, by passing a different dataset to code::kr::'s code::lookupDataSet:: argument instead, you can return different points, so long as the labels in the two datasets match. In this way, the FluidKDTree can be used to perform nearest neighbour mappings in a synth.
|
||||
|
||||
For instance, whilst fitting the tree against some n-dimensional descriptor data, our lookup dataset could use the same labels to map descriptor entries back to buffers, or locations in buffers, so that queries can be used to trigger audio.
|
||||
|
||||
code::
|
||||
|
||||
(
|
||||
Routine{
|
||||
var inputBuffer = Buffer.alloc(s,2);
|
||||
var outputBuffer = Buffer.alloc(s,10);//5 neighbours * 2D data points
|
||||
s.sync;
|
||||
{
|
||||
var trig = Impulse.kr(4); //can go as fast as ControlRate.ir/2
|
||||
var point = 2.collect{TRand.kr(0,1,trig)};
|
||||
point.collect{|p,i| BufWr.kr([p],inputBuffer,i)};
|
||||
~tree.kr(trig,inputBuffer,outputBuffer,5,nil);
|
||||
Poll.kr(trig, BufRd.kr(1,outputBuffer,Array.iota(10)),10.collect{|i| "Neighbour" + (i/2).asInteger ++ "-" ++ (i.mod(2))});
|
||||
Silent.ar;
|
||||
}.play;
|
||||
}.play;
|
||||
)
|
||||
|
||||
//Using a lookup data set instead:
|
||||
//here we populate with numbers that are in effect the indicies, but it could be anything numerical that will be returned on the server-side and would be usable on that side
|
||||
(
|
||||
fork{
|
||||
d = Dictionary.with(
|
||||
*[\cols -> 1,\data -> Dictionary.newFrom(
|
||||
100.collect{|i| [i, [ i ]]}.flatten)]);
|
||||
~dsL = FluidDataSet.new(s);
|
||||
~dsL.load(d, {~dsL.print});
|
||||
}
|
||||
)
|
||||
|
||||
// Create the buffers, and make a synth, querying our tree with some random points
|
||||
(
|
||||
Routine{
|
||||
var inputBuffer = Buffer.alloc(s,2);
|
||||
var outputBuffer = Buffer.alloc(s,5);//5 neighbours * 1D points
|
||||
s.sync;
|
||||
{
|
||||
var trig = Impulse.kr(4); //can go as fast as ControlRate.ir/2
|
||||
var point = 2.collect{TRand.kr(0,1,trig)};
|
||||
point.collect{|p,i| BufWr.kr([p],inputBuffer,i)};
|
||||
~tree.kr(trig,inputBuffer,outputBuffer,5,~dsL);
|
||||
Poll.kr(trig, BufRd.kr(1,outputBuffer,Array.iota(5)),5.collect{|i| "Neighbour" + i});
|
||||
Silent.ar;
|
||||
}.play;
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
@ -1,259 +0,0 @@
|
||||
TITLE:: FluidKMeans
|
||||
summary:: Cluster data points with K-Means
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidDataSet, Classes/FluidLabelSet, Classes/FluidKNNClassifier, Classes/FluidKNNRegressor
|
||||
|
||||
DESCRIPTION::
|
||||
Uses the K-Means algorithm to learn clusters from a link::Classes/FluidDataSet::
|
||||
|
||||
https://scikit-learn.org/stable/tutorial/statistical_inference/unsupervised_learning.html#clustering-grouping-observations-together
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Construct a new K Means model on the passed server. The parameters code::numClusters:: and code::maxIter:: can be modulated on an existing instance.
|
||||
ARGUMENT:: server
|
||||
If nil will use Server.default.
|
||||
ARGUMENT:: numClusters
|
||||
The number of clusters to classify data into.
|
||||
ARGUMENT:: maxIter
|
||||
The maximum number of iterations the algorithm will use whilst fitting.
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE::k
|
||||
|
||||
METHOD:: fit
|
||||
Identify code::numClusters:: clusters in a link::Classes/FluidDataSet::. It will optimise until no improvement is possible, or up to code::maxIter::, whichever comes first. Subsequent calls will continue training from the stopping point with the same conditions.
|
||||
ARGUMENT:: dataSet
|
||||
A link::Classes/FluidDataSet:: of data points.
|
||||
ARGUMENT:: action
|
||||
A function to run when fitting is complete, taking as its argument an array with the number of data points for each cluster.
|
||||
|
||||
METHOD:: predict
|
||||
Given a trained object, return the cluster ID for each data point in a link::Classes/FluidDataSet:: to a link::Classes/FluidLabelSet::.
|
||||
ARGUMENT:: dataSet
|
||||
A link::Classes/FluidDataSet:: containing the data to predict.
|
||||
ARGUMENT:: labelSet
|
||||
A link::Classes/FluidLabelSet:: to retrieve the predicted clusters.
|
||||
ARGUMENT:: action
|
||||
A function to run when the server responds.
|
||||
|
||||
METHOD:: fitPredict
|
||||
Run link::Classes/FluidKMeans#*fit:: and link::Classes/FluidKMeans#*predict:: in a single pass: i.e. train the model on the incoming link::Classes/FluidDataSet:: and then return the learned clustering to the passed link::Classes/FluidLabelSet::
|
||||
ARGUMENT:: dataSet
|
||||
a link::Classes/FluidDataSet:: containing the data to fit and predict.
|
||||
ARGUMENT:: labelSet
|
||||
a link::Classes/FluidLabelSet:: to retrieve the predicted clusters.
|
||||
ARGUMENT:: action
|
||||
A function to run when the server responds
|
||||
|
||||
METHOD:: predictPoint
|
||||
Given a trained object, return the cluster ID for a data point in a link::Classes/Buffer::
|
||||
ARGUMENT:: buffer
|
||||
A link::Classes/Buffer:: containing a data point.
|
||||
ARGUMENT:: action
|
||||
A function to run when the server responds, taking the ID of the cluster as its argument.
|
||||
|
||||
METHOD:: transform
|
||||
Given a trained object, return for each item of a provided link::Classes/FluidDataSet:: its distance to each cluster as an array, often reffered to as the cluster-distance space.
|
||||
ARGUMENT:: srcDataSet
|
||||
A link::Classes/FluidDataSet:: of data points to transform.
|
||||
ARGUMENT:: dstDataSet
|
||||
A link::Classes/FluidDataSet:: to contain the new cluster-distance space.
|
||||
ARGUMENT:: action
|
||||
A function to run when complete.
|
||||
|
||||
METHOD:: fitTransform
|
||||
Run link::Classes/FluidKMeans#*fit:: and link::Classes/FluidKMeans#*transform:: in a single pass: i.e. train the model on the incoming link::Classes/FluidDataSet:: and then return its cluster-distance space in the destination link::Classes/FluidDataSet::
|
||||
ARGUMENT:: srcDataSet
|
||||
A link::Classes/FluidDataSet:: containing the data to fit and transform.
|
||||
ARGUMENT:: dstDataSet
|
||||
A link::Classes/FluidDataSet:: to contain the new cluster-distance space.
|
||||
ARGUMENT:: action
|
||||
A function to run when complete.
|
||||
|
||||
METHOD:: transformPoint
|
||||
Given a trained object, return the distance of the provided point to each cluster. Both points are handled as link::Classes/Buffer::
|
||||
ARGUMENT:: sourceBuffer
|
||||
A link::Classes/Buffer:: containing a data point to query.
|
||||
ARGUMENT:: targetBuffer
|
||||
A link::Classes/Buffer:: containing a the distance of the source point to each cluster.
|
||||
ARGUMENT:: action
|
||||
A function to run when complete.
|
||||
|
||||
METHOD:: getMeans
|
||||
Given a trained object, retrieve the means (centroids) of each cluster as a link::Classes/FluidDataSet::
|
||||
ARGUMENT:: dataSet
|
||||
A link::Classes/FluidDataSet:: of clusers with a mean per column.
|
||||
ARGUMENT:: action
|
||||
A function to run when complete.
|
||||
|
||||
METHOD:: setMeans
|
||||
Overwrites the means (centroids) of each cluster, and declare the object trained.
|
||||
ARGUMENT:: dataSet
|
||||
A link::Classes/FluidDataSet:: of clusers with a mean per column.
|
||||
ARGUMENT:: action
|
||||
A function to run when complete.
|
||||
|
||||
METHOD:: clear
|
||||
Reset the object status to not fitted and untrained.
|
||||
ARGUMENT:: action
|
||||
A function to run when complete.
|
||||
|
||||
EXAMPLES::
|
||||
code::
|
||||
|
||||
(
|
||||
//Make some clumped 2D points and place into a DataSet
|
||||
~points = (4.collect{
|
||||
64.collect{(1.sum3rand) + [1,-1].choose}.clump(2)
|
||||
}).flatten(1) * 0.5;
|
||||
fork{
|
||||
~dataSet = FluidDataSet(s);
|
||||
d = Dictionary.with(
|
||||
*[\cols -> 2,\data -> Dictionary.newFrom(
|
||||
~points.collect{|x, i| [i, x]}.flatten)]);
|
||||
s.sync;
|
||||
~dataSet.load(d, {~dataSet.print});
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
// Create a KMeans instance and a LabelSet for the cluster labels in the server
|
||||
~clusters = FluidLabelSet(s);
|
||||
~kmeans = FluidKMeans(s);
|
||||
|
||||
// Fit into 4 clusters
|
||||
(
|
||||
~kmeans.fitPredict(~dataSet,~clusters,action: {|c|
|
||||
"Fitted.\n # Points in each cluster:".postln;
|
||||
c.do{|x,i|
|
||||
("Cluster" + i + "->" + x.asInteger + "points").postln;
|
||||
}
|
||||
});
|
||||
)
|
||||
|
||||
// Cols of kmeans should match DataSet, size is the number of clusters
|
||||
|
||||
~kmeans.cols;
|
||||
~kmeans.size;
|
||||
~kmeans.dump;
|
||||
|
||||
// Retrieve labels of clustered points
|
||||
(
|
||||
~assignments = Array.new(128);
|
||||
fork{
|
||||
128.do{ |i|
|
||||
~clusters.getLabel(i,{|clusterID|
|
||||
(i.asString+clusterID).postln;
|
||||
~assignments.add(clusterID)
|
||||
});
|
||||
s.sync;
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
//or faster by sorting the IDs
|
||||
~clusters.dump{|x|~assignments = x.at("data").atAll(x.at("data").keys.asArray.sort{|a,b|a.asInteger < b.asInteger}).flatten.postln;}
|
||||
|
||||
//Visualise: we're hoping to see colours neatly mapped to quandrants...
|
||||
(
|
||||
d = ((~points + 1) * 0.5).flatten(1).unlace;
|
||||
w = Window("scatter", Rect(128, 64, 200, 200));
|
||||
~colours = [Color.blue,Color.red,Color.green,Color.magenta];
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
d[0].size.do{|i|
|
||||
var x = (d[0][i]*200);
|
||||
var y = (d[1][i]*200);
|
||||
var r = Rect(x,y,5,5);
|
||||
Pen.fillColor = ~colours[~assignments[i].asInteger];
|
||||
Pen.fillOval(r);
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
// single point transform on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,0.5.dup);
|
||||
~kmeans.predictPoint(~inbuf,{|x|x.postln;});
|
||||
::
|
||||
|
||||
subsection:: Accessing the means
|
||||
|
||||
We can get and set the means for each cluster, their centroid.
|
||||
|
||||
code::
|
||||
// with the dataset and kmeans generated and trained in the code above
|
||||
~centroids = FluidDataSet(s);
|
||||
~kmeans.getMeans(~centroids, {~centroids.print});
|
||||
|
||||
// We can also set them to arbitrary values to seed the process
|
||||
~centroids.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\0, [0.5,0.5], \1, [-0.5,0.5], \2, [0.5,-0.5], \3, [-0.5,-0.5]])]));
|
||||
~centroids.print
|
||||
~kmeans.setMeans(~centroids, {~kmeans.predict(~dataSet,~clusters,{~clusters.dump{|x|var count = 0.dup(4); x["data"].keysValuesDo{|k,v|count[v[0].asInteger] = count[v[0].asInteger] + 1;};count.postln}})});
|
||||
|
||||
// We can further fit from the seeded means
|
||||
~kmeans.fit(~dataSet)
|
||||
// then retreive the improved means
|
||||
~kmeans.getMeans(~centroids, {~centroids.print});
|
||||
//subtle in this case but still.. each quadrant is where we seeded it.
|
||||
::
|
||||
|
||||
subsection:: Cluster-distance Space
|
||||
|
||||
We can get the euclidian distance of a given point to each cluster. This is often referred to as the cluster-distance space as it creates new dimensions for each given point, one distance per cluster.
|
||||
|
||||
code::
|
||||
// with the dataset and kmeans generated and trained in the code above
|
||||
b = Buffer.sendCollection(s,[0.5,0.5])
|
||||
c = Buffer(s)
|
||||
|
||||
// get the distance of our given point (b) to each cluster, thus giving us 4 dimensions in our cluster-distance space
|
||||
~kmeans.transformPoint(b,c,{|x|x.query;x.getn(0,x.numFrames,{|y|y.postln})})
|
||||
|
||||
// we can also transform a full dataset
|
||||
~srcDS = FluidDataSet(s)
|
||||
~cdspace = FluidDataSet(s)
|
||||
// make a new dataset with 4 points
|
||||
~srcDS.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\pp, [0.5,0.5], \np, [-0.5,0.5], \pn, [0.5,-0.5], \nn, [-0.5,-0.5]])]));
|
||||
~kmeans.transform(~srcDS, ~cdspace, {~cdspace.print})
|
||||
::
|
||||
|
||||
subsection:: Queries in a Synth
|
||||
|
||||
This is the equivalent of predictPoint, but wholly on the server
|
||||
|
||||
code::
|
||||
(
|
||||
{
|
||||
var trig = Impulse.kr(5);
|
||||
var point = WhiteNoise.kr(1.dup);
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(1);
|
||||
Poll.kr(trig, point, [\pointX,\pointY]);
|
||||
point.collect{ |p,i| BufWr.kr([p],inputPoint,i)};
|
||||
~kmeans.kr(trig,inputPoint,outputPoint);
|
||||
Poll.kr(trig,BufRd.kr(1,outputPoint,0,interpolation:0),\cluster);
|
||||
}.play;
|
||||
)
|
||||
|
||||
// to sonify the output, here are random values alternating quadrant, generated more quickly as the cursor moves rightwards
|
||||
(
|
||||
{
|
||||
var trig = Impulse.kr(MouseX.kr(0,1).exprange(0.5,ControlRate.ir / 2));
|
||||
var step = Stepper.kr(trig,max:3);
|
||||
var point = TRand.kr(-0.1, [0.1, 0.1], trig) + [step.mod(2).linlin(0,1,-0.6,0.6),step.div(2).linlin(0,1,-0.6,0.6)] ;
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(1);
|
||||
point.collect{|p,i| BufWr.kr([p],inputPoint,i)};
|
||||
~kmeans.kr(trig,inputPoint,outputPoint);
|
||||
SinOsc.ar((BufRd.kr(1,outputPoint,0,interpolation:0) + 69).midicps,mul: 0.1);
|
||||
}.play;
|
||||
)
|
||||
|
||||
::
|
||||
@ -1,181 +0,0 @@
|
||||
TITLE:: FluidKNNClassifier
|
||||
summary:: Classify data with K Nearest Neighbours
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidKNNRegressor, Classes/FluidDataSet, Classes/FluidLabelSet
|
||||
|
||||
DESCRIPTION::
|
||||
A nearest-neighbour classifier using link::Classes/FluidKDTree:: . Each point is assigned the class that is most common among its nearest neighbours.
|
||||
https://scikit-learn.org/stable/modules/neighbors.html#classification
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Create a new KNNClassifier
|
||||
ARGUMENT:: server
|
||||
The server to make the model on
|
||||
ARGUMENT:: numNeighbours
|
||||
the number of neighours to consider
|
||||
ARGUMENT:: weight
|
||||
true / false: whether the neighbours should be weighted by distance
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
METHOD:: fit
|
||||
Fit the model to a source link::Classes/FluidDataSet:: and a target link::Classes/FluidLabelSet::. These need to be the same size
|
||||
ARGUMENT:: dataSet
|
||||
Source data
|
||||
ARGUMENT:: labelSet
|
||||
Labels for the source data
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: predict
|
||||
Given a fitted model, predict labels for a link::Classes/FluidDataSet:: and write these to a link::Classes/FluidLabelSet::
|
||||
ARGUMENT:: dataSet
|
||||
data to predict labels for
|
||||
ARGUMENT:: labelSet
|
||||
place to write labels
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: predictPoint
|
||||
Given a fitted model, predict labels for a data point in a link::Classes/Buffer:: and return these to the caller
|
||||
ARGUMENT:: buffer
|
||||
A data point
|
||||
ARGUMENT:: action
|
||||
Run when done, passes predicted label as argument
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
|
||||
|
||||
// Make:
|
||||
// - A KNN Classifier
|
||||
// - A DataSet of example points, and a LabelSet of corresponding labels
|
||||
// - A DataSet of test data and a LabelSet for predicted labels
|
||||
|
||||
(
|
||||
~classifier = FluidKNNClassifier(s);
|
||||
~source= FluidDataSet(s);
|
||||
~labels = FluidLabelSet(s);
|
||||
~test = FluidDataSet(s);
|
||||
~mapping = FluidLabelSet(s);
|
||||
)
|
||||
|
||||
//Make some clumped 2D points and place into a DataSet
|
||||
(
|
||||
~examplepoints = [[0.5,0.5],[-0.5,0.5],[0.5,-0.5],[-0.5,-0.5]];
|
||||
~examplelabels = [\red,\orange,\green,\blue];
|
||||
d = Dictionary.new;
|
||||
d.add(\cols -> 2);
|
||||
d.add(\data -> Dictionary.newFrom(~examplepoints.collect{|x, i|[i.asString, x]}.flatten));
|
||||
~source.load(d);
|
||||
~examplelabels.collect{|x,i| ~labels.addLabel(i, x);};
|
||||
)
|
||||
|
||||
//Make some random, but clustered test points
|
||||
(
|
||||
~testpoints = (4.collect{
|
||||
64.collect{(1.sum3rand) + [1,-1].choose}.clump(2)
|
||||
}).flatten(1) * 0.5;
|
||||
d = Dictionary.with(
|
||||
*[\cols -> 2,\data -> Dictionary.newFrom(
|
||||
~testpoints.collect{|x, i| [i, x]}.flatten)]);
|
||||
~test.load(d);
|
||||
|
||||
)
|
||||
|
||||
|
||||
//Fit the classifier to the example DataSet and LabelSet, and then run prediction on the test data into our mapping LabelSet
|
||||
(
|
||||
~classifier.fit(~source,~labels);
|
||||
~classifier.predict(~test, ~mapping, 1);
|
||||
)
|
||||
|
||||
//Return labels of clustered points - wait for the dump to be done
|
||||
(
|
||||
~assignments = Array.new(~testpoints.size);
|
||||
fork{
|
||||
~testpoints.do{|x,i|
|
||||
~mapping.getLabel(i, action:{|l|
|
||||
~assignments.add(l);
|
||||
});
|
||||
s.sync;
|
||||
if(i==(~testpoints.size - 1)){"Got assignments".postln;}
|
||||
};
|
||||
~assignments.postln;
|
||||
}
|
||||
)
|
||||
|
||||
//Visualise: we're hoping to see colours neatly mapped to quandrants...
|
||||
(
|
||||
c = IdentityDictionary();
|
||||
|
||||
c.add(\red->Color.red);
|
||||
c.add(\blue->Color.blue);
|
||||
c.add(\green->Color.green);
|
||||
c.add(\orange-> Color.new255(255, 127, 0));
|
||||
|
||||
e = 200 * ((~examplepoints + 1) * 0.5).flatten(1).unlace;
|
||||
d = ((~testpoints + 1) * 0.5).flatten(1).unlace;
|
||||
// d = [20.collect{1.0.rand}, 20.collect{1.0.rand}];
|
||||
w = Window("scatter", Rect(128, 64, 200, 200));
|
||||
~colours = [Color.blue,Color.red,Color.green,Color.magenta];
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
e[0].size.do{|i|
|
||||
var r = Rect(e[0][i],e[1][i],10,10);
|
||||
Pen.fillColor = c[~examplelabels[i]];
|
||||
Pen.fillOval(r);
|
||||
};
|
||||
d[0].size.do{|i|
|
||||
var x = (d[0][i]*200);
|
||||
var y = (d[1][i]*200);
|
||||
var r = Rect(x,y,5,5);
|
||||
Pen.fillColor = c[~assignments[i].asSymbol].alpha_(0.3);
|
||||
Pen.fillOval(r);
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
// single point prediction on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,0.5.dup);
|
||||
~classifier.predictPoint(~inbuf,{|x|x.postln;});
|
||||
::
|
||||
|
||||
subsection::Server Side Queries
|
||||
This is the equivalent of predictPoint, but wholly on the server
|
||||
code::
|
||||
|
||||
//Generate a random point and sends a trigger to query, and return the class that point matches
|
||||
(
|
||||
{
|
||||
var trig = Impulse.kr(5);
|
||||
var point = WhiteNoise.kr(1.dup);
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(1);
|
||||
Poll.kr(trig, point, [\pointX,\pointY]);
|
||||
point.collect{ |p,i| BufWr.kr([p],inputPoint,i)};
|
||||
~classifier.kr(trig,inputPoint,outputPoint);
|
||||
Poll.kr(trig,BufRd.kr(1,outputPoint,0,interpolation:0),\cluster);
|
||||
}.play;
|
||||
)
|
||||
|
||||
// to sonify the output, here are random values alternating quadrant.
|
||||
(
|
||||
{
|
||||
var trig = Impulse.kr(MouseX.kr(0,1).exprange(0.5,ControlRate.ir /2).poll(trig:2, label: "Query Frequency"));
|
||||
var step = Stepper.kr(trig,max:3);
|
||||
var point = TRand.kr(-0.1, [0.1, 0.1], trig) + [step.mod(2).linlin(0,1,-0.6,0.6),step.div(2).linlin(0,1,-0.6,0.6)] ;
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(1);
|
||||
point.collect{|p,i| BufWr.kr([p],inputPoint,i)};
|
||||
~classifier.kr(trig,inputPoint,outputPoint);
|
||||
SinOsc.ar((BufRd.kr(1,outputPoint,0,interpolation:0) + 69).midicps, mul: 0.1);
|
||||
}.play
|
||||
)
|
||||
::
|
||||
@ -1,131 +0,0 @@
|
||||
TITLE:: FluidKNNRegressor
|
||||
summary:: Regression with K Nearest Neighbours
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidKNNClassifier, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
A nearest-neighbour regressor. A continuous value is predicted for each point as the (weighted) average value of its nearest neighbours.
|
||||
|
||||
https://scikit-learn.org/stable/modules/neighbors.html#regression
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Create a new KNN regressor on the server
|
||||
ARGUMENT:: server
|
||||
The server to run this model on.
|
||||
ARGUMENT:: numNeighbours
|
||||
number of neigbours to consider in mapping, min 1
|
||||
ARGUMENT:: weight
|
||||
Whether to weight neighbours by distance when producing new point
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
METHOD:: fit
|
||||
Map a source link::Classes/FluidDataSet:: to a one-dimensional target; both DataSets need to have the same number of points.
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data
|
||||
ARGUMENT:: targetDataSet
|
||||
Target data
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
|
||||
METHOD:: predict
|
||||
Apply learned mapping to a link::Classes/FluidDataSet:: and write to an output DataSet
|
||||
ARGUMENT:: sourceDataSet
|
||||
data to regress
|
||||
ARGUMENT:: targetDataSet
|
||||
output data
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: predictPoint
|
||||
Apply learned mapping to a data point in a link::Classes/Buffer::
|
||||
ARGUMENT:: buffer
|
||||
data point
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
|
||||
//Make a simple mapping between a ramp and a sine cycle, test with an exponentional ramp
|
||||
(
|
||||
~source = FluidDataSet(s);
|
||||
~target = FluidDataSet(s);
|
||||
~test = FluidDataSet(s);
|
||||
~output = FluidDataSet(s);
|
||||
~tmpbuf = Buffer.alloc(s,1);
|
||||
~regressor = FluidKNNRegressor(s);
|
||||
)
|
||||
|
||||
//Make source, target and test data
|
||||
(
|
||||
~sourcedata = 128.collect{|i|i/128};
|
||||
~targetdata = 128.collect{|i| sin(2*pi*i/128) };
|
||||
~testdata = 128.collect{|i|(i/128)**2};
|
||||
|
||||
~source.load(
|
||||
Dictionary.with(
|
||||
*[\cols -> 1,\data -> Dictionary.newFrom(
|
||||
~sourcedata.collect{|x, i| [i.asString, [x]]}.flatten)])
|
||||
);
|
||||
|
||||
~target.load(
|
||||
d = Dictionary.with(
|
||||
*[\cols -> 1,\data -> Dictionary.newFrom(
|
||||
~targetdata.collect{|x, i| [i.asString, [x]]}.flatten)]);
|
||||
);
|
||||
|
||||
~test.load(
|
||||
Dictionary.with(
|
||||
*[\cols -> 1,\data -> Dictionary.newFrom(
|
||||
~testdata.collect{|x, i| [i.asString, [x]]}.flatten)])
|
||||
);
|
||||
|
||||
~targetdata.plot;
|
||||
~source.print;
|
||||
~target.print;
|
||||
~test.print;
|
||||
|
||||
)
|
||||
|
||||
// Now make a regressor and fit it to the source and target, and predict against test
|
||||
//grab the output data whilst we're at it, so we can inspect
|
||||
(
|
||||
~outputdata = Array(128);
|
||||
~regressor.fit(~source, ~target);
|
||||
~regressor.predict(~test, ~output, 1, action:{
|
||||
~output.dump{|x| 128.do{|i|
|
||||
~outputdata.add(x["data"][i.asString][0])
|
||||
}};
|
||||
});
|
||||
)
|
||||
|
||||
//We should see a single cycle of a chirp
|
||||
~outputdata.plot;
|
||||
|
||||
// single point transform on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,[0.5]);
|
||||
~regressor.predictPoint(~inbuf,{|x|x.postln;});
|
||||
::
|
||||
|
||||
subsection:: Server Side Queries
|
||||
|
||||
code::
|
||||
//we are here querying with a saw in control rate, all on the server, via a buffer interface
|
||||
(
|
||||
{
|
||||
var input = Saw.kr(2).linlin(-1,1,0,1);
|
||||
var trig = Impulse.kr(ControlRate.ir/10);
|
||||
var inputPoint = LocalBuf(1);
|
||||
var outputPoint = LocalBuf(1);
|
||||
BufWr.kr(input,inputPoint,0);
|
||||
~regressor.kr(trig,inputPoint,outputPoint);
|
||||
BufRd.kr(1,outputPoint,0);
|
||||
}.scope
|
||||
)
|
||||
|
||||
::
|
||||
@ -1,75 +0,0 @@
|
||||
TITLE:: FluidLabelSet
|
||||
summary:: A set of labels associated with IDs
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidDataSet, Classes/FluidKMeans
|
||||
|
||||
DESCRIPTION::
|
||||
FluidLabelSet is a container associating identifiers with labels.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Make a new instance of a label set, uniquely identified by its name. Creating an instance with a name already in use will throw an exception. Use link::Classes/FluidLabelSet#*at:: or free the existing instance.
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to create the label set.
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init, id
|
||||
|
||||
METHOD:: addLabel
|
||||
Add a label to the label set.
|
||||
ARGUMENT:: identifier
|
||||
The identifier for this label.
|
||||
ARGUMENT:: label
|
||||
The label to add.
|
||||
ARGUMENT:: action
|
||||
A function to run when the operation completes.
|
||||
|
||||
METHOD:: updateLabel
|
||||
Change a label in the label set.
|
||||
ARGUMENT:: identifier
|
||||
The identifier for this label.
|
||||
ARGUMENT:: label
|
||||
The label to update.
|
||||
ARGUMENT:: action
|
||||
A function to run when the operation completes.
|
||||
|
||||
METHOD:: getLabel
|
||||
Retrieve the label associated with an identifier. Will report an error if the identifier is not present in the set.
|
||||
ARGUMENT:: identifier
|
||||
The identifier for the label to be retrieved.
|
||||
ARGUMENT:: action
|
||||
A function to run when the operation completes.
|
||||
|
||||
METHOD:: deleteLabel
|
||||
Delete a label given a certain identifier.
|
||||
ARGUMENT:: identifier
|
||||
The identifier to be deleted.
|
||||
ARGUMENT:: action
|
||||
A function to run when the operation completes.
|
||||
|
||||
METHOD:: clear
|
||||
Empty the label set.
|
||||
|
||||
METHOD:: getIds
|
||||
Export to the labelset IDs to a link::Classes/FluidLabelSet::.
|
||||
ARGUMENT:: labelSet
|
||||
The link::Classes/FluidLabelSet:: to export to. Its content will be replaced.
|
||||
ARGUMENT:: action
|
||||
A function to run when the export is done.
|
||||
|
||||
METHOD:: print
|
||||
Post an abbreviated content of the label set in the window by default, but you can supply a custom action instead.
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
~ls = FluidLabelSet.new(s);
|
||||
|
||||
["one", "two", "three"].collect{|x,i| ~ls.addLabel(i, x);};
|
||||
~ls.print;
|
||||
|
||||
~ls.free;
|
||||
::
|
||||
@ -1,161 +0,0 @@
|
||||
TITLE:: FluidLoudness
|
||||
SUMMARY:: A Loudness and True-Peak Descriptor in Real-Time
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Classes/FluidBufLoudness,Classes/FluidPitch,Classes/FluidMelBands,Classes/FluidMFCC,Classes/FluidSpectralShape,Guides/FluidCorpusManipulationToolkit
|
||||
DESCRIPTION::
|
||||
|
||||
|
||||
Two loudness descriptors, with a ITU BS 1770 mode
|
||||
|
||||
|
||||
|
||||
Computes the true peak of the signal as well as applying the filters proposed by broadcasting standards to emulate the perception of amplitude.
|
||||
|
||||
The process will return a multichannel control steam with [loudness, truepeak] values, both in dBFS, which will be repeated if no change happens within the algorithm, i.e. when the hopSize is larger than the signal vector size. More information on broadcasting standardisation of loudness measurement is available at the reference page (LINK::https://tech.ebu.ch/docs/tech/tech3341.pdf::) and in more musician-friendly explantions here (LINK::http://designingsound.org/2013/02/06/loudness-and-metering-part-1/::).
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: kr
|
||||
|
||||
ARGUMENT:: in
|
||||
|
||||
Audio-rate signal to analyze
|
||||
|
||||
ARGUMENT:: kWeighting
|
||||
|
||||
|
||||
A flag to switch the perceptual model of loudness. On by default, removing it makes the algorithm more CPU efficient by reverting to a simple RMS of the frame.
|
||||
|
||||
|
||||
ARGUMENT:: truePeak
|
||||
|
||||
|
||||
A flag to switch the computation of TruePeak. On by default, removing it makes the algorithm more CPU efficient by reverting to a simple absolute peak of the frame.
|
||||
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The size of the window on which the computation is done. By default 1024 to be similar with all other FluCoMa objects, the EBU specifies 400ms, which is 17640 samples at 44100.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Maximum: CODE::maxWindowSize::
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
How much the buffered window moves forward, in samples. By default 512 to be similar with all other FluCoMa objects, the EBU specifies 100ms, which is 4410 samples at 44100.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 1
|
||||
|
||||
::
|
||||
|
||||
|
||||
ARGUMENT:: maxWindowSize
|
||||
|
||||
|
||||
How large can the windowSize be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 4
|
||||
|
||||
##
|
||||
Snaps to powers of two
|
||||
|
||||
::
|
||||
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
|
||||
code::
|
||||
//create a monitoring bus for the descriptors
|
||||
b = Bus.new(\control,0,2);
|
||||
|
||||
//create a monitoring window for the values
|
||||
(
|
||||
w = Window("Loudness Monitor", Rect(10, 10, 220, 65)).front;
|
||||
|
||||
c = Array.fill(2, {arg i; StaticText(w, Rect(10, i * 25 + 10, 135, 20)).background_(Color.grey(0.7)).align_(\right)});
|
||||
c[0].string = ("Loudness: ");
|
||||
c[1].string = ("Peak: ");
|
||||
|
||||
a = Array.fill(2, {arg i;
|
||||
StaticText(w, Rect(150, i * 25 + 10, 60, 20)).background_(Color.grey(0.7)).align_(\center);
|
||||
});
|
||||
)
|
||||
|
||||
//routine to update the parameters
|
||||
(
|
||||
r = Routine {
|
||||
{
|
||||
b.get({ arg val;
|
||||
{
|
||||
if(w.isClosed.not) {
|
||||
val.do({arg item,index;
|
||||
a[index].string = item.round(0.01)})
|
||||
}
|
||||
}.defer
|
||||
});
|
||||
0.1.wait;
|
||||
}.loop
|
||||
}.play
|
||||
)
|
||||
|
||||
//basic test, with default values
|
||||
(
|
||||
x = {var source = PinkNoise.ar(0.25);
|
||||
Out.kr(b, FluidLoudness.kr(source));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
//free this
|
||||
x.free
|
||||
|
||||
//the EBU standard specifies that the window should be 400ms long, and update every 100ms, for instantaneous loudness. At SR=44100, this means the following settings. Various test signals are loaded.
|
||||
(
|
||||
x = {
|
||||
arg freq=220, type = 1, noise = 0;
|
||||
var source = PinkNoise.ar(noise) + Select.ar(type,[DC.ar(),SinOsc.ar(freq,mul:0.1), VarSaw.ar(freq,mul:0.1), Saw.ar(freq,0.1), Pulse.ar(freq,mul:0.1)]);
|
||||
Out.kr(b, FluidLoudness.kr(source,windowSize:17640,hopSize:4410,maxwindowSize:17640));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
// change the various frequencies to see the impact of the filter for the loudness. The TruePeak is steady.
|
||||
x.set(\freq, 440)
|
||||
x.set(\freq, 110)
|
||||
x.set(\freq, 55)
|
||||
x.set(\freq, 3000)
|
||||
x.set(\freq, 9000)
|
||||
|
||||
// adding harmonics, by changing to triangle (2), saw (3) or square (4) shows that spectral algo are more resilient when signal are richer
|
||||
x.set(\type, 2)
|
||||
x.set(\type, 3)
|
||||
x.set(\type, 4)
|
||||
|
||||
// adding noise shows its impact on loudness
|
||||
x.set(\noise, 0.25)
|
||||
|
||||
// and removing the oscilator
|
||||
x.set(\type, 0)
|
||||
|
||||
// and measuring silence
|
||||
x.set(\noise, 0)
|
||||
::
|
||||
@ -1,176 +0,0 @@
|
||||
TITLE:: FluidMDS
|
||||
summary:: Dimensionality Reduction with Multidimensional Scaling
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidPCA, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
|
||||
Multidimensional scaling of a link::Classes/FluidDataSet::
|
||||
|
||||
https://scikit-learn.org/stable/modules/manifold.html#multi-dimensional-scaling-mds
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
|
||||
METHOD:: new
|
||||
Make a new instance
|
||||
ARGUMENT:: server
|
||||
The server on which to run this model
|
||||
ARGUMENT:: numDimensions
|
||||
The number of dimensions to reduce to
|
||||
ARGUMENT:: distanceMetric
|
||||
The distance metric to use (integer, 0-6, see utility constants below)
|
||||
|
||||
METHOD:: euclidean
|
||||
Euclidean distance (default)
|
||||
|
||||
METHOD:: sqeuclidean
|
||||
Squared Euclidean distance
|
||||
|
||||
METHOD:: manhattan
|
||||
Manhattan distance
|
||||
|
||||
METHOD:: max
|
||||
Minkowski max
|
||||
|
||||
METHOD:: min
|
||||
Minkowski max
|
||||
|
||||
METHOD:: kl
|
||||
Symmetric Kulback Leiber divergance (only makes sense with non-negative data)
|
||||
|
||||
METHOD:: cosine
|
||||
Cosine distance
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init
|
||||
|
||||
METHOD:: fitTransform
|
||||
Fit the model to a link::Classes/FluidDataSet:: and write the new projected data to a destination FluidDataSet.
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: destDataSet
|
||||
Destination data, or the DataSet name
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
|
||||
//Preliminaries: we want some audio, a couple of FluidDataSets, some Buffers, a FluidStandardize and a FluidMDS
|
||||
(
|
||||
~audiofile = FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav");
|
||||
~raw = FluidDataSet(s);
|
||||
~standardized = FluidDataSet(s);
|
||||
~reduced = FluidDataSet(s);
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
~mfcc_feature = Buffer.new(s);
|
||||
~stats = Buffer.alloc(s, 7, 12);
|
||||
~standardizer = FluidStandardize(s);
|
||||
~mds = FluidMDS(s);
|
||||
)
|
||||
|
||||
// Load audio and run an mfcc analysis, which gives us 13 points (we'll throw the 0th away)
|
||||
(
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
FluidBufMFCC.process(s,~audio, features: ~mfcc_feature);
|
||||
)
|
||||
|
||||
// Divide the time series in 100, and take the mean of each segment and add this as a point to
|
||||
// the 'raw' FluidDataSet
|
||||
(
|
||||
{
|
||||
var trig = LocalIn.kr(1, 1);
|
||||
var buf = LocalBuf(12, 1);
|
||||
var count = PulseCount.kr(trig) - 1;
|
||||
var chunkLen = (~mfcc_feature.numFrames / 100).asInteger;
|
||||
var stats = FluidBufStats.kr(
|
||||
source: ~mfcc_feature, startFrame: count * chunkLen,
|
||||
startChan:1, numFrames: chunkLen, stats: ~stats, trig: trig, blocking:1
|
||||
);
|
||||
var rd = BufRd.kr(12, ~stats, DC.kr(0), 0, 1);
|
||||
var bufWr, dsWr;
|
||||
12.do{|i|
|
||||
bufWr = BufWr.kr(rd[i], buf, DC.kr(i));
|
||||
};
|
||||
dsWr = FluidDataSetWr.kr(~raw, buf: buf, idNumber: count, trig: Done.kr(stats),blocking:1);
|
||||
LocalOut.kr(Done.kr(dsWr));
|
||||
FreeSelf.kr(count - 99);
|
||||
Poll.kr(trig,(100-count));
|
||||
}.play;
|
||||
)
|
||||
// wait for the count to reaches 0 in the post window.
|
||||
|
||||
//First standardize our DataSet, so that the MFCC dimensions are on comensurate scales
|
||||
//Then apply the MDS in-place on the standardized data to get 2 dimensions, using a Euclidean distance metric
|
||||
//Download the DataSet contents into an array for plotting
|
||||
(
|
||||
~reducedarray = Array.new(100);
|
||||
~standardizer.fitTransform(~raw, ~standardized);
|
||||
~mds.fitTransform(~standardized, ~reduced, action:{
|
||||
~reduced.dump{|x| 100.do{|i|
|
||||
~reducedarray.add(x["data"][i.asString])
|
||||
}}});
|
||||
)
|
||||
|
||||
//Visualise the 2D projection of our original 12D data
|
||||
(
|
||||
d = ~reducedarray.flop.deepCollect(1, { |x| x.normalize});
|
||||
w = Window("scatter", Rect(128, 64, 200, 200));
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
d[0].size.do{|i|
|
||||
var x = (d[0][i]*200);
|
||||
var y = (d[1][i]*200);
|
||||
var r = Rect(x,y,5,5);
|
||||
Pen.fillColor = Color.blue;
|
||||
Pen.fillOval(r);
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
//we can change the distance computation
|
||||
~mds.distanceMetric = FluidMDS.kl;
|
||||
|
||||
//recompute the reduction and recover the points
|
||||
(
|
||||
~reducedarray2 = Array.new(100);
|
||||
~mds.fitTransform(~standardized, ~reduced, action:{
|
||||
~reduced.dump{|x| 100.do{|i|
|
||||
~reducedarray2.add(x["data"][i.asString])
|
||||
}}});
|
||||
)
|
||||
|
||||
//draw the new projection in red above the other
|
||||
//Visualise the 2D projection of our original 12D data
|
||||
(
|
||||
d = ~reducedarray.flop.deepCollect(1, { |x| x.normalize});
|
||||
e = ~reducedarray2.flop.deepCollect(1, { |x| x.normalize});
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
d[0].size.do{|i|
|
||||
var x = (d[0][i]*200);
|
||||
var y = (d[1][i]*200);
|
||||
var r = Rect(x,y,5,5);
|
||||
Pen.fillColor = Color.blue;
|
||||
Pen.fillOval(r);
|
||||
};
|
||||
e[0].size.do{|i|
|
||||
var x = (e[0][i]*200);
|
||||
var y = (e[1][i]*200);
|
||||
var r = Rect(x,y,5,5);
|
||||
Pen.fillColor = Color.red;
|
||||
Pen.fillOval(r);
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
::
|
||||
@ -1,383 +0,0 @@
|
||||
TITLE:: FluidMFCC
|
||||
SUMMARY:: Mel-Frequency Cepstral Coefficients as Spectral Descriptors in Real-Time
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Classes/FluidBufMFCC,Classes/FluidPitch,Classes/FluidMelBands,Classes/FluidLoudness,Classes/FluidSpectralShape,Guides/FluidCorpusManipulationToolkit
|
||||
DESCRIPTION::
|
||||
|
||||
|
||||
This class implements a classic spectral descriptor, the Mel-Frequency Cepstral Coefficients (MFCCs)
|
||||
|
||||
|
||||
|
||||
See LINK::https://en.wikipedia.org/wiki/Mel-frequency_cepstrum::. The input is first decomposed into perceptually spaced bands (the number of bands specified by numBands), just as in the MelBands object. It is then analysed in numCoefs number of cepstral coefficients. It has the avantage to be amplitude invarient, except for the first coefficient.
|
||||
|
||||
The process will return a multichannel control steam of maxNumCoeffs, which will be repeated if no change happens within the algorithm, i.e. when the hopSize is larger than the host vector size.
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: kr
|
||||
|
||||
ARGUMENT:: in
|
||||
|
||||
Audio-rate signal to analyze
|
||||
|
||||
ARGUMENT:: numCoeffs
|
||||
|
||||
|
||||
The number of cepstral coefficients to be outputed. It is limited by the maxNumCoefs parameter. When the number is smaller than the maximum, the output is zero-padded.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 2
|
||||
|
||||
##
|
||||
Maximum: MIN(CODE::numBands::, CODE::maxNumCoeffs::)
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: numBands
|
||||
|
||||
|
||||
The number of bands that will be perceptually equally distributed between minFreq and maxFreq to describe the spectral shape before it is converted to cepstral coefficients.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: MAX(CODE::numCoeffs::, 2)
|
||||
|
||||
##
|
||||
Maximum: CODE::(FFT Size / 2) + 1:: (see fft settings)
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: startCoeff
|
||||
|
||||
|
||||
The lowest index of the output cepstral coefficient, zero-counting.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
##
|
||||
Maximum: 1
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
|
||||
|
||||
The lower boundary of the lowest band of the model, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
|
||||
|
||||
The highest boundary of the highest band of the model, in Hz.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 0
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
|
||||
|
||||
The window size. As MFCC computation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::
|
||||
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
|
||||
|
||||
The window hop size. As MFCC computation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
|
||||
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize.
|
||||
|
||||
|
||||
|
||||
ARGUMENT:: maxNumCoeffs
|
||||
|
||||
|
||||
The maximum number of cepstral coefficients that can be computed. This sets the number of channels of the output, and therefore cannot be modulated.
|
||||
|
||||
STRONG::Constraints::
|
||||
|
||||
LIST::
|
||||
##
|
||||
Minimum: 2
|
||||
|
||||
##
|
||||
Maximum: (max FFFT Size / 2) + 1`` (see maxFFTSize)
|
||||
|
||||
::
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
|
||||
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//create a monitoring window for the values
|
||||
|
||||
(
|
||||
b = Bus.new(\control,0,13);
|
||||
w = Window("MFCCs Monitor", Rect(10, 10, 420, 320)).front;
|
||||
a = MultiSliderView(w,Rect(10, 10, 400, 300)).elasticMode_(1).isFilled_(1);
|
||||
a.reference_(Array.fill(13,{0.5})); //make a center line to show 0
|
||||
)
|
||||
|
||||
//run the window updating routine.
|
||||
(
|
||||
~winRange = 20;
|
||||
|
||||
r = Routine {
|
||||
{
|
||||
b.get({ arg val;
|
||||
{
|
||||
if(w.isClosed.not) {
|
||||
//val.postln;
|
||||
a.value = val.linlin(~winRange.neg,~winRange,0,1);
|
||||
}
|
||||
}.defer
|
||||
});
|
||||
0.01.wait;
|
||||
}.loop
|
||||
}.play
|
||||
)
|
||||
|
||||
//play a simple sound to observe the values
|
||||
(
|
||||
x = {arg type = 0;
|
||||
var source = Select.ar(type,[SinOsc.ar(220),Saw.ar(220),Pulse.ar(220)]) * LFTri.kr(0.1).exprange(0.01,0.1);
|
||||
Out.kr(b,FluidMFCC.kr(source,maxNumCoeffs:13));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// change the wave types, observe the amplitude invariance of the descriptors, apart from the leftmost coefficient
|
||||
x.set(\type, 1)
|
||||
~winRange = 40; //adjust the range above and below 0 to zoom in or out on the MFCC
|
||||
x.set(\type, 2)
|
||||
x.set(\type, 0)
|
||||
// free this source
|
||||
x.free
|
||||
|
||||
// load a more exciting one
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
|
||||
// analyse with parameters to be changed
|
||||
(
|
||||
x = {arg bands = 40, low = 20, high = 20000;
|
||||
var source = PlayBuf.ar(1,c,loop:1);
|
||||
Out.kr(b,FluidMFCC.kr(source, numCoeffs: 13, numBands: bands, minFreq: low, maxFreq: high, maxNumCoeffs: 13) / 10);
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
~winRange = 10; //adjust the range above and below 0 to zoom in or out on the MFCC
|
||||
|
||||
// observe the number of bands. The unused ones at the top are not updated
|
||||
x.set(\bands,20)
|
||||
|
||||
// back to the full range
|
||||
x.set(\bands,40)
|
||||
|
||||
// focus all the bands on a mid range
|
||||
x.set(\low,320, \high, 800)
|
||||
|
||||
// focusing on the low end shows the fft resolution issue. One could restart the analysis with a larger fft to show more precision
|
||||
x.set(\low,20, \high, 160)
|
||||
|
||||
// back to full range
|
||||
x.set(\low,20, \high, 20000)
|
||||
|
||||
// free everything
|
||||
x.free;b.free;c.free;r.stop;
|
||||
::
|
||||
|
||||
STRONG::A musical example::
|
||||
|
||||
CODE::
|
||||
//program that freezes mfcc spectra, then looks for matches between two frozen spectra
|
||||
(
|
||||
SynthDef("MFCCJamz", {arg freq=220, source = 0, buffer, mfccBus, distBus, t_freeze0=0, t_freeze1=0, onsetsOn0=0, onsetsOn1=0, restart = 1;
|
||||
var sound, mfcc, mfccFreeze0, mfccFreeze1, dist0, dist1, closest, slice;
|
||||
|
||||
sound = SelectX.ar(source, [
|
||||
SinOsc.ar(freq, 0, 0.1),
|
||||
LFTri.ar(freq, 0, 0.1),
|
||||
LFSaw.ar(freq, 0, 0.1),
|
||||
Pulse.ar(freq, 0.5, 0.1),
|
||||
WhiteNoise.ar(0.1),
|
||||
PinkNoise.ar(0.1),
|
||||
PlayBuf.ar(1, buffer, 1, loop:1, trigger:restart)
|
||||
]);
|
||||
|
||||
slice = FluidOnsetSlice.ar(sound); //onset detection for mfcc freeze on onset
|
||||
|
||||
mfcc = FluidMFCC.kr(sound,maxNumCoeffs:13);
|
||||
mfccFreeze0 = Latch.kr(mfcc, t_freeze0+(slice*onsetsOn0));
|
||||
mfccFreeze1 = Latch.kr(mfcc, t_freeze1+(slice*onsetsOn1));
|
||||
|
||||
Out.kr(mfccBus,mfcc.addAll(mfccFreeze0).addAll(mfccFreeze1));
|
||||
|
||||
//distance calculations
|
||||
|
||||
dist0 = Mix((mfcc.copyRange(1,12) - mfccFreeze0.copyRange(1,12)).squared).sqrt;
|
||||
dist1 = Mix((mfcc.copyRange(1,12) - mfccFreeze1.copyRange(1,12)).squared).sqrt;
|
||||
|
||||
Out.kr(distBus, [dist0, dist1]);
|
||||
|
||||
//sends a trigger when the item with a closer euclidean distance changes
|
||||
SendTrig.kr(Trig1.kr(dist1-dist0, 0.001)+Trig1.kr(dist0-dist1, 0.001), 0, dist1<dist0);
|
||||
|
||||
Out.ar(0, sound);
|
||||
}).load(s);
|
||||
)
|
||||
|
||||
|
||||
(
|
||||
var buffers, buffer, paths, mfccBus, freezeBus, distBus, win, sliders, updateRout, winRange, currentMFCC, synth, movingGUI, trainButtons, playbackButton, oscFunc, closestBus;
|
||||
|
||||
winRange = 100;
|
||||
|
||||
win = Window("MFCCs Monitor", Rect(10, 10, 450, 320)).front;
|
||||
sliders = List.newClear(0);
|
||||
3.do{|i|
|
||||
sliders.add(MultiSliderView().maxWidth_(150).maxHeight_(150)
|
||||
.elasticMode_(1).isFilled_(1)
|
||||
.reference_(Array.fill(13,{0.5})); //make a center line to show 0
|
||||
);
|
||||
};
|
||||
|
||||
//look in the directory for all .wav files
|
||||
paths = PathName(FluidFilesPath())
|
||||
.files.select({arg item; item.fullPath.contains(".wav")})
|
||||
.collect({arg item; item.fullPath});
|
||||
|
||||
|
||||
|
||||
s.waitForBoot({
|
||||
Routine({
|
||||
buffers = List.newClear(0);
|
||||
|
||||
paths.do{arg item;buffers.add(Buffer.read(s, item.postln));};
|
||||
|
||||
2.wait;
|
||||
s.sync;
|
||||
|
||||
mfccBus = Bus.control(s, 39);
|
||||
distBus = Bus.control(s, 2);
|
||||
closestBus = Bus.control(s);
|
||||
|
||||
synth = Synth("MFCCJamz", [\buffer, buffers[0], \mfccBus, mfccBus, \distBus, distBus, \closestBus, closestBus]);
|
||||
|
||||
//the slider and PopUps under the main mfcc display
|
||||
|
||||
movingGUI = [
|
||||
Slider().orientation_(\horizontal).action_{|sl| winRange = sl.value.linlin(0,1,50,500)},
|
||||
Slider().orientation_(\horizontal).action_{|sl| synth.set(\freq, sl.value.linexp(0,1,220,660))},
|
||||
PopUpMenu().items_(["sine", "tri", "saw", "square", "white", "pink", "buf"])
|
||||
.action_{|menu| synth.set(\source, menu.value)}
|
||||
.maxWidth_(150),
|
||||
PopUpMenu().items_(paths)
|
||||
.action_{|menu| synth.set(\buffer, buffers[menu.value])}
|
||||
.maxWidth_(150)
|
||||
];
|
||||
|
||||
//the buttons under the two frozen mfcc displays
|
||||
|
||||
trainButtons = List.newClear(0);
|
||||
2.do{arg i;
|
||||
i.postln;
|
||||
trainButtons.add(Button().states_([["train", Color.black, Color.green]])
|
||||
.action_{arg butt;
|
||||
//freezeBus.setn(currentMFCC);
|
||||
synth.set(("t_freeze"++i).asSymbol,1);
|
||||
};
|
||||
);
|
||||
trainButtons.add(Button().states_([["trainOnset", Color.black, Color.red],["trainOnset", Color.black, Color.green]])
|
||||
.action_{arg butt;
|
||||
synth.set(("onsetsOn"++i).asSymbol,butt.value);
|
||||
};
|
||||
);
|
||||
trainButtons.add(Button().states_([["", Color.black, Color.black],["", Color.black, Color.blue]]));
|
||||
trainButtons.add(StaticText());
|
||||
};
|
||||
|
||||
playbackButton = Button().states_([["restart", Color.black, Color.green]])
|
||||
.mouseDownAction_({arg butt;
|
||||
synth.set(\restart, 0.5);
|
||||
})
|
||||
.action_{arg butt;
|
||||
synth.set(\restart, -0.5);
|
||||
};
|
||||
|
||||
win.layout_(HLayout(
|
||||
VLayout(sliders[0], movingGUI[0], movingGUI[1], movingGUI[2]),
|
||||
VLayout(sliders[1],HLayout(trainButtons[0],trainButtons[1]), HLayout(trainButtons[2],trainButtons[3]), movingGUI[3]),
|
||||
VLayout(sliders[2],HLayout(trainButtons[4],trainButtons[5]), HLayout(trainButtons[6],trainButtons[7]), playbackButton)
|
||||
));
|
||||
win.front;
|
||||
win.onClose_{synth.free; oscFunc.free; updateRout.stop};
|
||||
|
||||
//receives a trigger when the nearest element changes
|
||||
oscFunc = OSCFunc({ arg msg, time;
|
||||
{
|
||||
trainButtons[2].value_(1-msg[3]);
|
||||
trainButtons[6].value_(msg[3]);
|
||||
}.defer
|
||||
},'/tr', s.addr);
|
||||
|
||||
//update the window
|
||||
updateRout = Routine {
|
||||
{arg i;
|
||||
mfccBus.get({ arg val;
|
||||
{
|
||||
currentMFCC = val;
|
||||
sliders[0].value = val.copyRange(0,12).linlin(winRange.neg,winRange,0,1);
|
||||
sliders[1].value = val.copyRange(13,25).linlin(winRange.neg,winRange,0,1);
|
||||
sliders[2].value = val.copyRange(26,38).linlin(winRange.neg,winRange,0,1);
|
||||
}.defer
|
||||
});
|
||||
distBus.get({ arg val;
|
||||
{
|
||||
trainButtons[3].string = val[0].round(0.001).asString;
|
||||
trainButtons[7].string = val[1].round(0.001).asString;
|
||||
|
||||
}.defer
|
||||
});
|
||||
0.05.wait;
|
||||
}.loop
|
||||
}.play;
|
||||
|
||||
}).play(AppClock);
|
||||
})
|
||||
)
|
||||
::
|
||||
@ -1,189 +0,0 @@
|
||||
TITLE:: FluidMLPClassifier
|
||||
summary:: Classification with a multi-layer perceptron
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidMLPRegressor, Classes/FluidDataSet
|
||||
|
||||
Perform classification between a link::Classes/FluidDataSet:: and a link::Classes/FluidLabelSet:: using a Multi-Layer Perception neural network.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Creates a new instance on the server.
|
||||
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run this model.
|
||||
|
||||
ARGUMENT:: hidden
|
||||
An link::Classes/Array:: that gives the sizes of any hidden layers in the network (default is two hidden layers of three units each).
|
||||
|
||||
ARGUMENT:: activation
|
||||
The activation function to use for the hidden layer units. Beware of the permitted ranges of each: relu (0->inf), sigmoid (0->1), tanh (-1,1).
|
||||
|
||||
ARGUMENT:: maxIter
|
||||
The maximum number of iterations to use in training.
|
||||
|
||||
ARGUMENT:: learnRate
|
||||
The learning rate of the network. Start small, increase slowly.
|
||||
|
||||
ARGUMENT:: momentum
|
||||
The training momentum, default 0.9
|
||||
|
||||
ARGUMENT:: batchSize
|
||||
The training batch size.
|
||||
|
||||
ARGUMENT:: validation
|
||||
The fraction of the DataSet size to hold back during training to validate the network against.
|
||||
|
||||
METHOD:: identity, sigmoid, relu, tanh
|
||||
A set of convinience constants for the available activation functions.
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init, uid
|
||||
|
||||
METHOD:: fit
|
||||
Train the network to map between a source link::Classes/FluidDataSet:: and a target link::Classes/FluidLabelSet::
|
||||
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data
|
||||
|
||||
ARGUMENT:: targetLabelSet
|
||||
Target data
|
||||
|
||||
ARGUMENT:: action
|
||||
Function to run when training is complete
|
||||
|
||||
returns:: The training loss, or -1 if training failed
|
||||
|
||||
METHOD:: predict
|
||||
Apply the learned mapping to a DataSet (given a trained network)
|
||||
|
||||
ARGUMENT:: sourceDataSet
|
||||
Input data
|
||||
|
||||
ARGUMENT:: targetLabelSet
|
||||
Output data
|
||||
|
||||
ARGUMENT:: action
|
||||
Function to run when complete
|
||||
|
||||
METHOD:: predictPoint
|
||||
Apply the learned mapping to a single data point in a link::Classes/Buffer::
|
||||
|
||||
ARGUMENT:: sourceBuffer
|
||||
Input point
|
||||
|
||||
ARGUMENT:: action
|
||||
A function to run when complete
|
||||
|
||||
METHOD:: clear
|
||||
This will erase all the learning done in the neural network.
|
||||
|
||||
ARGUMENT:: action
|
||||
A function to run when complete
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
(
|
||||
~classifier=FluidMLPClassifier(s).hidden_([6]).activation_(FluidMLPClassifier.tanh).maxIter_(1000).learnRate_(0.1).momentum_(0.1).batchSize_(50).validation_(0);
|
||||
~sourcedata= FluidDataSet(s);
|
||||
~labels = FluidLabelSet(s);
|
||||
~testdata = FluidDataSet(s);
|
||||
~predictedlabels = FluidLabelSet(s);
|
||||
)
|
||||
//Make some clumped 2D points and place into a DataSet
|
||||
(
|
||||
~centroids = [[0.5,0.5],[-0.5,0.5],[0.5,-0.5],[-0.5,-0.5]];
|
||||
~categories = [\red,\orange,\green,\blue];
|
||||
~trainingset = Dictionary();
|
||||
~labeldata = Dictionary();
|
||||
4.do{ |i|
|
||||
64.do{ |j|
|
||||
~trainingset.put("mlpclass"++i++\_++j, ~centroids[i].collect{|x| x.gauss(0.5/3)});
|
||||
~labeldata.put("mlpclass"++i++\_++j,[~categories[i]]);
|
||||
}
|
||||
};
|
||||
~sourcedata.load(Dictionary.with(*[\cols->2,\data->~trainingset]),action:{~sourcedata.print});
|
||||
~labels.load(Dictionary.with(*[\cols->1,\data->~labeldata]),action:{~labels.print});
|
||||
)
|
||||
|
||||
//Fit the classifier to the example DataSet and labels, and then run prediction on the test data into our mapping label set
|
||||
~classifier.fit(~sourcedata,~labels,action:{|loss| ("Trained"+loss).postln});
|
||||
|
||||
//make some test data
|
||||
(
|
||||
~testset = Dictionary();
|
||||
4.do{ |i|
|
||||
64.do{ |j|
|
||||
~testset.put("mlpclass_test"++i++\_++j, ~centroids[i].collect{|x| x.gauss(0.5/3)});
|
||||
}
|
||||
};
|
||||
~testdata.load(Dictionary.with(*[\cols->2,\data->~testset]),action:{~testdata.print});
|
||||
)
|
||||
|
||||
//Run the test data through the network, into the predicted labelset
|
||||
~classifier.predict(~testdata,~predictedlabels,action:{"Test complete".postln});
|
||||
|
||||
//get labels from server
|
||||
~predictedlabels.dump(action:{|d| ~labelsdict = d["data"]; ~labelsdict.postln});
|
||||
|
||||
//Visualise: we're hoping to see colours neatly mapped to quandrants...
|
||||
(
|
||||
c = Dictionary();
|
||||
c.add("red"->Color.red);
|
||||
c.add("blue"->Color.blue);
|
||||
c.add("green"->Color.green);
|
||||
c.add("orange"->Color.new255(255, 127, 0));
|
||||
e = 200 * ((~centroids + 1) * 0.5).flatten(1).unlace;
|
||||
w = Window("scatter", Rect(128, 64, 200, 200));
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
~testset.keysValuesDo{|k,v|
|
||||
var x = v[0].linlin(-1,1,200,0).asInteger;
|
||||
var y = v[1].linlin(-1,1,200,0).asInteger;
|
||||
var r = Rect(x,y,5,5);
|
||||
Pen.fillColor = c.at(~labelsdict[k][0]);
|
||||
Pen.fillOval(r);
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
// single point transform on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,0.5.dup);
|
||||
~classifier.predictPoint(~inbuf,{|x|x.postln;});
|
||||
::
|
||||
subsection::Querying in a Synth
|
||||
This is the equivalent of code::predictPoint::, but wholly on the server
|
||||
code::
|
||||
(
|
||||
{
|
||||
var trig = Impulse.kr(5);
|
||||
var point = WhiteNoise.kr(1.dup);
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(1);
|
||||
Poll.kr(trig, point, [\pointX,\pointY]);
|
||||
point.collect{ |p,i| BufWr.kr([p],inputPoint,i)};
|
||||
~classifier.kr(trig,inputPoint,outputPoint);
|
||||
Poll.kr(trig,BufRd.kr(1,outputPoint,0,interpolation:0),\cluster);
|
||||
Silent.ar;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// to sonify the output, here are random values alternating quadrant.
|
||||
(
|
||||
{
|
||||
var trig = Impulse.kr(MouseX.kr(0,1).exprange(0.5,ControlRate.ir /2).poll(trig: 2,label: "Query Frequency"));
|
||||
var step = Stepper.kr(trig,max:3);
|
||||
var point = TRand.kr(-0.1, [0.1, 0.1], trig) + [step.mod(2).linlin(0,1,-0.6,0.6),step.div(2).linlin(0,1,-0.6,0.6)] ;
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(1);
|
||||
point.collect{|p,i| BufWr.kr([p],inputPoint,i)};
|
||||
~classifier.kr(trig,inputPoint,outputPoint);
|
||||
SinOsc.ar((BufRd.kr(1,outputPoint,0,interpolation:0) + 69).midicps,mul: 0.1)
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
@ -1,191 +0,0 @@
|
||||
TITLE:: FluidMLPRegressor
|
||||
summary:: Regression with a multi-layer perceptron
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidMLPClassifier, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
Perform regression between link::Classes/FluidDataSet::s using a Multi-Layer Perception neural network.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Creates a new instance on the server.
|
||||
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run this model.
|
||||
|
||||
ARGUMENT:: hidden
|
||||
An link::Classes/Array:: that gives the sizes of any hidden layers in the network (default is two hidden layers of three units each).
|
||||
|
||||
ARGUMENT:: activation
|
||||
The activation function to use for the hidden layer units. Beware of the permitted ranges of each: relu (0->inf), sigmoid (0->1), tanh (-1,1).
|
||||
|
||||
ARGUMENT:: outputActivation
|
||||
The activation function to use for the final layer units. Beware of the permitted ranges of each: relu (0->inf), sigmoid (0->1), tanh (-1,1).
|
||||
|
||||
ARGUMENT:: tapIn
|
||||
The layer whose input is used to predict and predictPoint. It is 0 counting, where the default of 0 is the input layer, and 1 would be the first hidden layer, and so on.
|
||||
|
||||
ARGUMENT:: tapOut
|
||||
The layer whose output to return. It is counting from 0 as the input layer, and 1 would be the first hidden layer, and so on. The default of -1 is the last layer of the whole network.
|
||||
|
||||
ARGUMENT:: maxIter
|
||||
The maximum number of iterations to use in training.
|
||||
|
||||
ARGUMENT:: learnRate
|
||||
The learning rate of the network. Start small, increase slowly.
|
||||
|
||||
ARGUMENT:: momentum
|
||||
The training momentum, default 0.9
|
||||
|
||||
ARGUMENT:: batchSize
|
||||
The training batch size.
|
||||
|
||||
ARGUMENT:: validation
|
||||
The fraction of the DataSet size to hold back during training to validate the network against.
|
||||
|
||||
METHOD:: identity, sigmoid, relu, tanh
|
||||
A set of convinience constants for the available activation functions.
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init, uid
|
||||
|
||||
METHOD:: fit
|
||||
Train the network to map between a source and target link::Classes/FluidDataSet::
|
||||
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data
|
||||
|
||||
ARGUMENT:: targetDataSet
|
||||
Target data
|
||||
|
||||
ARGUMENT:: action
|
||||
Function to run when training is complete
|
||||
|
||||
returns:: The training loss, or -1 if training failed
|
||||
|
||||
METHOD:: predict
|
||||
Apply the learned mapping to a link::Classes/FluidDataSet:: (given a trained network)
|
||||
|
||||
ARGUMENT:: sourceDataSet
|
||||
Input data
|
||||
|
||||
ARGUMENT:: targetDataSet
|
||||
Output data
|
||||
|
||||
ARGUMENT:: action
|
||||
Function to run when complete
|
||||
|
||||
METHOD:: predictPoint
|
||||
Apply the learned mapping to a single data point in a link::Classes/Buffer::
|
||||
|
||||
ARGUMENT:: sourceBuffer
|
||||
Input point
|
||||
|
||||
ARGUMENT:: targetBuffer
|
||||
Output point
|
||||
|
||||
ARGUMENT:: action
|
||||
A function to run when complete
|
||||
|
||||
METHOD:: clear
|
||||
This will erase all the learning done in the neural network.
|
||||
|
||||
ARGUMENT:: action
|
||||
A function to run when complete
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
|
||||
//Make a simple mapping between a ramp and a sine cycle, test with an exponentional ramp
|
||||
(
|
||||
~source = FluidDataSet(s);
|
||||
~target = FluidDataSet(s);
|
||||
~test = FluidDataSet(s);
|
||||
~output = FluidDataSet(s);
|
||||
~tmpbuf = Buffer.alloc(s,1);
|
||||
~regressor=FluidMLPRegressor(s).hidden_([2]).activation_(FluidMLPRegressor.tanh).outputActivation_(FluidMLPRegressor.tanh).maxIter_(1000).learnRate_(0.1).momentum_(0.1).batchSize_(1).validation_(0);
|
||||
)
|
||||
|
||||
//Make source, target and test data
|
||||
(
|
||||
~sourcedata = 128.collect{|i|i/128};
|
||||
~targetdata = 128.collect{|i| sin(2*pi*i/128) };
|
||||
~testdata = 128.collect{|i|(i/128)**2};
|
||||
|
||||
~source.load(
|
||||
Dictionary.with(
|
||||
*[\cols -> 1,\data -> Dictionary.newFrom(
|
||||
~sourcedata.collect{|x, i| [i.asString, [x]]}.flatten)]);
|
||||
);
|
||||
|
||||
~target.load(
|
||||
d = Dictionary.with(
|
||||
*[\cols -> 1,\data -> Dictionary.newFrom(
|
||||
~targetdata.collect{|x, i| [i.asString, [x]]}.flatten)]);
|
||||
);
|
||||
|
||||
~test.load(
|
||||
Dictionary.with(
|
||||
*[\cols -> 1,\data -> Dictionary.newFrom(
|
||||
~testdata.collect{|x, i| [i.asString, [x]]}.flatten)]);
|
||||
);
|
||||
|
||||
~targetdata.plot;
|
||||
~source.print;
|
||||
~target.print;
|
||||
~test.print;
|
||||
)
|
||||
|
||||
// Now make a regressor and fit it to the source and target, and predict against test
|
||||
//grab the output data whilst we're at it, so we can inspect
|
||||
|
||||
// run this to train the network for up to 1000(max epochs to map source to target. fit() returns loss. If this is -1, then training has failed. Run until the printed error is satisfactory to you
|
||||
~regressor.fit(~source, ~target, {|x|x.postln;});
|
||||
|
||||
//you can change parameters of the MLPregressor with setters
|
||||
~regressor.learnRate = 0.01;
|
||||
~regressor.momentum = 0;
|
||||
~regressor.validation= 0.2;
|
||||
|
||||
(
|
||||
~outputdata = Array(128);
|
||||
~regressor.predict(~test, ~output, action:{
|
||||
~output.dump{|x| 128.do{|i|
|
||||
~outputdata.add(x["data"][i.asString][0])
|
||||
}};
|
||||
}
|
||||
);
|
||||
)
|
||||
|
||||
//We should see a single cycle of a chirp. If not, fit a little more epochs
|
||||
~outputdata.plot;
|
||||
|
||||
// single point transform on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,[0.5]);
|
||||
~outbuf = Buffer.new(s);
|
||||
~regressor.predictPoint(~inbuf,~outbuf,{|x|x.postln;x.getn(0,1,{|y|y.postln;};)});
|
||||
::
|
||||
|
||||
subsection:: Querying in a Synth
|
||||
|
||||
This is the equivalent of calling code::predictPoint::, except wholly on the server
|
||||
|
||||
code::
|
||||
(
|
||||
{
|
||||
var input = Saw.kr(2).linlin(-1,1,0,1);
|
||||
var trig = Impulse.kr(ControlRate.ir/10);
|
||||
var inputPoint = LocalBuf(1);
|
||||
var outputPoint = LocalBuf(1);
|
||||
BufWr.kr(input,inputPoint,0);
|
||||
~regressor.kr(trig,inputPoint,outputPoint,0,-1);
|
||||
Poll.kr(trig,BufRd.kr(1,outputPoint,0),"mapped value");
|
||||
}.scope;
|
||||
)
|
||||
|
||||
|
||||
|
||||
::
|
||||
@ -1,207 +0,0 @@
|
||||
TITLE:: FluidMelBands
|
||||
SUMMARY:: A Perceptually Spread Spectral Contour Descriptor in Real-Time
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Classes/FluidMFCC
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a spectral shape descriptor where the amplitude is given for a number of equally spread perceptual bands. The spread is based on the Mel scale (https://en.wikipedia.org/wiki/Mel_scale) which is one of the first attempt to mimic pitch perception scientifically. This implementation allows to select the range and number of bands dynamically. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return a multichannel control steam of size STRONG::maxNumBands::, which will be repeated if no change happens within the algorythm, i.e. when the hopSize is larger than the server's kr period.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: kr
|
||||
The audio rate in, control rate out version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The audio to be processed.
|
||||
|
||||
ARGUMENT:: numBands
|
||||
The number of bands that will be perceptually equally distributed between STRONG::minFreq:: and STRONG::maxFreq::. It is limited by the STRONG::maxNumBands:: parameter. When the number is smaller than the maximum, the output is zero-padded.
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
The lower boundary of the lowest band of the model, in Hz.
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
The highest boundary of the highest band of the model, in Hz.
|
||||
|
||||
ARGUMENT:: normalize
|
||||
This flag enables the scaling of the output to preserve the energy of the window. It is on (1) by default.
|
||||
|
||||
ARGUMENT:: scale
|
||||
This flag sets the scaling of the output value. It is either linear (0, by default) or in dB (1).
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As spectral description relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As spectral description relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
ARGUMENT:: maxNumBands
|
||||
The maximum number of Mel bands that can be modelled. This sets the number of channels of the output, and therefore cannot be modulated.
|
||||
|
||||
RETURNS::
|
||||
A KR signal of STRONG::maxNumBands:: channels, giving the measure amplitudes for each band. The latency is windowSize.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//create a monitoring bus for the descriptors
|
||||
b = Bus.new(\control,0,40);
|
||||
|
||||
//create a monitoring window for the values
|
||||
|
||||
(
|
||||
w = Window("Mel Bands Monitor", Rect(10, 10, 620, 320)).front;
|
||||
a = MultiSliderView(w,Rect(10, 10, 600, 300)).elasticMode_(1).isFilled_(1);
|
||||
)
|
||||
|
||||
//run the window updating routine.
|
||||
(
|
||||
~winRange = 0.1;
|
||||
r = Routine {
|
||||
{
|
||||
b.get({ arg val;
|
||||
{
|
||||
if(w.isClosed.not) {
|
||||
a.value = val/~winRange;
|
||||
}
|
||||
}.defer
|
||||
});
|
||||
0.01.wait;
|
||||
}.loop
|
||||
}.play
|
||||
)
|
||||
|
||||
//play a simple sound to observe the values
|
||||
(
|
||||
x = {
|
||||
var source = SinOsc.ar(LFTri.kr(0.1).exprange(80,800),0,0.1);
|
||||
Out.kr(b,FluidMelBands.kr(source,maxNumBands:40));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// free this source
|
||||
x.free
|
||||
|
||||
// load a more exciting one
|
||||
c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
|
||||
// analyse with parameters to be changed
|
||||
(
|
||||
x = {arg bands = 40, low = 20, high = 20000;
|
||||
var source = PlayBuf.ar(1,c,loop:1);
|
||||
Out.kr(b,FluidMelBands.kr(source, bands, low, high, 40) / 10);
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
//set the winRange to a more informative value
|
||||
~winRange = 0.05;
|
||||
|
||||
// observe the number of bands. The unused ones at the top are not updated
|
||||
x.set(\bands,20)
|
||||
|
||||
// back to the full range
|
||||
x.set(\bands,40)
|
||||
|
||||
// focus all the bands on a mid range: nothing to see!
|
||||
x.set(\low,320, \high, 800)
|
||||
|
||||
// focusing on the low end shows the fft resolution issue. One could restart the analysis with a larger fft to show more precision
|
||||
x.set(\low,20, \high, 160)
|
||||
|
||||
// back to full range
|
||||
x.set(\low,20, \high, 20000)
|
||||
|
||||
// free everything
|
||||
x.free;b.free;c.free;r.stop;
|
||||
::
|
||||
|
||||
STRONG::A musical example: a perceptually spread vocoder::
|
||||
|
||||
CODE::
|
||||
//load a source and define control bus for the resynthesis cluster
|
||||
(
|
||||
b = Bus.control(s,40);
|
||||
c = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
d = Group.new;
|
||||
)
|
||||
|
||||
//play the source and send the analysis on the
|
||||
(
|
||||
x = {
|
||||
arg dry = 0.2;
|
||||
var source = PlayBuf.ar(1,c,loop:1);
|
||||
Out.kr(b,FluidMelBands.kr(source,maxNumBands:40));
|
||||
Out.ar(0, DelayN.ar(source,delaytime:1024*SampleDur.ir,mul:dry));
|
||||
}.play;
|
||||
)
|
||||
|
||||
// set the dry playback volume
|
||||
x.set(\dry, 0.5)
|
||||
|
||||
// create a cluster of sines tuned on each MelBand center frequency, as a sort of vocoder.
|
||||
(
|
||||
var lowMel = 1127.010498 * ((20/700) + 1).log;
|
||||
var highMel = 1127.010498 * ((20000/700) + 1).log;
|
||||
var rangeMel = highMel - lowMel;
|
||||
var stepMel = rangeMel / 41;
|
||||
40.do({
|
||||
arg i;
|
||||
var freqMel = (stepMel * (i +1)) + lowMel;
|
||||
var freq = ((freqMel/ 1127.01048).exp - 1 ) * 700;
|
||||
{SinOsc.ar(freq,mul:Lag.kr(In.kr(b,40)[i],512*SampleDur.ir,0.5))}.play(d,1,addAction:\addToTail);
|
||||
});
|
||||
)
|
||||
|
||||
// free all
|
||||
d.free; x.free; b.free; c.free;
|
||||
|
||||
/////////////////////////////////////
|
||||
// instantiate a more dynamic vocoder:
|
||||
// MouseX defines the bottom frequency and MouseY define the top frequency, between which the 40 bands of analysis and synthesis are perceptually equally spred
|
||||
|
||||
// the bus, source and group
|
||||
(
|
||||
b = Bus.control(s,40);
|
||||
c = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
d = Group.new;
|
||||
)
|
||||
|
||||
// the modified source
|
||||
(
|
||||
x = {
|
||||
arg dry = 0.2;
|
||||
var source = PlayBuf.ar(1,c,loop:1);
|
||||
Out.kr(b,FluidMelBands.kr(source,maxNumBands:40,minFreq:MouseX.kr().exprange(20,600),maxFreq:MouseY.kr().exprange(650,20000)));
|
||||
Out.ar(0, DelayN.ar(source,delaytime:1024*SampleDur.ir,mul:dry));
|
||||
}.play;
|
||||
)
|
||||
|
||||
// the modified vocoder
|
||||
(
|
||||
40.do({
|
||||
arg i;
|
||||
{
|
||||
var lowMel = 1127.010498 * ((MouseX.kr().exprange(20,600)/700) + 1).log;
|
||||
var highMel = 1127.010498 * ((MouseY.kr().exprange(650,20000)/700) + 1).log;
|
||||
var rangeMel = highMel - lowMel;
|
||||
var stepMel = rangeMel / 41;
|
||||
var freqMel = (stepMel * (i +1)) + lowMel;
|
||||
var freq = ((freqMel/ 1127.01048).exp - 1 ) * 700;
|
||||
SinOsc.ar(freq,mul:Lag.kr(In.kr(b,40)[i],512*SampleDur.ir,0.5))}.play(d,1,addAction:\addToTail);
|
||||
});
|
||||
)
|
||||
|
||||
// free all
|
||||
d.free; x.free; b.free; c.free;
|
||||
::
|
||||
@ -1,93 +0,0 @@
|
||||
TITLE:: FluidNMFMorph
|
||||
summary:: Morph between sounds
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidAudioTransport,Classes/FluidBufNMFCross
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
Perform cross-synthesis using Nonnegative Matrix Factorization (NMF) and Optimal Transport
|
||||
(OT). NMF analyses of code::source:: and code::target:: sounds decompose their material in to a selectable number of components, which are in turn represented by their emphasis::bases:: (spectrum) and emphasis::activations:: (temporal pattern of each component).
|
||||
|
||||
code::FluidNMFMorph:: provides the ability to interpolate between code::source:: and code::target:: spectra using a technique called Optimal Transport, that provides richer results than a simple linear interpolation between spectral shapes. The resulting sound is built up using a buffer of temporal activations, then resynthesised using a phase estimate.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
Given buffers of spectral and temporal data from a NMF anlaysis such as produced by link::Classes/FluidBufNMF::, cross-synthesise a hybrid sound.
|
||||
|
||||
ARGUMENT:: source
|
||||
A link::Classes/Buffer:: with the spectral bases for the source sound.
|
||||
|
||||
ARGUMENT:: target
|
||||
A link::Classes/Buffer:: with the spectral bases for the target sound.
|
||||
|
||||
ARGUMENT:: activations
|
||||
A link::Classes/Buffer:: with the temporal activations for the target sound.
|
||||
|
||||
ARGUMENT:: autoassign
|
||||
If set to code::1:: the algorithm will attempt to optimally match which NMF basis components from source and target best match each other, and will use this mapping as its basis for interpolation. warning::changing this value re-initalizes the process::
|
||||
|
||||
ARGUMENT:: interp
|
||||
Set the relative contributions of code::source:: and code::target:: between 0 and 1.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The analysis window size in samples. Needs to match that of the seeding NMF analyses
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The analysis hop size in samples. Needs to match that of the seeding NMF analyses
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The analysis FFT size in samples. Needs to match that of the seeding NMF analyses
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
The maximum FFT size to allocate memory for
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
private:: checkInputs, init
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::FluidNMFMorph:: relies on preexisting NMF analyses to generate variations between sounds. We can produce these using link::Classes/FluidBufNMF::
|
||||
|
||||
code::
|
||||
//read some audio
|
||||
(
|
||||
~src1 = Buffer.readChannel(s,FluidFilesPath("Nicol-LoopE-M.wav"),channels:[0]); //some drums
|
||||
~src2 = Buffer.readChannel(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"),channels:[0]);//some piano
|
||||
|
||||
~src1Bases = Buffer.new;
|
||||
~src2Bases = Buffer.new;
|
||||
~src1Activations = Buffer.new;
|
||||
~src2Activations = Buffer.new;
|
||||
)
|
||||
//nmf analyses
|
||||
(
|
||||
FluidBufNMF.process(s,~src1,bases:~src1Bases,activations:~src1Activations,components:5, action:{"Analysed Source 1".postln});
|
||||
FluidBufNMF.process(s,~src2,bases:~src2Bases,activations:~src2Activations, components:5, action:{"Analysed Source 2".postln});
|
||||
)
|
||||
|
||||
(
|
||||
~morph = { |source, target, activations, interp, autoassign|
|
||||
FluidNMFMorph.ar(source,target,activations,autoassign,interp) * 80
|
||||
};
|
||||
)
|
||||
|
||||
~synth = ~morph.play(s,args:[\source,~src1Bases,\target,~src2Bases,\activations,~src2Activations,\interp,0.5,\autoassign,1]);
|
||||
|
||||
//Play with different interpolation values
|
||||
~synth.set(\interp,0.0);
|
||||
~synth.set(\interp,1.0);
|
||||
::
|
||||
warning::The following parameters current require one to change the 'autoassign' control to update the process::
|
||||
code::
|
||||
//Change the actvations
|
||||
~synth.set(\activations, ~src1Activations, \autoassign,0);
|
||||
~synth.set(\autoassign,1);
|
||||
~synth.set(\activations, ~src2Activations, \autoassign,0);
|
||||
~synth.set(\autoassign,1);
|
||||
|
||||
//Swap source and target
|
||||
~synth.set(\source,~src2Bases,\target,~src1Bases, \autoassign,0);
|
||||
~synth.set(\autoassign,1);
|
||||
::
|
||||
@ -1,156 +0,0 @@
|
||||
TITLE:: FluidNormalize
|
||||
summary:: Normalize a FluidDataSet
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidStandardize, Classes/FluidRobustScale, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
Normalize the entries of a link::Classes/FluidDataSet::, or normalize a data point according to the learned bounds of a data set. On the server.
|
||||
|
||||
See http://www.faqs.org/faqs/ai-faq/neural-nets/part2/section-16.html
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
private:: kr
|
||||
|
||||
METHOD:: new
|
||||
Create a new instance
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run
|
||||
ARGUMENT:: min
|
||||
Minimum output value, default 0
|
||||
ARGUMENT:: max
|
||||
Maximum output value, default 1
|
||||
ARGUMENT:: invert
|
||||
The direction in which the normalization will occur for transform and transformpoint. The default 0 is taking in the range of the input used to fit and transforms it towards the normalised range. A value of 1 will expect an input of the normalized range to transform back to the original range.
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
METHOD:: fit
|
||||
Compute the normalization factors from a link::Classes/FluidDataSet:: for later.
|
||||
ARGUMENT:: dataSet
|
||||
The link::Classes/FluidDataSet:: to normalize
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
METHOD:: transform
|
||||
Normalize a link::Classes/FluidDataSet:: into another link::Classes/FluidDataSet::, using the learned extrema from a previous call to link::Classes/FluidNormalize#fit::
|
||||
ARGUMENT:: sourceDataSet
|
||||
The link::Classes/FluidDataSet:: to normalize
|
||||
ARGUMENT:: destDataSet
|
||||
The link::Classes/FluidDataSet:: to populate with normalized data
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
|
||||
METHOD:: fitTransform
|
||||
Normalize a link::Classes/FluidDataSet::
|
||||
ARGUMENT:: sourceDataSet
|
||||
The link::Classes/FluidDataSet:: to normalize
|
||||
ARGUMENT:: destDataSet
|
||||
The link::Classes/FluidDataSet:: to populate with normalized data
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
|
||||
METHOD:: transformPoint
|
||||
Normalize a new data point, using the learned extrema from a previous call to link::Classes/FluidNormalize#fit::
|
||||
ARGUMENT:: sourceBuffer
|
||||
A link::Classes/Buffer:: with the new data point
|
||||
ARGUMENT:: destBuffer
|
||||
A link::Classes/Buffer:: to contain the normalized value
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
EXAMPLES::
|
||||
code::
|
||||
s.boot;
|
||||
//Preliminaries: we want some audio, a couple of FluidDataSets, some Buffers and a FluidNormalize
|
||||
// FluidNormalize.dumpAllMethods
|
||||
(
|
||||
~audiofile = FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav");
|
||||
~raw = FluidDataSet(s);
|
||||
~norm = FluidDataSet(s);
|
||||
~pitch_feature = Buffer.new(s);
|
||||
~stats = Buffer.alloc(s, 7, 2);
|
||||
~normalizer = FluidNormalize(s);
|
||||
)
|
||||
|
||||
// Load audio and run a pitch analysis, which gives us pitch and pitch confidence (so a 2D datum)
|
||||
(
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
FluidBufPitch.process(s,~audio, features: ~pitch_feature);
|
||||
)
|
||||
|
||||
// Divide the time series in to 10, and take the mean of each segment and add this as a point to
|
||||
// the 'raw' FluidDataSet
|
||||
(
|
||||
{
|
||||
var trig = LocalIn.kr(1, 1);
|
||||
var buf = LocalBuf(2, 1);
|
||||
var count = PulseCount.kr(trig) - 1;
|
||||
var chunkLen = (~pitch_feature.numFrames / 10).asInteger;
|
||||
var stats = FluidBufStats.kr(
|
||||
source: ~pitch_feature, startFrame: count * chunkLen,
|
||||
numFrames: chunkLen, stats: ~stats, trig: (trig * (count <=9)), blocking:1
|
||||
);
|
||||
var rd = BufRd.kr(2, ~stats, DC.kr(0), 0, 1);// pick only mean pitch and confidence
|
||||
var wr1 = BufWr.kr(rd[0], buf, DC.kr(0));
|
||||
var wr2 = BufWr.kr(rd[1], buf, DC.kr(1));
|
||||
var dsWr = FluidDataSetWr.kr(~raw, buf: buf, idNumber: count, trig: Done.kr(stats));
|
||||
LocalOut.kr( Done.kr(dsWr));
|
||||
Poll.kr(trig,count,\count);
|
||||
FreeSelf.kr(count - 9);
|
||||
}.play;
|
||||
)
|
||||
|
||||
// Normalize and load to language-side array
|
||||
(
|
||||
~rawarray = Array.new(10);
|
||||
~normedarray= Array.new(10);
|
||||
~normalizer.fitTransform(~raw,~norm, {
|
||||
~raw.dump{|x| 10.do{|i|
|
||||
~rawarray.add(x["data"][i.asString])
|
||||
}};
|
||||
~norm.dump{|x| 10.do{|i|
|
||||
~normedarray.add(x["data"][i.asString])
|
||||
}};
|
||||
});
|
||||
)
|
||||
|
||||
//Plot side by side. Before normalization the two dimensions have radically different scales
|
||||
//which can be unhelpful in many cases
|
||||
|
||||
(
|
||||
(~rawarray ++ 0).flop.plot("Unnormalized",Rect(0,0,400,400),minval:0,maxval:[5000,1]).plotMode=\bars;
|
||||
(~normedarray ++ 0).flop.plot("Normalized",Rect(410,0,400,400)).plotMode=\bars;
|
||||
)
|
||||
|
||||
// single point transform on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,0.5.dup);
|
||||
~outbuf = Buffer.new(s);
|
||||
~normalizer.transformPoint(~inbuf,~outbuf,{|x|x.postln;x.getn(0,2,{|y|y.postln;};)});
|
||||
|
||||
//Server side queries
|
||||
(
|
||||
{
|
||||
var audio = BufRd.ar(1,~audio,LFSaw.ar(BufDur.ir(~audio).reciprocal).range(0, BufFrames.ir(~audio)));
|
||||
var counter = Stepper.ar(Impulse.ar(ControlRate.ir),max:99);
|
||||
var trig = A2K.kr(HPZ1.ar(counter) < 0);
|
||||
//average 100 frames: one could use the MovingAverage extension here
|
||||
var avg;
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(2);
|
||||
var avgBuf = LocalBuf(100,2);
|
||||
//running average of pitch features
|
||||
BufWr.kr(FluidPitch.kr(audio),avgBuf,phase:counter);
|
||||
avg = Mix.new(BufRd.kr(2, avgBuf, phase:100.collect{|x|x})) * 0.01;
|
||||
//assemble data point
|
||||
BufWr.kr(avg[0],inputPoint,0);
|
||||
BufWr.kr(avg[1],inputPoint,1);
|
||||
~normalizer.kr(trig,inputPoint,outputPoint);
|
||||
Poll.kr(trig,BufRd.kr(1,inputPoint,[0,1]),["pitch (raw)", "confidence (raw)"]);
|
||||
Poll.kr(trig,BufRd.kr(1,outputPoint,[0,1]),["pitch (normalized)", "confidence (normalized)"])
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
@ -1,85 +0,0 @@
|
||||
TITLE:: FluidNoveltySlice
|
||||
SUMMARY:: Real-Time Novelty-Based Slicer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a real-time slicer using an algorithm assessing novelty in the signal to estimate the slicing points. A novelty curve is being derived from running a kernel across the diagonal of the similarity matrix, and looking for peak of changes. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return an audio steam with sample-long impulses at estimated starting points of the different slices.
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
The audio rate version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The audio to be processed.
|
||||
|
||||
ARGUMENT:: feature
|
||||
The feature on which novelty is computed.
|
||||
table::
|
||||
##0 || Spectrum || The magnitude of the full spectrum.
|
||||
##1 || MFCC || 13 Mel-Frequency Cepstrum Coefficients.
|
||||
##2 || Chroma || The contour of a 12 band Chromagram.
|
||||
##3 || Pitch || The pitch and its confidence.
|
||||
##4 || Loudness || The TruePeak and Loudness.
|
||||
::
|
||||
ARGUMENT:: kernelSize
|
||||
The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes.
|
||||
|
||||
ARGUMENT:: threshold
|
||||
The normalised threshold, between 0 an 1, on the novelty curve to consider it a segmentation point.
|
||||
|
||||
ARGUMENT:: filterSize
|
||||
The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes.
|
||||
|
||||
ARGUMENT:: minSliceLength
|
||||
The minimum duration of a slice in number of hopSize.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As sinusoidal estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
ARGUMENT:: maxKernelSize
|
||||
This cannot be modulated.
|
||||
|
||||
ARGUMENT:: maxFilterSize
|
||||
This cannot be modulated.
|
||||
|
||||
RETURNS::
|
||||
An audio stream with impulses at detected transients. The latency between the input and the output is STRONG::hopSize * (((kernelSize+1)/2).asInteger + ((filterSize + 1) / 2).asInteger + 1):: samples at minimum.
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//load some sounds
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
|
||||
// basic param (the process add a latency of windowSize samples
|
||||
{var sig = PlayBuf.ar(1,b,loop:1); [FluidNoveltySlice.ar(sig,0,11,0.33) * 0.5, DelayN.ar(sig, 1, (512 * (((11 + 1) / 2).asInteger + ((1 + 1) / 2).asInteger + 1)) / s.sampleRate, 0.2)]}.play
|
||||
|
||||
// other parameters
|
||||
{var sig = PlayBuf.ar(1,b,loop:1); [FluidNoveltySlice.ar(sig, 1, 31, 0.0035, 4, 100, 128, 32) * 0.5, DelayN.ar(sig, 1, (32 * (((31 + 1)/2).asInteger + ((4 + 1) / 2).asInteger + 1))/ s.sampleRate,0.2)]}.play
|
||||
|
||||
// More musical, novelty-trigged autopan
|
||||
(
|
||||
{
|
||||
var sig, trig, syncd, pan;
|
||||
sig = PlayBuf.ar(1,b,loop:1);
|
||||
trig = FluidNoveltySlice.ar(sig, 0, 11, 0.25, 5, 1, 128, 32);
|
||||
syncd = DelayN.ar(sig, 1, (32 * (((11 + 1)/2).asInteger + ((5 + 1) / 2).asInteger + 1))/ s.sampleRate);
|
||||
pan = TRand.ar(-1,1,trig);
|
||||
Pan2.ar(syncd,pan);
|
||||
}.play
|
||||
)
|
||||
::
|
||||
@ -1,85 +0,0 @@
|
||||
TITLE:: FluidOnsetSlice
|
||||
SUMMARY:: Spectral Difference-Based Real-Time Audio Slicer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements many spectral based onset detection metrics, most of them taken from the literature. (http://www.dafx.ca/proceedings/papers/p_133.pdf) Some are already available in SuperCollider's LINK::Classes/Onsets:: object. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return an audio steam with sample-long impulses at estimated starting points of the different slices.
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
The audio rate version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The audio to be processed.
|
||||
|
||||
ARGUMENT:: metric
|
||||
The metric used to derive a difference curve between spectral frames. It can be any of the following:
|
||||
TABLE::
|
||||
##0 || Energy || thresholds on (sum of squares of magnitudes / nBins) (like Onsets \power)
|
||||
##1 || HFC || thresholds on (sum of (squared magnitudes * binNum) / nBins)
|
||||
##2 || SpectralFlux || thresholds on (diffence in magnitude between consecutive frames, half rectified)
|
||||
##3 || MKL || thresholds on (sum of log of magnitude ratio per bin) (or equivalent: sum of difference of the log magnitude per bin) (like Onsets \mkl)
|
||||
##4 || IS || (WILL PROBABLY BE REMOVED) Itakura - Saito divergence (see literature)
|
||||
##5 || Cosine || thresholds on (cosine distance between comparison frames)
|
||||
##6 || PhaseDev || takes the past 2 frames, projects to the current, as anticipated if it was a steady state, then compute the sum of the differences, on which it thresholds (like Onsets \phase)
|
||||
##7 || WPhaseDev || same as PhaseDev, but weighted by the magnitude in order to remove chaos noise floor (like Onsets \wphase)
|
||||
##8 || ComplexDev || same as PhaseDev, but in the complex domain - the anticipated amp is considered steady, and the phase is projected, then a complex subtraction is done with the actual present frame. The sum of magnitudes is used to threshold (like Onsets \complex)
|
||||
##9 || RComplexDev || same as above, but rectified (like Onsets \rcomplex)
|
||||
::
|
||||
|
||||
ARGUMENT:: threshold
|
||||
The thresholding of a new slice. Value ranges are different for each metric, from 0 upwards.
|
||||
|
||||
ARGUMENT:: minSliceLength
|
||||
The minimum duration of a slice in number of hopSize.
|
||||
|
||||
ARGUMENT:: filterSize
|
||||
The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes.
|
||||
|
||||
ARGUMENT:: frameDelta
|
||||
For certain metrics (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distane between the comparison window in samples.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As sinusoidal estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
RETURNS::
|
||||
An audio stream with impulses at detected transients. The latency between the input and the output is windowSize at maximum.
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
//load some sounds
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
|
||||
// basic param (the process add a latency of windowSize samples
|
||||
{var sig = PlayBuf.ar(1,b,loop:1); [FluidOnsetSlice.ar(sig) * 0.5, DelayN.ar(sig, 1, 1024/ s.sampleRate)]}.play
|
||||
|
||||
// other parameters
|
||||
{var sig = PlayBuf.ar(1,b,loop:1); [FluidOnsetSlice.ar(sig, 9, 0.15, 45, 9, 0, 128, 64) * 0.5, DelayN.ar(sig, 1, (128)/ s.sampleRate)]}.play
|
||||
|
||||
// More musical, onset-trigged autopan
|
||||
(
|
||||
{
|
||||
var sig, trig, syncd, pan;
|
||||
sig = PlayBuf.ar(1,b,loop:1);
|
||||
trig = FluidOnsetSlice.ar(sig, 7, 0.2, 100, 9, 0, 128);
|
||||
syncd = DelayN.ar(sig, 1, (128 / s.sampleRate)); //one could add a bit more delay here to make sure to catch the attack
|
||||
pan = Stepper.ar(trig,-1,-1,1,1);
|
||||
Pan2.ar(syncd,pan);
|
||||
}.play
|
||||
)
|
||||
::
|
||||
@ -1,169 +0,0 @@
|
||||
TITLE:: FluidPCA
|
||||
summary:: Dimensionality Reduction with Principal Component Analysis
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidMDS, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
|
||||
Principal Components Analysis of a link::Classes/FluidDataSet::
|
||||
|
||||
https://scikit-learn.org/stable/modules/decomposition.html#principal-component-analysis-pca
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Make a new instance
|
||||
ARGUMENT:: server
|
||||
The server on which to run this model
|
||||
ARGUMENT:: numDimensions
|
||||
The number of dimensions to reduce to
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init
|
||||
|
||||
METHOD:: fit
|
||||
Train this model on a link::Classes/FluidDataSet:: but don't transform the data
|
||||
ARGUMENT:: dataSet
|
||||
A link::Classes/FluidDataSet:: to analyse
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: transform
|
||||
Given a trained model, apply the reduction to a source link::Classes/FluidDataSet:: and write to a destination. Can be the same for both (in-place)
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: destDataSet
|
||||
Destination data, or the DataSet name
|
||||
ARGUMENT:: action
|
||||
Run when done. The fraction of accounted variance is passed as an argument, aka the fidelity of the new representation: a value near 1.0 means a higher fidelity to the original.
|
||||
|
||||
METHOD:: fitTransform
|
||||
link::Classes/FluidPCA#fit:: and link::Classes/FluidPCA#transform:: in a single pass
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: destDataSet
|
||||
Destination data, or the DataSet name
|
||||
ARGUMENT:: action
|
||||
Run when done. The fraction of accounted variance is passed as an argument, aka the fidelity of the new representation: a value near 1.0 means a higher fidelity to the original.
|
||||
|
||||
METHOD:: transformPoint
|
||||
Given a trained model, transform the data point in a link::Classes/Buffer:: and write to an output
|
||||
ARGUMENT:: sourceBuffer
|
||||
Input data
|
||||
ARGUMENT:: destBuffer
|
||||
Output data
|
||||
ARGUMENT:: action
|
||||
Run when done. The function is passed code::destBuffer:: as argument.
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
s.reboot;
|
||||
//Preliminaries: we want some audio, a couple of FluidDataSets, some Buffers, a FluidStandardize and a FluidPCA
|
||||
(
|
||||
~audiofile = FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav");
|
||||
~raw = FluidDataSet(s);
|
||||
~standardized = FluidDataSet(s);
|
||||
~reduced = FluidDataSet(s);
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
~mfcc_feature = Buffer.new(s);
|
||||
~stats = Buffer.alloc(s, 7, 12);
|
||||
~datapoint = Buffer.alloc(s, 12);
|
||||
~standardizer = FluidStandardize(s);
|
||||
~pca = FluidPCA(s,2);
|
||||
)
|
||||
|
||||
|
||||
// Load audio and run an mfcc analysis, which gives us 13 points (we'll throw the 0th away)
|
||||
(
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
FluidBufMFCC.process(s,~audio, features: ~mfcc_feature,action:{"Done MFCCs".postln});
|
||||
)
|
||||
|
||||
// Divide the time series in 100, and take the mean of each segment and add this as a point to
|
||||
// the 'raw' FluidDataSet
|
||||
(
|
||||
{
|
||||
var trig = LocalIn.kr(1, 1);
|
||||
var buf = LocalBuf(12, 1);
|
||||
var count = PulseCount.kr(trig) - 1;
|
||||
var chunkLen = (~mfcc_feature.numFrames / 100).asInteger;
|
||||
var stats = FluidBufStats.kr(
|
||||
source: ~mfcc_feature, startFrame: count * chunkLen,
|
||||
startChan:1, numFrames: chunkLen, stats: ~stats,
|
||||
trig: trig * (count < 100), blocking: 1
|
||||
);
|
||||
var rd = BufRd.kr(12, ~stats, DC.kr(0), 0, 1);
|
||||
var bufWr, dsWr;
|
||||
12.do{|i|
|
||||
bufWr = BufWr.kr(rd[i], buf, DC.kr(i));
|
||||
};
|
||||
dsWr = FluidDataSetWr.kr(~raw, buf: buf, idNumber: count, trig: Done.kr(stats));
|
||||
LocalOut.kr( Done.kr(dsWr));
|
||||
FreeSelf.kr(count - 99);
|
||||
Poll.kr(trig,(100 - count));
|
||||
}.play;
|
||||
)
|
||||
// wait for the count to reaches 0 in the post window.
|
||||
|
||||
//First standardize our DataSet, so that the MFCC dimensions are on comensurate scales
|
||||
//Then apply the PCA in-place on the standardized data
|
||||
//Download the DataSet contents into an array for plotting
|
||||
(
|
||||
~reducedarray = Array.new(100);
|
||||
~standardizer.fitTransform(~raw, ~standardized);
|
||||
~pca.fitTransform(~standardized, ~reduced, action:{|x|
|
||||
x.postln; //pass on the variance
|
||||
~reduced.dump{|x| 100.do{|i|
|
||||
~reducedarray.add(x["data"][i.asString])
|
||||
}};
|
||||
});
|
||||
)
|
||||
|
||||
//Visualise the 2D projection of our original 12D data
|
||||
(
|
||||
d = ~reducedarray.flop.deepCollect(1, { |x| x.normalize});
|
||||
w = Window("scatter", Rect(128, 64, 200, 200));
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
d[0].size.do{|i|
|
||||
var x = (d[0][i]*200);
|
||||
var y = (d[1][i]*200);
|
||||
var r = Rect(x,y,5,5);
|
||||
Pen.fillColor = Color.blue;
|
||||
Pen.fillOval(r);
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
// transform a single point with arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,0.5.dup(12));
|
||||
~outbuf = Buffer.new(s);
|
||||
~pca.transformPoint(~inbuf,~outbuf,{|x|x.postln;x.getn(0,1,{|y|y.postln;};)});
|
||||
::
|
||||
|
||||
subsection:: Server Side Queries
|
||||
|
||||
Let's map our learned PCA dimensions to the controls of a processor
|
||||
|
||||
code::
|
||||
(
|
||||
{
|
||||
var mapped;
|
||||
var audio = BufRd.ar(1,~audio,LFSaw.ar(BufDur.ir(~audio).reciprocal).range(0, BufFrames.ir(~audio)));
|
||||
var mfcc = FluidMFCC.kr(audio)[1..12];
|
||||
var smoothed = LagUD.kr(mfcc,1*ControlDur.ir,500*ControlDur.ir);
|
||||
var trig = Impulse.kr(ControlRate.ir / 2);
|
||||
var inputPoint = LocalBuf(12);
|
||||
var outputPoint = LocalBuf(2);
|
||||
smoothed.collect{|coeff,i| BufWr.kr([coeff],inputPoint,i)};
|
||||
~pca.kr(trig, inputPoint, outputPoint, 2);
|
||||
mapped = BufRd.kr(1,outputPoint, phase:[0,1]).linlin(-3,3,0,3);
|
||||
CombC.ar(audio,3,mapped[0],mapped[1]*3)
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
@ -1,164 +0,0 @@
|
||||
TITLE:: FluidPitch
|
||||
SUMMARY:: A Selection of Pitch Descriptors in Real-Time
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Classes/Pitch
|
||||
|
||||
DESCRIPTION::
|
||||
FluidPitch implements three popular pitch descriptors. It outputs two values: the computed frequency and the confidence in the accuracy of that frequency. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning materials, and discussions on its musicianly uses, visit https://www.flucoma.org/reference/pitch
|
||||
|
||||
FluidPitch outputs a multichannel control steam of [pitch, confidence] values. The 'unit' argument changes the unit of the pitch output. 'unit' set to 0 will return a frequency in Hz, 'unit' set to 1 will return the MIDI note (with a decimal for microtonal values). If the chosen algorithm cannot determine a fundamental pitch at all, a frequency of 0 Hz is returned (or -999.0 when the unit is in MIDI note).
|
||||
|
||||
When the hopSize is larger than the server's kr period, the returned values will be repeated until the next window is computed.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: kr
|
||||
The audio rate in, control rate out
|
||||
|
||||
ARGUMENT:: in
|
||||
The audio to be processed.
|
||||
|
||||
ARGUMENT:: algorithm
|
||||
The algorithm to estimate the pitch. The options are:
|
||||
TABLE::
|
||||
## 0 || Cepstrum: Returns a pitch estimate as the location of the second highest peak in the Cepstrum of the signal (after DC).
|
||||
## 1 || Harmonic Product Spectrum: Implements the Harmonic Product Spectrum algorithm for pitch detection . See e.g. FOOTNOTE:: A. Lerch, "An Introduction to Audio Content Analysis: Applications in Signal Processing and Music Informatics." John Wiley & Sons, 2012.https://onlinelibrary.wiley.com/doi/book/10.1002/9781118393550 ::
|
||||
## 2 || YinFFT: Implements the frequency domain version of the YIN algorithm, as described in FOOTNOTE::P. M. Brossier, "Automatic Annotation of Musical Audio for Interactive Applications.” QMUL, London, UK, 2007. :: See also FOOTNOTE::https://essentia.upf.edu/documentation/reference/streaming_PitchYinFFT.html::
|
||||
::
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
The minimum frequency that the algorithm will search for an estimated fundamental. This sets the lowest value that will be generated. The default is 20.
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
The maximum frequency that the algorithm will search for an estimated fundamental. This sets the highest value that will be generated. The default is 10000.
|
||||
|
||||
ARGUMENT:: unit
|
||||
The unit of the estimated frequency. The default of 0 is in Hz. A value of 1 will convert to MIDI note values.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. The number of samples used to calculate the FFT.
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It must be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal to or above the windowSize.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
RETURNS::
|
||||
A 2-channel KR signal with the [pitch, confidence] descriptors. The latency is windowSize.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
|
||||
code::
|
||||
//create a monitoring bus for the descriptors
|
||||
b = Bus.new(\control,0,4);
|
||||
|
||||
//create a monitoring window for the values
|
||||
(
|
||||
w = Window("Frequency Monitor", Rect(10, 10, 220, 115)).front;
|
||||
|
||||
c = Array.fill(4, {arg i; StaticText(w, Rect(10, i * 25 + 10, 135, 20)).background_(Color.grey(0.7)).align_(\right)});
|
||||
c[0].string = ("FluidPitch: ");
|
||||
c[1].string = ("confidence: ");
|
||||
c[2].string = ("SC Pitch: ");
|
||||
c[3].string = ("Confidence: ");
|
||||
|
||||
a = Array.fill(4, {arg i;
|
||||
StaticText(w, Rect(150, i * 25 + 10, 60, 20)).background_(Color.grey(0.7)).align_(\center);
|
||||
});
|
||||
)
|
||||
|
||||
//routine to update the parameters
|
||||
(
|
||||
r = Routine {
|
||||
{
|
||||
|
||||
b.get({ arg val;
|
||||
{
|
||||
if(w.isClosed.not) {
|
||||
val.do({arg item,index;
|
||||
a[index].string = item.round(0.01)})
|
||||
}
|
||||
}.defer
|
||||
});
|
||||
|
||||
0.1.wait;
|
||||
}.loop
|
||||
|
||||
}.play
|
||||
)
|
||||
|
||||
//test signals, all in one synth
|
||||
(
|
||||
x = {
|
||||
arg freq=220, type = 0, noise = 0;
|
||||
var source = PinkNoise.ar(noise) + Select.ar(type,[SinOsc.ar(freq,mul:0.1), VarSaw.ar(freq,mul:0.1), Saw.ar(freq,0.1), Pulse.ar(freq,mul:0.1), Mix.new(Array.fill(8, {arg i; SinOsc.ar(LFNoise1.kr(0.1.rand,10,220*(i+1)),mul:(i+1).reciprocal * 0.1)}))]);
|
||||
Out.kr(b, FluidPitch.kr(source) ++ Pitch.kr(source));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// the built-in is slightly better on pure sinewaves
|
||||
x.set(\freq, 440)
|
||||
|
||||
// adding harmonics, by changing to triangle (1), saw (2) or square (3) shows that spectral algo are more resilient when signal are richer
|
||||
x.set(\type, 1)
|
||||
x.set(\type, 2)
|
||||
x.set(\type, 3)
|
||||
|
||||
// adding noise shows the comparative sturdiness of the spectral pitch tracker
|
||||
x.set(\noise, 0.05)
|
||||
|
||||
//if latency is no issue, getting a higher windowSize will stabilise the algorithm even more
|
||||
::
|
||||
|
||||
STRONG::a more musical example::
|
||||
CODE::
|
||||
// play a noisy synth file
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav"));
|
||||
b.play(true);
|
||||
|
||||
//insert a selective reverb - sending only the material with very high pitch confidence
|
||||
(
|
||||
f = {var source, rev;
|
||||
source = In.ar(0,2);
|
||||
rev = FreeVerb.ar(DelayN.ar(source,delaytime:1024/s.sampleRate) * Lag.kr((FluidPitch.kr(source.sum)[1] > 0.98),0.01), 1);
|
||||
ReplaceOut.ar(0, rev+ source);
|
||||
}.play(addAction:\addToTail);
|
||||
)
|
||||
|
||||
// free the effect
|
||||
f.free
|
||||
|
||||
// insert a stereo delay instead (as well) using the same
|
||||
(
|
||||
f = {
|
||||
var source, todelay, delay1, delay2, delay3, feedback, mod1, mod2, mod3, mod4;
|
||||
//read the source
|
||||
source = In.ar(0,2);
|
||||
|
||||
// generate modulators that are coprime in frequency
|
||||
mod1 = SinOsc.ar(1, 0, 0.001);
|
||||
mod2 = SinOsc.ar(((617 * 181) / (461 * 991)), 0, 0.001);
|
||||
mod3 = SinOsc.ar(((607 * 193) / (491 * 701)), 0, 0.001);
|
||||
mod4 = SinOsc.ar(((613 * 191) / (463 * 601)), 0, 0.001);
|
||||
|
||||
// gate the signal to send to the delays
|
||||
todelay = DelayN.ar(source,delaytime:1024/s.sampleRate) * Lag.kr((FluidPitch.kr(source.sum)[1] > 0.98),0.01);
|
||||
|
||||
// delay network
|
||||
feedback = LocalIn.ar(3);// take the feedback in for the delays
|
||||
delay1 = DelayC.ar(BPF.ar(todelay+feedback[1]+(feedback[2] * 0.3), 987, 6.7,0.35),0.123,0.122+(mod1*mod2));
|
||||
delay2 = DelayC.ar(BPF.ar(todelay+feedback[0]+(feedback[2] * 0.3), 1987, 6.7,0.35),0.345,0.344+(mod3*mod4));
|
||||
delay3 = DelayC.ar(BPF.ar(todelay+feedback[1], 1456, 6.7,0.35),0.567,0.566+(mod1*mod3),0.6);
|
||||
LocalOut.ar([delay1,delay2, delay3]); // write the feedback for the delays
|
||||
|
||||
ReplaceOut.ar(0, source + [delay1+delay3,delay2+delay3]);
|
||||
}.play(addAction:\addToTail);
|
||||
)
|
||||
::
|
||||
@ -1,158 +0,0 @@
|
||||
TITLE:: FluidRobustScale
|
||||
summary:: Apply Robust Scaling to FluidDataSet
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidStandardize, Classes/FluidNormalize, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
Scales the entries of a link::Classes/FluidDataSet::, or scales a data point according to the learned values of a data set. It will centre the median of each dimension to 0, and will scale the data to +/- the provided centiles, by default providing the first and third qartile (25 and 75).All happens on the server.
|
||||
|
||||
See https://scikit-learn.org/stable/auto_examples/preprocessing/plot_all_scaling.html#robustscaler
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
private:: kr
|
||||
|
||||
METHOD:: new
|
||||
Create a new instance
|
||||
ARGUMENT:: server
|
||||
The link::Classes/Server:: on which to run
|
||||
ARGUMENT:: low
|
||||
The low centile boundary, default 25.
|
||||
ARGUMENT:: high
|
||||
The high centile boundary, default 75.
|
||||
ARGUMENT:: invert
|
||||
The direction in which the scaling will occur for transform and transformpoint. The default 0 is taking in the range of the input used to fit and transforms it towards the robust scaling range. A value of 1 will expect an input of the scaling range to transform back to the original range.
|
||||
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
METHOD:: fit
|
||||
Compute the scaling factors from a link::Classes/FluidDataSet:: for later.
|
||||
ARGUMENT:: dataSet
|
||||
The link::Classes/FluidDataSet:: to scale
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
METHOD:: transform
|
||||
Scale a link::Classes/FluidDataSet:: into another link::Classes/FluidDataSet::, using the learned range from a previous call to link::Classes/FluidRobustScale#fit::
|
||||
ARGUMENT:: sourceDataSet
|
||||
The link::Classes/FluidDataSet:: to scale
|
||||
ARGUMENT:: destDataSet
|
||||
The link::Classes/FluidDataSet:: to populate with scaled data
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
|
||||
METHOD:: fitTransform
|
||||
Scale a link::Classes/FluidDataSet::
|
||||
ARGUMENT:: sourceDataSet
|
||||
The link::Classes/FluidDataSet:: to scale
|
||||
ARGUMENT:: destDataSet
|
||||
The link::Classes/FluidDataSet:: to populate with scaled data
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
|
||||
METHOD:: transformPoint
|
||||
Scale a new data point, using the learned scaling from a previous call to link::Classes/FluidRobustScale#fit::
|
||||
ARGUMENT:: sourceBuffer
|
||||
A link::Classes/Buffer:: with the new data point
|
||||
ARGUMENT:: destBuffer
|
||||
A link::Classes/Buffer:: to contain the scaled value
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
EXAMPLES::
|
||||
code::
|
||||
s.boot;
|
||||
//Preliminaries: we want some audio, a couple of FluidDataSets, some Buffers and a FluidRobustScale
|
||||
(
|
||||
~audiofile = FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav");
|
||||
~raw = FluidDataSet(s);
|
||||
~scaled = FluidDataSet(s);
|
||||
~pitch_feature = Buffer.new(s);
|
||||
~stats = Buffer.alloc(s, 7, 2);
|
||||
~robust = FluidRobustScale(s);
|
||||
)
|
||||
|
||||
// Load audio and run a pitch analysis, which gives us pitch and pitch confidence (so a 2D datum)
|
||||
(
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
FluidBufPitch.process(s,~audio, features: ~pitch_feature);
|
||||
)
|
||||
|
||||
// Divide the time series in to 10, and take the mean of each segment and add this as a point to
|
||||
// the 'raw' FluidDataSet
|
||||
(
|
||||
{
|
||||
var trig = LocalIn.kr(1, 1);
|
||||
var buf = LocalBuf(2, 1);
|
||||
var count = PulseCount.kr(trig) - 1;
|
||||
var chunkLen = (~pitch_feature.numFrames / 10).asInteger;
|
||||
var stats = FluidBufStats.kr(
|
||||
source: ~pitch_feature, startFrame: count * chunkLen,
|
||||
numFrames: chunkLen, stats: ~stats, trig: (trig * (count <=9)), blocking:1
|
||||
);
|
||||
var rd = BufRd.kr(2, ~stats, DC.kr(0), 0, 1);// pick only mean pitch and confidence
|
||||
var wr1 = BufWr.kr(rd[0], buf, DC.kr(0));
|
||||
var wr2 = BufWr.kr(rd[1], buf, DC.kr(1));
|
||||
var dsWr = FluidDataSetWr.kr(~raw, buf: buf, idNumber: count, trig: Done.kr(stats));
|
||||
LocalOut.kr( Done.kr(dsWr));
|
||||
Poll.kr(trig,count,\count);
|
||||
FreeSelf.kr(count - 9);
|
||||
}.play;
|
||||
)
|
||||
|
||||
//check the dataset
|
||||
~raw.print;
|
||||
|
||||
// Scale and load to language-side array
|
||||
(
|
||||
~rawarray = Array.new(10);
|
||||
~scaledarray= Array.new(10);
|
||||
~robust.fitTransform(~raw,~scaled, {
|
||||
~raw.dump{|x| 10.do{|i|
|
||||
~rawarray.add(x["data"][i.asString])
|
||||
}};
|
||||
~scaled.dump{|x| 10.do{|i|
|
||||
~scaledarray.add(x["data"][i.asString])
|
||||
}};
|
||||
});
|
||||
)
|
||||
|
||||
//Plot side by side. Before normalization the two dimensions have radically different scales
|
||||
//which can be unhelpful in many cases
|
||||
|
||||
(
|
||||
(~rawarray ++ 0).flop.plot("Raw Data",Rect(0,0,400,400),minval:0,maxval:[5000,1]).plotMode=\bars;
|
||||
(~scaledarray ++ 0).flop.plot("Scaled",Rect(410,0,400,400), minval:-2,maxval:2).plotMode=\bars;
|
||||
)
|
||||
|
||||
// single point transform on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,0.5.dup);
|
||||
~outbuf = Buffer.new(s);
|
||||
~robust.transformPoint(~inbuf,~outbuf,{|x|x.postln;x.getn(0,2,{|y|y.postln;};)});
|
||||
|
||||
//Server side queries
|
||||
(
|
||||
{
|
||||
var audio = BufRd.ar(1,~audio,LFSaw.ar(BufDur.ir(~audio).reciprocal).range(0, BufFrames.ir(~audio)));
|
||||
var counter = Stepper.ar(Impulse.ar(ControlRate.ir),max:99);
|
||||
var trig = A2K.kr(HPZ1.ar(counter) < 0);
|
||||
//average 100 frames: one could use the MovingAverage extension here
|
||||
var avg;
|
||||
var inputPoint = LocalBuf(2);
|
||||
var outputPoint = LocalBuf(2);
|
||||
var avgBuf = LocalBuf(100,2);
|
||||
//running average of pitch features
|
||||
BufWr.kr(FluidPitch.kr(audio),avgBuf,phase:counter);
|
||||
avg = Mix.new(BufRd.kr(2, avgBuf, phase:100.collect{|x|x})) * 0.01;
|
||||
//assemble data point
|
||||
BufWr.kr(avg[0],inputPoint,0);
|
||||
BufWr.kr(avg[1],inputPoint,1);
|
||||
~robust.kr(trig,inputPoint,outputPoint);
|
||||
Poll.kr(trig,BufRd.kr(1,inputPoint,[0,1]),["pitch (raw)", "confidence (raw)"]);
|
||||
Poll.kr(trig,BufRd.kr(1,outputPoint,[0,1]),["pitch (scaled)", "confidence (scaled)"])
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
@ -1,61 +0,0 @@
|
||||
TITLE:: FluidSTFTPass
|
||||
SUMMARY:: Real-Time FFT/IFFT return trip.
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation,Classes/UnaryOpFunction
|
||||
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a sanity test for the FluCoMa Real-Time Client FFT/IFFT Wrapper. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
The audio rate version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The input to be passed-through
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The size of the buffered window to be analysed, in samples. It will add that much latency to the signal.
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
How much the buffered window moves forward, in samples. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
How large will the FFT be, zero-padding the buffer to the right size, which should be bigger than the windowSize argument, bigger than 4 samples, and should be a power of 2. This is a way to oversample the FFT for extra precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
|
||||
RETURNS::
|
||||
Same as input, delayed by the windowSize.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
Summing with the inverse (gain of -1) with a delay of the latency gives us CPU-expensive silence.
|
||||
CODE::
|
||||
{ var source = PinkNoise.ar(0.1); DelayN.ar(source, delaytime:1024/s.sampleRate, mul: -1) + FluidSTFTPass.ar(source, 1024); }.play
|
||||
::
|
||||
Larger, oversampled, FFT
|
||||
CODE::
|
||||
{ FluidSTFTPass.ar(PinkNoise.ar(0.1), 2048, 128, 8192) }.play
|
||||
::
|
||||
Stereo Input Tests.
|
||||
CODE::
|
||||
{ FluidSTFTPass.ar([SinOsc.ar(222,mul: 0.1), PinkNoise.ar(Decay.ar(Impulse.ar(0.666,mul: 0.2), 0.5))], fftSize:1024)}.play
|
||||
::
|
||||
Stereo Parameter Tests.
|
||||
CODE::
|
||||
{ FluidSTFTPass.ar(SinOsc.ar(222,mul: 0.1), [1024,8192],256,8192)}.play
|
||||
::
|
||||
Modulating Window Param Tests.
|
||||
CODE::
|
||||
{ var source = SinOsc.ar(222,mul: 0.1); [source, FluidSTFTPass.ar(source,LFNoise0.kr(1).range(10,10000))] }.play
|
||||
::
|
||||
Very Short FFT test.
|
||||
CODE::
|
||||
{ var source = SinOsc.ar(222,mul: 0.1); [source, FluidSTFTPass.ar(source,10)] }.play
|
||||
::
|
||||
@ -1,93 +0,0 @@
|
||||
TITLE:: FluidSines
|
||||
SUMMARY:: Sinusoidal Modelling and Resynthesis
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation
|
||||
|
||||
DESCRIPTION::
|
||||
This class applies a Sinusoidal Modelling process on its audio input. It implements a mix of algorithms taken from classic papers. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The algorithm will take an audio in, and will divide it in two parts: LIST::
|
||||
## a reconstruction of what it detects as sinusoidal;
|
||||
## a residual derived from the previous signal to allow null-summing::
|
||||
|
||||
The whole process is based on the assumption that signal is made of pitched steady components that have a long-enough duration and are periodic enough to be perceived as such, that can be tracked, resynthesised and removed from the original, leaving behind what is considered as non-pitched, noisy, and/or transient. It first tracks the peaks, then checks if they are the continuation of a peak in previous spectral frames, by assigning them a track.
|
||||
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
The audio rate version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The input to be processed
|
||||
|
||||
ARGUMENT:: bandwidth
|
||||
The number of bins used to resynthesises a peak. It has an effect on CPU cost: the widest is more accurate but more computationally expensive. It is capped at (fftSize / 2) + 1.
|
||||
|
||||
ARGUMENT:: detectionThreshold
|
||||
The threshold in dB above which a magnitude peak is considered to be a sinusoidal component.
|
||||
|
||||
ARGUMENT:: birthLowThreshold
|
||||
The threshold in dB above which to consider a peak to start a sinusoidal component tracking, for the low end of the spectrum. It is interpolated across the spectrum until birthHighThreshold at half-Nyquist.
|
||||
|
||||
ARGUMENT:: birthHighThreshold
|
||||
The threshold in dB above which to consider a peak to start a sinusoidal component tracking, for the high end of the spectrum. It is interpolated across the spectrum until birthLowThreshold at DC.
|
||||
|
||||
ARGUMENT:: minTrackLen
|
||||
The minimum duration, in spectral frames, for a sinusoidal track to be accepted as a partial. It allows to remove bubbly pitchy artefacts, but is more CPU intensive and might reject quick pitch material.
|
||||
|
||||
ARGUMENT:: trackingMethod
|
||||
The algorithm used to track the sinusoidal continuity between spectral frames. 0 is the default, "Greedy", and 1 is a more expensive "Hungarian" one. footnote::Neri, J., and Depalle, P., "Fast Partial Tracking of Audio with Real-Time Capability through Linear Programming". Proceedings of DAFx-2018.::
|
||||
|
||||
ARGUMENT:: trackMagRange
|
||||
The amplitude difference allowed for a track to diverge between frames, in dB.
|
||||
|
||||
ARGUMENT:: trackFreqRange
|
||||
The frequency difference allowed for a track to diverge between frames, in Hertz.
|
||||
|
||||
ARGUMENT:: trackProb
|
||||
The propensity of peaks to become enrolled into tracks.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As sinusoidal estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As sinusoidal estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to the next power of 2 equal or above the highest of the windowSize and (bandwidth - 1) * 2.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
RETURNS::
|
||||
An array of two audio streams: [0] is the harmonic part extracted, [1] is the rest. The latency between the input and the output is (( hopSize * minTrackLen) + windowSize) samples.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
CODE::
|
||||
// load some audio to play
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
|
||||
// run with large parameters - left is sinusoidal model, right is residual
|
||||
{FluidSines.ar(PlayBuf.ar(1,b,loop:1),detectionThreshold: -40, minTrackLen: 2, windowSize: 2048, fftSize: 8192)}.play
|
||||
|
||||
// interactive parameters with a narrower bandwidth
|
||||
{FluidSines.ar(PlayBuf.ar(1,b,loop:1), 30, MouseX.kr(-140,-10),MouseY.kr(-110,-10),MouseY.kr(-140,-40), 10 , windowSize: 1000, hopSize: 200, fftSize: 4096)}.play
|
||||
|
||||
// null test (the process add a latency of (( hopSize * minTrackLen) + windowSize) samples
|
||||
{var sig = PlayBuf.ar(1,b,loop:1); [FluidSines.ar(sig).sum - DelayN.ar(sig, 1, ((( 512 * 15) + 1024)/ s.sampleRate))]}.play
|
||||
|
||||
// as the algorithm resynthesize the sinusoidal peaks, we would expect to get it to work almost perfectly on a sine wave, with these settings that tell the process to tolerate everything as a sinusoid, even short and quiet peaks
|
||||
{FluidSines.ar(SinOsc.ar(mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 200,trackFreqRange: 1000,trackProb: 0)}.play;
|
||||
|
||||
// we can listen to the artefact in solo, amplifying it by 30dB, to hear the 'lobes' - not bad at all!
|
||||
{FluidSines.ar(SinOsc.ar(mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 200,trackFreqRange: 1000,trackProb: 0)[1].dup * Line.ar(0,30,1).dbamp}.play;
|
||||
|
||||
// as this is a windowed process, the frequency of the peak is good for that full window, and therefore interesting artefacts appear when the pitch is changing.
|
||||
{FluidSines.ar(SinOsc.ar(LFTri.kr(0.1).exprange(220,880),mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 300,trackFreqRange: 1000,trackProb: 0)}.play;
|
||||
|
||||
// if we solo and amplify the artefacts, they are much more apparent (and interesting)
|
||||
{FluidSines.ar(SinOsc.ar(LFTri.kr(0.1).exprange(220,880),mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 300,trackFreqRange: 1000,trackProb: 0)[1].dup * Line.ar(0,30,1).dbamp}.play;
|
||||
::
|
||||
@ -1,325 +0,0 @@
|
||||
TITLE:: FluidSpectralShape
|
||||
SUMMARY:: Seven Spectral Shape Descriptors in Real-Time
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation, Classes/SpecCentroid, Classes/SpecFlatness, Classes/SpecCentroid, Classes/SpecPcile
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements seven of the most popular spectral shape descriptors, computed on a linear scale for both amplitude and frequency. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The descriptors are:
|
||||
LIST::
|
||||
##the four first statistical moments (https://en.wikipedia.org/wiki/Moment_(mathematics) ), more commonly known as:
|
||||
LIST::
|
||||
## the spectral centroid (1) in Hz. This is the point that splits the spectrum in 2 halves of equal energy. It is the weighted average of the magnitude spectrum.
|
||||
## the spectral spread (2) in Hz. This is the standard deviation of the spectrum envelop, or the average of the distance to the centroid.
|
||||
## the normalised skewness (3) as ratio. This indicates how tilted is the spectral curve in relation to the middle of the spectral frame, i.e. half of the Nyquist frequency. If it is below that frequency, i.e. the central bin of the magnitude spectrum, it is positive.
|
||||
## the normalised kurtosis (4) as ratio. This indicates how focused is the spectral curve. If it is peaky, it is high.
|
||||
::
|
||||
## the rolloff (5) in Hz. This indicates the frequency under which code::rolloffPercent:: of the energy is included. The default is 95%.
|
||||
## the flatness (6) in dB. This is the ratio of geometric mean of the magnitude, over the arithmetic mean of the magnitudes. It yields a very approximate measure on how noisy a signal is.
|
||||
## the crest (7) in dB. This is the ratio of the loudest magnitude over the RMS of the whole frame. A high number is an indication of a loud peak poking out from the overal spectral curve.
|
||||
|
||||
The drawings in Peeters 2003 (http://recherche.ircam.fr/anasyn/peeters/ARTICLES/Peeters_2003_cuidadoaudiofeatures.pdf) are useful, as are the commented examples below. For the mathematically-inclined reader, the tutorials and code offered here (https://www.audiocontentanalysis.org/) are interesting to further the understanding.
|
||||
::
|
||||
|
||||
The process will return a multichannel control steam with the seven values, which will be repeated if no change happens within the algorythm, i.e. when the hopSize is larger than the server's kr period.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: kr
|
||||
The audio rate in, control rate out version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The audio to be processed.
|
||||
|
||||
ARGUMENT:: minFreq
|
||||
The minimum frequency that the algorithm will consider for computing the spectral shape. Frequencies below will be ignored. The default of 0 goes down to DC when possible.
|
||||
|
||||
ARGUMENT:: maxFreq
|
||||
The maximum frequency that the algorithm will consider for computing the spectral shape. Frequencies above will be ignored. The default of -1 goes up to Nyquist.
|
||||
|
||||
ARGUMENT:: rolloffPercent
|
||||
This sets the percentage of the frame's energy that will be reported as the rolloff frequency. The default is 95%.
|
||||
|
||||
ARGUMENT:: unit
|
||||
The frequency unit for the spectral shapes to be computed upon, and outputted at. The default (0) is in Hertz and computes the moments on a linear spectrum. The alternative is in MIDI note numbers(1), which compute the moments on an exponential spectrum.
|
||||
|
||||
ARGUMENT:: power
|
||||
This flag sets the scaling of the magnitudes in the moment calculation. It uses either its amplitude (0, by default) or its power (1).
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The window size. As spectral shape estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty
|
||||
|
||||
ARGUMENT:: hopSize
|
||||
The window hop size. As spectral shape estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).
|
||||
|
||||
ARGUMENT:: fftSize
|
||||
The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will use the next power of 2 equal or above the windowSize.
|
||||
|
||||
ARGUMENT:: maxFFTSize
|
||||
How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.
|
||||
|
||||
RETURNS::
|
||||
A 7-channel KR signal with the seven spectral shape descriptors. The latency is windowSize.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
|
||||
code::
|
||||
//create a monitoring bus for the descriptors
|
||||
b = Bus.new(\control,0,7);
|
||||
|
||||
//create a monitoring window for the values
|
||||
|
||||
(
|
||||
w = Window("spectral Shape Monitor", Rect(10, 10, 220, 190)).front;
|
||||
|
||||
c = Array.fill(7, {arg i; StaticText(w, Rect(10, i * 25 + 10, 135, 20)).background_(Color.grey(0.7)).align_(\right)});
|
||||
c[0].string = ("Centroid: ");
|
||||
c[1].string = ("Spread: ");
|
||||
c[2].string = ("Skewness: ");
|
||||
c[3].string = ("Kurtosis: ");
|
||||
c[4].string = ("Rolloff: ");
|
||||
c[5].string = ("Flatness: ");
|
||||
c[6].string = ("Crest: ");
|
||||
|
||||
a = Array.fill(7, {arg i;
|
||||
StaticText(w, Rect(150, i * 25 + 10, 60, 20)).background_(Color.grey(0.7)).align_(\center);
|
||||
});
|
||||
)
|
||||
|
||||
//run the window updating routine.
|
||||
(
|
||||
r = Routine {
|
||||
{
|
||||
|
||||
b.get({ arg val;
|
||||
{
|
||||
if(w.isClosed.not) {
|
||||
val.do({arg item,index;
|
||||
a[index].string = item.round(0.01)})
|
||||
}
|
||||
}.defer
|
||||
});
|
||||
|
||||
0.01.wait;
|
||||
}.loop
|
||||
|
||||
}.play
|
||||
)
|
||||
|
||||
//play a simple sound to observe the values
|
||||
(
|
||||
{
|
||||
var source;
|
||||
source = BPF.ar(WhiteNoise.ar(), 330, 55/330);
|
||||
Out.kr(b,FluidSpectralShape.kr(source));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
|
||||
STRONG::A commented tutorial on how each descriptor behaves with test signals: ::
|
||||
|
||||
CODE::
|
||||
// as above, create a monitoring bus for the descriptors
|
||||
b = Bus.new(\control,0,7);
|
||||
|
||||
//again, create a monitoring window for the values
|
||||
(
|
||||
w = Window("Spectral Shape Monitor", Rect(10, 10, 220, 190)).front;
|
||||
|
||||
c = Array.fill(7, {arg i; StaticText(w, Rect(10, i * 25 + 10, 135, 20)).background_(Color.grey(0.7)).align_(\right)});
|
||||
c[0].string = ("Centroid: ");
|
||||
c[1].string = ("Spread: ");
|
||||
c[2].string = ("Skewness: ");
|
||||
c[3].string = ("Kurtosis: ");
|
||||
c[4].string = ("Rolloff: ");
|
||||
c[5].string = ("Flatness: ");
|
||||
c[6].string = ("Crest: ");
|
||||
|
||||
a = Array.fill(7, {arg i;
|
||||
StaticText(w, Rect(150, i * 25 + 10, 60, 20)).background_(Color.grey(0.7)).align_(\center);
|
||||
});
|
||||
)
|
||||
|
||||
// this time, update a little more slowly.
|
||||
(
|
||||
r = Routine {
|
||||
{
|
||||
|
||||
b.get({ arg val;
|
||||
{
|
||||
if(w.isClosed.not) {
|
||||
val.do({arg item,index;
|
||||
a[index].string = item.round(0.01)})
|
||||
}
|
||||
}.defer
|
||||
});
|
||||
|
||||
0.2.wait;
|
||||
}.loop
|
||||
|
||||
}.play
|
||||
)
|
||||
|
||||
// first, a sine wave
|
||||
(
|
||||
x = {
|
||||
arg freq=220;
|
||||
var source;
|
||||
source = SinOsc.ar(freq,mul:0.1);
|
||||
Out.kr(b, VarLag.kr(FluidSpectralShape.kr(source),1024/s.sampleRate));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// at 220, the centroid is on the frequency, the spread is narrow, but as wide as the FFT Hann window ripples, the skewness is high as we are low and therefore far left of the middle bin (aka half-Nyquist), the Kurtosis is incredibly high as we have a very peaky spectrum. The rolloff is slightly higher than the frequency, taking into account the FFT windowing ripples, the flatness is incredibly low, as we have one peak and not much else, and the crest is quite high, because most of the energy is in a few peaky bins.
|
||||
|
||||
x.set(\freq, 440)
|
||||
|
||||
// at 440, the skewness has changed (we are nearer the middle of the spectrogram) and the Kurtosis too, although it is still so high it is quite in the same order of magnitude. The rest is stable, as expected.
|
||||
|
||||
x.set(\freq, 11000)
|
||||
|
||||
// at 11kHz, kurtosis is still in the thousand, but skewness is almost null, as expected.
|
||||
|
||||
x.free
|
||||
|
||||
// second, broadband noise
|
||||
(
|
||||
x = {
|
||||
arg type = 0;
|
||||
var source;
|
||||
source = Select.ar(type,[WhiteNoise.ar(0.1),PinkNoise.ar(0.1)]);
|
||||
Out.kr(b, VarLag.kr(FluidSpectralShape.kr(source),1024/s.sampleRate));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// white noise has a linear repartition of energy, so we would expect a centroid in the middle bin (aka half-Nyquist) with a spread covering the full range (+/- a quarter-Nyquist), with a skewness almost null since we are centered, and a very low Kurtosis since we are flat. The rolloff should be almost at Nyquist, the flatness as high as it gets, and the crest quite low.
|
||||
|
||||
x.set(\type, 1)
|
||||
|
||||
// pink noise has a drop of 3dB per octave across the spectrum, so we would, by comparison, expect a lower centroid, a slighly higher skewness and kurtosis, a lower rolloff, a slighly lower flatness and a higher crest for the larger low-end energy.
|
||||
|
||||
x.free
|
||||
|
||||
// third, bands of noise
|
||||
(
|
||||
x = {
|
||||
arg type = 0;
|
||||
var source, chain;
|
||||
chain = FFT(LocalBuf(1024), WhiteNoise.ar(0.5));
|
||||
chain = chain.pvcollect(1024, {arg mag,phase;[mag,phase]},5,11,1);
|
||||
source = Select.ar(type,[
|
||||
BPF.ar(BPF.ar(WhiteNoise.ar(0.5),330,0.666),330,0.666),
|
||||
IFFT(chain)]);
|
||||
Out.kr(b, VarLag.kr(FluidSpectralShape.kr(source),1024/s.sampleRate));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// a second-order bandpass filter on whitenoise, centred on 330Hz with one octave bandwidth, gives us a centroid quite high. This is due to the exponential behaviour of the filter, with a gentle slope. Observe the spectral analyser:
|
||||
|
||||
s.freqscope
|
||||
|
||||
// at first it seems quite centred, but then flip the argument FrqScl to lin(ear) and observe how high the spectrum goes. If we set it to a brickwall spectral filter tuned on the same frequencies:
|
||||
|
||||
x.set(\type, 1)
|
||||
|
||||
// we have a much narrower register, and our centroid and spread, as well as the kurtosis and flatness, agrees with this reading.
|
||||
|
||||
x.free
|
||||
|
||||
//fourth, equally spaced sines
|
||||
(
|
||||
x = {
|
||||
arg freq = 220;
|
||||
var source;
|
||||
source = Mix.fill(7, {arg ind; SinOsc.ar(freq + (ind * (220 / 6)), 0, 0.02)});
|
||||
Out.kr(b,FluidSpectralShape.kr(source));
|
||||
source.dup;
|
||||
}.play;
|
||||
)
|
||||
|
||||
// this example shows a similar result to the brickwall spectral bandpass above. If we move the central frequency nearer the half-Nyquist:
|
||||
|
||||
x.set(\freq, 8800)
|
||||
|
||||
// we can observe that the linear spread is kept the same, since there is the same linear distance in Hz between our frequencies. Skewness is a good indication here of where we are in the spectrum with the shape.
|
||||
::
|
||||
|
||||
|
||||
STRONG::A few notes on the impact of the scale options::
|
||||
|
||||
CODE::
|
||||
// The computation of the centroids and other moments can also be done considering a logarithmic pitch scale, and/or the power of the magnitudes. This yields values that are more in line with the expectation of the users of equalisers for instance, where the shape is often drawn and described in logairhmic terms, i.e. dB per octave.
|
||||
|
||||
// For instance, compare the values of the centroid and the spread in both scales:
|
||||
(
|
||||
{
|
||||
var source = BPF.ar(PinkNoise.ar(0.1),MouseX.kr().exprange(300,3000).poll(1,label: "filter frequency"), 0.5);
|
||||
FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 0, power: 0)[0].lag.poll(1,"linear centroid");
|
||||
FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 1, power: 1)[0].lag.midicps.poll(1,"exponential centroid");//convert from midi to Hz
|
||||
source.dup
|
||||
}.play
|
||||
)
|
||||
|
||||
// The lower one gets in frequency, the more the linear spectral bias shows. The same applies to the spread:
|
||||
|
||||
(
|
||||
{
|
||||
var source = BPF.ar(PinkNoise.ar(0.1),440, MouseX.kr().exprange(0.1,4).poll(1,label: "filter RQ"));
|
||||
FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 0, power: 0)[1].lag.poll(1,"linear spread");
|
||||
FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 1, power: 1)[1].lag.poll(1,"exponential spread");
|
||||
source.dup
|
||||
}.play
|
||||
)
|
||||
|
||||
// The logarythmic unit is in semitones. To convert, either divide by 12 to get the octave of one standard deviation, or divide by 6 to get the width of the filter in octaves. One clear observation is that the width is now in a range that scales with what we hear, growing fourfold as the filter goes from resonating to more broadband.
|
||||
|
||||
// An example of productive mapping between filters parameters and logarithmic centroid values allows to make a simple automatic subtractive noise resynthesis
|
||||
|
||||
// load a beat
|
||||
b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav"));
|
||||
|
||||
//logcentroid version
|
||||
(
|
||||
{
|
||||
var source = PlayBuf.ar(1,b,loop: 1);
|
||||
var loudness, centroid, spread;
|
||||
|
||||
#centroid,spread = Lag.kr(FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 1, power: 1, hopSize: 128),128/SampleRate.ir);
|
||||
loudness = Lag.kr(FluidLoudness.kr(source,hopSize: 128),128/SampleRate.ir);
|
||||
[
|
||||
DelayN.ar(source,delaytime: 1024/SampleRate.ir),
|
||||
BBandPass.ar(WhiteNoise.ar(),
|
||||
centroid.midicps,
|
||||
(spread/6),
|
||||
loudness[0].dbamp * 2
|
||||
)
|
||||
]
|
||||
}.scope;
|
||||
)
|
||||
|
||||
//lincentroid version for comparison
|
||||
(
|
||||
{
|
||||
var source = PlayBuf.ar(1,b,loop: 1);
|
||||
var loudness, centroid, spread;
|
||||
|
||||
#centroid,spread = Lag.kr(FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 0, power: 0, hopSize: 128),128/SampleRate.ir);
|
||||
loudness = Lag.kr(FluidLoudness.kr(source,hopSize: 128),128/SampleRate.ir);
|
||||
[
|
||||
DelayN.ar(source,delaytime: 1024/SampleRate.ir),
|
||||
Sanitize.ar(BBandPass.ar(WhiteNoise.ar(),
|
||||
centroid,
|
||||
(spread * 2/centroid).max(0.001),
|
||||
loudness[0].dbamp * 2
|
||||
))
|
||||
]
|
||||
}.scope;
|
||||
)
|
||||
::
|
||||
@ -1,150 +0,0 @@
|
||||
TITLE:: FluidStandardize
|
||||
summary:: Standardize a FluidDataSet
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidDataSet, Classes/FluidNormalize, Classes/FluidRobustScale
|
||||
|
||||
DESCRIPTION::
|
||||
Standardize a link::Classes/FluidDataSet::, i.e. rescale using its mean(s) and standard deviation(s) in each dimension.
|
||||
|
||||
See http://www.faqs.org/faqs/ai-faq/neural-nets/part2/section-16.html
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Create a new instance
|
||||
ARGUMENT:: server
|
||||
The server for this model
|
||||
ARGUMENT:: invert
|
||||
The direction in which the standardization will occur for transform and transformpoint. The default 0 is taking in the range of the input used to fit and transforms it towards the standardized range. A value of 1 will expect an input of the standardized range to transform back to the original range.
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
METHOD:: fit
|
||||
Fit model to a DataSet without applying scaling
|
||||
ARGUMENT:: dataSet
|
||||
The link::Classes/FluidDataSet:: to standardize
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
METHOD:: transform
|
||||
Standardize a link::Classes/FluidDataSet::, using the learned statistics from a previous call to link::Classes/FluidStandardize#fit::
|
||||
ARGUMENT:: sourceDataSet
|
||||
The link::Classes/FluidDataSet:: to standardize
|
||||
ARGUMENT:: destDataSet
|
||||
The link::Classes/FluidDataSet:: to populate with standardized data
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
METHOD:: fitTransform
|
||||
Standardize a link::Classes/FluidDataSet:: into another link::Classes/FluidDataSet::
|
||||
ARGUMENT:: sourceDataSet
|
||||
The link::Classes/FluidDataSet:: to standardize
|
||||
ARGUMENT:: destDataSet
|
||||
The link::Classes/FluidDataSet:: to populate with standardized data
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
METHOD:: transformPoint
|
||||
Standardize a new data point, using the learned statistics from a previous call to link::Classes/FluidStandardize#fit::
|
||||
ARGUMENT:: sourceBuffer
|
||||
A link::Classes/Buffer:: with the new data point
|
||||
ARGUMENT:: destBuffer
|
||||
A link::Classes/Buffer:: to contain the standardize value
|
||||
ARGUMENT:: action
|
||||
A function to run when processing is complete
|
||||
|
||||
EXAMPLES::
|
||||
code::
|
||||
s.boot;
|
||||
//Preliminaries: we want some audio, a couple of FluidDataSets, some Buffers and a FluidStandardize
|
||||
(
|
||||
~audiofile = FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav");
|
||||
~raw = FluidDataSet(s);
|
||||
~stand = FluidDataSet(s);
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
~pitch_feature = Buffer.new(s);
|
||||
~stats = Buffer.alloc(s, 7, 2);
|
||||
~standardizer = FluidStandardize(s);
|
||||
)
|
||||
|
||||
|
||||
// Load audio and run a pitch analysis, which gives us pitch and pitch confidence (so a 2D datum)
|
||||
(
|
||||
~audio = Buffer.read(s,~audiofile);
|
||||
FluidBufPitch.process(s,~audio, features: ~pitch_feature,action:{"Analysed Pitch".postln});
|
||||
)
|
||||
|
||||
// Divide the time series in to 10, and take the mean of each s"egment and add this as a point to
|
||||
// the 'raw' FluidDataSet
|
||||
(
|
||||
{
|
||||
var trig = LocalIn.kr(1, 1);
|
||||
var buf = LocalBuf(2, 1);
|
||||
var count = PulseCount.kr(trig) - 1;
|
||||
var chunkLen = (~pitch_feature.numFrames / 10).asInteger;
|
||||
var stats = FluidBufStats.kr(
|
||||
source: ~pitch_feature, startFrame: count * chunkLen,
|
||||
numFrames: chunkLen, stats: ~stats,
|
||||
trig: trig * (count < 10), blocking: 1
|
||||
);
|
||||
var rd = BufRd.kr(2, ~stats, DC.kr(0), 0, 1);// pick only mean pitch and confidence
|
||||
var wr1 = BufWr.kr(rd[0], buf, DC.kr(0));
|
||||
var wr2 = BufWr.kr(rd[1], buf, DC.kr(1));
|
||||
var dsWr = FluidDataSetWr.kr(~raw, buf: buf, idNumber: count, trig: Done.kr(stats));
|
||||
LocalOut.kr( Done.kr(dsWr));
|
||||
FreeSelf.kr(count - 9);
|
||||
Poll.kr(trig,count, \count);
|
||||
}.play;
|
||||
)
|
||||
|
||||
// Standardize and load to language-side array
|
||||
(
|
||||
~rawarray = Array.new(10);
|
||||
~stdarray= Array.new(10);
|
||||
~standardizer.fitTransform(~raw,~stand, {
|
||||
~raw.dump{|x| 10.do{|i|
|
||||
~rawarray.add(x["data"][i.asString])
|
||||
}};
|
||||
~stand.dump{|x| 10.do{|i|
|
||||
~stdarray.add(x["data"][i.asString])
|
||||
}};
|
||||
});
|
||||
)
|
||||
|
||||
(
|
||||
(~rawarray ++ 0).flop.plot("Unstandardized",Rect(0,0,400,400),minval:0,maxval:[5000,1]).plotMode=\bars;
|
||||
(~stdarray ++ 0).flop.plot("Standardized",Rect(410,0,400,400), minval:-2,maxval:2).plotMode=\bars;
|
||||
)
|
||||
|
||||
// single point transform on arbitrary value
|
||||
~inbuf = Buffer.loadCollection(s,0.5.dup);
|
||||
~outbuf = Buffer.new(s);
|
||||
~standardizer.transformPoint(~inbuf,~outbuf,{|x|x.postln;x.getn(0,2,{|y|y.postln;};)});
|
||||
|
||||
::
|
||||
|
||||
subsection::Server Side Querying
|
||||
code::
|
||||
(
|
||||
// read frames out of buffer and pass to standardize
|
||||
{
|
||||
var audio = BufRd.ar(1,~audio,LFSaw.ar(BufDur.ir(~audio).reciprocal).range(0, BufFrames.ir(~audio)));
|
||||
var counter = Stepper.ar(Impulse.ar(ControlRate.ir),max:99);
|
||||
var trig = A2K.kr(HPZ1.ar(counter) < 0);
|
||||
//average 10 frames: one could use the MovingAverage extension here
|
||||
var avg;
|
||||
var inputPoint= LocalBuf(2);
|
||||
var outputPoint = LocalBuf(2);
|
||||
var avgBuf = LocalBuf(100,2);
|
||||
//average of pitch features
|
||||
BufWr.kr(FluidPitch.kr(audio),avgBuf,phase:counter);
|
||||
avg = Mix.new(BufRd.kr(2, avgBuf, phase:100.collect{|x|x})) * 0.01;
|
||||
//assemble data point
|
||||
BufWr.kr(avg[0],inputPoint,0);
|
||||
BufWr.kr(avg[1],inputPoint,1);
|
||||
Poll.kr(trig,BufRd.kr(1,inputPoint,[0,1]),["pitch (raw)", "confidence (raw)"]);
|
||||
~standardizer.kr(trig,inputPoint,outputPoint);
|
||||
Poll.kr(trig,BufRd.kr(1,outputPoint,[0,1]),["pitch (standardized)", "confidence (standardized)"]);
|
||||
}.play;
|
||||
)
|
||||
::
|
||||
@ -1,47 +0,0 @@
|
||||
TITLE:: FluidStats
|
||||
summary:: Rolling mean and standard devication on kr inputs
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidBufStats, Classes/FluidStandardize
|
||||
|
||||
DESCRIPTION::
|
||||
Computes the rolling mean and sample standard deviation over a given window for multichannel kr inputs.
|
||||
|
||||
This UGen does not perform multichannel expansion because (like link::Classes/BufWr::) it takes an array as input.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: kr
|
||||
Run the UGen
|
||||
|
||||
ARGUMENT:: inputsArray
|
||||
An array (or just one) kr input stream.
|
||||
|
||||
ARGUMENT:: size
|
||||
The size of the history window to use
|
||||
|
||||
returns:: A 2D array of kr outputs code::[[means][standard deviations]]::
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: numOutputs, initOutputs, init, channels,checkInputs
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
(
|
||||
~gaussianNoise = { |a,b|
|
||||
var mag = (-2 * a.abs.log).sqrt;
|
||||
var f = 2 * pi * b.abs;
|
||||
[mag * f.cos, mag * f.sin]
|
||||
}
|
||||
|
||||
|
||||
|
||||
{
|
||||
var src = ~gaussianNoise.value(WhiteNoise.kr,WhiteNoise.kr);
|
||||
var stats = FluidStats.kr(src,20);
|
||||
stats[0].poll(label:'means');
|
||||
stats[1].poll(label:'standard deviations');
|
||||
}.play
|
||||
)
|
||||
::
|
||||
@ -1,73 +0,0 @@
|
||||
TITLE:: FluidTransientSlice
|
||||
SUMMARY:: Transient-Based Real-Time Audio Slicer
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation
|
||||
|
||||
DESCRIPTION::
|
||||
This class implements a real-time transient-based slice extractor relying on the same algorithm than Classes/FluidBufTransients using clicks/transients/derivation/anomaly in the signal to estimate the slicing points. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The process will return an audio steam with sample-long impulses at estimated starting points of the different slices.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
The audio rate version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The audio to be processed.
|
||||
|
||||
ARGUMENT:: order
|
||||
The order in samples of the impulse response filter used to model the estimated continuous signal. It is how many previous samples are used by the algorithm to predict the next one as reference for the model. The higher the order, the more accurate is its spectral definition, not unlike fft, improving low frequency resolution, but it differs in that it is not connected to its temporal resolution.
|
||||
|
||||
ARGUMENT:: blockSize
|
||||
The size in samples of frame on which it the algorithm is operating. High values are more cpu intensive, and also determines the maximum transient size, which will not be allowed to be more than half that lenght in size.
|
||||
|
||||
ARGUMENT:: padSize
|
||||
The size of the handles on each sides of the block simply used for analysis purpose and avoid boundary issues.
|
||||
|
||||
ARGUMENT:: skew
|
||||
The nervousness of the bespoke detection function with values from -10 to 10. It allows to decide how peaks are amplified or smoothed before the thresholding. High values increase the sensitivity to small variations.
|
||||
|
||||
ARGUMENT:: threshFwd
|
||||
The threshold of the onset of the smoothed error function. It allows tight start of the identification of the anomaly as it proceeds forward.
|
||||
|
||||
ARGUMENT:: threshBack
|
||||
The threshold of the offset of the smoothed error function. As it proceeds backwards in time, it allows tight ending of the identification of the anomaly.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The averaging window of the error detection function. It needs smoothing as it is very jittery. The longer the window, the less precise, but the less false positives.
|
||||
|
||||
ARGUMENT:: clumpLength
|
||||
The window size in sample within with positive detections will be clumped together to avoid overdetecting in time.
|
||||
|
||||
ARGUMENT:: minSliceLength
|
||||
The minimum duration of a slice in samples.
|
||||
|
||||
RETURNS::
|
||||
An audio stream with impulses at detected transients. The latency between the input and the output is (blockSize + padSize - order) samples.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
CODE::
|
||||
//load some sounds
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
|
||||
// basic param (the process add a latency of (blockSize + padSize - order) samples
|
||||
{var sig = PlayBuf.ar(1,b,loop:1); [FluidTransientSlice.ar(sig) * 0.5, DelayN.ar(sig, 1, ((256 + 128 - 20)/ s.sampleRate))]}.play
|
||||
|
||||
// other parameters
|
||||
{var sig = PlayBuf.ar(1,b,loop:1); [FluidTransientSlice.ar(sig,order:80,minSliceLength:2205) * 0.5, DelayN.ar(sig, 1, ((256 + 128 - 80)/ s.sampleRate))]}.play
|
||||
|
||||
// More musical, transient-trigged autopan
|
||||
(
|
||||
{
|
||||
var sig, trig, syncd, pan;
|
||||
sig = PlayBuf.ar(1,b,loop:1);
|
||||
trig = FluidTransientSlice.ar(sig,order:10,minSliceLength:4410);
|
||||
syncd = DelayN.ar(sig, 1, ((256 + 128 - 20)/ s.sampleRate));
|
||||
pan = TRand.ar(-1,1,trig);
|
||||
Pan2.ar(syncd,pan);
|
||||
}.play
|
||||
)
|
||||
::
|
||||
@ -1,65 +0,0 @@
|
||||
TITLE:: FluidTransients
|
||||
SUMMARY:: Real-Time Transient Modeller and Extractor
|
||||
CATEGORIES:: Libraries>FluidCorpusManipulation
|
||||
RELATED:: Guides/FluidCorpusManipulation
|
||||
|
||||
DESCRIPTION::
|
||||
This class applies a real-time transient extractor on its input. It implements declicking algorithm from chapter 5 of the classic Digital Audio Restoration by Godsill, Simon J., Rayner, Peter J.W. with some bespoke improvements on the detection function tracking. It is part of the LINK:: Guides/FluidCorpusManipulation##Fluid Corpus Manipulation Toolkit::. For more explanations, learning material, and discussions on its musicianly uses, visit http://www.flucoma.org/
|
||||
|
||||
The algorithm will take an audio in, and will divide it in two outputs: LIST::
|
||||
## the transients, estimated from the signal and extracted from it;
|
||||
## the remainder of the material, as estimated by the algorithm. ::
|
||||
|
||||
The whole process is based on the assumption that a transient is an element that is deviating from the surrounding material, as sort of click or anomaly. The algorithm then estimates what should have happened if the signal had followed its normal path, and resynthesises this estimate, removing the anomaly which is considered as the transient.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: ar
|
||||
The audio rate version of the object.
|
||||
|
||||
ARGUMENT:: in
|
||||
The audio to be processed.
|
||||
|
||||
ARGUMENT:: order
|
||||
The order in samples of the impulse response filter used to model the estimated continuous signal. It is how many previous samples are used by the algorithm to predict the next one as reference for the model. The higher the order, the more accurate is its spectral definition, not unlike fft, improving low frequency resolution, but it differs in that it is not conected to its temporal resolution.
|
||||
|
||||
ARGUMENT:: blockSize
|
||||
The size in samples of frame on which it the algorithm is operating. High values are more cpu intensive, and also determines the maximum transient size, which will not be allowed to be more than half that lenght in size.
|
||||
|
||||
ARGUMENT:: padSize
|
||||
The size of the handles on each sides of the block simply used for analysis purpose and avoid boundary issues.
|
||||
|
||||
ARGUMENT:: skew
|
||||
The nervousness of the bespoke detection function with values from -10 to 10. It allows to decide how peaks are amplified or smoothed before the thresholding. High values increase the sensitivity to small variations.
|
||||
|
||||
ARGUMENT:: threshFwd
|
||||
The threshold of the onset of the smoothed error function. It allows tight start of the identification of the anomaly as it proceeds forward.
|
||||
|
||||
ARGUMENT:: threshBack
|
||||
The threshold of the offset of the smoothed error function. As it proceeds backwards in time, it allows tight ending of the identification of the anomaly.
|
||||
|
||||
ARGUMENT:: windowSize
|
||||
The averaging window of the error detection function. It needs smoothing as it is very jittery. The longer the window, the less precise, but the less false positive.
|
||||
|
||||
ARGUMENT:: clumpLength
|
||||
The window size in sample within which positive detections will be clumped together to avoid overdetecting in time.
|
||||
|
||||
RETURNS::
|
||||
An array of two audio streams: [0] is the transient extracted, [1] is the rest. The latency between the input and the output is (blockSize + padSize - order) samples.
|
||||
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
CODE::
|
||||
//load some buffer
|
||||
b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav"));
|
||||
|
||||
// basic parameters
|
||||
{FluidTransients.ar(PlayBuf.ar(1, b, loop:1))}.play
|
||||
|
||||
// tweaked parameters
|
||||
{FluidTransients.ar(PlayBuf.ar(1, b, loop:1), 80, threshFwd:MouseX.kr(0,5), threshBack:MouseY.kr(0,2))}.play
|
||||
|
||||
// null test (the process add a latency of (blockSize + padding - order) samples
|
||||
{var sig = PlayBuf.ar(1, b, loop:1); [FluidTransients.ar(sig).sum - DelayN.ar(sig, 1, ((256 + 128 - 20)/ s.sampleRate))]}.play
|
||||
::
|
||||
@ -1,247 +0,0 @@
|
||||
TITLE:: FluidUMAP
|
||||
summary:: Dimensionality Reduction with Uniform Manifold Approximation and Projection
|
||||
categories:: Libraries>FluidCorpusManipulation
|
||||
related:: Classes/FluidMDS, Classes/FluidPCA, Classes/FluidDataSet
|
||||
|
||||
DESCRIPTION::
|
||||
|
||||
Multidimensional scaling of a link::Classes/FluidDataSet:: using Uniform Manifold Approximation and Projection (UMAP)
|
||||
|
||||
Please refer to https://umap-learn.readthedocs.io/ for more information on the algorithm.
|
||||
|
||||
CLASSMETHODS::
|
||||
|
||||
METHOD:: new
|
||||
Make a new instance
|
||||
ARGUMENT:: server
|
||||
The server on which to run this model
|
||||
ARGUMENT:: numDimensions
|
||||
The number of dimensions to reduce to
|
||||
ARGUMENT:: numNeighbours
|
||||
The number of neighbours considered by the algorithm to balance local vs global structures to conserve. Low values will prioritise preserving local structure, high values will help preserving the global structure.
|
||||
ARGUMENT:: minDist
|
||||
The minimum distance each point is allowed to be from the others in the low dimension space. Low values will make tighter clumps, and higher will spread the points more.
|
||||
ARGUMENT:: iterations
|
||||
The number of iterations that the algorithm will go through to optimise the new representation
|
||||
ARGUMENT:: learnRate
|
||||
The learning rate of the algorithm, aka how much of the error it uses to estimate the next iteration.
|
||||
|
||||
INSTANCEMETHODS::
|
||||
|
||||
PRIVATE:: init
|
||||
|
||||
METHOD:: fit
|
||||
Train this model on a link::Classes/FluidDataSet:: but don't transform the data
|
||||
ARGUMENT:: dataSet
|
||||
A link::Classes/FluidDataSet:: to analyse
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
METHOD:: transform
|
||||
Given a trained model, apply the reduction to a source link::Classes/FluidDataSet:: and write to a destination. Can be the same for both (in-place)
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: destDataSet
|
||||
Destination data, or the DataSet name
|
||||
ARGUMENT:: action
|
||||
Run when done.
|
||||
|
||||
METHOD:: fitTransform
|
||||
Fit the model to a link::Classes/FluidDataSet:: and write the new projected data to a destination FluidDataSet.
|
||||
ARGUMENT:: sourceDataSet
|
||||
Source data, or the DataSet name
|
||||
ARGUMENT:: destDataSet
|
||||
Destination data, or the DataSet name
|
||||
ARGUMENT:: action
|
||||
Run when done
|
||||
|
||||
EXAMPLES::
|
||||
|
||||
code::
|
||||
|
||||
//Preliminaries: we want some points, a couple of FluidDataSets, a FluidStandardize and a FluidUMAP
|
||||
(
|
||||
~raw = FluidDataSet(s);
|
||||
~standardized = FluidDataSet(s);
|
||||
~reduced = FluidDataSet(s);
|
||||
~normalized = FluidDataSet(s);
|
||||
~standardizer = FluidStandardize(s);
|
||||
~normalizer = FluidNormalize(s, 0.05, 0.95);
|
||||
~umap = FluidUMAP(s).numDimensions_(2).numNeighbours_(5).minDist_(0.2).iterations_(50).learnRate_(0.2);
|
||||
)
|
||||
|
||||
|
||||
// build a dataset of 400 points in 3D (colour in RGB)
|
||||
~colours = Dictionary.newFrom(400.collect{|i|[("entry"++i).asSymbol, 3.collect{1.0.rand}]}.flatten(1));
|
||||
~raw.load(Dictionary.newFrom([\cols, 3, \data, ~colours]));
|
||||
|
||||
// check the entries
|
||||
~raw.print;
|
||||
|
||||
//First standardize our DataSet, then apply the UMAP to get 2 dimensions, then normalise these 2 for drawing in the full window size
|
||||
(
|
||||
~standardizer.fitTransform(~raw,~standardized,action:{"Standardized".postln});
|
||||
~umap.fitTransform(~standardized,~reduced,action:{"Finished UMAP".postln});
|
||||
~normalizer.fitTransform(~reduced,~normalized,action:{"Normalized Output".postln});
|
||||
)
|
||||
//we recover the reduced dataset
|
||||
~normalized.dump{|x| ~normalizedDict = x["data"]};
|
||||
|
||||
~normalized.print
|
||||
~normalizedDict.postln
|
||||
|
||||
//Visualise the 2D projection of our original 4D data
|
||||
(
|
||||
w = Window("a perspective", Rect(128, 64, 200, 200));
|
||||
w.drawFunc = {
|
||||
Pen.use {
|
||||
~normalizedDict.keysValuesDo{|key, val|
|
||||
Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]);
|
||||
Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5));
|
||||
~colours[key.asSymbol].flat;
|
||||
}
|
||||
}
|
||||
};
|
||||
w.refresh;
|
||||
w.front;
|
||||
)
|
||||
|
||||
//play with parameters
|
||||
~umap.numNeighbours_(10).minDist_(0.5).iterations_(100).learnRate_(0.1);
|
||||
|
||||
//rerun the UMAP
|
||||
~umap.fitTransform(~standardized,~reduced,action:{"Finished UMAP".postln});
|
||||
|
||||
//draw to compare
|
||||
(
|
||||
~normalizer.fitTransform(~reduced,~normalized,action:{
|
||||
"Normalized Output".postln;
|
||||
~normalized.dump{|x|
|
||||
~normalizedDict = x["data"];
|
||||
|
||||
{
|
||||
u = Window("another perspective", Rect(328, 64, 200, 200));
|
||||
u.drawFunc = {
|
||||
Pen.use {
|
||||
~normalizedDict.keysValuesDo{|key, val|
|
||||
Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]);
|
||||
Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5));
|
||||
~colours[key.asSymbol].flat;
|
||||
};
|
||||
};
|
||||
};
|
||||
u.refresh;
|
||||
u.front;
|
||||
|
||||
}.defer;
|
||||
|
||||
};
|
||||
});
|
||||
)
|
||||
|
||||
// now run new random points on the same training material. Colours should be scattered around the same space
|
||||
|
||||
~newDS = FluidDataSet(s);
|
||||
~colours2 = Dictionary.newFrom(400.collect{|i|[("entry"++i).asSymbol, 3.collect{1.0.rand}]}.flatten(1));
|
||||
~newDS.load(Dictionary.newFrom([\cols, 3, \data, ~colours2]));
|
||||
|
||||
//we need to standardize to the same space
|
||||
~newDSstan = FluidDataSet(s);
|
||||
~standardizer.transform(~newDS, ~newDSstan);
|
||||
|
||||
//then we can run the umap
|
||||
~newDSmap = FluidDataSet(s);
|
||||
~umap.transform(~newDSstan, ~newDSmap);
|
||||
|
||||
//then we can draw and look
|
||||
(
|
||||
~normalizer.transform(~newDSmap,~normalized,action:{
|
||||
"Normalized Output".postln;
|
||||
~normalized.dump{|x|
|
||||
~normalizedDict = x["data"];
|
||||
{
|
||||
t = Window("new material", Rect(528, 64, 200, 200));
|
||||
t.drawFunc = {
|
||||
Pen.use {
|
||||
~normalizedDict.keysValuesDo{|key, val|
|
||||
Pen.fillColor = Color.new(~colours2[key.asSymbol][0], ~colours2[key.asSymbol][1],~colours2[key.asSymbol][2]);
|
||||
Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5));
|
||||
~colours2[key.asSymbol].flat;
|
||||
};
|
||||
};
|
||||
};
|
||||
t.refresh;
|
||||
t.front;
|
||||
}.defer;
|
||||
};
|
||||
});
|
||||
)
|
||||
|
||||
//if we process the original dataset, we will see small differences in positions
|
||||
~reduced2 = FluidDataSet(s);
|
||||
~umap.transform(~standardized, ~reduced2, action: {\done.postln;});
|
||||
|
||||
//then we can draw and look
|
||||
(
|
||||
~normalizer.transform(~reduced2,~normalized,action:{
|
||||
"Normalized Output".postln;
|
||||
~normalized.dump{|x|
|
||||
~normalizedDict = x["data"];
|
||||
{
|
||||
z = Window("old material", Rect(728, 64, 200, 200));
|
||||
z.drawFunc = {
|
||||
Pen.use {
|
||||
~normalizedDict.keysValuesDo{|key, val|
|
||||
Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]);
|
||||
Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5));
|
||||
~colours[key.asSymbol].flat;
|
||||
};
|
||||
};
|
||||
};
|
||||
z.refresh;
|
||||
z.front;
|
||||
}.defer;
|
||||
};
|
||||
});
|
||||
)
|
||||
|
||||
//this is because the fitTransform method has the advantage of being certain that the data it transforms is the one that has been used to fit the model. This allows for more accurate distance measurement.
|
||||
|
||||
//to check, let's retrieve a single point and predict its position
|
||||
(
|
||||
~sourcePoint = Buffer(s);
|
||||
~original = Buffer(s);
|
||||
~standed = Buffer(s);
|
||||
~umaped = Buffer(s);
|
||||
)
|
||||
|
||||
//retrieve the 3D original
|
||||
~raw.getPoint("entry49",~sourcePoint)
|
||||
//retrieve the fitTransformed point as the most accurate point
|
||||
~reduced.getPoint("entry49",~original, {~original.getn(0,2,{|x|x.postln})})
|
||||
//retreive the transformed point, via the standardizer
|
||||
~standardizer.transformPoint(~sourcePoint,~standed);
|
||||
~umap.transformPoint(~standed, ~umaped, {~umaped.getn(0,2,{|x|x.postln})})
|
||||
|
||||
// one can also retrieve in control rate with Server Side Queries
|
||||
// Let's map our learned UMAP dimensions to the controls of a processor
|
||||
|
||||
(
|
||||
{
|
||||
var trig = Impulse.kr(1);
|
||||
var point = WhiteNoise.kr(1.dup(3));
|
||||
var inputPoint = LocalBuf(3);
|
||||
var standPoint = LocalBuf(3);
|
||||
var outputPoint = LocalBuf(2);
|
||||
var cue1, cue2;
|
||||
Poll.kr(trig, point, [\pointX,\pointY,\pointZ]);
|
||||
point.collect{ |p,i| BufWr.kr([p],inputPoint,i)};
|
||||
cue1 = ~standardizer.kr(trig,inputPoint,standPoint);
|
||||
Poll.kr(cue1,BufRd.kr(1,standPoint,(0..2),interpolation:0),[\stdX,\stdY, \stdZ]);
|
||||
cue2 = ~umap.kr(cue1, standPoint, outputPoint);
|
||||
Poll.kr(cue2,BufRd.kr(1,outputPoint,[0,1],interpolation:0),[\newDimA,\newDimB]);
|
||||
Silent.ar;
|
||||
}.play;
|
||||
)
|
||||
|
||||
::
|
||||
Loading…
Reference in New Issue