TITLE:: FluidMFCC
SUMMARY:: Mel-Frequency Cepstral Coefficients as Spectral Descriptors in Real-Time
CATEGORIES:: Libraries>FluidCorpusManipulation
RELATED:: Classes/FluidBufMFCC,Classes/FluidPitch,Classes/FluidMelBands,Classes/FluidLoudness,Classes/FluidSpectralShape,Guides/FluidCorpusManipulationToolkit
DESCRIPTION::

    
    This class implements a classic spectral descriptor, the Mel-Frequency Cepstral Coefficients (MFCCs)


    
    See LINK::https://en.wikipedia.org/wiki/Mel-frequency_cepstrum::. The input is first decomposed into perceptually spaced bands (the number of bands specified by numBands), just as in the MelBands object. It is then analysed in numCoefs number of cepstral coefficients. It has the avantage to be amplitude invarient, except for the first coefficient.

    The process will return a multichannel control steam of maxNumCoeffs, which will be repeated if no change happens within the algorithm, i.e. when the hopSize is larger than the host vector size.


CLASSMETHODS::

METHOD:: kr

ARGUMENT:: in

  Audio-rate signal to analyze

ARGUMENT:: numCoeffs

    
    The number of cepstral coefficients to be outputed. It is limited by the maxNumCoefs parameter. When the number is smaller than the maximum, the output is zero-padded.

    STRONG::Constraints::

    LIST::
    ## 
    Minimum: 2

    ## 
    Maximum: MIN(CODE::numBands::, CODE::maxNumCoeffs::)

    ::

ARGUMENT:: numBands

    
    The number of bands that will be perceptually equally distributed between minFreq and maxFreq to describe the spectral shape before it is converted to cepstral coefficients.

    STRONG::Constraints::

    LIST::
    ## 
    Minimum: MAX(CODE::numCoeffs::, 2)

    ## 
    Maximum: CODE::(FFT Size / 2) + 1:: (see fft settings)

    ::

ARGUMENT:: startCoeff

    
    The lowest index of the output cepstral coefficient, zero-counting.

    STRONG::Constraints::

    LIST::
    ## 
    Minimum: 0

    ## 
    Maximum: 1

    ::

ARGUMENT:: minFreq

    
    The lower boundary of the lowest band of the model, in Hz.

    STRONG::Constraints::

    LIST::
    ## 
    Minimum: 0

    ::

ARGUMENT:: maxFreq

    
    The highest boundary of the highest band of the model, in Hz.

    STRONG::Constraints::

    LIST::
    ## 
    Minimum: 0

    ::

ARGUMENT:: windowSize

    
    The window size. As MFCC computation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. LINK::http://www.subsurfwiki.org/wiki/Gabor_uncertainty::


ARGUMENT:: hopSize

    
    The window hop size. As MFCC computation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2).


ARGUMENT:: fftSize

    
    The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize.


 
ARGUMENT:: maxNumCoeffs

    
    The maximum number of cepstral coefficients that can be computed. This sets the number of channels of the output, and therefore cannot be modulated.

    STRONG::Constraints::

    LIST::
    ## 
    Minimum: 2

    ## 
    Maximum: (max FFFT Size / 2) + 1`` (see maxFFTSize)

    ::

ARGUMENT:: maxFFTSize

    
    How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated.


 

INSTANCEMETHODS::
  
EXAMPLES::

code::
//create a monitoring window for the values

(
b = Bus.new(\control,0,13);
w = Window("MFCCs Monitor", Rect(10, 10, 420, 320)).front;
a = MultiSliderView(w,Rect(10, 10, 400, 300)).elasticMode_(1).isFilled_(1);
a.reference_(Array.fill(13,{0.5})); //make a center line to show 0
)

//run the window updating routine.
(
~winRange = 20;

r = Routine {
	{
		b.get({ arg val;
			{
				if(w.isClosed.not) {
					//val.postln;
					a.value = val.linlin(~winRange.neg,~winRange,0,1);
				}
			}.defer
		});
		0.01.wait;
	}.loop
}.play
)

//play a simple sound to observe the values
(
x = {arg type = 0;
	var source = Select.ar(type,[SinOsc.ar(220),Saw.ar(220),Pulse.ar(220)]) * LFTri.kr(0.1).exprange(0.01,0.1);
	Out.kr(b,FluidMFCC.kr(source,maxNumCoeffs:13));
	source.dup;
}.play;
)

// change the wave types, observe the amplitude invariance of the descriptors, apart from the leftmost coefficient
x.set(\type, 1)
~winRange = 40; //adjust the range above and below 0 to zoom in or out on the MFCC
x.set(\type, 2)
x.set(\type, 0)
// free this source
x.free

// load a more exciting one
c = Buffer.read(s,File.realpath(FluidMFCC.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/Tremblay-AaS-SynthTwoVoices-M.wav");

// analyse with parameters to be changed
(
x = {arg bands = 40, low = 20, high = 20000;
	var source = PlayBuf.ar(1,c,loop:1);
	Out.kr(b,FluidMFCC.kr(source, numCoeffs: 13, numBands: bands, minFreq: low, maxFreq: high, maxNumCoeffs: 13) / 10);
	source.dup;
}.play;
)
~winRange = 10; //adjust the range above and below 0 to zoom in or out on the MFCC

// observe the number of bands. The unused ones at the top are not updated
x.set(\bands,20)

// back to the full range
x.set(\bands,40)

// focus all the bands on a mid range
x.set(\low,320, \high, 800)

// focusing on the low end shows the fft resolution issue. One could restart the analysis with a larger fft to show more precision
x.set(\low,20, \high, 160)

// back to full range
x.set(\low,20, \high, 20000)

// free everything
x.free;b.free;c.free;r.stop;
::

STRONG::A musical example::

CODE::
//program that freezes mfcc spectra, then looks for matches between two frozen spectra
(
SynthDef("MFCCJamz", {arg freq=220, source = 0, buffer, mfccBus, distBus, t_freeze0=0, t_freeze1=0, onsetsOn0=0, onsetsOn1=0, restart = 1;
	var sound, mfcc, mfccFreeze0, mfccFreeze1, dist0, dist1, closest, slice;

	sound = SelectX.ar(source, [
		SinOsc.ar(freq, 0, 0.1),
		LFTri.ar(freq, 0, 0.1),
		LFSaw.ar(freq, 0, 0.1),
		Pulse.ar(freq, 0.5, 0.1),
		WhiteNoise.ar(0.1),
		PinkNoise.ar(0.1),
		PlayBuf.ar(1, buffer, 1, loop:1, trigger:restart)
	]);

	slice = FluidOnsetSlice.ar(sound); //onset detection for mfcc freeze on onset

	mfcc = FluidMFCC.kr(sound,maxNumCoeffs:13);
	mfccFreeze0 = Latch.kr(mfcc, t_freeze0+(slice*onsetsOn0));
	mfccFreeze1 = Latch.kr(mfcc, t_freeze1+(slice*onsetsOn1));

	Out.kr(mfccBus,mfcc.addAll(mfccFreeze0).addAll(mfccFreeze1));

	//distance calculations

	dist0 = Mix((mfcc.copyRange(1,12) - mfccFreeze0.copyRange(1,12)).squared).sqrt;
	dist1 = Mix((mfcc.copyRange(1,12) - mfccFreeze1.copyRange(1,12)).squared).sqrt;

	Out.kr(distBus, [dist0, dist1]);

	//sends a trigger when the item with a closer euclidean distance changes
	SendTrig.kr(Trig1.kr(dist1-dist0, 0.001)+Trig1.kr(dist0-dist1, 0.001), 0, dist1<dist0);

	Out.ar(0, sound);
}).load(s);
)


(
var buffers, buffer, paths, mfccBus, freezeBus, distBus, win, sliders, updateRout, winRange, currentMFCC, synth, movingGUI, trainButtons, playbackButton, oscFunc, closestBus;

winRange = 100;

win = Window("MFCCs Monitor", Rect(10, 10, 450, 320)).front;
sliders = List.newClear(0);
3.do{|i|
	sliders.add(MultiSliderView().maxWidth_(150).maxHeight_(150)
		.elasticMode_(1).isFilled_(1)
		.reference_(Array.fill(13,{0.5})); //make a center line to show 0
	);
};

//look in the directory for all .wav files
paths = PathName(File.realpath(FluidMFCC.class.filenameSymbol)
	.dirname.withTrailingSlash ++ "../AudioFiles/")
.files.select({arg item; item.fullPath.contains(".wav")})
.collect({arg item; item.fullPath});



s.waitForBoot({
	Routine({
		buffers = List.newClear(0);

		paths.do{arg item;buffers.add(Buffer.read(s, item.postln));};

		2.wait;
		s.sync;

		mfccBus = Bus.control(s, 39);
		distBus = Bus.control(s, 2);
		closestBus = Bus.control(s);

		synth = Synth("MFCCJamz", [\buffer, buffers[0], \mfccBus, mfccBus, \distBus, distBus, \closestBus, closestBus]);

		//the slider and PopUps under the main mfcc display

		movingGUI = [
			Slider().orientation_(\horizontal).action_{|sl| winRange = sl.value.linlin(0,1,50,500)},
			Slider().orientation_(\horizontal).action_{|sl| synth.set(\freq, sl.value.linexp(0,1,220,660))},
			PopUpMenu().items_(["sine", "tri", "saw", "square", "white", "pink", "buf"])
			.action_{|menu| synth.set(\source, menu.value)}
			.maxWidth_(150),
			PopUpMenu().items_(paths)
			.action_{|menu| synth.set(\buffer, buffers[menu.value])}
			.maxWidth_(150)
		];

		//the buttons under the two frozen mfcc displays

		trainButtons = List.newClear(0);
		2.do{arg i;
			i.postln;
			trainButtons.add(Button().states_([["train", Color.black, Color.green]])
				.action_{arg butt;
					//freezeBus.setn(currentMFCC);
					synth.set(("t_freeze"++i).asSymbol,1);
				};
			);
			trainButtons.add(Button().states_([["trainOnset", Color.black, Color.red],["trainOnset", Color.black, Color.green]])
				.action_{arg butt;
					synth.set(("onsetsOn"++i).asSymbol,butt.value);
				};
			);
			trainButtons.add(Button().states_([["", Color.black, Color.black],["", Color.black, Color.blue]]));
			trainButtons.add(StaticText());
		};

		playbackButton = Button().states_([["restart", Color.black, Color.green]])
				.mouseDownAction_({arg butt;
					synth.set(\restart, 0.5);
				})
				.action_{arg butt;
					synth.set(\restart, -0.5);
				};

		win.layout_(HLayout(
			VLayout(sliders[0], movingGUI[0], movingGUI[1], movingGUI[2]),
			VLayout(sliders[1],HLayout(trainButtons[0],trainButtons[1]), HLayout(trainButtons[2],trainButtons[3]), movingGUI[3]),
			VLayout(sliders[2],HLayout(trainButtons[4],trainButtons[5]), HLayout(trainButtons[6],trainButtons[7]), playbackButton)
		));
		win.front;
		win.onClose_{synth.free; oscFunc.free; updateRout.stop};

		//receives a trigger when the nearest element changes
		oscFunc = OSCFunc({ arg msg, time;
			{
				trainButtons[2].value_(1-msg[3]);
				trainButtons[6].value_(msg[3]);
			}.defer
		},'/tr', s.addr);

		//update the window
		updateRout = Routine {
			{arg i;
				mfccBus.get({ arg val;
					{
						currentMFCC = val;
						sliders[0].value = val.copyRange(0,12).linlin(winRange.neg,winRange,0,1);
						sliders[1].value = val.copyRange(13,25).linlin(winRange.neg,winRange,0,1);
						sliders[2].value = val.copyRange(26,38).linlin(winRange.neg,winRange,0,1);
					}.defer
				});
				distBus.get({ arg val;
					{
						trainButtons[3].string = val[0].round(0.001).asString;
						trainButtons[7].string = val[1].round(0.001).asString;

					}.defer
				});
				0.05.wait;
			}.loop
		}.play;

	}).play(AppClock);
})
)
::