flucoma-sc/release-packaging/Examples/Guides/Classify Timbre with a Neural Network.scd

(
// 1. Instantiate some of the things we need.
Window.closeAll;
s.waitForBoot{
	Task{
		var win;
		~nMFCCs = 13;
		~trombone = Buffer.read(s,FluidFilesPath("Olencki-TenTromboneLongTones-M.wav"));
		~oboe = Buffer.read(s,FluidFilesPath("Harker-DS-TenOboeMultiphonics-M.wav"));
		~timbre_buf = Buffer.alloc(s,~nMFCCs);
		~ds = FluidDataSet(s);
		~labels = FluidLabelSet(s);
		~point_counter = 0;

		s.sync;

		win = Window("MFCCs",Rect(0,0,800,300));

		~mfcc_multislider = MultiSliderView(win,win.bounds)
		.elasticMode_(true)
		.size_(~nMFCCs);

		win.front;

	}.play(AppClock);
};
)

/*
2. Play some trombone sounds.
*/
(
{
	var sig = PlayBuf.ar(1,~trombone,BufRateScale.ir(~trombone),doneAction:2);
	var mfccs = FluidMFCC.kr(sig,~nMFCCs,40,1,maxNumCoeffs:~nMFCCs);
	SendReply.kr(Impulse.kr(30),"/mfccs",mfccs);
	FluidKrToBuf.kr(mfccs,~timbre_buf);
	sig.dup;
}.play;

OSCFunc({
	arg msg;
	{~mfcc_multislider.value_(msg[3..].linlin(-30,30,0,1))}.defer;
},"\mfccs");
)

/*
3. When you know the MFCC buf has trombone timbre data in it
(because you hear trombone and see it in the multislider),
execute this next block to add points to the dataset and
labels to the label set.

Avoid adding points when there is silence inbetween trombone
tones, because... silence isn't trombone, so we don't want
to label it that way.

Try adding points continuously during the first three or so
trombone tones. We'll save the rest to test on later.
*/
(
var id = "example-%".format(~point_counter);
~ds.addPoint(id,~timbre_buf);
~labels.addLabel(id,"trombone");
~point_counter = ~point_counter + 1;
)

/*
4. Play some oboe sounds.
*/
(
{
	var sig = PlayBuf.ar(1,~oboe,BufRateScale.ir(~oboe),doneAction:2);
	var mfccs = FluidMFCC.kr(sig,~nMFCCs,40,1,maxNumCoeffs:~nMFCCs);
	SendReply.kr(Impulse.kr(30),"/mfccs",mfccs);
	FluidKrToBuf.kr(mfccs,~timbre_buf);
	sig.dup;
}.play;

OSCFunc({
	arg msg;
	{~mfcc_multislider.value_(msg[3..].linlin(-30,30,0,1))}.defer;
},"\mfccs");
)

/*
5. All same as before with the trombone sounds.
*/
(
var id = "example-%".format(~point_counter);
~ds.addPoint(id,~timbre_buf);
~labels.addLabel(id,"oboe");
~point_counter = ~point_counter + 1;
)

/*
6. Make an MLPClassifier (neural network) to train. For more information about the parameters
visit: https://learn.flucoma.org/reference/mlpclassifier
*/
~mlpclassifier = FluidMLPClassifier(s,[5],1,learnRate:0.05,batchSize:5,validation:0.1);

/*
7. You may want to do a ".fit" more than once. For this task a loss value less than 0.01 would
be pretty good. Loss values however are always very relative so it's not really possible
to make objective observations about what one should "aim" for with a loss value. The best
way to know if a neural network is successfully performing the task you would like it to
is to test it. Probably using examples that it has never seen before.
*/
(
~mlpclassifier.fit(~ds,~labels,{
	arg loss;
	loss.postln;
});
)

/*
8. Make a prediction buffer to write the MLPClassifier's predictions into. The predictions that
it outputs to a buffer are integers. "0" will be represent what ever the "zeroth" example
label it saw was (because we always start counting from zero in these cases). "1" will represent
the "first" example label it saw, etc.
*/
~prediction_buf = Buffer.alloc(s,1);

/*
9. Play some trombone sounds and make some predictions. It should show a 0.
*/
(
{
	var sig = PlayBuf.ar(1,~trombone,BufRateScale.ir(~trombone),doneAction:2);
	var mfccs = FluidMFCC.kr(sig,~nMFCCs,40,1,maxNumCoeffs:~nMFCCs);
	FluidKrToBuf.kr(mfccs,~timbre_buf);
	~mlpclassifier.kr(Impulse.kr(30),~timbre_buf,~prediction_buf);
	FluidBufToKr.kr(~prediction_buf).poll;
	sig.dup;
}.play;
)

/*
10. Play some oboe sounds and make some predictions. It should show a 1.
*/
(
{
	var sig = PlayBuf.ar(1,~oboe,BufRateScale.ir(~oboe),doneAction:2);
	var mfccs = FluidMFCC.kr(sig,~nMFCCs,40,1,maxNumCoeffs:~nMFCCs);
	FluidKrToBuf.kr(mfccs,~timbre_buf);
	~mlpclassifier.kr(Impulse.kr(30),~timbre_buf,~prediction_buf);
	FluidBufToKr.kr(~prediction_buf).poll;
	sig.dup;
}.play;
)

/*
11. During the silences it is reporting either trombone or oboe, because that's all
it knows about, let's zero out the timbre_buf to simulate silence and then add
some points that are labeled "silence".
*/
~timbre_buf.setn(0,0.dup(~nMFCCs))

(
100.do{
	var id = "example-%".format(~point_counter);
	~ds.addPoint(id,~timbre_buf);
	~labels.addLabel(id,"silence");
	~point_counter = ~point_counter + 1;
};
)

~ds.print;
~labels.print;

~ds.write("tmp/%_ds.json".format(Date.localtime.stamp));
~labels.write("tmp/%_labels.json".format(Date.localtime.stamp))

/*
12. Now go retrain some more and do some more predictions. The silent gaps between
tones should now report a "2".
*/