added example files for human testing
Ted Moore
4 years ago
parent
70f071c749
commit
930858bd08
@ -0,0 +1,235 @@
|
|||||||
|
// define a few processes
|
||||||
|
(
|
||||||
|
~ds = FluidDataSet(s);
|
||||||
|
~dsW = FluidDataSet(s);
|
||||||
|
~dsL = FluidDataSet(s);
|
||||||
|
//define as many buffers as we have parallel voices/threads in the extractor processing (default is 4)
|
||||||
|
~loudbuf = 4.collect{Buffer.new};
|
||||||
|
~weightbuf = 4.collect{Buffer.new};
|
||||||
|
~mfccbuf = 4.collect{Buffer.new};
|
||||||
|
~statsbuf = 4.collect{Buffer.new};
|
||||||
|
~flatbuf = 4.collect{Buffer.new};
|
||||||
|
|
||||||
|
// here we instantiate a loader as per example 0
|
||||||
|
~loader = FluidLoadFolder(File.realpath(FluidBufMFCC.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/");
|
||||||
|
|
||||||
|
// here we instantiate a further slicing step as per example 0
|
||||||
|
~slicer = FluidSliceCorpus({ |src,start,num,dest|
|
||||||
|
FluidBufOnsetSlice.kr(src,start,num,metric: 9, minSliceLength: 17, indices:dest, threshold:0.2,blocking: 1)
|
||||||
|
});
|
||||||
|
|
||||||
|
// here we instantiate a process of description and dataset writing, as per example 0
|
||||||
|
~extractor = FluidProcessSlices({|src,start,num,data|
|
||||||
|
var identifier, voice, mfcc, stats, flatten;
|
||||||
|
identifier = data.key;
|
||||||
|
voice = data.value[\voice];
|
||||||
|
mfcc = FluidBufMFCC.kr(src, startFrame:start, numFrames:num, numChans:1, features:~mfccbuf[voice], padding: 2, trig:1, blocking: 1);
|
||||||
|
stats = FluidBufStats.kr(~mfccbuf[voice], stats:~statsbuf[voice], numDerivs: 1, trig:Done.kr(mfcc), blocking: 1);
|
||||||
|
flatten = FluidBufFlatten.kr(~statsbuf[voice], destination:~flatbuf[voice], trig:Done.kr(stats), blocking: 1);
|
||||||
|
FluidDataSetWr.kr(~ds, identifier, nil, ~flatbuf[voice], Done.kr(flatten), blocking: 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
// here we make another processor, this time with doing an amplitude weighing
|
||||||
|
~extractorW = FluidProcessSlices({|src,start,num,data|
|
||||||
|
var identifier, voice, loud, weights, mfcc, stats, flatten;
|
||||||
|
identifier = data.key;
|
||||||
|
voice = data.value[\voice];
|
||||||
|
mfcc = FluidBufMFCC.kr(src, startFrame:start, numFrames:num, numChans:1, features:~mfccbuf[voice], padding: 2, trig:1, blocking: 1);
|
||||||
|
loud = FluidBufLoudness.kr(src, startFrame:start, numFrames:num, numChans:1, features:~loudbuf[voice], padding: 2, trig:Done.kr(mfcc), blocking: 1);
|
||||||
|
weights = FluidBufScale.kr(~loudbuf[voice], numChans: 1, destination: ~weightbuf[voice], inputLow: -70, inputHigh: 0, trig: Done.kr(loud), blocking: 1);
|
||||||
|
stats = FluidBufStats.kr(~mfccbuf[voice], stats:~statsbuf[voice], numDerivs: 1, weights: ~weightbuf[voice], trig:Done.kr(weights), blocking: 1);
|
||||||
|
flatten = FluidBufFlatten.kr(~statsbuf[voice], destination:~flatbuf[voice], trig:Done.kr(stats), blocking: 1);
|
||||||
|
FluidDataSetWr.kr(~dsW, identifier, nil, ~flatbuf[voice], Done.kr(flatten), blocking: 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
// and here we make a little processor for loudness if we want to poke at it
|
||||||
|
~extractorL = FluidProcessSlices({|src,start,num,data|
|
||||||
|
var identifier, voice, loud, stats, flatten;
|
||||||
|
identifier = data.key;
|
||||||
|
voice = data.value[\voice];
|
||||||
|
loud = FluidBufLoudness.kr(src, startFrame:start, numFrames:num, numChans:1, features:~mfccbuf[voice], trig:1, padding: 2, blocking: 1);
|
||||||
|
stats = FluidBufStats.kr(~mfccbuf[voice], stats:~statsbuf[voice], numDerivs: 1, trig:Done.kr(loud), blocking: 1);
|
||||||
|
flatten = FluidBufFlatten.kr(~statsbuf[voice], destination:~flatbuf[voice], trig:Done.kr(stats), blocking: 1);
|
||||||
|
FluidDataSetWr.kr(~dsL, identifier, nil, ~flatbuf[voice], Done.kr(flatten), blocking: 1);
|
||||||
|
});
|
||||||
|
)
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////
|
||||||
|
//loading process
|
||||||
|
|
||||||
|
//load and play to test if it is that quick - it is!
|
||||||
|
(
|
||||||
|
t = Main.elapsedTime;
|
||||||
|
~loader.play(s,action:{(Main.elapsedTime - t).postln;"Loaded".postln;{var start, stop; PlayBuf.ar(~loader.index[~loader.index.keys.asArray.last.asSymbol][\numchans],~loader.buffer,startPos: ~loader.index[~loader.index.keys.asArray.last.asSymbol][\bounds][0])}.play;});
|
||||||
|
)
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////
|
||||||
|
// slicing process
|
||||||
|
|
||||||
|
// run the slicer
|
||||||
|
(
|
||||||
|
t = Main.elapsedTime;
|
||||||
|
~slicer.play(s,~loader.buffer,~loader.index,action:{(Main.elapsedTime - t).postln;"Slicing done".postln});
|
||||||
|
)
|
||||||
|
|
||||||
|
//slice count
|
||||||
|
~slicer.index.keys.size
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////
|
||||||
|
// description process
|
||||||
|
|
||||||
|
// run both descriptor extractor - here they are separate to the batch process duration
|
||||||
|
(
|
||||||
|
t = Main.elapsedTime;
|
||||||
|
~extractor.play(s,~loader.buffer,~slicer.index,action:{(Main.elapsedTime - t).postln;"Features done".postln});
|
||||||
|
)
|
||||||
|
|
||||||
|
(
|
||||||
|
t = Main.elapsedTime;
|
||||||
|
~extractorW.play(s,~loader.buffer,~slicer.index,action:{(Main.elapsedTime - t).postln;"Features done".postln});
|
||||||
|
)
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////
|
||||||
|
// manipulating and querying the data
|
||||||
|
|
||||||
|
// extracting whatever stats we want. In this case, mean/std/lowest/highest, and the same on the first derivative - excluding MFCC0 as it is mostly volume, keeping MFCC1-12
|
||||||
|
|
||||||
|
(
|
||||||
|
~curated = FluidDataSet(s);
|
||||||
|
~curatedW = FluidDataSet(s);
|
||||||
|
~curator = FluidDataSetQuery.new(s);
|
||||||
|
)
|
||||||
|
|
||||||
|
(
|
||||||
|
~curator.addRange(1,12,{
|
||||||
|
~curator.addRange(14,12,{
|
||||||
|
~curator.addRange(53,12,{
|
||||||
|
~curator.addRange(79,12,{
|
||||||
|
~curator.addRange(92,12,{
|
||||||
|
~curator.addRange(105,12,{
|
||||||
|
~curator.addRange(144,12,{
|
||||||
|
~curator.addRange(170,12);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
)
|
||||||
|
~curator.transform(~ds,~curated)
|
||||||
|
~curator.transform(~dsW,~curatedW)
|
||||||
|
|
||||||
|
//check the dimension count
|
||||||
|
~ds.print
|
||||||
|
~dsW.print
|
||||||
|
~curated.print
|
||||||
|
~curatedW.print
|
||||||
|
|
||||||
|
//building a tree for each dataset
|
||||||
|
~tree = FluidKDTree(s,5);
|
||||||
|
~tree.fit(~ds,{"Fitted".postln;});
|
||||||
|
~treeW = FluidKDTree(s,5);
|
||||||
|
~treeW.fit(~dsW,{"Fitted".postln;});
|
||||||
|
~treeC = FluidKDTree(s,5);
|
||||||
|
~treeC.fit(~curated,{"Fitted".postln;});
|
||||||
|
~treeCW = FluidKDTree(s,5);
|
||||||
|
~treeCW.fit(~curatedW,{"Fitted".postln;});
|
||||||
|
|
||||||
|
//select a sound to match
|
||||||
|
// EITHER retrieve a random slice
|
||||||
|
~targetsound = Buffer(s);
|
||||||
|
~targetname = ~slicer.index.keys.asArray.scramble.last.asSymbol;
|
||||||
|
#a,b = ~slicer.index[~targetname][\bounds];
|
||||||
|
FluidBufCompose.process(s,~loader.buffer,a,(b-a),numChans: 1, destination: ~targetsound,action: {~targetsound.play;})
|
||||||
|
|
||||||
|
// OR just load a file in that buffer
|
||||||
|
~targetsound = Buffer.read(s,Platform.resourceDir +/+ "sounds/a11wlk01.wav");
|
||||||
|
|
||||||
|
//describe the sound to match
|
||||||
|
(
|
||||||
|
{
|
||||||
|
var loud, weights, mfcc, stats, flatten, stats2, written;
|
||||||
|
mfcc = FluidBufMFCC.kr(~targetsound,features:~mfccbuf[0],padding: 2, trig:1);
|
||||||
|
stats = FluidBufStats.kr(~mfccbuf[0],stats:~statsbuf[0], numDerivs: 1,trig:Done.kr(mfcc));
|
||||||
|
flatten = FluidBufFlatten.kr(~statsbuf[0],destination:~flatbuf[0],trig:Done.kr(stats));
|
||||||
|
loud = FluidBufLoudness.kr(~targetsound,features:~loudbuf[0],padding: 2,trig:Done.kr(flatten),blocking: 1);
|
||||||
|
weights = FluidBufScale.kr(~loudbuf[0],numChans: 1,destination: ~weightbuf[0],inputLow: -70,inputHigh: 0,trig: Done.kr(loud),blocking: 1);
|
||||||
|
stats2 = FluidBufStats.kr(~mfccbuf[0],stats:~statsbuf[0], numDerivs: 1, weights: ~weightbuf[0], trig:Done.kr(weights),blocking: 1);
|
||||||
|
written = FluidBufFlatten.kr(~statsbuf[0],destination:~flatbuf[1],trig:Done.kr(stats2));
|
||||||
|
FreeSelf.kr(Done.kr(written));
|
||||||
|
}.play;
|
||||||
|
)
|
||||||
|
|
||||||
|
//go language side to extract the right dimensions
|
||||||
|
~flatbuf[0].getn(0,182,{|x|~curatedBuf = Buffer.loadCollection(s, x[[0,1,4,6,7,8,11,13].collect{|x|var y=x*13+1;(y..(y+11))}.flat].postln)})
|
||||||
|
~flatbuf[1].getn(0,182,{|x|~curatedWBuf = Buffer.loadCollection(s, x[[0,1,4,6,7,8,11,13].collect{|x|var y=x*13+1;(y..(y+11))}.flat].postln)})
|
||||||
|
|
||||||
|
//find its nearest neighbours
|
||||||
|
~tree.kNearest(~flatbuf[0],{|x| ~friends = x.postln;})
|
||||||
|
~treeW.kNearest(~flatbuf[1],{|x| ~friendsW = x.postln;})
|
||||||
|
~treeC.kNearest(~curatedBuf,{|x| ~friendsC = x.postln;})
|
||||||
|
~treeCW.kNearest(~curatedWBuf,{|x| ~friendsCW = x.postln;})
|
||||||
|
|
||||||
|
|
||||||
|
// play them in a row
|
||||||
|
(
|
||||||
|
Routine{
|
||||||
|
5.do{|i|
|
||||||
|
var dur;
|
||||||
|
v = ~slicer.index[~friends[i].asSymbol];
|
||||||
|
dur = (v[\bounds][1] - v[\bounds][0]) / s.sampleRate;
|
||||||
|
{BufRd.ar(v[\numchans],~loader.buffer,Line.ar(v[\bounds][0],v[\bounds][1],dur, doneAction: 2))}.play;
|
||||||
|
~friends[i].postln;
|
||||||
|
dur.wait;
|
||||||
|
};
|
||||||
|
}.play;
|
||||||
|
)
|
||||||
|
|
||||||
|
(
|
||||||
|
Routine{
|
||||||
|
5.do{|i|
|
||||||
|
var dur;
|
||||||
|
v = ~slicer.index[~friendsW[i].asSymbol];
|
||||||
|
dur = (v[\bounds][1] - v[\bounds][0]) / s.sampleRate;
|
||||||
|
{BufRd.ar(v[\numchans],~loader.buffer,Line.ar(v[\bounds][0],v[\bounds][1],dur, doneAction: 2))}.play;
|
||||||
|
~friendsW[i].postln;
|
||||||
|
dur.wait;
|
||||||
|
};
|
||||||
|
}.play;
|
||||||
|
)
|
||||||
|
|
||||||
|
(
|
||||||
|
Routine{
|
||||||
|
5.do{|i|
|
||||||
|
var dur;
|
||||||
|
v = ~slicer.index[~friendsC[i].asSymbol];
|
||||||
|
dur = (v[\bounds][1] - v[\bounds][0]) / s.sampleRate;
|
||||||
|
{BufRd.ar(v[\numchans],~loader.buffer,Line.ar(v[\bounds][0],v[\bounds][1],dur, doneAction: 2))}.play;
|
||||||
|
~friendsC[i].postln;
|
||||||
|
dur.wait;
|
||||||
|
};
|
||||||
|
}.play;
|
||||||
|
)
|
||||||
|
|
||||||
|
(
|
||||||
|
Routine{
|
||||||
|
5.do{|i|
|
||||||
|
var dur;
|
||||||
|
v = ~slicer.index[~friendsCW[i].asSymbol];
|
||||||
|
dur = (v[\bounds][1] - v[\bounds][0]) / s.sampleRate;
|
||||||
|
{BufRd.ar(v[\numchans],~loader.buffer,Line.ar(v[\bounds][0],v[\bounds][1],dur, doneAction: 2))}.play;
|
||||||
|
~friendsCW[i].postln;
|
||||||
|
dur.wait;
|
||||||
|
};
|
||||||
|
}.play;
|
||||||
|
)
|
||||||
|
|
||||||
|
//explore dynamic range (changing the weigting's value of 0 in lines 39 and 157 will change the various weights given to quieter parts of the signal
|
||||||
|
(
|
||||||
|
t = Main.elapsedTime;
|
||||||
|
~extractorL.play(s,~loader.buffer,~slicer.index,action:{(Main.elapsedTime - t).postln;"Features done".postln});
|
||||||
|
)
|
||||||
|
~norm = FluidNormalize.new(s)
|
||||||
|
~norm.fit(~dsL)
|
||||||
|
~norm.dump({|x|x["data_min"][[8,12]].postln;x["data_max"][[8,12]].postln;})//here we extract the stats from the dataset by retrieving the stored maxima of the fitting process in FluidNormalize
|
||||||
@ -0,0 +1,73 @@
|
|||||||
|
//load a part of a sound that has 3 clear components: a clear pitch component to start, a noisy pitchless ending and DC offset silence on both ends
|
||||||
|
(
|
||||||
|
b = Buffer.read(s,File.realpath(FluidBufPitch.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/Tremblay-ASWINE-ScratchySynth-M.wav",42250,44100);
|
||||||
|
~pitches = Buffer(s);
|
||||||
|
~stats = Buffer(s);
|
||||||
|
~loud = Buffer(s);
|
||||||
|
~scaled = Buffer(s);
|
||||||
|
~brutePitchStats = Array;
|
||||||
|
~loudnessWeighedPitchStats = Array;
|
||||||
|
~confidenceWeighedPitchStats = Array;
|
||||||
|
~condidenceWeightedPitchIQRStats = Array;
|
||||||
|
~pitchIQRStats = Array;
|
||||||
|
)
|
||||||
|
|
||||||
|
b.play
|
||||||
|
b.plot
|
||||||
|
|
||||||
|
FluidBufPitch.process(s,b,features: ~pitches)
|
||||||
|
FluidBufStats.process(s,~pitches,stats:~stats)
|
||||||
|
~stats.getn(0,14,{|x|~brutePitchStats = x; x.reshape(7,2).do{|y| "%\t\t\t%\n".postf(y[0].round(0.1),y[1].round(0.01))}})
|
||||||
|
|
||||||
|
//observe the data - there are something clearly "wrong" in there - mostly, should we consider the stats on pitch when the confidence is low?
|
||||||
|
~pitches.plot(separately: true)
|
||||||
|
|
||||||
|
//let's check the loudness
|
||||||
|
FluidBufLoudness.process(s,b,features: ~loud)
|
||||||
|
FluidBufStats.process(s,~loud,stats:~stats)
|
||||||
|
~stats.getn(0,14,{|x|x.reshape(7,2).do{|y| "%\t\t\t%\n".postf(y[0].round(0.1),y[1].round(0.01))}})
|
||||||
|
|
||||||
|
~loud.plot(separately: true)
|
||||||
|
|
||||||
|
//it seems the loudness of the noisy section will not help us reject the silence. But let's try
|
||||||
|
FluidBufScale.process(s,~loud,numChans: 1,destination: ~scaled,inputLow: -60,inputHigh: -20)
|
||||||
|
FluidBufStats.process(s,~pitches, stats:~stats,weights: ~scaled)
|
||||||
|
~stats.getn(0,14,{|x|~loudnessWeighedPitchStats = x; x.reshape(7,2).do{|y| "%\t\t\t%\n".postf(y[0].round(0.1),y[1].round(0.01))}})
|
||||||
|
//not much difference but let's listen
|
||||||
|
|
||||||
|
//average pitch
|
||||||
|
c = {SinOsc.ar(~brutePitchStats[0],mul: 0.05)}.play
|
||||||
|
//compare with the source
|
||||||
|
b.play
|
||||||
|
c.free
|
||||||
|
//loudness-weighted average
|
||||||
|
c = {SinOsc.ar(~loudnessWeighedPitchStats[0],mul: 0.05)}.play
|
||||||
|
//compare with the source
|
||||||
|
b.play
|
||||||
|
c.free
|
||||||
|
//hmmm, worse! That is because we did remove the low amplitude skewing to wards the default value (high) which was balancing our noisy peak with low pitch and low pitch confidence...
|
||||||
|
|
||||||
|
//let's instead weight against the pitch confidence, first applying a threshold to so we pull down any middle value we want to ignore
|
||||||
|
FluidBufThresh.process(s, ~pitches, startChan: 1, numChans: 1, destination: ~scaled, threshold: 0.8)
|
||||||
|
FluidBufStats.process(s,~pitches, stats:~stats,weights: ~scaled)
|
||||||
|
~stats.getn(0,14,{|x|~confidenceWeighedPitchStats = x;x.reshape(7,2).do{|y| "%\t\t\t%\n".postf(y[0].round(0.1),y[1].round(0.01))}})
|
||||||
|
|
||||||
|
//let's listen
|
||||||
|
c = {SinOsc.ar(~confidenceWeighedPitchStats[0],mul: 0.05)}.play
|
||||||
|
//compare with the source
|
||||||
|
b.play
|
||||||
|
c.free
|
||||||
|
// much better! it is even better when we move the threshold above but 0.8 confidence is quite high... If we look at our stats we see that there are still minima in the low hundreds, and maxima in the very top...These must be statistically far enough and few enough just to mess a bit our stats, so let's use the inter-quantile range to first remove them then compute the stats.
|
||||||
|
FluidBufStats.process(s,~pitches, stats:~stats,weights: ~scaled,outliersCutoff: 1.5)
|
||||||
|
~stats.getn(0,14,{|x|~confidenceWeightedPitchIQRStats = x;x.reshape(7,2).do{|y| "%\t\t\t%\n".postf(y[0].round(0.1),y[1].round(0.01))}})
|
||||||
|
//now that is impressive!
|
||||||
|
c = {SinOsc.ar(~confidenceWeightedPitchIQRStats[0],mul: 0.05)}.play
|
||||||
|
b.play
|
||||||
|
c.free
|
||||||
|
|
||||||
|
//for completion, here is just with rejection of outliers - not as good, but a decent second best!
|
||||||
|
FluidBufStats.process(s,~pitches, stats:~stats,outliersCutoff: 1.5)
|
||||||
|
~stats.getn(0,14,{|x|~pitchIQRStats = x;x.reshape(7,2).do{|y| "%\t\t\t%\n".postf(y[0].round(0.1),y[1].round(0.01))}})
|
||||||
|
c = {SinOsc.ar(~pitchIQRStats[0],mul: 0.05)}.play
|
||||||
|
b.play
|
||||||
|
c.free
|
||||||
@ -0,0 +1,230 @@
|
|||||||
|
// load a source folder
|
||||||
|
~loader = FluidLoadFolder(File.realpath(FluidBufMFCC.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/");
|
||||||
|
~loader.play;
|
||||||
|
|
||||||
|
//slightly oversegment with novelty
|
||||||
|
//segments should still make sense but might cut a few elements in 2 or 3
|
||||||
|
~slicer = FluidSliceCorpus({ |src,start,num,dest| FluidBufNoveltySlice.kr(src,start,num,indices:dest, feature: 1, kernelSize: 29, threshold: 0.1, filterSize: 5, hopSize: 128, blocking: 1)});
|
||||||
|
~slicer.play(s, ~loader.buffer,~loader.index);
|
||||||
|
|
||||||
|
//test the segmentation by looping them
|
||||||
|
(
|
||||||
|
~originalindices = Array.newFrom(~slicer.index.keys).sort{|a,b| ~slicer.index[a][\bounds][0]< ~slicer.index[b][\bounds][0]}.collect{|x|~slicer.index[x][\bounds]};
|
||||||
|
d = {arg start=0, end = 44100;
|
||||||
|
BufRd.ar(1, ~loader.buffer, Phasor.ar(0,1,start,end,start),0,1);
|
||||||
|
}.play;
|
||||||
|
|
||||||
|
w = Window.new(bounds:Rect(100,100,400,60)).front;
|
||||||
|
b = ControlSpec(0, ~originalindices.size - 1, \linear, 1); // min, max, mapping, step
|
||||||
|
c = StaticText(w, Rect(340, 20, 50, 20)).align_(\center);
|
||||||
|
a = Slider(w, Rect(10, 20, 330, 20))
|
||||||
|
.action_({var val = b.map(a.value).asInteger;
|
||||||
|
c.string_(val.asString);
|
||||||
|
d.set(\start,~originalindices[val][0], \end, ~originalindices[val][1]);
|
||||||
|
});
|
||||||
|
)
|
||||||
|
|
||||||
|
//analyse each segment with 20 MFCCs in a dataset and spectralshapes in another one
|
||||||
|
(
|
||||||
|
~featuresbuf = 4.collect{Buffer.new};
|
||||||
|
~statsbuf = 4.collect{Buffer.new};
|
||||||
|
~flatbuf = 4.collect{Buffer.new};
|
||||||
|
~slicesMFCC = FluidDataSet(s);
|
||||||
|
~slicesShapes = FluidDataSet(s);
|
||||||
|
~extractor = FluidProcessSlices({|src,start,num,data|
|
||||||
|
var features, stats, writer, flatten,mfccBuf, statsBuf, flatBuf, identifier, voice;
|
||||||
|
identifier = data.key;
|
||||||
|
voice = data.value[\voice];
|
||||||
|
features = FluidBufMFCC.kr(src,startFrame:start,numFrames:num,numChans:1, numCoeffs: 20, features:~featuresbuf[voice],trig:1,blocking: 1);
|
||||||
|
stats = FluidBufStats.kr(~featuresbuf[voice],stats:~statsbuf[voice],trig:Done.kr(features),blocking: 1);
|
||||||
|
flatten = FluidBufFlatten.kr(~statsbuf[voice],destination:~flatbuf[voice],trig:Done.kr(stats),blocking: 1);
|
||||||
|
writer = FluidDataSetWr.kr(~slicesMFCC,identifier, nil, ~flatbuf[voice], Done.kr(flatten),blocking: 1);
|
||||||
|
features = FluidBufSpectralShape.kr(src,startFrame:start,numFrames:num,numChans:1, features:~featuresbuf[voice],trig:Done.kr(writer),blocking: 1);
|
||||||
|
stats = FluidBufStats.kr(~featuresbuf[voice],stats:~statsbuf[voice],trig:Done.kr(features),blocking: 1);
|
||||||
|
flatten = FluidBufFlatten.kr(~statsbuf[voice],destination:~flatbuf[voice],trig:Done.kr(stats),blocking: 1);
|
||||||
|
writer = FluidDataSetWr.kr(~slicesShapes,identifier, nil, ~flatbuf[voice], Done.kr(flatten),blocking: 1);
|
||||||
|
});
|
||||||
|
)
|
||||||
|
|
||||||
|
(
|
||||||
|
t = Main.elapsedTime;
|
||||||
|
~extractor.play(s,~loader.buffer, ~slicer.index, action:{(Main.elapsedTime - t).postln;"Analysis done".postln});
|
||||||
|
)
|
||||||
|
|
||||||
|
~originalindices.size
|
||||||
|
~slicesMFCC.print
|
||||||
|
~slicesShapes.print
|
||||||
|
|
||||||
|
//run a window over consecutive segments, forcing them in 2 classes, and merging the consecutive segments of similar class
|
||||||
|
//we overlap the analysis with the last (original) slice to check for continuity
|
||||||
|
(
|
||||||
|
~winSize = 4;//the number of consecutive items to split in 2 classes;
|
||||||
|
~curated = FluidDataSet(s);
|
||||||
|
~query = FluidDataSetQuery(s);
|
||||||
|
~stan = FluidStandardize(s);
|
||||||
|
~kmeans = FluidKMeans(s,2,1000);
|
||||||
|
~windowDS = FluidDataSet(s);
|
||||||
|
~windowLS = FluidLabelSet(s);
|
||||||
|
)
|
||||||
|
|
||||||
|
//curate stats (MFCCs)
|
||||||
|
~query.clear
|
||||||
|
~query.addRange((0*20)+1,10);
|
||||||
|
~query.transform(~slicesMFCC,~curated);
|
||||||
|
|
||||||
|
//OR
|
||||||
|
//curate stats (moments)
|
||||||
|
~query.clear
|
||||||
|
~query.addRange(0,3);
|
||||||
|
~query.transform(~slicesShapes,~curated);
|
||||||
|
|
||||||
|
//OR
|
||||||
|
//curate both
|
||||||
|
~query.clear
|
||||||
|
~query.addColumn(0);//add col 0 (mean of mfcc0 as 'loudness')
|
||||||
|
~query.transform(~slicesMFCC,~curated);//mfcc0 as loudness
|
||||||
|
~query.clear;
|
||||||
|
~query.addRange(0,3);//add some spectral moments
|
||||||
|
~query.transformJoin(~slicesShapes, ~curated, ~curated);//join in centroids
|
||||||
|
|
||||||
|
//optionally standardize in place
|
||||||
|
~stan.fitTransform(~curated, ~curated);
|
||||||
|
|
||||||
|
~curated.print
|
||||||
|
|
||||||
|
//retrieve the dataset as dictionary
|
||||||
|
~curated.dump{|x|~sliceDict = x;};
|
||||||
|
|
||||||
|
~originalslicesarray = ~originalindices.flop[0] ++ ~loader.buffer.numFrames
|
||||||
|
~orginalkeys = Array.newFrom(~slicer.index.keys).sort{|a,b| ~slicer.index[a][\bounds][0]< ~slicer.index[b][\bounds][0]}
|
||||||
|
|
||||||
|
//the windowed function, recursive to deal with sync dependencies
|
||||||
|
(
|
||||||
|
~windowedFunct = {arg head, winSize, overlap;
|
||||||
|
var nbass = [], assignments = [], tempDict = ();
|
||||||
|
//check the size of everything to not overrun
|
||||||
|
winSize = (~originalslicesarray.size - head).min(winSize);
|
||||||
|
//copy the items to a subdataset from hear
|
||||||
|
winSize.do{|i|
|
||||||
|
tempDict.put((i.asString), ~sliceDict["data"][(~orginalkeys[(i+head)]).asString]);//here one could curate which stats to take
|
||||||
|
// "whichslices:%\n".postf(i+head);
|
||||||
|
};
|
||||||
|
~windowDS.load(Dictionary.newFrom([\cols, ~sliceDict["cols"].asInteger, \data, tempDict]), action: {
|
||||||
|
// "% - loaded\n".postf(head);
|
||||||
|
|
||||||
|
//kmeans 2 and retrieve ordered array of class assignations
|
||||||
|
~kmeans.fitPredict(~windowDS, ~windowLS, action: {|x|
|
||||||
|
nbass = x;
|
||||||
|
// "% - fitted1: ".postf(head); nbass.postln;
|
||||||
|
|
||||||
|
if (nbass.includes(winSize.asFloat), {
|
||||||
|
~kmeans.fitPredict(~windowDS, ~windowLS, {|x|
|
||||||
|
nbass = x;
|
||||||
|
// "% - fitted2: ".postf(head); nbass.postln;
|
||||||
|
if (nbass.includes(winSize.asFloat), {
|
||||||
|
~kmeans.fitPredict(~windowDS, ~windowLS, {|x|
|
||||||
|
nbass = x;
|
||||||
|
// "% - fitted3: ".postf(head); nbass.postln;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
~windowLS.dump{|x|
|
||||||
|
var assignments = x.at("data").asSortedArray.flop[1].flatten;
|
||||||
|
"% - assigned ".postf(head);
|
||||||
|
|
||||||
|
assignments.postln;
|
||||||
|
|
||||||
|
(winSize-1).do{|i|
|
||||||
|
if (assignments[i+1] != assignments[i], {
|
||||||
|
~newindices= ~newindices ++ (~originalslicesarray[head+i+1]).asInteger;
|
||||||
|
~newkeys = ~newkeys ++ (~orginalkeys[head+i+1]);
|
||||||
|
});
|
||||||
|
|
||||||
|
};
|
||||||
|
//if we still have some frames to do, do them
|
||||||
|
if (((winSize + head) < ~originalslicesarray.size), {
|
||||||
|
"-----------------".postln;
|
||||||
|
~windowedFunct.value(head + winSize - overlap, winSize, overlap);
|
||||||
|
}, {~newindices = (~newindices ++ ~loader.buffer.numFrames); "done".postln;});//if we're done close the books
|
||||||
|
};
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
//the job
|
||||||
|
|
||||||
|
//test 1 - start at the begining, consider 4 items at a time, make 2 clusters, overlap 1
|
||||||
|
~newindices = [~originalslicesarray[0]]; ~newkeys = [~orginalkeys[0]];
|
||||||
|
~windowedFunct.value(0, 4, 1);
|
||||||
|
|
||||||
|
//OPTIONAL: try again with more clusters (3) and a wider window (6) and more overlap (2)
|
||||||
|
~newindices = [~originalslicesarray[0]]; ~newkeys = [~orginalkeys[0]];
|
||||||
|
~kmeans.numClusters = 3;
|
||||||
|
~windowedFunct.value(0,6,2);
|
||||||
|
|
||||||
|
//compare sizes
|
||||||
|
~orginalkeys.size
|
||||||
|
~newkeys.size;
|
||||||
|
|
||||||
|
//export to reaper
|
||||||
|
(
|
||||||
|
//first create a new file that ends with rpp - it will overwrite if the file exists
|
||||||
|
f = File.new(Platform.defaultTempDir ++ "clusteredslices-" ++ Date.getDate.stamp ++".rpp","w+");
|
||||||
|
|
||||||
|
if (f.isOpen , {
|
||||||
|
var path, prevpath ="", sr, count, dur, realDur;
|
||||||
|
//write the header
|
||||||
|
f.write("<REAPER_PROJECT 0.1 \"5.99/OSX64\" 1603037150\n\n");
|
||||||
|
|
||||||
|
//a first track with the originalslicearray
|
||||||
|
//write the track header
|
||||||
|
f.write("<TRACK\nNAME \"novelty output\"\n");
|
||||||
|
// iterate through the items in the track
|
||||||
|
~orginalkeys.do{|v, i|
|
||||||
|
path = ~slicer.index[v][\path];
|
||||||
|
if (path != prevpath, {
|
||||||
|
sr = ~slicer.index[v][\sr];
|
||||||
|
prevpath = path;
|
||||||
|
count = 0;
|
||||||
|
});
|
||||||
|
dur = ~originalslicesarray[i+1] - ~originalslicesarray[i];
|
||||||
|
if ( dur > 0, {
|
||||||
|
f.write("<ITEM\nPOSITION " ++ (~originalslicesarray[i] / sr) ++ "\nLENGTH " ++ (dur / sr) ++ "\nNAME \"" ++ v ++ "\"\nSOFFS " ++ (count / sr) ++ "\n<SOURCE WAVE\nFILE \"" ++ path ++ "\"\n>\n>\n");
|
||||||
|
});
|
||||||
|
count = count + dur;
|
||||||
|
};
|
||||||
|
//write the track footer
|
||||||
|
f.write(">\n");
|
||||||
|
|
||||||
|
// a second track with the new ~indices
|
||||||
|
prevpath = "";
|
||||||
|
//write the track header
|
||||||
|
f.write("<TRACK\nNAME \"clustered output\"\n");
|
||||||
|
// iterate through the items in the track
|
||||||
|
~newkeys.do{|v, i|
|
||||||
|
dur = ~newindices[i+1] - ~newindices[i];
|
||||||
|
if (dur > 0, {
|
||||||
|
path = ~slicer.index[v][\path];
|
||||||
|
if (path != prevpath, {
|
||||||
|
sr = ~slicer.index[v][\sr];
|
||||||
|
prevpath = path;
|
||||||
|
count = 0;
|
||||||
|
});
|
||||||
|
f.write("<ITEM\nPOSITION " ++ (~newindices[i] / sr) ++ "\nLENGTH " ++ (dur / sr) ++ "\nNAME \"" ++ v ++ "\"\nSOFFS " ++ (count / sr) ++ "\n<SOURCE WAVE\nFILE \"" ++ path ++ "\"\n>\n>\n");
|
||||||
|
count = count + dur;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
//write the track footer
|
||||||
|
f.write(">\n");
|
||||||
|
|
||||||
|
//write the footer
|
||||||
|
f.write(">\n");
|
||||||
|
f.close;
|
||||||
|
});
|
||||||
|
)
|
||||||
|
|
||||||
|
(then open the time-stamped reaper file clusterdslice in the folder tmp)
|
||||||
|
Platform.defaultTempDir.openOS
|
||||||
@ -0,0 +1,324 @@
|
|||||||
|
// Lookup in a KDTree using melbands
|
||||||
|
// Demonstration of a massive parallel approach to batch process swiftly in SC
|
||||||
|
|
||||||
|
s.options.numBuffers = 16384 //The method below for doing the analysus quickly needs lots of buffers
|
||||||
|
s.reboot
|
||||||
|
|
||||||
|
//Step 0: Make a corpus
|
||||||
|
|
||||||
|
//We'll jam together some random flucoma sounds for illustrative purposes
|
||||||
|
//Get some files
|
||||||
|
(
|
||||||
|
~audioexamples_path = File.realpath(FluidBufMelBands.class.filenameSymbol).dirname.withTrailingSlash +/+ "../AudioFiles/*.wav";
|
||||||
|
~allTheSounds = SoundFile.collect(~audioexamples_path);
|
||||||
|
~testSounds = ~allTheSounds;
|
||||||
|
~testSounds.do{|f| f.path.postln}; // print out the files that are loaded
|
||||||
|
)
|
||||||
|
|
||||||
|
//Load the files into individual buffers:
|
||||||
|
(
|
||||||
|
~audio_buffers = ~testSounds.collect{|f|
|
||||||
|
Buffer.readChannel(
|
||||||
|
server: s,
|
||||||
|
path:f.path,
|
||||||
|
channels:[0],
|
||||||
|
action:{("Loaded" + f.path).postln;}
|
||||||
|
)
|
||||||
|
};
|
||||||
|
)
|
||||||
|
|
||||||
|
//Do a segmentation of each buffer, in parallel
|
||||||
|
(
|
||||||
|
fork{
|
||||||
|
~index_buffers = ~audio_buffers.collect{Buffer.new};
|
||||||
|
s.sync;
|
||||||
|
~count = ~audio_buffers.size;
|
||||||
|
~audio_buffers.do{|src,i|
|
||||||
|
FluidBufOnsetSlice.process(
|
||||||
|
server:s,
|
||||||
|
source:src,
|
||||||
|
indices:~index_buffers[i],
|
||||||
|
metric: 9,
|
||||||
|
threshold:0.2,
|
||||||
|
minSliceLength: 17,
|
||||||
|
action:{
|
||||||
|
(~testSounds[i].path ++ ":" + ~index_buffers[i].numFrames + "slices").postln;
|
||||||
|
~count = ~count - 1;
|
||||||
|
if(~count == 0){"Done slicing".postln};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// we now have an array of index buffers, one per source buffer, each containing the segmentation points as a frame positions
|
||||||
|
// this allows us to make an array of sizes
|
||||||
|
~index_buffers.collect{|b| b.numFrames}.sum
|
||||||
|
|
||||||
|
//For each of these segments, let's make a datapoint using the mean melbands.
|
||||||
|
// There's a number of ways of skinning this cat w/r/t telling the server what to do, but here we want to minimize traffic between language and server, and also produce undertsandable code
|
||||||
|
|
||||||
|
//First, we'll grab the onset points as language-side arrays, then scroll through each slice getting the mean melbands
|
||||||
|
(
|
||||||
|
// - a dataset to keep the mean melbands in
|
||||||
|
~mels = FluidDataSet(s);
|
||||||
|
// - a dictionary to keep the slice points in for later playback
|
||||||
|
~slices = Dictionary();
|
||||||
|
//The code below (as well as needing lots of buffers), creates lots of threads and we need a big ass scheduling queue
|
||||||
|
~clock = TempoClock(queueSize:8192);
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
// Do the Mel analysis in a cunning parallel fashion
|
||||||
|
(
|
||||||
|
{
|
||||||
|
var counter, remaining;
|
||||||
|
var condition = Condition.new; // used to create a test condition to pause the routine ...
|
||||||
|
var index_arrays = Dictionary();
|
||||||
|
|
||||||
|
"Process started. Please wait.".postln;
|
||||||
|
|
||||||
|
~total_slice_count = ~index_buffers.collect{|b| b.numFrames}.sum + ~index_buffers.size; //we get an extra slice in buffer
|
||||||
|
~featurebuffers = ~total_slice_count.collect{Buffer.new}; // create a buffer per slice
|
||||||
|
|
||||||
|
//Make our dictionary FluidDataSet-shaped
|
||||||
|
~slices.put("cols",3);//[bufnum,start,end] for each slice
|
||||||
|
~slices.put("data",Dictionary());
|
||||||
|
|
||||||
|
//Collect each set of onsets into a language side array and store them in a dict
|
||||||
|
~index_buffers.do{|b,i| // iterate over the input buffer array
|
||||||
|
{
|
||||||
|
b.loadToFloatArray( // load to language side array
|
||||||
|
action:{|indices|
|
||||||
|
//Glue the first and last samples of the buffer on to the index list, and place in dictionary with the
|
||||||
|
//Buffer object as a key
|
||||||
|
|
||||||
|
index_arrays.put(~audio_buffers[i], Array.newFrom([0] ++ indices ++ (~audio_buffers[i].numFrames - 1)));
|
||||||
|
|
||||||
|
if(i==(~index_buffers.size-1)) {condition.unhang};
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}.fork(stackSize:~total_slice_count);
|
||||||
|
};
|
||||||
|
condition.hang; //Pause until all the callbacks above have completed
|
||||||
|
"Arrays loaded. Starting on the analysis, please wait.".postln;
|
||||||
|
|
||||||
|
//For each of these lists of points, we want to scroll over the indices in pairs and get some mel bands
|
||||||
|
counter = 0;
|
||||||
|
remaining = ~total_slice_count;
|
||||||
|
|
||||||
|
s.sync;
|
||||||
|
|
||||||
|
// now iterate over Dict and calc melbands
|
||||||
|
|
||||||
|
index_arrays.keysValuesDo{|buffer, indices|
|
||||||
|
indices.doAdjacentPairs{|start,end,num|
|
||||||
|
var analysis = Routine({|counter|
|
||||||
|
FluidBufMelBands.processBlocking(
|
||||||
|
server:s,
|
||||||
|
source:buffer,
|
||||||
|
startFrame:start,
|
||||||
|
numFrames:(end-1) - start,
|
||||||
|
features:~featurebuffers[counter],
|
||||||
|
action:{
|
||||||
|
remaining = remaining - 1;
|
||||||
|
if(remaining == 0) { ~numMelBands = ~featurebuffers[0].numChannels;condition.unhang };
|
||||||
|
}
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
~slices["data"].put(counter,[buffer.bufnum,start,end]);
|
||||||
|
|
||||||
|
//I'm spawning new threads to wait for the analysis callback from the server. The final callback will un-hang this thread
|
||||||
|
analysis.value(counter); //Done differently to other blocks because I need to pass in the value of counter
|
||||||
|
counter = counter + 1;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
condition.hang;
|
||||||
|
"Analysis of % slices done.\n".postf(~total_slice_count);
|
||||||
|
}.fork(clock:~clock);
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
// Run stats on each mel buffer
|
||||||
|
|
||||||
|
// create a stats buffer for each of the slices
|
||||||
|
~statsbuffers = ~total_slice_count.collect{Buffer.new}; // create n Slices buffers - to be filled with (40 mel bands * 7 stats)
|
||||||
|
|
||||||
|
// run stats on all the buffers
|
||||||
|
(
|
||||||
|
{
|
||||||
|
var remaining = ~total_slice_count;
|
||||||
|
~featurebuffers.do{|buffer,i|
|
||||||
|
FluidBufStats.processBlocking(
|
||||||
|
server:s,
|
||||||
|
source:buffer,
|
||||||
|
stats:~statsbuffers[i],
|
||||||
|
action:{
|
||||||
|
remaining = remaining - 1;
|
||||||
|
if(remaining == 0) { "done".postln};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
}.fork(clock:~clock);
|
||||||
|
)
|
||||||
|
|
||||||
|
~featurebuffers.size
|
||||||
|
|
||||||
|
//Flatten each stats buffer into a data point
|
||||||
|
~flatbuffers = ~total_slice_count.collect{Buffer.new};// create an array of flatten stats
|
||||||
|
|
||||||
|
(
|
||||||
|
{
|
||||||
|
var remaining = ~total_slice_count;
|
||||||
|
~statsbuffers.do{|buffer,i|
|
||||||
|
FluidBufFlatten.processBlocking(
|
||||||
|
server:s,
|
||||||
|
source:buffer,
|
||||||
|
destination:~flatbuffers[i],
|
||||||
|
action:{
|
||||||
|
remaining = remaining - 1;
|
||||||
|
if(remaining == 0) { "Got flat points".postln; };
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
}.fork(clock:~clock);
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
//Ram each flat point into a data set. At this point we have more data than we need, but we'll prune in moment
|
||||||
|
(
|
||||||
|
"Filling dataset".postln;
|
||||||
|
~mels.clear;
|
||||||
|
|
||||||
|
// ~flatbuffers = flatbuffers;
|
||||||
|
~flatbuffers.do{|buf,i|
|
||||||
|
~mels.addPoint(i,buf);
|
||||||
|
};
|
||||||
|
|
||||||
|
~mels.print;
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
// Prune & standardise
|
||||||
|
|
||||||
|
// Tidy up the temp arrays of buffers we do not need anymore
|
||||||
|
|
||||||
|
(
|
||||||
|
"Cleaning".postln;
|
||||||
|
(~featurebuffers ++ ~statsbuffers ++ ~flatbuffers).do{|buf| buf.free};
|
||||||
|
)
|
||||||
|
|
||||||
|
//Above we sneakily made a dictionary of slice data for playback (bufnum,start,end). Let's throw it in a dataset
|
||||||
|
~slicedata = FluidDataSet(s); // will hold slice data (bufnum,start,end) for playback
|
||||||
|
|
||||||
|
//dict -> dataset
|
||||||
|
(
|
||||||
|
~slicedata.load(~slices);
|
||||||
|
~slicedata.print;
|
||||||
|
)
|
||||||
|
|
||||||
|
// Step 1. Let's prune and standardize before fitting to a tree
|
||||||
|
(
|
||||||
|
~meanmels = FluidDataSet(s);//will hold pruned mel data
|
||||||
|
~stdmels = FluidDataSet(s);//will standardised, pruned mel data
|
||||||
|
~standardizer = FluidStandardize(s);
|
||||||
|
~pruner = FluidDataSetQuery(s);
|
||||||
|
~tree = FluidKDTree(s,numNeighbours:10,lookupDataSet:~slicedata);//we have to supply the lookup data set when we make the tree (boo!)
|
||||||
|
)
|
||||||
|
|
||||||
|
//Prune, standardize and fit KDTree
|
||||||
|
(
|
||||||
|
{
|
||||||
|
~meanmels.clear;
|
||||||
|
~stdmels.clear;
|
||||||
|
~pruner.addRange(0,~numMelBands).transform(~mels,~meanmels); //prune with a 'query' -- so this is dropping all but ~meanmels
|
||||||
|
~standardizer.fitTransform(~meanmels,~stdmels);
|
||||||
|
~tree.fit(~stdmels,{"KDTree ready".postln});
|
||||||
|
}.fork(clock:~clock);
|
||||||
|
)
|
||||||
|
|
||||||
|
~meanmels.print
|
||||||
|
|
||||||
|
//Step 2: Set the FluidStandardizer and FluidKDTree up for listening
|
||||||
|
//set the buffers and busses needed
|
||||||
|
(
|
||||||
|
~stdInputPoint = Buffer.alloc(s,40);
|
||||||
|
~stdOutputPoint = Buffer.alloc(s,40);
|
||||||
|
~treeOutputPoint = Buffer.alloc(s,3 * 10);//numNeighbours x triples of bufnum,start,end
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
// let's play a random sound (to make sure we understand our data structure!
|
||||||
|
(
|
||||||
|
{
|
||||||
|
var randPoint, buf, start, stop, dur;
|
||||||
|
|
||||||
|
randPoint = ~slices["data"].keys.asArray.scramble[0]; // this good way of getting - but recast as strong
|
||||||
|
|
||||||
|
buf= ~slices["data"][randPoint][0];
|
||||||
|
start = ~slices["data"][randPoint][1];
|
||||||
|
stop = ~slices["data"][randPoint][2];
|
||||||
|
|
||||||
|
dur = stop - start;
|
||||||
|
|
||||||
|
BufRd.ar(1,buf, Line.ar(start,stop,dur/s.sampleRate, doneAction: 2), 0, 2);
|
||||||
|
}.play
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
// Query KD tree
|
||||||
|
|
||||||
|
// a target sound from outside our dataset
|
||||||
|
~inBuf = Buffer.readChannel(s, Platform.resourceDir +/+ "sounds/a11wlk01.wav", numFrames:15000, channels:[0]);
|
||||||
|
~inBuf.play
|
||||||
|
|
||||||
|
//OR one from within (but just the begining so beware of the difference!)
|
||||||
|
~inBuf = Buffer.alloc(s,15000);
|
||||||
|
~randomSlice = ~slices["data"].keys.asArray.scramble[0];
|
||||||
|
~audio_buffers[~slices["data"][~randomSlice][0]].copyData(~inBuf,srcStartAt: ~slices["data"][~randomSlice][1], numSamples: 15000.min(~slices["data"][~randomSlice][2] - (~slices["data"][~randomSlice][1])));
|
||||||
|
~inBuf.play
|
||||||
|
|
||||||
|
// now try getting a point, playing it, grabbing nearest neighbour and playing it ...
|
||||||
|
|
||||||
|
(
|
||||||
|
~inBufMels = Buffer(s);
|
||||||
|
~inBufStats = Buffer(s);
|
||||||
|
~inBufFlat = Buffer(s);
|
||||||
|
~inBufComp = Buffer(s);
|
||||||
|
~inBufStand = Buffer(s);
|
||||||
|
)
|
||||||
|
|
||||||
|
// FluidBuf Compose is buf version of dataSetQuery
|
||||||
|
|
||||||
|
(
|
||||||
|
FluidBufMelBands.process(s, ~inBuf, features: ~inBufMels, action: {
|
||||||
|
FluidBufStats.process(s, ~inBufMels, stats:~inBufStats, action: {
|
||||||
|
FluidBufFlatten.process(s, ~inBufStats, destination:~inBufFlat, action: {
|
||||||
|
FluidBufCompose.process(s, ~inBufFlat, numFrames: ~numMelBands, destination: ~inBufComp, action: {
|
||||||
|
~standardizer.transformPoint(~inBufComp, ~inBufStand, {
|
||||||
|
~tree.kNearest(~inBufStand,{ |a|a.postln;~nearest = a;})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
// playback nearest in order
|
||||||
|
(
|
||||||
|
fork{
|
||||||
|
~nearest.do{|i|
|
||||||
|
var buf, start, stop, dur;
|
||||||
|
|
||||||
|
buf= ~slices["data"][i.asInteger][0];
|
||||||
|
start = ~slices["data"][i.asInteger][1];
|
||||||
|
stop = ~slices["data"][i.asInteger][2];
|
||||||
|
dur = (stop - start)/ s.sampleRate;
|
||||||
|
{BufRd.ar(1,buf, Line.ar(start,stop,dur, doneAction: 2), 0, 2);}.play;
|
||||||
|
|
||||||
|
i.postln;
|
||||||
|
dur.wait;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
)
|
||||||
@ -0,0 +1,90 @@
|
|||||||
|
//1- make the gui then the synth below
|
||||||
|
(
|
||||||
|
var trained = 0, entering = 0;
|
||||||
|
var va = Array.fill(10,{0.5});
|
||||||
|
var input = Buffer.alloc(s,2);
|
||||||
|
var output = Buffer.alloc(s,10);
|
||||||
|
var mlp = FluidMLPRegressor(s,[6],activation: 1,outputActivation: 1,maxIter: 1000,learnRate: 0.1,momentum: 0,batchSize: 1,validation: 0);
|
||||||
|
var entry = 0;
|
||||||
|
|
||||||
|
~inData = FluidDataSet(s);
|
||||||
|
~outData = FluidDataSet(s);
|
||||||
|
|
||||||
|
w = Window("ChaosSynth", Rect(10, 10, 790, 320)).front;
|
||||||
|
a = MultiSliderView(w,Rect(10, 10, 400, 300)).elasticMode_(1).isFilled_(1);
|
||||||
|
a.value=va;
|
||||||
|
a.action = {arg q;
|
||||||
|
b.set(\val, q.value);
|
||||||
|
va = q.value;};
|
||||||
|
f = Slider2D(w,Rect(420,10,300, 300));
|
||||||
|
f.x = 0.5;
|
||||||
|
f.y = 0.5;
|
||||||
|
f.action = {arg x,y; //if trained, predict the point f.x f.y
|
||||||
|
if (entering == 1, { //if entering a point, add to the the database f.x f.y against the array va
|
||||||
|
input.setn(0, [f.x, f.y]);
|
||||||
|
output.setn(0, va);
|
||||||
|
~inData.addPoint(entry.asSymbol,input);
|
||||||
|
~outData.addPoint(entry.asSymbol,output);
|
||||||
|
entering = 0;
|
||||||
|
entry = entry + 1;
|
||||||
|
{d.value = 0;}.defer;
|
||||||
|
}, { //if not entering a point
|
||||||
|
if (trained == 1, { //if trained
|
||||||
|
input.setn(0, [f.x, f.y]);
|
||||||
|
mlp.predictPoint(input,output,{
|
||||||
|
output.getn(0,10,{
|
||||||
|
|x|va = x; b.set(\val, va); {a.value = va;}.defer;});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
c = Button(w, Rect(730,240,50, 20)).states_([["train", Color.red, Color.white], ["trained", Color.white, Color.grey]]).action_{
|
||||||
|
mlp.fit(~inData,~outData,{|x|
|
||||||
|
trained = 1;
|
||||||
|
{
|
||||||
|
c.value = 1;
|
||||||
|
e.value = x.round(0.001).asString;
|
||||||
|
}.defer;
|
||||||
|
});//train the network
|
||||||
|
};
|
||||||
|
d = Button(w, Rect(730,10,50, 20)).states_([["entry", Color.white, Color.grey], ["entry", Color.red, Color.white]]).action_{
|
||||||
|
entering = 1;
|
||||||
|
};
|
||||||
|
StaticText(w,Rect(732,260,50,20)).string_("Error:");
|
||||||
|
e = TextField(w,Rect(730,280,50,20)).string_(0.asString);
|
||||||
|
StaticText(w,Rect(732,70,50,20)).string_("rate:");
|
||||||
|
TextField(w,Rect(730,90,50,20)).string_(0.1.asString).action_{|in|mlp.learnRate = in.value.asFloat.postln;};
|
||||||
|
StaticText(w,Rect(732,110,50,20)).string_("momentum:");
|
||||||
|
TextField(w,Rect(730,130,50,20)).string_(0.0.asString).action_{|in|mlp.momentum = in.value.asFloat.postln;};
|
||||||
|
StaticText(w,Rect(732,150,50,20)).string_("maxIter:");
|
||||||
|
TextField(w,Rect(730,170,50,20)).string_(1000.asString).action_{|in| mlp.maxIter = in.value.asInteger.postln;};
|
||||||
|
StaticText(w,Rect(732,190,50,20)).string_("validation:");
|
||||||
|
TextField(w,Rect(730,210,50,20)).string_(0.0.asString).action_{|in|mlp.validation = in.value.asFloat.postln;};
|
||||||
|
)
|
||||||
|
|
||||||
|
//2- the synth
|
||||||
|
(
|
||||||
|
b = {
|
||||||
|
arg val = #[0,0,0,0,0,0,0,0,0,0];
|
||||||
|
var osc1, osc2, feed1, feed2, base1=69, base2=69, base3 = 130;
|
||||||
|
#feed2,feed1 = LocalIn.ar(2);
|
||||||
|
osc1 = MoogFF.ar(SinOsc.ar((((feed1 * val[0]) + val[1]) * base1).midicps,mul: (val[2] * 50).dbamp).atan,(base3 - (val[3] * (FluidLoudness.kr(feed2, 1, 0, hopSize: 64)[0].clip(-120,0) + 120))).lag(128/44100).midicps, val[4] * 3.5);
|
||||||
|
osc2 = MoogFF.ar(SinOsc.ar((((feed2 * val[5]) + val[6]) * base2).midicps,mul: (val[7] * 50).dbamp).atan,(base3 - (val[8] * (FluidLoudness.kr(feed1, 1, 0, hopSize: 64)[0].clip(-120,0) + 120))).lag(128/44100).midicps, val[9] * 3.5);
|
||||||
|
Out.ar(0,LeakDC.ar([osc1,osc2],mul: 0.1));
|
||||||
|
LocalOut.ar([osc1,osc2]);
|
||||||
|
}.play;
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
~inData.print;
|
||||||
|
~outData.print;
|
||||||
|
|
||||||
|
/////////
|
||||||
|
//3 - play with the multislider
|
||||||
|
//4 - when you like a spot, click entry (become read) then a position in the 2D graph where this point should be
|
||||||
|
//5 - do that for a few points
|
||||||
|
//6 - click train
|
||||||
|
//7 - the 2D graph controls the 10D
|
||||||
|
//8 - if you like a new sound and you want to update the graph, just click entry, then where it should be in the 2D, then retrain when you are happy
|
||||||
@ -0,0 +1,161 @@
|
|||||||
|
s.reboot;
|
||||||
|
//Preliminaries: we want some audio, a couple of FluidDataSets, some Buffers
|
||||||
|
(
|
||||||
|
~raw = FluidDataSet(s);
|
||||||
|
~norm = FluidDataSet(s);
|
||||||
|
~retrieved = FluidDataSet(s);
|
||||||
|
~audio = Buffer.read(s,File.realpath(FluidBufMelBands.class.filenameSymbol).dirname +/+ "../AudioFiles/Tremblay-ASWINE-ScratchySynth-M.wav");
|
||||||
|
~melfeatures = Buffer.new(s);
|
||||||
|
~stats = Buffer.alloc(s, 7, 40);
|
||||||
|
~datapoint = Buffer.alloc(s, 40);
|
||||||
|
~queryPoint = Buffer.alloc(s, 2);
|
||||||
|
~dQueryPoint = Buffer.alloc(s, 2);
|
||||||
|
~dpN =Buffer.alloc(s, 40);
|
||||||
|
~dpMLPn =Buffer.alloc(s, 40);
|
||||||
|
~dpMLP =Buffer.alloc(s, 40);
|
||||||
|
)
|
||||||
|
|
||||||
|
// process the melbands
|
||||||
|
FluidBufMelBands.process(s,~audio, features: ~melfeatures,action: {\done.postln;});
|
||||||
|
|
||||||
|
// Divide the time series in 100, and take the mean of each segment and add this as a point to the 'raw' FluidDataSet
|
||||||
|
(
|
||||||
|
{
|
||||||
|
var trig = LocalIn.kr(1, 1);
|
||||||
|
var buf = LocalBuf(40, 1);
|
||||||
|
var count = PulseCount.kr(trig) - 1;
|
||||||
|
var chunkLen = (~melfeatures.numFrames / 100).asInteger;
|
||||||
|
var stats = FluidBufStats.kr(source: ~melfeatures, startFrame: count * chunkLen, numFrames: chunkLen, stats: ~stats, trig: trig, blocking: 1);
|
||||||
|
var rd = BufRd.kr(40, ~stats, DC.kr(0), 0, 1);
|
||||||
|
var bufWr, dsWr;
|
||||||
|
40.do{|i|
|
||||||
|
bufWr = BufWr.kr(rd[i], buf, DC.kr(i));
|
||||||
|
};
|
||||||
|
dsWr = FluidDataSetWr.kr(~raw, buf: buf, idNumber: count, trig: Done.kr(stats));
|
||||||
|
LocalOut.kr( Done.kr(dsWr));
|
||||||
|
FreeSelf.kr(count - 99);
|
||||||
|
Poll.kr(trig,(100-count));
|
||||||
|
}.play;
|
||||||
|
)
|
||||||
|
// wait for the count to reaches 0 in the post window. Check the dataset if curious (loads of small numbers)
|
||||||
|
~raw.print;
|
||||||
|
|
||||||
|
// normalize the input
|
||||||
|
~normalizer = FluidNormalize(s);
|
||||||
|
~normalizer.fitTransform(~raw,~norm);
|
||||||
|
~norm.print; //a more decent range
|
||||||
|
|
||||||
|
//we can then run the AE - the server might become yellow :)
|
||||||
|
~mlp = FluidMLPRegressor(s,[9,2,9],activation: 1,outputActivation: 1,tapIn: 0,tapOut: 2,maxIter: 10000,learnRate: 0.1,momentum: 0.1,batchSize: 10,validation: 0.1);
|
||||||
|
~mlp.fit(~norm,~norm,{|x|x.postln;});// run this a few times, until you are happy with the error
|
||||||
|
|
||||||
|
//we can then retrieve the hidden layer #2 because of the tapOut parameter
|
||||||
|
~mlp.predict(~norm,~retrieved);
|
||||||
|
|
||||||
|
//check the structure of retrieved
|
||||||
|
~retrieved.print;
|
||||||
|
|
||||||
|
//let's normalise it for display
|
||||||
|
~normData = FluidDataSet(s);
|
||||||
|
~reducedarray = Array.new(100);
|
||||||
|
~normalView = FluidNormalize(s,0.1,0.9);
|
||||||
|
(
|
||||||
|
~normalView.fitTransform(~retrieved,~normData, action:{
|
||||||
|
~normData.dump{|x| 100.do{|i|
|
||||||
|
~reducedarray.add(x["data"][i.asString])
|
||||||
|
}};
|
||||||
|
});
|
||||||
|
)
|
||||||
|
~normData.print;
|
||||||
|
|
||||||
|
//make a basic KD tree to retrieve the nearest entry in 2D
|
||||||
|
~kdtree = FluidKDTree(s,numNeighbours: 1);
|
||||||
|
~kdtree.fit(~normData);
|
||||||
|
|
||||||
|
//prepare the normalizers and the neural net for inverse query
|
||||||
|
(
|
||||||
|
~normalView.invert = 1;
|
||||||
|
~normalizer.invert = 1;
|
||||||
|
~mlp.tapIn = 2;
|
||||||
|
~mlp.tapOut = -1;
|
||||||
|
)
|
||||||
|
|
||||||
|
//Visualise and query the 2D projection of our original 40D data
|
||||||
|
(
|
||||||
|
var w,v,myx,myy,vRNN, vRN, vMN, vM;
|
||||||
|
|
||||||
|
~arrayRawNn = Array.new(40);
|
||||||
|
~arrayRawN = Array.new(40);
|
||||||
|
~arrayMLPn = Array.new(40);
|
||||||
|
~arrayMLP = Array.new(40);
|
||||||
|
|
||||||
|
//initialise the mouse position holder
|
||||||
|
myx=130;
|
||||||
|
myy=130;
|
||||||
|
|
||||||
|
w = Window("AutoEncoder", Rect(64, 64, 770, 270));
|
||||||
|
v = View.new(w,Rect(0,0, 310, 310));
|
||||||
|
|
||||||
|
vRNN = MultiSliderView(w,Rect(270,8,240,115)).value_(~arrayRawNn).readOnly_(true).elasticMode_(1).isFilled_(true);
|
||||||
|
vRN = MultiSliderView(w,Rect(270,147,240,115)).value_(~arrayRawN).readOnly_(true).elasticMode_(1).isFilled_(true);
|
||||||
|
vMN = MultiSliderView(w,Rect(520,10,240,115)).value_(~arrayMLPn).readOnly_(true).elasticMode_(1).isFilled_(true);
|
||||||
|
vM = MultiSliderView(w,Rect(520,147,240,115)).value_(~arrayMLP).readOnly_(true).elasticMode_(1).isFilled_(true);
|
||||||
|
|
||||||
|
StaticText(w,Rect(275,120,490,30)).string_("above: normalised nearest neighbour\nbelow: original nearest neighbour").font_(Font("Monaco", 10));
|
||||||
|
StaticText(w,Rect(525,120,490,30)).string_("above: regressed values at coordinates\nbelow: denormalised regressed values").font_(Font("Monaco", 10));
|
||||||
|
|
||||||
|
//creates a function that reacts to mousedown
|
||||||
|
v.mouseMoveAction = {|view, x, y|
|
||||||
|
myx=x.clip(10,260);
|
||||||
|
myy=y.clip(10,260);
|
||||||
|
w.refresh;
|
||||||
|
Routine{
|
||||||
|
~queryPoint.setn(0,([myx,myy] - 10 / 250));//set the query point to the coordinate
|
||||||
|
~kdtree.kNearest(~queryPoint, action: {|nearest| //retrieve the nearest point
|
||||||
|
~norm.getPoint(nearest, ~dpN, action: { //get the normalised 40d
|
||||||
|
~raw.getPoint(nearest, ~datapoint, action: { // get the original 40d
|
||||||
|
~normalView.transformPoint(~queryPoint, ~dQueryPoint, action: { //denormalise the 2d coordinate to get the right range of values for the MLP
|
||||||
|
~mlp.predictPoint(~dQueryPoint, ~dpMLPn, action: { //predict from the middle (2d) to the normalised output (40d)
|
||||||
|
~normalizer.transformPoint(~dpMLPn, ~dpMLP, action: { //denormalised the 40d
|
||||||
|
~datapoint.getn(0,40,{|x|~arrayRawN = x; //retrieve the nearest
|
||||||
|
~dpN.getn(0,40,{|x|~arrayRawNn = x; // retrieve the normalised nearest
|
||||||
|
~dpMLPn.getn(0,40,{|x|~arrayMLPn = x; //retrieve the predicted normalised 40d
|
||||||
|
~dpMLP.getn(0,40,{|x|~arrayMLP = x; //retrieve the denormalised predicted 40d
|
||||||
|
AppClock.sched(0,{ // update the visualisation of the 4 arrays
|
||||||
|
vRNN.value=~arrayRawNn;
|
||||||
|
vRN.value=~arrayRawN * 15;
|
||||||
|
vMN.value=~arrayMLPn;
|
||||||
|
vM.value=~arrayMLP * 15;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}.play;
|
||||||
|
};
|
||||||
|
//custom redraw function
|
||||||
|
w.drawFunc = {
|
||||||
|
Pen.use {
|
||||||
|
~reducedarray.size.do{|i|
|
||||||
|
var coord = (~reducedarray[i] * 250) + 7;
|
||||||
|
var r = Rect(coord[0],coord[1],6,6);
|
||||||
|
Pen.fillColor = Color.blue;
|
||||||
|
Pen.fillOval(r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
Pen.color = Color.red;
|
||||||
|
Pen.addOval(Rect(myx-4, myy-4,8,8));
|
||||||
|
Pen.perform(\stroke);
|
||||||
|
Pen.color = Color.black;
|
||||||
|
Pen.addRect(Rect(10,10,250,250));
|
||||||
|
Pen.perform(\stroke);
|
||||||
|
};
|
||||||
|
w.refresh;
|
||||||
|
w.front;
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue