diff --git a/release-packaging/Examples/dataset/1-learning examples/11-compositing-datasets.scd b/release-packaging/Examples/dataset/1-learning examples/11-compositing-datasets.scd index 643e118..537df87 100644 --- a/release-packaging/Examples/dataset/1-learning examples/11-compositing-datasets.scd +++ b/release-packaging/Examples/dataset/1-learning examples/11-compositing-datasets.scd @@ -1,11 +1,11 @@ -// define a few processes +// define a few datasets ( -~globalDS = FluidDataSet(s,\global11); ~pitchDS = FluidDataSet(s,\pitch11); ~loudDS = FluidDataSet(s,\loud11); ~mfccDS = FluidDataSet(s,\mfcc11); -~timbreDS = FluidDataSet(s,\timbre11); +~durDS = FluidDataSet(s,\dur11); + //define as many buffers as we have parallel voices/threads in the extractor processing (default is 4) ~pitchbuf = 4.collect{Buffer.new}; @@ -22,8 +22,8 @@ // here we instantiate a loader as per example 0 // ~loader = FluidLoadFolder(File.realpath(FluidBufPitch.class.filenameSymbol).dirname.withTrailingSlash ++ "../AudioFiles/"); -~loader = FluidLoadFolder("/Volumes/machins/projets/newsfeed/sons/smallnum/"); -// ~loader = FluidLoadFolder("/Volumes/machins/projets/newsfeed/sons/segments/"); +// ~loader = FluidLoadFolder("/Volumes/machins/projets/newsfeed/sons/smallnum/"); +~loader = FluidLoadFolder("/Volumes/machins/projets/newsfeed/sons/segments/"); // here we instantiate a further slicing step as per example 0 ~slicer = FluidSliceCorpus({ |src,start,num,dest| @@ -38,8 +38,8 @@ voice = data.value[\voice]; // the pitch computation is independant so it starts right away pitch = FluidBufPitch.kr(src, startFrame:start, numFrames:num, numChans:1, features:~pitchbuf[voice], unit: 1, trig:1, blocking: 1); - // pitchweights = FluidBufThresh.kr(~pitchbuf[voice], numChans: 1, startChan: 1, destination: ~weightPitchbuf[voice], threshold: 0.1, trig:Done.kr(pitch), blocking: 1);//pull down low conf - pitchweights = FluidBufScale.kr(~pitchbuf[voice], numChans: 1, startChan: 1, destination: ~weightPitchbuf[voice],inputLow: -1, trig:Done.kr(pitch), blocking: 1); + pitchweights = FluidBufThresh.kr(~pitchbuf[voice], numChans: 1, startChan: 1, destination: ~weightPitchbuf[voice], threshold: 0.1, trig:Done.kr(pitch), blocking: 1);//pull down low conf + // pitchweights = FluidBufScale.kr(~pitchbuf[voice], numChans: 1, startChan: 1, destination: ~weightPitchbuf[voice],inputLow: -1, trig:Done.kr(pitch), blocking: 1); pitchstats = FluidBufStats.kr(~pitchbuf[voice], stats:~statsPitchbuf[voice], numDerivs: 1, weights: ~weightPitchbuf[voice], outliersCutoff: 1.5, trig:Done.kr(pitchweights), blocking: 1); pitchflat = FluidBufFlatten.kr(~statsPitchbuf[voice],~flatPitchbuf[voice],trig:Done.kr(pitchstats),blocking: 1); writePitch = FluidDataSetWr.kr(~pitchDS,label, -1, ~flatPitchbuf[voice], Done.kr(pitchflat),blocking: 1); @@ -81,29 +81,101 @@ t = Main.elapsedTime; ////////////////////////////////////////////////////////////////////////// // description process -// run the descriptor extractor +// run the descriptor extractor (errors will be given, this is normal: the pitch conditions are quite exacting and therefore many slices are not valid) ( t = Main.elapsedTime; ~extractor.play(s,~loader.buffer,~slicer.index,action:{(Main.elapsedTime - t).postln;"Features done".postln}); ) +// make a dataset of durations for querying that too (it could have been made in the process loop, but hey, we have dictionaries we can manipulate too!) +( +~dict = Dictionary.new; +~temp = ~slicer.index.collect{ |k| [k[\bounds][1] - k[\bounds][0]]}; +~dict.add(\data -> ~temp); +~dict.add(\cols -> 1); +~durDS.load(~dict) +) + ////////////////////////////////////////////////////////////////////////// // manipulating and querying the data ~pitchDS.print; ~loudDS.print; ~mfccDS.print; +~durDS.print; + +//reduce the MFCC timbral space stats (4 potential ways to explore here...) +~tempDS = FluidDataSet(s,\temp11); + +~query = FluidDataSetQuery(s); +~query.addRange(0,24);//add only means and stddev of the 12 coeffs... +~query.addRange((7*12),24);// and the same stats of the first derivative (moving 7 stats x 12 mfccs to the right) +~query.transform(~mfccDS, ~tempDS); + +//check +~tempDS.print; + +//shrinking A: PCA then standardize +~pca = FluidPCA(s,4);//shrink to 4 dimensions + +~timbreDSp = FluidDataSet(s,\timbreDSp11); +~pca.fitTransform(~tempDS,~timbreDSp,{|x|x.postln;})//accuracy + +// shrinking B: standardize then PCA +// https://scikit-learn.org/stable/auto_examples/preprocessing/plot_scaling_importance.html +~stan = FluidStandardize(s); +~stanDS = FluidDataSet(s,\stan11); +~stan.fitTransform(~tempDS,~stanDS) +~timbreDSsp = FluidDataSet(s,\timbreDSsp11); +~pca.fitTransform(~stanDS,~timbreDSsp,{|x|x.postln;})//accuracy + +// comparing NN for fun +~targetDSp = Buffer(s) +~targetDSsp = Buffer(s) +~tree = FluidKDTree(s,5) -~loudbuf[0].query -~statsLoudbuf[0].query -~flatLoudbuf[0].query +// you can run this a few times to have fun +( +~target = ~slicer.index.keys.asArray.scramble.[0].asSymbol; +~timbreDSp.getPoint(~target, ~targetDSp); +~timbreDSsp.getPoint(~target, ~targetDSsp); +) + +~tree.fit(~timbreDSp,{~tree.kNearest(~targetDSp,{|x|~nearestDSp = x.postln;})}) +~tree.fit(~timbreDSsp,{~tree.kNearest(~targetDSsp,{|x|~nearestDSsp = x.postln;})}) + +// play them in a row +( +Routine{ +5.do{|i| + var dur; + v = ~slicer.index[~nearestDSp[i].asSymbol]; + dur = (v[\bounds][1] - v[\bounds][0]) / s.sampleRate; + {BufRd.ar(v[\numchans],~loader.buffer,Line.ar(v[\bounds][0],v[\bounds][1],dur, doneAction: 2))}.play; + ~nearestDSp[i].postln; + dur.wait; + }; +}.play; +) + +( +Routine{ +5.do{|i| + var dur; + v = ~slicer.index[~nearestDSsp[i].asSymbol]; + dur = (v[\bounds][1] - v[\bounds][0]) / s.sampleRate; + {BufRd.ar(v[\numchans],~loader.buffer,Line.ar(v[\bounds][0],v[\bounds][1],dur, doneAction: 2))}.play; + ~nearestDSsp[i].postln; + dur.wait; + }; +}.play; +) + + +// for compositing +~globalDS = FluidDataSet(s,\global11); -~pitchbuf[0].query -~statsPitchbuf[0].query -~weightPitchbuf[0].query -~flatPitchbuf[0].query -//standardise //AE ou PCA //normalize