From 4b88b405d782d30e939b9a99fa4b69264ad755fe Mon Sep 17 00:00:00 2001 From: Owen Green Date: Wed, 6 May 2020 15:13:04 +0100 Subject: [PATCH] Enable persistence for Dataset, Labelset and 'model' objects viz. KDTree, KMeans,KNN, Normalize, PCA and Standardize. Most changes in FluidManipulationClient --- release-packaging/Classes/FluidDataSet.sc | 112 ++++++---- release-packaging/Classes/FluidKDTree.sc | 51 +++-- release-packaging/Classes/FluidKMeans.sc | 25 ++- release-packaging/Classes/FluidKNN.sc | 26 ++- release-packaging/Classes/FluidLabelSet.sc | 106 ++++++---- .../Classes/FluidManipulationClient.sc | 191 +++++++++++++++--- release-packaging/Classes/FluidNormalize.sc | 25 +-- release-packaging/Classes/FluidPCA.sc | 28 ++- .../Classes/FluidProviderTest.sc | 8 +- release-packaging/Classes/FluidStandardize.sc | 23 ++- .../Classes/FluidSubscriberTest.sc | 4 +- test/TestFluidManipulationLifecyle.sc | 105 ++++++++++ 12 files changed, 524 insertions(+), 180 deletions(-) create mode 100644 test/TestFluidManipulationLifecyle.sc diff --git a/release-packaging/Classes/FluidDataSet.sc b/release-packaging/Classes/FluidDataSet.sc index d519509..6d270cd 100644 --- a/release-packaging/Classes/FluidDataSet.sc +++ b/release-packaging/Classes/FluidDataSet.sc @@ -1,64 +1,88 @@ + +FluidDataSetExistsError : Exception{ +} + FluidDataSet : FluidManipulationClient { - var <>synth, <>server, <>id; + var k; - + + *new {|server| + var uid = UniqueID.next; + ^super.new(server,uid).init(uid); + } + + init {|uid| + id = uid; + } + fit{|dataset,k, maxIter = 100, buffer, action| buffer = buffer ? -1; this.k = k; - this.pr_sendMsg(\fit,[dataset.asString, k,maxIter, buffer.asUGenInput],action,[numbers(FluidMessageResponse,_,k,_)]); + this.prSendMsg(\fit,[dataset.asString, k,maxIter, buffer.asUGenInput],action,[numbers(FluidMessageResponse,_,k,_)]); } predict{ |dataset, labelset,action| - this.pr_sendMsg(\predict,[dataset.asString, labelset.asString],action,[numbers(FluidMessageResponse,_,this.k,_)]); + this.prSendMsg(\predict,[dataset.asString, labelset.asString],action,[numbers(FluidMessageResponse,_,this.k,_)]); } getClusters{ |dataset, labelset,action| - this.pr_sendMsg(\getClusters,[dataset.asString, labelset.asString],action); + this.prSendMsg(\getClusters,[dataset.asString, labelset.asString],action); } predictPoint { |buffer, action| - this.pr_sendMsg(\predictPoint,[buffer.asUGenInput],action,[number(FluidMessageResponse,_,_)]); + this.prSendMsg(\predictPoint,[buffer.asUGenInput],action,[number(FluidMessageResponse,_,_)]); } cols { |action| - this.pr_sendMsg(\cols,[],action,[number(FluidMessageResponse,_,_)]); + this.prSendMsg(\cols,[],action,[number(FluidMessageResponse,_,_)]); } read{ |filename,action| - this.pr_sendMsg(\read,[filename.asString],action); + this.prSendMsg(\read,[filename.asString],action); } write{ |filename,action| - this.pr_sendMsg(\write,[filename.asString],action); + this.prSendMsg(\write,[filename.asString],action); } } diff --git a/release-packaging/Classes/FluidKNN.sc b/release-packaging/Classes/FluidKNN.sc index 92591bc..f1e8260 100644 --- a/release-packaging/Classes/FluidKNN.sc +++ b/release-packaging/Classes/FluidKNN.sc @@ -1,15 +1,23 @@ FluidKNN : FluidManipulationClient { - fit{|dataset, action| - this.pr_sendMsg(\fit,[dataset.asString],action); - } + *new {|server| + var uid = UniqueID.next; + ^super.new(server,uid).init(uid); + } - classifyPoint{ |buffer, labelset, k, action| - this.pr_sendMsg(\classify,[buffer.asUGenInput, labelset.asString, k],action,[string(FluidMessageResponse,_,_)]); - } + init {|uid| + id = uid; + } - regressPoint { |buffer,dataset, k, action| - this.pr_sendMsg(\regress,[buffer.asUGenInput, dataset.asString,k],action,[number(FluidMessageResponse,_,_)]); - } + fit{|dataset, action| + this.prSendMsg(\fit,[dataset.asString],action); + } + classifyPoint{ |buffer, labelset, k, action| + this.prSendMsg(\classify,[buffer.asUGenInput, labelset.asString, k],action,[string(FluidMessageResponse,_,_)]); + } + + regressPoint { |buffer,dataset, k, action| + this.prSendMsg(\regress,[buffer.asUGenInput, dataset.asString,k],action,[number(FluidMessageResponse,_,_)]); + } } diff --git a/release-packaging/Classes/FluidLabelSet.sc b/release-packaging/Classes/FluidLabelSet.sc index 147c3a5..edc0220 100644 --- a/release-packaging/Classes/FluidLabelSet.sc +++ b/release-packaging/Classes/FluidLabelSet.sc @@ -1,55 +1,83 @@ +FluidLabelSetExistsError : Exception{ +} + FluidLabelSet : FluidManipulationClient { - var <> synth, <> server, <>id; + var synth, <> server; + var <>pluginname; - *kr { - ^this.multiNew('control', Done.none, nonBlocking); - } + *kr { |pluginname...args| + ^this.new1('control', pluginname,*args) + } + + init { |pluginname...args| + this.pluginname = pluginname; + inputs = args++Done.none++0; + } + + name{ + ^pluginname.asString; + } +} + +FluidManipulationClient { + + var ugen, updateFunc; + var nodeResponder; + + var initTreeCondition; + var synthBeenSet = false; + var serverListener; + + *prServerString{ |s| + var ascii = s.ascii; + ^[ascii.size].addAll(ascii) + } + + sendSynthDef { |...args| + var plugin = this.class.name.asSymbol; + if(server.hasBooted) + { + fork{ + SynthDef(defName.asSymbol,{ + var ugen = FluidProxyUgen.kr(plugin, *args); + this.ugen = ugen; + ugen + }).send(server); + + server.sync; + + synthDefLoaded = true; + updateFunc = { + //Sometimes Server.initTree seems to get called a bunch of + //times during boot: we can't be having extra instances + //However, once boot has finished, ending up here means cmd-. or server.freeAll + //has happened, and we just need to run + + var shouldRun = (synthBeenSet.not.and(server.serverBooting)) + .or(server.serverRunning.and(server.serverBooting.not)); + + if(shouldRun) { + synthBeenSet = true; + synth = nil; + this.updateSynth; + } + }; + updateFunc.value; + ServerTree.add(updateFunc, server); + }; + }; + } + + updateSynth { + if(server.hasBooted){ + if(synthDefLoaded){ + if(synth.isNil){ + synth = Synth.after(server.defaultGroup,defName.asSymbol); + synth.register; + } + } + }{ + synth !? {synth.free}; + } + } *new{ |server...args| - var synth, instance; server = server ? Server.default; - if(server.serverRunning.not,{("ERROR:" + this.asString + "– server not running").postln; ^nil}); - synth = {instance = this.kr(*args)}.play(server); - instance.server = server; - instance.synth = synth; - ^instance + if(server.serverRunning.not,{ + (this.asString + "– server not running").warn; + + }); + ^super.newCopyArgs(server ?? {Server.default}).baseinit(*args) } - pr_sendMsg { |msg, args, action,parser| - var c = Condition.new(false); + baseinit { |...args| + + id = UniqueID.next; + synthDefLoaded = false; + defName = (this.class.name.asString ++ id); - OSCFunc( - { |msg| - forkIfNeeded{ - var result; - // msg.postln; - result = FluidMessageResponse.collectArgs(parser,msg.drop(3)); - this.server.sync; - c.test = true; - c.signal; - if(action.notNil){action.value(result)}{action.value}; - } - },'/'++msg).oneShot; + if(server.serverRunning){ this.sendSynthDef(*args);}; - this.server.listSendMsg(['/u_cmd',this.synth.nodeID,this.synthIndex,msg].addAll(args)); + bootFunc = { + ServerBoot.remove(bootFunc,server); + synth = nil; + this.sendSynthDef(*args); + }; - forkIfNeeded { c.wait }; + ServerBoot.add(bootFunc,server); + ServerQuit.add({this.free;},server); + } + free{ + ServerTree.remove(updateFunc,server); + ServerBoot.remove(bootFunc, server); + updateFunc = nil; + // synth !? {synth.tryPerform(\free)};// + synth = nil; + } + + prSendMsg { |msg, args, action,parser| + if(this.server.serverRunning.not,{(this.asString + "– server not running").error; ^nil}); + synth !? { + OSCFunc( + { |msg| + forkIfNeeded{ + var result; + result = FluidMessageResponse.collectArgs(parser,msg.drop(3)); + if(action.notNil){action.value(result)}{action.value}; + } + },'/'++msg, server.addr, nil,[synth.nodeID]).oneShot; + server.listSendMsg(['/u_cmd',synth.nodeID,ugen.synthIndex,msg].addAll(args)); + } } } + +FluidServerCache { + + var cache; + + *new{ ^super.new.init } + + init{ + cache = IdentityDictionary.new; + } + + at { |server,id| + ^cache[server].tryPerform(\at,id) + } + + includesKey{|server,key| + ^cache[server].tryPerform(\includesKey,key) + } + + put {|server,id,x| + cache[server][id] = x; + } + + remove { |server,id| + cache[server]!? {cache[server].removeAt(id)}; + } + + initCache {|server| + cache[server] ?? { + cache[server] = IdentityDictionary.new; + ServerQuit.add({this.clearCache(server)},server); + NotificationCenter.register(server,\newAllocators,this, { + this.clearCache(server); + }); + } + } + + clearCache { |server| + cache[server] !? { cache.removeAt(server) !? {|x| x.tryPerform(\free) } }; + } +} + diff --git a/release-packaging/Classes/FluidNormalize.sc b/release-packaging/Classes/FluidNormalize.sc index ed67f10..1f3cffb 100644 --- a/release-packaging/Classes/FluidNormalize.sc +++ b/release-packaging/Classes/FluidNormalize.sc @@ -1,35 +1,36 @@ FluidNormalize : FluidManipulationClient { - *kr{ |min = 0, max = 1| - ^this.multiNew('control',min, max, Done.none, super.nonBlocking); + *new {|server| + var uid = UniqueID.next; + ^super.new(server,uid).init(uid); } - *new { |server,min = 0 ,max = 1| - ^super.new(server,min,max); - } + init {|uid| + id = uid; + } fit{|dataset, action| - this.pr_sendMsg(\fit,[dataset.asString],action); + this.prSendMsg(\fit,[dataset.asString],action); } normalize{|sourceDataset, destDataset, action| - this.pr_sendMsg(\normalize,[sourceDataset.asString, destDataset.asString],action); + this.prSendMsg(\normalize,[sourceDataset.asString, destDataset.asString],action); } normalizePoint{|sourceBuffer, destBuffer, action| - this.pr_sendMsg(\normalizePoint,[sourceBuffer.asUGenInput, destBuffer.asUGenInput],action); + this.prSendMsg(\normalizePoint,[sourceBuffer.asUGenInput, destBuffer.asUGenInput],action); } cols {|action| - this.pr_sendMsg(\cols,[],action,[numbers(FluidMessageResponse,_,1,_)]); + this.prSendMsg(\cols,[],action,[numbers(FluidMessageResponse,_,1,_)]); } read{|filename,action| - this.pr_sendMsg(\read,[filename.asString],action); + this.prSendMsg(\read,[filename.asString],action); } write{|filename,action| - this.pr_sendMsg(\write,[filename.asString],action); + this.prSendMsg(\write,[filename.asString],action); } -} \ No newline at end of file +} diff --git a/release-packaging/Classes/FluidPCA.sc b/release-packaging/Classes/FluidPCA.sc index 51bb06d..519acb9 100644 --- a/release-packaging/Classes/FluidPCA.sc +++ b/release-packaging/Classes/FluidPCA.sc @@ -1,36 +1,46 @@ FluidPCA : FluidManipulationClient { + + *new {|server| + var uid = UniqueID.next; + ^super.new(server,uid).init(uid); + } + + init {|uid| + id = uid; + } + fit{|dataset, k, action| - this.pr_sendMsg(\fit,[dataset.asString, k],action); + this.prSendMsg(\fit,[dataset.asString, k],action); } transform{|sourceDataset, destDataset, action| - this.pr_sendMsg(\transform,[sourceDataset.asString, destDataset.asString],action); + this.prSendMsg(\transform,[sourceDataset.asString, destDataset.asString],action); } fitTransform{|sourceDataset, k, destDataset, action| - this.pr_sendMsg(\fitTransform,[sourceDataset.asString,k, destDataset.asString],action); + this.prSendMsg(\fitTransform,[sourceDataset.asString,k, destDataset.asString],action); } transformPoint{|sourceBuffer, destBuffer, action| - this.pr_sendMsg(\transformPoint,[sourceBuffer.asUGenInput, destBuffer.asUGenInput],action); + this.prSendMsg(\transformPoint,[sourceBuffer.asUGenInput, destBuffer.asUGenInput],action); } cols {|action| - this.pr_sendMsg(\cols,[],action,[numbers(FluidMessageResponse,_,1,_)]); + this.prSendMsg(\cols,[],action,[numbers(FluidMessageResponse,_,1,_)]); } rows {|action| - this.pr_sendMsg(\rows,[],action,[numbers(FluidMessageResponse,_,1,_)]); + this.prSendMsg(\rows,[],action,[numbers(FluidMessageResponse,_,1,_)]); } read{|filename,action| - this.pr_sendMsg(\read,[filename],action); + this.prSendMsg(\read,[filename],action); } write{|filename,action| - this.pr_sendMsg(\write,[filename],action); + this.prSendMsg(\write,[filename],action); } -} \ No newline at end of file +} diff --git a/release-packaging/Classes/FluidProviderTest.sc b/release-packaging/Classes/FluidProviderTest.sc index d9589b2..bcc434e 100644 --- a/release-packaging/Classes/FluidProviderTest.sc +++ b/release-packaging/Classes/FluidProviderTest.sc @@ -26,18 +26,18 @@ FluidProviderTest : UGen { } addPoint{|server, nodeID, args, action| - this.pr_sendMsg(server, nodeID, 'addPoint',args,action); + this.prSendMsg(server, nodeID, 'addPoint',args,action); } updatePoint{|server, nodeID, args, action| - this.pr_sendMsg(server, nodeID, 'updatePoint',args,action); + this.prSendMsg(server, nodeID, 'updatePoint',args,action); } deletePoint{|server, nodeID, args, action| - this.pr_sendMsg(server,nodeID, 'deletePoint',args,action); + this.prSendMsg(server,nodeID, 'deletePoint',args,action); } - pr_sendMsg { |server, nodeID, msg, args, action,parser| + prSendMsg { |server, nodeID, msg, args, action,parser| server = server ? Server.default; diff --git a/release-packaging/Classes/FluidStandardize.sc b/release-packaging/Classes/FluidStandardize.sc index ab192e7..deee70e 100644 --- a/release-packaging/Classes/FluidStandardize.sc +++ b/release-packaging/Classes/FluidStandardize.sc @@ -1,27 +1,36 @@ FluidStandardize : FluidManipulationClient { + *new {|server| + var uid = UniqueID.next; + ^super.new(server,uid).init(uid); + } + + init {|uid| + id = uid; + } + fit{|dataset, action| - this.pr_sendMsg(\fit,[dataset.asString],action); + this.prSendMsg(\fit,[dataset.asString],action); } standardize{|sourceDataset, destDataset, action| - this.pr_sendMsg(\standardize,[sourceDataset.asString, destDataset.asString],action); + this.prSendMsg(\standardize,[sourceDataset.asString, destDataset.asString],action); } standardizePoint{|sourceBuffer, destBuffer, action| - this.pr_sendMsg(\standardizePoint,[sourceBuffer.asUGenInput, destBuffer.asUGenInput],action); + this.prSendMsg(\standardizePoint,[sourceBuffer.asUGenInput, destBuffer.asUGenInput],action); } cols {|action| - this.pr_sendMsg(\cols,[],action,[numbers(FluidMessageResponse,_,1,_)]); + this.prSendMsg(\cols,[],action,[numbers(FluidMessageResponse,_,1,_)]); } read{|filename,action| - this.pr_sendMsg(\read,[filename.asString],action); + this.prSendMsg(\read,[filename.asString],action); } write{|filename,action| - this.pr_sendMsg(\write,[filename.asString],action); + this.prSendMsg(\write,[filename.asString],action); } -} \ No newline at end of file +} diff --git a/release-packaging/Classes/FluidSubscriberTest.sc b/release-packaging/Classes/FluidSubscriberTest.sc index 8ea6070..04a799d 100644 --- a/release-packaging/Classes/FluidSubscriberTest.sc +++ b/release-packaging/Classes/FluidSubscriberTest.sc @@ -21,11 +21,11 @@ FluidSubscriberTest : UGen { } providerLookup { |server, nodeID, label, action| - this.pr_sendMsg(server, nodeID, 'providerLookup', label, action, + this.prSendMsg(server, nodeID, 'providerLookup', label, action, [string(FluidMessageResponse,_,_),numbers(FluidMessageResponse,_,2,_)] ); } - pr_sendMsg { |server, nodeID, msg, args, action,parser| + prSendMsg { |server, nodeID, msg, args, action,parser| server = server ? Server.default; diff --git a/test/TestFluidManipulationLifecyle.sc b/test/TestFluidManipulationLifecyle.sc new file mode 100644 index 0000000..8209341 --- /dev/null +++ b/test/TestFluidManipulationLifecyle.sc @@ -0,0 +1,105 @@ + +TestFluidCorpusManipulationServer : UnitTest +{ + var waitForCounts, countsListener; + + setUp{ + waitForCounts = Condition.new(false); + countsListener = { |s,changed| + if(changed == \counts) { + waitForCounts.test = true; + waitForCounts.signal; + } + }; + Server.default.addDependant(countsListener); + } + + tearDown{ + Server.default.removeDependant(countsListener); + Server.default.quit; + } + + test_DataSetPersistence{ + var foo, bar, tree, testPoint; + + foo = FluidDataSet(Server.default,\foo); + + this.bootServer(Server.default); + while {Server.default.serverRunning.not}{0.2.wait}; + waitForCounts.test = false; + waitForCounts.wait; + + this.assertEquals(Server.default.numSynths,1,"Dataset: One Synth present after deferred boot"); + + waitForCounts.test = false; + foo.free; + Server.default.freeAll; + waitForCounts.wait; + + this.assertEquals(Server.default.numSynths,0,"Dataset: One Synth present via cretation after boot"); + + //Uniqueness test (difficult to run with previous instance of foo, because + //UnitTest.bootServer messes with Server alloctors and screws up the ID cache + foo = FluidDataSet(Server.default,\foo); + this.assertException({ + bar = FluidDataSet(Server.default,\foo); + },FluidDataSetExistsError,"DataSetDuplicateError on reused name", onFailure:{ + "Exception fail".postln; + }); + + waitForCounts.test = false; + bar = FluidDataSet(Server.default,\bar); + waitForCounts.wait; + + this.assertEquals(Server.default.numSynths,2,"Dataset: Two Synths present after new Dataset added"); + + testPoint = Buffer.alloc(Server.default,8); + Server.default.sync; + testPoint.setn(0,[1,2,3,4,5,6,7,8]); + Server.default.sync; + foo.addPoint(\one,testPoint); + Server.default.sync; + foo.size({|size| + this.assertEquals(size,1,"Dataset size is 1"); + }); + Server.default.sync; + foo.cols({|cols| + this.assertEquals(cols,8,"Dataset cols is 8"); + }); + + Server.default.sync; + waitForCounts.test = false; + + tree = FluidKDTree(Server.default); + waitForCounts.wait; + + this.assert(tree.synth.notNil,"Tree should have a valid synth"); + this.assertEquals(Server.default.numSynths,3,"Dataset: Three Synths remain after cmd-."); + + tree.fit(foo); + Server.default.sync; + tree.cols({|cols| + this.assertEquals(cols,8,"KDTree correct dims after fit") + }); + Server.default.sync; + + //Test cmd-period resistance + waitForCounts.test = false; + Server.default.freeAll; + Server.default.sync; + Server.default.sync; + waitForCounts.wait; + + this.assertEquals(Server.default.numSynths,3,"Dataset: Three Synths remain after cmd-."); + foo.size({|size| + this.assertEquals(size,1,"Dataset size is still 1 after Cmd-."); + }); + Server.default.sync; + foo.cols({|cols| + this.assertEquals(cols,8,"Dataset cols is still 8 after Cmd-."); + }); + Server.default.sync; + tree.cols({|cols| this.assertEquals(cols,8,"KDTree correct dims after Cmd-.")}); + Server.default.sync; + } +} \ No newline at end of file