From a783c4af619227c0f217da9ad537e573a6669353 Mon Sep 17 00:00:00 2001 From: Pierre Alexandre Tremblay Date: Wed, 15 Aug 2018 16:57:15 +0100 Subject: [PATCH] now working with the tensors and the share client code --- CMakeLists.txt | 18 ++++---- fdNMF.cpp | 83 +++++++++++----------------------- fluid_client_nmf.cc | 100 +++++++++++++++++++++++++++++++++++++++++ fluid_client_nmf.h | 96 +++++++++++++++++++++++++++++++++++++++ fluid_nmf_tilde_util.h | 89 ++++++++++++++++++++++++++++++++++++ tests.scd | 2 +- 6 files changed, 319 insertions(+), 69 deletions(-) create mode 100644 fluid_client_nmf.cc create mode 100644 fluid_client_nmf.h create mode 100644 fluid_nmf_tilde_util.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ad09a29..abb9ab0 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,10 @@ ####### added the eingenmf -set(EIGENMF_DIR ../eigenmf) +set(FLUID_DECOMP_DIR ../fluid_decomposition) include_directories( - "${EIGENMF_DIR}" - "${EIGENMF_DIR}/Eigen" - "${EIGENMF_DIR}/HISSTools_FFT" + "${FLUID_DECOMP_DIR}" + "${FLUID_DECOMP_DIR}/include" + "${FLUID_DECOMP_DIR}/3rdparty" ) ####### original SC Cmake file starts here @@ -90,16 +90,14 @@ if(MINGW) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mstackrealign") endif() -####### added the eingenmf +####### added the fluid_decomposition add_library( ${PROJECT} MODULE ${FILENAME} -"${EIGENMF_DIR}/nmf.cc" -"${EIGENMF_DIR}/nmf.h" -"${EIGENMF_DIR}/stft.cc" -"${EIGENMF_DIR}/stft.h" - "${EIGENMF_DIR}/HISSTools_FFT/HISSTools_FFT.cpp" +"${FLUID_DECOMP_DIR}/3rdparty/HISSTools_FFT/HISSTools_FFT.cpp" +"fluid_client_nmf.cc" +"fluid_client_nmf.h" ) if(SUPERNOVA) add_library(${PROJECT}_supernova MODULE ${FILENAME}) diff --git a/fdNMF.cpp b/fdNMF.cpp index b5f03db..b5d099b 100644 --- a/fdNMF.cpp +++ b/fdNMF.cpp @@ -2,27 +2,16 @@ // A tool from the FluCoMa project, funded by the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation programme (grant agreement No 725899) #include "SC_PlugIn.h" -#include -#include -#include "stft.h" -#include "nmf.h" -#include - - -//Using statements for eigenmf. These will change -using stft::STFT; -using stft::ISTFT; -using stft::Spectrogram; -using stft:: audio_buffer_t; -using stft:: magnitude_t; -using nmf::NMF; -using nmf::NMFModel; -using Eigen::MatrixXcd; -using Eigen::MatrixXd; -using std::complex; -using util::stlVecVec2Eigen; -using util::Eigen2StlVecVec; -using std::numeric_limits; +#include "STFT.hpp" +#include "FluidTensor.hpp" +#include "fluid_client_nmf.h" +#include "fluid_nmf_tilde_util.h" + +//Using statements for fluidtensor +using fluid::FluidTensor; +using fluid::FluidTensorView; +using fluid::nmf::error_strings; +using fluid::nmf::NMFClient; static InterfaceTable *ft; @@ -31,8 +20,8 @@ namespace gaffatape { void BufNMF(World *world, struct SndBuf *dstBuf, struct sc_msg_iter *msg) { - int dstFrameCount = dstBuf->frames; - int dstChanCount = dstBuf->channels; + size_t dstFrameCount = dstBuf->frames; + size_t dstChanCount = dstBuf->channels; uint32 srcBufNum = msg->geti(); long rank = msg->geti(); @@ -53,8 +42,8 @@ void BufNMF(World *world, struct SndBuf *dstBuf, struct sc_msg_iter *msg) return; } - int srcFrameCount = srcBuf->frames; - int srcChanCount = srcBuf->channels; + size_t srcFrameCount = srcBuf->frames; + size_t srcChanCount = srcBuf->channels; if (dstChanCount < rank) { Print("fdNMF is not happy because the destination buffer has a lower channel count than the number of ranks.\n"); @@ -67,47 +56,25 @@ void BufNMF(World *world, struct SndBuf *dstBuf, struct sc_msg_iter *msg) return; } - // make a vector of doubles for the samples - // padding by half a fft frame each sides - std::vector audio_in(srcFrameCount+fftSize); - long halfFftSize = fftSize / 2; + // make fuildtensorviewers of my SC interleaved buffers + FluidTensorView in_view ({0,{srcFrameCount, srcChanCount}},srcBuf->data); + FluidTensorView out_view ({0,{dstFrameCount, dstChanCount}},dstBuf->data); - //copied as is from max source (setting up the different variables and processes) - STFT stft(windowSize, fftSize, hopSize); - NMF nmfProcessor(rank, iterations); - ISTFT istft(windowSize, fftSize, hopSize); + //setup the nmf + NMFClient nmf(rank ,iterations, fftSize, windowSize, hopSize); //for each channels // for (int j=0;jdata[(i*srcChanCount)+j]; - } - - Spectrogram spec = stft.process(audio_in); - magnitude_t mag = spec.magnitude(); - NMFModel decomposition = nmfProcessor.process(mag); - MatrixXd W = stlVecVec2Eigen(decomposition.W); - MatrixXd H = stlVecVec2Eigen(decomposition.H); - - MatrixXd V = W * H; - - for (int i = 0; i < rank; i++) + FluidTensor audio_in(in_view.col(j)); + //Process, with resynthesis + nmf.process(audio_in,true); + //Copy output + for (int i = 0; i < rank; ++i) { - MatrixXd source = W.col(i) * H.row(i); - MatrixXd filter = source.cwiseQuotient(V); - MatrixXcd specMatrix = stlVecVec2Eigen(spec.mData); - specMatrix = specMatrix.cwiseProduct(filter); - Spectrogram resultS(Eigen2StlVecVec>(specMatrix)); - - audio_buffer_t result = istft.process(resultS); - - //writes the output - for (int k=0;kdata[(k*rank)+i] = (float)result[k+halfFftSize]; - } + out_view.col(i) = nmf.source(i); } } } diff --git a/fluid_client_nmf.cc b/fluid_client_nmf.cc new file mode 100644 index 0000000..b7eb841 --- /dev/null +++ b/fluid_client_nmf.cc @@ -0,0 +1,100 @@ +#include "fluid_client_nmf.h" +#include "STFT.hpp" +#include "RatioMask.hpp" + +namespace fluid{ +namespace nmf{ + using fluid::nmf::NMF; + using fluid::stft::STFT; + using fluid::stft::ISTFT; + using fluid::stft::Spectrogram; + using fluid::FluidTensor; + + + NMFClient::NMFClient(size_t rank,size_t iterations, size_t fft_size, size_t window_size, size_t hop_size): + m_rank(rank),m_iterations(iterations), m_fft_size(fft_size), m_window_size(window_size), m_hop_size(hop_size), m_has_processed(false), m_has_resynthed(false) + {} + + + void NMFClient::process(const FluidTensor &data, bool resynthesise) + { + m_audio_buffers.resize(m_rank,data.extent(0)); + + m_has_processed = false; + m_has_resynthed = false; + STFT stft(m_window_size,m_fft_size,m_hop_size); + Spectrogram spec = stft.process(data); + FluidTensor mag = spec.getMagnitude(); + NMF nmf(m_rank,m_iterations); + m_model = nmf.process(spec.getMagnitude()); + m_has_processed = true; + + if(resynthesise) + { + ratiomask::RatioMask mask(m_model.getMixEstimate(),1); + ISTFT istft(m_window_size, m_fft_size, m_hop_size); + for(int i = 0; i < m_rank; ++i) + { + RealMatrix estimate = m_model.getEstimate(i); + Spectrogram result(mask.process(spec.mData, estimate)); + RealVector audio = istft.process(result); + m_audio_buffers.row(i) = audio; + } + m_has_resynthed = true; + } + } + + size_t NMFClient::dictionary_size() const + { + return m_has_processed ? m_model.getW().extent(0) : 0 ; + } + size_t NMFClient::activations_length() const{ + return m_has_processed ? m_model.getH().extent(1) : 0; + } + size_t NMFClient::num_sources() const + { + return m_has_resynthed ? m_audio_buffers.size() : 0; + } + + const FluidTensorView NMFClient::dictionary(const size_t idx) const + { + assert(m_has_processed && idx < m_model.W.cols()); + return m_model.getW().col(idx); + } + + const FluidTensorView NMFClient::activation(const size_t idx) const + { + assert(m_has_processed && idx < m_model.H.rows()); + return m_model.getH().row(idx); + } + + const FluidTensor NMFClient::dictionaries() const + { + return m_model.getW(); + } + const FluidTensor NMFClient::activations()const + { + return m_model.getH(); + } + + + + const FluidTensorView NMFClient::source(const size_t idx) const + { + assert(idx < m_audio_buffers.rows() && "Range Error"); + return m_audio_buffers.row(idx); + } + +// source_iterator NMFClient::sources_begin() const +// { +// return m_audio_buffers.cbegin(); +// } +// +// source_iterator NMFClient::sources_end() const +// { +// return m_audio_buffers.cend(); +// } + + +}//namespace nmf +}//namespace fluid diff --git a/fluid_client_nmf.h b/fluid_client_nmf.h new file mode 100644 index 0000000..df7e812 --- /dev/null +++ b/fluid_client_nmf.h @@ -0,0 +1,96 @@ +#pragma once + +#include "FluidTensor.hpp" +#include "NMF.hpp" +#include +#include + +using fluid::FluidTensor; + +namespace fluid { +namespace nmf{ + /** + Integration class for doing NMF filtering and resynthesis + **/ + class NMFClient + { +// using vec_iterator = std::vector::const_iterator; +// using source_iterator = std::vector>::const_iterator; + + public: + //No, you may not construct an empty instance, or copy this, or move this + NMFClient() = delete; + NMFClient(NMFClient&)=delete; + NMFClient(NMFClient&&)=delete; + NMFClient operator=(NMFClient&)=delete; + NMFClient operator=(NMFClient&&)=delete; + + /** + You may constrct one by supplying some senisble numbers here + rank: NMF rank + iterations: max nmf iterations + fft_size: power 2 pls + **/ + NMFClient(size_t rank, size_t iterations, size_t fft_size, size_t window_size, size_t hop_size); + ~NMFClient()= default; + //Not implemented + //void reset(); + //bool isReady() const; + + /*** + Take some data, NMF it + ***/ + void process(const FluidTensor &data, bool resynthesise); + + /*** + Report the size of a dictionary, in bins (= fft_size/2) + ***/ + size_t dictionary_size() const; + + /*** + Report the length of an activation, in frames + ***/ + size_t activations_length() const; + + /*** + Report the number of sources (i.e. the rank + ***/ + size_t num_sources() const; +// size_t rank() const; + + /*** + Retreive the dictionary at the given index + ***/ + const FluidTensorView dictionary(const size_t idx) const; + + /*** + Retreive the activation at the given index + ***/ + const FluidTensorView activation(const size_t idx) const; + + /*** + Retreive the resynthesized source at the given index (so long as resyntheiss has happened, mind + ***/ + const FluidTensorView source(const size_t idx) const; + +// source_iterator sources_begin() const ; +// source_iterator sources_end()const; + + /*** + Get the whole of dictionaries / activations as a 2D structure + ***/ + const FluidTensor dictionaries() const; + const FluidTensor activations() const; + private: + size_t m_rank; + size_t m_iterations; + size_t m_fft_size; + size_t m_window_size; + size_t m_hop_size; + bool m_has_processed; + bool m_has_resynthed; + fluid::nmf::NMFModel m_model; + FluidTensor m_audio_buffers; + }; +} //namespace max +} //namesapce fluid diff --git a/fluid_nmf_tilde_util.h b/fluid_nmf_tilde_util.h new file mode 100644 index 0000000..94f0015 --- /dev/null +++ b/fluid_nmf_tilde_util.h @@ -0,0 +1,89 @@ +// +// fluid_nmf_tilde_util.h +// fluid.nmf~ +// +// Created by Owen Green on 02/07/2018. +// + +#ifndef fluid_nmf_tilde_util_h +#define fluid_nmf_tilde_util_h + +#include "FluidTensor.hpp" +#include +#include +namespace fluid{ + namespace nmf{ + + using real_matrix = fluid::FluidTensor; + + struct error_strings + { + static constexpr const char* rank_below_one = "Rank < 1 makes no sense"; + static constexpr const char* no_input_buffer_symbol = "Could not extract symbol from input buffer name"; + static constexpr const char* no_out_polybuf_symbol = "Could not extract symbol from output polybuffer name"; + + static constexpr const char* sources_args = "At least three arguments required: NMF rank, input buffer, and output polybuffer."; + + static constexpr const char* filters_args = "At least four arguments required: NMF rank, input buffer, output dictionary polybuffer, and output activations polybuffer."; + + static constexpr const char* no_filter_polybuf_symbol = "Could not extract symbol from filters polybuffer name"; + static constexpr const char* no_env_polybuf_symbol = "Could not extract symbol from envelopes polybuffer name"; + static constexpr const char* get_buffer_fail = "Could not get buffer for name %s"; + + private: + error_strings(){} + }; + + + + + +// /***************** +// A template function to reduce duplication in copying back to Max buffers +// My hope is to render this either obsolete or much simpler once algo code +// is refactored. +// At present it needs to be fed a vector of vectors (ugh) and a function that +// does the copying (see under) +// *******************/ +// template +// void copy_to_buffer(t_object* obj, int rank, t_symbol* polybuf_name,real_matrix &mtrx,F&& copy_fn, double scale=1.) +// { +// for(int i = 0; i < rank; i++) +// { +// std::ostringstream ss; +// ss << polybuf_name->s_name << "." << i+1; +// const char* buffername = ss.str().c_str(); +// +// t_buffer_ref* ref = buffer_ref_new(obj, gensym(buffername)); +// t_buffer_obj* this_buffer = buffer_ref_getobject(ref); +// +// if(this_buffer) +// { +// float* samps = buffer_locksamples(this_buffer); +// if(samps) +// { +// copy_fn(samps,mtrx,i,scale); +// buffer_unlocksamples(this_buffer); +// } +// } +// +// if(ref) +// object_free(ref); +// +// } +// } +// +// void from_cols(float* dst, real_matrix &m, long i,double scale) +// { +// for(long j = 0; j< m[0].size(); j++) +// dst[j] = m[i][j] * scale; +// } +// +// void from_rows(float* dst,real_matrix &m, long i,double scale) +// { +// for(long j = 0; j < m.size(); j++) +// dst[j] = m[j][i] * scale; +// } + } +} +#endif /* fluid_nmf_tilde_util_h */ diff --git a/tests.scd b/tests.scd index 92d205d..8aefe7d 100644 --- a/tests.scd +++ b/tests.scd @@ -15,7 +15,7 @@ c.plot; {PlayBuf.ar(2,c.bufnum,doneAction:2)}.play // run the code -b.fdNMF(e,5, 150, 2048,1024,256,{|x| "itworks".postln; x.postln;}) +b.fdNMF(e,5, 100, 1024,1024,256,{|x| "itworks".postln; x.postln;}) // read to check e.plot;