Skip to content
2 changes: 2 additions & 0 deletions include/flucoma/algorithms/public/KMeans.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,8 @@ class KMeans
mMeans = _impl::asEigen<Eigen::Array>(means);
mDims = mMeans.cols();
mK = mMeans.rows();
assert(mDims > 0);
assert(mK > 0);
mEmpty = std::vector<bool>(asUnsigned(mK), false);
mTrained = true;
}
Expand Down
33 changes: 28 additions & 5 deletions include/flucoma/clients/nrt/KMeansClient.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class KMeansClient : public FluidBaseClient,
public DataClient<algorithm::KMeans>
{
enum {kName, kNumClusters, kMaxIter, kInit, kRandomSeed};
ParameterTrackChanges<index> mTracker;
public:
using string = std::string;
using BufferPtr = std::shared_ptr<BufferAdaptor>;
Expand All @@ -49,6 +49,8 @@ class KMeansClient : public FluidBaseClient,
using ParamDescType = decltype(KMeansParams);

using ParamSetViewType = ParameterSetView<ParamDescType>;
using ParamValues = typename ParamSetViewType::ValueTuple;

std::reference_wrapper<ParamSetViewType> mParams;

void setParams(ParamSetViewType& p) { mParams = p; }
Expand Down Expand Up @@ -204,18 +206,16 @@ class KMeansClient : public FluidBaseClient,
return OK();
}

MessageResult<void> setMeans(InputDataSetClientRef srcClient)
MessageResult<ParamValues> setMeans(InputDataSetClientRef srcClient)
{
auto srcPtr = srcClient.get().lock();
if (!srcPtr) return Error(NoDataSet);
auto dataSet = srcPtr->getDataSet();
if (dataSet.size() == 0) return Error(EmptyDataSet);
if (dataSet.size() != get<kNumClusters>()) return Error(WrongNumInitial);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Presumably the numClusters parameter can now be incorrect after setMeans. So this method actually needs to update the params and return these.

Something like (untested)

  MessageResult<ParamValues> setMeans(InputDataSetClientRef srcClient)
  {
    auto srcPtr = srcClient.get().lock();
    if (!srcPtr) return Error(NoDataSet);
    auto dataSet = srcPtr->getDataSet();
    if (dataSet.size() == 0) return Error(EmptyDataSet);
    auto numClusters = dataSet.size(); 
    mAlgorithm.setMeans(dataSet.getData()); 
    // update params and return to host
    ParamValues newParams = mParams.get().toTuple();
    std::get<kNumClusters>(newParams) = numClusters; 
    return newParams; 
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ParamValues will need to be defined near the top of the class, like in the MLP objects:

using ParamValues = typename ParamSetViewType::ValueTuple;

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is interesting, as the current public code does not behave the way I thought it would - changing the numCluster should always reset the algo to unFitted, but it currently doesn't.

could we fix that the same way? In effect, the expected behaviour for me:

  • set does what I did here
  • changing numCluster always invalidates the fitting (it doesn't now)

should i just put a separate bug report and a fix for that? Or am I just all confused?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

studying MLP code, I can see that the clients check if (!mAlgorithm.trained()) return Error(NoDataFitted); but i cannot find how it makes it 'untrained' when any parameter (like network structure) is changed. mTracker is only checked once in fit.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

an example of the misunderstood behaviour by yours truly:


----------begin_max5_patcher----------
1884.3oc0ZksbihCE8YmuBU7raWrujm546nyTtjAYa0AKwHDwISWc+sOZCLf
gDCMwS09ABn064bWzUR4GOrxZG8UToE3Qv2.qV8iGVsRUjrfUluWYcB9ZZNr
T0LqT5oSHB2ZstNN5UtpbmMfSvmQ.HvGTPwDNHq7Ih6FPACkgS4OBPLFksFP
nbvdLmiXOQ71He8Ih+FvYJ6YQGB1.xomQL.o5TZdUopYga.kbbdttQqAnWPD
.dOn7HsJOCrCApHpgLqVrxwDTJshnjsPSghgjVwyQbERrMkV.4oGwjCaYnTt
lIh7B2XuFjjHe5Fp9HNZiM3uuLRXR8.4XJDmoHB5tu+E2HKYY+7gGjOV+aRs
gVSE.9g9RY1KzU9Gm.aEPrmBBBVRDDLWD3p0AyBA9KIB7mMBzh9rPf2Rh.uo
if.OknG6OaD3tjHvc1Hv2Y1HvYIQfyjQfSTfTl8muir8b.vITYI7.5J.XBlq
Cv+OUH1aCinZAQWD+sBjFNVVMB9PpqjnMAxvVZ0ks1AxcLz5dMZcB+iCst9I
yFsAKIZurhaInwUSPAkBKYHGSIa6rpp6m.UjDqVp0a5Lg+mld2dI04N50y78
i0qMDNcj5rjHUj0DfyfXhPL+DrsMKcn8mmDLS9iyO1w2dt9wwyArDzYQeuBq
h7tqNU.xGFgtigPYmrV+AHM1jJiZwnfOPoNvZQNtKHPK.OYkxPPNBbBAIkfL
HGVh3OYsjJWCjcMIP3OYLGY0dvQLCPMHckvCLWrcFVoH9ZqQXkErnnUwqZ0E
I87cpZfhW2TDlnKJroHF5Ebc+CZJExDPjKvWESQ.VuFVmaqbXnYHFoBqDEcg
BEkQjTpDBT3kU.S0cVp4pqtEooSXzSQZdd5jVbBtvZBU9gbZ5ynrVo+HTMEH
Qfn1K4zo5LzdXUNe6dJgWh+WkD3HsEGn98FIbvJkXPI9+ECCya.vAFNiRjBQ
GMgr35oSDAOPaIzFLpVPfECzYgIffVFoxRAHqJ2AYRE0tbTKOTggNkl2spl9
ki1yMUWfIjdrHmVLdkL7giuSe2QEUd58FaUMkaqH5Z2JrI3aKguzks4v7bie
a2g+UHAeR3yxwZUfqcSkHBT.zikoLZddG7pq4kApISXimhNiy3GUSTaiAQyw
E0FQVMZ4L7ATIuaYb3gxtkTxeSS5sJpZmwGdKGcpHWfhtMP3cfK4kGomKMMr
1PqMAb43VZ6S2NHXmxe2cVzMhnNZXI.Bjq1I.Nj8FftWebLk.NEjSgY.wGT.
+HpNfI37Qb5QvY4ArrCAnULc.01yQmL+7aUwfajYzreBUNOQgs1LSXqMyLZ.
09AU0Y6spdsDSXpYRocVeoKiJ4wMhfcOq3pGAoz7xGaRNd7EYFYM1FshUGDO
.SoWuIIbiqLyBuqV14ZlxaXlJ9dxTGXzph6E4DnRgNL7iolQLhLYR+4yMhEz
DQ9yTdeyjctQNQZoHLXh8mMoDrnjhFGVyMdgATQIaRB7C78Rz+7cEYUnBd3M
mfGN1sEn5.psijq5.IC8ZqzfVD9PIae1l.arE+b7MjU3MRV1CSVg+uwUEPBJ
e1lNNpyPvWuyLyQv5D0i.1IhrKiYQx5y96nrLct5t2h0kt4hUGz4f+MQhH9t
dAtI9hIMzOxNbsnn.GuH2.+DamjP6nXgDoDqdBU2SypUExby6ibH4fIswnK4
9nyutiz3ttyityXAiVPYMoauIX.s3kjckoBLRJLJ8fr9gUukhTKRqMmMQfER
SGYQj1FGSZx8+a0qhoZ2MYcMUgH7FkgZg8SQHhuQgH3yTHBtU0gLB5HRwE6D
UFziXnzrALgGeXq8PdYeZCKxvzTQ7mNV1wR64XuDWGkOVjcRrs+0dUc7Gp86
LODV7cZqbgaUq2hIx8fipmKUaqenlgKLv5VzNSHjWuABoKr.Yp8Va+NpsNjS
uULde5Yf.PRgUG0Q47G4G3buD8tmF2GJ76NrWrCjFHr5pVLdzNQMUb5AFLCa
VdpSkciDpB+pnEioh7MQQNWYzX5oSioVn3Wz5Aea3t5V20n3Xg5X8fucUW6G
O1Kocs0oDX0f2lJaoJWbq3q8TmtEQ+zAmf0rRd8kKdpV.yWn5DzxRYIaFi5A
vRddFYa0mcyVHmyv6p35fYsO6tIcFCGxo6f48NXfgNBhGtDRcgN+aaUHNaUt
PfuHe+KpOb0end2Cbo7cvk81.L66W+z07mIcoGdK3AGuqZ+dD6W5KB.70R3o
hVW30Mdd46EaliKOvbEW8dfOzQQ0gZ2M8kDLoSPdIw997Jb1lmUmuyu.e8Lj
Iu4mRfyDg+GdUAdhTuU2as4JCBmLrcWbXaNuqe8A2307Acj4540oV35M8KHo
MlU8pa51ZJnedcFh3p74RZmN2Hox4dIStqn5acdhuSyi6MLOprzc9MmHG+6D
hbBtWST38Zhb07+6qjVpIx9dLQd2vDE0oMmvY5ybu9+gG0ox4Xq+uaHo4ieS
IK5VXfqUp53J8teQ4bz6dE6cmhWeehieWh8uGQUhUic+gO7yG9OP.s.FO
-----------end_max5_patcher-----------

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

load and read have absolutely horrible special case handling in the host wrappers that happens to deal with updating the parameters. There's not any machinery for ad hoc messages that would return updated parameters (like setMeans).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I say it works, I mean that I have a debugger with the process and I go line for line, I see in all cases the mAlgo being changed, and the newParams too. the only diff is in the 'updated' client - one updates, the other not.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. It will 'work', but only up to the point where the host needs to actually store the updated parameters, which it won't do without a new overload being added.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok so should I put a bug report for load/read not working for kmeans, or will sorting this PR sort the other case too?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

load and read not working is its own issue, though I suspect it's because the default implementation for data clients doesn't return updated parameters (MLP overrides the default).

mAlgorithm.setMeans(dataSet.getData());
return OK();
return updateParameters();
}


MessageResult<void> transformPoint(InputBufferPtr in, BufferPtr out) const
{
if (!mAlgorithm.initialized()) return Error(NoDataFitted);
Expand All @@ -234,6 +234,20 @@ class KMeansClient : public FluidBaseClient,
return OK();
}

MessageResult<ParamValues> read(string fileName)
{
auto result = DataClient::read(fileName);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

MessageResult<ParamValues> load(string s)
{
auto result = DataClient::load(s);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

static auto getMessageDescriptors()
{
return defineMessages(
Expand All @@ -259,6 +273,15 @@ class KMeansClient : public FluidBaseClient,
private:
using DataSet = FluidDataSet<std::string, double, 1>;

ParameterTrackChanges<index> mTracker;

MessageResult<ParamValues> updateParameters()
{
ParamValues newParams = mParams.get().toTuple();
std::get<kNumClusters>(newParams) = mAlgorithm.getK();
return newParams;
}

std::pair<MessageResult<IndexVector>, IndexVector>
train(DataSet const& dataSet, index k, index maxIter , index initMethod, index randomSeed)
{
Expand Down
40 changes: 30 additions & 10 deletions include/flucoma/clients/nrt/NormalizeClient.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ class NormalizeClient : public FluidBaseClient,
using StringVector = FluidTensor<string, 1>;

using ParamDescType = decltype(NormalizeParams);

using ParamSetViewType = ParameterSetView<ParamDescType>;
using ParamValues = typename ParamSetViewType::ValueTuple;

std::reference_wrapper<ParamSetViewType> mParams;

void setParams(ParamSetViewType& p) { mParams = p; }
Expand Down Expand Up @@ -75,6 +76,7 @@ class NormalizeClient : public FluidBaseClient,
}
return {};
}

MessageResult<void> transform(InputDataSetClientRef sourceClient,
DataSetClientRef destClient)
{
Expand All @@ -91,19 +93,27 @@ class NormalizeClient : public FluidBaseClient,
}

MessageResult<void> transformPoint(InputBufferPtr in, BufferPtr out)
{
return _transformPoint(in, out, false);
}

{ return _transformPoint(in, out, false); }

MessageResult<void> inverseTransform(InputDataSetClientRef sourceClient,
DataSetClientRef destClient)
DataSetClientRef destClient)
{ return _transform(sourceClient, destClient, true); }

MessageResult<void> inverseTransformPoint(InputBufferPtr in, BufferPtr out)
{ return _transformPoint(in, out, true); }

MessageResult<ParamValues> read(string fileName)
{
return _transform(sourceClient, destClient, true);
auto result = DataClient::read(fileName);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}
MessageResult<void> inverseTransformPoint(InputBufferPtr in, BufferPtr out)

MessageResult<ParamValues> load(string s)
{
return _transformPoint(in, out, true);
auto result = DataClient::load(s);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

static auto getMessageDescriptors()
Expand All @@ -126,6 +136,16 @@ class NormalizeClient : public FluidBaseClient,
}

private:
ParameterTrackChanges<double, double> mTracker;

MessageResult<ParamValues> updateParameters()
{
ParamValues newParams = mParams.get().toTuple();
std::get<kMin>(newParams) = mAlgorithm.getMin();
std::get<kMax>(newParams) = mAlgorithm.getMax();
return newParams;
}

MessageResult<void> _transform(InputDataSetClientRef sourceClient,
DataSetClientRef destClient, bool invert)
{
Expand Down
29 changes: 27 additions & 2 deletions include/flucoma/clients/nrt/RobustScaleClient.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ class RobustScaleClient : public FluidBaseClient,
using StringVector = FluidTensor<string, 1>;

using ParamDescType = decltype(RobustScaleParams);

using ParamSetViewType = ParameterSetView<ParamDescType>;
using ParamValues = typename ParamSetViewType::ValueTuple;

std::reference_wrapper<ParamSetViewType> mParams;

void setParams(ParamSetViewType& p) { mParams = p; }
Expand Down Expand Up @@ -76,6 +77,7 @@ class RobustScaleClient : public FluidBaseClient,
}
return {};
}

MessageResult<void> transform(InputDataSetClientRef sourceClient,
DataSetClientRef destClient)
{
Expand Down Expand Up @@ -106,6 +108,20 @@ class RobustScaleClient : public FluidBaseClient,
{
return _transformPoint(in, out, true);
}

MessageResult<ParamValues> read(string fileName)
{
auto result = DataClient::read(fileName);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

MessageResult<ParamValues> load(string s)
{
auto result = DataClient::load(s);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

static auto getMessageDescriptors()
{
Expand All @@ -127,6 +143,16 @@ class RobustScaleClient : public FluidBaseClient,
}

private:
ParameterTrackChanges<double, double> mTracker;

MessageResult<ParamValues> updateParameters()
{
ParamValues newParams = mParams.get().toTuple();
std::get<kLow>(newParams) = mAlgorithm.getLow();
std::get<kHigh>(newParams) = mAlgorithm.getHigh();
return newParams;
}

MessageResult<void> _transform(InputDataSetClientRef sourceClient,
DataSetClientRef destClient, bool invert)
{
Expand Down Expand Up @@ -168,7 +194,6 @@ class RobustScaleClient : public FluidBaseClient,
outBuf.samps(0, mAlgorithm.dims(), 0) <<= dest;
return OK();
}

};

using RobustScaleRef = SharedClientRef<const RobustScaleClient>;
Expand Down
53 changes: 39 additions & 14 deletions include/flucoma/clients/nrt/SKMeansClient.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ class SKMeansClient : public FluidBaseClient,
ModelObject,
public DataClient<algorithm::SKMeans>
{
ParameterTrackChanges<index> mTracker;
public:
using string = std::string;
using BufferPtr = std::shared_ptr<BufferAdaptor>;
Expand All @@ -48,6 +47,8 @@ class SKMeansClient : public FluidBaseClient,

using ParamDescType = decltype(SKMeansParams);
using ParamSetViewType = ParameterSetView<ParamDescType>;
using ParamValues = typename ParamSetViewType::ValueTuple;

std::reference_wrapper<ParamSetViewType> mParams;

void setParams(ParamSetViewType& p) { mParams = p; }
Expand Down Expand Up @@ -204,18 +205,16 @@ auto [result, _] = train(dataSet, k, maxIter, get<kInit>(), get<kRandomSeed>());
return OK();
}

MessageResult<void> setMeans(DataSetClientRef srcClient)
MessageResult<ParamValues> setMeans(DataSetClientRef srcClient)
{
auto srcPtr = srcClient.get().lock();
if (!srcPtr) return Error(NoDataSet);
auto dataSet = srcPtr->getDataSet();
if (dataSet.size() == 0) return Error(EmptyDataSet);
if (dataSet.size() != get<kNumClusters>()) return Error(WrongNumInitial);
mAlgorithm.setMeans(dataSet.getData());
return OK();
return updateParameters();
}


MessageResult<void> encodePoint(BufferPtr in, BufferPtr out) const
{
if (!mAlgorithm.initialized()) return Error(NoDataFitted);
Expand All @@ -234,6 +233,20 @@ auto [result, _] = train(dataSet, k, maxIter, get<kInit>(), get<kRandomSeed>());
return OK();
}

MessageResult<ParamValues> read(string fileName)
{
auto result = DataClient::read(fileName);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

MessageResult<ParamValues> load(string s)
{
auto result = DataClient::load(s);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

static auto getMessageDescriptors()
{
return defineMessages(
Expand All @@ -259,21 +272,33 @@ auto [result, _] = train(dataSet, k, maxIter, get<kInit>(), get<kRandomSeed>());
private:
using DataSet = FluidDataSet<std::string, double, 1>;

ParameterTrackChanges<index> mTracker;

MessageResult<ParamValues> updateParameters()
{
ParamValues newParams = mParams.get().toTuple();
std::get<kNumClusters>(newParams) = mAlgorithm.getK();
return newParams;
}

std::pair<MessageResult<IndexVector>, IndexVector>
train(DataSet const& dataSet, index k, index maxIter , index initMethod, index randomSeed)
train(DataSet const& dataSet, index k, index maxIter, index initMethod,
index randomSeed)
{
mAlgorithm.train(
dataSet, k, maxIter,
static_cast<algorithm::KMeans::InitMethod>(initMethod), randomSeed);
mAlgorithm.train(dataSet, k, maxIter,
static_cast<algorithm::KMeans::InitMethod>(initMethod),
randomSeed);
IndexVector assignments(dataSet.size());
mAlgorithm.getAssignments(assignments);
auto training_result = MessageResult<IndexVector>(getCounts(assignments,k));
if(mAlgorithm.nEmpty() > 0)
auto training_result =
MessageResult<IndexVector>(getCounts(assignments, k));
if (mAlgorithm.nEmpty() > 0)
{
training_result.set(Result::Status::kWarning);
training_result.addMessage("There were empty clusters; perhaps numClusters is too high.");
training_result.set(Result::Status::kWarning);
training_result.addMessage(
"There were empty clusters; perhaps numClusters is too high.");
}
return {training_result, assignments};
return {training_result, assignments};
}

IndexVector getCounts(IndexVector assignments, index k) const
Expand Down
27 changes: 26 additions & 1 deletion include/flucoma/clients/nrt/UMAPClient.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,9 @@ class UMAPClient : public FluidBaseClient,
using StringVector = FluidTensor<string, 1>;

using ParamDescType = decltype(UMAPParams);

using ParamSetViewType = ParameterSetView<ParamDescType>;
using ParamValues = typename ParamSetViewType::ValueTuple;

std::reference_wrapper<ParamSetViewType> mParams;

void setParams(ParamSetViewType& p) { mParams = p; }
Expand Down Expand Up @@ -156,6 +157,20 @@ class UMAPClient : public FluidBaseClient,
return OK();
}

MessageResult<ParamValues> read(string fileName)
{
auto result = DataClient::read(fileName);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

MessageResult<ParamValues> load(string s)
{
auto result = DataClient::load(s);
if (result.ok()) return updateParameters();
return {result.status(), result.message()};
}

static auto getMessageDescriptors()
{
return defineMessages(
Expand All @@ -171,6 +186,16 @@ class UMAPClient : public FluidBaseClient,
makeMessage("write", &UMAPClient::write),
makeMessage("read", &UMAPClient::read));
}

private:
ParameterTrackChanges<index> mTracker;

MessageResult<ParamValues> updateParameters()
{
ParamValues newParams = mParams.get().toTuple();
std::get<kNumDimensions>(newParams) = mAlgorithm.dims();
return newParams;
}
};

using UMAPRef = SharedClientRef<const UMAPClient>;
Expand Down
Loading