schema.proto

// Copyright (c) 2014, Salesforce.com, Inc.  All rights reserved.
// Copyright (c) 2015, Google, Inc.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
// - Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
// - Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
// - Neither the name of Salesforce.com nor the names of its contributors
//   may be used to endorse or promote products derived from this
//   software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

syntax = "proto2";

import "distributions/io/schema.proto";

package protobuf.loom;

//----------------------------------------------------------------------------

message HyperPrior {
  message BetaBernoulli {
    repeated float alpha = 1 [packed = true];
    repeated float beta = 2 [packed = true];
  }
  message DirichletDiscrete {
    repeated float alpha = 1 [packed = true];
  }
  message DirichletProcessDiscrete {
    repeated float gamma = 1 [packed = true];
    repeated float alpha = 2 [packed = true];
  }
  message GammaPoisson {
    repeated float alpha = 1 [packed = true];
    repeated float inv_beta = 2 [packed = true];
  }
  message BetaNegativeBinomial {
    repeated float alpha = 1 [packed = true];
    repeated float beta = 2 [packed = true];
    repeated uint64 r = 3 [packed = true];
  }
  message NormalInverseChiSq {
    repeated float mu = 1 [packed = true];
    repeated float kappa = 2 [packed = true];
    repeated float sigmasq = 3 [packed = true];
    repeated float nu = 4 [packed = true];
  }

  repeated distributions.Clustering.PitmanYor topology = 1;
  repeated distributions.Clustering.PitmanYor clustering = 2;
  optional BetaBernoulli bb = 3;
  optional DirichletDiscrete dd = 4;
  optional DirichletProcessDiscrete dpd = 5;
  optional GammaPoisson gp = 6;
  optional BetaNegativeBinomial bnb = 7;
  optional NormalInverseChiSq nich = 8;
}

//----------------------------------------------------------------------------

message ProductValue {
  message Observed {
    enum Sparsity {
      NONE = 0;
      SPARSE = 1;
      DENSE = 2;
      ALL = 3;
    }
    required Sparsity sparsity = 1;
    repeated bool dense = 2 [packed = true];
    repeated uint32 sparse = 3 [packed = true];
  }

  message Diff {
    required ProductValue pos = 1;
    required ProductValue neg = 2;
    repeated uint32 tares = 3 [packed = true];
  }

  required Observed observed = 1;
  repeated bool booleans = 2 [packed = true];  // including bb
  repeated uint32 counts = 3 [packed = true];  // including dd, dpd, gp, bnb
  repeated float reals = 4 [packed = true];  // including nich
}

//----------------------------------------------------------------------------

message ProductModel {

  message Shared {
    required distributions.Clustering.PitmanYor clustering = 1;
    repeated distributions.BetaBernoulli.Shared bb = 2;
    repeated distributions.DirichletDiscrete.Shared dd = 3;  // ordered by increasing dim
    repeated distributions.DirichletProcessDiscrete.Shared dpd = 4;
    repeated distributions.GammaPoisson.Shared gp = 5;
    repeated distributions.BetaNegativeBinomial.Shared bnb = 6;
    repeated distributions.NormalInverseChiSq.Shared nich = 7;
  }

  message Group {
    required uint64 count = 1;
    repeated distributions.BetaBernoulli.Group bb = 2;
    repeated distributions.DirichletDiscrete.Group dd = 3;
    repeated distributions.DirichletProcessDiscrete.Group dpd = 4;
    repeated distributions.GammaPoisson.Group gp = 5;
    repeated distributions.BetaNegativeBinomial.Group bnb = 6;
    repeated distributions.NormalInverseChiSq.Group nich = 7;
  }
}

//----------------------------------------------------------------------------

message CrossCat {
  message Kind {
    required ProductModel.Shared product_model = 1;
    repeated uint32 featureids = 2 [packed = true];
  }

  repeated Kind kinds = 1;
  required distributions.Clustering.PitmanYor topology = 2;
  optional HyperPrior hyper_prior = 3;
}

//----------------------------------------------------------------------------

message CrossCatTree {
  message Parent {
    required uint32 id = 1;
    repeated uint32 child_ids = 2 [packed = true];
    required ProductModel.Shared shared = 3;
  }

  repeated Parent parents = 1;
  required distributions.Clustering.PitmanYor topology = 2;
  optional HyperPrior hyper_prior = 3;
}

//----------------------------------------------------------------------------

message Row {
  required uint64 id = 1;
  required ProductValue.Diff diff = 2;
}

//----------------------------------------------------------------------------

message Assignment {
  required uint64 rowid = 1;
  repeated uint32 groupids = 2 [packed = true];
}

//----------------------------------------------------------------------------

message Config
{
  message Schedule
  {
    required float extra_passes = 1;
    required float small_data_size = 2;
    required float big_data_size = 3;
    required uint32 max_reject_iters = 4;
    required float checkpoint_period_sec = 5;
  }
  message Kernels
  {
    message Cat
    {
      required uint32 empty_group_count = 1;
      required uint32 row_queue_capacity = 2;
      required uint32 parser_threads = 3;
    }
    message Hyper
    {
      required bool run = 1;
      required bool parallel = 2;
    }
    message Kind
    {
      required uint32 iterations = 1;
      required uint32 empty_kind_count = 2;
      required uint32 row_queue_capacity = 3;
      required uint32 parser_threads = 4;
      required bool score_parallel = 5;
    }

    required Cat cat = 1;
    required Hyper hyper = 2;
    required Kind kind = 3;
  }
  message Sparsify
  {
    required bool run = 1;
  }
  message PosteriorEnum
  {
    required uint32 sample_count = 1;
    required uint32 sample_skip = 2;
  }
  message Generate
  {
    required uint64 row_count = 1;
    required float density = 2;
    required uint32 sample_skip = 3;
  }
  message Query
  {
    required bool parallel = 1;
  }

  required uint64 seed = 1;
  required Schedule schedule = 2;
  required Kernels kernels = 3;
  required PosteriorEnum posterior_enum = 4;
  required Generate generate = 5;
  required float target_mem_bytes = 6;
  optional Query query = 7;
}

//----------------------------------------------------------------------------

message Checkpoint
{
  message Schedule {
    required double annealing_state = 1;
    required uint64 row_count = 2;
    required uint64 reject_iters = 3;
  };

  message StreamInterval {
    required uint64 unassigned_pos = 1;
    required uint64 assigned_pos = 2;
  };

  required bool finished = 1;
  required uint64 seed = 2;
  required uint64 tardis_iter = 3;
  required Schedule schedule = 4;
  required uint64 row_count = 5;
  required StreamInterval rows = 6;
}

//----------------------------------------------------------------------------

message LogMessage
{
  message Rusage
  {
    required uint64 max_resident_size_kb = 1;
    required double user_time_sec = 2;
    required double sys_time_sec = 3;
  }

  message Args
  {
    message Summary
    {
      required distributions.Clustering.PitmanYor model_hypers = 1;
      repeated distributions.Clustering.PitmanYor kind_hypers = 2;
      repeated uint32 feature_counts = 3 [packed = true];
      repeated uint32 category_counts = 4 [packed = true];
    }
    message Scores
    {
      optional float score = 1;
      optional float kl_divergence = 2;
      optional uint64 total_object_count = 3;
      optional uint64 assigned_object_count = 4;
      repeated float features = 5 [packed = true];
    }
    message KernelStatus
    {
      message Cat {
        required uint64 total_time = 1;
      }
      message Hyper {
        required uint64 total_time = 1;
      }
      message Kind
      {
        required uint64 total_count = 1;
        required uint64 change_count = 2;
        required uint64 birth_count = 3;
        required uint64 death_count = 4;
        required uint64 tare_time = 5;
        required uint64 score_time = 6;
        required uint64 sample_time = 7;
        required uint64 total_time = 8;
      }
      message ParCat {
        repeated uint64 times = 1 [packed = true];
        repeated uint64 counts = 2 [packed = true];
      }

      optional Cat cat = 1;
      optional Hyper hyper = 2;
      optional Kind kind = 3;
      optional ParCat parcat = 4;
    }

    optional uint32 iter = 1;
    optional Summary summary = 2;
    optional Scores scores = 3;
    optional KernelStatus kernel_status = 4;
  }

  required uint64 timestamp_usec = 1;
  required Rusage rusage = 2;
  required Args args = 3;
}

//----------------------------------------------------------------------------

message PosteriorEnum {
  message Group {
    repeated uint32 rowids = 1 [packed = true];
  }
  message Kind {
    repeated uint32 featureids = 1 [packed = true];
    repeated Group groups = 2;
  }
  message Sample {
    repeated Kind kinds = 1;
    optional float score = 2;
  }
}

//----------------------------------------------------------------------------

message Query
{
  message Sample
  {
    message Request
    {
      required ProductValue.Diff data = 1;
      required ProductValue.Observed to_sample = 2;
      required uint32 sample_count = 3;
    }
    message Response
    {
      repeated ProductValue.Diff samples = 1;
    }
  }

  message Score
  {
    message Request
    {
      required ProductValue.Diff data = 1;
    }
    message Response
    {
      required float score = 1;
    }
  }

  message Entropy
  {
    message Request
    {
      repeated ProductValue.Observed row_sets = 1;
      repeated ProductValue.Observed col_sets = 2;
      required ProductValue.Diff conditional = 3;
      required uint32 sample_count = 4;
    }
    message Response
    {
      repeated float means = 1 [packed = true];
      repeated float variances = 2 [packed = true];
    }
  }

  message ScoreDerivative
  {
    message Request
    {
      repeated ProductValue.Diff score_data = 1;
      required ProductValue.Diff update_data = 2;
      required uint32 row_limit = 3;
    }
    message Response
    {
      repeated uint64 ids = 1 [packed = true];
      repeated float score_diffs = 2 [packed = true];
    }
  }

  message Request
  {
    required string id = 1;
    optional Sample.Request sample = 2;
    optional Score.Request score = 3;
    optional Entropy.Request entropy = 4;
    optional ScoreDerivative.Request score_derivative = 5;
  }

  message Response
  {
    required string id = 1;
    repeated string error = 2;
    optional Sample.Response sample = 3;
    optional Score.Response score = 4;
    optional Entropy.Response entropy = 5;
    optional ScoreDerivative.Response score_derivative = 6;
  }
}