Data generation logic

This data all fits into memory in one run and takes less than a minute
to load into snowflake so it will completely reset nightly.

Overview of (max) granularity of each table

- One time:
-- Databases, Tables
- Daily:
-- DatabaseStorageUsageHistory, StorageUsage
- Hourly:
-- LoadHistory, LoginHistory
- Minute:
-- QueryHistory

data_rows/DatabaseStorageUsageHistory.js

@@ -0,0 +1,89 @@
+import DataRow from "./DataRow";
+import { TYPES } from "./Types";
+import { randomFromArray } from "../helpers";
+
+class InitializeError extends Error {}
+
+/**
+ * DATABASE_STORAGE_USAGE_HISTORY View
+ *
+ * https://docs.snowflake.net/manuals/sql-reference/account-usage/database_storage_usage_history.html
+ */
+export class DatabaseStorageUsageHistory extends DataRow {
+  constructor(databaseID, databaseName, date) {
+    super();
+    if (!this.constructor._sizes) {
+      throw new InitializeError(
+        `Must call ${this.constructor.name}.initialize() first`
+      );
+    }
+    this.USAGE_DATE = date.toISOString();
+    this.DATABASE_ID = databaseID;
+    this.DATABASE_NAME = databaseName;
+    this._setAverageDatabaseBytes();
+    this._setAverageFailsafeBytes();
+  }
+
+  _setAverageDatabaseBytes() {
+    let size = this.constructor._sizes[this.DATABASE_NAME];
+    if (Math.random() < 0.1) {
+      const changeDegree = Math.random() * (0.2 - 0.05) + 0.05;
+      const change = Math.round(size * changeDegree);
+      if (Math.random() < 0.5) {
+        size -= change;
+      } else {
+        size += change;
+      }
+    }
+    this.AVERAGE_DATABASE_BYTES = size;
+  }
+
+  _setAverageFailsafeBytes() {
+    this.AVERAGE_FAILSAFE_BYTES = 0;
+    if (Math.random() < 0.1) {
+      this.AVERAGE_FAILSAFE_BYTES = Math.round(
+        Math.random() * this.AVERAGE_DATABASE_BYTES
+      );
+    }
+  }
+
+  static initialize(dbs) {
+    if (this._sizes) {
+      throw new InitializeError(
+        `Must call ${this.name}.initialize() only once`
+      );
+    }
+    const seedSizes = [1e12, 1e13, 1e14, 1e15];
+    const sizes = {};
+    for (const db of dbs) {
+      const seed = randomFromArray(seedSizes);
+      sizes[db.DATABASE_NAME] = Math.round(Math.random() * seed);
+    }
+    this._sizes = sizes;
+  }
+
+  static types() {
+    return [
+      {
+        name: "USAGE_DATE",
+        type: TYPES.timestamp
+      },
+      {
+        name: "DATABASE_ID",
+        type: TYPES.integer
+      },
+      {
+        name: "DATABASE_NAME",
+        type: TYPES.string
+      },
+      {
+        name: "AVERAGE_DATABASE_BYTES",
+        type: TYPES.integer
+      },
+      {
+        name: "AVERAGE_FAILSAFE_BYTES",
+        type: TYPES.integer
+      }
+    ];
+  }
+}

data_rows/Databases.js

@@ -0,0 +1,40 @@
+import DataRow from "./DataRow";
+import { TYPES } from "./Types";
+
+/**
+ * DATABASES View
+ *
+ * https://docs.snowflake.net/manuals/sql-reference/account-usage/databases.html
+ */
+export class Databases extends DataRow {
+  constructor(name, id) {
+    super();
+    this.DATABASE_NAME = name;
+    this.DATABASE_ID = id;
+  }
+
+  static generate() {
+    const names = [
+      "squiggly_database",
+      "jims_database",
+      "jacksonbase",
+      "prod",
+      "staging_db",
+      "dev1"
+    ];
+    return names.map((name, idx) => new this(name, idx + 1));
+  }
+
+  static types() {
+    return [
+      {
+        name: "DATABASE_NAME",
+        type: TYPES.string
+      },
+      {
+        name: "DATABASE_ID",
+        type: TYPES.integer
+      }
+    ];
+  }
+}

data_rows/LoadHistory.js

@@ -0,0 +1,42 @@
+import DataRow from "./DataRow";
+import { TYPES } from "./Types";
+
+/**
+ * LOAD_HISTORY View
+ *
+ * https://docs.snowflake.net/manuals/sql-reference/account-usage/load_history.html
+ */
+export class LoadHistory extends DataRow {
+  constructor(tableID, date) {
+    super();
+    this.TABLE_ID = tableID;
+    this.LAST_LOAD_TIME = date.toISOString();
+    this.ROW_COUNT = Math.round(Math.random() * 100);
+    this.ERROR_COUNT = Math.random() < 0.1 ? Math.round(Math.random() * 10) : 0;
+  }
+
+  static oddsNew() {
+    return 0.2;
+  }
+
+  static types() {
+    return [
+      {
+        name: "TABLE_ID",
+        type: TYPES.integer
+      },
+      {
+        name: "LAST_LOAD_TIME",
+        type: TYPES.timestamp
+      },
+      {
+        name: "ROW_COUNT",
+        type: TYPES.integer
+      },
+      {
+        name: "ERROR_COUNT",
+        type: TYPES.integer
+      }
+    ];
+  }
+}

data_rows/LoginHistory.js

@@ -0,0 +1,54 @@
+import DataRow from "./DataRow";
+import { TYPES } from "./Types";
+import * as helpers from "../helpers";
+
+/**
+ * LOGIN_HISTORY View
+ *
+ * https://docs.snowflake.net/manuals/sql-reference/account-usage/login_history.html
+ */
+export class LoginHistory extends DataRow {
+  constructor(date) {
+    super();
+    this.EVENT_ID = helpers.getID();
+    this.EVENT_TIMESTAMP = date.toISOString();
+    const user = helpers.randomFromArray(this.constructor.getUsers());
+    this.USER_NAME = user.name;
+    this.REPORTED_CLIENT_TYPE = user.driver;
+    this.IS_SUCCESS = Math.random() < 0.9 ? "YES" : "NO";
+  }
+
+  static getUsers() {
+    return [
+      { name: "WEB_CLIENT", driver: "JAVASCRIPT_DRIVER", querySpeed: 1.2 },
+      { name: "BOB", driver: "OTHER", querySpeed: 2.0 },
+      { name: "BI_APP", driver: "JDBC_DRIVER", querySpeed: 0.8 },
+      { name: "JANE", driver: "SNOWFLAKE_UI", querySpeed: 1.0 }
+    ];
+  }
+
+  static types() {
+    return [
+      {
+        name: "EVENT_ID",
+        type: TYPES.string
+      },
+      {
+        name: "EVENT_TIMESTAMP",
+        type: TYPES.timestamp
+      },
+      {
+        name: "USER_NAME",
+        type: TYPES.string
+      },
+      {
+        name: "REPORTED_CLIENT_TYPE",
+        type: TYPES.string
+      },
+      {
+        name: "IS_SUCCESS",
+        type: TYPES.string
+      }
+    ];
+  }
+}

data_rows/QueryHistory.js

@@ -0,0 +1,147 @@
+import uuid4 from "uuid/v4";
+
+import DataRow from "./DataRow";
+import { LoginHistory } from "./LoginHistory";
+import { TYPES } from "./Types";
+import * as helpers from "../helpers";
+
+/**
+ * QUERY_HISTORY View
+ *
+ * https://docs.snowflake.net/manuals/sql-reference/account-usage/query_history.html
+ */
+export class QueryHistory extends DataRow {
+  constructor(databaseName, date) {
+    super();
+    this.QUERY_ID = uuid4();
+    this.setQueryText();
+    this.DATABASE_NAME = databaseName;
+    const queryType = helpers.randomFromArray(this._getQueryTypes());
+    this.QUERY_TYPE = queryType.type;
+    const user = helpers.randomFromArray(LoginHistory.getUsers());
+    this.USER_NAME = user.name;
+    [this.WAREHOUSE_NAME, this.WAREHOUSE_SIZE] = helpers.randomFromArray([
+      ["BIG_WH", "X-Large"],
+      ["SMALL_WH", "Small"]
+    ]);
+    this.setExecutionStatus();
+    this.START_TIME = date.toISOString();
+    this.COMPILATION_TIME = Math.round(Math.random() * 1000);
+    this.setExecutionTime(user.querySpeed, queryType.querySpeed);
+    this.QUEUED_REPAIR_TIME = 0;
+    this.QUEUED_OVERLOAD_TIME =
+      Math.random() < 0.2 ? Math.round(Math.random() * 1000) : 0;
+    this.setTransactionBlockedtime();
+  }
+
+  setQueryText() {
+    const queries = ["SHOW USERS", "SHOW WAREHOUSES", "SELECT foo FROM bar"];
+    this.QUERY_TEXT = helpers.randomFromArray(queries);
+  }
+
+  _getQueryTypes() {
+    return [
+      { type: "WITH", querySpeed: 1.2 },
+      { type: "REPLACE", querySpeed: 0.5 },
+      { type: "SHOW", querySpeed: 0.2 },
+      { type: "CREATE", querySpeed: 0.1 },
+      { type: "COPY", querySpeed: 1.5 },
+      { type: "SELECT", querySpeed: 1.9 },
+      { type: "UNKNOWN", querySpeed: 2.2 }
+    ];
+  }
+
+  setExecutionStatus() {
+    const statuses = [
+      { weight: 0.9, name: "SUCCESS" },
+      { weight: 0.31, name: "RUNNING" },
+      { weight: 0.28, name: "QUEUED" },
+      { weight: 0.24, name: "BLOCKED" },
+      { weight: 0.19, name: "RESUMING_WAREHOUSE" },
+      { weight: 0.15, name: "FAILED_WITH_ERROR" },
+      { weight: 0.1, name: "FAILED_WITH_INCIDENT" }
+    ];
+    this.EXECUTION_STATUS = helpers.randomFromArrayByWeight(statuses);
+  }
+
+  setExecutionTime(userQuerySpeed, queryTypeSpeed) {
+    const warehouseQuerySpeed = {
+      BIG_WH: 0.5,
+      SMALL_WH: 1.5
+    }[this.WAREHOUSE_NAME];
+    const factor = warehouseQuerySpeed * userQuerySpeed * queryTypeSpeed * 1000;
+    this.EXECUTION_TIME = Math.round(Math.random() * factor);
+  }
+
+  setTransactionBlockedtime() {
+    this.TRANSACTION_BLOCKED_TIME = 0;
+    if (Math.random() < 0.05) {
+      this.TRANSACTION_BLOCKED_TIME = Math.round(Math.random() * 1000);
+    }
+  }
+
+  static oddsNew() {
+    return 0.3;
+  }
+
+  static types() {
+    return [
+      {
+        name: "QUERY_ID",
+        type: TYPES.string
+      },
+      {
+        name: "QUERY_TEXT",
+        type: TYPES.string
+      },
+      {
+        name: "DATABASE_NAME",
+        type: TYPES.string
+      },
+      {
+        name: "QUERY_TYPE",
+        type: TYPES.string
+      },
+      {
+        name: "USER_NAME",
+        type: TYPES.string
+      },
+      {
+        name: "WAREHOUSE_NAME",
+        type: TYPES.string
+      },
+      {
+        name: "WAREHOUSE_SIZE",
+        type: TYPES.string
+      },
+      {
+        name: "EXECUTION_STATUS",
+        type: TYPES.string
+      },
+      {
+        name: "START_TIME",
+        type: TYPES.timestamp
+      },
+      {
+        name: "COMPILATION_TIME",
+        type: TYPES.integer
+      },
+      {
+        name: "EXECUTION_TIME",
+        type: TYPES.integer
+      },
+      {
+        name: "QUEUED_REPAIR_TIME",
+        type: TYPES.integer
+      },
+      {
+        name: "QUEUED_OVERLOAD_TIME",
+        type: TYPES.integer
+      },
+      {
+        name: "TRANSACTION_BLOCKED_TIME",
+        type: TYPES.integer
+      }
+    ];
+  }
+}