feat(cubesqlplanner): Base joins support

packages/cubejs-schema-compiler/package.json

@@ -23,6 +23,7 @@
     "watch": "tsc -w",
     "test": "npm run unit && npm run integration",
     "unit": "TZ=UTC jest --coverage dist/test/unit",
+    "tst": "jest",
     "integration": "TZ=UTC jest dist/test/integration/*",
     "integration:mssql": "TZ=UTC jest dist/test/integration/mssql",
     "integration:mysql": "TZ=UTC jest dist/test/integration/mysql",

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -262,6 +262,7 @@ export class BaseQuery {
     this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters);
 
     this.join = this.joinGraph.buildJoin(this.allJoinHints);
+
     this.cubeAliasPrefix = this.options.cubeAliasPrefix;
     this.preAggregationsSchemaOption = this.options.preAggregationsSchema ?? DEFAULT_PREAGGREGATIONS_SCHEMA;
     this.externalQueryClass = this.options.externalQueryClass;
@@ -609,11 +610,18 @@ export class BaseQuery {
     const queryParams = {
       measures: this.options.measures,
       dimensions: this.options.dimensions,
+      timeDimensions: this.options.timeDimensions,
+      timezone: this.options.timezone,
       joinRoot: this.join.root,
+      joinGraph: this.joinGraph,
       cubeEvaluator: this.cubeEvaluator,
+      filters: this.options.filters,
+      baseTools: this,
 
     };
     const res = nativeBuildSqlAndParams(queryParams);
+    // FIXME
+    res[1] = [...res[1]];
     return res;
   }
 
@@ -2144,7 +2152,9 @@ export class BaseQuery {
   }
 
   measureSql(measure) {
-    return this.evaluateSymbolSql(measure.path()[0], measure.path()[1], measure.measureDefinition());
+    const res = this.evaluateSymbolSql(measure.path()[0], measure.path()[1], measure.measureDefinition());
+
+    return res;
   }
 
   autoPrefixWithCubeName(cubeName, sql, isMemberExpr = false) {
@@ -2369,6 +2379,29 @@ export class BaseQuery {
     return `${cubeName}.${primaryKey}`;
   }
 
+  resolveSymbolsCallDeps(cubeName, sql) {
+    const self = this;
+    const { cubeEvaluator } = this;
+    const deps = [];
+    cubeEvaluator.resolveSymbolsCall(sql, (name) => {
+      const resolvedSymbol = cubeEvaluator.resolveSymbol(
+        cubeName,
+        name
+      );
+      if (resolvedSymbol._objectWithResolvedProperties) {
+        return resolvedSymbol;
+      }
+      return '';
+    }, {
+      depsResolveFn: (name, parent) => {
+        deps.push({ name, parent });
+        return deps.length - 1;
+      },
+      contextSymbols: this.parametrizedContextSymbols(),
+    });
+    return deps;
+  }
+
   evaluateSql(cubeName, sql, options) {
     options = options || {};
     const self = this;
@@ -3233,6 +3266,21 @@ export class BaseQuery {
         ilike: '{{ expr }} {% if negated %}NOT {% endif %}ILIKE {{ pattern }}',
         like_escape: '{{ like_expr }} ESCAPE {{ escape_char }}',
       },
+      filters: {
+        equals: '{{ column }} = {{ value }}{{ is_null_check }}',
+        not_equals: '{{ column }} <> {{ value }}{{ is_null_check }}',
+        or_is_null_check: ' OR {{ column }} IS NULL',
+        set_where: '{{ column }} IS NOT NULL',
+        not_set_where: '{{ column }} IS NULL',
+        in: '{{ column }} IN ({{ values_concat }}){{ is_null_check }}',
+        not_in: '{{ column }} NOT IN ({{ values_concat }}){{ is_null_check }}',
+        time_range_filter: '{{ column }} >= {{ from_timestamp }} AND {{ column }} <= {{ to_timestamp }}',
+        gt: '{{ column }} > {{ param }}',
+        gte: '{{ column }} >= {{ param }}',
+        lt: '{{ column }} < {{ param }}',
+        lte: '{{ column }} <= {{ param }}'
+
+      },
       quotes: {
         identifiers: '"',
         escape: '""'
@@ -3661,6 +3709,10 @@ export class BaseQuery {
     };
   }
 
+  securityContextForRust() {
+    return this.contextSymbolsProxy(this.contextSymbols.securityContext);
+  }
+
   contextSymbolsProxy(symbols) {
     return BaseQuery.contextSymbolsProxyFrom(symbols, this.paramAllocator.allocateParam.bind(this.paramAllocator));
   }

packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts

@@ -49,7 +49,8 @@ export class BaseTimeDimension extends BaseFilter {
       return null;
     }
 
-    return super.selectColumns();
+    const res = super.selectColumns();
+    return res;
   }
 
   public hasNoRemapping() {

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -462,9 +462,10 @@ export class CubeEvaluator extends CubeSymbols {
 
   public isInstanceOfType(type: 'measures' | 'dimensions' | 'segments', path: string | string[]): boolean {
     const cubeAndName = Array.isArray(path) ? path : path.split('.');
-    return this.evaluatedCubes[cubeAndName[0]] &&
+    const symbol = this.evaluatedCubes[cubeAndName[0]] &&
       this.evaluatedCubes[cubeAndName[0]][type] &&
       this.evaluatedCubes[cubeAndName[0]][type][cubeAndName[1]];
+    return symbol !== undefined;
   }
 
   public byPathAnyType(path: string[]) {

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -477,6 +477,7 @@ export class CubeSymbols {
     const funcDefinition = func.toString();
     if (!this.funcArgumentsValues[funcDefinition]) {
       const match = funcDefinition.match(FunctionRegex);
+
       if (match && (match[1] || match[2] || match[3])) {
         this.funcArgumentsValues[funcDefinition] = (match[1] || match[2] || match[3]).split(',').map(s => s.trim());
       } else if (match) {
@@ -493,8 +494,81 @@ export class CubeSymbols {
     return joinHints;
   }
 
+  resolveSymbolsCallDeps(cubeName, sql) {
+    try {
+      return this.resolveSymbolsCallDeps2(cubeName, sql);
+    } catch (e) {
+      console.log(e);
+      return [];
+    }
+  }
+
+  resolveSymbolsCallDeps2(cubeName, sql) {
+    const deps = [];
+    this.resolveSymbolsCall(sql, (name) => {
+      deps.push({ name, undefined });
+      const resolvedSymbol = this.resolveSymbol(
+        cubeName,
+        name
+      );
+      if (resolvedSymbol._objectWithResolvedProperties) {
+        return resolvedSymbol;
+      }
+      return '';
+    }, {
+      depsResolveFn: (name, parent) => {
+        deps.push({ name, parent });
+        return deps.length - 1;
+      },
+      currResolveIndexFn: () => deps.length - 1,
+      contextSymbols: this.depsContextSymbols(),
+
+    });
+    return deps;
+  }
+
+  depsContextSymbols() {
+    return Object.assign({
+      filterParams: this.filtersProxyDep(),
+      filterGroup: this.filterGroupFunctionDep(),
+      securityContext: BaseQuery.contextSymbolsProxyFrom({}, (param) => param)
+    });
+  }
+
+  filtersProxyDep() {
+    return new Proxy({}, {
+      get: (target, name) => {
+        if (name === '_objectWithResolvedProperties') {
+          return true;
+        }
+        // allFilters is null whenever it's used to test if the member is owned by cube so it should always render to `1 = 1`
+        // and do not check cube validity as it's part of compilation step.
+        const cubeName = this.cubeNameFromPath(name);
+        return new Proxy({ cube: cubeName }, {
+          get: (cubeNameObj, propertyName) => ({
+            filter: (column) => ({
+              __column() {
+                return column;
+              },
+              __member() {
+                return this.pathFromArray([cubeNameObj.cube, propertyName]);
+              },
+              toString() {
+                return '';
+              }
+            })
+          })
+        });
+      }
+    });
+  }
+
+  filterGroupFunctionDep() {
+    return (...filterParamArgs) => '';
+  }
+
   resolveSymbol(cubeName, name) {
-    const { sqlResolveFn, contextSymbols, collectJoinHints } = this.resolveSymbolsCallContext || {};
+    const { sqlResolveFn, contextSymbols, collectJoinHints, depsResolveFn, currResolveIndexFn } = this.resolveSymbolsCallContext || {};
 
     if (name === 'USER_CONTEXT') {
       throw new Error('Support for USER_CONTEXT was removed, please migrate to SECURITY_CONTEXT.');
@@ -528,8 +602,14 @@ export class CubeSymbols {
           name
         );
       }
+    } else if (depsResolveFn) {
+      if (cube) {
+        const newCubeName = this.isCurrentCube(name) ? cubeName : name;
+        const parentIndex = currResolveIndexFn();
+        cube = this.cubeDependenciesProxy(parentIndex, newCubeName);
+        return cube;
+      }
     }
-
     return cube || (this.symbols[cubeName] && this.symbols[cubeName][name]);
   }
 
@@ -623,6 +703,44 @@ export class CubeSymbols {
     return cube && cube[dimName] && cube[dimName][gr] && cube[dimName][gr][granName];
   }
 
+  cubeDependenciesProxy(parentIndex, cubeName) {
+    const self = this;
+    const { depsResolveFn } = self.resolveSymbolsCallContext || {};
+    return new Proxy({}, {
+      get: (v, propertyName) => {
+        if (propertyName === '__cubeName') {
+          depsResolveFn('__cubeName', parentIndex);
+          return cubeName;
+        }
+        const cube = self.symbols[cubeName];
+
+        if (propertyName === 'toString') {
+          depsResolveFn('toString', parentIndex);
+          return () => '';
+        }
+        if (propertyName === 'sql') {
+          depsResolveFn('sql', parentIndex);
+          return () => '';
+        }
+        if (propertyName === '_objectWithResolvedProperties') {
+          return true;
+        }
+        if (cube[propertyName]) {
+          depsResolveFn(propertyName, parentIndex);
+          return '';
+        }
+        if (self.symbols[propertyName]) {
+          const index = depsResolveFn(propertyName, parentIndex);
+          return this.cubeDependenciesProxy(index, propertyName);
+        }
+        if (typeof propertyName === 'string') {
+          throw new UserError(`${cubeName}.${propertyName} cannot be resolved. There's no such member or cube.`);
+        }
+        return undefined;
+      }
+    });
+  }
+
   isCurrentCube(name) {
     return CURRENT_CUBE_CONSTANTS.indexOf(name) >= 0;
   }

packages/cubejs-schema-compiler/test/integration/mssql/MSSqlDbRunner.js

@@ -84,7 +84,7 @@ export class MSSqlDbRunner extends BaseDbRunner {
     (2, 1, 2),
     (3, 3, 6)
     `);
-    await query(`CREATE TABLE ##numbers (num INT);`);
+    await query('CREATE TABLE ##numbers (num INT);');
     await query(`
     INSERT INTO ##numbers (num) VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
                                   (10), (11), (12), (13), (14), (15), (16), (17), (18), (19),

packages/cubejs-schema-compiler/test/integration/mysql/MySqlDbRunner.js

@@ -73,7 +73,7 @@ export class MySqlDbRunner extends BaseDbRunner {
     (2, 1, 2),
     (3, 3, 6)
     `);
-    await query(`CREATE TEMPORARY TABLE numbers (num INT);`);
+    await query('CREATE TEMPORARY TABLE numbers (num INT);');
     await query(`
     INSERT INTO numbers (num) VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
                                   (10), (11), (12), (13), (14), (15), (16), (17), (18), (19),

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts

@@ -401,8 +401,6 @@ describe('SQL Generation', () => {
       timezone: 'America/Los_Angeles'
     });
 
-    console.log(query.buildSqlAndParams());
-
     return dbRunner.testQuery(query.buildSqlAndParams()).then(res => {
       console.log(JSON.stringify(res));
       expect(res).toEqual(