Merge branch 'finos:main' into main

.github/workflows/test-ui.yml

@@ -11,6 +11,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Install Node & dependencies
         uses: ./.github/actions/setup-vuu-ui
+      - run: cd ./vuu-ui && npm run build:worker
       - run: cd ./vuu-ui && npm run lint
       - run: cd ./vuu-ui && npm run typecheck
 
@@ -23,57 +24,28 @@ jobs:
         uses: ./.github/actions/setup-vuu-ui
       - run: cd ./vuu-ui && npm run test:vite
 
-  playwright:
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Node & dependencies
-        uses: ./.github/actions/setup-vuu-ui
-      - name: Install Playwright Chromium Browser
-        run: cd ./vuu-ui && npx playwright install --with-deps chromium
-      - name: Build Showcase
-        run: cd ./vuu-ui && npm run showcase:build
-      - name: Run tests
-        run: |
-          cd ./vuu-ui && 
-          npm run test:playwright
-      - uses: actions/upload-artifact@v4
-        if: ${{ !cancelled() }}
-        with:
-          name: playwright-report
-          path: ./vuu-ui/playwright/reports/
-          retention-days: 10
-
-  cypress-e2e:
-    # As a third party action, cypress-io is pinned to a full length commit SHA for security purposes.
-    # This is also a requirement for the semgrep (static code analysis) scan to pass.
-    # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-third-party-actions
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Node & dependencies
-        uses: ./.github/actions/setup-vuu-ui
-      - name: Run end-to-end tests in Chrome
-        uses: cypress-io/github-action@bd9dda317ed2d4fbffc808ba6cdcd27823b2a13b
-        with:
-          install: false
-          working-directory: ./vuu-ui
-          browser: chrome
-          build: npm run showcase:build
-          start: npm run showcase:preview
-          wait-on: "http://localhost:4173"
-      - name: Run end-to-end tests in Edge
-        uses: cypress-io/github-action@bd9dda317ed2d4fbffc808ba6cdcd27823b2a13b
-        with:
-          install: false
-          working-directory: ./vuu-ui
-          browser: edge
-          build: npm run showcase:build
-          start: npm run showcase:preview
-          wait-on: "http://localhost:4173"
+  # playwright:
+  #   timeout-minutes: 10
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #     - name: Install Node & dependencies
+  #       uses: ./.github/actions/setup-vuu-ui
+  #     - name: Install Playwright Chromium Browser
+  #       run: cd ./vuu-ui && npx playwright install --with-deps chromium
+  #     - name: Build Showcase
+  #       run: cd ./vuu-ui && npm run showcase:build
+  #     - name: Run tests
+  #       run: |
+  #         cd ./vuu-ui &&
+  #         npm run test:playwright
+  #     - uses: actions/upload-artifact@v4
+  #       if: ${{ !cancelled() }}
+  #       with:
+  #         name: playwright-report
+  #         path: ./vuu-ui/playwright/reports/
+  #         retention-days: 10
 
   cypress-component:
     runs-on: ubuntu-latest
@@ -82,6 +54,8 @@ jobs:
         uses: actions/checkout@v3
       - name: Install Node & dependencies
         uses: ./.github/actions/setup-vuu-ui
+      - name: Build Worker
+        run: cd ./vuu-ui && npm run build:worker
       - name: Run component tests in Chrome
         uses: cypress-io/github-action@bd9dda317ed2d4fbffc808ba6cdcd27823b2a13b
         with:

.gitignore

@@ -38,4 +38,4 @@ website-desktop/package-lock.json
 
 # Generated docs
 docs/contributing.md
-vuu-ui/packages/vuu-data/src/inlined-worker.js
+vuu-ui/packages/vuu-data-remote/src/inlined-worker.js

.semgrepignore

@@ -26,8 +26,10 @@ vuu-ui/sample-apps/app-vuu-basket-trader/scripts/build.mjs
 vuu-ui/sample-apps/app-vuu-example/scripts/build.mjs
 vuu-ui/showcase/scripts/build-docs.mjs
 vuu-ui/showcase/scripts/build.mjs
+vuu-ui/showcase/scripts/build-rollup.mjs
 vuu-ui/showcase/scripts/parse-stories.mjs
 vuu-ui/showcase/scripts/utils.mjs
+vuu-ui/scripts/build-rollup.mjs
 vuu-ui/scripts/utils.mjs
 vuu-ui/tools/vuu-showcase/src/showcase-utils.ts
 example/order/src/main/scala/org/finos/vuu/order/oms/impl/InMemOmsApi.scala

docs/rpc/service.md

@@ -26,13 +26,13 @@ object TypeAheadModule extends DefaultModule {
 
   def apply()(implicit clock: Clock, lifecycle: LifecycleContainer): ViewServerModule = {
     ModuleFactory.withNamespace(NAME)
-      .addRpcHandler(server => new TypeAheadRpcHandlerImpl(server.tableContainer))
+      .addRpcHandler(server => new GenericTypeAheadRpcHandler(server.tableContainer))
       .asModule()
   }
 }
 ```
 
-You can see we've defined an RpcHandler called TypeAheadRpcHandlerImpl, which implements the interface:
+You can see we've defined an RpcHandler called GenericTypeAheadRpcHandler, which implements the interface:
 
 ```scala
 trait TypeAheadRpcHandler{

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/IgniteOrderStore.scala

@@ -6,7 +6,9 @@ import org.apache.ignite.cache.query._
 import org.apache.ignite.cluster.ClusterState
 import org.apache.ignite.{IgniteCache, Ignition}
 import org.finos.vuu.core.module.simul.model.{ChildOrder, OrderStore, ParentOrder}
-import org.finos.vuu.example.ignite.utils.getListToObjectConverter
+import org.finos.vuu.example.ignite.schema.ChildOrderSchema.toChildOrder
+import org.finos.vuu.feature.ignite.IgniteSqlQuery
+import org.finos.vuu.feature.ignite.IgniteSqlQuery.QuerySeparator
 
 import java.util
 import scala.collection.mutable
@@ -87,27 +89,32 @@ class IgniteOrderStore(private val parentOrderCache: IgniteCache[Int, ParentOrde
 
   }
 
-  def getCount(sqlFilterQueries: String): Long = {
+  def getCount(filterSql: IgniteSqlQuery): Long = {
     //todo should this be COUNT_BIG?
-    val whereClause = if(sqlFilterQueries == null || sqlFilterQueries.isEmpty) "" else s" where $sqlFilterQueries"
-    val query = new SqlFieldsQuery(s"select COUNT(1) from ChildOrder$whereClause")
-    val cursor = childOrderCache.query(query)
+    val whereClause = if (filterSql.isEmpty) filterSql else filterSql.prependSql("WHERE", QuerySeparator.SPACE)
+    val query = IgniteSqlQuery("SELECT COUNT(1) FROM ChildOrder").appendQuery(whereClause, QuerySeparator.SPACE)
 
-    val countValue = cursor.getAll().get(0).get(0)
+    val cursor = childOrderCache.query(query.buildFieldsQuery())
+    val countValue = cursor.getAll.get(0).get(0)
     val totalCount = countValue.asInstanceOf[Long]
 
-    logger.info(s"Ignite returned total count of $totalCount for ChildOrder with filter $sqlFilterQueries")
+    logger.info(s"Ignite returned total count of `$totalCount` for ChildOrder with filter `$filterSql`")
     totalCount
   }
 
-  def findChildOrder(sqlFilterQueries: String, sqlSortQueries: String, rowCount: Int, startIndex: Long): Iterator[ChildOrder] = {
-    val whereClause = if(sqlFilterQueries == null || sqlFilterQueries.isEmpty) "" else s" where $sqlFilterQueries"
-    val orderByClause = if(sqlSortQueries == null || sqlSortQueries.isEmpty) " order by id" else s" order by $sqlSortQueries"
-    val query = new SqlFieldsQuery(s"select * from ChildOrder$whereClause$orderByClause limit ? offset ?")
-    query.setArgs(rowCount, startIndex)
+  def findChildOrder(filterSql: IgniteSqlQuery, sortSql: IgniteSqlQuery, rowCount: Int, startIndex: Long): Iterator[ChildOrder] = {
+    val whereClause = if (filterSql.isEmpty) filterSql else filterSql.prependSql("WHERE", QuerySeparator.SPACE)
+    val orderByClause = if (sortSql.isEmpty) IgniteSqlQuery("ORDER BY id") else sortSql.prependSql("ORDER BY", QuerySeparator.SPACE)
+    val limitAndOffsetClause = IgniteSqlQuery("limit ? offset ?", List(rowCount, startIndex))
+
+    val query = IgniteSqlQuery("SELECT * FROM ChildOrder")
+      .appendQuery(whereClause, QuerySeparator.SPACE)
+      .appendQuery(orderByClause, QuerySeparator.SPACE)
+      .appendQuery(limitAndOffsetClause, QuerySeparator.SPACE)
 
-    val results = childOrderCache.query(query).asScala.iterator.map(i => toChildOrder(i.asScala.toList))
-    logger.info(s"Loaded Ignite ChildOrder for $rowCount rows, from index : $startIndex where $whereClause order by $sqlSortQueries")
+    val results = childOrderCache.query(query.buildFieldsQuery()).asScala.iterator.map(i => toChildOrder(i.asScala.toList))
+    logger.info(s"Loaded Ignite ChildOrder for $rowCount rows, from index : $startIndex with " +
+      s"WHERE CLAUSE: `$whereClause` | ORDER BY CLAUSE: `$orderByClause`")
 
     results
   }
@@ -136,8 +143,6 @@ class IgniteOrderStore(private val parentOrderCache: IgniteCache[Int, ParentOrde
       .map(x => x.getValue)
   }
 
-  private def toChildOrder = getListToObjectConverter[ChildOrder](ChildOrder)
-
   private def mapToString(results: FieldsQueryCursor[util.List[_]]): (Int, ListBuffer[String]) = {
     var counter = 0
     val buffer = mutable.ListBuffer[String]()

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/module/IgniteOrderDataModule.scala

@@ -5,7 +5,7 @@ import org.finos.toolbox.time.Clock
 import org.finos.vuu.api.ViewPortDef
 import org.finos.vuu.core.module.{DefaultModule, ModuleFactory, TableDefContainer, ViewServerModule}
 import org.finos.vuu.core.table.{Column, Columns}
-import org.finos.vuu.example.ignite.{IgniteColumnValueProvider, IgniteOrderStore}
+import org.finos.vuu.example.ignite.IgniteOrderStore
 import org.finos.vuu.example.ignite.provider.IgniteOrderDataProvider
 import org.finos.vuu.net.rpc.RpcHandler
 import org.finos.vuu.plugin.virtualized.api.VirtualizedSessionTableDef

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/provider/IgniteOrderDataProvider.scala

@@ -8,17 +8,20 @@ import org.finos.vuu.example.ignite.module.IgniteOrderDataModule
 import org.finos.vuu.example.ignite.provider.IgniteOrderDataProvider.columnNameByExternalField
 import org.finos.vuu.example.ignite.query.IndexCalculator
 import org.finos.vuu.example.ignite.schema.ChildOrderSchema
+import org.finos.vuu.feature.ignite.IgniteSqlQuery
 import org.finos.vuu.plugin.virtualized.table.{VirtualizedRange, VirtualizedSessionTable, VirtualizedViewPortKeys}
 import org.finos.vuu.provider.VirtualizedProvider
-import org.finos.vuu.util.schema.SchemaMapper
+import org.finos.vuu.util.schema.SchemaMapperBuilder
 import org.finos.vuu.viewport.ViewPort
 
 import java.util.concurrent.atomic.AtomicInteger
 
 class IgniteOrderDataProvider(final val igniteStore: IgniteOrderStore)
                              (implicit clock: Clock) extends VirtualizedProvider with StrictLogging {
 
-  private val schemaMapper = SchemaMapper(ChildOrderSchema.schema, IgniteOrderDataModule.columns, columnNameByExternalField)
+  private val schemaMapper = SchemaMapperBuilder(ChildOrderSchema.schema, IgniteOrderDataModule.columns)
+    .withFieldsMap(columnNameByExternalField)
+    .build()
   private val dataQuery = IgniteOrderDataQuery(igniteStore, schemaMapper)
   private val indexCalculator = IndexCalculator(extraRowsCount = 5000)
 
@@ -50,25 +53,17 @@ class IgniteOrderDataProvider(final val igniteStore: IgniteOrderStore)
     viewPort.setKeys(new VirtualizedViewPortKeys(internalTable.primaryKeys))
   }
 
-  private def getTotalSize(filter: String): Long =
-      igniteStore.getCount(filter)
+  private def getTotalSize(filterSql: IgniteSqlQuery): Long = igniteStore.getCount(filterSql)
 
   private def tableUpdater(table: VirtualizedSessionTable): (Int, Map[String, Any]) => Unit = {
     val keyField = table.tableDef.keyField
-    def hasRowChangedAtIndex = getHasRowChanged(table)
 
-    (index, rowMap) => {
+    (idx, rowMap) => {
       val newRow = RowWithData(rowMap(keyField).toString, rowMap)
-      if (hasRowChangedAtIndex(index, newRow)) table.processUpdateForIndex(index, newRow.key, newRow, clock.now())
+      if (table.hasRowChangedAtIndex(idx, newRow)) table.processUpdateForIndex(idx, newRow.key, newRow, clock.now())
     }
   }
 
-  private def getHasRowChanged(table: VirtualizedSessionTable) = (index: Int, newRow: RowWithData) => {
-    val existingKeyAtIndex = table.primaryKeys.get(index)
-    val existingRow = table.pullRow(existingKeyAtIndex)
-    !existingRow.equals(newRow)
-  }
-
   override def subscribe(key: String): Unit = {}
 
   override def doStart(): Unit = {}

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/provider/IgniteOrderDataQuery.scala

@@ -3,7 +3,7 @@ package org.finos.vuu.example.ignite.provider
 import org.finos.vuu.core.module.simul.model.ChildOrder
 import org.finos.vuu.core.sort.ModelType.SortSpecInternal
 import org.finos.vuu.example.ignite.IgniteOrderStore
-import org.finos.vuu.feature.ignite.FilterAndSortSpecToSql
+import org.finos.vuu.feature.ignite.{FilterAndSortSpecToSql, IgniteSqlQuery}
 import org.finos.vuu.net.FilterSpec
 import org.finos.vuu.util.schema.SchemaMapper
 
@@ -12,7 +12,7 @@ class IgniteOrderDataQuery private (private val igniteOrderStore: IgniteOrderSto
 
   private val filterAndSortSpecToSql = FilterAndSortSpecToSql(schemaMapper)
 
-  def getFilterSql(filterSpec: FilterSpec): String =
+  def getFilterSql(filterSpec: FilterSpec): IgniteSqlQuery =
     filterAndSortSpecToSql.filterToSql(filterSpec)
 
   def fetch(filterSpec: FilterSpec, sortSpec: SortSpecInternal, startIndex: Long, rowCount: Int): Iterator[ChildOrder] = {

example/apache-ignite/src/main/scala/org/finos/vuu/example/ignite/schema/ChildOrderSchema.scala

@@ -1,16 +1,27 @@
 package org.finos.vuu.example.ignite.schema
 
+import com.fasterxml.jackson.databind.json.JsonMapper
+import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule}
 import org.apache.ignite.cache.QueryEntity
 import org.finos.vuu.core.module.simul.model.ChildOrder
 import org.finos.vuu.feature.ignite.utils.buildQueryEntity
 import org.finos.vuu.util.schema.{ExternalEntitySchema, ExternalEntitySchemaBuilder}
 
 object ChildOrderSchema {
   val schema: ExternalEntitySchema = ExternalEntitySchemaBuilder()
-    .withCaseClass[ChildOrder]
+    .withEntity(classOf[ChildOrder])
     .withIndex("PARENTID_IDX", List("parentId"))
     .withIndex("CHILDID_IDX", List("id"))
     .build()
 
   val queryEntity: QueryEntity = buildQueryEntity(schema, classOf[Int], classOf[ChildOrder])
+
+  object toChildOrder {
+    private val mapper: JsonMapper with ClassTagExtensions = JsonMapper
+      .builder()
+      .addModule(DefaultScalaModule)
+      .build() :: ClassTagExtensions
+
+    def apply(l: List[_]): ChildOrder = mapper.convertValue(schema.toMap(l), classOf[ChildOrder])
+  }
 }