add sharding, search, sorting, soft removal

.github/workflows/test.yml

@@ -1,19 +1,17 @@
 name: test
 
-on: [ push ]
+on: push
 
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v4.1.1
       with:
         clean: false
-        fetch-depth: 0 # with tags
         submodules: 'recursive'
-    - uses: actions/setup-java@v1.4.3
+    - uses: actions/setup-java@v4.1.0
       with:
-        java-version: 11
-    - env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: sbt +compile +test
+        java-version: 21
+        distribution: 'zulu'
+    - run: sbt test

.gitignore

@@ -1,9 +1,5 @@
 .DS_Store
-target/
-/tmp
 /.bsp/
-.bloop
-.metals
-.vscode
-project/project
-project/metals.sbt
+/data/example-*/
+/data/test-*/
+target/

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "proto"]
 	path = deps/proto
-	url = https://github.com/zero-deps/proto
+	url = git@github.com:zero-deps/proto.git
 	branch = main

.sbtopts

@@ -1 +1 @@
--J-XX:MaxMetaspaceSize=512m
+-J-XX:MaxMetaspaceSize=1g

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 zero-deps
+Copyright (c) 2015–2024 zero-deps
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,49 +1,33 @@
-# Abstract scala type database
+# Scala Abstract Type Database
 
-![ci](https://github.com/zero-deps/kvs/workflows/ci/badge.svg)
+![Production Ready](https://img.shields.io/badge/Project%20Stage-Production%20Ready%20(main)-brightgreen.svg)
+![Development](https://img.shields.io/badge/Project%20Stage-Development%20(series/5.x)-yellowgreen.svg)
 
-Abstract Scala storage framework with high-level API for handling linked lists of polymorphic data (feeds).
+[![Documentation](https://img.shields.io/badge/documentation-pdf-yellow)](kvs.pdf)
+[![Paper](https://img.shields.io/badge/paper-pdf-lightgrey)](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf)
 
-KVS is highly available distributed (AP) strong eventual consistent (SEC) and sequentially consistent (via cluster sharding) storage. It is used for data from sport and games events. In some configurations used as distributed network file system. Also can be a generic storage for application.
+This open-source project presents an abstract storage framework in Scala, offering a high-level API tailored for managing linked lists of polymorphic data, referred to as 'feeds.' The system, known as KVS (Key-Value Storage), boasts attributes such as high availability, distributed architecture (AP), strong eventual consistency (SEC), and sequential consistency achieved through cluster sharding. Its primary application involves handling data from sports and gaming events, but it can also serve as a distributed network file system or a versatile general-purpose storage solution for various applications.
 
-Designed with various backends in mind and to work in pure JVM environment. Implementation based on top of KAI (implementation of Amazon DynamoDB in Erlang) port with modification to use akka-cluster infrastructure.
+The design philosophy behind KVS encompasses versatility, with support for multiple backend implementations and compatibility within a pure JVM environment. The implementation is grounded in the KAI framework (an Erlang-based Amazon DynamoDB implementation), adapted to utilize the pekko-cluster infrastructure.
 
-Currently main backend is RocksDB to support embedded setup alongside application. Feed API (add/entries/remove) is built on top of Key-Value API (put/get/delete).
+At its core, KVS relies on RocksDB as the primary backend, enabling seamless integration in embedded setups alongside applications. The central Feed API, facilitating operations like addition, entry retrieval, and removal, is constructed upon the foundation of the Key-Value API, which includes functions for putting, getting, and deleting data.
 
 ## Usage
 
-Add project as a git module.
+Add project as a git submodule.
 
-## Backend
+## Project Structure
 
-* Ring
-* RocksDB
-* Memory
-* FS
-* SQL
-* etc.
+* `./feed`: Introduces the Feed over Ring concept
+* `./search`: Offers Search over Ring functionality
+* `./sort`: Implements a Sorted Set on Ring
+* `./ring`: Establishes a Ring structure using Pekko Cluster
+* `./sharding`: Addresses Sequential Consistency & Cluster Sharding aspects
+* `./src`: Contains illustrative sample applications and comprehensive tests
 
-## Test
+## Test & Demo
 
 ```bash
-sbt> test
+sbt test
+sbt run
 ```
-
-## Resources
-
-### Chain Replication
-
-[Chain Replication in Theory and in Practice](http://www.snookles.com/scott/publications/erlang2010-slf.pdf)
-
-[Chain Replication for Supporting High Throughput and Availability](http://www.cs.cornell.edu/home/rvr/papers/OSDI04.pdf)
-
-[High-throughput chain replication for read-mostly workload](https://www.cs.princeton.edu/courses/archive/fall15/cos518/papers/craq.pdf)
-
-[Leveraging Sharding in the Design of Scalable Replication Protocols](https://ymsir.com/papers/sharding-socc.pdf)
-
-[Byzantine Chain Replication](http://www.cs.cornell.edu/home/rvr/newpapers/opodis2012.pdf)
-
-### Consensus Algorithm
-
-[RAFT](https://raft.github.io/raft.pdf)
-[SWIM](https://www.cs.cornell.edu/projects/Quicksilver/public_pdfs/SWIM.pdf)

build.sbt

@@ -1,53 +1,89 @@
-val scalav = "3.2.2"
-val zio = "2.0.10"
-val akka = "2.6.20"
-val rocks = "7.10.2"
-val protoj = "3.22.2"
-val lucene = "8.11.2"
+val scalav = "3.3.3"
+val zio = "2.0.21"
+val pekko = "1.0.2"
+val rocks = "8.11.3"
+val protoj = "3.25.3"
+val lucene = "9.10.0"
 
-lazy val root = project.in(file(".") ).aggregate(kvs)
+lazy val kvsroot = project.in(file(".")).settings(
+  scalaVersion := scalav
+, libraryDependencies ++= Seq(
+    "dev.zio" %% "zio-test-sbt" % zio % Test
+  , "org.apache.pekko" %% "pekko-cluster-sharding" % pekko
+  )
+, testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework")
+, scalacOptions ++= scalacOptionsCommon
+, Test / fork := true
+, run / fork := true
+, run / connectInput := true
+).dependsOn(feed, search, sort).aggregate(ring, feed, search, sort)
 
-lazy val kvs = project.in(file("kvs")).settings(
+lazy val ring = project.in(file("ring")).settings(
   scalaVersion := scalav
+, Compile / scalaSource := baseDirectory.value / "src"
 , libraryDependencies ++= Seq(
-    "com.typesafe.akka" % "akka-cluster-sharding_2.13" % akka
-  , "com.typesafe.akka" % "akka-slf4j_2.13" % akka
-  , "ch.qos.logback" % "logback-classic" % "1.4.5"
-  , "com.github.jnr" % "jnr-ffi" % "2.2.2"
-  , "org.apache.lucene" % "lucene-analyzers-common" % lucene
-  , "dev.zio" %% "zio" % zio
-  , "dev.zio" %% "zio-nio" % "2.0.0"
+    "org.apache.pekko" %% "pekko-cluster" % pekko
   , "org.rocksdb" % "rocksdbjni" % rocks
-  , "org.scalatest" %% "scalatest" % "3.2.14" % Test
-  , "com.typesafe.akka" % "akka-testkit_2.13" % akka % Test
+  , "dev.zio" %% "zio" % zio
   )
-, scalacOptions ++= scalacOptions3
+, scalacOptions ++= scalacOptionsCommon diff Seq("-language:strictEquality") :+ "-nowarn"
 ).dependsOn(proto)
 
-lazy val proto = project.in(file("deps/proto/proto")).settings(
+lazy val sharding = project.in(file("sharding")).settings(
   scalaVersion := scalav
-, crossScalaVersions := scalav :: Nil
-, libraryDependencies += "com.google.protobuf" % "protobuf-java" % protoj
-).dependsOn(protoops)
+, Compile / scalaSource := baseDirectory.value / "src"
+, libraryDependencies ++= Seq(
+    "org.apache.pekko" %% "pekko-cluster-sharding" % pekko
+  )
+, scalacOptions ++= scalacOptionsCommon
+).dependsOn(ring)
+
+lazy val feed = project.in(file("feed")).settings(
+  scalaVersion := scalav
+, Compile / scalaSource := baseDirectory.value / "src"
+, libraryDependencies ++= Seq(
+    "dev.zio" %% "zio-streams" % zio
+  )
+, scalacOptions ++= scalacOptionsCommon
+).dependsOn(sharding)
+
+lazy val search = project.in(file("search")).settings(
+  scalaVersion := scalav
+, Compile / scalaSource := baseDirectory.value / "src"
+, libraryDependencies ++= Seq(
+    "dev.zio" %% "zio-streams" % zio
+  , "org.apache.lucene" % "lucene-analysis-common" % lucene
+  )
+, scalacOptions ++= scalacOptionsCommon
+).dependsOn(sharding)
+
+lazy val sort = project.in(file("sort")).settings(
+  scalaVersion := scalav
+, Compile / scalaSource := baseDirectory.value / "src"
+, libraryDependencies ++= Seq(
+    "dev.zio" %% "zio-streams" % zio
+  )
+, scalacOptions ++= scalacOptionsCommon
+).dependsOn(sharding)
 
-lazy val protoops = project.in(file("deps/proto/ops")).settings(
+lazy val proto = project.in(file("deps/proto/proto")).settings(
   scalaVersion := scalav
 , crossScalaVersions := scalav :: Nil
+, libraryDependencies ++= Seq(
+    "com.google.protobuf" % "protobuf-java" % protoj
+  )
+, scalacOptions ++= scalacOptionsCommon diff Seq("-language:strictEquality") :+ "-Xcheck-macros"
 ).dependsOn(protosyntax)
 
 lazy val protosyntax = project.in(file("deps/proto/syntax")).settings(
   scalaVersion := scalav
 , crossScalaVersions := scalav :: Nil
+, scalacOptions ++= scalacOptionsCommon diff Seq("-language:strictEquality")
 )
 
-val scalacOptions3 = Seq(
-  "-source:future", "-nowarn"
+val scalacOptionsCommon = Seq(
+  "-Yexplicit-nulls"
 , "-language:strictEquality"
-, "-language:postfixOps"
-, "-Yexplicit-nulls"
-, "-encoding", "UTF-8"
 )
 
-turbo := true
-useCoursier := true
 Global / onChangedBuildSource := ReloadOnSourceChanges