diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82bc673e..4eb5f367 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,19 +1,17 @@ name: test -on: [ push ] +on: push jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4.1.1 with: clean: false - fetch-depth: 0 # with tags submodules: 'recursive' - - uses: actions/setup-java@v1.4.3 + - uses: actions/setup-java@v4.1.0 with: - java-version: 11 - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: sbt +compile +test + java-version: 21 + distribution: 'zulu' + - run: sbt test diff --git a/.gitignore b/.gitignore index 8debe97e..eeb153cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,5 @@ .DS_Store -target/ -/tmp /.bsp/ -.bloop -.metals -.vscode -project/project -project/metals.sbt \ No newline at end of file +/data/example-*/ +/data/test-*/ +target/ diff --git a/.gitmodules b/.gitmodules index 55c94b7a..0505b9b2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "proto"] path = deps/proto - url = https://github.com/zero-deps/proto + url = git@github.com:zero-deps/proto.git branch = main diff --git a/.sbtopts b/.sbtopts index 559ca2bf..ca8c8341 100644 --- a/.sbtopts +++ b/.sbtopts @@ -1 +1 @@ --J-XX:MaxMetaspaceSize=512m +-J-XX:MaxMetaspaceSize=1g diff --git a/LICENSE b/LICENSE index 2a026d2a..2614b4da 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 zero-deps +Copyright (c) 2015–2024 zero-deps Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 9aa1e8e0..45b46319 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,33 @@ -# Abstract scala type database +# Scala Abstract Type Database -![ci](https://github.com/zero-deps/kvs/workflows/ci/badge.svg) +![Production Ready](https://img.shields.io/badge/Project%20Stage-Production%20Ready%20(main)-brightgreen.svg) +![Development](https://img.shields.io/badge/Project%20Stage-Development%20(series/5.x)-yellowgreen.svg) -Abstract Scala storage framework with high-level API for handling linked lists of polymorphic data (feeds). +[![Documentation](https://img.shields.io/badge/documentation-pdf-yellow)](kvs.pdf) +[![Paper](https://img.shields.io/badge/paper-pdf-lightgrey)](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf) -KVS is highly available distributed (AP) strong eventual consistent (SEC) and sequentially consistent (via cluster sharding) storage. It is used for data from sport and games events. In some configurations used as distributed network file system. Also can be a generic storage for application. +This open-source project presents an abstract storage framework in Scala, offering a high-level API tailored for managing linked lists of polymorphic data, referred to as 'feeds.' The system, known as KVS (Key-Value Storage), boasts attributes such as high availability, distributed architecture (AP), strong eventual consistency (SEC), and sequential consistency achieved through cluster sharding. Its primary application involves handling data from sports and gaming events, but it can also serve as a distributed network file system or a versatile general-purpose storage solution for various applications. -Designed with various backends in mind and to work in pure JVM environment. Implementation based on top of KAI (implementation of Amazon DynamoDB in Erlang) port with modification to use akka-cluster infrastructure. +The design philosophy behind KVS encompasses versatility, with support for multiple backend implementations and compatibility within a pure JVM environment. The implementation is grounded in the KAI framework (an Erlang-based Amazon DynamoDB implementation), adapted to utilize the pekko-cluster infrastructure. -Currently main backend is RocksDB to support embedded setup alongside application. Feed API (add/entries/remove) is built on top of Key-Value API (put/get/delete). +At its core, KVS relies on RocksDB as the primary backend, enabling seamless integration in embedded setups alongside applications. The central Feed API, facilitating operations like addition, entry retrieval, and removal, is constructed upon the foundation of the Key-Value API, which includes functions for putting, getting, and deleting data. ## Usage -Add project as a git module. +Add project as a git submodule. -## Backend +## Project Structure -* Ring -* RocksDB -* Memory -* FS -* SQL -* etc. +* `./feed`: Introduces the Feed over Ring concept +* `./search`: Offers Search over Ring functionality +* `./sort`: Implements a Sorted Set on Ring +* `./ring`: Establishes a Ring structure using Pekko Cluster +* `./sharding`: Addresses Sequential Consistency & Cluster Sharding aspects +* `./src`: Contains illustrative sample applications and comprehensive tests -## Test +## Test & Demo ```bash -sbt> test +sbt test +sbt run ``` - -## Resources - -### Chain Replication - -[Chain Replication in Theory and in Practice](http://www.snookles.com/scott/publications/erlang2010-slf.pdf) - -[Chain Replication for Supporting High Throughput and Availability](http://www.cs.cornell.edu/home/rvr/papers/OSDI04.pdf) - -[High-throughput chain replication for read-mostly workload](https://www.cs.princeton.edu/courses/archive/fall15/cos518/papers/craq.pdf) - -[Leveraging Sharding in the Design of Scalable Replication Protocols](https://ymsir.com/papers/sharding-socc.pdf) - -[Byzantine Chain Replication](http://www.cs.cornell.edu/home/rvr/newpapers/opodis2012.pdf) - -### Consensus Algorithm - -[RAFT](https://raft.github.io/raft.pdf) -[SWIM](https://www.cs.cornell.edu/projects/Quicksilver/public_pdfs/SWIM.pdf) diff --git a/build.sbt b/build.sbt index de050414..d306cb35 100644 --- a/build.sbt +++ b/build.sbt @@ -1,53 +1,89 @@ -val scalav = "3.2.2" -val zio = "2.0.10" -val akka = "2.6.20" -val rocks = "7.10.2" -val protoj = "3.22.2" -val lucene = "8.11.2" +val scalav = "3.3.3" +val zio = "2.0.21" +val pekko = "1.0.2" +val rocks = "8.11.3" +val protoj = "3.25.3" +val lucene = "9.10.0" -lazy val root = project.in(file(".") ).aggregate(kvs) +lazy val kvsroot = project.in(file(".")).settings( + scalaVersion := scalav +, libraryDependencies ++= Seq( + "dev.zio" %% "zio-test-sbt" % zio % Test + , "org.apache.pekko" %% "pekko-cluster-sharding" % pekko + ) +, testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework") +, scalacOptions ++= scalacOptionsCommon +, Test / fork := true +, run / fork := true +, run / connectInput := true +).dependsOn(feed, search, sort).aggregate(ring, feed, search, sort) -lazy val kvs = project.in(file("kvs")).settings( +lazy val ring = project.in(file("ring")).settings( scalaVersion := scalav +, Compile / scalaSource := baseDirectory.value / "src" , libraryDependencies ++= Seq( - "com.typesafe.akka" % "akka-cluster-sharding_2.13" % akka - , "com.typesafe.akka" % "akka-slf4j_2.13" % akka - , "ch.qos.logback" % "logback-classic" % "1.4.5" - , "com.github.jnr" % "jnr-ffi" % "2.2.2" - , "org.apache.lucene" % "lucene-analyzers-common" % lucene - , "dev.zio" %% "zio" % zio - , "dev.zio" %% "zio-nio" % "2.0.0" + "org.apache.pekko" %% "pekko-cluster" % pekko , "org.rocksdb" % "rocksdbjni" % rocks - , "org.scalatest" %% "scalatest" % "3.2.14" % Test - , "com.typesafe.akka" % "akka-testkit_2.13" % akka % Test + , "dev.zio" %% "zio" % zio ) -, scalacOptions ++= scalacOptions3 +, scalacOptions ++= scalacOptionsCommon diff Seq("-language:strictEquality") :+ "-nowarn" ).dependsOn(proto) -lazy val proto = project.in(file("deps/proto/proto")).settings( +lazy val sharding = project.in(file("sharding")).settings( scalaVersion := scalav -, crossScalaVersions := scalav :: Nil -, libraryDependencies += "com.google.protobuf" % "protobuf-java" % protoj -).dependsOn(protoops) +, Compile / scalaSource := baseDirectory.value / "src" +, libraryDependencies ++= Seq( + "org.apache.pekko" %% "pekko-cluster-sharding" % pekko + ) +, scalacOptions ++= scalacOptionsCommon +).dependsOn(ring) + +lazy val feed = project.in(file("feed")).settings( + scalaVersion := scalav +, Compile / scalaSource := baseDirectory.value / "src" +, libraryDependencies ++= Seq( + "dev.zio" %% "zio-streams" % zio + ) +, scalacOptions ++= scalacOptionsCommon +).dependsOn(sharding) + +lazy val search = project.in(file("search")).settings( + scalaVersion := scalav +, Compile / scalaSource := baseDirectory.value / "src" +, libraryDependencies ++= Seq( + "dev.zio" %% "zio-streams" % zio + , "org.apache.lucene" % "lucene-analysis-common" % lucene + ) +, scalacOptions ++= scalacOptionsCommon +).dependsOn(sharding) + +lazy val sort = project.in(file("sort")).settings( + scalaVersion := scalav +, Compile / scalaSource := baseDirectory.value / "src" +, libraryDependencies ++= Seq( + "dev.zio" %% "zio-streams" % zio + ) +, scalacOptions ++= scalacOptionsCommon +).dependsOn(sharding) -lazy val protoops = project.in(file("deps/proto/ops")).settings( +lazy val proto = project.in(file("deps/proto/proto")).settings( scalaVersion := scalav , crossScalaVersions := scalav :: Nil +, libraryDependencies ++= Seq( + "com.google.protobuf" % "protobuf-java" % protoj + ) +, scalacOptions ++= scalacOptionsCommon diff Seq("-language:strictEquality") :+ "-Xcheck-macros" ).dependsOn(protosyntax) lazy val protosyntax = project.in(file("deps/proto/syntax")).settings( scalaVersion := scalav , crossScalaVersions := scalav :: Nil +, scalacOptions ++= scalacOptionsCommon diff Seq("-language:strictEquality") ) -val scalacOptions3 = Seq( - "-source:future", "-nowarn" +val scalacOptionsCommon = Seq( + "-Yexplicit-nulls" , "-language:strictEquality" -, "-language:postfixOps" -, "-Yexplicit-nulls" -, "-encoding", "UTF-8" ) -turbo := true -useCoursier := true Global / onChangedBuildSource := ReloadOnSourceChanges diff --git a/deps/proto b/deps/proto index 1ba7e91b..3f764a72 160000 --- a/deps/proto +++ b/deps/proto @@ -1 +1 @@ -Subproject commit 1ba7e91b7d925feb900cb43aed58278ee2b76c1a +Subproject commit 3f764a72ddec87f9eb8d07b94b1f1f7e1e5008b8 diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index fb1aa38d..00000000 --- a/docs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -version.tex diff --git a/docs/about.tex b/docs/about.tex deleted file mode 100644 index 9ccf8dd3..00000000 --- a/docs/about.tex +++ /dev/null @@ -1,6 +0,0 @@ -\paragraph{KVS} -Abstract Scala Types database wich allow to build the storage schemes around the linked list of entities (data feeds). -Powered by several backend storage engines it can suite variety of needs. -With the RING backend its a great tool for managing the distributed data and provide sequential consistency when used with FeedServer. - -\footahref{https://github.com/zero-deps/kvs}{https://github.com/zero-deps/kvs} diff --git a/docs/info.tex b/docs/info.tex deleted file mode 100644 index db4e5703..00000000 --- a/docs/info.tex +++ /dev/null @@ -1,18 +0,0 @@ -\section*{KVS - Abstract Scala Data Types Key-Value Storage} -\paragraph{} -The Key-Value Database for storing scala value types. It has the simple put/get API plus some extended operation to manage the data feeds. - -\section*{Backend Storage Engine} -\paragraph{} -The KVS can be backed by the various storage engines. -\begin{description} -\item[RING] For distributed, scalable, fault tolerant key-value store. -\item[LevelDB] For usage in simple non-clustered environment -\item[Memory] In memory storage for chache or testing purposes -\item[FS] Filesystem storage. -\end{description} - -\section{Services Handlers} -\paragraph{} -Particular Services can operate with specific data types. Storing every specific type may require some additional logic. -You should think about the handlers as some kind of serializators/picklers/marshallers for KVS. diff --git a/docs/kvs.tex b/docs/kvs.tex deleted file mode 100644 index 7e83d63e..00000000 --- a/docs/kvs.tex +++ /dev/null @@ -1,60 +0,0 @@ -\section{} -KVS its the Key-Value storage framework which provide high level API for handling `feeds` of the data. -Something about sequential consistency...last operation for the operation. -Features: -Managing linked list -various backend support:leveldb, ring -sequential consistency via feed server -basic schema -extendable schema? -secondary indexes? -mupltiple backends -the application is kind of framework for seq consistency via feed server or something else. -all the operation that are make without secondary indexes,i.e not linked can be used without feed server. -The data in kvs presented as plain scala case classes - -iterators, all records can be chained into double-linked list. so you can inherit from entry class and provide the data type. -kvs use scala pickling library for serialization so the picklers must be defined on compile time for you kind of data. - -the table will support add/remove linked list operation. - -kvs.entries(kvs.get(feed,users), user, undefined) - -kvs.all(user) - real flat values of all keys from table. - -containers are just boxed for storing top of lists - -database init kvs.join - - -put operation is mean put into the database, but add is adding to the list - -leveldb - for secondary indexes -consistency - put each in the box - - -statically typed. mutable, key-value,case class store -schemas -for each container loc process should be spawned - handle feed operations ordering and consistency - - -\section{} - -after handler for entries with particular payload is defined -it can be reused for specific tagged type -Scalaz tygged type is used for specify the "new type" from existing type without the needs to actually create the types. - - -For example the Message and Metrics are the same kind of entry with string payload. -Typicaly they are marked by some empty trait - -trait Msg -trait Mtr - -and the tagged types can be created as follows - -type Message = En[String] @@ Msg -type Metrics = En[String] @@ Mtr - -actually as soon as you defined the En[String] handler implicitly you are ready to define the new handlers. -type tags will help to catch the error about incompatible operations on compile time. diff --git a/docs/kvs_jmx_api.tex b/docs/kvs_jmx_api.tex deleted file mode 100644 index e6185d98..00000000 --- a/docs/kvs_jmx_api.tex +++ /dev/null @@ -1,65 +0,0 @@ -\section{Datastore JMX interface} -KVS application registers MBeans called Kvs and Ring. MBean is object that simlular to JavaBean that represent resourse -that can be management using JMX techlology. -The Java Management Extensions (JMX) technology is a part of Java Platform that gives abillity to manage aplication remotely. -In order to connect to Kvs MBean you can use standart application as jconsole that provided with JDK and located in -\$JDK_HOME/bin, or others that complies to the JMX specification. - - -\paragraph{KVS JMX interfase} -\begin{description} - -\item [Read all feed as string] -allStr(fid:String):String - -Return string representation of all entities in spesified feed. - -\item [Export all data] - -Current version of KVS using RNG application as backend layer. As far as RNG is distributed data store it's not possible -to backup or migrate data form server to server only with copy-past directory with persisted data. Even more any copy -of RNG's data will has no any sense becase data in RNG storage is partitioned and particular node can be acquired only for -defined range of keys in store (approximately equals N from quorum configuration divided by number of reachable nodes). -Use save method to merge data from all nodes. This method returns path to zip file with dump. - -\begin{lstlisting}[language=bash] -Kvs:save -\end{lstlisting} - -Be aware that RNG become readonly after save trigger in order to keep consistency. When archive will composed RNG become writable and -readable again. - -\item [Load file] -Archive from save operation can be loaded to application with load method. - -\begin{lstlisting}[language=bash] -Kvs:load(path) -\end{lstlisting} - - -After load opperation triggered, RNG state become readonly. Data from loaded file has higher priority compare to already stored. -KVS become readable and writeable when loading is finished. -Be sure that quorum write configuration should be satisfied otherwise data will be ignored on write opperation. This condition can be checked by comparing - W property from quorum configuration and currently reachable nodes in cluster. - Important note that loading of file that was created on differ version of services can lead to broken data. It's caused - because RNG persist data in bytes so all entities that serilisated before saving goes throught serialisation - deserialisation . - That's why KVS not supported compatability between differ version of schema. - -\end{description} - -\paragraph{RNG JMX interfase} -RNG JMX interface is not for production usage. As far as RNG is lowest level of application, it's not aware about data schema and works -only with arrays of Byte. Those method can be used only if caller aware about data schema and keys composition. - -\begin{description} -\item -[Ring:get(key:String): String] -Get value by key. -\item - [Ring:put(key:String, data: String):String] - This method has value only for testing. Put value associated with key. -\item -[Ring:delete(key:String):Unit] -Delete value associated with key. -\end{description} - diff --git a/docs/ring_about.tex b/docs/ring_about.tex deleted file mode 100644 index d7ee57fe..00000000 --- a/docs/ring_about.tex +++ /dev/null @@ -1,5 +0,0 @@ -\paragraph{Ring} -Scala implementation of Kai AP, key-value storage which is inspired by Amazon's Dynamo. It's service distributed datastore that -follows CAP theorem and provides consistency, partition tolerance and data availability among cluster. Currently it is a part of KVS codebase. - -\footahref{/ring.html}{Detailed documentation} diff --git a/docs/ring_config.tex b/docs/ring_config.tex deleted file mode 100644 index 8105b930..00000000 --- a/docs/ring_config.tex +++ /dev/null @@ -1,57 +0,0 @@ -\section{Ring Datastore} -\paragraph{} -Ring Application (also referred by the inner name RNG) is available as akka extension and makes your system part of the highly available, fault tolerant data distrubution cluster. - -\paragraph{} -To configure RNG application on your cluster the next config options are available: - -\begin{description} - \item[quorum] Configured by array of three integer parameters N,W,R where - \begin{description} - \item[N] Number of nodes in bucket(in other words the number of copies). - \item[R] Number of nodes that must be participated in successful read operation. - \item[W] Number of nodes for successful write. - \end{description} - To keep data consistent the quorums have to obey the following rules: - \begin{enumerate} - \item R + W > N - \item W > N/2 - \end{enumerate} - Or use the next hint: - \begin{itemize} - \item single node cluster [1,1,1] - \item two nodes cluster [2,2,1] - \item 3 and more nodes cluster [3,2,2] - \end{itemize} - if quorum fails on write operation, data will not be saved. So in case if 2 nodes and [2,2,1] after 1 node down the cluster becomes not writeable and readable. - \item[buckets] Number of buckets for key. Think about this as about size of HashMap. In current implementation this value should not be changed after first setup. - \item[virtual-nodes] Number of virtual nodes for each physical. In current implementation this value should not be changed after first setup. - \item[hash-length] Lengths of hash from key. In current implementation this value should not be changed after first setup. - \item[gather-timeout] Number of seconds that requested cluster will wait for response from another nodes. - \item[ring-node-name] Role name that mark node as part of ring. - \item[leveldb] Configuration of levelDB database used as backend for ring. - \begin{description} - \item[dir] directory location for levelDB storage. - \item[fsync] if true levelDB will synchronise data to disk immediately. - \end{description} -\end{description} - -\paragraph{} -RNG provides default configuration for single node mode: - -\begin{lstlisting}[language=json,caption=Example] -ring { - quorum = [1, 1, 1] //N,R,W - buckets = 1024 - virtual-nodes = 128 - hash-length = 32 - gather-timeout = 3 - leveldb { - dir = "rng_data_"${akka.remote.netty.tcp.hostname}"_"${akka.remote.netty.tcp.port} - fsync = false - } -} -\end{lstlisting} - -\paragraph{} -In case default values are suitable for particular deployment, rewrite is not needed. diff --git a/docs/ring_health.tex b/docs/ring_health.tex deleted file mode 100644 index 830282e5..00000000 --- a/docs/ring_health.tex +++ /dev/null @@ -1,30 +0,0 @@ -\subsection*{KVS/RING Data Metrics} - -\paragraph{} -Services uses the data storage so its helpfull to track the state of the different data distribution metrics. -The RING distributed key-value storage also provide the node extension to collect and propogate different metrics. - -\paragraph{} -These various stats give a picture of the general level of activity or load on the node at specified moment. - -\begin{description} -\item [Disk/Memory usage] -Available disk space. Used file descriptors. Swap Usage. IO wait. -\item [Read operations] -Consistent reads coordinated by this node. Number of local replicas participating in secondary index reads. -Number of siblings encountered during all GET operations by this node within the specific time. -\item [Write operations] -Consistent writes coordinated by this node. -Object size encountered by this node within the specific time. -Abnormally large objects (especially paired with high sibling counts) can indicate sibling explosion. -\item [Network] -Throughput metrics. Latency metrics. General load/health metrics. Network errors. -\item [Search] -Documents indexed by search. Search queries on the node. Number of “failed to index” errors Search encountered for specific time. -\item [General Load/Health Metrics] -Watch for abnormally high sibling counts, especially max ones. -Number of unprocessed messages in the vnode message queues of the Search subsystem on this node in the specific time. -\end{description} - -\paragraph{} -The KVS/RING Metrics Extension provide JMX control over data distribution system. diff --git a/docs/ring_info.tex b/docs/ring_info.tex deleted file mode 100644 index e33fa317..00000000 --- a/docs/ring_info.tex +++ /dev/null @@ -1,51 +0,0 @@ -\section{Ring datastore} - -Scala implementation of Kai (originally implemented in erlang). -Kai is a distributed key-value datastore, which is mainly inspired -by Amazon's Dynamo. Ring is implemented on top of akka and injected as akka extension. - -\section*{Overview} - -To reach fault tolerance and scalability ring resolve next problems: - -\begin{description} - -\item[Problems:] Technique -\item[Membership and failure detection: ] reused akka's membership events that uses gossip for communication. FD also reused from akka. -\item[Data partitioning:] consistent hashing. -\item[High availability to wright:] vector clocks increase number of write opperation to merge data on read opperation. -\item[Handling nodes failures:] gossip protocol. - -\end{description} - -\section*{Consistent hashing} - -To figure out where the data for a particular key goes in that cluster you need to apply a hash function to the key. -Just like a hashtable, a unique key maps to a value and of course the same key will always return the same hash code. -In very first and simple version of this algorithm the node for particular key is determined by hash(key) mod n, where n is a number of -nodes in cluster. This works well and trivial in implementation but when new node join or removed from cluster we got a problem, every object is hashed to a new location. -The idea of the consistent hashing algorithm is to hash both node and key using the same hash function. -As result we can map the node to an interval, which will contain a number of key hashes. If the node is removed -then its interval is taken over by a node with an adjacent interval. - -\section*{Vector clocks} - -Vector clocks is an algorithm for generating a partial ordering of events in a distributed system and detecting causality violations. (from wikipedia.org) -Vector clocks help us to determine order in which data writes was occurred. This provide ability to write data from one node and after that -merge version of data. Vector clock is a list of pairs of node name and number of changes from this node. -When data writs first time the vector clock will have one entity ( node-A : 1). Each time data amended the counter is incremented. - -\section*{Quorum} - -Quorum determines the number of nodes that should be participated in operation. Quorum-like system configured by values: R ,W and N. R is the -minimum number of nodes that must participate in a successful read operation. W is the minimum number of nodes that must participate. -N is a preference list, the max number of nodes that can be participated in operation. Also quorum can configure balance of latency -for read and write operation. - In order to keep data strongly consistent configuration should obey rules: - \begin{lstlisting}[language=bash] - 1) R + W > N - 2) W > V/2 - \end{lstlisting} - - - diff --git a/feed/src/feed.scala b/feed/src/feed.scala new file mode 100644 index 00000000..61bef270 --- /dev/null +++ b/feed/src/feed.scala @@ -0,0 +1,65 @@ +package kvs.feed + +import kvs.rng.Dba +import proto.* +import zio.*, stream.* + +/* Abstract type feed */ +trait Feed: + def all[A: Codec](fid: Fid, eid: Option[Eid]=None): Stream[Err, (Eid, A)] + def get[A: Codec](fid: Fid, eid: Eid): IO[Err, Option[A]] + def add[A: Codec](fid: Fid, a: A): IO[Err, Eid] + def remove(fid: Fid, eid: Eid): IO[Err, Boolean] + def cleanup(fid: Fid): IO[Err, Unit] +end Feed + +def all[A: Codec](fid: Fid, eid: Option[Eid]=None): ZStream[Feed, Err, (Eid, A)] = + ZStream.serviceWithStream(_.all(fid, eid)) + +def get[A: Codec](fid: Fid, eid: Eid): ZIO[Feed, Err, Option[A]] = + ZIO.serviceWithZIO(_.get(fid, eid)) + +def add[A: Codec](fid: Fid, a: A): ZIO[Feed, Err, Eid] = + ZIO.serviceWithZIO(_.add(fid, a)) + +def remove(fid: Fid, eid: Eid): ZIO[Feed, Err, Boolean] = + ZIO.serviceWithZIO(_.remove(fid, eid)) + +def cleanup(fid: Fid): ZIO[Feed, Err, Unit] = + ZIO.serviceWithZIO(_.cleanup(fid)) + +val live: URLayer[Dba, Feed] = + ZLayer( + for + dba <- ZIO.service[Dba] + yield + new Feed: + def all[A: Codec](fid: Fid, eid: Option[Eid]=None): Stream[Err, (Eid, A)] = + eid.fold(ops.all(fid))(ops.all(fid, _))(dba).mapZIO{ case (k, a) => + for + b <- unpickle(a) + yield k -> b + } + + def get[A: Codec](fid: Fid, eid: Eid): IO[Err, Option[A]] = + for + res <- ops.get(fid, eid)(dba) + a <- + (for + b <- ZIO.fromOption(res) + a <- unpickle[A](b) + yield a).unsome + yield a + + def add[A: Codec](fid: Fid, a: A): IO[Err, Eid] = + for + b <- pickle(a) + eid <- ops.add(fid, b)(dba) + yield eid + + def remove(fid: Fid, eid: Eid): IO[Err, Boolean] = + ops.remove(fid, eid)(dba) + + def cleanup(fid: Fid): IO[Err, Unit] = + ops.cleanup(fid)(dba) + ) diff --git a/feed/src/ops.scala b/feed/src/ops.scala new file mode 100644 index 00000000..eaf7f4c1 --- /dev/null +++ b/feed/src/ops.scala @@ -0,0 +1,229 @@ +package kvs.feed + +import org.apache.pekko.actor.Actor +import kvs.rng.{Dba, Key, AckQuorumFailed, AckTimeoutFailed} +import proto.* +import zio.*, stream.* + +type Eid = Long // entry id +type Err = AckQuorumFailed | AckTimeoutFailed + +private[feed] type Fid = String // feed id +private[feed] type Data = Array[Byte] + +private[feed] def pickle[A : Codec](e: A): UIO[Array[Byte]] = ZIO.succeed(encode[A](e)) +private[feed] def unpickle[A : Codec](a: Array[Byte]): UIO[A] = ZIO.attempt(decode[A](a)).orDie // is defect + +/* + * Feed: + * [head] -->next--> [en] -->next--> (nothing) + */ +object ops: + private[feed] case class Fd + ( @N(1) head: Option[Eid] + , @N(2) length: Long + , @N(3) removed: Long + , @N(4) maxid: Eid + ) + + private[feed] object Fd: + val empty = Fd(head=None, length=0, removed=0, maxid=0) + + private[feed] case class En + ( @N(1) next: Option[Eid] + , @N(2) data: Data + , @N(3) removed: Boolean = false + ) + + private[feed] given Codec[Fd] = caseCodecAuto + private[feed] given Codec[En] = caseCodecAuto + private[feed] given Codec[(Fid, Eid)] = caseCodecIdx + + private[feed] object meta: + def len(id: Fid)(dba: Dba): IO[Err, Long] = + get(id)(dba).map(_.fold(0L)(_.length)) + + def delete(id: Fid)(dba: Dba): IO[Err, Unit] = + dba.delete(stob(id)) + + def put(id: Fid, el: Fd)(dba: Dba): IO[Err, Unit] = + for + p <- pickle(el) + x <- dba.put(stob(id), p) + yield x + + def get(id: Fid)(dba: Dba): IO[Err, Option[Fd]] = + dba.get(stob(id)).flatMap{ + case Some(x) => unpickle[Fd](x).map(Some(_)) + case None => ZIO.succeed(None) + } + end meta + + private def _get(key: Key)(dba: Dba): IO[Err, Option[En]] = + dba.get(key).flatMap(_ match + case Some(x) => + unpickle[En](x).map{ + case en if en.removed => None + case en => Some(en) + } + case None => ZIO.succeed(None) + ) + + private[feed] def get(fid: Fid, eid: Eid)(dba: Dba): IO[Err, Option[Data]] = + for + key <- pickle((fid, eid)) + x <- _get(key)(dba).map(_.map(_.data)) + yield x + + /** + * Mark entry for removal. O(1) complexity. + * @return true if marked for removal + */ + private[feed] def remove(fid: Fid, eid: Eid)(dba: Dba): IO[Err, Boolean] = + for + key <- pickle((fid, eid)) + en1 <- _get(key)(dba) + res <- + en1.fold(ZIO.succeed(false))(en => + for + fd <- meta.get(fid)(dba).flatMap(_.fold(ZIO.dieMessage("feed is not exists"))(ZIO.succeed)) + _ <- meta.put(fid, fd.copy(length=fd.length-1, removed=fd.removed+1))(dba) + p <- pickle(en.copy(removed=true)) + _ <- dba.put(key, p) + yield true + ) + yield res + + /** + * Adds the entry to the container. Creates the container if it's absent. + * ID will be generated. + */ + private[feed] def add(fid: Fid, data: Data)(dba: Dba): IO[Err, Eid] = + for { + fd1 <- meta.get(fid)(dba) + fd <- fd1.fold(meta.put(fid, Fd.empty)(dba).map(_ => Fd.empty))(ZIO.succeed) + id = fd.maxid + 1 + en = En(next=fd.head, data=data) + _ <- meta.put(fid, fd.copy(maxid=id))(dba) // in case kvs will fail after adding the en + key <- pickle((fid, id)) + p <- pickle(en) + _ <- dba.put(key, p) + _ <- meta.put(fid, fd.copy(head=Some(id), length=fd.length+1, maxid=id))(dba) + } yield id + + /* all items with removed starting from specified key */ + private def entries(fid: Fid, start: Eid)(dba: Dba): Stream[Err, (Eid, En)] = + ZStream. + unfoldZIO(Some(start): Option[Eid]){ + case None => ZIO.succeed(None) + case Some(id) => + for + k <- pickle((fid, id)) + bs <- + dba.get(k).flatMap{ + case None => ZIO.dieMessage("feed is corrupted") + case Some(bs) => ZIO.succeed(bs) + } + en <- unpickle[En](bs) + yield Some((id -> en) -> en.next) + } + + /* all items with removed starting from beggining */ + private def entries(fid: Fid)(dba: Dba): Stream[Err, (Eid, En)] = + ZStream.fromZIO(meta.get(fid)(dba)).flatMap{ + case None => ZStream.empty + case Some(a) => a.head.fold(ZStream.empty)(entries(fid, _)(dba)) + } + + /* all items without removed starting from specified key */ + private def entries_live(fid: Fid, eid: Eid)(dba: Dba): Stream[Err, (Eid, En)] = + entries(fid, eid)(dba). + filterNot{ case (_,en) => en.removed } + + /* all items without removed starting from beggining */ + private def entries_live(fid: Fid)(dba: Dba): Stream[Err, (Eid, En)] = + entries(fid)(dba). + filterNot{ case (_,en) => en.removed } + + /* all data without removed from beggining */ + private[feed] def all(fid: Fid)(dba: Dba): Stream[Err, (Eid, Data)] = + entries_live(fid)(dba).map{ case (id, en) => id -> en.data } + + /* all data without removed from specified key */ + private[feed] def all(fid: Fid, eid: Eid)(dba: Dba): Stream[Err, (Eid, Data)] = + entries_live(fid, eid)(dba).map{ case (id, en) => id -> en.data } + + /* delete all entries marked for removal, O(n) complexity */ + private[feed] def cleanup(fid: Fid)(dba: Dba): IO[Err, Unit] = + meta.get(fid)(dba).flatMap{ + case None => ZIO.unit + case Some(fd) => + for + /* remove from head */ + x <- + entries(fid)(dba). + takeWhile{ case (_, en) => en.removed }. + mapZIO{ case (id, en) => + for + k <- pickle((fid, id)) + _ <- dba.delete(k) + yield en.next + }. + runLast + /* fix head */ + _ <- x.fold(ZIO.unit)(id => meta.put(fid, fd.copy(head=id))(dba)) + /* remove in the middle */ + _ <- + entries(fid)(dba). + /* zip with stream of last live entry before removed ones */ + /* [1(live),2(removed),3(removed),4(live),5(removed)] becomes [1,1,1,4,4] */ + zip(entries(fid)(dba).scan(None: Option[(Eid,En)]){ + case (None, (id, en)) => Some(id->en) + case ( x, ( _, en)) if en.removed => x + case ( _, (id, en)) => Some(id->en) + }.collect{ case Some(x) => x }). + filter{ case (_, en, _) => en.removed }. + mapZIO{ case (id, en, (id2, en2)) => + for + /* change link */ + p <- pickle(en2.copy(next=en.next)) + k2 <- pickle((fid, id2)) + _ <- dba.put(k2, p) + /* delete entry */ + k <- pickle((fid, id)) + _ <- dba.delete(k) + yield () + }. + runDrain + /* fix removed/maxid */ + _ <- fix(fid, fd)(dba) + yield () + } + + /* fix length, removed and maxid for feed */ + private def fix(fid: Fid, fd: Fd)(dba: Dba): IO[Err, Unit] = + for + x <- + entries(fid)(dba).runFold((0L, 0L, 0L)){ + case ((l, r, m), (id, en)) => + val l2 = en.removed.fold(l, l+1) + val r2 = en.removed.fold(r+1, r) + val m2 = Math.max(m, id) + (l2, r2, m2) + } + (l, r, m) = x + _ <- meta.put(fid, fd.copy(length=l, removed=r, maxid=m))(dba) + yield () + + private[feed] inline def stob(s: String): Array[Byte] = + s.getBytes("utf8").nn + + extension (x: Boolean) + private[feed] inline def fold[A](t: => A, f: => A): A = + if x then t else f + + given CanEqual[Nothing, kvs.rng.Value] = CanEqual.derived + given CanEqual[Nothing, (Eid, ops.En)] = CanEqual.derived + given CanEqual[Nothing, ops.Fd] = CanEqual.derived + +end ops diff --git a/kvs.pdf b/kvs.pdf new file mode 100644 index 00000000..ffa8d9bd Binary files /dev/null and b/kvs.pdf differ diff --git a/kvs/src/main/resources/lib/README.md b/kvs/src/main/resources/lib/README.md deleted file mode 100644 index d93d1571..00000000 --- a/kvs/src/main/resources/lib/README.md +++ /dev/null @@ -1 +0,0 @@ -Compiled levelDB 1.20 for 64-bit platforms from leveldb-jna project. diff --git a/kvs/src/main/resources/lib/leveldb.dll b/kvs/src/main/resources/lib/leveldb.dll deleted file mode 100644 index f9c6f53e..00000000 Binary files a/kvs/src/main/resources/lib/leveldb.dll and /dev/null differ diff --git a/kvs/src/main/resources/lib/libleveldb.dylib b/kvs/src/main/resources/lib/libleveldb.dylib deleted file mode 100644 index 04249216..00000000 Binary files a/kvs/src/main/resources/lib/libleveldb.dylib and /dev/null differ diff --git a/kvs/src/main/resources/lib/libleveldb.so b/kvs/src/main/resources/lib/libleveldb.so deleted file mode 100644 index f3779518..00000000 Binary files a/kvs/src/main/resources/lib/libleveldb.so and /dev/null differ diff --git a/kvs/src/main/resources/reference.conf b/kvs/src/main/resources/reference.conf deleted file mode 100644 index 0aa968e8..00000000 --- a/kvs/src/main/resources/reference.conf +++ /dev/null @@ -1,73 +0,0 @@ -ring { - quorum = [1, 1, 1] // N, W, R - buckets = 32768 // 2^15 - virtual-nodes = 128 - hash-length = 32 - # ring-timeout is bigger than gather-timeout - ring-timeout = 11 seconds - gather-timeout = 10 seconds - dump-timeout = 1 hour - repl-timeout = 1 minute - iter-timeout = 10 minutes - leveldb { - dir = rng_data - fsync = false - } -} - -rks { - dump-timeout = 1 hour -} - -akka { - actor { - provider = cluster - - deployment { - /ring_readonly_store { - router = round-robin-pool - nr-of-instances = 5 - } - } - - debug { - receive = off - lifecycle = off - } - - serializers { - kvsproto = zd.kvs.Serializer - } - - serialization-identifiers { - "zd.kvs.Serializer" = 50 - } - - serialization-bindings { - "zd.rng.model.ChangeState" = kvsproto - "zd.rng.model.StoreGetAck" = kvsproto - "zd.rng.model.StoreDelete" = kvsproto - "zd.rng.model.StoreGet" = kvsproto - "zd.rng.model.StorePut" = kvsproto - "zd.rng.model.DumpBucketData" = kvsproto - "zd.rng.model.DumpEn" = kvsproto - "zd.rng.model.DumpGet" = kvsproto - "zd.rng.model.DumpGetBucketData" = kvsproto - "zd.rng.model.ReplBucketPut" = kvsproto - "zd.rng.model.ReplBucketUpToDate" = kvsproto - "zd.rng.model.ReplGetBucketIfNew" = kvsproto - "zd.rng.model.ReplNewerBucketData" = kvsproto - } - } - - coordinated-shutdown { - phases { - stop-kvs-jmx { - depends-on = [before-cluster-shutdown] - } - cluster-sharding-shutdown-region { - depends-on = [stop-kvs-jmx] - } - } - } -} diff --git a/kvs/src/main/scala/dump/DumpIO.scala b/kvs/src/main/scala/dump/DumpIO.scala deleted file mode 100644 index 583ac864..00000000 --- a/kvs/src/main/scala/dump/DumpIO.scala +++ /dev/null @@ -1,75 +0,0 @@ -package zd -package dump - -import akka.actor.{Actor, ActorLogging, Props} -import java.nio.ByteBuffer -import java.nio.channels.FileChannel -import java.nio.file.Paths -import java.nio.file.StandardOpenOption.{READ, WRITE, CREATE} -import codec.* -import scala.util.Try -import proto.* - -type Key = Array[Byte] -type Value = Array[Byte] - -object DumpIO { - def props(ioPath: String): Throwable Either Props = { - Try(FileChannel.open(Paths.get(ioPath), READ, WRITE, CREATE).nn).toEither.map(channel => Props(new DumpIO(ioPath, channel))) - } - - case object ReadNext - final case class ReadNextRes(kv: Vector[(Key, Value)]) - final case object ReadNextLast - - final case class Put(kv: Vector[(Key, Value)]) - final case object PutDone -} - -class DumpIO(ioPath: String, channel: FileChannel) extends Actor with ActorLogging { - - def receive = { - case _: DumpIO.ReadNext.type => - val key = ByteBuffer.allocateDirect(4).nn - val keyRead = channel.read(key) - if (keyRead == 4) { - val blockSize: Int = key.flip.asInstanceOf[ByteBuffer].getInt - val value: Array[Byte] = new Array[Byte](blockSize) - val valueRead: Int = channel.read(ByteBuffer.wrap(value)) - if (valueRead == blockSize) { - val kv: Vector[(Key, Value)] = decode[DumpKV](value).kv.view.map(d => d.k -> d.v).to(Vector) - sender ! DumpIO.ReadNextRes(kv) - } else { - log.error(s"failed to read dump io, blockSize=${blockSize}, valueRead=${valueRead}") - sender ! DumpIO.ReadNextLast - } - } else if (keyRead == -1) { - sender ! DumpIO.ReadNextLast - } else { - log.error(s"failed to read dump io, keyRead=${keyRead}") - sender ! DumpIO.ReadNextLast - } - case msg: DumpIO.Put => - val data = encode(DumpKV(msg.kv.map(e => KV(e._1, e._2)))) - channel.write(ByteBuffer.allocateDirect(4).nn.putInt(data.size).nn.flip.nn.asInstanceOf[ByteBuffer]) - channel.write(ByteBuffer.wrap(data)) - sender ! DumpIO.PutDone - } - - override def postStop(): Unit = { - channel.close() - super.postStop() - } -} - -object DumpIterate { - def props(f: (Key, Value) => Unit): Props = Props(new DumpIterate(f)) -} - -class DumpIterate(f: (Key, Value) => Unit) extends Actor { - def receive = { - case msg: DumpIO.Put => - msg.kv.foreach(e => f(e._1, e._2)) - sender ! DumpIO.PutDone - } -} diff --git a/kvs/src/main/scala/dump/DumpKV.scala b/kvs/src/main/scala/dump/DumpKV.scala deleted file mode 100644 index 353256a3..00000000 --- a/kvs/src/main/scala/dump/DumpKV.scala +++ /dev/null @@ -1,37 +0,0 @@ -package zd -package dump - -import proto.* -import java.util.Arrays - -final case class DumpKV - ( @N(1) kv: Vector[KV] - ) - -final class KV - ( @N(1) val k: Array[Byte] - , @N(2) val v: Array[Byte] - ) { - override def equals(other: Any): Boolean = other match { - case that: KV => - Arrays.equals(k, that.k) && - Arrays.equals(v, that.v) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(k, v) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"KV(k=$k, v=$v)" -} - -object KV { - def apply(k: Array[Byte], v: Array[Byte]): KV = { - new KV(k=k, v=v) - } -} - -object codec { - implicit val dumpKVCodec: MessageCodec[DumpKV] = caseCodecAuto[DumpKV] - implicit val kVCodec: MessageCodec[KV] = classCodecAuto[KV] -} diff --git a/kvs/src/main/scala/el.scala b/kvs/src/main/scala/el.scala deleted file mode 100644 index d61d1e7a..00000000 --- a/kvs/src/main/scala/el.scala +++ /dev/null @@ -1,23 +0,0 @@ -package zd.kvs - -trait ElHandler[T]: - def pickle(e: T): Array[Byte] - def unpickle(a: Array[Byte]): T - - def put(k: String, el: T)(using dba: Dba): Either[Err, T] = - dba.put(k, pickle(el)).map(_ => el) - - def get(k: String)(using dba: Dba): Either[Err, Option[T]] = - dba.get(k).map(_.map(unpickle)) - - def delete(k: String)(using dba: Dba): Either[Err, Unit] = - dba.delete(k).void - -object ElHandler: - given ElHandler[Array[Byte]] = new ElHandler: - def pickle(e: Array[Byte]): Array[Byte] = e - def unpickle(a: Array[Byte]): Array[Byte] = a - - given ElHandler[String] = new ElHandler: - def pickle(e: String): Array[Byte] = e.getBytes("utf8").nn - def unpickle(a: Array[Byte]): String = String(a, "utf8").nn diff --git a/kvs/src/main/scala/err.scala b/kvs/src/main/scala/err.scala deleted file mode 100644 index d7736fcd..00000000 --- a/kvs/src/main/scala/err.scala +++ /dev/null @@ -1,15 +0,0 @@ -package zd.kvs - -sealed trait Err derives CanEqual - -case class EntryExists(key: String) extends Err -case object KeyNotFound extends Err -case class FileNotExists(dir: String, name: String) extends Err -case class FileAlreadyExists(dir: String, name: String) extends Err -case class Fail(r: String) extends Err -case class Failed(t: Throwable) extends Err -case class InvalidArgument(d: String) extends Err - -case class RngAskQuorumFailed(why: String) extends Err -case class RngAskTimeoutFailed(op: String, key: String) extends Err -case class RngFail(m: String) extends Err diff --git a/kvs/src/main/scala/ext.scala b/kvs/src/main/scala/ext.scala deleted file mode 100644 index e615743e..00000000 --- a/kvs/src/main/scala/ext.scala +++ /dev/null @@ -1,33 +0,0 @@ -package zd.kvs - -extension [A,B,U](x: Option[A]) - inline def cata(f: A => B, b: => B): B = x.fold(b)(f) - -extension [A](x: Boolean) - inline def fold(t: => A, f: => A): A = if x then t else f - -extension [A,B](xs: Seq[Either[A, B]]) - @annotation.tailrec private def _sequence(ys: Seq[Either[A, B]], acc: Vector[B]): Either[A, Vector[B]] = - ys.headOption match - case None => Right(acc) - case Some(Left(e)) => Left(e) - case Some(Right(z)) => _sequence(ys.tail, acc :+ z) - inline def sequence: Either[A, Vector[B]] = _sequence(xs, Vector.empty) - - @annotation.tailrec private def _sequence_(ys: Seq[Either[A, B]]): Either[A, Unit] = - ys.headOption match - case None => Right(()) - case Some(Left(e)) => Left(e) - case Some(Right(z)) => _sequence_(ys.tail) - inline def sequence_ : Either[A, Unit] = _sequence_(xs) - -extension [L,R,U,L2,R2](x: Either[L,R]) - inline def leftMap(f: L => L2): Either[L2,R] = x match - case Right(a) => Right(a) - case y@Left(l) => Left(f(l)) - inline def recover(pf: PartialFunction[L,R]): Either[L,R] = x match - case Left(l) if pf isDefinedAt l => Right(pf(l)) - case _ => x - inline def void: Either[L, Unit] = x.map(_ => unit) - -inline def unit: Unit = () diff --git a/kvs/src/main/scala/file.fs.scala b/kvs/src/main/scala/file.fs.scala deleted file mode 100644 index cccbc67f..00000000 --- a/kvs/src/main/scala/file.fs.scala +++ /dev/null @@ -1,56 +0,0 @@ -package zd.kvs - -import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.{StandardOpenOption, SimpleFileVisitor, FileVisitResult, Files as JFiles, Path as JPath} -import java.io.IOException - -import zio.* -import zio.nio.file.* -import zio.stream.* - -class FileFs(val root: JPath): - - private val ROOT = Path.fromJava(root) - - def create(path: List[String]): ZIO[Any, Exception, Unit] = - for - _ <- Files.createDirectories(ROOT / path.init) - _ <- Files.createFile(ROOT / path) - yield unit - - def createDir(path: List[String]): ZIO[Any, Exception, Unit] = - Files.createDirectories(ROOT / path) - - def append(path: List[String], data: Chunk[Byte]): ZIO[Any, Exception, Unit] = - Files.writeBytes(ROOT / path, data, StandardOpenOption.APPEND) - - def size(path: List[String]): ZIO[Any, IOException, Long] = - Files.size(ROOT / path) - - def stream(path: List[String]): ZStream[Any, Throwable, Byte] = - val p = root.resolve(path.mkString("/")).nn - ZStream.fromFile(p.toFile.nn) - - def bytes(path: List[String]): ZIO[Any, IOException, Chunk[Byte]] = - Files.readAllBytes(ROOT / path) - - def delete(path: List[String]): ZIO[Any, Exception, Unit] = - val p = root.resolve(path.mkString("/")).nn - ZIO.attemptBlocking(JFiles.walkFileTree(p, new SimpleFileVisitor[JPath]() { - override def visitFile(file: JPath, attrs: BasicFileAttributes): FileVisitResult = { - JFiles.delete(file) - FileVisitResult.CONTINUE - } - override def postVisitDirectory(file: JPath, e: IOException): FileVisitResult = { - JFiles.delete(file) - FileVisitResult.CONTINUE - } - })).unit - .refineToOrDie[Exception] - - def move(pathFrom: List[String], pathTo: List[String]): ZIO[Any, Exception, Unit] = - Files.move(ROOT / pathFrom, ROOT / pathTo) - - extension (p: Path) - def / (xs: List[String]): Path = xs.foldLeft(p)((acc, x) => acc / x) - diff --git a/kvs/src/main/scala/file.scala b/kvs/src/main/scala/file.scala deleted file mode 100644 index 4140819e..00000000 --- a/kvs/src/main/scala/file.scala +++ /dev/null @@ -1,101 +0,0 @@ -package zd.kvs - -import scala.annotation.tailrec -import scala.util.{Try, Success, Failure} -import proto.* - -case class File - ( @N(1) name: String // name – unique value inside directory - , @N(2) count: Int // count – number of chunks - , @N(3) size: Long // size - size of file in bytes - , @N(4) dir: Boolean // true if directory - ) - -trait FileHandler: - protected val chunkLength: Int - - given MessageCodec[File] = caseCodecAuto - - private inline def pickle(e: File): Array[Byte] = encode(e) - private def unpickle(a: Array[Byte]): Either[Err, File] = Try(decode[File](a)) match - case Success(x) => Right(x) - case Failure(x) => Left(Failed(x)) - - private def get(dir: String, name: String)(using dba: Dba): Either[Err, File] = - dba.get(s"${dir}/${name}") match - case Right(Some(x)) => unpickle(x) - case Right(None) => Left(FileNotExists(dir, name)) - case Left(e) => Left(e) - - def create(dir: String, name: String)(using dba: Dba): Either[Err, File] = - dba.get(s"${dir}/${name}") match - case Right(Some(_)) => Left(FileAlreadyExists(dir, name)) - case Right(None) => - val f = File(name, count=0, size=0L, dir=false) - val x = pickle(f) - for { - _ <- dba.put(s"${dir}/${name}", x) - } yield f - case Left(e) => Left(e) - - def append(dir: String, name: String, data: Array[Byte])(using dba: Dba): Either[Err, File] = - append(dir, name, data, data.length) - - def append(dir: String, name: String, data: Array[Byte], length: Int)(using dba: Dba): Either[Err, File] = - @tailrec def writeChunks(count: Int, rem: Array[Byte]): Either[Err, Int] = - rem.splitAt(chunkLength) match - case (xs, _) if xs.length == 0 => Right(count) - case (xs, ys) => - dba.put(s"${dir}/${name}_chunk_${count+1}", xs) match - case Right(_) => writeChunks(count+1, rem=ys) - case Left(e) => Left(e) - for { - _ <- (length == 0).fold(Left(InvalidArgument("data is empty")), Right(())) - file <- get(dir, name) - count <- writeChunks(file.count, rem=data) - file1 = file.copy(count=count, size=file.size+length) - file2 = pickle(file1) - _ <- dba.put(s"${dir}/${name}", file2) - } yield file1 - - def size(dir: String, name: String)(using Dba): Either[Err, Long] = - get(dir, name).map(_.size) - - def stream(dir: String, name: String)(using dba: Dba): Either[Err, LazyList[Either[Err, Array[Byte]]]] = - get(dir, name).map(_.count).flatMap{ - case n if n < 0 => Left(Fail(s"impossible count=${n}")) - case 0 => Right(LazyList.empty) - case n if n > 0 => - def k(i: Int) = s"${dir}/${name}_chunk_${i}" - Right(LazyList.range(1, n+1).map(i => dba.get(k(i)).flatMap(_.cata(Right(_), Left(KeyNotFound))))) - } - - def delete(dir: String, name: String)(using dba: Dba): Either[Err, File] = - for { - file <- get(dir, name) - _ <- LazyList.range(1, file.count+1).map(i => dba.delete(s"${dir}/${name}_chunk_${i}")).sequence_ - _ <- dba.delete(s"${dir}/${name}") - } yield file - - def copy(dir: String, name: (String, String))(using dba: Dba): Either[Err, File] = - val (fromName, toName) = name - for { - from <- get(dir, fromName) - _ <- get(dir, toName).fold( - l => l match { - case _: FileNotExists => Right(()) - case _ => Left(l) - }, - _ => Left(FileAlreadyExists(dir, toName)) - ) - _ <- LazyList.range(1, from.count+1).map(i => for { - x <- { - val k = s"${dir}/${fromName}_chunk_${i}" - dba.get(k).flatMap(_.cata(Right(_), Left(KeyNotFound))) - } - _ <- dba.put(s"${dir}/${toName}_chunk_${i}", x) - } yield ()).sequence_ - to = File(toName, from.count, from.size, from.dir) - x = pickle(to) - _ <- dba.put(s"${dir}/${toName}", x) - } yield to diff --git a/kvs/src/main/scala/idx.scala b/kvs/src/main/scala/idx.scala deleted file mode 100644 index 903c4848..00000000 --- a/kvs/src/main/scala/idx.scala +++ /dev/null @@ -1,149 +0,0 @@ -package zd.kvs -package idx - -import proto.* - -/** - * Feed of entries: [top] ----prev-> [en] ----prev-> [empty] - */ -object IdxHandler: - opaque type Fid = String - def Fid(fid: String): Fid = fid - - case class Fd - ( @N(1) id: Fid - , @N(2) top: String = empty - ) - - case class Idx - ( @N(1) fid: Fid - , @N(2) id: String - , @N(3) prev: String - ) - - object Idx: - def apply(fid: Fid, id: String): Idx = Idx(fid, id, empty) - - given MessageCodec[Fd] = caseCodecAuto - - def update(fd: Fd, top: String)(using dba: Dba): Either[Err, Unit] = - dba.put(fd.id, encode(fd.copy(top=top))).void - - def create(fid: Fid)(using dba: Dba): Either[Err, Fd] = - val fd = Fd(fid) - dba.put(fd.id, encode(fd)).map(_ => fd) - - def get(fid: Fid)(using dba: Dba): Either[Err, Option[Fd]] = - dba.get(fid).map(_.map(decode)) - - def delete(fid: Fid)(using dba: Dba): Either[Err, Unit] = - dba.delete(fid).void - - type A = Idx - given MessageCodec[A] = caseCodecAuto - - private def key(fid: Fid, id: String): String = s"${fid}.${id}" - - private def _put(en: A)(using dba: Dba): Either[Err, A] = - dba.put(key(en.fid, en.id), encode(en)).map(_ => en) - - def get(fid: Fid, id: String)(using dba: Dba): Either[Err, Option[A]] = - dba.get(key(fid, id)).map(_.map(decode)) - - private def getOrFail(fid: Fid, id: String)(using dba: Dba): Either[Err, A] = - val k = key(fid, id) - dba.get(k).flatMap{ - case Some(x) => Right(decode(x)) - case None => Left(KeyNotFound) - } - - private def delete(fid: Fid, id: String)(using dba: Dba): Either[Err, Unit] = - dba.delete(key(fid, id)).void - - private def nextid(fid: Fid)(using dba: Dba): Either[Err, String] = - def key(fid: String): String = s"IdCounter.${fid}" - dba.get(key(fid)).flatMap{ v => - val prev = v.map(String(_, "utf8").nn).getOrElse("0") - val next = (prev.toLong+1).toString - dba.put(key(fid), next.getBytes("utf8").nn).map(_ => next) - } - - /** - * Adds the entry to the container - * Creates the container if it's absent - * @param en entry to add (prev is ignored). If id is empty it will be generated - */ - def add(en: A)(using dba: Dba): Either[Err, A] = - get(en.fid).flatMap(_.cata(Right(_), create(en.fid))).flatMap{ (fd: Fd) => - ( if (en.id == empty) - nextid(en.fid) // generate ID if it is empty - else - get(en.fid, en.id).flatMap( // id of entry must be unique - _.cata(_ => Left(EntryExists(key(en.fid, en.id))), Right(en.id)) - ) - ).map(id => en.copy(id=id, prev=fd.top)).flatMap{ en => - // add new entry with prev pointer - _put(en).flatMap{ en => - // update feed's top - update(fd, top=en.id).map(_ => en) - } - } - } - - /** - * Puts the entry to the container - * If entry don't exists in containter create container and add it to the head - * If entry exists in container, put it in the same place - * @param en entry to put (prev is ignored) - */ - def put(en: A)(using Dba): Either[Err, A] = - get(en.fid, en.id).fold( - l => Left(l), - r => r.cata(x => _put(en.copy(x.prev)), add(en)) - ) - - /** Iterates through container and return the stream of entries. - * - * Stream is FILO ordered (most recent is first). - * @param from if specified then return entries after this entry - */ - def all(fid: Fid)(using Dba): Either[Err, LazyList[Either[Err, A]]] = - all(fid, from=None) - - def all(fid: Fid, from: Option[A])(using Dba): Either[Err, LazyList[Either[Err, A]]] = - def _stream(id: String): LazyList[Either[Err, A]] = - id match - case `empty` => LazyList.empty - case _ => - val en = getOrFail(fid, id) - en match - case Right(e) => LazyList.cons(en, _stream(e.prev)) - case _ => LazyList(en) - from match - case None => get(fid).map(_.cata(x => _stream(x.top), LazyList.empty)) - case Some(en) => Right(_stream(en.prev)) - - def remove(_fid: Fid, _id: String)(using Dba): Either[Err, Unit] = - // get entry to delete - getOrFail(_fid, _id).flatMap{ en => - val id = en.id - val fid = en.fid - val prev = en.prev - get(fid).flatMap(_.cata(Right(_), Left(KeyNotFound))).flatMap{ fd => - val top = fd.top - ( if (id == top) - // change top and decrement - update(fd, top=prev) - else - // find entry which points to this one (next) - LazyList.iterate(start=getOrFail(fid,top))(_.flatMap(x=>getOrFail(fid,x.prev))) - .takeWhile(_.isRight) - .flatMap(_.toOption) - .find(_.prev==id) - .toRight(KeyNotFound) - // change link - .flatMap(next => _put(next.copy(prev=prev))) - ).flatMap(_ => delete(fid, id)) // delete entry - } - } -end IdxHandler diff --git a/kvs/src/main/scala/jmx.scala b/kvs/src/main/scala/jmx.scala deleted file mode 100644 index 817bbd3e..00000000 --- a/kvs/src/main/scala/jmx.scala +++ /dev/null @@ -1,36 +0,0 @@ -package zd.kvs - -import java.lang.management.ManagementFactory -import javax.management.{ObjectName,StandardMBean} -import scala.util.* - -/** Kvs management access */ -trait KvsMBean { - def unsafe_save(path: String): String - def unsafe_load(path: String): String - def unsafe_compact(): String -} - -class KvsJmx(kvs: Kvs) { - private val server = ManagementFactory.getPlatformMBeanServer.nn - private val name = new ObjectName("kvs:type=Kvs") - - def registerMBean(): Unit = { - val mbean = new StandardMBean(classOf[KvsMBean]) with KvsMBean { - def unsafe_save(path: String): String = kvs.dump.save(path).fold(_.toString, identity) - def unsafe_load(path: String): String = kvs.dump.load(path).fold(_.toString, identity) - - def unsafe_compact(): String = { - val t = System.nanoTime - kvs.compact() - s"done in ${(System.nanoTime - t) / 1000000} ms" - } - } - Try(server.registerMBean(mbean,name)) - () - } - - def unregisterMBean(): Unit = { - Try(server.unregisterMBean(name)) - } -} diff --git a/kvs/src/main/scala/kvs.scala b/kvs/src/main/scala/kvs.scala deleted file mode 100644 index 84ce9985..00000000 --- a/kvs/src/main/scala/kvs.scala +++ /dev/null @@ -1,96 +0,0 @@ -package zd.kvs - -import akka.actor.* -import akka.event.* -import scala.concurrent.* - -trait ReadableEl: - def get[A: ElHandler](k: String): Either[Err, Option[A]] - -trait WritableEl extends ReadableEl: - def put[A: ElHandler](k: String, el: A): Either[Err, A] - def delete[A: ElHandler](k: String): Either[Err, Unit] - -trait ReadableFile: - def stream(dir: String, name: String)(using FileHandler): Either[Err, LazyList[Either[Err, Array[Byte]]]] - def size(dir: String, name: String)(using FileHandler): Either[Err, Long] - -trait WritableFile extends ReadableFile: - def create(dir: String, name: String)(using FileHandler): Either[Err, File] - def append(dir: String, name: String, chunk: Array[Byte])(using FileHandler): Either[Err, File] - def delete(dir: String, name: String)(using FileHandler): Either[Err, File] - def copy(dir: String, name: (String, String))(using FileHandler): Either[Err, File] - -class ReadableIdx(using Dba): - type Fid = idx.IdxHandler.Fid - type Fd = idx.IdxHandler.Fd - type A = idx.IdxHandler.Idx - protected val h = idx.IdxHandler - def get(fid: Fid): Either[Err, Option[Fd]] = h.get(fid) - def all(fid: Fid, from: Option[A]=None): Either[Err, LazyList[Either[Err, A]]] = h.all(fid, from) - def get(fid: Fid, id: String): Either[Err, Option[A]] = h.get(fid, id) - def getOrFail(fid: Fid, id: String): Either[Err, A] = - get(fid, id).flatMap{ - case None => Left(KeyNotFound) - case Some(x) => Right(x) - } - def exists_b(fid: Fid, id: String): Either[Err, Boolean] = h.get(fid, id).map(_.cata(_ => true, false)) - def not_exists(fid: Fid, id: String): Either[Err, Unit] = h.get(fid, id).flatMap(_.cata(_ => Left(EntryExists(id)), Right(unit))) -end ReadableIdx - -class WritableIdx(using Dba) extends ReadableIdx: - def update(fd: Fd, top: String): Either[Err, Unit] = h.update(fd, top) - def create(fid: Fid): Either[Err, Fd] = h.create(fid) - def delete(fid: Fid): Either[Err, Unit] = h.delete(fid) - - def add(a: A): Either[Err, A] = h.add(a) - def put(a: A): Either[Err, A] = h.put(a) - def remove(fid: Fid, id: String): Either[Err, Unit] = h.remove(fid, id) - -trait ReadableKvs: - val el: ReadableEl - val file: ReadableFile - val index: ReadableIdx - -trait WritableKvs extends ReadableKvs: - val el: WritableEl - val file: WritableFile - val index: WritableIdx - -object Kvs: - def apply(system: ActorSystem): Kvs = rng(system) - def rng(system: ActorSystem): Kvs = new Kvs(using Rng(system)) - def mem(): Kvs = new Kvs(using Mem()) - def fs(): Kvs = ??? - def sql(): Kvs = ??? - def rks(system: ActorSystem, dir: String): Kvs = new Kvs(using Rks(system, dir)) -end Kvs - -class Kvs(using val dba: Dba) extends WritableKvs, AutoCloseable: - import dba.R - - val el = new WritableEl: - def put[A](k: String, el: A)(using h: ElHandler[A]): Either[Err, A] = h.put(k,el) - def get[A](k: String)(using h: ElHandler[A]): Either[Err, Option[A]] = h.get(k) - def delete[A](k: String)(using h: ElHandler[A]): Either[Err, Unit] = h.delete(k) - - val file = new WritableFile: - def create(dir: String, name: String)(using h: FileHandler): Either[Err, File] = h.create(dir, name) - def append(dir: String, name: String, chunk: Array[Byte])(using h: FileHandler): Either[Err, File] = h.append(dir, name, chunk) - def stream(dir: String, name: String)(using h: FileHandler): Either[Err, LazyList[Either[Err, Array[Byte]]]] = h.stream(dir, name) - def size(dir: String, name: String)(using h: FileHandler): Either[Err, Long] = h.size(dir, name) - def delete(dir: String, name: String)(using h: FileHandler): Either[Err, File] = h.delete(dir, name) - def copy(dir: String, name: (String, String))(using h: FileHandler): Either[Err, File] = h.copy(dir, name) - - val index = new WritableIdx - - object dump: - def save(path: String): R[String] = dba.save(path) - def load(path: String): R[String] = dba.load(path) - - def onReady(): Future[Unit] = dba.onReady() - def compact(): Unit = dba.compact() - def deleteByKeyPrefix(k: dba.K): R[Unit] = dba.deleteByKeyPrefix(k) - - def close(): Unit = dba.close() -end Kvs diff --git a/kvs/src/main/scala/leveldb/leveldb.scala b/kvs/src/main/scala/leveldb/leveldb.scala deleted file mode 100644 index 4b0c8180..00000000 --- a/kvs/src/main/scala/leveldb/leveldb.scala +++ /dev/null @@ -1,195 +0,0 @@ -package leveldbjnr - -import java.io.File -import java.nio.file.{Files, StandardCopyOption} -import jnr.ffi.byref.{NumberByReference, PointerByReference} -import jnr.ffi.{LibraryLoader, LibraryOption, Pointer, TypeAlias} -import scala.util.{Try} - -object LevelDb { - private def copyLib(name: String): Either[Throwable, Long] = { - val is = classOf[LevelDb].getResourceAsStream(s"/lib/${name}") - val dest = new File(s"./tmp/${name}") - Try { - dest.mkdirs() - Files.copy(is, dest.toPath(), StandardCopyOption.REPLACE_EXISTING) - }.toEither - } - - copyLib("libleveldb.dylib") - copyLib("libleveldb.so") - copyLib("leveldb.dll") - sys.props += "java.library.path" -> "./tmp/" - val lib = LibraryLoader.create(classOf[Api]).nn.option(LibraryOption.IgnoreError, null).nn.failImmediately().nn.load("leveldb").nn - - private[leveldbjnr] def checkError(error: PointerByReference): Either[Throwable, Unit] = { - val str = error.getValue - val x = if (str != null) Left(new Exception(str.getString(0))) - else Right(()) - lib.leveldb_free(str) - x - } - - def open(path: String): Either[Throwable, LevelDb] = { - val opts = lib.leveldb_options_create() - lib.leveldb_options_set_create_if_missing(opts, 1) - lib.leveldb_options_set_write_buffer_size(opts, 200*1024*1024) - lib.leveldb_options_set_max_open_files(opts, 2500) - lib.leveldb_options_set_block_size(opts, 64*1024) - val filterpolicy = lib.leveldb_filterpolicy_create_bloom(10) - lib.leveldb_options_set_filter_policy(opts, filterpolicy) - val cache = lib.leveldb_cache_create_lru(500*1024*1024) - lib.leveldb_options_set_cache(opts, cache) - val error = new PointerByReference - val leveldb = lib.leveldb_open(opts, path, error) - lib.leveldb_options_destroy(opts) - // lib.leveldb_cache_destroy(cache) - // lib.leveldb_filterpolicy_destroy(filterpolicy) - checkError(error).map(_ => LevelDb(leveldb)) - } - - def destroy(path: String): Either[Throwable, Unit] = { - val opts = lib.leveldb_options_create() - val error = new PointerByReference - lib.leveldb_destroy_db(opts, path, error) - lib.leveldb_options_destroy(opts) - checkError(error) - } - - def version: (Int, Int) = { - (lib.leveldb_major_version(), lib.leveldb_minor_version()) - } -} - -case class LevelDb(leveldb: Pointer) { - import LevelDb.{lib, checkError} - - def get(key: Array[Byte], readOptions: ReadOpts): Either[Throwable, Option[Array[Byte]]] = { - val resultLengthPointer = new NumberByReference(TypeAlias.size_t) - val error = new PointerByReference - val result = Option(lib.leveldb_get(leveldb, readOptions.pointer, key, key.length.toLong, resultLengthPointer, error)) - checkError(error).map{ _ => - result.map{ result => - val resultLength = resultLengthPointer.intValue - val resultAsByteArray = new Array[Byte](resultLength) - result.nn.get(0, resultAsByteArray, 0, resultLength) - lib.leveldb_free(result) - resultAsByteArray - } - } - } - - def put(key: Array[Byte], value: Array[Byte], writeOptions: WriteOpts): Either[Throwable, Unit] = { - val error = new PointerByReference - lib.leveldb_put(leveldb, writeOptions.pointer, key, key.length.toLong, value, value.length.toLong, error) - checkError(error) - } - - def write(writeBatch: WriteBatch, writeOptions: WriteOpts): Either[Throwable, Unit] = { - val error = new PointerByReference - lib.leveldb_write(leveldb, writeOptions.pointer, writeBatch.pointer, error) - checkError(error) - } - - def delete(key: Array[Byte], writeOptions: WriteOpts): Either[Throwable, Unit] = { - val error = new PointerByReference - lib.leveldb_delete(leveldb, writeOptions.pointer, key, key.length.toLong, error) - checkError(error) - } - - def iter(): Iter = { - new Iter(leveldb) - } - - def compact(): Unit = { - lib.leveldb_compact_range(leveldb, null, 0L, null, 0L) - } - - def close(): Unit = { - lib.leveldb_close(leveldb) - } -} - -class Iter(leveldb: Pointer) { - import LevelDb.lib - - val p = lib.leveldb_create_iterator(leveldb, new ReadOpts().pointer) - - def seek(key: Array[Byte]): Unit = { - lib.leveldb_iter_seek(p, key, key.length.toLong) - } - - def seek_to_first(): Unit = { - lib.leveldb_iter_seek_to_first(p) - } - - def valid(): Boolean = { - lib.leveldb_iter_valid(p) != 0.toByte - } - - def next(): Unit = { - lib.leveldb_iter_next(p) - } - - def key(): Array[Byte] = { - val resultLengthPointer = new NumberByReference(TypeAlias.size_t) - val resultPointer = lib.leveldb_iter_key(p, resultLengthPointer) - val resultLength = resultLengthPointer.intValue - val resultAsByteArray = new Array[Byte](resultLength) - resultPointer.get(0, resultAsByteArray, 0, resultLength) - resultAsByteArray - } - - def close(): Unit = { - val _ = lib.leveldb_iter_destroy(p) - } -} - -case class WriteOpts(sync: Boolean = false) { - import LevelDb.lib - - private[leveldbjnr] val pointer: Pointer = lib.leveldb_writeoptions_create() - - lib.leveldb_writeoptions_set_sync(pointer, if (sync) 1 else 0) - - def close(): Unit = { - lib.leveldb_writeoptions_destroy(pointer) - } -} - -case class ReadOpts(verifyChecksum: Boolean = false, fillCache: Boolean = true) { - import LevelDb.lib - - private[leveldbjnr] val pointer: Pointer = lib.leveldb_readoptions_create() - - lib.leveldb_readoptions_set_verify_checksums(pointer, if (verifyChecksum) 1 else 0) - lib.leveldb_readoptions_set_fill_cache(pointer, if (fillCache) 1 else 0) - - def close(): Unit = { - lib.leveldb_readoptions_destroy(pointer) - } -} - -case class WriteBatch() { - import LevelDb.lib - - private[leveldbjnr] val pointer: Pointer = lib.leveldb_writebatch_create() - - def put(key: Array[Byte], value: Array[Byte]): Unit = { - lib.leveldb_writebatch_put(pointer, key, key.length.toLong, value, value.length.toLong) - } - - def delete(key: Array[Byte]): Unit = { - lib.leveldb_writebatch_delete(pointer, key, key.length.toLong) - } - - def clear(): Unit = { - lib.leveldb_writebatch_clear(pointer) - } - - def close(): Unit = { - lib.leveldb_writebatch_destroy(pointer) - } -} - -given [A]: CanEqual[A, A | Null] = CanEqual.derived \ No newline at end of file diff --git a/kvs/src/main/scala/leveldb/native.scala b/kvs/src/main/scala/leveldb/native.scala deleted file mode 100644 index 2c6595fd..00000000 --- a/kvs/src/main/scala/leveldb/native.scala +++ /dev/null @@ -1,181 +0,0 @@ -package leveldbjnr - -import jnr.ffi.Pointer -import jnr.ffi.byref.{PointerByReference, NumberByReference} -import jnr.ffi.annotations.{In, Out} - -trait Api { - def leveldb_open(@In options: Pointer, - @In name: String, - errptr: PointerByReference): Pointer - - def leveldb_close(levelDB: Pointer): Unit - - def leveldb_get(levelDB: Pointer, - @In options: Pointer, - @In key: Array[Byte], - @In keylen: Long, - @Out vallen: NumberByReference, - errptr: PointerByReference): Pointer - - def leveldb_put(levelDB: Pointer, - @In options: Pointer, - @In key: Array[Byte], - @In keylen: Long, - @In `val`: Array[Byte], - @In vallen: Long, - errptr: PointerByReference): Unit - - def leveldb_delete(levelDB: Pointer, - @In options: Pointer, - @In key: Array[Byte], - @In keylen: Long, - errptr: PointerByReference): Unit - - def leveldb_write(levelDB: Pointer, - @In options: Pointer, - @In batch: Pointer, - errptr: PointerByReference): Unit - - def leveldb_property_value(levelDB: Pointer, @In propname: String): String - - def leveldb_compact_range(levelDB: Pointer, - @In start_key: Array[Byte] | Null, - @In start_key_len: Long, - @In limit_key: Array[Byte] | Null, - @In limit_key_len: Long): Unit - - def leveldb_destroy_db(@In options: Pointer, - @In name: String, - errptr: PointerByReference): Unit - - def leveldb_repair_db(@In options: Pointer, - @In name: String, - errptr: PointerByReference): Unit - - def leveldb_major_version(): Int - - def leveldb_minor_version(): Int - - def leveldb_options_create(): Pointer - - def leveldb_options_destroy(options: Pointer): Unit - - def leveldb_options_set_comparator(options: Pointer, - @In comparator: Pointer): Unit - - def leveldb_options_set_filter_policy(options: Pointer, - @In filterPolicy: Pointer): Unit - - def leveldb_options_set_create_if_missing(options: Pointer, - @In value: Byte): Unit - - def leveldb_options_set_error_if_exists(options: Pointer, - @In value: Byte): Unit - - def leveldb_options_set_paranoid_checks(options: Pointer, - @In value: Byte): Unit - - def leveldb_options_set_env(options: Pointer, @In env: Pointer): Unit - - def leveldb_options_set_info_log(options: Pointer, @In logger: Pointer): Unit - - def leveldb_options_set_write_buffer_size(options: Pointer, - @In writeBufferSize: Long): Unit - - def leveldb_options_set_max_open_files(options: Pointer, - @In maxOpenFiles: Int): Unit - - def leveldb_options_set_cache(options: Pointer, @In cache: Pointer): Unit - - def leveldb_options_set_block_size(options: Pointer, - @In blockSize: Long): Unit - - def leveldb_options_set_block_restart_interval( - options: Pointer, - @In blockRestartInterval: Int): Unit - - def leveldb_options_set_compression(options: Pointer, - @In compressionType: Int): Unit - - def leveldb_readoptions_create(): Pointer - - def leveldb_readoptions_destroy(readOptions: Pointer): Unit - - def leveldb_readoptions_set_verify_checksums(readOptions: Pointer, - @In value: Byte): Unit - - def leveldb_readoptions_set_fill_cache(readOptions: Pointer, - @In value: Byte): Unit - - def leveldb_readoptions_set_snapshot(readOptions: Pointer, - @In snapshot: Pointer): Unit - - def leveldb_writeoptions_create(): Pointer - - def leveldb_writeoptions_destroy(writeOptions: Pointer): Unit - - def leveldb_writeoptions_set_sync(writeOptions: Pointer, - @In value: Byte): Unit - - def leveldb_cache_create_lru(@In capacity: Long): Pointer - - def leveldb_cache_destroy(cache: Pointer): Unit - - def leveldb_create_default_env(): Pointer - - def leveldb_env_destroy(env: Pointer): Unit - - def leveldb_filterpolicy_destroy(filterPolicy: Pointer): Unit - - def leveldb_filterpolicy_create_bloom(@In bits_per_key: Int): Pointer - - def leveldb_create_iterator(levelDB: Pointer, @In options: Pointer): Pointer - - def leveldb_iter_destroy(iterator: Pointer): Pointer - - def leveldb_iter_valid(@In iterator: Pointer): Byte - - def leveldb_iter_seek_to_first(iterator: Pointer): Unit - - def leveldb_iter_seek_to_last(iterator: Pointer): Unit - - def leveldb_iter_seek(iterator: Pointer, - @In k: Array[Byte], - @In klen: Long): Unit - - def leveldb_iter_next(iterator: Pointer): Unit - - def leveldb_iter_prev(iterator: Pointer): Unit - - def leveldb_iter_key(@In iterator: Pointer, - @Out klen: NumberByReference): Pointer - - def leveldb_iter_value(@In iterator: Pointer, - @Out vlen: NumberByReference): Pointer - - def leveldb_iter_get_error(@In iterator: Pointer, - errptr: PointerByReference): Unit - - def leveldb_create_snapshot(levelDB: Pointer): Pointer - - def leveldb_release_snapshot(levelDB: Pointer, @In snapshot: Pointer): Unit - - def leveldb_writebatch_create(): Pointer - - def leveldb_writebatch_destroy(writeBatch: Pointer): Unit - - def leveldb_writebatch_clear(writeBatch: Pointer): Unit - - def leveldb_writebatch_put(writeBatch: Pointer, - @In key: Array[Byte], - @In klen: Long, - @In `val`: Array[Byte], - @In vlen: Long): Unit - - def leveldb_writebatch_delete(writeBatch: Pointer, - @In key: Array[Byte], - @In klen: Long): Unit - - def leveldb_free(pointer: Pointer | Null): Unit -} diff --git a/kvs/src/main/scala/package.scala b/kvs/src/main/scala/package.scala deleted file mode 100644 index 367acb0e..00000000 --- a/kvs/src/main/scala/package.scala +++ /dev/null @@ -1,5 +0,0 @@ -package zd.kvs - -val empty = "empty_8fc62083-b0d1-49cc-899c-fbb9ab177241" // DON'T change ever - -given [A]: CanEqual[None.type, A] = CanEqual.derived diff --git a/kvs/src/main/scala/rks/Dump.scala b/kvs/src/main/scala/rks/Dump.scala deleted file mode 100644 index 777e7ca8..00000000 --- a/kvs/src/main/scala/rks/Dump.scala +++ /dev/null @@ -1,95 +0,0 @@ -package zd.rks - -import akka.actor.{Actor, ActorLogging, ActorRef, Props, PoisonPill} -import java.time.format.{DateTimeFormatter} -import java.time.{LocalDateTime} -import org.rocksdb.* -import zd.dump.{DumpIO} - -object DumpProcessor { - def props(db: RocksDB): Props = Props(new DumpProcessor(db)) - - case class Load(path: String) - case class Save(path: String) -} - -class DumpProcessor(db: RocksDB) extends Actor with ActorLogging { - def receive = { - case DumpProcessor.Load(path) => - log.info(s"Loading dump: path=${path}".green) - DumpIO.props(path) match { - case Left(t) => - val msg = s"invalid path=$path" - log.error(cause=t, message=msg) - // TODO unlock - sender ! msg - context.stop(self) - case Right(a) => - val dumpIO = context.actorOf(a) - dumpIO ! DumpIO.ReadNext - context.become(load(dumpIO, sender)()) - } - case DumpProcessor.Save(path) => - log.info(s"Saving dump: path=${path}".green) - val timestamp = LocalDateTime.now.nn.format(DateTimeFormatter.ofPattern("yyyy.MM.dd-HH.mm.ss")) - val dumpPath = s"${path}/rks_dump_${timestamp}" - DumpIO.props(dumpPath) match { - case Left(t) => - val msg = s"invalid path=$path" - log.error(cause=t, message=msg) - // TODO unlock - sender ! msg - context.stop(self) - case Right(a) => - val dumpIO = context.actorOf(a) - self ! DumpIO.PutDone - val it = db.newIterator().nn - it.seekToFirst() - context.become(save(it, dumpIO, sender, dumpPath)()) - } - } - - def save(it: RocksIterator, dumpIO: ActorRef, client: ActorRef, dumpPath: String): () => Receive = { - () => { - case _: DumpIO.PutDone.type => - if it.isValid() then - dumpIO ! DumpIO.Put(Vector(it.key().nn -> it.value().nn)) - it.next() - else - log.info(s"Dump is saved: path=${dumpPath}".green) - // TODO unlock - client ! dumpPath - dumpIO ! PoisonPill - context.stop(self) - } - } - - def load(dumpIO: ActorRef, client: ActorRef): () => Receive = { - var keysNumber: Long = 0L - var size: Long = 0L - var ksize: Long = 0L - - () => { - case res: DumpIO.ReadNextRes => - dumpIO ! DumpIO.ReadNext - keysNumber = keysNumber + res.kv.size - res.kv.foreach{ d => - ksize = ksize + d._1.size - size = size + d._2.size - - db.put(d._1, d._2) - } - - case _: DumpIO.ReadNextLast.type => - log.info(s"load info: load is completed, total keys=${keysNumber}, size=${size}, ksize=${ksize}") - log.info("Dump is loaded".green) - client ! "done" - // TODO unlock - dumpIO ! PoisonPill - context.stop(self) - } - } -} - -extension (value: String) - def green: String = s"\u001B[32m${value}\u001B[0m" \ No newline at end of file diff --git a/kvs/src/main/scala/rng/Data.scala b/kvs/src/main/scala/rng/Data.scala deleted file mode 100644 index c8ccf582..00000000 --- a/kvs/src/main/scala/rng/Data.scala +++ /dev/null @@ -1,47 +0,0 @@ -package zd.rng -package data - -import proto.* -import java.util.Arrays - -final class Data - ( @N(1) val key: Array[Byte] - , @N(2) val bucket: Int - , @N(3) val lastModified: Long - , @N(4) val vc: VectorClock - , @N(5) val value: Array[Byte] - ) { - def copy(vc: VectorClock): Data = { - new Data(key=key, bucket=bucket, lastModified=lastModified, vc=vc, value=value) - } - override def equals(other: Any): Boolean = other match { - case that: Data => - Arrays.equals(key, that.key) && - bucket == that.bucket && - lastModified == that.lastModified && - vc == that.vc && - Arrays.equals(value, that.value) - case _ => false - } - override def hashCode(): Int = { - val state = Seq[Any](key, bucket, lastModified, vc, value) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"Data(key=$key, bucket=$bucket, lastModified=$lastModified, vc=$vc, value=$value)" -} -object Data { - def apply(key: Array[Byte], bucket: Int, lastModified: Long, vc: VectorClock, value: Array[Byte]): Data = { - new Data(key=key, bucket=bucket, lastModified=lastModified, vc=vc, value=value) - } -} - -final case class BucketInfo - ( @N(1) vc: VectorClock - , @N(2) keys: Vector[Array[Byte]] - ) - -object codec { - import akka.cluster.given - implicit val bucketInfoCodec: MessageCodec[BucketInfo] = caseCodecAuto[BucketInfo] - implicit val dataCodec: MessageCodec[Data] = classCodecAuto[Data] -} diff --git a/kvs/src/main/scala/rng/Dump.scala b/kvs/src/main/scala/rng/Dump.scala deleted file mode 100644 index 5ca96687..00000000 --- a/kvs/src/main/scala/rng/Dump.scala +++ /dev/null @@ -1,202 +0,0 @@ -package zd.rng - -import akka.actor.{Actor, ActorLogging, ActorRef, Props, PoisonPill} -import akka.pattern.ask -import akka.util.Timeout -import java.time.format.{DateTimeFormatter} -import java.time.{LocalDateTime} -import zd.rng.data.{Data} -import zd.rng.model.{DumpBucketData, DumpGetBucketData} -import zd.dump.{DumpIO, DumpIterate} -import scala.collection.immutable.{SortedMap} -import scala.concurrent.duration.* -import scala.concurrent.{Await} -import scala.util.{Try} - -object DumpProcessor { - def props(): Props = Props(new DumpProcessor) - - final case class Load(path: String) - final case class Save(buckets: SortedMap[Bucket, PreferenceList], path: String) - final case class Iterate(buckets: SortedMap[Bucket, PreferenceList], f: (Key, Value) => Unit) -} - -class DumpProcessor extends Actor with ActorLogging { - implicit val timeout: Timeout = Timeout(120 seconds) - val maxBucket: Bucket = context.system.settings.config.getInt("ring.buckets") - 1 - val stores = SelectionMemorize(context.system) - def receive = waitForStart - - def waitForStart: Receive = { - case DumpProcessor.Load(path) => - log.info(s"Loading dump: path=${path}".green) - DumpIO.props(path) match { - case Left(t) => - val msg = s"invalid path=$path" - log.error(cause=t, message=msg) - sender ! msg - stores.get(addr(self), "ring_hash").fold( - _ ! RestoreState, - _ ! RestoreState, - ) - context.stop(self) - case Right(a) => - val dumpIO = context.actorOf(a) - dumpIO ! DumpIO.ReadNext - context.become(load(dumpIO, sender)()) - } - - case DumpProcessor.Save(buckets, path) => - log.info(s"Saving dump: path=${path}".green) - val timestamp = LocalDateTime.now.nn.format(DateTimeFormatter.ofPattern("yyyy.MM.dd-HH.mm.ss")) - val dumpPath = s"${path}/rng_dump_${timestamp}" - DumpIO.props(dumpPath) match { - case Left(t) => - val msg = s"invalid path=$path" - log.error(cause=t, message=msg) - stores.get(addr(self), "ring_hash").fold( - _ ! RestoreState, - _ ! RestoreState, - ) - sender ! msg - context.stop(self) - case Right(a) => - val dumpIO = context.actorOf(a) - buckets(0).foreach(n => stores.get(n, "ring_readonly_store").fold( - _ ! DumpGetBucketData(0), - _ ! DumpGetBucketData(0), - )) - context.become(save(buckets, dumpIO, sender, dumpPath)()) - } - - case DumpProcessor.Iterate(buckets, f) => - log.info(s"DumpProcessor. Iterating...".green) - val dumpIterate = context.actorOf(DumpIterate.props(f)) - buckets(0).foreach(n => stores.get(n, "ring_readonly_store").fold( - _ ! DumpGetBucketData(0), - _ ! DumpGetBucketData(0), - )) - context.become(save(buckets, dumpIterate, sender, "dump_iterate")()) - } - - def load(dumpIO: ActorRef, client: ActorRef): () => Receive = { - var keysNumber: Long = 0L - var size: Long = 0L - var ksize: Long = 0L - () => { - case res: DumpIO.ReadNextRes => - dumpIO ! DumpIO.ReadNext - keysNumber = keysNumber + res.kv.size - res.kv.to(LazyList).map{ d => - ksize = ksize + d._1.size - size = size + d._2.size - val putF = stores.get(addr(self), "ring_hash").fold( - _.ask(InternalPut(d._1, d._2)), - _.ask(InternalPut(d._1, d._2)), - ) - Try(Await.result(putF, timeout.duration)).toEither - }.sequence_ match { - case Right(_) => - if (keysNumber % 1000 == 0L) { - log.info(s"load info: write done, total keys=${keysNumber}, size=${size}, ksize=${ksize}") - } - case Left(t) => - log.error(cause=t, message="can't put") - client ! "can't put" - stores.get(addr(self), "ring_hash").fold( - _ ! RestoreState, - _ ! RestoreState, - ) - context.stop(dumpIO) - context.stop(self) - } - case _: DumpIO.ReadNextLast.type => - log.info(s"load info: load is completed, total keys=${keysNumber}, size=${size}, ksize=${ksize}") - log.info("Dump is loaded".green) - client ! "done" - stores.get(addr(self), "ring_hash").fold( - _ ! RestoreState, - _ ! RestoreState, - ) - dumpIO ! PoisonPill - context.stop(self) - } - } - - def save(buckets: SortedMap[Bucket, PreferenceList], dumpIO: ActorRef, client: ActorRef, dumpPath: String): () => Receive = { - var processBucket: Int = 0 - var keysNumber: Long = 0 - var collected: Vector[Vector[Data]] = Vector.empty - - var putQueue: Vector[DumpIO.Put] = Vector.empty - var readyToPut: Boolean = true - var pullWorking: Boolean = false - - def pull(): Unit = { - if (!pullWorking && putQueue.size < 50 && processBucket < maxBucket) { - processBucket = processBucket + 1 - pullWorking = true - buckets(processBucket).foreach(n => stores.get(n, "ring_readonly_store").fold( - _ ! DumpGetBucketData(processBucket), - _ ! DumpGetBucketData(processBucket), - )) - } - } - - def showInfo(msg: String): Unit = { - if (processBucket == maxBucket && putQueue.isEmpty) { - log.info(s"dump done: msg=${msg}, bucket=${processBucket}/${maxBucket}, total=${keysNumber}, putQueue=${putQueue.size}") - } else if (keysNumber % 10000 == 0L) { - log.info(s"dump info: msg=${msg}, bucket=${processBucket}/${maxBucket}, total=${keysNumber}, putQueue=${putQueue.size}") - } - } - - () => { - case res: (DumpBucketData) if processBucket == res.b => - collected = res.items.toVector +: collected - if (collected.size == buckets(processBucket).size) { - pullWorking = false - pull() - - val merged: Vector[Data] = MergeOps.forDump(collected.flatten) - collected = Vector.empty - keysNumber = keysNumber + merged.size - if (readyToPut) { - readyToPut = false - dumpIO ! DumpIO.Put(merged.map(x => x.key -> x.value)) - } else { - putQueue = DumpIO.Put(merged.map(x => x.key -> x.value)) +: putQueue - } - showInfo("main") - } - case res: DumpBucketData => - log.error(s"wrong bucket response, expected=${processBucket}, actual=${res.b}") - stores.get(addr(self), "ring_hash").fold( - _ ! RestoreState, - _ ! RestoreState, - ) - client ! "failed" - context.stop(dumpIO) - context.stop(self) - case _: DumpIO.PutDone.type => - if (putQueue.isEmpty) { - if (processBucket == maxBucket) { - log.info(s"Dump is saved: path=${dumpPath}".green) - stores.get(addr(self), "ring_hash").fold( - _ ! RestoreState, - _ ! RestoreState - ) - client ! dumpPath - dumpIO ! PoisonPill - context.stop(self) - } - readyToPut = true - } else { - putQueue.headOption.map(dumpIO ! _) - putQueue = putQueue.tail - } - pull() - showInfo("io") - } - } -} diff --git a/kvs/src/main/scala/rng/GatherDel.scala b/kvs/src/main/scala/rng/GatherDel.scala deleted file mode 100644 index ce9b94ac..00000000 --- a/kvs/src/main/scala/rng/GatherDel.scala +++ /dev/null @@ -1,52 +0,0 @@ -package zd.rng - -import akka.actor.* -import akka.cluster.Cluster -import scala.concurrent.duration.* - -class GatherDel(client: ActorRef, t: FiniteDuration, prefList: Set[Node], k: Key) extends FSM[FsmState, Set[Node]] with ActorLogging { - import context.system - - val config = system.settings.config.getConfig("ring").nn - val quorum = config.getIntList("quorum").nn - val W: Int = quorum.get(1).nn - val local: Address = Cluster(context.system).selfAddress - setTimer("send_by_timeout", "timeout", t) - - startWith(Collecting, prefList) - - when(Collecting){ - case Event("ok", nodesLeft) => - nodesLeft - addr1(sender) match { - case enough if prefList.size - enough.size == W => // W nodes removed key - client ! AckSuccess(None) - goto(Sent) using(enough) - case less => stay using(less) - } - - case Event("timeout", nodesLeft) => - //politic of revert is not needed because on read opperation removed data will be saved again, - //only notify client about failed opperation. - //deleted on other nodes but we don't know about it ? sorry, eventually consistency - client ! AckTimeoutFailed("del", new String(k, "UTF-8")) - stop() - } - - when(Sent){ - case Event("ok", data) => - data - addr1(sender) match { - case none if none.isEmpty => stop() - case nodes => stay using(nodes) - } - - case Event("timeout", _) => stop() - } - - def addr1(s: ActorRef) = if (addr(s).hasLocalScope) local else addr(s) - - initialize() -} - -object GatherDel { - def props(client: ActorRef, t: FiniteDuration, prefList: Set[Node], k: Key): Props = Props(new GatherDel(client, t, prefList, k)) -} diff --git a/kvs/src/main/scala/rng/GatherPut.scala b/kvs/src/main/scala/rng/GatherPut.scala deleted file mode 100644 index 1e7e44e3..00000000 --- a/kvs/src/main/scala/rng/GatherPut.scala +++ /dev/null @@ -1,97 +0,0 @@ -package zd.rng - -import akka.actor.{ActorLogging, ActorRef, FSM, Props, RootActorPath} -import zd.rng.data.Data -import zd.rng.model.{StoreGetAck, StorePut} -import scala.concurrent.duration.* -import java.util.Arrays - -final class PutInfo( - val key: Key - , val v: Value - , val N: Int - , val W: Int - , val bucket: Bucket - , val localAdr: Node - , val nodes: Set[Node] - ) { - override def equals(other: Any): Boolean = other match { - case that: PutInfo => - Arrays.equals(key, that.key) && - Arrays.equals(v, that.v) && - N == that.N && - W == that.W && - bucket == that.bucket && - localAdr == that.localAdr && - nodes == that.nodes - case _ => false - } - override def hashCode(): Int = { - val state = Seq[Any](key, v, N, W, bucket, localAdr, nodes) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"PutInfo(key=$key, v=$v, N=$N, W=$W, bucket=$bucket, localAdr=$localAdr, nodes=$nodes)" -} - -object PutInfo { - def apply(key: Key, v: Value, N: Int, W: Int, bucket: Bucket, localAdr: Node, nodes: Set[Node]): PutInfo = { - new PutInfo(key=key, v=v, N=N, W=W, bucket=bucket, localAdr=localAdr, nodes=nodes) - } -} - -object GatherPut { - def props(client: ActorRef, t: FiniteDuration, putInfo: PutInfo): Props = Props(new GatherPut(client, t, putInfo)) -} - -class GatherPut(client: ActorRef, t: FiniteDuration, putInfo: PutInfo) extends FSM[FsmState, Int] with ActorLogging { - - startWith(Collecting, 0) - setTimer("send_by_timeout", "timeout", t) - - when(Collecting){ - case Event(StoreGetAck(data), _) => - val vc = if (data.size == 1) { - data.head.vc - } else if (data.size > 1) { - data.map(_.vc).foldLeft(emptyVC)((sum, i) => sum.merge(i)) - } else { - emptyVC - } - val updatedData = Data(putInfo.key, putInfo.bucket, now_ms(), vc.:+(putInfo.localAdr.toString), putInfo.v) - mapInPut(putInfo.nodes, updatedData) - stay() - - case Event("ok", n) => - val n1 = n + 1 - if (n1 == putInfo.N) { - client ! AckSuccess(None) - stop() - } else if (n1 == putInfo.W) { - client ! AckSuccess(None) - goto (Sent) using n1 - } else { - stay using n1 - } - - case Event("timeout", _) => - client ! AckTimeoutFailed("put", new String(putInfo.key, "UTF-8")) - stop() - } - - // keep fsm running to avoid dead letter warnings - when(Sent){ - case Event("ok", n) => - val n1 = n + 1 - if (n1 == putInfo.N) stop() - else stay using n1 - case Event("timeout", _) => - stop() - } - - def mapInPut(nodes: Set[Node], d: Data) = { - val storeList = nodes.map(n => RootActorPath(n) / "user" / "ring_write_store") - storeList.foreach(ref => context.system.actorSelection(ref).tell(StorePut(d), self)) - } - - initialize() -} diff --git a/kvs/src/main/scala/rng/Hash.scala b/kvs/src/main/scala/rng/Hash.scala deleted file mode 100644 index e9f154dd..00000000 --- a/kvs/src/main/scala/rng/Hash.scala +++ /dev/null @@ -1,410 +0,0 @@ -package zd.rng - -import akka.actor.* -import akka.cluster.ClusterEvent.* -import akka.cluster.{Member, Cluster} -import com.typesafe.config.Config -import zd.rng.model.{StoreDelete, StoreGet, QuorumState, ChangeState} -import zd.rng.model.QuorumState.{QuorumStateUnsatisfied, QuorumStateReadonly, QuorumStateEffective} -import scala.collection.immutable.{SortedMap, SortedSet} -import scala.concurrent.duration.* -import java.util.Arrays -import leveldbjnr.* - -class Put(val k: Key, val v: Value) { - override def equals(other: Any): Boolean = other match { - case that: Put => - Arrays.equals(k, that.k) && - Arrays.equals(v, that.v) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(k, v) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"Put(k=$k, v=$v)" -} - -object Put { - def apply(k: Key, v: Value): Put = { - new Put(k=k, v=v) - } -} - -class Get(val k: Key) { - override def equals(other: Any): Boolean = other match { - case that: Get => - Arrays.equals(k, that.k) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(k) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"Get(k=$k)" -} - -object Get { - def apply(k: Key): Get = { - new Get(k=k) - } -} - -class Delete(val k: Key) { - override def equals(other: Any): Boolean = other match { - case that: Delete => - Arrays.equals(k, that.k) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(k) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"Delete(k=$k)" -} - -object Delete { - def apply(k: Key): Delete = { - new Delete(k=k) - } -} - -case class Save(path: String) -case class Iterate(f: (Key, Value) => Unit) -case class Load(path: String) -case class Iter(keyPrefix: Array[Byte]) -case class IterRes(keys: List[String]) - -case object RestoreState - -case object Ready - -class InternalPut(val k: Key, val v: Value) { - override def equals(other: Any): Boolean = other match { - case that: InternalPut => - Arrays.equals(k, that.k) && - Arrays.equals(v, that.v) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(k, v) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"InternalPut(k=$k, v=$v)" -} - -object InternalPut { - def apply(k: Key, v: Value): InternalPut = { - new InternalPut(k=k, v=v) - } -} - -case class HashRngData( - nodes: Set[Node], - buckets: SortedMap[Bucket, PreferenceList], - vNodes: SortedMap[Bucket, Node], - replication: Option[ActorRef], -) - -object Hash { - def props(leveldb: LevelDb): Props = Props(new Hash(leveldb)) -} - -// TODO available/not avaiable nodes -class Hash(leveldb: LevelDb) extends FSM[QuorumState, HashRngData] with ActorLogging { - import context.system - - val config: Config = system.settings.config.getConfig("ring").nn - - val quorum = config.getIntList("quorum").nn - val N: Int = quorum.get(0).nn - val W: Int = quorum.get(1).nn - val R: Int = quorum.get(2).nn - val gatherTimeout = Duration.fromNanos(config.getDuration("gather-timeout").nn.toNanos) - val vNodesNum = config.getInt("virtual-nodes") - val bucketsNum = config.getInt("buckets") - val cluster = Cluster(system) - val local: Node = cluster.selfAddress - val hashing = HashingExtension(system) - val actorsMem = SelectionMemorize(system) - - log.info(s"Ring configuration:".blue) - log.info(s"ring.quorum.N = ${N}".blue) - log.info(s"ring.quorum.W = ${W}".blue) - log.info(s"ring.quorum.R = ${R}".blue) - log.info(s"ring.leveldb.dir = ${config.getString("leveldb.dir")}".blue) - - startWith(QuorumStateUnsatisfied, HashRngData(Set.empty[Node], SortedMap.empty[Bucket, PreferenceList], SortedMap.empty[Bucket, Node], replication=None)) - - override def preStart() = { - cluster.subscribe(self, initialStateMode = InitialStateAsEvents, classOf[MemberUp], classOf[MemberRemoved]) - } - - override def postStop(): Unit = cluster.unsubscribe(self) - - when(QuorumStateUnsatisfied){ - case Event(_: Get, _) => - sender ! AckQuorumFailed("QuorumStateUnsatisfied") - stay() - case Event(_: Put, _) => - sender ! AckQuorumFailed("QuorumStateUnsatisfied") - stay() - case Event(_: Delete, _) => - sender ! AckQuorumFailed("QuorumStateUnsatisfied") - stay() - case Event(Save(_), _) => - sender ! AckQuorumFailed("QuorumStateUnsatisfied") - stay() - case Event(Iterate(_), _) => - sender ! AckQuorumFailed("QuorumStateUnsatisfied") - stay() - case Event(Load(_), _) => - sender ! AckQuorumFailed("QuorumStateUnsatisfied") - stay() - case Event(Iter(_), _) => - sender ! AckQuorumFailed("QuorumStateUnsatisfied") - stay() - case Event(RestoreState, _) => - log.warning("Don't know how to restore state when quorum is unsatisfied") - stay() - case Event(Ready, _) => - sender ! false - stay() - } - - when(QuorumStateReadonly){ - case Event(x: Get, data) => - doGet(x.k, sender, data) - stay() - case Event(RestoreState, data) => - val s = state(data.nodes.size) - data.nodes.foreach(n => actorsMem.get(n, "ring_hash").fold( - _ ! ChangeState(s), - _ ! ChangeState(s), - )) - goto(s) - case Event(Ready, _) => - sender ! false - stay() - - case Event(_: Put, _) => - sender ! AckQuorumFailed("QuorumStateReadonly") - stay() - case Event(_: Delete, _) => - sender ! AckQuorumFailed("QuorumStateReadonly") - stay() - case Event(Save(_), _) => - sender ! AckQuorumFailed("QuorumStateReadonly") - stay() - case Event(Iterate(_), _) => - sender ! AckQuorumFailed("QuorumStateReadonly") - stay() - case Event(Load(_), _) => - sender ! AckQuorumFailed("QuorumStateReadonly") - stay() - case Event(Iter(_), _) => - sender ! AckQuorumFailed("QuorumStateReadonly") - stay() - } - - when(QuorumStateEffective){ - case Event(Ready, _) => - sender ! true - stay() - - case Event(x: Get, data) => - doGet(x.k, sender, data) - stay() - case Event(x: Put, data) => - doPut(x.k, x.v, sender, data) - stay() - case Event(x: Delete, data) => - doDelete(x.k, sender, data) - stay() - - case Event(Save(path), data) => - data.nodes.foreach(n => actorsMem.get(n, "ring_hash").fold( - _ ! ChangeState(QuorumStateReadonly), - _ ! ChangeState(QuorumStateReadonly), - )) - val x = system.actorOf(DumpProcessor.props(), s"dump_wrkr-${now_ms()}") - x.forward(DumpProcessor.Save(data.buckets, path)) - goto(QuorumStateReadonly) - case Event(Iterate(f), data) => - data.nodes.foreach(n => actorsMem.get(n, "ring_hash").fold( - _ ! ChangeState(QuorumStateReadonly), - _ ! ChangeState(QuorumStateReadonly), - )) - val x = system.actorOf(DumpProcessor.props(), s"dump_wrkr-${now_ms()}") - x.forward(DumpProcessor.Iterate(data.buckets, f)) - goto(QuorumStateReadonly) - case Event(Load(path), data) => - data.nodes.foreach(n => actorsMem.get(n, "ring_hash").fold( - _ ! ChangeState(QuorumStateReadonly), - _ ! ChangeState(QuorumStateReadonly), - )) - val x = system.actorOf(DumpProcessor.props(), s"load_wrkr-${now_ms()}") - x.forward(DumpProcessor.Load(path)) - goto(QuorumStateReadonly) - case Event(Iter(keyPrefix), data) => - val it = leveldb.iter() - it.seek_to_first() - var keys = collection.mutable.ListBuffer.empty[String] - while (it.valid()) { - val key = it.key() - val idx = key.indexOfSlice(keyPrefix) - if (idx >= 0) { - val slice = key.slice(idx, key.length) - keys += new String(slice, "utf8") - } - it.next() - } - it.close() - sender ! IterRes(keys.toList) - stay() - case Event(RestoreState, _) => - log.info("State is already OK") - stay() - } - - /* COMMON FOR ALL STATES*/ - whenUnhandled { - case Event(MemberUp(member), data) => - val next = joinNodeToRing(member, data) - goto(next._1) using next._2 - case Event(MemberRemoved(member, prevState), data) => - val next = removeNodeFromRing(member, data) - goto(next._1) using next._2 - case Event(Ready, data) => - sender ! false - stay() - case Event(ChangeState(s), data) => - state(data.nodes.size) match { - case QuorumStateUnsatisfied => stay() - case _ => goto(s) - } - case Event(x: InternalPut, data) => - doPut(x.k, x.v, sender, data) - stay() - } - - def doDelete(k: Key, client: ActorRef, data: HashRngData): Unit = { - val nodes = nodesForKey(k, data) - val gather = system.actorOf(GatherDel.props(client, gatherTimeout, nodes, k)) - val stores = nodes.map{actorsMem.get(_, "ring_write_store")} - stores.foreach(_.fold( - _.tell(StoreDelete(k), gather), - _.tell(StoreDelete(k), gather), - )) - } - - def doPut(k: Key, v: Value, client: ActorRef, data: HashRngData): Unit = { - val nodes = availableNodesFrom(nodesForKey(k, data)) - val M = nodes.size - if (M >= W) { - val bucket = hashing `findBucket` k - val info = PutInfo(k, v, N, W, bucket, local, data.nodes) - val gather = system.actorOf(GatherPut.props(client, gatherTimeout, info)) - val node = if (nodes contains local) local else nodes.head - actorsMem.get(node, "ring_readonly_store").fold( - _.tell(StoreGet(k), gather), - _.tell(StoreGet(k), gather), - ) - } else { - client ! AckQuorumFailed("M >= W") - } - } - - def doGet(k: Key, client: ActorRef, data: HashRngData): Unit = { - val nodes = availableNodesFrom(nodesForKey(k, data)) - val M = nodes.size - if (M >= R) { - val gather = system.actorOf(GatherGet.props(client, gatherTimeout, M, R, k)) - val stores = nodes map { actorsMem.get(_, "ring_readonly_store") } - stores foreach (_.fold( - _.tell(StoreGet(k), gather), - _.tell(StoreGet(k), gather), - )) - } else { - client ! AckQuorumFailed("M >= R") - } - } - - def availableNodesFrom(l: Set[Node]): Set[Node] = { - val unreachableMembers = cluster.state.unreachable.map(m => m.address) - l filterNot (node => unreachableMembers contains node) - } - - def joinNodeToRing(member: Member, data: HashRngData): (QuorumState, HashRngData) = { - val newvNodes: Map[VNode, Node] = (1 to vNodesNum).view.map(vnode => { - hashing.hash(stob(member.address.hostPort).++(itob(vnode))) -> member.address - }).to(Map) - val updvNodes = data.vNodes ++ newvNodes - val nodes = data.nodes + member.address - val moved = bucketsToUpdate(bucketsNum - 1, Math.min(nodes.size,N), updvNodes, data.buckets) - data.replication map (context stop _) - val repl = syncNodes(moved) - val updData = HashRngData(nodes, data.buckets++moved, updvNodes, Some(repl)) - log.info(s"Node ${member.address} is joining ring. Nodes in ring = ${updData.nodes.size}, state = ${state(updData.nodes.size)}") - state(updData.nodes.size) -> updData - } - - def removeNodeFromRing(member: Member, data: HashRngData): (QuorumState, HashRngData) = { - log.info(s"Removing ${member} from ring") - val unusedvNodes: Set[VNode] = (1 to vNodesNum).view.map(vnode => hashing.hash(stob(member.address.hostPort).++(itob(vnode)))).to(Set) - val updvNodes = data.vNodes.filterNot(vn => unusedvNodes.contains(vn._1)) - val nodes = data.nodes - member.address - val moved = bucketsToUpdate(bucketsNum - 1, Math.min(nodes.size,N), updvNodes, data.buckets) - log.info(s"Will update ${moved.size} buckets") - data.replication map (context stop _) - val repl = syncNodes(moved) - val updData = HashRngData(nodes, data.buckets++moved, updvNodes, Some(repl)) - state(updData.nodes.size) -> updData - } - - def syncNodes(_buckets: SortedMap[Bucket,PreferenceList]): ActorRef = { - val empty = SortedMap.empty[Bucket,PreferenceList] - val buckets = _buckets.foldLeft(empty){ case (acc, (b, prefList)) => - if (prefList contains local) { - prefList.filterNot(_ == local) match { - case empty if empty.isEmpty => acc - case prefList => acc + (b -> prefList) - } - } else acc - } - val replication = context.actorOf(ReplicationSupervisor.props(buckets), s"repl-${now_ms()}") - replication ! "go-repl" - replication - } - - def state(nodes: Int): QuorumState = nodes match { - case 0 => QuorumStateUnsatisfied - case n if n >= Math.max(R, W) => QuorumStateEffective - case _ => QuorumStateReadonly - } - - def bucketsToUpdate( - maxBucket: Bucket - , nodesNumber: Int - , vNodes: SortedMap[Bucket, Node] - , buckets: SortedMap[Bucket, PreferenceList] - ): SortedMap[Bucket, PreferenceList] = - (0 to maxBucket).foldLeft(SortedMap.empty[Bucket, PreferenceList])( - (acc, b) => { - val prefList = hashing.findNodes(b * hashing.bucketRange, vNodes, nodesNumber) - buckets.get(b) match { - case None => acc + (b -> prefList) - case Some(`prefList`) => acc - case _ => acc + (b -> prefList) - } - } - ) - - implicit val ord: Ordering[Node] = Ordering.by[Node, String](n => n.hostPort) - def nodesForKey(k: Key, data: HashRngData): PreferenceList = data.buckets.get(hashing.findBucket(k)) match { - case None => SortedSet.empty[Node] - case Some(nods) => nods - } - - initialize() -} diff --git a/kvs/src/main/scala/rng/Msg.scala b/kvs/src/main/scala/rng/Msg.scala deleted file mode 100644 index 93750d3d..00000000 --- a/kvs/src/main/scala/rng/Msg.scala +++ /dev/null @@ -1,150 +0,0 @@ -package zd.rng -package model - -import proto.* -import zd.rng.data.* -import java.util.Arrays - -sealed trait Msg - -@N(1) final case class ChangeState - ( @N(1) s: QuorumState - ) extends Msg - -@N(2) final case class DumpBucketData - ( @N(1) b: Int - , @N(2) items: Vector[Data] - ) extends Msg - -@N(3) final class DumpEn - ( @N(1) val k: Array[Byte] - , @N(2) val v: Array[Byte] - , @N(3) val nextKey: Array[Byte] - ) extends Msg { - override def equals(other: Any): Boolean = other match { - case that: DumpEn => - Arrays.equals(k, that.k) && - Arrays.equals(v, that.v) && - Arrays.equals(nextKey, that.nextKey) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(k, v, nextKey) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"DumpEn(k=$k, v=$v, nextKey=$nextKey)" -} - -object DumpEn { - def apply(k: Array[Byte], v: Array[Byte], nextKey: Array[Byte]): DumpEn = { - new DumpEn(k=k, v=v, nextKey=nextKey) - } -} - -@N(4) final class DumpGet - ( @N(1) val k: Array[Byte] - ) extends Msg { - override def equals(other: Any): Boolean = other match { - case that: DumpGet => - Arrays.equals(k, that.k) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(k) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"DumpGet(k=$k)" -} - -object DumpGet { - def apply(k: Array[Byte]): DumpGet = { - new DumpGet(k=k) - } -} - -@N(5) final case class DumpGetBucketData - ( @N(1) b: Int - ) extends Msg - -@N(7) final case class ReplBucketPut - ( @N(1) b: Int - , @N(2) bucketVc: VectorClock - , @N(3) items: Vector[Data] - ) extends Msg - -@N(8) case object ReplBucketUpToDate extends Msg - -@N(9) final case class ReplGetBucketIfNew - ( @N(1) b: Int - , @N(2) vc: VectorClock - ) extends Msg - -@N(10) final case class ReplNewerBucketData - ( @N(1) vc: VectorClock - , @N(2) items: Vector[Data] - ) extends Msg - -final case class ReplBucketsVc - ( @N(1) bvcs: Map[Int, VectorClock] - ) - -@N(11) final class StoreDelete - ( @N(1) val key: Array[Byte] - ) extends Msg { - override def equals(other: Any): Boolean = other match { - case that: StoreDelete => - Arrays.equals(key, that.key) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(key) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"StoreDelete(key=$key)" -} - -object StoreDelete { - def apply(key: Array[Byte]): StoreDelete = { - new StoreDelete(key=key) - } -} - -@N(12) final class StoreGet - ( @N(1) val key: Array[Byte] - ) extends Msg { - override def equals(other: Any): Boolean = other match { - case that: StoreGet => - Arrays.equals(key, that.key) - case _ => false - } - override def hashCode(): Int = { - val state = Seq(key) - state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b) - } - override def toString = s"StoreGet(key=$key)" -} - -object StoreGet { - def apply(key: Array[Byte]): StoreGet = { - new StoreGet(key=key) - } -} - -@N(13) final case class StoreGetAck - ( @N(1) data: Option[Data] - ) extends Msg - -@N(14) final case class StorePut - ( @N(1) data: Data - ) extends Msg - -final case class ReplGetBucketsVc - ( @N(1) bs: Vector[Int] - ) - -sealed trait QuorumState -object QuorumState { - @N(1) case object QuorumStateUnsatisfied extends QuorumState - @N(2) case object QuorumStateReadonly extends QuorumState - @N(3) case object QuorumStateEffective extends QuorumState -} diff --git a/kvs/src/main/scala/rng/StoreReadonly.scala b/kvs/src/main/scala/rng/StoreReadonly.scala deleted file mode 100644 index 2541b6e6..00000000 --- a/kvs/src/main/scala/rng/StoreReadonly.scala +++ /dev/null @@ -1,97 +0,0 @@ -package zd.rng -package store - -import akka.actor.{Actor, ActorLogging, Props} -import akka.cluster.{VectorClock} -import leveldbjnr.* -import zd.rng.data.{Data, BucketInfo} -import zd.rng.data.codec.* -import zd.rng.model.{StoreGet, StoreGetAck} -import zd.rng.model.{DumpGet, DumpEn} -import zd.rng.model.{DumpGetBucketData, DumpBucketData} -import zd.rng.model.{ReplGetBucketsVc, ReplBucketsVc, ReplGetBucketIfNew, ReplBucketUpToDate, ReplNewerBucketData} -import proto.* - -object ReadonlyStore { - def props(leveldb: LevelDb): Props = Props(new ReadonlyStore(leveldb)) -} - -class ReadonlyStore(leveldb: LevelDb) extends Actor with ActorLogging { - val hashing = HashingExtension(context.system) - val ro = ReadOpts() - - def get(k: Key): Option[Array[Byte]] = leveldb.get(k, ro).fold(l => throw l, r => r) - - val `:key:` = stob(":key:") - val `:keys` = stob(":keys") - val `readonly_dummy` = stob("readonly_dummy") - - override def receive: Receive = { - case x: StoreGet => - val k = itob(hashing.findBucket(x.key)) ++ `:key:` ++ x.key - val result: Option[Data] = get(k).map(decode[Data](_)) - sender ! StoreGetAck(result) - - case DumpGetBucketData(b) => - val k = itob(b) ++ `:keys` - val b_info = get(k).map(decode[BucketInfo](_)) - val keys: Vector[Key] = b_info.map(_.keys.toVector).getOrElse(Vector.empty) - val items: Vector[Data] = keys.flatMap(key => - get(itob(b)++`:key:`++key).map(decode[Data](_)) - ) - sender ! DumpBucketData(b, items) - case ReplGetBucketIfNew(b, vc) => - val vc_other: VectorClock = vc - val k = itob(b) ++ `:keys` - val b_info = get(k).map(decode[BucketInfo](_)) - b_info match { - case Some(b_info) => - val vc_local: VectorClock = b_info.vc - vc_other == vc_local || vc_other > vc_local match { - case true => sender ! ReplBucketUpToDate - case false => - val keys = b_info.keys - val items: Vector[Data] = keys.view.flatMap(key => - get(itob(b)++`:key:`++key).map(decode[Data](_)) - ).to(Vector) - sender ! ReplNewerBucketData(b_info.vc, items) - } - case None => - sender ! ReplBucketUpToDate - } - case ReplGetBucketsVc(bs) => - val bvcs: Bucket Map VectorClock = bs.view.flatMap{ b => - val k = itob(b) ++ `:keys` - get(k).map(x => b -> decode[BucketInfo](x).vc) - }.to(Map) - sender ! ReplBucketsVc(bvcs) - - case x: DumpGet => - import java.io.{ByteArrayOutputStream, ObjectOutputStream, ObjectInputStream, ByteArrayInputStream} - val key: Array[Byte] = { - val bos = new ByteArrayOutputStream - val out = new ObjectOutputStream(bos) - out.writeObject(new String(x.k, "UTF-8")) - out.close() - bos.toByteArray.nn - } - val data: Option[Array[Byte]] = leveldb.get(key, ro).fold(l => throw l, r => r) - val res: DumpEn = data match { - case None => - DumpEn(x.k, `readonly_dummy`, Array.empty[Byte]) - case Some(data) => - val in = new ObjectInputStream(new ByteArrayInputStream(data)) - val obj = in.readObject - in.close() - val decoded = obj.asInstanceOf[(akka.util.ByteString, Option[String])] - DumpEn(x.k, decoded._1.toArray, decoded._2.fold(Array.empty[Byte])(stob)) - } - sender ! res - case _ => - } - - override def postStop(): Unit = { - ro.close() - super.postStop() - } -} diff --git a/kvs/src/main/scala/rng/ack.scala b/kvs/src/main/scala/rng/ack.scala deleted file mode 100644 index 64d9dcaf..00000000 --- a/kvs/src/main/scala/rng/ack.scala +++ /dev/null @@ -1,6 +0,0 @@ -package zd.rng - -sealed trait Ack -final case class AckSuccess(v: Option[Value]) extends Ack -final case class AckQuorumFailed(why: String) extends Ack -final case class AckTimeoutFailed(op: String, k: String) extends Ack diff --git a/kvs/src/main/scala/rng/akkacluster.scala b/kvs/src/main/scala/rng/akkacluster.scala deleted file mode 100644 index 66110a8b..00000000 --- a/kvs/src/main/scala/rng/akkacluster.scala +++ /dev/null @@ -1,9 +0,0 @@ -package akka - -import proto.* - -package object cluster { - implicit val vcodec: MessageCodec[(String, Long)] = caseCodecNums[Tuple2[String,Long]]("_1"->1,"_2"->2) - implicit val vccodec: MessageCodec[akka.cluster.VectorClock] = caseCodecNums[akka.cluster.VectorClock]("versions"->1) - val emptyVC = VectorClock() -} diff --git a/kvs/src/main/scala/rng/ext.scala b/kvs/src/main/scala/rng/ext.scala deleted file mode 100644 index 8e538854..00000000 --- a/kvs/src/main/scala/rng/ext.scala +++ /dev/null @@ -1,9 +0,0 @@ -package zd.rng - -extension [A,B](xs: Seq[Either[A, B]]) - @annotation.tailrec private def _sequence_(ys: Seq[Either[A, B]]): Either[A, Unit] = - ys.headOption match - case None => Right(()) - case Some(Left(e)) => Left(e) - case Some(Right(z)) => _sequence_(ys.tail) - inline def sequence_ : Either[A, Unit] = _sequence_(xs) \ No newline at end of file diff --git a/kvs/src/main/scala/rng/merge.scala b/kvs/src/main/scala/rng/merge.scala deleted file mode 100644 index 3c6978d9..00000000 --- a/kvs/src/main/scala/rng/merge.scala +++ /dev/null @@ -1,125 +0,0 @@ -package zd.rng - -import zd.rng.data.{Data} -import zd.rng.GatherGet.AddrOfData -import scala.annotation.tailrec -import scala.collection.immutable.{HashMap, HashSet} - -object MergeOps { - case class ArrayWrap(a: Array[Byte]) { - override def equals(other: Any): Boolean = { - if (!other.isInstanceOf[ArrayWrap]) false - else java.util.Arrays.equals(a, other.asInstanceOf[ArrayWrap].a) - } - override def hashCode(): Int = java.util.Arrays.hashCode(a) - } - - def forDump(xs: Vector[Data]): Vector[Data] = { - @tailrec - def loop(xs: Vector[Data], acc: HashMap[ArrayWrap, Data]): Vector[Data] = { - xs match { - case xs if xs.isEmpty => acc.values.toVector - case received +: t => - val k = ArrayWrap(received.key) - acc.get(k) match { - case None => - loop(t, acc + (k -> received)) - case Some(stored) => - (stored < received) match { - case OkLess(true) => loop(t, acc + (k -> received)) - case OkLess(false) => loop(t, acc) - case ConflictLess(true, vc) => loop(t, acc + (k -> received.copy(vc=vc))) - case ConflictLess(false, vc) => loop(t, acc + (k -> stored.copy(vc=vc))) - } - } - } - } - loop(xs, acc=HashMap.empty) - } - - def forRepl(xs: Vector[Data]): Vector[Data] = { - @tailrec - def loop(xs: Vector[Data], acc: HashMap[ArrayWrap, Data]): Vector[Data] = { - xs match { - case xs if xs.isEmpty => acc.values.toVector - case received +: t => - val k = ArrayWrap(received.key) - acc.get(k) match { - case None => - loop(t, acc + (k -> received)) - case Some(stored) => - (stored < received) match { - case OkLess(true) => loop(t, acc + (k -> received)) - case OkLess(false) => loop(t, acc) - case ConflictLess(true, vc) => loop(t, acc + (k -> received.copy(vc=vc))) - case ConflictLess(false, vc) => loop(t, acc + (k -> stored.copy(vc=vc))) - } - } - } - } - loop(xs, acc=HashMap.empty) - } - - /* returns (actual data, list of outdated nodes) */ - def forGatherGet(xs: Vector[AddrOfData]): (Option[Data], HashSet[Node]) = { - @tailrec - def loop(xs: Vector[Option[Data]], newest: Option[Data]): Option[Data] = { - xs match { - case xs if xs.isEmpty => newest - case None +: t => loop(t, newest) - case (r@Some(received)) +: t => - newest match { - case None => loop(t, r) - case s@Some(saved) => - (saved < received) match { - case OkLess(true) => loop(t, r) - case OkLess(false) => loop(t, s) - case ConflictLess(true, _) => loop(t, r) - case ConflictLess(false, _) => loop(t, s) - } - } - } - } - xs match { - case Seq() => None -> HashSet.empty - case h +: t => - val correct = loop(t.map(_._1), h._1) - def makevc1(x: Option[Data]): VectorClock = x.map(_.vc).getOrElse(emptyVC) - val correct_vc = makevc1(correct) - correct -> xs.view.filterNot(x => makevc1(x._1) == correct_vc).map(_._2).to(HashSet) - } - } - - def forPut(stored: Option[Data], received: Data): Option[Data] = { - stored match { - case None => - Some(received) - case Some(stored) => - (stored < received) match { - case OkLess(true) => Some(received) - case OkLess(false) => None - case ConflictLess(true, vc) => Some(received.copy(vc=vc)) - case ConflictLess(false, vc) => Some(stored.copy(vc=vc)) - } - } - } - - sealed trait LessComp - final case class OkLess(res: Boolean) extends LessComp - final case class ConflictLess(res: Boolean, vc: VectorClock) extends LessComp - - implicit class DataExt(x: Data) { - def <(o: Data): LessComp = { - val xvc = x.vc - val ovc = o.vc - if (xvc < ovc) OkLess(true) - else if (xvc == ovc) OkLess(x.lastModified < o.lastModified) - else if (xvc > ovc) OkLess(false) - else { // xvc <> ovc - val mergedvc = xvc merge ovc - if (x.lastModified < o.lastModified) ConflictLess(true, mergedvc) - else ConflictLess(false, mergedvc) - } - } - } -} diff --git a/kvs/src/main/scala/rng/package.scala b/kvs/src/main/scala/rng/package.scala deleted file mode 100644 index 77dac79c..00000000 --- a/kvs/src/main/scala/rng/package.scala +++ /dev/null @@ -1,35 +0,0 @@ -package zd - -import akka.actor.{Address, ActorRef} - -package object rng { - type Bucket = Int - type VNode = Int - type Node = Address - type Key = Array[Byte] - type Value = Array[Byte] - type VectorClock = akka.cluster.VectorClock - type Age = (VectorClock, Long) - type PreferenceList = Set[Node] - - val emptyVC = akka.cluster.emptyVC - - def stob(s: String): Array[Byte] = s.getBytes("UTF-8").nn - def itob(v: Int): Array[Byte] = Array[Byte]((v >> 24).toByte, (v >> 16).toByte, (v >> 8).toByte, v.toByte) - - extension (value: String) - def blue: String = s"\u001B[34m${value}\u001B[0m" - def green: String = s"\u001B[32m${value}\u001B[0m" - - def now_ms(): Long = System.currentTimeMillis - - def addr(s: ActorRef): Node = s.path.address - - given [A]: CanEqual[None.type, Option[A]] = CanEqual.derived - given CanEqual[String, Any] = CanEqual.derived - given CanEqual[Node, Node] = CanEqual.derived - given CanEqual[RestoreState.type, Any] = CanEqual.derived - given CanEqual[Ready.type, Any] = CanEqual.derived - given CanEqual[model.QuorumState, model.QuorumState] = CanEqual.derived - given CanEqual[model.ReplBucketUpToDate.type, Any] = CanEqual.derived -} diff --git a/kvs/src/main/scala/search/dir.scala b/kvs/src/main/scala/search/dir.scala deleted file mode 100644 index ceb2e812..00000000 --- a/kvs/src/main/scala/search/dir.scala +++ /dev/null @@ -1,283 +0,0 @@ -package zd.kvs -package search - -import java.io.{IOException, ByteArrayOutputStream} -import java.nio.file.{NoSuchFileException, FileAlreadyExistsException} -import java.util.{Collection, Collections, Arrays} -import java.util.concurrent.atomic.AtomicLong -import org.apache.lucene.store.* -import scala.annotation.tailrec -import scala.collection.concurrent.TrieMap -import scala.collection.JavaConverters.* - -class KvsDirectory(dir: String)(kvs: WritableFile)(using Dba) extends BaseDirectory(KvsLockFactory(dir)) { - given FileHandler = new FileHandler { - override val chunkLength = 10_000_000 // 10 MB - } - val h = EnHandler - - private val outs = TrieMap.empty[String, ByteArrayOutputStream] - private val nextTempFileCounter = AtomicLong() - - def exists: Either[Err, Boolean] = - h.get(Fd(dir)).map(_.isDefined) - - def deleteAll(): Either[Err, Unit] = - for { - xs <- h.all(dir) - ys <- xs.sequence - _ <- ys.map{ x => - val name = x.id - for { - _ <- kvs.delete(dir, name).map(_ => ()).recover{ case _: FileNotExists => () } - _ <- h.remove(dir, name) - } yield unit - }.sequence_ - _ <- h.delete(Fd(dir)).map(_ => ()).recover{ case KeyNotFound => () } - } yield unit - - /** - * Returns names of all files stored in this directory. - * The output must be in sorted (UTF-16, java's {//link String#compareTo}) order. - * - * //throws IOException in case of I/O error - */ - override - def listAll(): Array[String | Null] | Null = { - ensureOpen() - h.all(dir).flatMap(_.sequence).fold( - l => throw new IOException(l.toString) - , r => r.map(_.id).sorted.toArray - ) - } - - /** - * Removes an existing file in the directory. - * - * This method must throw {//link NoSuchFileException} - * if {@code name} points to a non-existing file. - * - * @param name the name of an existing file. - * //throws IOException in case of I/O error - */ - override - def deleteFile(name: String | Null): Unit = { - sync(Collections.singletonList(name.nn).nn) - val r = for { - _ <- kvs.delete(dir, name.nn) - _ <- h.remove(dir, name.nn) - } yield () - r.fold( - l => l match { - case _: FileNotExists => throw new NoSuchFileException(s"${dir}/${name.nn}") - case KeyNotFound => throw new NoSuchFileException(s"${dir}/${name.nn}") - case x => throw new IOException(x.toString) - }, - _ => () - ) - } - - /** - * Returns the byte length of a file in the directory. - * - * This method must throw {//link NoSuchFileException} - * if {@code name} points to a non-existing file. - * - * @param name the name of an existing file. - * //throws IOException in case of I/O error - */ - override - def fileLength(name: String | Null): Long = { - ensureOpen() - sync(Collections.singletonList(name.nn).nn) - kvs.size(dir, name.nn).fold( - l => l match { - case _: FileNotExists => throw new NoSuchFileException(s"${dir}/${name.nn}") - case _ => throw new IOException(l.toString) - }, - r => r - ) - } - - /** - * Creates a new, empty file in the directory and returns an {//link IndexOutput} - * instance for appending data to this file. - * - * This method must throw {//link java.nio.file.FileAlreadyExistsException} if the file - * already exists. - * - * @param name the name of the file to create. - * //throws IOException in case of I/O error - */ - override - def createOutput(name: String | Null, context: IOContext | Null): IndexOutput | Null = { - ensureOpen() - val r = for { - _ <- h.add(En(dir, name.nn)) - _ <- kvs.create(dir, name.nn) - } yield () - r.fold( - l => l match { - case _: EntryExists => throw new FileAlreadyExistsException(s"${dir}/${name.nn}") - case _: FileAlreadyExists => throw new FileAlreadyExistsException(s"${dir}/${name.nn}") - case _ => throw new IOException(l.toString) - }, - _ => { - val out = new ByteArrayOutputStream; - outs += ((name.nn, out)) - new OutputStreamIndexOutput(s"${dir}/${name.nn}", name.nn, out, 8192) - } - ) - } - - /** - * Creates a new, empty, temporary file in the directory and returns an {//link IndexOutput} - * instance for appending data to this file. - * - * The temporary file name (accessible via {//link IndexOutput#getName()}) will start with - * {@code prefix}, end with {@code suffix} and have a reserved file extension {@code .tmp}. - */ - override - def createTempOutput(prefix: String | Null, suffix: String | Null, context: IOContext | Null): IndexOutput | Null = { - ensureOpen() - @tailrec def loop(): Either[Err, File] = { - val name = Directory.getTempFileName(prefix, suffix, nextTempFileCounter.getAndIncrement).nn - val res = for { - _ <- h.add(En(dir, name)) - r <- kvs.create(dir, name) - } yield r - res match { - case Left(_: EntryExists) => loop() - case Left(_: FileAlreadyExists) => loop() - case x => x - } - } - val res = loop() - res.fold( - l => throw new IOException(l.toString), - r => { - val out = new ByteArrayOutputStream; - outs += ((r.name, out)) - new OutputStreamIndexOutput(s"${dir}/${r.name}", r.name, out, 8192) - } - ) - } - - /** - * Ensures that any writes to these files are moved to - * stable storage (made durable). - * - * Lucene uses this to properly commit changes to the index, to prevent a machine/OS crash - * from corrupting the index. - */ - override - def sync(names: Collection[String] | Null): Unit = { - ensureOpen() - names.nn.asScala.foreach{ (name: String) => - outs.get(name).map(_.toByteArray.nn).foreach{ xs => - kvs.append(dir, name, xs).fold( - l => throw new IOException(l.toString), - _ => () - ) - outs -= name - } - } - } - - override - def syncMetaData(): Unit = - ensureOpen() - - /** - * Renames {@code source} file to {@code dest} file where - * {@code dest} must not already exist in the directory. - * - * It is permitted for this operation to not be truly atomic, for example - * both {@code source} and {@code dest} can be visible temporarily in {//link #listAll()}. - * However, the implementation of this method must ensure the content of - * {@code dest} appears as the entire {@code source} atomically. So once - * {@code dest} is visible for readers, the entire content of previous {@code source} - * is visible. - * - * This method is used by IndexWriter to publish commits. - */ - override - def rename(source: String | Null, dest: String | Null): Unit = { - ensureOpen() - sync(Arrays.asList(source, dest).nn) - val res = for { - _ <- kvs.copy(dir, source.nn -> dest.nn) - _ <- h.add(En(dir, dest.nn)) - _ <- kvs.delete(dir, source.nn) - _ <- h.remove(dir, source.nn) - } yield () - res.fold( - l => throw new IOException(l.toString), - _ => () - ) - } - - /** - * Opens a stream for reading an existing file. - * - * This method must throw {//link NoSuchFileException} - * if {@code name} points to a non-existing file. - * - * @param name the name of an existing file. - * //throws IOException in case of I/O error - */ - override - def openInput(name: String | Null, context: IOContext | Null): IndexInput | Null = { - sync(Collections.singletonList(name.nn).nn) - val res = for { - bs <- kvs.stream(dir, name.nn) - bs1 <- bs.sequence - } yield new BytesIndexInput(s"${dir}/${name.nn}", bs1) - res.fold( - l => l match { - case FileNotExists(dir, name) => throw new NoSuchFileException(s"${dir}/${name}") - case _ => throw new IOException(l.toString) - }, - r => r - ) - } - - override def close(): Unit = synchronized { - isOpen = false - } - - override - def getPendingDeletions(): java.util.Set[String] = { - Collections.emptySet[String].nn - } -} - -class KvsLockFactory(dir: String) extends LockFactory { - private val locks = TrieMap.empty[String, Unit] - - override def obtainLock(d: Directory | Null, lockName: String | Null): Lock | Null = { - val key = dir + lockName - locks.putIfAbsent(key, ()) match { - case None => return new KvsLock(key) - case Some(_) => throw new LockObtainFailedException(key) - } - } - - private class KvsLock(key: String) extends Lock { - @volatile private var closed = false - - override def ensureValid(): Unit = { - if (closed) { - throw new AlreadyClosedException(key) - } - if (!locks.contains(key)) { - throw new AlreadyClosedException(key) - } - } - - override def close(): Unit = { - locks -= key - closed = true - } - } -} diff --git a/kvs/src/main/scala/search/en.scala b/kvs/src/main/scala/search/en.scala deleted file mode 100644 index 4bb75282..00000000 --- a/kvs/src/main/scala/search/en.scala +++ /dev/null @@ -1,130 +0,0 @@ -package zd.kvs -package search - -import scala.annotation.tailrec -import proto.* - -final case class Fd - ( @N(1) id: String - , @N(2) top: String = empty - ) - -case class En - ( @N(1) fid: String - , @N(2) id: String - , @N(3) prev: String = empty - ) - -given MessageCodec[En] = caseCodecAuto - -/** - * Linked list of entries - * - * [top] -->prev--> [entry] -->prev--> [empty] - */ -object EnHandler: - given MessageCodec[Fd] = caseCodecAuto[Fd] - - def put(fd: Fd)(using dba: Dba): Either[Err, Fd] = - dba.put(fd.id, encode(fd)).map(_ => fd) - - def get(fd: Fd)(using dba: Dba): Either[Err, Option[Fd]] = - dba.get(fd.id).map(_.map(decode)) - - def delete(fd: Fd)(using dba: Dba): Either[Err, Unit] = - dba.delete(fd.id).void - - private inline def key(fid: String, id: String): String = s"${fid}.${id}" - private inline def key(en: En): String = key(fid=en.fid, id=en.id) - - private def _put(en: En)(using dba: Dba): Either[Err, En] = - dba.put(key(en), encode(en)).map(_ => en) - - def get(fid: String, id: String)(using dba: Dba): Either[Err, Option[En]] = - dba.get(key(fid, id)).map(_.map(decode)) - - private def getOrFail(fid: String, id: String)(using dba: Dba): Either[Err, En] = - val k = key(fid, id) - dba.get(k).flatMap{ - case Some(x) => Right(decode(x)) - case None => Left(KeyNotFound) - } - - private def delete(fid: String, id: String)(using dba: Dba): Either[Err, Unit] = - dba.delete(key(fid, id)).void - - private def nextid(fid: String)(using dba: Dba): Either[Err, String] = - def key(fid: String): String = s"IdCounter.${fid}" - dba.get(key(fid)).flatMap{ v => - val prev = v.map(String(_, "utf8").nn).getOrElse("0") - val next = (prev.toLong+1).toString - dba.put(key(fid), next.getBytes("utf8").nn).map(_ => next) - } - - /** - * Adds the entry to the container - * Creates the container if it's absent - * @param en entry to add (prev is ignored). If id is empty it will be generated - */ - def add(en: En)(using dba: Dba): Either[Err, En] = - get(Fd(en.fid)).flatMap(_.cata(Right(_), put(Fd(en.fid)))).flatMap{ (fd: Fd) => - ( if (en.id == empty) - nextid(en.fid) // generate ID if it is empty - else - get(en.fid, en.id).flatMap( // id of entry must be unique - _.cata(_ => Left(EntryExists(key(en))), Right(en.id)) - ) - ).map(id => en.copy(id=id, prev=fd.top)).flatMap{ en => - // add new entry with prev pointer - _put(en).flatMap{ en => - // update feed's top - put(fd.copy(top=en.id)).map(_ => en) - } - } - } - - /** - * Iterates through container and return the stream of entries. - * - * Stream is FILO ordered (most recent is first). - * @param from if specified then return entries after this entry - */ - def all(fid: String)(using Dba): Either[Err, LazyList[Either[Err, En]]] = - all(fid, from=None) - - def all(fid: String, from: Option[En])(using Dba): Either[Err, LazyList[Either[Err, En]]] = - def _stream(id: String): LazyList[Either[Err, En]] = - id match - case `empty` => LazyList.empty - case _ => - val en = getOrFail(fid, id) - en match - case Right(e) => LazyList.cons(en, _stream(e.prev)) - case _ => LazyList(en) - from match - case None => get(Fd(fid)).map(_.cata(x => _stream(x.top), LazyList.empty)) - case Some(en) => Right(_stream(en.prev)) - - def remove(_fid: String, _id: String)(using Dba): Either[Err, Unit] = - // get entry to delete - getOrFail(_fid, _id).flatMap{ en => - val id = en.id - val fid = en.fid - val prev = en.prev - get(Fd(fid)).flatMap(_.cata(Right(_), Left(KeyNotFound))).flatMap{ fd => - val top = fd.top - ( if (id == top) - // change top and decrement - put(fd.copy(top=prev)) - else - // find entry which points to this one (next) - LazyList.iterate(start=getOrFail(fid,top))(_.flatMap(x=>getOrFail(fid,x.prev))) - .takeWhile(_.isRight) - .flatMap(_.toOption) - .find(_.prev==id) - .toRight(KeyNotFound) - // change link - .flatMap(next => _put(next.copy(prev=prev))) - ).flatMap(_ => delete(fid, id)) // delete entry - } - } diff --git a/kvs/src/main/scala/search/in.scala b/kvs/src/main/scala/search/in.scala deleted file mode 100644 index abb5b1f0..00000000 --- a/kvs/src/main/scala/search/in.scala +++ /dev/null @@ -1,68 +0,0 @@ -package zd.kvs -package search - -import org.apache.lucene.store.IndexInput -import scala.annotation.tailrec -import java.io.* - -class BytesIndexInput(resourceDescription: String, xs: Vector[Array[Byte]], offset: Long, len: Long) - extends IndexInput(resourceDescription) { - - def this(d: String, xs: Vector[Array[Byte]]) = this(d, xs, 0, xs.foldLeft(0L)((acc, x) => acc + x.length)) - - private var open = true - private var pos = offset - - override def close(): Unit = open = false - override def getFilePointer(): Long = { ensureOpen(); pos - offset } - override def length(): Long = { ensureOpen(); len } - override def readByte(): Byte = { - ensureOpen() - @tailrec def loop(remaining: Vector[Array[Byte]], p: Long): Byte = { - remaining.headOption match { - case Some(head) if p < head.length => head(Math.toIntExact(p)) - case Some(head) => loop(remaining.tail, p-head.length) - case None => throw new EOFException - } - } - val b = loop(xs, pos) - pos += 1 - b - } - override def readBytes(ys: Array[Byte] | Null, ys_offset: Int, ys_len: Int): Unit = { - ensureOpen() - @tailrec def loop_copy(src: Array[Byte], remaining: Vector[Array[Byte]], src_offset: Int, dst_offset: Int, len_remaining: Int): Unit = { - val len1 = Math.min(len_remaining, src.length-src_offset) - System.arraycopy(src, src_offset, ys, dst_offset, len1) - if (len1 < len_remaining) { - remaining.headOption match { - case Some(head2) => loop_copy(head2, remaining.tail, 0, dst_offset+len1, len_remaining-len1) - case None => throw new EOFException - } - } - } - @tailrec def loop_find(remaining: Vector[Array[Byte]], p: Long): Unit = { - remaining.headOption match { - case Some(head) if p < head.length => - loop_copy(src=head, remaining=remaining.tail, src_offset=Math.toIntExact(p), dst_offset=ys_offset, len_remaining=ys_len) - case Some(head) => loop_find(remaining.tail, p-head.length) - case None => throw new EOFException - } - } - loop_find(xs, pos) - pos += ys_len - } - override def seek(p: Long): Unit = { - ensureOpen() - pos = p + offset - if (p < 0 || p > len) throw new EOFException - } - override def slice(sliceDescription: String | Null, o: Long, l: Long): IndexInput | Null = { - ensureOpen() - new BytesIndexInput(sliceDescription.nn, xs, offset+o, l) - } - - private def ensureOpen(): Unit = { - if (!open) throw new IOException("closed") - } -} \ No newline at end of file diff --git a/kvs/src/main/scala/store/dba.scala b/kvs/src/main/scala/store/dba.scala deleted file mode 100644 index 8f1cd46a..00000000 --- a/kvs/src/main/scala/store/dba.scala +++ /dev/null @@ -1,30 +0,0 @@ -package zd.kvs - -import scala.concurrent.Future - -/** Database Application Interface */ -trait Dba { self: AutoCloseable => - type K = String - type V = Array[Byte] - type R[A] = Either[Err, A] - - def put(key: K, value: V): R[Unit] - def get(key: K): R[Option[V]] - def delete(key: K): R[Unit] - - def save(path: String): R[String] - def load(path: String): R[String] - - def onReady(): Future[Unit] - def compact(): Unit - def deleteByKeyPrefix(keyPrefix: K): R[Unit] - - def close(): Unit -} - -extension (x: Dba) - def getOrFail(key: x.K): x.R[x.V] = - x.get(key).flatMap{ - case None => Left(KeyNotFound) - case Some(x) => Right(x) - } diff --git a/kvs/src/main/scala/store/mem.scala b/kvs/src/main/scala/store/mem.scala deleted file mode 100644 index 42646626..00000000 --- a/kvs/src/main/scala/store/mem.scala +++ /dev/null @@ -1,26 +0,0 @@ -package zd.kvs - -import scala.collection.concurrent.TrieMap -import scala.concurrent.Future -import util.chaining.* - -class Mem extends Dba, AutoCloseable: - private val db = TrieMap[String, V]() - - override def get(key: String): R[Option[V]] = - db.get(key).pipe(Right.apply) - - override def put(key: String, value: V): R[Unit] = - db.put(key, value).pipe(Right.apply).map(_ => unit) - - override def delete(key: String): R[Unit] = - db.remove(key).pipe(Right.apply).map(_ => unit) - - override def onReady(): Future[Unit] = Future.successful(()) - override def compact(): Unit = () - - override def load(path: String): R[String] = ??? - override def save(path: String): R[String] = ??? - override def deleteByKeyPrefix(k: K): R[Unit] = ??? - - override def close(): Unit = unit diff --git a/kvs/src/main/scala/store/rks.scala b/kvs/src/main/scala/store/rks.scala deleted file mode 100644 index 0ba3ca37..00000000 --- a/kvs/src/main/scala/store/rks.scala +++ /dev/null @@ -1,84 +0,0 @@ -package zd.kvs - -import akka.actor.* -import akka.event.Logging -import akka.pattern.ask -import akka.util.{Timeout} -import akka.event.LoggingAdapter -import proto.* -import org.rocksdb.* -import zio.* -import scala.concurrent.* -import scala.concurrent.duration.* -import scala.util.{Try, Success, Failure} -import scala.util.chaining.* -import scala.concurrent.Future -import zd.rks.DumpProcessor - -class Rks(system: ActorSystem, dir: String) extends Dba, AutoCloseable: - RocksDB.loadLibrary() - private val logging: LoggingAdapter = Logging(system, "rks") - private val cfg = system.settings.config.getConfig("rks").nn - private val opts = Options().nn - .setCreateIfMissing(true).nn - .setCompressionType(CompressionType.LZ4_COMPRESSION).nn - .setLogger(new Logger(Options()) { - def log(infoLogLevel: InfoLogLevel, logMsg: String): Unit = - infoLogLevel match - case InfoLogLevel.DEBUG_LEVEL => logging.debug(logMsg) - case InfoLogLevel.INFO_LEVEL => logging.debug(logMsg) - case InfoLogLevel.WARN_LEVEL => logging.warning(logMsg) - case _ => logging.error(logMsg) - }).nn - private val db = RocksDB.open(opts, dir).nn - - private def withRetryOnce[A](op: Array[Byte] => A, key: K): R[A] = - val eff = for { - k <- ZIO.succeed(zd.rng.stob(key)) - x <- ZIO.attempt(op(k)).retry(Schedule.fromDuration(100 milliseconds)) - } yield x - Try( - Unsafe.unsafe { implicit unsafe => - Runtime.default.unsafe.run(eff.either).getOrThrowFiberFailure() - } - ).toEither.flatten.leftMap(Failed) - - override def get(key: K): R[Option[V]] = - for { - x <- withRetryOnce(db.get, key) - } yield if x != null then Some(x) else None - - override def put(key: K, value: V): R[Unit] = - withRetryOnce(db.put(_, value), key) - - override def delete(key: K): R[Unit] = - withRetryOnce(db.delete, key) - - def compact(): Unit = db.compactRange() - - def save(path: String): R[String] = - val d = FiniteDuration(1, HOURS) - val dump = system.actorOf(DumpProcessor.props(db), s"dump_wrkr-${java.lang.System.currentTimeMillis}") - val x = dump.ask(DumpProcessor.Save(path))(Timeout(d)) - Try(Await.result(x, d)) match - case Success(v: String) => Right(v) - case Success(v) => Left(RngFail(s"Unexpected response: ${v}")) - case Failure(t) => Left(Failed(t)) - - def load(path: String): R[String] = - val d = concurrent.duration.Duration.fromNanos(cfg.getDuration("dump-timeout").nn.toNanos) - val dump = system.actorOf(DumpProcessor.props(db), s"dump_wrkr-${java.lang.System.currentTimeMillis}") - val x = dump.ask(DumpProcessor.Load(path))(Timeout(d)) - Try(Await.result(x, d)) match - case Success(v: String) => Right(v) - case Success(v) => Left(RngFail(s"Unexpected response: ${v}")) - case Failure(t) => Left(Failed(t)) - - def onReady(): Future[Unit] = Future.successful(unit) - def deleteByKeyPrefix(keyPrefix: K): R[Unit] = ??? - - override def close(): Unit = - try { db.close() } catch { case _: Throwable => unit } - try { opts.close() } catch { case _: Throwable => unit } - - given [A]: CanEqual[A, A | Null] = CanEqual.derived diff --git a/kvs/src/main/scala/store/rng.scala b/kvs/src/main/scala/store/rng.scala deleted file mode 100644 index c8d324f2..00000000 --- a/kvs/src/main/scala/store/rng.scala +++ /dev/null @@ -1,129 +0,0 @@ -package zd.kvs - -import akka.actor.* -import akka.event.Logging -import akka.pattern.ask -import akka.routing.FromConfig -import akka.util.{Timeout} -import leveldbjnr.LevelDb -import zd.rng -import zd.rng.store.{ReadonlyStore, WriteStore} -import zd.rng.stob -import scala.concurrent.* -import scala.concurrent.duration.* -import scala.util.{Try, Success, Failure} - -class Rng(system: ActorSystem) extends Dba, AutoCloseable: - private val log = Logging(system, "hash-ring") - - private val cfg = system.settings.config.getConfig("ring").nn - - system.eventStream - - private val leveldbPath = cfg.getString("leveldb.dir").nn - private val db: LevelDb = LevelDb.open(leveldbPath).fold(l => throw l, r => r) - - private val writeStore = system.actorOf(WriteStore.props(db).withDeploy(Deploy.local), name="ring_write_store") - system.actorOf(FromConfig.props(ReadonlyStore.props(db)).withDeploy(Deploy.local), name="ring_readonly_store") - - private val hash = system.actorOf(rng.Hash.props(db).withDeploy(Deploy.local), name="ring_hash") - - override def put(key: K, value: V): R[Unit] = - val d = Duration.fromNanos(cfg.getDuration("ring-timeout").nn.toNanos) - val t = Timeout(d) - val putF = hash.ask(rng.Put(stob(key), value))(t).mapTo[rng.Ack] - Try(Await.result(putF, d)) match - case Success(rng.AckSuccess(_)) => Right(()) - case Success(rng.AckQuorumFailed(why)) => Left(RngAskQuorumFailed(why)) - case Success(rng.AckTimeoutFailed(op, k)) => Left(RngAskTimeoutFailed(op, k)) - case Failure(t) => Left(Failed(t)) - - private def isReady(): Future[Boolean] = - val d = Duration.fromNanos(cfg.getDuration("ring-timeout").nn.toNanos) - val t = Timeout(d) - hash.ask(rng.Ready)(t).mapTo[Boolean] - - override def onReady(): Future[Unit] = - val p = Promise[Unit]() - def loop(): Unit = - import system.dispatcher - system.scheduler.scheduleOnce(1 second){ - isReady() onComplete { - case Success(true) => - log.info("KVS is ready") - p.success(()) - case _ => - log.info("KVS isn't ready yet...") - loop() - } - } - loop() - p.future - - override def get(key: K): R[Option[V]] = - val d = Duration.fromNanos(cfg.getDuration("ring-timeout").nn.toNanos) - val t = Timeout(d) - val fut = hash.ask(rng.Get(stob(key)))(t).mapTo[rng.Ack] - Try(Await.result(fut, d)) match - case Success(rng.AckSuccess(v)) => Right(v) - case Success(rng.AckQuorumFailed(why)) => Left(RngAskQuorumFailed(why)) - case Success(rng.AckTimeoutFailed(op, k)) => Left(RngAskTimeoutFailed(op, k)) - case Failure(t) => Left(Failed(t)) - - override def delete(key: K): R[Unit] = - val d = Duration.fromNanos(cfg.getDuration("ring-timeout").nn.toNanos) - val t = Timeout(d) - val fut = hash.ask(rng.Delete(stob(key)))(t).mapTo[rng.Ack] - Try(Await.result(fut, d)) match - case Success(rng.AckSuccess(_)) => Right(()) - case Success(rng.AckQuorumFailed(why)) => Left(RngAskQuorumFailed(why)) - case Success(rng.AckTimeoutFailed(op, k)) => Left(RngAskTimeoutFailed(op, k)) - case Failure(t) => Left(Failed(t)) - - override def save(path: String): R[String] = - val d = 1 hour - val x = hash.ask(rng.Save(path))(Timeout(d)) - Try(Await.result(x, d)) match - case Success(rng.AckQuorumFailed(why)) => Left(RngAskQuorumFailed(why)) - case Success(v: String) => Right(v) - case Success(v) => Left(RngFail(s"Unexpected response: ${v}")) - case Failure(t) => Left(Failed(t)) - - def iterate(f: (K, V) => Unit): R[String] = - val d = 1 hour - val x = hash.ask(rng.Iterate((key, value) => f(new String(key, "UTF-8"), value)))(Timeout(d)) - Try(Await.result(x, d)) match - case Success(rng.AckQuorumFailed(why)) => Left(RngAskQuorumFailed(why)) - case Success(v: String) => Right(v) - case Success(v) => Left(RngFail(s"Unexpected response: ${v}")) - case Failure(t) => Left(Failed(t)) - - override def load(path: String): R[String] = - val d = Duration.fromNanos(cfg.getDuration("dump-timeout").nn.toNanos) - val t = Timeout(d) - val x = hash.ask(rng.Load(path))(t) - Try(Await.result(x, d)) match - case Success(rng.AckQuorumFailed(why)) => Left(RngAskQuorumFailed(why)) - case Success(v: String) => Right(v) - case Success(v) => Left(RngFail(s"Unexpected response: ${v}")) - case Failure(t) => Left(Failed(t)) - - override def compact(): Unit = - db.compact() - - override def deleteByKeyPrefix(k: K): R[Unit] = - val d = Duration.fromNanos(cfg.getDuration("iter-timeout").nn.toNanos) - val t = Timeout(d) - val x = hash.ask(rng.Iter(stob(k)))(t) - Try(Await.result(x, d)) match - case Success(rng.AckQuorumFailed(why)) => Left(RngAskQuorumFailed(why)) - case Success(res: zd.rng.IterRes) => - res.keys.foreach(log.info) - res.keys.map(delete).sequence_ - case Success(v) => Left(RngFail(s"Unexpected response: ${v}")) - case Failure(t) => Left(Failed(t)) - - override def close(): Unit = - hash ! PoisonPill - writeStore ! PoisonPill -end Rng diff --git a/kvs/src/test/resources/application.conf b/kvs/src/test/resources/application.conf deleted file mode 100644 index bf7b5ae2..00000000 --- a/kvs/src/test/resources/application.conf +++ /dev/null @@ -1,16 +0,0 @@ -akka { - loglevel = off - - remote.artery.canonical { - hostname = 127.0.0.1 - port = 4460 - } - - cluster { - seed-nodes = [ - "akka.tcp://Test@127.0.0.1:4460", - ] - } -} - -ring.leveldb.dir = "rng_data_test" diff --git a/kvs/src/test/resources/logback.xml b/kvs/src/test/resources/logback.xml deleted file mode 100644 index 5d16a1e1..00000000 --- a/kvs/src/test/resources/logback.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - %date{HH:mm:ss.SSS} %highlight(%-5level) %cyan(%logger{36}) %X{akkaSource} - %msg%n - - - - - - - diff --git a/kvs/src/test/scala/conf.scala b/kvs/src/test/scala/conf.scala deleted file mode 100644 index 9e650992..00000000 --- a/kvs/src/test/scala/conf.scala +++ /dev/null @@ -1,24 +0,0 @@ -package zd.kvs - -object conf { - def tmpl(port: Int) = s""" - |akka { - | loglevel = off - | - | actor.provider = cluster - | - | remote.artery.canonical { - | hostname = 127.0.0.1 - | port = ${port} - | } - | - | cluster { - | seed-nodes = [ - | "akka.tcp://Test@127.0.0.1:${port}", - | ] - | } - |} - | - |ring.leveldb.dir = "rng_data_test_${port}" - """.stripMargin -} diff --git a/kvs/src/test/scala/el.test.scala b/kvs/src/test/scala/el.test.scala deleted file mode 100644 index c6eae599..00000000 --- a/kvs/src/test/scala/el.test.scala +++ /dev/null @@ -1,35 +0,0 @@ -package zd.kvs - -import akka.actor.* -import com.typesafe.config.{ConfigFactory} -import org.scalatest.freespec.AnyFreeSpecLike -import org.scalatest.matchers.should.Matchers -import org.scalatest.* -import scala.concurrent.Await -import scala.concurrent.duration.* -import scala.util.Try - -class ElHandlerTest extends AnyFreeSpecLike with Matchers with EitherValues with BeforeAndAfterAll { - val kvs = Kvs.mem() - - "el handler should" - { - "return error when element is absent" in { - kvs.el.get[String]("key").getOrElse(???) should be (None) - } - "save value" in { - kvs.el.put("key","value").getOrElse(???) should be ("value") - } - "retrieve value" in { - kvs.el.get[String]("key").getOrElse(???) should be (Some("value")) - } - "override value" in { - kvs.el.put("key","value2").getOrElse(???) should be ("value2") - } - "delete value" in { - kvs.el.delete[String]("key").getOrElse(???) should be (()) - } - "clean up" in { - kvs.el.get[String]("key").getOrElse(???) should be (None) - } - } -} diff --git a/kvs/src/test/scala/file.test.scala b/kvs/src/test/scala/file.test.scala deleted file mode 100644 index 3883d157..00000000 --- a/kvs/src/test/scala/file.test.scala +++ /dev/null @@ -1,60 +0,0 @@ -package zd.kvs - -import akka.actor.* -import com.typesafe.config.{ConfigFactory} -import org.scalatest.freespec.AnyFreeSpecLike -import org.scalatest.matchers.should.Matchers -import org.scalatest.* -import scala.concurrent.Await -import scala.concurrent.duration.* -import scala.util.Try - -class FileHandlerTest extends AnyFreeSpecLike with Matchers with EitherValues with BeforeAndAfterAll { - val kvs = Kvs.mem() - - val dir = "dir" - val name = "name" + java.util.UUID.randomUUID.toString - - implicit val fh: FileHandler = new FileHandler { - override val chunkLength = 5 - } - - "file" - { - "create" in { - kvs.file.create(dir, name).isRight should be (true) - } - "create if exists" in { - kvs.file.create(dir, name).left.value should be (FileAlreadyExists(dir, name)) - } - "append" in { - val r = kvs.file.append(dir, name, Array[Byte](1, 2, 3, 4, 5, 6)) - r.isRight should be (true) - r.getOrElse(???).size should be (6) - r.getOrElse(???).count should be (2) - } - "size" in { - val r = kvs.file.size(dir, name) - r.isRight should be (true) - r.getOrElse(???) should be (6) - } - "size if absent" in { - kvs.file.size(dir, name + "1").left.value should be (FileNotExists(dir, name + "1")) - } - "content" in { - val r = kvs.file.stream(dir, name) - r.isRight should be (true) - val r1 = r.getOrElse(???).sequence - r1.isRight should be (true) - r1.getOrElse(???).toArray.flatten should be (Array[Byte](1, 2, 3, 4, 5, 6)) - } - "content if absent" in { - kvs.file.stream(dir, name + "1").left.value should be (FileNotExists(dir, name + "1")) - } - "delete" in { - kvs.file.delete(dir, name).isRight should be (true) - } - "delete if absent" in { - kvs.file.delete(dir, name).left.value should be (FileNotExists(dir, name)) - } - } -} diff --git a/kvs/src/test/scala/in.test.scala b/kvs/src/test/scala/in.test.scala deleted file mode 100644 index e113de85..00000000 --- a/kvs/src/test/scala/in.test.scala +++ /dev/null @@ -1,54 +0,0 @@ -package zd.kvs -package search - -import org.scalatest.freespec.AnyFreeSpecLike -import org.scalatest.matchers.should.Matchers -import java.util.Arrays - -class InTest extends AnyFreeSpecLike with Matchers { - val in = new BytesIndexInput("test", Vector( - Array[Byte](1,2,3) - , Array[Byte](4,5,6) - , Array[Byte](7,8,9) - , Array[Byte](0,1,2) - )) - "read byte" in { - in.readByte shouldBe 1 - in.readByte shouldBe 2 - in.readByte shouldBe 3 - in.readByte shouldBe 4 - in.seek(0) - in.readByte shouldBe 1 - in.seek(2) - in.readByte shouldBe 3 - in.seek(3) - in.readByte shouldBe 4 - in.seek(7) - in.readByte shouldBe 8 - in.seek(5) - in.readByte shouldBe 6 - in.readByte shouldBe 7 - } - "read bytes across arrays" in { - in.seek(4) - val res_offset = 2 - val res_len = 6 - val res = new Array[Byte](res_offset+res_len) - in.readBytes(res, res_offset, res_len) - assert(Arrays.equals(res, Array.fill[Byte](res_offset)(0)++Array[Byte](5,6,7,8,9,0)), "bad: "+res.mkString("[",",","]")) - } - "slice" in { - val slice = in.slice("slice", 4, 3).nn - slice.length shouldBe 3 - slice.getFilePointer shouldBe 0 - slice.readByte shouldBe 5 - slice.readByte shouldBe 6 - slice.readByte shouldBe 7 - slice.seek(1) - slice.readByte shouldBe 6 - slice.seek(0) - val res = new Array[Byte](3) - slice.readBytes(res, 0, 3) - assert(Arrays.equals(res, Array[Byte](5,6,7)), "bad: "+res.mkString("[",",","]")) - } -} \ No newline at end of file diff --git a/kvs/src/test/scala/index.test.scala b/kvs/src/test/scala/index.test.scala deleted file mode 100644 index 063590bb..00000000 --- a/kvs/src/test/scala/index.test.scala +++ /dev/null @@ -1,114 +0,0 @@ -package zd.kvs -package idx - -import akka.actor.ActorSystem -import com.typesafe.config.ConfigFactory -import org.scalatest.matchers.should.Matchers -import org.scalatest.freespec.AnyFreeSpecLike -import org.scalatest.* -import scala.concurrent.Await -import scala.concurrent.duration.* -import scala.util.Try - -class IdxHandlerTest extends AnyFreeSpecLike with Matchers with BeforeAndAfterAll with EitherValues { - type Fid = IdxHandler.Fid - val Fid = IdxHandler.Fid - type Fd = IdxHandler.Fd - val Fd = IdxHandler.Fd - type En = IdxHandler.Idx - val En = IdxHandler.Idx - val fid = Fid("index") - def entry(n: Int): En = En(fid, id=n.toString, prev=zd.kvs.empty) - - val e1 = entry(1) - val e2 = entry(2) - val e3 = entry(3) - val e5 = entry(5) - - val kvs = Kvs.mem() - - "Feed should" - { - "be empty at creation" in { - kvs.index.all(fid) shouldBe (Right(LazyList.empty)) - } - - "should save e1" in { - val saved = kvs.index.add(e1).getOrElse(???) - (saved.fid, saved.id) shouldBe ((e1.fid, "1")) - } - - "should save e2" in { - val saved = kvs.index.add(e2).getOrElse(???) - (saved.fid, saved.id) shouldBe ((e2.fid, "2")) - } - - "should get e1 and e2 from feed" in { - val stream = kvs.index.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e2.copy(prev="1")), Right(e1))) - } - - "should save entry(3)" in { - val saved = kvs.index.add(e3).getOrElse(???) - (saved.fid, saved.id) shouldBe ((e3.fid, "3")) - } - - "should not save entry(2) again" in { - kvs.index.add(e2).left.getOrElse(???) shouldBe EntryExists(s"${fid}.2") - } - - "should get 3 values from feed" in { - val stream = kvs.index.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e3.copy(prev="2")), Right(e2.copy(prev="1")), Right(e1))) - } - - "should not remove unexisting entry from feed" in { - kvs.index.remove(fid,"5").left.value shouldBe KeyNotFound - } - - "should remove entry(2) from feed without prev/next" in { - kvs.index.remove(e2.fid,"2").getOrElse(???) - } - - "should get 2 values from feed" in { - val stream = kvs.index.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e3.copy(prev="1")), Right(e1))) - } - - "should remove entry(1) from feed" in { - kvs.index.remove(fid,"1").getOrElse(???) - } - - "should get 1 values from feed" in { - val stream = kvs.index.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e3))) - } - - "should remove entry(3) from feed" in { - kvs.index.remove(fid,"3").getOrElse(???) - } - - "should be empty" in { - kvs.index.all(fid).getOrElse(???) shouldBe empty - } - - "should not create stack overflow" in { - val limit = 100 - LazyList.from(1,1).takeWhile( _.<=(limit)).foreach{ n => - val toadd = entry(n) - val added = kvs.index.add(toadd).getOrElse(???) - (added.fid, added.id) shouldBe ((toadd.fid, n.toString)) - } - LazyList.from(1,1).takeWhile( _.<=(limit)).foreach{ n => - val toremove = entry(n) - kvs.index.remove(toremove.fid, toremove.id).getOrElse(???) - } - } - - "feed should be empty at the end test" in { - kvs.el.delete[String](s"IdCounter.$fid") - kvs.index.all(fid).getOrElse(???) shouldBe empty - kvs.index.delete(fid) - kvs.index.all(fid) shouldBe (Right(LazyList.empty)) - } - } -} diff --git a/kvs/src/test/scala/leveldb.test.scala b/kvs/src/test/scala/leveldb.test.scala deleted file mode 100644 index 747eed42..00000000 --- a/kvs/src/test/scala/leveldb.test.scala +++ /dev/null @@ -1,46 +0,0 @@ -package leveldbjnr - -import org.scalatest.freespec.AnyFreeSpec -import org.scalatest.matchers.should.Matchers -import org.scalatest.* - -class LeveldbTest extends AnyFreeSpec with Matchers with EitherValues { - - var leveldb: LevelDb | Null = null - val path = "leveldb_test" - val ro = ReadOpts() - val wo = WriteOpts() - - "leveldb" - { - "version" in { - LevelDb.version should be ((1,20)) - } - "destroy" in { - LevelDb.destroy(path) - } - "create" in { - leveldb = LevelDb.open(path).fold(l => throw l, r => r) - } - "no value" in { - leveldb.nn.get(Array(1,2,3), ro) should be (Right(None)) - } - "put" in { - leveldb.nn.put(Array(1,2,3), Array(11,22,33), wo) should be (Right(())) - } - "read" in { - leveldb.nn.get(Array(1,2,3), ro).map(_.map(_.toList)) should be (Right(Some(List(11,22,33)))) - } - "delete" in { - leveldb.nn.delete(Array(1,2,3), wo) should be (Right(())) - } - "read2" in { - leveldb.nn.get(Array(1,2,3), ro) should be (Right(None)) - } - "close & destroy" in { - leveldb.nn.close() - wo.close() - ro.close() - LevelDb.destroy(path) should be (Right(())) - } - } -} diff --git a/kvs/src/test/scala/merge.test.scala b/kvs/src/test/scala/merge.test.scala deleted file mode 100644 index f5ef6432..00000000 --- a/kvs/src/test/scala/merge.test.scala +++ /dev/null @@ -1,217 +0,0 @@ -package zd.kvs - -import zd.rng.* -import zd.rng.data.* -import org.scalatest.freespec.AnyFreeSpec -import org.scalatest.matchers.should.Matchers -import org.scalatest.* -import scala.collection.immutable.{HashSet, TreeMap} - -class MergeTest extends AnyFreeSpec with Matchers with EitherValues with BeforeAndAfterAll { - def v1(v: Long) = "n1" -> v - def v2(v: Long) = "n2" -> v - def vc(v: Tuple2[String,Long]*) = new VectorClock(TreeMap.empty[String,Long] ++ v) - - "forRepl" - { - import zd.rng.MergeOps.forRepl - "empty" in { - val xs = Vector.empty - forRepl(xs) should be (empty) - } - "single item" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=1, vc=vc(v1(1), v2(1)), stob("v1")), - ) - val ys = Set( - xs(0), - ) - forRepl(xs).toSet should be (ys) - } - "no conflict" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=1, vc=vc(v1(1), v2(1)), stob("v1")), - Data(stob("k2"), bucket=1, lastModified=1, vc=vc(v1(1), v2(1)), stob("v2")), - Data(stob("k3"), bucket=1, lastModified=1, vc=vc(v1(1), v2(1)), stob("v3")), - ) - val ys = Set( - xs(0), - xs(1), - xs(2), - ) - forRepl(xs).toSet should be (ys) - } - "same vc" - { - val vcs = vc(v1(1), v2(1)) - "old then new" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=1, vcs, stob("v11")), - Data(stob("k1"), bucket=1, lastModified=2, vcs, stob("v12")), - Data(stob("k2"), bucket=1, lastModified=1, vcs, stob("v2")), - ) - val ys = Set( - xs(1), - xs(2), - ) - assert(forRepl(xs).toSet.size == ys.size) - forRepl(xs).toSet.zip(ys).foreach{ case (e1, e2) => assert(e1 == e2) } - } - "new then old" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=2, vcs, stob("v11")), - Data(stob("k1"), bucket=1, lastModified=1, vcs, stob("v12")), - Data(stob("k2"), bucket=1, lastModified=1, vcs, stob("v2")), - ) - val ys = Set( - xs(0), - xs(2), - ) - assert(forRepl(xs).toSet.size == ys.size) - forRepl(xs).toSet.zip(ys).foreach{ case (e1, e2) => assert(e1 == e2) } - } - } - "new vc" - { - val vc1s = vc(v1(1), v2(1)) - val vc2s = vc(v1(2), v2(2)) - "old then new" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=2, vc1s, stob("v11")), - Data(stob("k1"), bucket=1, lastModified=1, vc2s, stob("v12")), - Data(stob("k2"), bucket=1, lastModified=1, vc1s, stob("v2")), - ) - val ys = Set( - xs(1), - xs(2), - ) - assert(forRepl(xs).toSet.size == ys.size) - forRepl(xs).toSet.zip(ys).foreach{ case (e1, e2) => assert(e1 == e2) } - } - "new then old" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=1, vc2s, stob("v11")), - Data(stob("k1"), bucket=1, lastModified=2, vc1s, stob("v12")), - Data(stob("k2"), bucket=1, lastModified=1, vc1s, stob("v2")), - ) - val ys = Set( - xs(0), - xs(2), - ) - assert(forRepl(xs).toSet.size == ys.size) - forRepl(xs).toSet.zip(ys).foreach{ case (e1, e2) => assert(e1 == e2) } - } - } - "conflict" - { - val vc1s = vc(v1(1), v2(2)) - val vc2s = vc(v1(2), v2(1)) - "seq" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=2, vc1s, stob("v11")), - Data(stob("k1"), bucket=1, lastModified=1, vc2s, stob("v12")), - Data(stob("k2"), bucket=1, lastModified=1, vc1s, stob("v2")), - ) - val ys = Set( - xs(0).copy(vc=vc(v1(2), v2(2))), - xs(2), - ) - assert(forRepl(xs).toSet.size == ys.size) - forRepl(xs).toSet.zip(ys).foreach{ case (e1, e2) => assert(e1 == e2) } - } - "reversed" in { - val xs = Vector( - Data(stob("k1"), bucket=1, lastModified=1, vc2s, stob("v11")), - Data(stob("k1"), bucket=1, lastModified=2, vc1s, stob("v12")), - Data(stob("k2"), bucket=1, lastModified=1, vc1s, stob("v2")), - ) - val ys = Set( - xs(1).copy(vc=vc(v1(2), v2(2))), - xs(2), - ) - assert(forRepl(xs).toSet.size == ys.size) - forRepl(xs).toSet.zip(ys).foreach{ case (e1, e2) => assert(e1 == e2) } - } - } - } - - "forPut" - { - import zd.rng.MergeOps.forPut - "stored is none" in { - val vc1 = vc(v1(1)) - val x = Data(stob("k1"), bucket=1, lastModified=1, vc1, stob("v1")) - forPut(None, x) should be (Some(x)) - } - "stored vc is older" in { - val vc1 = vc(v1(1)) - val vc2 = vc(v1(2)) - val x = Data(stob("k1"), bucket=1, lastModified=2, vc1, stob("v1")) - val y = Data(stob("k2"), bucket=1, lastModified=1, vc2, stob("v2")) - forPut(Some(x), y) should be (Some(y)) - } - "stored vc is newer" in { - val vc1 = vc(v1(2)) - val vc2 = vc(v1(1)) - val x = Data(stob("k1"), bucket=1, lastModified=1, vc1, stob("v1")) - val y = Data(stob("k2"), bucket=1, lastModified=2, vc2, stob("v2")) - forPut(Some(x), y) should be (None) - } - "vcs are the same" - { - val vc1 = vc(v1(1)) - "direct order" in { - val x = Data(stob("k1"), bucket=1, lastModified=1, vc1, stob("v1")) - val y = Data(stob("k2"), bucket=1, lastModified=2, vc1, stob("v2")) - forPut(Some(x), y) should be (Some(y)) - } - "reverse order" in { - val x = Data(stob("k1"), bucket=1, lastModified=2, vc1, stob("v1")) - val y = Data(stob("k2"), bucket=1, lastModified=1, vc1, stob("v2")) - forPut(Some(x), y) should be (None) - } - } - "vcs in conflict" - { - val vc1 = vc(v1(1), v2(2)) - val vc2 = vc(v1(2), v2(1)) - val mergedvc = vc1 merge vc2 - "direct order" in { - val x = Data(stob("k1"), bucket=1, lastModified=1, vc1, stob("v1")) - val y = Data(stob("k2"), bucket=1, lastModified=2, vc2, stob("v2")) - forPut(Some(x), y) should be (Some(y.copy(vc=mergedvc))) - } - "reverse order" in { - val x = Data(stob("k1"), bucket=1, lastModified=2, vc2, stob("v1")) - val y = Data(stob("k2"), bucket=1, lastModified=1, vc1, stob("v2")) - forPut(Some(x), y) should be (Some(x.copy(vc=mergedvc))) - } - } - } - - "forGatherGet" - { - import zd.rng.MergeOps.forGatherGet - import akka.actor.{Address} - def addr(n: Int): Address = Address("","","",n) - "empty" in { - forGatherGet(Vector.empty) should be (None -> HashSet.empty) - } - "newer in tail" in { - val xs = Vector( - Some(Data(stob("k1"), bucket=1, lastModified=1, vc(v1(2)), stob("v1"))) -> addr(1), - Some(Data(stob("k2"), bucket=1, lastModified=1, vc(v1(3)), stob("v2"))) -> addr(2), - Some(Data(stob("k3"), bucket=1, lastModified=1, vc(v1(1)), stob("v3"))) -> addr(3), - ) - forGatherGet(xs) should be (xs(1)._1 -> HashSet(addr(1), addr(3))) - } - "conflict" in { - val xs = Vector( - Some(Data(stob("k1"), bucket=1, lastModified=1, vc(v1(1)), stob("v1"))) -> addr(1), - Some(Data(stob("k2"), bucket=1, lastModified=2, vc(v2(1)), stob("v2"))) -> addr(2), - ) - forGatherGet(xs) should be (xs(1)._1 -> HashSet(addr(1))) - } - "none" in { - val xs = Vector( - None -> addr(1), - Some(Data(stob("k2"), bucket=1, lastModified=1, vc(v1(3)), stob("v2"))) -> addr(2), - ) - forGatherGet(xs) should be (xs(1)._1 -> HashSet(addr(1))) - } - } -} - -given [A, B]: CanEqual[A, B] = CanEqual.derived diff --git a/kvs/src/test/scala/rks.test.scala b/kvs/src/test/scala/rks.test.scala deleted file mode 100644 index e466933b..00000000 --- a/kvs/src/test/scala/rks.test.scala +++ /dev/null @@ -1,35 +0,0 @@ -package zd.kvs - -import akka.actor.ActorSystem -import akka.testkit.TestKit -import org.scalatest.freespec.AnyFreeSpecLike -import org.scalatest.matchers.should.Matchers -import org.scalatest.* - -class RksTest extends TestKit(ActorSystem("test")), AnyFreeSpecLike, Matchers, EitherValues, BeforeAndAfterAll { - val kvs = Kvs.rks(system, "target/rkstest") - - "return error when element is absent" in { - kvs.el.get[String]("key").right.value shouldBe None - } - "save value" in { - kvs.el.put("key", "value").right.value shouldBe "value" - } - "retrieve value" in { - kvs.el.get[String]("key").right.value shouldBe Some("value") - } - "override value" in { - kvs.el.put("key", "value2").right.value shouldBe "value2" - } - "delete value" in { - kvs.el.delete[String]("key").right.value shouldBe () - } - "clean up" in { - kvs.el.get[String]("key").right.value shouldBe None - kvs.close() - } - - override def afterAll() = { - TestKit.shutdownActorSystem(system) - } -} diff --git a/kvs/src/test/scala/search.test.scala b/kvs/src/test/scala/search.test.scala deleted file mode 100644 index 0a13f759..00000000 --- a/kvs/src/test/scala/search.test.scala +++ /dev/null @@ -1,112 +0,0 @@ -package zd.kvs -package search - -import akka.actor.ActorSystem -import com.typesafe.config.ConfigFactory -import org.scalatest.matchers.should.Matchers -import org.scalatest.freespec.AnyFreeSpecLike -import org.scalatest.* -import scala.concurrent.Await -import scala.concurrent.duration.* -import scala.util.Try -import proto.* - -class EnHandlerTest extends AnyFreeSpecLike with Matchers with BeforeAndAfterAll with EitherValues: - val fid = "files" - def entry(n: Int): En = En(fid=fid, id=n.toString, prev=zd.kvs.empty) - - val e1 = entry(1) - val e2 = entry(2) - val e3 = entry(3) - val e5 = entry(5) - - val kvs = Kvs.mem() - val h = EnHandler - given Dba = kvs.dba - - "Feed should" - { - "be empty at creation" in { - h.all(fid) shouldBe (Right(LazyList.empty)) - } - - "should save e1" in { - val saved = h.add(e1).getOrElse(???) - (saved.fid, saved.id) shouldBe ((e1.fid, "1")) - } - - "should save e2" in { - val saved = h.add(e2).getOrElse(???) - (saved.fid, saved.id) shouldBe ((e2.fid, "2")) - } - - "should get e1 and e2 from feed" in { - val stream = h.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e2.copy(prev="1")), Right(e1))) - } - - "should save entry(3)" in { - val saved = h.add(e3).getOrElse(???) - (saved.fid, saved.id) shouldBe ((e3.fid, "3")) - } - - "should not save entry(2) again" in { - h.add(e2).left.getOrElse(???) shouldBe EntryExists(s"${fid}.2") - } - - "should get 3 values from feed" in { - val stream = h.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e3.copy(prev="2")), Right(e2.copy(prev="1")), Right(e1))) - } - - "should not remove unexisting entry from feed" in { - h.remove(fid,"5").left.value shouldBe KeyNotFound - } - - "should remove entry(2) from feed without prev/next/data" in { - h.remove(e2.fid,"2").getOrElse(???) - } - - "should get 2 values from feed" in { - val stream = h.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e3.copy(prev="1")), Right(e1))) - } - - "should remove entry(1) from feed" in { - h.remove(fid,"1").getOrElse(???) - } - - "should get 1 values from feed" in { - val stream = h.all(fid) - stream.map(_.toList) shouldBe Right(List(Right(e3))) - } - - "should remove entry(3) from feed" in { - h.remove(fid,"3").getOrElse(???) - } - - "should be empty" in { - h.all(fid).getOrElse(???) shouldBe empty - } - - "should not create stack overflow" in { - val limit = 100 - LazyList.from(1,1).takeWhile( _.<=(limit)).foreach{ n => - val toadd = entry(n) - val added = h.add(toadd).getOrElse(???) - (added.fid, added.id) shouldBe ((toadd.fid, n.toString)) - } - LazyList.from(1,1).takeWhile( _.<=(limit)).foreach{ n => - val toremove = entry(n) - h.remove(toremove.fid, toremove.id).getOrElse(???) - } - } - - "feed should be empty at the end test" in { - kvs.el.delete[String](s"IdCounter.$fid") - h.all(fid).getOrElse(???) shouldBe empty - h.delete(Fd(fid)) - h.all(fid) shouldBe (Right(LazyList.empty)) - } - } - -given [A, B]: CanEqual[A, B] = CanEqual.derived diff --git a/project/build.properties b/project/build.properties index 875272df..04267b14 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.8.2 \ No newline at end of file +sbt.version=1.9.9 diff --git a/project/plugins.sbt b/project/plugins.sbt deleted file mode 100644 index d5462617..00000000 --- a/project/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("io.github.zero-deps" % "sbt-git" % "2.5.3.gd2541c1") diff --git a/ring/src/Data.scala b/ring/src/Data.scala new file mode 100644 index 00000000..80e63b45 --- /dev/null +++ b/ring/src/Data.scala @@ -0,0 +1,36 @@ +package kvs.rng +package data + +import proto.* + +import org.apache.pekko.cluster.given + +sealed trait StoreKey + +@N(1) final case class DataKey(@N(1) bucket: Int, @N(2) key: Array[Byte]) extends StoreKey + +final case class Data + ( @N(1) lastModified: Long + , @N(2) vc: VectorClock + , @N(3) value: Array[Byte] + ) + +@N(2) final case class BucketInfoKey(@N(1) bucket: Int) extends StoreKey + +final case class BucketInfo + ( @N(1) vc: VectorClock + , @N(2) keys: Vector[Array[Byte]] + ) + +object keycodec { + implicit val StoreKeyC: MessageCodec[StoreKey] = { + implicit val BucketInfoKeyC: MessageCodec[BucketInfoKey] = caseCodecAuto + implicit val DataKeyC: MessageCodec[DataKey] = caseCodecAuto + sealedTraitCodecAuto[StoreKey] + } +} + +object codec { + implicit val bucketInfoCodec: MessageCodec[BucketInfo] = caseCodecAuto + implicit val dataCodec: MessageCodec[Data] = caseCodecAuto +} diff --git a/ring/src/GatherDel.scala b/ring/src/GatherDel.scala new file mode 100644 index 00000000..4688da55 --- /dev/null +++ b/ring/src/GatherDel.scala @@ -0,0 +1,50 @@ +package kvs.rng + +import org.apache.pekko.actor.* +import org.apache.pekko.cluster.Cluster +import scala.concurrent.duration.* + +class GatherDel(client: ActorRef, t: FiniteDuration, prefList: Set[Node], k: Array[Byte], conf: Conf) extends FSM[FsmState, Set[Node]] with ActorLogging { + val quorum = conf.quorum + val W: Int = quorum.W + val local: Address = Cluster(context.system).selfAddress + setTimer("send_by_timeout", "timeout", t) + + startWith(Collecting, prefList) + + when(Collecting){ + case Event("ok", nodesLeft) => + nodesLeft - addr1(sender()) match { + case enough if prefList.size - enough.size == W => // W nodes removed key + client ! AckSuccess(None) + goto(Sent) using(enough) + case less => stay() using(less) + } + + case Event("timeout", nodesLeft) => + /* politic of revert is not needed because on read opperation removed data will be saved again, + * only notify client about failed opperation. + * deleted on other nodes but we don't know about it ? sorry, eventually consistency + */ + client ! AckTimeoutFailed("del", k) + stop() + } + + when(Sent){ + case Event("ok", data) => + data - addr1(sender()) match { + case x if x.isEmpty => stop() + case nodes => stay() using(nodes) + } + + case Event("timeout", _) => stop() + } + + def addr1(s: ActorRef) = if (addr(s).hasLocalScope) local else addr(s) + + initialize() +} + +object GatherDel { + def props(client: ActorRef, t: FiniteDuration, prefList: Set[Node], k: Array[Byte], conf: Conf): Props = Props(new GatherDel(client, t, prefList, k, conf)) +} diff --git a/kvs/src/main/scala/rng/GatherGet.scala b/ring/src/GatherGet.scala similarity index 62% rename from kvs/src/main/scala/rng/GatherGet.scala rename to ring/src/GatherGet.scala index 3e89b6a9..e7bf252a 100644 --- a/kvs/src/main/scala/rng/GatherGet.scala +++ b/ring/src/GatherGet.scala @@ -1,29 +1,28 @@ -package zd.rng +package kvs.rng import annotation.unused -import akka.actor.* -import zd.rng.data.Data -import zd.rng.model.{StoreGetAck, StorePut, StoreDelete} +import org.apache.pekko.actor.* import scala.concurrent.duration.* import scala.collection.immutable.{HashSet} +import model.{StoreGetAck, StorePut, StoreDelete}, data.Data import GatherGet.DataCollection -class GatherGet(client: ActorRef, t: FiniteDuration, M: Int, @unused R: Int, k: Key) extends FSM[FsmState, DataCollection] with ActorLogging { +class GatherGet(client: ActorRef, t: FiniteDuration, M: Int, @unused R: Int, k: Array[Byte]) extends FSM[FsmState, DataCollection] with ActorLogging { val stores = SelectionMemorize(context.system) startWith(Collecting, DataCollection(Vector.empty, 0)) setTimer("send_by_timeout", "timeout", t) when(Collecting) { - case Event(StoreGetAck(data), DataCollection(perNode, nodes)) => - val xs = (data -> addr(sender)) +: perNode + case Event(StoreGetAck(key, bucket, data), DataCollection(perNode, nodes)) => + val xs = (data -> addr(sender())) +: perNode nodes + 1 match { - case `M` => //todo; wait for first R same answers? + case `M` => // alternative is to wait for first R same answers cancelTimer("send_by_timeout") val (correct: Option[Data], outdated: HashSet[Node]) = MergeOps.forGatherGet(xs) ;{ // update outdated nodes with correct data - val msg = correct.fold[Any](StoreDelete(k))(d => StorePut(d)) + val msg = correct.fold[Any](StoreDelete(k))(d => StorePut(key, bucket, d)) outdated foreach { node => stores.get(node, "ring_write_store").fold(_ ! msg, _ ! msg) } @@ -31,19 +30,19 @@ class GatherGet(client: ActorRef, t: FiniteDuration, M: Int, @unused R: Int, k: client ! AckSuccess(correct.map(_.value)) stop() case ns => - stay using DataCollection(xs, ns) + stay() using DataCollection(xs, ns) } case Event("timeout", _) => - client ! AckTimeoutFailed("get", new String(k, "UTF-8")) + client ! AckTimeoutFailed("get", k) stop() } } object GatherGet { - def props(client: ActorRef, t: FiniteDuration, M: Int, R: Int, k: Key): Props = Props(new GatherGet(client, t, M, R, k)) + def props(client: ActorRef, t: FiniteDuration, M: Int, R: Int, k: Array[Byte]): Props = Props(new GatherGet(client, t, M, R, k)) type AddrOfData = (Option[Data], Node) - final case class DataCollection(perNode: Vector[AddrOfData], nodes: Int) + case class DataCollection(perNode: Vector[AddrOfData], nodes: Int) } diff --git a/ring/src/GatherPut.scala b/ring/src/GatherPut.scala new file mode 100644 index 00000000..29d05ee0 --- /dev/null +++ b/ring/src/GatherPut.scala @@ -0,0 +1,73 @@ +package kvs.rng + +import org.apache.pekko.actor.{ActorLogging, ActorRef, FSM, Props, RootActorPath} +import scala.concurrent.duration.* + +import data.Data, model.{StoreGetAck, StorePut} + +case class PutInfo( + key: Array[Byte] + , v: Array[Byte] + , N: Int + , W: Int + , bucket: Bucket + , localAdr: Node + , nodes: Set[Node] + ) + +object GatherPut { + def props(client: ActorRef, t: FiniteDuration, putInfo: PutInfo): Props = Props(new GatherPut(client, t, putInfo)) +} + +class GatherPut(client: ActorRef, t: FiniteDuration, putInfo: PutInfo) extends FSM[FsmState, Int] with ActorLogging { + + startWith(Collecting, 0) + setTimer("send_by_timeout", "timeout", t) + + when(Collecting){ + case Event(StoreGetAck(key, bucket, data), _) => + val vc = if (data.size == 1) { + data.head.vc + } else if (data.size > 1) { + data.map(_.vc).foldLeft(emptyVC)((sum, i) => sum.merge(i)) + } else { + emptyVC + } + val updatedData = Data(now_ms(), vc.:+(putInfo.localAdr.toString), putInfo.v) + mapInPut(putInfo.nodes, key=key, bucket=bucket, updatedData) + stay() + + case Event("ok", n) => + val n1 = n + 1 + if (n1 == putInfo.N) { + client ! AckSuccess(None) + stop() + } else if (n1 == putInfo.W) { + client ! AckSuccess(None) + goto (Sent) using n1 + } else { + stay() using n1 + } + + case Event("timeout", _) => + client ! AckTimeoutFailed("put", putInfo.key) + stop() + } + + // keep fsm running to avoid dead letter warnings + when(Sent){ + case Event("ok", n) => + val n1 = n + 1 + if (n1 == putInfo.N) stop() + else stay() using n1 + case Event("timeout", _) => + stop() + } + + def mapInPut(nodes: Set[Node], key: Array[Byte], bucket: Int, d: Data) = { + val storeList = nodes.map(n => RootActorPath(n) / "user" / "ring_write_store") + storeList.foreach(ref => context.system.actorSelection(ref).tell(StorePut(key=key, bucket=bucket, d), self)) + } + + initialize() +} diff --git a/ring/src/Hash.scala b/ring/src/Hash.scala new file mode 100644 index 00000000..70cb8a3d --- /dev/null +++ b/ring/src/Hash.scala @@ -0,0 +1,241 @@ +package kvs.rng + +import org.apache.pekko.actor.* +import org.apache.pekko.cluster.ClusterEvent.* +import org.apache.pekko.cluster.{Member, Cluster} +import scala.collection.immutable.{SortedMap, SortedSet} +import proto.* + +import model.{StoreDelete, StoreGet, QuorumState, ChangeState}, model.QuorumState.{QuorumStateUnsatisfied, QuorumStateReadonly, QuorumStateEffective} + +case class Put(k: Array[Byte], v: Array[Byte]) +case class Get(k: Array[Byte]) +case class Delete(k: Array[Byte]) + +case object RestoreState + +case class InternalPut(k: Array[Byte], v: Array[Byte]) + +case class HashRngData( + nodes: Set[Node], + buckets: SortedMap[Bucket, PreferenceList], + vNodes: SortedMap[Bucket, Node], + replication: Option[ActorRef], +) + +case class PortVNode( + @N(1) port: String +, @N(2) vnode: Int +) + +object Hash { + def props(conf: Conf, hashing: Hashing): Props = Props(new Hash(conf, hashing)) +} + +class Hash(conf: Conf, hashing: Hashing) extends FSM[QuorumState, HashRngData] with ActorLogging { + import context.system + + val quorum = conf.quorum + val N = quorum.N + val W = quorum.W + val R = quorum.R + val gatherTimeout = conf.gatherTimeout + val vNodesNum = conf.virtualNodes + val bucketsNum = conf.buckets + val cluster = Cluster(system) + val local: Node = cluster.selfAddress + val actorsMem = SelectionMemorize(system) + + startWith(QuorumStateUnsatisfied, HashRngData(Set.empty[Node], SortedMap.empty[Bucket, PreferenceList], SortedMap.empty[Bucket, Node], replication=None)) + + override def preStart() = { + cluster.subscribe(self, initialStateMode = InitialStateAsEvents, classOf[MemberUp], classOf[MemberRemoved]) + } + + override def postStop(): Unit = cluster.unsubscribe(self) + + when(QuorumStateUnsatisfied){ + case Event(_: Get, _) => + sender() ! AckQuorumFailed("QuorumStateUnsatisfied") + stay() + case Event(_: Put, _) => + sender() ! AckQuorumFailed("QuorumStateUnsatisfied") + stay() + case Event(_: Delete, _) => + sender() ! AckQuorumFailed("QuorumStateUnsatisfied") + stay() + case Event(RestoreState, _) => + log.warning("Don't know how to restore state when quorum is unsatisfied") + stay() + } + + when(QuorumStateReadonly){ + case Event(x: Get, data) => + doGet(x.k, sender(), data) + stay() + case Event(_: Put, _) => + sender() ! AckQuorumFailed("QuorumStateReadonly") + stay() + case Event(_: Delete, _) => + sender() ! AckQuorumFailed("QuorumStateReadonly") + stay() + case Event(RestoreState, data) => + val s = state(data.nodes.size) + data.nodes.foreach(n => actorsMem.get(n, "ring_hash").fold( + _ ! ChangeState(s), + _ ! ChangeState(s), + )) + goto(s) + } + + when(QuorumStateEffective){ + case Event(x: Get, data) => + doGet(x.k, sender(), data) + stay() + case Event(x: Put, data) => + doPut(x.k, x.v, sender(), data) + stay() + case Event(x: Delete, data) => + doDelete(x.k, sender(), data) + stay() + case Event(RestoreState, _) => + log.info("State is already OK") + stay() + } + + /* common for all states */ + whenUnhandled { + case Event(MemberUp(member), data) => + val next = joinNodeToRing(member, data) + goto(next._1) using next._2 + case Event(MemberRemoved(member, prevState), data) => + val next = removeNodeFromRing(member, data) + goto(next._1) using next._2 + case Event(ChangeState(s), data) => + state(data.nodes.size) match { + case QuorumStateUnsatisfied => stay() + case _ => goto(s) + } + case Event(x: InternalPut, data) => + doPut(x.k, x.v, sender(), data) + stay() + } + + def doDelete(k: Array[Byte], client: ActorRef, data: HashRngData): Unit = { + val nodes = nodesForKey(k, data) + val gather = system.actorOf(GatherDel.props(client, gatherTimeout, nodes, k, conf)) + val stores = nodes.map{actorsMem.get(_, "ring_write_store")} + stores.foreach(_.fold( + _.tell(StoreDelete(k), gather), + _.tell(StoreDelete(k), gather), + )) + } + + def doPut(k: Array[Byte], v: Array[Byte], client: ActorRef, data: HashRngData): Unit = { + val nodes = availableNodesFrom(nodesForKey(k, data)) + val M = nodes.size + if (M >= W) { + val bucket = hashing.findBucket(k) + val info = PutInfo(k, v, N, W, bucket, local, data.nodes) + val gather = system.actorOf(GatherPut.props(client, gatherTimeout, info)) + val node = if (nodes contains local) local else nodes.head + actorsMem.get(node, "ring_readonly_store").fold( + _.tell(StoreGet(k), gather), + _.tell(StoreGet(k), gather), + ) + } else { + client ! AckQuorumFailed("M >= W") + } + } + + def doGet(k: Array[Byte], client: ActorRef, data: HashRngData): Unit = { + val nodes = availableNodesFrom(nodesForKey(k, data)) + val M = nodes.size + if (M >= R) { + val gather = system.actorOf(GatherGet.props(client, gatherTimeout, M, R, k)) + val stores = nodes map { actorsMem.get(_, "ring_readonly_store") } + stores foreach (_.fold( + _.tell(StoreGet(k), gather), + _.tell(StoreGet(k), gather), + )) + } else { + client ! AckQuorumFailed("M >= R") + } + } + + def availableNodesFrom(l: Set[Node]): Set[Node] = { + val unreachableMembers = cluster.state.unreachable.map(m => m.address) + l filterNot (node => unreachableMembers contains node) + } + + def joinNodeToRing(member: Member, data: HashRngData): (QuorumState, HashRngData) = { + val newvNodes: Map[VNode, Node] = (1 to vNodesNum).view.map(vnode => + hashing.hash(encode(PortVNode(port=member.address.hostPort, vnode=vnode))) -> member.address + ).to(Map) + val updvNodes = data.vNodes ++ newvNodes + val nodes = data.nodes + member.address + val moved = bucketsToUpdate(bucketsNum - 1, Math.min(nodes.size,N), updvNodes, data.buckets) + data.replication map (context stop _) + val repl = syncNodes(moved) + val updData = HashRngData(nodes, data.buckets++moved, updvNodes, Some(repl)) + log.info(s"Node ${member.address} is joining ring. Nodes in ring = ${updData.nodes.size}, state = ${state(updData.nodes.size)}") + state(updData.nodes.size) -> updData + } + + def removeNodeFromRing(member: Member, data: HashRngData): (QuorumState, HashRngData) = { + log.info(s"Removing ${member} from ring") + val unusedvNodes: Set[VNode] = (1 to vNodesNum).view.map(vnode => + hashing.hash(encode(PortVNode(port=member.address.hostPort, vnode=vnode))) + ).to(Set) + val updvNodes = data.vNodes.filterNot(vn => unusedvNodes.contains(vn._1)) + val nodes = data.nodes - member.address + val moved = bucketsToUpdate(bucketsNum - 1, Math.min(nodes.size,N), updvNodes, data.buckets) + log.info(s"Will update ${moved.size} buckets") + data.replication map (context stop _) + val repl = syncNodes(moved) + val updData = HashRngData(nodes, data.buckets++moved, updvNodes, Some(repl)) + state(updData.nodes.size) -> updData + } + + def itob(v: Int): Array[Byte] = Array[Byte]((v >> 24).toByte, (v >> 16).toByte, (v >> 8).toByte, v.toByte) + + def syncNodes(_buckets: SortedMap[Bucket,PreferenceList]): ActorRef = { + val empty = SortedMap.empty[Bucket,PreferenceList] + val buckets = _buckets.foldLeft(empty){ case (acc, (b, prefList)) => + if (prefList contains local) { + prefList.filterNot(_ == local) match { + case empty if empty.isEmpty => acc + case prefList => acc + (b -> prefList) + } + } else acc + } + val replication = context.actorOf(ReplicationSupervisor.props(buckets, conf), s"repl-${now_ms()}") + replication ! "go-repl" + replication + } + + def state(nodes: Int): QuorumState = nodes match { + case 0 => QuorumStateUnsatisfied + case n if n >= Math.max(R, W) => QuorumStateEffective + case _ => QuorumStateReadonly + } + + def bucketsToUpdate(maxBucket: Bucket, nodesNumber: Int, vNodes: SortedMap[Bucket, Node], buckets: SortedMap[Bucket, PreferenceList]): SortedMap[Bucket, PreferenceList] = { + (0 to maxBucket).foldLeft(SortedMap.empty[Bucket, PreferenceList])((acc, b) => { + val prefList = hashing.findNodes(b * hashing.bucketRange, vNodes, nodesNumber) + buckets.get(b) match + case None => acc + (b -> prefList) + case Some(`prefList`) => acc + case _ => acc + (b -> prefList) + }) + } + + implicit val ord: Ordering[Node] = Ordering.by[Node, String](n => n.hostPort) + + def nodesForKey(k: Array[Byte], data: HashRngData): PreferenceList = data.buckets.get(hashing.findBucket(k)) match { + case None => SortedSet.empty[Node] + case Some(nods) => nods + } + + initialize() +} diff --git a/kvs/src/main/scala/rng/HashingExtension.scala b/ring/src/Hashing.scala similarity index 54% rename from kvs/src/main/scala/rng/HashingExtension.scala rename to ring/src/Hashing.scala index 0f0467c8..e31d0e23 100644 --- a/kvs/src/main/scala/rng/HashingExtension.scala +++ b/ring/src/Hashing.scala @@ -1,27 +1,26 @@ -package zd.rng +package kvs.rng -import akka.actor.* -import com.typesafe.config.Config +import org.apache.pekko.actor.* import java.security.MessageDigest import scala.annotation.tailrec -import scala.collection.{SortedMap} +import scala.collection.SortedMap -class HashingImpl(config: Config) extends Extension { - val hashLen = config.getInt("hash-length") - val bucketsNum = config.getInt("buckets") +class Hashing(conf: Conf) { + val hashLen = conf.hashLength + val bucketsNum = conf.buckets val bucketRange = (math.pow(2, hashLen.toDouble) / bucketsNum).ceil.toInt def hash(word: Array[Byte]): Int = { - implicit val digester: MessageDigest = MessageDigest.getInstance("MD5").nn - digester `update` word - val digest: Array[Byte] = digester.digest.nn + val digester = MessageDigest.getInstance("MD5").nn + digester.update(word) + val digest = digester.digest.nn (0 to hashLen / 8 - 1).foldLeft(0)((acc, i) => acc | ((digest(i) & 0xff) << (8 * (hashLen / 8 - 1 - i))) ) //take first 4 byte } - def findBucket(key: Key): Bucket = (hash(key) / bucketRange).abs + def findBucket(key: Array[Byte]): Bucket = (hash(key) / bucketRange).abs def findNodes(hashKey: Int, vNodes: SortedMap[Bucket, Address], nodesNumber: Int): PreferenceList = { @tailrec @@ -39,11 +38,3 @@ class HashingImpl(config: Config) extends Extension { findBucketNodes(hashKey, Set.empty[Node]) } } - -object HashingExtension extends ExtensionId[HashingImpl] with ExtensionIdProvider { - - override def createExtension(system: ExtendedActorSystem): HashingImpl = - new HashingImpl(system.settings.config.getConfig("ring").nn) - - override def lookup(): HashingExtension.type = HashingExtension -} diff --git a/ring/src/Msg.scala b/ring/src/Msg.scala new file mode 100644 index 00000000..933f841f --- /dev/null +++ b/ring/src/Msg.scala @@ -0,0 +1,76 @@ +package kvs.rng +package model + +import proto.N + +import kvs.rng.data.* + +sealed trait Msg + +@N(1) case class ChangeState + ( @N(1) s: QuorumState + ) extends Msg + +@N(2) case class DumpBucketData + ( @N(1) b: Int + , @N(2) items: Vector[KeyBucketData] + ) extends Msg + +@N(5) case class DumpGetBucketData + ( @N(1) b: Int + ) extends Msg + +@N(7) case class ReplBucketPut + ( @N(1) b: Int + , @N(2) bucketVc: VectorClock + , @N(3) items: Vector[KeyBucketData] + ) extends Msg + +@N(8) case object ReplBucketUpToDate extends Msg + +@N(9) case class ReplGetBucketIfNew + ( @N(1) b: Int + , @N(2) vc: VectorClock + ) extends Msg + +@N(10) case class ReplNewerBucketData + ( @N(1) vc: VectorClock + , @N(2) items: Vector[KeyBucketData] + ) extends Msg + +case class ReplBucketsVc + ( @N(1) bvcs: Map[Int, VectorClock] + ) + +@N(11) case class StoreDelete + ( @N(1) key: Array[Byte] + ) extends Msg + +@N(12) case class StoreGet + ( @N(1) key: Array[Byte] + ) extends Msg + +@N(13) case class StoreGetAck + ( @N(1) key: Array[Byte] + , @N(2) bucket: Int + , @N(3) data: Option[Data] + ) extends Msg + +@N(14) case class StorePut + ( @N(1) key: Array[Byte] + , @N(2) bucket: Int + , @N(3) data: Data + ) extends Msg + +case class KeyBucketData + ( @N(1) key: Array[Byte] + , @N(2) bucket: Int + , @N(3) data: Data + ) + +sealed trait QuorumState +object QuorumState { + @N(1) case object QuorumStateUnsatisfied extends QuorumState + @N(2) case object QuorumStateReadonly extends QuorumState + @N(3) case object QuorumStateEffective extends QuorumState +} diff --git a/kvs/src/main/scala/rng/Replication.scala b/ring/src/Replication.scala similarity index 69% rename from kvs/src/main/scala/rng/Replication.scala rename to ring/src/Replication.scala index 82aae4ba..10c20ef3 100644 --- a/kvs/src/main/scala/rng/Replication.scala +++ b/ring/src/Replication.scala @@ -1,24 +1,24 @@ -package zd.rng +package kvs.rng -import akka.actor.{ActorLogging, Props, FSM} -import akka.cluster.{Cluster} -import zd.rng.data.{Data} -import zd.rng.model.{ReplBucketPut, ReplGetBucketsVc, ReplBucketsVc, ReplGetBucketIfNew, ReplBucketUpToDate, ReplNewerBucketData} -import scala.collection.immutable.{SortedMap} -import scala.concurrent.duration.{Duration} -import zd.rng.ReplicationSupervisor.{State} +import scala.collection.immutable.SortedMap +import org.apache.pekko.actor.{ActorLogging, Props, FSM} +import org.apache.pekko.cluster.Cluster + +import model.{ReplBucketPut, ReplBucketsVc, ReplGetBucketIfNew, ReplBucketUpToDate, ReplNewerBucketData, KeyBucketData} +import ReplicationSupervisor.{State, ReplGetBucketsVc} object ReplicationSupervisor { - final case class Progress(done: Int, total: Int, step: Int) - final case class State(buckets: SortedMap[Bucket, PreferenceList], bvcs: Map[Bucket, VectorClock], progress: Progress) + case class Progress(done: Int, total: Int, step: Int) + case class State(buckets: SortedMap[Bucket, PreferenceList], bvcs: Map[Bucket, VectorClock], progress: Progress) + case class ReplGetBucketsVc(bs: Vector[Int]) - def props(buckets: SortedMap[Bucket, PreferenceList]): Props = { + def props(buckets: SortedMap[Bucket, PreferenceList], conf: Conf): Props = { val len = buckets.size - Props(new ReplicationSupervisor(State(buckets, bvcs=Map.empty, Progress(done=0, total=len, step=len/4)))) + Props(new ReplicationSupervisor(State(buckets, bvcs=Map.empty, Progress(done=0, total=len, step=len/4)), conf)) } } -class ReplicationSupervisor(initialState: State) extends FSM[FsmState, State] with ActorLogging { +class ReplicationSupervisor(initialState: State, conf: Conf) extends FSM[FsmState, State] with ActorLogging { val actorMem = SelectionMemorize(context.system) val local: Node = Cluster(context.system).selfAddress @@ -57,7 +57,7 @@ class ReplicationSupervisor(initialState: State) extends FSM[FsmState, State] wi } def getBucketIfNew(b: Bucket, prefList: PreferenceList, bvc: Option[VectorClock]): Unit = { - val worker = context.actorOf(ReplicationWorker.props(b, prefList, bvc.getOrElse(emptyVC))) + val worker = context.actorOf(ReplicationWorker.props(b, prefList, bvc.getOrElse(emptyVC), conf)) worker ! "start" } @@ -73,7 +73,7 @@ class ReplicationSupervisor(initialState: State) extends FSM[FsmState, State] wi val (b, prefList) = remaining.head // safe val bvc = state.bvcs.get(b) getBucketIfNew(b, prefList, bvc) - stay using state.copy(buckets=remaining, progress=pr.copy(done=pr.done+1)) + stay() using state.copy(buckets=remaining, progress=pr.copy(done=pr.done+1)) } } } @@ -81,18 +81,18 @@ class ReplicationSupervisor(initialState: State) extends FSM[FsmState, State] wi import ReplicationWorker.{ReplState} object ReplicationWorker { - final case class ReplState(prefList: PreferenceList, info: Vector[Vector[Data]], vc: VectorClock) + case class ReplState(prefList: PreferenceList, info: Vector[Vector[KeyBucketData]], vc: VectorClock) - def props(b: Bucket, prefList: PreferenceList, vc: VectorClock): Props = Props(new ReplicationWorker(b, prefList, vc)) + def props(b: Bucket, prefList: PreferenceList, vc: VectorClock, conf: Conf): Props = Props(new ReplicationWorker(b, prefList, vc, conf)) } -class ReplicationWorker(b: Bucket, _prefList: PreferenceList, _vc: VectorClock) extends FSM[FsmState, ReplState] with ActorLogging { +class ReplicationWorker(b: Bucket, _prefList: PreferenceList, _vc: VectorClock, conf: Conf) extends FSM[FsmState, ReplState] with ActorLogging { import context.system val cluster = Cluster(system) val local = cluster.selfAddress val actorMem = SelectionMemorize(system) - setTimer("send_by_timeout", "timeout", Duration.fromNanos(context.system.settings.config.getDuration("ring.repl-timeout").nn.toNanos), repeat=true) + setTimer("send_by_timeout", "timeout", conf.replTimeout, repeat=true) startWith(Collecting, ReplState(_prefList, info=Vector.empty, _vc)) when(Collecting){ @@ -102,12 +102,11 @@ class ReplicationWorker(b: Bucket, _prefList: PreferenceList, _vc: VectorClock) _ ! ReplGetBucketIfNew(b, _vc), _ ! ReplGetBucketIfNew(b, _vc), )) - stay using state + stay() using state - case Event(ReplNewerBucketData(vc, _items), state) => - val items = _items.toVector - if (state.prefList contains addr(sender)) { - state.prefList - addr(sender) match { + case Event(ReplNewerBucketData(vc, items), state) => + if (state.prefList contains addr(sender())) { + state.prefList - addr(sender()) match { case empty if empty.isEmpty => val all = state.info.foldLeft(items)((acc, list) => list ++ acc) val merged = MergeOps.forRepl(all) @@ -120,7 +119,7 @@ class ReplicationWorker(b: Bucket, _prefList: PreferenceList, _vc: VectorClock) context.parent ! b stop() case nodes => - stay using state.copy( + stay() using state.copy( prefList = nodes, info = items +: state.info, vc = state.vc merge vc, @@ -128,17 +127,17 @@ class ReplicationWorker(b: Bucket, _prefList: PreferenceList, _vc: VectorClock) } } else { // after restart it is possible to receive multiple answers from same node - stay using state + stay() using state } case Event(ReplBucketUpToDate, state) => self forward ReplNewerBucketData(vc=emptyVC, items=Vector.empty) - stay using state + stay() using state case Event("timeout", state) => log.info(s"no answer. repeat with=${state.prefList}") self ! "start" - stay using state + stay() using state } initialize() diff --git a/kvs/src/main/scala/rng/SelectionMemorize.scala b/ring/src/SelectionMemorize.scala similarity index 93% rename from kvs/src/main/scala/rng/SelectionMemorize.scala rename to ring/src/SelectionMemorize.scala index e8792c30..5d686196 100644 --- a/kvs/src/main/scala/rng/SelectionMemorize.scala +++ b/ring/src/SelectionMemorize.scala @@ -1,6 +1,6 @@ -package zd.rng +package kvs.rng -import akka.actor.* +import org.apache.pekko.actor.* case class Watch(a: ActorRef) case class Select(node: Node, path: String) @@ -10,7 +10,7 @@ object SelectionMemorize extends ExtensionId[SelectionMemorize] with ExtensionId override def createExtension(system: ExtendedActorSystem): SelectionMemorize = new SelectionMemorize(system) - override def lookup(): SelectionMemorize.type = SelectionMemorize + override def lookup: SelectionMemorize.type = SelectionMemorize } trait ActorRefStorage { diff --git a/ring/src/StoreReadonly.scala b/ring/src/StoreReadonly.scala new file mode 100644 index 00000000..fa4ac6c0 --- /dev/null +++ b/ring/src/StoreReadonly.scala @@ -0,0 +1,61 @@ +package kvs.rng +package store + +import org.apache.pekko.actor.{Actor, ActorLogging, Props} +import org.apache.pekko.cluster.{VectorClock} +import org.rocksdb.* +import proto.{encode, decode} + +import data.{Data, BucketInfo, StoreKey, DataKey, BucketInfoKey}, data.codec.*, data.keycodec.*, model.{DumpGetBucketData, DumpBucketData}, model.{ReplBucketsVc, ReplGetBucketIfNew, ReplBucketUpToDate, ReplNewerBucketData, KeyBucketData}, model.{StoreGet, StoreGetAck}, ReplicationSupervisor.ReplGetBucketsVc + +object ReadonlyStore { + def props(db: RocksDB, hashing: Hashing): Props = Props(new ReadonlyStore(db, hashing)) +} + +class ReadonlyStore(db: RocksDB, hashing: Hashing) extends Actor with ActorLogging { + def get(k: Key): Option[Array[Byte]] = Option(db.get(k)).map(_.nn) + + override def receive: Receive = { + case x: StoreGet => + val k = DataKey(bucket=hashing.findBucket(x.key), key=x.key) + val result: Option[Data] = get(encode[StoreKey](k)).map(decode[Data](_)) + sender() ! StoreGetAck(bucket=k.bucket, key=x.key, data=result) + + case DumpGetBucketData(b) => + val k = encode[StoreKey](BucketInfoKey(bucket=b)) + val b_info = get(k).map(decode[BucketInfo](_)) + val keys = b_info.map(_.keys).getOrElse(Vector.empty) + val items = keys.flatMap(key => + get(encode[StoreKey](DataKey(bucket=b, key=key))).map(decode[Data](_)).map(data => KeyBucketData(key=key, bucket=b, data=data)) + ) + sender() ! DumpBucketData(b, items) + case ReplGetBucketIfNew(b, vc) => + val vc_other: VectorClock = vc + val k = encode[StoreKey](BucketInfoKey(bucket=b)) + val b_info = get(k).map(decode[BucketInfo](_)) + b_info match { + case Some(b_info) => + val vc_local: VectorClock = b_info.vc + vc_other == vc_local || vc_other > vc_local match { + case true => sender() ! ReplBucketUpToDate + case false => + val keys = b_info.keys + val items = keys.flatMap{ key => + val data = get(encode[StoreKey](DataKey(bucket=b, key=key))).map(decode[Data](_)) + data.map(data => KeyBucketData(key=key, bucket=b, data=data)) + } + sender() ! ReplNewerBucketData(b_info.vc, items) + } + case None => + sender() ! ReplBucketUpToDate + } + case ReplGetBucketsVc(bs) => + val bvcs: Bucket Map VectorClock = bs.view.flatMap{ b => + val k = encode[StoreKey](BucketInfoKey(bucket=b)) + get(k).map(x => b -> decode[BucketInfo](x).vc) + }.to(Map) + sender() ! ReplBucketsVc(bvcs) + + case _ => + } +} diff --git a/kvs/src/main/scala/rng/StoreWrite.scala b/ring/src/StoreWrite.scala similarity index 50% rename from kvs/src/main/scala/rng/StoreWrite.scala rename to ring/src/StoreWrite.scala index 1b9eb6a1..385df7fd 100644 --- a/kvs/src/main/scala/rng/StoreWrite.scala +++ b/ring/src/StoreWrite.scala @@ -1,50 +1,45 @@ -package zd.rng +package kvs.rng package store -import akka.actor.{Actor, ActorLogging, Props} -import akka.cluster.{Cluster, VectorClock} -import java.util.Arrays -import leveldbjnr.* -import proto.* -import zd.rng.data.codec.* -import zd.rng.data.{Data, BucketInfo} -import zd.rng.model.{ReplBucketPut, StorePut, StoreDelete} +import org.apache.pekko.actor.{Actor, ActorLogging, Props} +import org.apache.pekko.cluster.{Cluster, VectorClock} +import org.rocksdb.* +import proto.{encode, decode} -class WriteStore(leveldb: LevelDb) extends Actor with ActorLogging { +import data.codec.*, data.keycodec.*, data.{Data, BucketInfo, StoreKey, DataKey, BucketInfoKey}, model.{ReplBucketPut, StorePut, StoreDelete, KeyBucketData} + +object WriteStore { + def props(db: RocksDB, hashing: Hashing): Props = Props(new WriteStore(db, hashing)) +} + +class WriteStore(db: RocksDB, hashing: Hashing) extends Actor with ActorLogging { import context.system - val config = system.settings.config.getConfig("ring.leveldb").nn - val ro = ReadOpts() - val wo = WriteOpts(config.getBoolean("fsync")) - val hashing = HashingExtension(system) + val wo = new WriteOptions val local: Node = Cluster(system).selfAddress - def get(k: Key): Option[Array[Byte]] = leveldb.get(k, ro).fold(l => throw l, r => r) - - val `:key:` = stob(":key:") - val `:keys` = stob(":keys") + def get(k: Key): Option[Array[Byte]] = Option(db.get(k)).map(_.nn) override def postStop(): Unit = { - try { leveldb.close() } catch { case _: Throwable => () } - ro.close() + db.close() wo.close() super.postStop() } def receive: Receive = { - case StorePut(data) => - doPut(data) - sender ! "ok" - case x: StoreDelete => sender ! doDelete(x.key) - case ReplBucketPut(b, bucketVc, items) => replBucketPut(b, bucketVc, items.toVector) + case StorePut(key, bucket, data) => + doPut(key, bucket, data) + sender() ! "ok" + case x: StoreDelete => sender() ! doDelete(x.key) + case ReplBucketPut(b, bucketVc, items) => replBucketPut(b, bucketVc, items) case unhandled => log.warning(s"unhandled message: ${unhandled}") } - def replBucketPut(b: Bucket, bucketVc: VectorClock, items: Vector[Data]): Unit = { + def replBucketPut(b: Bucket, bucketVc: VectorClock, items: Vector[KeyBucketData]): Unit = { withBatch{ batch => { // updating bucket info - val bucketId: Key = itob(b) ++ `:keys` + val bucketId: Key = encode[StoreKey](BucketInfoKey(bucket=b)) val bucketInfo = get(bucketId).map(decode[BucketInfo](_)) val newKeys = items.map(_.key) val v = bucketInfo match { @@ -57,70 +52,62 @@ class WriteStore(leveldb: LevelDb) extends Actor with ActorLogging { } // saving keys data items.foreach{ data => - val keyPath: Key = itob(b) ++ `:key:` ++ data.key + val keyPath: Key = encode[StoreKey](DataKey(bucket=b, key=data.key)) val keyData: Option[Data] = get(keyPath).map(decode[Data](_)) - val v: Option[Data] = MergeOps.forPut(stored=keyData, received=data) + val v: Option[Data] = MergeOps.forPut(stored=keyData, received=data.data) v.map(v => batch.put(keyPath, encode(v))) } } } - def doPut(data: Data): Unit = { + def doPut(key: Array[Byte], bucket: Int, data: Data): Unit = { val _ = withBatch{ batch => { // updating bucket info - val bucketId: Key = itob(data.bucket) ++ `:keys` + val bucketId: Key = encode[StoreKey](BucketInfoKey(bucket=bucket)) val bucketInfo = get(bucketId).map(decode[BucketInfo](_)) val v = bucketInfo match { - case Some(x) if x.keys contains data.key => + case Some(x) if x.keys contains key => val vc = x.vc :+ local.toString x.copy(vc=vc) case Some(x) => val vc = x.vc :+ local.toString - x.copy(vc=vc, keys=(data.key +: x.keys)) + x.copy(vc=vc, keys=(key +: x.keys)) case None => val vc = emptyVC :+ local.toString - BucketInfo(vc=vc, keys=Vector(data.key)) + BucketInfo(vc=vc, keys=Vector(key)) } batch.put(bucketId, encode(v)) } // saving key data - val keyPath: Key = itob(data.bucket) ++ `:key:` ++ data.key + val keyPath: Key = encode[StoreKey](DataKey(bucket=bucket, key=key)) val keyData: Option[Data] = get(keyPath).map(decode[Data](_)) val v: Option[Data] = MergeOps.forPut(stored=keyData, received=data) v.map(v => batch.put(keyPath, encode(v))) } } - def doDelete(key: Key): String = { + def doDelete(key: Array[Byte]): String = { val b = hashing.findBucket(key) - val b_info = get(itob(b) ++ `:keys`).map(decode[BucketInfo](_)) - b_info match { - case Some(b_info) => - val vc = b_info.vc :+ local.toString - val keys = b_info.keys.filterNot(xs => Arrays.equals(xs, key)) - withBatch(batch => { - batch.delete((itob(b) ++ `:key:` ++ key)) - batch.put((itob(b) ++ `:keys`), encode(BucketInfo(vc, keys))) - }) - "ok" - case None => - "ok" + val b_info = get(encode[StoreKey](BucketInfoKey(bucket=b))).map(decode[BucketInfo](_)) + b_info.foreach{ b_info => + val vc = b_info.vc :+ local.toString + val keys = b_info.keys.filterNot(_ == key) + withBatch(batch => { + batch.delete(encode[StoreKey](DataKey(bucket=b, key=key))) + batch.put(encode[StoreKey](BucketInfoKey(bucket=b)), encode(BucketInfo(vc, keys))) + }) } + "ok" } def withBatch[R](body: WriteBatch => R): R = { val batch = new WriteBatch try { val r = body(batch) - leveldb.write(batch, wo) + db.write(wo, batch) r } finally { batch.close() } } } - -object WriteStore { - def props(leveldb: LevelDb): Props = Props(new WriteStore(leveldb)) -} - diff --git a/ring/src/ack.scala b/ring/src/ack.scala new file mode 100644 index 00000000..ab8af58f --- /dev/null +++ b/ring/src/ack.scala @@ -0,0 +1,6 @@ +package kvs.rng + +sealed trait Ack +case class AckSuccess(v: Option[Array[Byte]]) extends Ack +case class AckQuorumFailed(why: String) extends Ack +case class AckTimeoutFailed(op: String, k: Array[Byte]) extends Ack diff --git a/ring/src/cluster.scala b/ring/src/cluster.scala new file mode 100644 index 00000000..c2c0baf2 --- /dev/null +++ b/ring/src/cluster.scala @@ -0,0 +1,9 @@ +package org.apache.pekko + +import proto.* + +package object cluster { + implicit val vcodec: MessageCodec[(String,Long)] = caseCodecNums[(String,Long)]("_1"->1, "_2"->2) + implicit val vccodec: MessageCodec[org.apache.pekko.cluster.VectorClock] = caseCodecNums[org.apache.pekko.cluster.VectorClock]("versions"->1) + val emptyVC = VectorClock() +} diff --git a/ring/src/conf.scala b/ring/src/conf.scala new file mode 100644 index 00000000..744ea478 --- /dev/null +++ b/ring/src/conf.scala @@ -0,0 +1,64 @@ +package kvs.rng + +import scala.concurrent.*, duration.* +import scala.language.postfixOps + +case class Quorum + ( N: Int + , W: Int + , R: Int + ) + +case class Conf( + quorum: Quorum = Quorum(N=1, W=1, R=1) +, buckets: Int = 32768 /* 2^15 */ +, virtualNodes: Int = 128 +, hashLength: Int = 32 +, ringTimeout: FiniteDuration = 11 seconds /* bigger than gatherTimeout */ +, gatherTimeout: FiniteDuration = 10 seconds +, dumpTimeout: FiniteDuration = 1 hour +, replTimeout: FiniteDuration = 1 minute +, dir: String = "data_rng" +) + +def pekkoConf(name: String, host: String, port: Int): String = s""" + pekko { + actor { + provider = cluster + deployment { + /ring_readonly_store { + router = round-robin-pool + nr-of-instances = 5 + } + } + debug { + receive = off + lifecycle = off + } + serializers { + kvsproto = kvs.rng.Serializer + } + serialization-identifiers { + "kvs.rng.Serializer" = 50 + } + serialization-bindings { + "kvs.rng.model.ChangeState" = kvsproto + "kvs.rng.model.StoreGetAck" = kvsproto + "kvs.rng.model.StoreDelete" = kvsproto + "kvs.rng.model.StoreGet" = kvsproto + "kvs.rng.model.StorePut" = kvsproto + "kvs.rng.model.DumpBucketData" = kvsproto + "kvs.rng.model.DumpGetBucketData" = kvsproto + "kvs.rng.model.ReplBucketPut" = kvsproto + "kvs.rng.model.ReplBucketUpToDate" = kvsproto + "kvs.rng.model.ReplGetBucketIfNew" = kvsproto + "kvs.rng.model.ReplNewerBucketData" = kvsproto + } + } + remote.artery.canonical { + hostname = $host + port = $port + } + cluster.seed-nodes = [ "pekko://$name@$host:$port" ] + } + """ diff --git a/ring/src/dba.scala b/ring/src/dba.scala new file mode 100644 index 00000000..00a6a481 --- /dev/null +++ b/ring/src/dba.scala @@ -0,0 +1,92 @@ +package kvs.rng + +import org.apache.pekko.actor.{Actor, ActorLogging, Props, Deploy} +import org.apache.pekko.event.Logging +import org.apache.pekko.routing.FromConfig +import kvs.rng.store.{ReadonlyStore, WriteStore} +import org.rocksdb.{util as _, *} +import proto.* +import scala.language.postfixOps +import zio.* + +/* Database API */ +trait Dba: + def put(key: Key, value: Value): IO[DbaErr, Unit] + def get(key: Key): IO[DbaErr, Option[Value]] + def delete(key: Key): IO[DbaErr, Unit] +end Dba + +type DbaErr = AckQuorumFailed | AckTimeoutFailed + +object Dba: + val live: ZLayer[ActorSystem & Conf, Throwable, Dba] = + ZLayer.scoped( + for + as <- ZIO.service[ActorSystem] + conf <- ZIO.service[Conf] + _ <- ZIO.attempt(RocksDB.loadLibrary()) + opts <- + ZIO.fromAutoCloseable( + ZIO.attempt{ + Options().nn + .setCreateIfMissing(true).nn + .setCompressionType(CompressionType.LZ4_COMPRESSION).nn + } + ) + db <- + ZIO.fromAutoCloseable( + ZIO.attempt( + RocksDB.open(opts, conf.dir).nn + ) + ) + _ <- ZIO.attempt(as.eventStream) + dba <- + ZIO.attempt( + new Dba: + val hashing = Hashing(conf) + as.actorOf(WriteStore.props(db, hashing).withDeploy(Deploy.local), name="ring_write_store") + as.actorOf(FromConfig.props(ReadonlyStore.props(db, hashing)).withDeploy(Deploy.local), name="ring_readonly_store") + + val hash = as.actorOf(Hash.props(conf, hashing).withDeploy(Deploy.local), name="ring_hash") + + def put(key: Key, value: Value): IO[DbaErr, Unit] = + withRetryOnce(Put(key, value)).unit + + def get(key: Key): IO[DbaErr, Option[Value]] = + withRetryOnce(Get(key)) + + def delete(key: Key): IO[DbaErr, Unit] = + withRetryOnce(Delete(key)).unit + + private def withRetryOnce[A](v: => A): IO[DbaErr, Option[Array[Byte]]] = + ZIO.async{ + (callback: IO[DbaErr, Option[Array[Byte]]] => Unit) => + val receiver = as.actorOf(AckReceiver.props{ + case Right(a) => callback(ZIO.succeed(a)) + case Left(e) => callback(ZIO.fail(e)) + }) + hash.tell(v, receiver) + }.retry(Schedule.fromDuration(100 milliseconds)) + ) + yield dba + ) +end Dba + +type AckReceiverCallback = Either[DbaErr, Option[Value]] => Unit + +object AckReceiver: + def props(cb: AckReceiverCallback): Props = Props(AckReceiver(cb)) + +class AckReceiver(cb: AckReceiverCallback) extends Actor with ActorLogging: + def receive: Receive = + case x: Ack => + val res = x match + case AckSuccess(v) => Right(v) + case x: AckQuorumFailed => Left(x) + case x: AckTimeoutFailed => Left(x) + cb(res) + context.stop(self) + case x => + log.error(x.toString) + context.stop(self) +end AckReceiver diff --git a/ring/src/merge.scala b/ring/src/merge.scala new file mode 100644 index 00000000..20d70d44 --- /dev/null +++ b/ring/src/merge.scala @@ -0,0 +1,123 @@ +package kvs.rng + +import scala.annotation.tailrec +import scala.collection.immutable.{HashMap, HashSet} + +import data.Data, GatherGet.AddrOfData, model.KeyBucketData + +private final class Bytes private (a: Array[Byte]) { + lazy val length: Int = a.length + lazy val isEmpty: Boolean = a.isEmpty + lazy val nonEmpty: Boolean = a.nonEmpty + lazy val mkString: String = new String(a, "utf8") + val unsafeArray: Array[Byte] = a + override def equals(other: Any): Boolean = { + if (!other.isInstanceOf[Bytes]) false + else java.util.Arrays.equals(a, other.asInstanceOf[Bytes].unsafeArray) + } + override def hashCode(): Int = java.util.Arrays.hashCode(a) +} + +private object Bytes { + val empty = new Bytes(Array.emptyByteArray) + def unsafeWrap(a: Array[Byte]): Bytes = new Bytes(a) + def apply(bs: Byte*): Bytes = new Bytes(bs.toArray) +} + +object MergeOps { + def forDump(xs: Vector[KeyBucketData]): Vector[KeyBucketData] = { + @tailrec def loop(xs: Vector[KeyBucketData], acc: HashMap[Bytes, KeyBucketData]): Vector[KeyBucketData] = { + xs match + case Vector() => acc.values.toVector + case Vector(received, s*) => + val t = s.toVector + val k = Bytes.unsafeWrap(received.key) + acc.get(k) match + case None => + loop(t, acc + (k -> received)) + case Some(stored) => + (stored.data < received.data) match + case OkLess(true) => loop(t, acc + (k -> received)) + case OkLess(false) => loop(t, acc) + case ConflictLess(true, vc) => loop(t, acc + (k -> received.copy(data=received.data.copy(vc=vc)))) + case ConflictLess(false, vc) => loop(t, acc + (k -> stored.copy(data=stored.data.copy(vc=vc)))) + } + loop(xs, acc=HashMap.empty) + } + + def forRepl(xs: Vector[KeyBucketData]): Vector[KeyBucketData] = + @tailrec + def loop(xs: Vector[KeyBucketData], acc: HashMap[Bytes, KeyBucketData]): Vector[KeyBucketData] = + xs match + case Vector() => acc.values.toVector + case Vector(received, s*) => + val t = s.toVector + val k = Bytes.unsafeWrap(received.key) + acc.get(k) match + case None => + loop(t, acc + (k -> received)) + case Some(stored) => + (stored.data < received.data) match + case OkLess(true) => loop(t, acc + (k -> received)) + case OkLess(false) => loop(t, acc) + case ConflictLess(true, vc) => loop(t, acc + (k -> received.copy(data=received.data.copy(vc=vc)))) + case ConflictLess(false, vc) => loop(t, acc + (k -> stored.copy(data=stored.data.copy(vc=vc)))) + loop(xs, acc=HashMap.empty) + + /* returns (actual data, list of outdated nodes) */ + def forGatherGet(xs: Vector[AddrOfData]): (Option[Data], HashSet[Node]) = + @tailrec + def loop(xs: Vector[Option[Data]], newest: Option[Data]): Option[Data] = + xs match + case Vector() => newest + case Vector(x, s*) => + x match + case None => loop(s.toVector, newest) + case r@Some(received) => + val t = s.toVector + newest match + case None => loop(t, r) + case s@Some(saved) => + saved < received match + case OkLess(true) => loop(t, r) + case OkLess(false) => loop(t, s) + case ConflictLess(true, _) => loop(t, r) + case ConflictLess(false, _) => loop(t, s) + xs match + case Vector() => None -> HashSet.empty + case Vector(h, t*) => + val correct = loop(t.view.map(_._1).toVector, h._1) + def makevc1(x: Option[Data]): VectorClock = x.map(_.vc).getOrElse(emptyVC) + val correct_vc = makevc1(correct) + correct -> xs.view.filterNot(x => makevc1(x._1) == correct_vc).map(_._2).to(HashSet) + + def forPut(stored: Option[Data], received: Data): Option[Data] = + stored match + case None => + Some(received) + case Some(stored) => + (stored < received) match + case OkLess(true) => Some(received) + case OkLess(false) => None + case ConflictLess(true, vc) => Some(received.copy(vc=vc)) + case ConflictLess(false, vc) => Some(stored.copy(vc=vc)) + + sealed trait LessComp + case class OkLess(res: Boolean) extends LessComp + case class ConflictLess(res: Boolean, vc: VectorClock) extends LessComp + + implicit class DataExt(x: Data) { + def <(o: Data): LessComp = { + val xvc = x.vc + val ovc = o.vc + if (xvc < ovc) OkLess(true) + else if (xvc == ovc) OkLess(x.lastModified < o.lastModified) + else if (xvc > ovc) OkLess(false) + else { // xvc <> ovc + val mergedvc = xvc merge ovc + if (x.lastModified < o.lastModified) ConflictLess(true, mergedvc) + else ConflictLess(false, mergedvc) + } + } + } +} diff --git a/ring/src/package.scala b/ring/src/package.scala new file mode 100644 index 00000000..064e7b58 --- /dev/null +++ b/ring/src/package.scala @@ -0,0 +1,25 @@ +package kvs.rng + +import org.apache.pekko.actor.{Address, ActorRef} +import proto.* + +type Bucket = Int +type VNode = Int +type Node = Address +type Key = Array[Byte] +type Value = Array[Byte] +type VectorClock = org.apache.pekko.cluster.VectorClock +type Age = (VectorClock, Long) +type PreferenceList = Set[Node] + +val emptyVC = org.apache.pekko.cluster.emptyVC + +extension (value: String) + def blue: String = s"\u001B[34m${value}\u001B[0m" + def green: String = s"\u001B[32m${value}\u001B[0m" + +def now_ms(): Long = System.currentTimeMillis + +def addr(s: ActorRef): Node = s.path.address + +given MessageCodec[PortVNode] = caseCodecAuto diff --git a/kvs/src/main/scala/pickle.scala b/ring/src/pickle.scala similarity index 53% rename from kvs/src/main/scala/pickle.scala rename to ring/src/pickle.scala index 64d204f8..63b6e46a 100644 --- a/kvs/src/main/scala/pickle.scala +++ b/ring/src/pickle.scala @@ -1,39 +1,35 @@ -package zd.kvs +package kvs.rng -import akka.actor.{ExtendedActorSystem} -import akka.serialization.{BaseSerializer} -import akka.cluster.given -import zd.rng.model.* +import org.apache.pekko.actor.{ExtendedActorSystem} +import org.apache.pekko.serialization.{BaseSerializer} import proto.* -class Serializer(val system: ExtendedActorSystem) extends BaseSerializer: +import kvs.rng.model.*, kvs.rng.data.codec.* +import org.apache.pekko.cluster.given + +class Serializer(val system: ExtendedActorSystem) extends BaseSerializer { implicit val msgCodec: MessageCodec[Msg] = { implicit def tuple2IntACodec[A:MessageCodec]: MessageCodec[Tuple2[Int, A]] = caseCodecIdx[Tuple2[Int, A]] - implicit val replBucketUpToDateCodec: MessageCodec[ReplBucketUpToDate.type] = caseCodecAuto[ReplBucketUpToDate.type] + implicit val KeyBucketDataC: MessageCodec[KeyBucketData] = caseCodecAuto + implicit val replBucketUpToDateCodec: MessageCodec[ReplBucketUpToDate.type] = caseCodecAuto implicit val quorumStateCodec: MessageCodec[QuorumState] = { import QuorumState.* - implicit val quorumStateUnsatisfiedCodec: MessageCodec[QuorumStateUnsatisfied.type] = caseCodecAuto[QuorumStateUnsatisfied.type] - implicit val quorumStateReadonlyCodec: MessageCodec[QuorumStateReadonly.type] = caseCodecAuto[QuorumStateReadonly.type] - implicit val quorumStateEffectiveCodec: MessageCodec[QuorumStateEffective.type] = caseCodecAuto[QuorumStateEffective.type] + implicit val quorumStateUnsatisfiedCodec: MessageCodec[QuorumStateUnsatisfied.type] = caseCodecAuto + implicit val quorumStateReadonlyCodec: MessageCodec[QuorumStateReadonly.type] = caseCodecAuto + implicit val quorumStateEffectiveCodec: MessageCodec[QuorumStateEffective.type] = caseCodecAuto sealedTraitCodecAuto[QuorumState] } - implicit val changeStateCodec: MessageCodec[ChangeState] = caseCodecAuto[ChangeState] - implicit val dataCodec: MessageCodec[zd.rng.data.Data] = zd.rng.data.codec.dataCodec - implicit val dumpBucketDataCodec: MessageCodec[DumpBucketData] = caseCodecAuto[DumpBucketData] - implicit val dumpEnCodec: MessageCodec[DumpEn] = classCodecAuto[DumpEn] - implicit val dumpGetCodec: MessageCodec[DumpGet] = classCodecAuto[DumpGet] - implicit val dumpGetBucketDataCodec: MessageCodec[DumpGetBucketData] = caseCodecAuto[DumpGetBucketData] - implicit val vcodec: MessageCodec[(String, Long)] = caseCodecNums[Tuple2[String,Long]]("_1"->1,"_2"->2) - implicit val vccodec: MessageCodec[akka.cluster.VectorClock] = caseCodecNums[akka.cluster.VectorClock]("versions"->1) - implicit val replBucketPutCodec: MessageCodec[ReplBucketPut] = caseCodecAuto[ReplBucketPut] - implicit val replGetBucketIfNewCodec: MessageCodec[ReplGetBucketIfNew] = caseCodecAuto[ReplGetBucketIfNew] - implicit val replNewerBucketDataCodec: MessageCodec[ReplNewerBucketData] = caseCodecAuto[ReplNewerBucketData] - implicit val replBucketsVcCodec: MessageCodec[ReplBucketsVc] = caseCodecAuto[ReplBucketsVc] - implicit val storeDeleteCodec: MessageCodec[StoreDelete] = classCodecAuto[StoreDelete] - implicit val storeGetCodec: MessageCodec[StoreGet] = classCodecAuto[StoreGet] - implicit val storeGetAckCodec: MessageCodec[StoreGetAck] = caseCodecAuto[StoreGetAck] - implicit val storePutCodec: MessageCodec[StorePut] = caseCodecAuto[StorePut] - implicit val replGetBucketsVcCodec: MessageCodec[ReplGetBucketsVc] = caseCodecAuto[ReplGetBucketsVc] + implicit val changeStateCodec: MessageCodec[ChangeState] = caseCodecAuto + implicit val dumpBucketDataCodec: MessageCodec[DumpBucketData] = caseCodecAuto + implicit val dumpGetBucketDataCodec: MessageCodec[DumpGetBucketData] = caseCodecAuto + implicit val replBucketPutCodec: MessageCodec[ReplBucketPut] = caseCodecAuto + implicit val replGetBucketIfNewCodec: MessageCodec[ReplGetBucketIfNew] = caseCodecAuto + implicit val replNewerBucketDataCodec: MessageCodec[ReplNewerBucketData] = caseCodecAuto + implicit val replBucketsVcCodec: MessageCodec[ReplBucketsVc] = caseCodecAuto + implicit val storeDeleteCodec: MessageCodec[StoreDelete] = caseCodecAuto + implicit val storeGetCodec: MessageCodec[StoreGet] = caseCodecAuto + implicit val storeGetAckCodec: MessageCodec[StoreGetAck] = caseCodecAuto + implicit val storePutCodec: MessageCodec[StorePut] = caseCodecAuto sealedTraitCodecAuto[Msg] } @@ -46,6 +42,7 @@ class Serializer(val system: ExtendedActorSystem) extends BaseSerializer: override val includeManifest: Boolean = false - override def fromBinary(data: Array[Byte], manifest: Option[Class[?]]): AnyRef = { + override def fromBinary(data: Array[Byte], manifest: Option[Class[_]]): AnyRef = { decode[Msg](data) } +} diff --git a/kvs/src/main/scala/rng/state.scala b/ring/src/state.scala similarity index 89% rename from kvs/src/main/scala/rng/state.scala rename to ring/src/state.scala index db1c2602..08562b30 100644 --- a/kvs/src/main/scala/rng/state.scala +++ b/ring/src/state.scala @@ -1,4 +1,4 @@ -package zd.rng +package kvs.rng sealed trait FsmState case object ReadyCollect extends FsmState diff --git a/ring/src/system.scala b/ring/src/system.scala new file mode 100644 index 00000000..2c24f843 --- /dev/null +++ b/ring/src/system.scala @@ -0,0 +1,24 @@ +package kvs.rng + +import com.typesafe.config.{ConfigFactory, Config} +import zio.* + +type ActorSystem = org.apache.pekko.actor.ActorSystem + +object ActorSystem: + case class Conf(name: String, config: Config) + + def staticConf(name: String, cfg: String): ULayer[Conf] = + ZLayer(ZIO.succeed(ConfigFactory.parseString(cfg).nn).map(Conf(name, _))) + + val live: RLayer[Conf, ActorSystem] = + ZLayer.scoped( + ZIO.acquireRelease( + for + conf <- ZIO.service[Conf] + system <- ZIO.attempt(org.apache.pekko.actor.ActorSystem(conf.name, conf.config)) + yield system + )( + system => ZIO.fromFuture(_ => system.terminate()).either + ) + ) diff --git a/search/src/dba.scala b/search/src/dba.scala new file mode 100644 index 00000000..54180a49 --- /dev/null +++ b/search/src/dba.scala @@ -0,0 +1,21 @@ +package kvs.search + +import kvs.rng.{Dba, DbaErr} +import scala.util.Try +import zio.* + +class DbaEff(dba: Dba): + type K = String + type V = Array[Byte] + type Err = DbaErr | Throwable + type R[A] = Either[Err, A] + + def put(key: K, value: V): R[Unit] = run(dba.put(stob(key), value)) + def get(key: K): R[Option[V]] = run(dba.get(stob(key))) + def delete(key: K): R[Unit] = run(dba.delete(stob(key))) + + private def run[A](eff: IO[Err, A]): R[A] = + Unsafe.unsafely(Runtime.default.unsafe.run(eff.either).toEither).flatten + + private inline def stob(s: String): Array[Byte] = + s.getBytes("utf8").nn diff --git a/search/src/dir.scala b/search/src/dir.scala new file mode 100644 index 00000000..e9ea9ba7 --- /dev/null +++ b/search/src/dir.scala @@ -0,0 +1,243 @@ +package kvs.search + +import java.io.{IOException, ByteArrayOutputStream} +import java.nio.file.{NoSuchFileException, FileAlreadyExistsException} +import java.util.concurrent.atomic.AtomicLong +import java.util.{Collection, Collections, Arrays, Set} +import org.apache.lucene.store.* +import scala.annotation.tailrec +import scala.collection.concurrent.TrieMap +import scala.collection.JavaConverters.* +import zio.* +import kvs.rng.Dba + +object KvsDirectory: + type DirName = String + + val live: ZLayer[Dba & DirName, Nothing, KvsDirectory] = + ZLayer( + for + dba <- ZIO.service[Dba] + dirname <- ZIO.service[DirName] + yield + KvsDirectory(dirname)(using DbaEff(dba)) + ) + +class KvsDirectory(val dir: String)(using dba: DbaEff) extends BaseDirectory(NoLockFactory.INSTANCE): + private val outs = TrieMap.empty[String, ByteArrayOutputStream] + private val nextTempFileCounter = AtomicLong() + + /** + * Returns names of all files stored in this directory. + * The output must be in sorted (UTF-16, java's {@link String#compareTo}) order. + * + * @throws IOException in case of I/O error + */ + override + def listAll(): Array[String | Null] | Null = { + ensureOpen() + Files.all(dir).fold(l => throw IOException(l.toString), identity) + } + + /** + * Removes an existing file in the directory. + * + * This method must throw {@link NoSuchFileException} + * if {@code name} points to a non-existing file. + * + * @param name the name of an existing file. + * @throws IOException in case of I/O error + */ + override + def deleteFile(name: String | Null): Unit = { + sync(Collections.singletonList(name.nn).nn) + val r: Either[FileNotExists.type | dba.Err | BrokenListing.type, Unit] = for { + _ <- File.delete(dir, name.nn) + _ <- Files.remove(dir, name.nn) + } yield () + r.fold( + _ match + case FileNotExists => throw NoSuchFileException(name) + case BrokenListing => throw NoSuchFileException(name) + case x => throw IOException(x.toString) + , identity + ) + } + + /** + * Returns the byte length of a file in the directory. + * + * This method must throw {@link NoSuchFileException} + * if {@code name} points to a non-existing file. + * + * @param name the name of an existing file. + * @throws IOException in case of I/O error + */ + override + def fileLength(name: String | Null): Long = { + ensureOpen() + sync(Collections.singletonList(name.nn).nn) + File.size(dir, name.nn).fold( + l => l match { + case FileNotExists => throw NoSuchFileException(name) + case _ => throw IOException(l.toString) + }, + r => r + ) + } + + /** + * Creates a new, empty file in the directory and returns an {@link IndexOutput} + * instance for appending data to this file. + * + * This method must throw {@link FileAlreadyExistsException} if the file + * already exists. + * + * @param name the name of the file to create. + * @throws IOException in case of I/O error + */ + override + def createOutput(name: String | Null, context: IOContext | Null): IndexOutput | Null = { + ensureOpen() + val r: Either[dba.Err | FileExists.type, Unit] = + for + _ <- Files.add(dir, name.nn) + _ <- File.create(dir, name.nn) + yield () + r.fold( + l => l match + case FileExists => throw FileAlreadyExistsException(name) + case _ => throw IOException(l.toString) + , _ => { + val out = ByteArrayOutputStream() + outs += ((name.nn, out)) + OutputStreamIndexOutput(name, name.nn, out, 8192) + } + ) + } + + /** + * Creates a new, empty, temporary file in the directory and returns an {@link IndexOutput} + * instance for appending data to this file. + * + * The temporary file name (accessible via {@link IndexOutput#getName()}) will start with + * {@code prefix}, end with {@code suffix} and have a reserved file extension {@code .tmp}. + */ + override + def createTempOutput(prefix: String | Null, suffix: String | Null, context: IOContext | Null): IndexOutput | Null = { + ensureOpen() + @tailrec + def loop(): Either[?, File] = + val name = Directory.getTempFileName(prefix, suffix, nextTempFileCounter.getAndIncrement).nn + val res: Either[dba.Err | FileExists.type, File] = + for + _ <- Files.add(dir, name) + r <- File.create(dir, name) + yield r + res match + case Left(FileExists) => loop() + case x => x + + val res = loop() + res.fold( + l => throw IOException(l.toString), + r => { + val out = ByteArrayOutputStream() + outs += ((r.name, out)) + OutputStreamIndexOutput(r.name, r.name, out, 8192) + } + ) + } + + /** + * Ensures that any writes to these files are moved to + * stable storage (made durable). + * + * Lucene uses this to properly commit changes to the index, to prevent a machine/OS crash + * from corrupting the index. + */ + override + def sync(names: Collection[String] | Null): Unit = { + ensureOpen() + names.nn.asScala.foreach{ (name: String) => + outs.get(name).map(_.toByteArray.nn).foreach{ xs => + File.append(dir, name, xs).fold( + l => throw IOException(l.toString), + _ => () + ) + outs -= name + } + } + } + + override + def syncMetaData(): Unit = + ensureOpen() + + /** + * Renames {@code source} file to {@code dest} file where + * {@code dest} must not already exist in the directory. + * + * It is permitted for this operation to not be truly atomic, for example + * both {@code source} and {@code dest} can be visible temporarily in {@link #listAll()}. + * However, the implementation of this method must ensure the content of + * {@code dest} appears as the entire {@code source} atomically. So once + * {@code dest} is visible for readers, the entire content of previous {@code source} + * is visible. + * + * This method is used by IndexWriter to publish commits. + */ + override + def rename(source: String | Null, dest: String | Null): Unit = { + ensureOpen() + sync(Arrays.asList(source, dest).nn) + val res = + for + _ <- File.copy(dir, source.nn -> dest.nn) + _ <- Files.add(dir, dest.nn) + _ <- File.delete(dir, source.nn) + _ <- Files.remove(dir, source.nn) + yield () + res.fold( + l => throw IOException(l.toString), + _ => () + ) + } + + /** + * Opens a stream for reading an existing file. + * + * This method must throw {@link NoSuchFileException} + * if {@code name} points to a non-existing file. + * + * @param name the name of an existing file. + * @throws IOException in case of I/O error + */ + override + def openInput(name: String | Null, context: IOContext | Null): IndexInput | Null = { + sync(Collections.singletonList(name.nn).nn) + val res = + for + bs <- File.stream(dir, name.nn) + yield ByteBuffersIndexInput(ByteBuffersDataInput(bs), name) + res.fold( + l => l match + case BrokenFile => throw NoSuchFileException(name) + case _ => throw IOException(l.toString) + , identity + ) + } + + override def close(): Unit = + synchronized { + isOpen = false + } + + override + def getPendingDeletions(): Set[String] = + Collections.emptySet[String].nn + + given [A]: CanEqual[BrokenListing.type, BrokenListing.type | A] = CanEqual.derived + given [A]: CanEqual[BrokenFile.type, BrokenFile.type | A] = CanEqual.derived + given [A]: CanEqual[FileNotExists.type, FileNotExists.type | A] = CanEqual.derived + given [A]: CanEqual[FileExists.type, FileExists.type | A] = CanEqual.derived \ No newline at end of file diff --git a/search/src/err.scala b/search/src/err.scala new file mode 100644 index 00000000..4480f3e1 --- /dev/null +++ b/search/src/err.scala @@ -0,0 +1,6 @@ +package kvs.search + +case object BrokenListing +case object BrokenFile +case object FileNotExists +case object FileExists diff --git a/search/src/file.scala b/search/src/file.scala new file mode 100644 index 00000000..2f5f60f7 --- /dev/null +++ b/search/src/file.scala @@ -0,0 +1,137 @@ +package kvs.search + +import proto.* +import scala.annotation.tailrec +import scala.util.{Try, Success, Failure} +import java.nio.ByteBuffer +import java.util.ArrayList + +case class File + ( @N(1) name: String // name – unique value inside directory + , @N(2) count: Int // count – number of chunks + , @N(3) size: Long // size - size of file in bytes + ) + +object File: + case object NoData + + private val chunkLength: Int = 10_000_000 // 10 MB + + given MessageCodec[File] = caseCodecAuto + + private inline def pickle(e: File): Array[Byte] = encode(e) + + private def unpickle(a: Array[Byte]): Either[Throwable, File] = + Try(decode[File](a)) match + case Success(x) => Right(x) + case Failure(x) => Left(x) + + private def get(dir: String, name: String)(using dba: DbaEff): Either[FileNotExists.type | dba.Err, File] = + given CanEqual[None.type, Option[dba.V]] = CanEqual.derived + val path = s"/search/file/$dir/$name" + dba.get(path) match + case Right(Some(x)) => unpickle(x) + case Right(None) => Left(FileNotExists) + case Left(e) => Left(e) + + def create(dir: String, name: String)(using dba: DbaEff): Either[dba.Err | FileExists.type, File] = + given CanEqual[None.type, Option[dba.V]] = CanEqual.derived + dba.get(s"/search/file/$dir/${name}") match + case Right(Some(_)) => Left(FileExists) + case Right(None) => + val f = File(name, count=0, size=0L) + val x = pickle(f) + for + _ <- dba.put(s"/search/file/$dir/${name}", x) + yield f + case Left(e) => Left(e) + + def append(dir: String, name: String, data: Array[Byte])(using dba: DbaEff): Either[dba.Err | NoData.type | FileNotExists.type, File] = + val length = data.length + @tailrec + def writeChunks(count: Int, rem: Array[Byte]): Either[dba.Err, Int] = + rem.splitAt(chunkLength) match + case (xs, _) if xs.length == 0 => Right(count) + case (xs, ys) => + dba.put(s"/search/file/$dir/${name}_chunk_${count+1}", xs) match + case Right(_) => writeChunks(count+1, rem=ys) + case Left(e) => Left(e) + for + _ <- (if length == 0 then Left(NoData) else Right(())): Either[dba.Err | NoData.type, Unit] + file <- get(dir, name): Either[dba.Err | NoData.type | FileNotExists.type, File] + count <- writeChunks(file.count, rem=data) + file1 = file.copy(count=count, size=file.size+length) + file2 = pickle(file1) + _ <- dba.put(s"/search/file/$dir/${name}", file2) + yield file1 + + def size(dir: String, name: String)(using dba: DbaEff): Either[dba.Err | FileNotExists.type, Long] = + get(dir, name).map(_.size) + + def stream(dir: String, name: String)(using dba: DbaEff): Either[dba.Err | BrokenFile.type | FileNotExists.type, ArrayList[ByteBuffer]] = + for + file <- (get(dir, name): Either[dba.Err | FileNotExists.type, File]) + xs <- + (file.count match + case 0 => Right(ArrayList(0)) + case n => + inline def k(i: Int) = s"/search/file/$dir/${name}_chunk_${i}" + @tailrec + def loop(i: Int, acc: ArrayList[ByteBuffer]): Either[dba.Err | BrokenFile.type, ArrayList[ByteBuffer]] = + if i <= n then + (dba.get(k(i)).flatMap(_.fold(Left(BrokenFile))(x => Right(ByteBuffer.wrap(x).nn))): Either[dba.Err | BrokenFile.type, ByteBuffer]) match + case Right(x) => + acc.add(x) + loop(i + 1, acc) + case Left(e) => Left(e) + else + Right(acc) + loop(1, ArrayList[ByteBuffer](n))): Either[dba.Err | BrokenFile.type, ArrayList[ByteBuffer]] + yield xs + + def delete(dir: String, name: String)(using dba: DbaEff): Either[FileNotExists.type | dba.Err, File] = + for + file <- get(dir, name) + _ <- LazyList.range(1, file.count+1).map(i => dba.delete(s"/search/file/$dir/${name}_chunk_${i}")).sequence_ + _ <- dba.delete(s"/search/file/$dir/${name}") + yield file + + def copy(dir: String, name: (String, String))(using dba: DbaEff): Either[dba.Err | FileNotExists.type | BrokenFile.type | FileExists.type, File] = + val (fromName, toName) = name + for + from <- get(dir, fromName): Either[dba.Err | FileNotExists.type | BrokenFile.type, File] + _ <- + get(dir, toName).fold( + l => l match { + case FileNotExists => Right(()) + case _ => Left(l) + }, + _ => Left(FileExists) + ): Either[dba.Err | FileExists.type | BrokenFile.type | FileNotExists.type, Unit] + _ <- + (LazyList.range(1, from.count+1).map(i => + (for + x <- ({ + val k = s"/search/file/$dir/${fromName}_chunk_${i}" + dba.get(k).flatMap(_.fold(Left(BrokenFile))(Right(_))) + }: Either[dba.Err | FileNotExists.type | BrokenFile.type | FileExists.type, dba.V]) + _ <- dba.put(s"/search/file/$dir/${toName}_chunk_${i}", x) + yield ()): Either[dba.Err | FileNotExists.type | BrokenFile.type | FileExists.type, Unit] + ).sequence_ : Either[dba.Err | FileNotExists.type | BrokenFile.type | FileExists.type, Unit]) + to = File(toName, from.count, from.size) + x = pickle(to) + _ <- dba.put(s"/search/file/$dir/${toName}", x) + yield to + + extension [A, B](xs: Seq[Either[A, B]]) + @tailrec + private def _sequence_(ys: Seq[Either[A, B]]): Either[A, Unit] = + ys.headOption match + case None => Right(()) + case Some(Left(e)) => Left(e) + case Some(Right(z)) => _sequence_(ys.tail) + def sequence_ : Either[A, Unit] = _sequence_(xs) + + given [A]: CanEqual[FileNotExists.type, FileNotExists.type | A] = CanEqual.derived + +end File diff --git a/search/src/files.scala b/search/src/files.scala new file mode 100644 index 00000000..3e13256c --- /dev/null +++ b/search/src/files.scala @@ -0,0 +1,124 @@ +package kvs.search + +import proto.* +import scala.annotation.tailrec +import scala.collection.immutable.TreeSet + +case class Fd + ( @N(1) dirname: String + , @N(2) head: Option[String] + ) + +case class En + ( @N(1) filename: String + , @N(2) next: Option[String] + ) + +object Files: + given MessageCodec[Fd] = caseCodecAuto + given MessageCodec[En] = caseCodecAuto + + def put(dirname: String)(using dba: DbaEff): Either[dba.Err, Fd] = + put(Fd(dirname, head=None)) + + def put(fd: Fd)(using dba: DbaEff): Either[dba.Err, Fd] = + dba.put(fd.dirname, encode(fd)).map(_ => fd) + + def get(dirname: String)(using dba: DbaEff): Either[dba.Err, Option[Fd]] = + val fd = Fd(dirname, head=None) + dba.get(fd.dirname).map(_.map(decode)) + + def delete(dirname: String)(using dba: DbaEff): Either[dba.Err, Unit] = + dba.delete(dirname) + + private inline def key(dirname: String, filename: String): String = s"/search/files/$dirname/$filename" + + private def _put(dirname: String, en: En)(using dba: DbaEff): Either[dba.Err, En] = + dba.put(key(dirname=dirname, filename=en.filename), encode(en)).map(_ => en) + + def get(dirname: String, filename: String)(using dba: DbaEff): Either[dba.Err, Option[En]] = + dba.get(key(dirname=dirname, filename=filename)).map(_.map(decode)) + + private def getOrFail[E](dirname: String, filename: String, err: => E)(using dba: DbaEff): Either[dba.Err | E, En] = + given CanEqual[None.type, Option[dba.V]] = CanEqual.derived + val k = key(dirname=dirname, filename=filename) + dba.get(k).flatMap{ + case Some(x) => Right(decode(x)) + case None => Left(err) + } + + private def delete(dirname: String, filename: String)(using dba: DbaEff): Either[dba.Err, Unit] = + dba.delete(key(dirname=dirname, filename=filename)) + + /** + * Adds the entry to the container + * Creates the container if it's absent + */ + def add(dirname: String, filename: String)(using dba: DbaEff): Either[dba.Err | FileExists.type, En] = + get(dirname).flatMap(_.fold(put(dirname))(Right(_))).flatMap{ (fd: Fd) => + (get(dirname=dirname, filename=filename).flatMap( // id of entry must be unique + _.fold(Right(()))(_ => Left(FileExists)) + ): Either[dba.Err | FileExists.type, Unit]) + .map(_ => En(filename, next=fd.head)).flatMap{ en => + // add new entry with next pointer + _put(dirname, en).flatMap{ en => + // update feed's head + put(fd.copy(head=Some(filename))).map(_ => en) + } + } + } + + def all(dirname: String)(using dba: DbaEff): Either[dba.Err | BrokenListing.type, Array[String | Null]] = + @tailrec + def loop(id: Option[String], acc: TreeSet[String]): Either[dba.Err | BrokenListing.type, Array[String | Null]] = + id match + case None => Right(acc.toArray) + case Some(id) => + val en = getOrFail(dirname, id, BrokenListing) + en match + case Right(e) => loop(e.next, acc + e.filename) + case Left(e) => Left(e) + get(dirname).flatMap(_.fold(Right(Array.empty[String | Null]))(x => loop(x.head, TreeSet.empty))) + + def remove(dirname: String, filename: String)(using dba: DbaEff): Either[dba.Err | FileNotExists.type | BrokenListing.type, Unit] = + for + // get entry to delete + en <- getOrFail(dirname, filename, FileNotExists) + fdOpt <- get(dirname) + _ <- + fdOpt match + case None => + // tangling en + delete(dirname, filename) + case Some(fd) => + (fd.head match + case None => Right(()) + case Some(head) => + for + _ <- + if filename == head then + put(fd.copy(head=en.next)) + else + @tailrec + def loop(id: Option[String]): Either[dba.Err | BrokenListing.type, En] = + id match + case None => Left(BrokenListing) + case Some(id) => + val en = getOrFail(dirname, id, BrokenListing) + en match + case Right(e) if e.next == Some(filename) => Right(e) + case Right(e) => loop(e.next) + case Left(e) => Left(e) + (for + // find entry which points to this one (next) + next <- loop(Some(head)) + // change link + _ <- _put(dirname, next.copy(next=en.next)) + yield ()): Either[dba.Err | FileNotExists.type | BrokenListing.type, Unit] + _ <- delete(dirname, filename) + yield ()): Either[dba.Err | FileNotExists.type | BrokenListing.type, Unit] + yield () + + given CanEqual[None.type, Option[Fd]] = CanEqual.derived + +end Files diff --git a/search/src/search.scala b/search/src/search.scala new file mode 100644 index 00000000..79b0e2e1 --- /dev/null +++ b/search/src/search.scala @@ -0,0 +1,59 @@ +package kvs.search + +import java.util.Collections +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.document.Document +import org.apache.lucene.document.StoredField +import org.apache.lucene.index.DirectoryReader +import org.apache.lucene.index.IndexWriterConfig.OpenMode +import org.apache.lucene.index.{IndexWriterConfig, IndexWriter, Term} +import org.apache.lucene.search.{IndexSearcher, Query} +import org.apache.lucene.store.Directory +import proto.* +import scala.reflect.ClassTag +import zio.*, stream.* + +trait Search: + def run[A : Codec : ClassTag](q: Query, limit: Int=10): ZStream[Any, Throwable, A] + def index[R, E, A : Codec](xs: ZStream[R, E, A], `a->doc`: A => Document): ZIO[R, E | Throwable, Unit] +end Search + +case class SearchLive(dir: KvsDirectory) extends Search: + def run[A : Codec : ClassTag](q: Query, limit: Int): ZStream[Any, Throwable, A] = + for + reader <- ZStream.scoped(ZIO.fromAutoCloseable(ZIO.attempt(DirectoryReader.open(dir).nn))) + searcher <- ZStream.fromZIO(ZIO.attempt(IndexSearcher(reader))) + x <- ZStream.fromIterableZIO(ZIO.attempt(searcher.search(q, limit).nn.scoreDocs.nn)) + doc <- ZStream.fromZIO(ZIO.attempt(searcher.doc(x.nn.doc).nn)) + a <- + ZStream.fromZIO{ + for + bs <- ZIO.attempt(doc.getBinaryValue("obj").nn) + obj <- ZIO.attempt(decode[A](bs.bytes.nn)) + yield obj + } + yield a + + def index[R, E, A : Codec](xs: ZStream[R, E, A], `a->doc`: A => Document): ZIO[R, E | Throwable, Unit] = + ZIO.scoped( + for + a <- ZIO.fromAutoCloseable(ZIO.attempt(StandardAnalyzer())) + c <- ZIO.attempt(IndexWriterConfig(a)) + _ <- ZIO.attempt(c.setOpenMode(OpenMode.CREATE)) + w <- ZIO.fromAutoCloseable(ZIO.attempt(IndexWriter(dir, c))) + _ <- + xs.mapZIO(a => ZIO.attempt(w.addDocument{ + val doc = `a->doc`(a) + doc.add(StoredField("obj", encode[A](a))) + doc + }): ZIO[R, E | Throwable, Unit]).runDrain + yield () + ) +end SearchLive + +val layer: ZLayer[KvsDirectory, Nothing, Search] = + ZLayer { + for + dir <- ZIO.service[KvsDirectory] + yield SearchLive(dir) + } diff --git a/sharding/src/consistency.scala b/sharding/src/consistency.scala new file mode 100644 index 00000000..e7f8cb91 --- /dev/null +++ b/sharding/src/consistency.scala @@ -0,0 +1,30 @@ +package kvs.sharding + +import org.apache.pekko.actor.{Actor, Props} +import kvs.rng.DbaErr +import zio.* + +trait SeqConsistency: + def send(msg: Any): IO[DbaErr, Any] + +object SeqConsistency: + case class Config(name: String, f: Any => IO[Any, Any], id: Any => String) + + val live: ZLayer[ClusterSharding & Config, Nothing, SeqConsistency] = + ZLayer( + for + sharding <- ZIO.service[ClusterSharding] + cfg <- ZIO.service[Config] + shards <- + sharding.start( + cfg.name + , Props(new Actor: + def receive: Receive = + a => sender() ! Unsafe.unsafely(Runtime.default.unsafe.run(cfg.f(a))) + ) + , cfg.id) + yield + new SeqConsistency: + def send(msg: Any): IO[DbaErr, Any] = + sharding.send(shards, msg) + ) diff --git a/sharding/src/sharding.scala b/sharding/src/sharding.scala new file mode 100644 index 00000000..008d7244 --- /dev/null +++ b/sharding/src/sharding.scala @@ -0,0 +1,53 @@ +package kvs.sharding + +import org.apache.pekko.actor.{Actor, ActorRef, Props} +import org.apache.pekko.cluster.sharding.{ClusterSharding as PekkoClusterSharding, ClusterShardingSettings, ShardRegion} +import kvs.rng.ActorSystem +import zio.* + +trait ClusterSharding: + def start[A](name: String, props: Props, id: A => String): UIO[ActorRef] + def send[A, E](shardRegion: ActorRef, msg: Any): IO[E, A] + +val live: URLayer[ActorSystem, ClusterSharding] = + ZLayer( + for + system <- ZIO.service[ActorSystem] + sharding <- ZIO.succeed(PekkoClusterSharding(system)) + yield + new ClusterSharding: + def start[A](name: String, props: Props, id: A => String): UIO[ActorRef] = + ZIO.succeed( + sharding.start( + typeName = name, + entityProps = props, + settings = ClusterShardingSettings(system), + extractEntityId = { + case msg: A => (id(msg), msg) + }: ShardRegion.ExtractEntityId, + extractShardId = { + case msg => (math.abs(id(msg.asInstanceOf[A]).hashCode) % 100).toString + }: ShardRegion.ExtractShardId, + ) + ) + + def send[A, E](shardRegion: ActorRef, msg: Any): IO[E, A] = + ZIO.asyncZIO{ (callback: IO[E, A] => Unit) => + for + receiver <- ZIO.succeed(system.actorOf(Props(Receiver[A, E]{ + case Exit.Success(a) => callback(ZIO.succeed(a)) + case Exit.Failure(e) => callback(ZIO.failCause(e)) + }))) + _ <- ZIO.succeed(shardRegion.tell(msg, receiver)) + yield () + } + ) + +class Receiver[A, E](handler: Exit[E, A] => Unit) extends Actor: + def receive: Receive = + case r: Exit[E, A] => + handler(r) + context.stop(self) + case x => + println(x.toString) + context.stop(self) diff --git a/sort/src/sort.scala b/sort/src/sort.scala new file mode 100644 index 00000000..2371a71d --- /dev/null +++ b/sort/src/sort.scala @@ -0,0 +1,239 @@ +package kvs.sort + +import com.google.protobuf.{CodedOutputStream, CodedInputStream} +import java.io.IOException +import kvs.rng.{Dba, DbaErr} +import proto.* +import scala.math.Ordering +import scala.math.Ordering.Implicits.infixOrderingOps +import zio.*, stream.* + +trait Sort: + def insert[A: Codec: Ordering](ns: String, x: A)(using CanEqual[A, A]): IO[Err, Unit] + def remove[A: Codec: Ordering](ns: String, x: A)(using CanEqual[A, A]): IO[Err, Unit] + def flatten[A: Codec](ns: String): Stream[Err, A] +end Sort + +type Err = DbaErr | IOException + +def insert[A: Codec: Ordering](ns: String, x: A)(using CanEqual[A, A]): ZIO[Sort, Err, Unit] = + ZIO.serviceWithZIO(_.insert(ns, x)) + +def remove[A: Codec: Ordering](ns: String, x: A)(using CanEqual[A, A]): ZIO[Sort, Err, Unit] = + ZIO.serviceWithZIO(_.remove(ns, x)) + +def flatten[A: Codec](ns: String): ZStream[Sort, Err, A] = + ZStream.serviceWithStream(_.flatten(ns)) + +class SortLive(dba: Dba) extends Sort: + def insert[A: Codec: Ordering](ns: String, x: A)(using CanEqual[A, A]): IO[Err, Unit] = + for + h <- dba_head + node <- dba_get(ns, h) + _ <- + node match + case None => dba_add(ns, toNode(x)) + case Some(node) => insert(ns, x, node, h) + yield () + + private def insert[A: Codec: Ordering](ns: String, x: A, node: Node[A], nodeKey: Key)(using CanEqual[A, A]): IO[Err, Unit] = + node match + case Node(_, y, _, false) if x == y => + dba_put(ns, nodeKey, node.copy(active=true)) + + case Node(_, y, _, true) if x == y => ZIO.unit + + case Node(None, y, _, _) if x < y => + for + k <- dba_add(ns, toNode(x)) + _ <- dba_put(ns, nodeKey, node.copy(left=Some(k))) + yield () + + case Node(Some(t), y, _, _) if x < y => + for + node1 <- dba_get(ns, t) + _ <- + node1 match + case None => + for + k <- dba_add(ns, toNode(x)) + _ <- dba_put(ns, nodeKey, node.copy(left=Some(k))) + yield () + case Some(node1) => + insert(ns, x, node1, t) + yield () + + case Node(_, _, None, _) => + for + k <- dba_add(ns, toNode(x)) + _ <- dba_put(ns, nodeKey, node.copy(right=Some(k))) + yield () + + case Node(_, _, Some(s), _) => + for + node1 <- dba_get(ns, s) + _ <- + node1 match + case None => + for + k <- dba_add(ns, toNode(x)) + _ <- dba_put(ns, nodeKey, node.copy(right=Some(k))) + yield () + case Some(node1) => + insert(ns, x, node1, s) + yield () + + def remove[A: Codec: Ordering](ns: String, x: A)(using CanEqual[A, A]): IO[Err, Unit] = + for + h <- dba_head + node <- dba_get(ns, h) + _ <- + node match + case None => ZIO.unit + case Some(node) => remove(ns, x, node, h) + yield () + + private def remove[A: Codec: Ordering](ns: String, x: A, node: Node[A], nodeKey: Key)(using CanEqual[A, A]): IO[Err, Unit] = + node match + case Node(_, y, _, false) if x == y => ZIO.unit + + case Node(_, y, _, true) if x == y => + dba_put(ns, nodeKey, node.copy(active=false)) + + case Node(None, y, _, _) if x < y => ZIO.unit + + case Node(Some(t), y, _, _) if x < y => + for + node1 <- dba_get(ns, t) + _ <- + node1 match + case None => ZIO.unit + case Some(node1) => + remove(ns, x, node1, t) + yield () + + case Node(_, _, None, _) => ZIO.unit + + case Node(_, _, Some(s), _) => + for + node1 <- dba_get(ns, s) + _ <- + node1 match + case None => ZIO.unit + case Some(node1) => + remove(ns, x, node1, s) + yield () + + def flatten[A: Codec](ns: String): Stream[Err, A] = + for + h <- ZStream.fromZIO(dba_head) + xs <- flatten(ns, h) + yield xs + + private def flatten[A: Codec](ns: String, nodeKey: Key): Stream[Err, A] = + for + node <- ZStream.fromZIO(dba_get(ns, nodeKey)) + xs <- + node match + case None => ZStream.empty + case Some(Node(t, x, s, a)) => + val xs = ZStream(x).filter(_ => a) + val ts = ZStream(t).collectSome.flatMap(flatten(ns, _)) + val ss = ZStream(s).collectSome.flatMap(flatten(ns, _)) + ts ++ xs ++ ss + yield xs + + inline private def toNode[A](x: A): Node[A] = Node(None, x, None, active=true) + + type Key = Long + + case class Node[A](@N(1) left: Option[Key], @N(2) x: A, @N(3) right: Option[Key], @N(4) active: Boolean) + + def dba_add[A: Codec](ns: String, v: Node[A]): IO[Err, Key] = + given CanEqual[None.type, Option[kvs.rng.Value]] = CanEqual.derived + for + nse <- encodeNS(ns) + ide <- dba.get(nse) + id <- + ide match + case Some(ide) => decodeKeyAsValue(ide) + case None => ZIO.succeed(0L) + id1 <- ZIO.succeed(id + 1L) + id1e <- encodeKeyAsValue(id1) + _ <- dba.put(nse, id1e) + _ <- dba_put(ns, id1, v) + yield id1 + + def dba_put[A: Codec](ns: String, k: Key, v: Node[A]): IO[Err, Unit] = + for + ke <- encodeKey(ns, k) + ve <- encodeNode(v) + _ <- dba.put(ke, ve) + yield () + + def dba_get[A: Codec](ns: String, k: Key): IO[Err, Option[Node[A]]] = + for + ke <- encodeKey(ns, k) + ve <- dba.get(ke) + v <- + ve match + case None => ZIO.none + case Some(ve) => decodeNode(ve).asSome + yield v + + def dba_head: UIO[Key] = ZIO.succeed(1L) + + def encodeNode[A: Codec](x: Node[A]): UIO[Array[Byte]] = + ZIO.succeed(encode(x)) + + def decodeNode[A: Codec](bs: Array[Byte]): UIO[Node[A]] = + ZIO.succeed(decode(bs)) + + def decodeKeyAsValue(bs: Array[Byte]): IO[IOException, Key] = + for + cis <- ZIO.succeed(CodedInputStream.newInstance(bs).nn) + k <- ZIO.attempt(cis.readUInt64).refineToOrDie[IOException] + yield k + + def encodeKeyAsValue(k: Key): IO[IOException, Array[Byte]] = + for + _ <- ZIO.when(k <= 0)(ZIO.dieMessage("key is not positive")) + size <- ZIO.succeed(CodedOutputStream.computeUInt64SizeNoTag(k)) + bs <- ZIO.succeed(new Array[Byte](size)) + cos <- ZIO.succeed(CodedOutputStream.newInstance(bs).nn) + _ <- ZIO.attempt(cos.writeUInt64NoTag(k)).refineToOrDie[IOException] + yield bs + + def encodeKey(ns: String, k: Key): IO[IOException, Array[Byte]] = + for + _ <- ZIO.when(k <= 0)(ZIO.dieMessage("key is not positive")) + size <- + ZIO.succeed( + CodedOutputStream.computeStringSizeNoTag(ns) + + 1 + + CodedOutputStream.computeUInt64SizeNoTag(k) + ) + bs <- ZIO.succeed(new Array[Byte](size)) + cos <- ZIO.succeed(CodedOutputStream.newInstance(bs).nn) + _ <- ZIO.attempt(cos.writeStringNoTag(ns)).refineToOrDie[IOException] + _ <- ZIO.attempt(cos.write(0x9: Byte)).refineToOrDie[IOException] + _ <- ZIO.attempt(cos.writeUInt64NoTag(k)).refineToOrDie[IOException] + yield bs + + def encodeNS(ns: String): IO[IOException, Array[Byte]] = + for + size <- ZIO.succeed(CodedOutputStream.computeStringSizeNoTag(ns)) + bs <- ZIO.succeed(new Array[Byte](size)) + cos <- ZIO.succeed(CodedOutputStream.newInstance(bs).nn) + _ <- ZIO.attempt(cos.writeStringNoTag(ns)).refineToOrDie[IOException] + yield bs + + given [A: Codec]: Codec[Node[A]] = caseCodecAuto + + given ce1: CanEqual[None.type, Option[Node[?]]] = CanEqual.derived + given ce2: CanEqual[None.type, Option[kvs.rng.Value]] = CanEqual.derived +end SortLive + +object SortLive: + val layer: URLayer[Dba, Sort] = + ZLayer.fromFunction(SortLive(_)) diff --git a/src/main/scala/feed_app.scala b/src/main/scala/feed_app.scala new file mode 100644 index 00000000..1ae5b105 --- /dev/null +++ b/src/main/scala/feed_app.scala @@ -0,0 +1,100 @@ +package kvs.feed +package app + +import org.apache.pekko.actor.{Actor, Props} +import java.io.IOException +import kvs.rng.{ActorSystem, Dba} +import kvs.sharding.* +import proto.* +import zio.*, stream.* +import zio.Console.{printLine, readLine} + +object FeedApp extends ZIOAppDefault: + def run = + val io: ZIO[Feed & SeqConsistency, Any, Unit] = + for + feed <- ZIO.service[Feed] + seqc <- ZIO.service[SeqConsistency] + user <- ZIO.succeed("guest") + _ <- printLine(s"welcome, $user") + _ <- + (for + _ <- printLine("add/all/q?") + s <- readLine + _ <- + s match + case "add" => + for + bodyRef <- Ref.make("") + _ <- printLine("enter post") + _ <- + (for + s <- readLine + _ <- + s match + case "" => ZIO.unit + case s => bodyRef.update(_ + "\n" + s) + yield s).repeatUntilEquals("") + body <- bodyRef.get + _ <- + body.isEmpty match + case true => ZIO.unit + case false => + for + post <- ZIO.succeed(Post(body)) + answer <- seqc.send(Add(user, post)) + _ <- printLine(answer.toString) + yield () + yield () + case "all" => + all(user).take(5).tap(x => printLine(x._2.body + "\n" + "-" * 10)).runDrain + case _ => ZIO.unit + yield s).repeatUntilEquals("q") + yield () + + val pekkoConfig: ULayer[ActorSystem.Conf] = + val name = "app" + ActorSystem.staticConf(name, kvs.rng.pekkoConf(name, "127.0.0.1", 4343) ++ "pekko.loglevel=off") + val dbaConfig: ULayer[kvs.rng.Conf] = + ZLayer.succeed(kvs.rng.Conf(dir = "target/data")) + val seqConsistencyConfig: URLayer[Feed, SeqConsistency.Config] = + ZLayer.fromFunction((feed: Feed) => + SeqConsistency.Config( + "Posts" + , { + case Add(user, post) => + (for + _ <- kvs.feed.add(fid(user), post) + yield "added").provideLayer(ZLayer.succeed(feed)) + } + , { + case Add(user, _) => user + } + ) + ) + + io.provide( + SeqConsistency.live + , seqConsistencyConfig + , kvs.sharding.live + , kvs.feed.live + , Dba.live + , dbaConfig + , ActorSystem.live + , pekkoConfig + ) +end FeedApp + +case class Post(@N(1) body: String) + +given Codec[Post] = caseCodecAuto + +case class Add(user: String, post: Post) + +def all(user: String): ZStream[Feed, Err, (Eid, Post)] = + for + r <- kvs.feed.all(fid(user)) + yield r + +def fid(user: String): String = s"posts.$user" + diff --git a/src/main/scala/search_app.scala b/src/main/scala/search_app.scala new file mode 100644 index 00000000..d1c52af2 --- /dev/null +++ b/src/main/scala/search_app.scala @@ -0,0 +1,143 @@ +package kvs.search +package app + +import org.apache.pekko.actor.{Actor, Props} +import java.io.IOException +import kvs.feed.* +import kvs.rng.{ActorSystem, Dba} +import kvs.sharding.* +import org.apache.lucene.document.{Document, TextField, Field} +import org.apache.lucene.index.Term +import org.apache.lucene.search.BooleanClause.Occur +import org.apache.lucene.search.{BooleanQuery, WildcardQuery} +import org.apache.lucene.store.Directory +import proto.* +import zio.*, stream.* +import zio.Console.{printLine, readLine} + +case class PostsSearch(s: Search) +case class NotesSearch(s: Search) + +object SearchApp extends ZIOAppDefault: + def run = + println("starting...") + val io: ZIO[PostsSearch & NotesSearch & SeqConsistency, Any, Unit] = + for + posts <- ZIO.service[PostsSearch].map(_.s) + notes <- ZIO.service[NotesSearch].map(_.s) + seqc <- ZIO.service[SeqConsistency] + _ <- printLine("indexing...") + _ <- seqc.send(IndexPosts).flatMap(x => printLine(x.toString)) + _ <- seqc.send(IndexNotes).flatMap(x => printLine(x.toString)) + _ <- printLine("welcome!") + _ <- printLine("enter 'q' to quit") + _ <- + (for + _ <- printLine("search?") + word <- readLine + _ <- + if word == "q" then ZIO.unit + else + for + xs <- + posts.run[Post]{ + val b = BooleanQuery.Builder() + b.add(WildcardQuery(Term("title", s"*${word}*")), Occur.SHOULD) + b.add(WildcardQuery(Term("content", s"*${word}*")), Occur.SHOULD) + b.build.nn + }.runCollect + _ <- printLine("posts> " + xs) + ys <- + notes.run[Note]( + WildcardQuery(Term("text", s"*${word}*")) + ).runCollect + _ <- printLine("notes> " + ys) + yield () + yield word).repeatUntilEquals("q") + yield () + + val pekkoConfig: ULayer[ActorSystem.Conf] = + val name = "app" + ActorSystem.staticConf(name, kvs.rng.pekkoConf(name, "127.0.0.1", 4343) ++ "pekko.loglevel=off") + val dbaConfig: ULayer[kvs.rng.Conf] = + ZLayer.succeed(kvs.rng.Conf(dir = "target/data")) + val postsDir: URLayer[Dba, KvsDirectory] = + ZLayer.succeed("posts_index") >>> KvsDirectory.live.fresh + val notesDir: URLayer[Dba, KvsDirectory] = + ZLayer.succeed("notes_index") >>> KvsDirectory.live.fresh + val postsSearch: URLayer[Dba, PostsSearch] = + postsDir >>> kvs.search.layer.fresh.project(PostsSearch(_)) + val notesSearch: URLayer[Dba, NotesSearch] = + notesDir >>> kvs.search.layer.fresh.project(NotesSearch(_)) + val seqConsistencyConfig: URLayer[PostsSearch & NotesSearch, SeqConsistency.Config] = + ZLayer( + for + posts <- ZIO.service[PostsSearch] + notes <- ZIO.service[NotesSearch] + yield + SeqConsistency.Config( + "Search" + , { + case IndexPosts => + (for + _ <- + posts.s.index[Any, Nothing, Post]( + ZStream( + Post("What is Lorem Ipsum?", "Lorem Ipsum is simply dummy text of the printing and typesetting industry.") + , Post("Where does it come from?", "It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old.") + ) + , p => { + val doc = Document() + doc.add(TextField("title", p.title, Field.Store.NO)) + doc.add(TextField("content", p.content, Field.Store.NO)) + doc + }) + yield "posts are indexed") + + case IndexNotes => + (for + _ <- + notes.s.index[Any, Nothing, Note]( + ZStream( + Note("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") + , Note("Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.") + ) + , n => { + val doc = Document() + doc.add(TextField("text", n.text, Field.Store.NO)) + doc + }) + yield "notes are indexed") + } + , { + case IndexPosts => "posts" + case IndexNotes => "notes" + } + ) + ) + + io.provide( + SeqConsistency.live + , seqConsistencyConfig + , postsSearch + , notesSearch + , kvs.sharding.live + , Dba.live + , dbaConfig + , ActorSystem.live + , pekkoConfig + ) +end SearchApp + +case class Post(@N(1) title: String, @N(2) content: String) +case class Note(@N(1) text: String) + +given Codec[Post] = caseCodecAuto +given Codec[Note] = caseCodecAuto + +case object IndexPosts +case object IndexNotes + +given CanEqual[IndexPosts.type, Any] = CanEqual.derived +given CanEqual[IndexNotes.type, Any] = CanEqual.derived + diff --git a/src/main/scala/sort_app.scala b/src/main/scala/sort_app.scala new file mode 100644 index 00000000..78ffb686 --- /dev/null +++ b/src/main/scala/sort_app.scala @@ -0,0 +1,99 @@ +package kvs.sort +package app + +import org.apache.pekko.actor.{Actor, Props} +import java.io.IOException +import kvs.rng.{ActorSystem, Dba} +import kvs.sharding.* +import proto.* +import zio.*, stream.* +import zio.Console.{printLine, readLine} + +object SortApp extends ZIOAppDefault: + def run = + val io: ZIO[Sort & SeqConsistency, Any, Unit] = + for + sort <- ZIO.service[Sort] + seqc <- ZIO.service[SeqConsistency] + _ <- printLine("welcome!") + _ <- + (for + _ <- printLine("add/all/q?") + s <- readLine + _ <- + s match + case "add" => + for + bodyRef <- Ref.make("") + _ <- printLine("enter post") + _ <- + (for + s <- readLine + _ <- + s match + case "" => ZIO.unit + case s => bodyRef.update(_ + "\n" + s) + yield s).repeatUntilEquals("") + body <- bodyRef.get + _ <- + body.isEmpty match + case true => ZIO.unit + case false => + for + post <- ZIO.succeed(Post(body)) + answer <- seqc.send(Add(post)) + _ <- printLine(answer.toString) + yield () + yield () + case "all" => + all().take(5).tap(x => printLine(x.body + "\n" + "-" * 10)).runDrain + case _ => ZIO.unit + yield s).repeatUntilEquals("q") + yield () + + val pekkoConfig: ULayer[ActorSystem.Conf] = + val name = "app" + ActorSystem.staticConf(name, kvs.rng.pekkoConf(name, "127.0.0.1", 4343) ++ "pekko.loglevel=off") + val dbaConfig: ULayer[kvs.rng.Conf] = + ZLayer.succeed(kvs.rng.Conf(dir = "target/data")) + val seqConsistencyConfig: URLayer[Sort, SeqConsistency.Config] = + ZLayer.fromFunction((sort: Sort) => + SeqConsistency.Config( + "Posts" + , { + case Add(post) => + (for + _ <- kvs.sort.insert(ns, post) + yield "added").provideLayer(ZLayer.succeed(sort)) + } + , _ => "shard" + ) + ) + + io.provide( + SeqConsistency.live + , seqConsistencyConfig + , kvs.sharding.live + , SortLive.layer + , Dba.live + , dbaConfig + , ActorSystem.live + , pekkoConfig + ) +end SortApp + +case class Post(@N(1) body: String) + +given Codec[Post] = caseCodecAuto +given Ordering[Post] = Ordering.by(_.body) +given CanEqual[Post, Post] = CanEqual.derived + +case class Add(post: Post) + +def all(): ZStream[Sort, Err, Post] = + for + r <- kvs.sort.flatten(ns) + yield r + +val ns = "posts_sorted" + diff --git a/src/test/scala/feed.test.scala b/src/test/scala/feed.test.scala new file mode 100644 index 00000000..682ea63a --- /dev/null +++ b/src/test/scala/feed.test.scala @@ -0,0 +1,37 @@ +package kvs.feed + +import kvs.rng.{ActorSystem, Dba} +import proto.* +import scala.concurrent.duration.* +import scala.language.postfixOps +import zio.* +import zio.test.*, Assertion.* +import zio.test.ZIOSpecDefault + +case class Entry(@N(1) i: Int) + +given MessageCodec[Entry] = caseCodecAuto + +object FeedSpec extends ZIOSpecDefault: + val name = "test" + val pekkoConf: ULayer[ActorSystem.Conf] = + ActorSystem.staticConf(name, kvs.rng.pekkoConf(name, "127.0.0.1", 4344) ++ "pekko.loglevel=off") + val actorSystem: TaskLayer[ActorSystem] = + pekkoConf >>> ActorSystem.live + val dbaConf: ULayer[kvs.rng.Conf] = + ZLayer.succeed(kvs.rng.Conf(dir = s"target/data-${java.util.UUID.randomUUID}")) + val dba: TaskLayer[Dba] = + actorSystem ++ dbaConf >>> Dba.live + val feedLayer: TaskLayer[Feed] = + actorSystem ++ dba >>> kvs.feed.live + + def spec = suite("FeedSpec")( + test("FILO") { + val fid = "feed" + (for + _ <- add(fid, Entry(1)) + _ <- add(fid, Entry(2)) + xs <- all(fid).map(_._2.i).runCollect + yield assert(xs)(equalTo(Seq(2, 1)))).provideLayer(feedLayer) + } + ) \ No newline at end of file