Created ValueStore and began work on AirportSpec

all/src/test/scala/spec/AirportSpec.scala

@@ -0,0 +1,149 @@
+package spec
+
+import cats.effect.IO
+import cats.effect.testing.scalatest.AsyncIOSpec
+import fabric.rw.RW
+import lightdb.halo.HaloDBSupport
+import lightdb.lucene.LuceneSupport
+import lightdb.{Document, Id, LightDB, StoredValue}
+import lightdb.model.Collection
+import lightdb.upgrade.DatabaseUpgrade
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AsyncWordSpec
+
+import java.nio.file.{Path, Paths}
+import scala.collection.mutable
+import scala.collection.mutable.ListBuffer
+import scala.io.Source
+
+class AirportSpec extends AsyncWordSpec with AsyncIOSpec with Matchers {
+  "AirportSpec" should {
+    "initialize the database" in {
+      DB.init(truncate = true)
+    }
+    // TODO: Test ValueStore
+    // TODO: the other stuff
+    "dispose" in {
+      DB.dispose()
+    }
+  }
+
+  object DB extends LightDB with HaloDBSupport {
+    override lazy val directory: Path = Paths.get("airports")
+
+    override lazy val userCollections: List[Collection[_]] = List(
+      Airport, Flight
+    )
+
+    override def upgrades: List[DatabaseUpgrade] = List(DataImportUpgrade)
+  }
+
+  case class Airport(name: String,
+                     city: String,
+                     state: String,
+                     country: String,
+                     lat: Double,
+                     long: Double,
+                     vip: Boolean,
+                     _id: Id[Airport] = Airport.id()) extends Document[Airport]
+
+  object Airport extends Collection[Airport]("airports", DB) with LuceneSupport[Airport] {
+    override implicit val rw: RW[Airport] = RW.gen
+
+    val name: I[String] = index.one[String]("name", _.name)
+  }
+
+  case class Flight(from: Id[Airport],
+                    to: Id[Airport],
+                    year: Int,
+                    month: Int,
+                    day: Int,
+                    dayOfWeek: Int,
+                    depTime: Int,
+                    arrTime: Int,
+                    depTimeUTC: String,
+                    arrTimeUTC: String,
+                    uniqueCarrier: String,
+                    flightNum: Int,
+                    tailNum: String,
+                    distance: Int,
+                    _id: Id[Flight] = Flight.id()) extends Document[Flight]
+
+  object Flight extends Collection[Flight]("flights", DB) {
+    override implicit val rw: RW[Flight] = RW.gen
+  }
+
+  object DataImportUpgrade extends DatabaseUpgrade {
+    override def applyToNew: Boolean = true
+    override def blockStartup: Boolean = true
+    override def alwaysRun: Boolean = false
+
+    override def upgrade(db: LightDB): IO[Unit] = for {
+      insertedAirports <- {
+        val airports = csv2Stream("airports.csv").map { d =>
+          Airport(
+            name = d(1),
+            city = d(2),
+            state = d(3),
+            country = d(4),
+            lat = d(5).toDouble,
+            long = d(6).toDouble,
+            vip = d(7).toBoolean,
+            _id = Airport.id(d(0))
+          )
+        }
+        airports.evalMap(Airport.set(_)).compile.count.map(_.toInt)
+      }
+      _ = insertedAirports should be(3375)
+      insertedFlights <- {
+        val flights = csv2Stream("flights.csv").map { d =>
+          Flight(
+            from = Airport.id(d(0)),
+            to = Airport.id(d(1)),
+            year = d(2).toInt,
+            month = d(3).toInt,
+            day = d(4).toInt,
+            dayOfWeek = d(5).toInt,
+            depTime = d(6).toInt,
+            arrTime = d(7).toInt,
+            depTimeUTC = d(8),
+            arrTimeUTC = d(9),
+            uniqueCarrier = d(10),
+            flightNum = d(11).toInt,
+            tailNum = d(12),
+            distance = d(13).toInt
+          )
+        }
+        flights.evalMap(Flight.set(_)).compile.count.map(_.toInt)
+      }
+      _ = insertedFlights should be(286463)
+    } yield {
+      ()
+    }
+
+    def csv2Stream(fileName: String): fs2.Stream[IO, Vector[String]] = {
+      val source = Source.fromURL(getClass.getClassLoader.getResource(fileName))
+      val iterator = source.getLines()
+      iterator.next() // Skip heading
+      fs2.Stream.fromIterator[IO](iterator.map { s =>
+        var open = false
+        val entries = ListBuffer.empty[String]
+        val b = new mutable.StringBuilder
+        s.foreach { c =>
+          if (c == '"') {
+            open = !open
+          } else if (c == ',' && !open) {
+            if (b.nonEmpty) {
+              entries += b.toString().trim
+              b.clear()
+            }
+          } else {
+            b.append(c)
+          }
+        }
+        if (b.nonEmpty) entries += b.toString().trim
+        entries.toVector
+      }, 1000)
+    }
+  }
+}

core/src/main/scala/lightdb/IndexedLinks.scala

@@ -18,6 +18,8 @@ case class IndexedLinks[V, D <: Document[D]](name: String,
                                              createKey: V => String,
                                              collection: AbstractCollection[D],
                                              maxLinks: MaxLinks = MaxLinks.OverflowWarn()) {
+  private lazy val store: Store = collection.db.createStoreInternal(s"${collection.collectionName}.indexedLinks.$name")
+
   collection.postSet.add((_: DocumentAction, doc: D, _: AbstractCollection[D]) => {
     add(doc).map(_ => Some(doc))
   })
@@ -26,8 +28,6 @@ case class IndexedLinks[V, D <: Document[D]](name: String,
   })
   collection.truncateActions += clear()
 
-  private lazy val store: Store = collection.db.createStoreInternal(s"${collection.collectionName}.indexedLinks.$name")
-
   protected[lightdb] def add(doc: D): IO[Unit] = {
     val v = createV(doc)
     for {

core/src/main/scala/lightdb/LightDB.scala

@@ -168,7 +168,7 @@ abstract class LightDB {
     IO.unit
   }
 
-  protected object stored {
+  object stored {
     def apply[T](key: String,
                  default: => T,
                  cache: Boolean = true,

core/src/main/scala/lightdb/StoredValue.scala

@@ -1,6 +1,7 @@
 package lightdb
 
 import cats.effect.IO
+import cats.implicits.catsSyntaxApplicativeByName
 import fabric.rw._
 import lightdb.model.Collection
 
@@ -25,14 +26,20 @@ case class StoredValue[T](key: String,
 
   def exists(): IO[Boolean] = collection.get(id).map(_.nonEmpty)
 
-  def clear(): IO[Unit] = collection.delete(id).map { _ =>
-    if (cache) cached = None
-  }
-
   def set(value: T): IO[T] = collection
     .set(KeyValue(id, value.asJson))
     .map { _ =>
       if (cache) cached = Some(value)
       value
     }
+
+  def modify(f: T => IO[T]): IO[T] = for {
+    current <- get()
+    modified <- f(current)
+    _ <- set(modified).whenA(current != modified)
+  } yield modified
+
+  def clear(): IO[Unit] = collection.delete(id).map { _ =>
+    if (cache) cached = None
+  }
 }

core/src/main/scala/lightdb/ValueStore.scala

@@ -1,11 +1,35 @@
 package lightdb
 
+import cats.effect.IO
 import fabric.rw.RW
-import lightdb.model.AbstractCollection
+import lightdb.model.{AbstractCollection, DocumentAction}
 
 case class ValueStore[V, D <: Document[D]](key: String,
                                            createV: D => V,
                                            loadStore: () => Store,
                                            collection: AbstractCollection[D],
+                                           cached: Boolean = false,
                                            distinct: Boolean = true)
-                                          (implicit rw: RW[V])
+                                          (implicit rw: RW[V]) {
+  private lazy val stored = collection.db.stored[List[V]](
+    key = s"${collection.collectionName}.valueStore.$key",
+    default = Nil,
+    cache = cached
+  )
+
+  collection.postSet.add((_: DocumentAction, doc: D, _: AbstractCollection[D]) => {
+    stored.modify { list => IO {
+      val v = createV(doc)
+      var l = v :: list
+      if (distinct) l = l.distinct
+      l
+    }}.map(_ => Some(doc))
+  })
+  collection.postDelete.add((_: DocumentAction, doc: D, _: AbstractCollection[D]) => {
+    stored.modify { list => IO {
+      val v = createV(doc)
+      list.filterNot(_ == v)
+    }}.map(_ => Some(doc))
+  })
+  collection.truncateActions += stored.clear()
+}