Skip list impl test and benchmark, bloom benchmark

README.md

@@ -86,5 +86,14 @@ SSTableBenchmark.randomAccess    thrpt   10    57254.444 ±   401.035  ops/s
     - [x] Bloom filter
     - [x] Indexes persistence
     - [x] File initialization
-- [ ] Memtable
+- [ ] Skip-List
+    - [x] Operations
+    - [ ] Iterator
 - [ ] Tree
+    - [ ] Operations
+    - [ ] Background compaction
+- [ ] Benchmarks
+    - [x] SSTable
+    - [x] Bloom filter
+    - [x] Skip-List
+    - [ ] Tree

build.gradle

@@ -12,6 +12,7 @@ repositories {
 
 dependencies {
     implementation 'it.unimi.dsi:fastutil:8.5.12'
+    implementation 'it.unimi.dsi:dsiutils:2.7.3'
     implementation 'commons-codec:commons-codec:1.16.0'
 
     testImplementation(platform("org.junit:junit-bom:5.9.1"))

src/jmh/java/com/tomfran/lsm/bloom/BloomFilterBenchmark.java

@@ -0,0 +1,46 @@
+package com.tomfran.lsm.bloom;
+
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+import static com.tomfran.lsm.TestUtils.getRandomByteArray;
+
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+public class BloomFilterBenchmark {
+
+    BloomFilter bf;
+
+    byte[][] keys;
+
+    int N = 1000000;
+    int index = 0;
+
+    @Setup
+    public void setup() {
+
+        bf = new BloomFilter(N, 0.01);
+
+        keys = new byte[N][];
+
+        for (int i = 0; i < N; i++)
+            keys[i] = getRandomByteArray();
+    }
+
+    @Benchmark
+    public void add() {
+        bf.add(keys[index]);
+
+        index = (index + 1) % N;
+    }
+
+    @Benchmark
+    public void contains(Blackhole bh) {
+        bh.consume(bf.mightContain(keys[index]));
+
+        index = (index + 1) % N;
+    }
+
+}

src/jmh/java/com/tomfran/lsm/memtable/SkipListBenchmark.java

@@ -0,0 +1,71 @@
+package com.tomfran.lsm.memtable;
+
+import com.tomfran.lsm.types.Item;
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+import static com.tomfran.lsm.TestUtils.getRandomItem;
+
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+public class SkipListBenchmark {
+
+    SkipList l;
+    Item[] items;
+
+    int NUM_ITEMS = 200000;
+    int index = 0;
+
+    boolean[] addRemove;
+
+    @Setup
+    public void setup() {
+
+        l = new SkipList(NUM_ITEMS / 2);
+
+        // generate random items and insert half
+        ObjectArrayList<Item> tmp = new ObjectArrayList<>();
+        for (int i = 0; i < NUM_ITEMS; i++) {
+            var it = getRandomItem();
+            if (i < NUM_ITEMS / 2)
+                l.add(it);
+
+            tmp.add(it);
+        }
+
+        items = tmp.toArray(new Item[0]);
+
+        // generate sequence of add/remove operations
+        addRemove = new boolean[NUM_ITEMS];
+        for (int i = 0; i < NUM_ITEMS; i++) {
+            addRemove[i] = Math.random() < 0.5;
+        }
+    }
+
+    @Benchmark
+    public void get(Blackhole bh) {
+        var key = items[index].key();
+        var found = l.get(key);
+
+        bh.consume(found);
+
+        index = (index + 1) % NUM_ITEMS;
+    }
+
+    @Benchmark
+    public void addRemove(Blackhole bh) {
+        var key = items[index].key();
+
+        if (addRemove[index]) {
+            l.add(items[index]);
+        } else {
+            l.remove(key);
+        }
+
+        index = (index + 1) % NUM_ITEMS;
+    }
+
+}

src/jmh/java/com/tomfran/lsm/sstable/SSTableBenchmark.java

@@ -90,7 +90,7 @@ private void deleteDir() throws IOException {
     @Benchmark
     public void randomAccess(Blackhole bh) {
         var item = insertedArray[index];
-        var it = sstable.getItem(item.key());
+        var it = sstable.get(item.key());
 
         bh.consume(it);
 
@@ -100,7 +100,7 @@ public void randomAccess(Blackhole bh) {
     @Benchmark
     public void negativeAccess(Blackhole bh) {
         var item = skippedArray[index];
-        var it = sstable.getItem(item.key());
+        var it = sstable.get(item.key());
 
         bh.consume(it);
 

src/main/java/com/tomfran/lsm/comparator/ByteArrayComparator.java

@@ -3,6 +3,10 @@
 public class ByteArrayComparator {
 
     static public int compare(byte[] a, byte[] b) {
+
+        if (a == null)
+            return b == null ? 0 : -1;
+
         int aLen = a.length;
         int bLen = b.length;
 

src/main/java/com/tomfran/lsm/memtable/Memtable.java

@@ -0,0 +1,44 @@
+package com.tomfran.lsm.memtable;
+
+import com.tomfran.lsm.sstable.SSTable;
+import com.tomfran.lsm.types.Item;
+
+import java.util.LinkedList;
+
+public class Memtable {
+
+    SkipList mutableData;
+    LinkedList<SkipList> immutableData;
+    LinkedList<LinkedList<SSTable>> sstables;
+
+    public Memtable() {
+        mutableData = new SkipList();
+        immutableData = new LinkedList<>();
+    }
+
+    public void add(Item item) {
+        mutableData.add(item);
+    }
+
+    public void get(byte[] key) {
+        mutableData.get(key);
+    }
+
+    public void remove(byte[] key) {
+        mutableData.remove(key);
+    }
+
+    public int size() {
+        return mutableData.size();
+    }
+
+    private void replaceMutableData() {
+        immutableData.addFirst(mutableData);
+        mutableData = new SkipList();
+    }
+
+    public SSTable flush() {
+        return null;
+    }
+
+}

src/main/java/com/tomfran/lsm/memtable/SkipList.java

@@ -0,0 +1,127 @@
+package com.tomfran.lsm.memtable;
+
+import com.tomfran.lsm.types.Item;
+import it.unimi.dsi.util.XoRoShiRo128PlusRandom;
+
+import java.util.Iterator;
+
+import static com.tomfran.lsm.comparator.ByteArrayComparator.compare;
+import static java.lang.Math.ceil;
+import static java.lang.Math.log;
+
+public class SkipList implements Iterable<Item> {
+
+    static final int DEFAULT_ELEMENTS = 1 << 16;
+    final Node sentinel;
+    private final Node[] buffer;
+    private final XoRoShiRo128PlusRandom rn;
+    int levels;
+    int size;
+
+    public SkipList() {
+        this(DEFAULT_ELEMENTS);
+    }
+
+    public SkipList(int numElements) {
+        levels = (int) ceil(log(numElements) / log(2));
+        size = 0;
+        sentinel = new Node(null, levels);
+        rn = new XoRoShiRo128PlusRandom();
+        buffer = new Node[levels];
+    }
+
+    public void add(Item item) {
+        Node current = sentinel;
+        for (int i = levels - 1; i >= 0; i--) {
+            while (current.next[i] != null && current.next[i].value.compareTo(item) < 0)
+                current = current.next[i];
+            buffer[i] = current;
+        }
+
+        if (current.next[0] != null && current.next[0].value.compareTo(item) == 0) {
+            current.next[0].value = item;
+            return;
+        }
+
+        Node newNode = new Node(item, levels);
+        for (int i = 0; i < randomLevel(); i++) {
+            newNode.next[i] = buffer[i].next[i];
+            buffer[i].next[i] = newNode;
+        }
+        size++;
+    }
+
+    private int randomLevel() {
+        int level = 1;
+        while (rn.nextBoolean() && level < levels)
+            level++;
+        return level;
+    }
+
+    public Item get(byte[] key) {
+        Node current = sentinel;
+        for (int i = levels - 1; i >= 0; i--) {
+            while (current.next[i] != null && compare(current.next[i].value.key(), key) < 0)
+                current = current.next[i];
+        }
+
+        if (current.next[0] != null && compare(current.next[0].value.key(), key) == 0)
+            return current.next[0].value;
+
+        return null;
+    }
+
+    public void remove(byte[] key) {
+        Node current = sentinel;
+        for (int i = levels - 1; i >= 0; i--) {
+            while (current.next[i] != null && compare(current.next[i].value.key(), key) < 0)
+                current = current.next[i];
+            buffer[i] = current;
+        }
+
+        if (current.next[0] != null && compare(current.next[0].value.key(), key) == 0) {
+            boolean last = current.next[0].next[0] == null;
+            for (int i = 0; i < levels; i++) {
+                if (buffer[i].next[i] != current.next[0])
+                    break;
+                buffer[i].next[i] = last ? null : current.next[0].next[i];
+            }
+            size--;
+        }
+    }
+
+    public int size() {
+        return size;
+    }
+
+    @Override
+    public Iterator<Item> iterator() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        for (int i = levels - 1; i >= 0; i--) {
+            sb.append(String.format("Level %2d: ", i));
+            Node current = sentinel;
+            while (current.next[i] != null) {
+                sb.append(current.next[i].value).append(" -> ");
+                current = current.next[i];
+            }
+            sb.append("END\n");
+        }
+        return sb.toString();
+    }
+
+    static final class Node {
+        Item value;
+        Node[] next;
+
+        Node(Item value, int numLevels) {
+            this.value = value;
+            this.next = new Node[numLevels];
+        }
+    }
+
+}

src/main/java/com/tomfran/lsm/sstable/SSTable.java

@@ -106,7 +106,7 @@ private void initializeFromDisk(String filename) {
      * @param key The key of the item to read.
      * @return The item with the given key, or null if no such item exists.
      */
-    public Item getItem(byte[] key) {
+    public Item get(byte[] key) {
         if (!bloomFilter.mightContain(key))
             return null;
 

src/main/java/com/tomfran/lsm/types/Item.java

@@ -15,4 +15,20 @@ public int hashCode() {
     public int compareTo(Item o) {
         return compare(key, o.key);
     }
+
+    @Override
+    public String toString() {
+        // binary representation of key and value, e.g. (1010101, 010101010)
+        StringBuilder sb = new StringBuilder();
+        sb.append("(");
+        for (byte b : key) {
+            sb.append(b);
+        }
+        sb.append(", ");
+        for (byte b : value) {
+            sb.append(b);
+        }
+        sb.append(")");
+        return sb.toString();
+    }
 }

src/test/java/com/tomfran/lsm/TestUtils.java

@@ -20,7 +20,13 @@ public static Item getRandomItem() {
         );
     }
 
-    static byte[] getRandomByteArray() {
+    public static byte[] getRandomByteArray(int length) {
+        byte[] bytes = new byte[length];
+        rn.nextBytes(bytes);
+        return bytes;
+    }
+
+    public static byte[] getRandomByteArray() {
         byte[] bytes = new byte[rn.nextInt(MIN_BYTES_LENGTH, MAX_BYTES_LENGTH)];
         rn.nextBytes(bytes);
         return bytes;