Merge pull request #1 from damus-io/master

Update

.github/workflows/c-cpp.yml

@@ -29,5 +29,8 @@ jobs:
         ./scripts/build.sh
         sudo cp bin/flatcc /usr/bin
 
+    - name: make all
+      run: make
+
     - name: make check
       run: make check

Makefile

@@ -1,5 +1,5 @@
 CFLAGS = -Wall -Wno-misleading-indentation -Wno-unused-function -Werror -O2 -g -Ideps/secp256k1/include -Ideps/lmdb -Ideps/flatcc/include
-HEADERS = sha256.h nostrdb.h cursor.h hex.h jsmn.h config.h sha256.h random.h memchr.h $(C_BINDINGS)
+HEADERS = sha256.h nostrdb.h cursor.h hex.h jsmn.h config.h sha256.h random.h memchr.h cpu.h $(C_BINDINGS)
 FLATCC_SRCS=deps/flatcc/src/runtime/json_parser.c deps/flatcc/src/runtime/verifier.c deps/flatcc/src/runtime/builder.c deps/flatcc/src/runtime/emitter.c deps/flatcc/src/runtime/refmap.c
 SRCS = nostrdb.c sha256.c bech32.c $(FLATCC_SRCS)
 LDS = $(SRCS) $(ARS) 
@@ -18,7 +18,7 @@ BIN=ndb
 
 CHECKDATA=testdata/db/v0/data.mdb
 
-all: $(BIN) lib bindings
+all: $(BIN) lib bindings ndb
 
 lib: benches test
 
@@ -136,9 +136,6 @@ deps/lmdb/liblmdb.a: deps/lmdb/lmdb.h
 bench: bench.c $(DEPS)
 	$(CC) $(CFLAGS) bench.c $(LDS) -o $@
 
-bench-ingest: bench-ingest.c $(DEPS)
-	$(CC) $(CFLAGS) bench-ingest.c $(LDS) -o $@
-
 testdata/db/ndb-v0.tar.zst:
 	curl https://cdn.jb55.com/s/ndb-v0.tar.zst -o $@
 

README.md

@@ -19,3 +19,50 @@ functionality (yet?)
 
 [1]: https://github.com/hoytech/strfry
 
+
+## API
+
+The API is *very* unstable. nostrdb is in heavy development mode so don't
+expect any of the interfaces to be stable at this time.
+
+## CLI
+
+nostrdb comes with a handy `ndb` command line tool for interacting with nostrdb
+databases. The tool is relatively new, and only supports a few commands.
+
+### Usage
+
+```
+usage: ndb [--skip-verification] [-d db_dir] <command>
+
+commands
+
+	stat
+	search [--oldest-first] [--limit 42] <fulltext query>
+	import <line-delimited json file>
+
+settings
+
+	--skip-verification  skip signature validation
+	-d <db_dir>          set database directory
+```
+
+### Building
+
+```bash
+$ make ndb
+```
+
+### Fulltext Queries
+
+nostrdb supports fulltext queries. You can import some test events like so:
+
+```
+$ make testdata/many-events.json
+$ ndb --skip-verification import testdata/many-events.json
+$ ndb search --limit 2 --oldest-first 'nosy ostrich'
+
+[01] K<'ostrich' 7 1671217526 note_id:253309>
+Q: What do you call a nosy ostrich?
+A: A nosTrich!
+```

bench-ingest-many.c

@@ -27,18 +27,20 @@ int map_file(const char *filename, unsigned char **p, size_t *flen)
 
 static int bench_parser()
 {
-	int ingester_threads;
 	long nanos, ms;
-	size_t written, mapsize;
+	size_t written;
 	struct ndb *ndb;
 	struct timespec t1, t2;
 	char *json;
 	int times = 1;
+	struct ndb_config config;
+	ndb_default_config(&config);
 
-	mapsize = 1024ULL * 1024ULL * 400ULL * 10ULL;
-	ingester_threads = 8;
-	assert(ndb_init(&ndb, "testdata/db", mapsize, ingester_threads,
-			NDB_FLAG_SKIP_NOTE_VERIFY));
+	ndb_config_set_mapsize(&config, 1024ULL * 1024ULL * 400ULL * 10ULL);
+	ndb_config_set_ingest_threads(&config, 8);
+	ndb_config_set_flags(&config, NDB_FLAG_SKIP_NOTE_VERIFY);
+
+	assert(ndb_init(&ndb, "testdata/db", &config));
 	const char *filename = "testdata/many-events.json";
 	if (!map_file(filename, (unsigned char**)&json, &written)) {
 		printf("mapping testdata/many-events.json failed\n");

cpu.h

@@ -0,0 +1,34 @@
+
+#if defined(_WIN32) || defined(_WIN64)
+	#include <windows.h>
+#elif defined(__linux__)
+	#include <unistd.h>
+#elif defined(__APPLE__)
+	#include <sys/types.h>
+	#include <sys/sysctl.h>
+#else
+	#error "Unsupported platform"
+#endif
+
+static inline int get_cpu_cores() {
+	int num_cores = 0;
+
+	// Windows
+	#if defined(_WIN32) || defined(_WIN64)
+		SYSTEM_INFO sysinfo;
+		GetSystemInfo(&sysinfo);
+		num_cores = sysinfo.dwNumberOfProcessors; // This returns logical processors
+		// Further use GetLogicalProcessorInformation for physical cores...
+	// Linux
+	#elif defined(__linux__)
+		num_cores = sysconf(_SC_NPROCESSORS_ONLN); // This returns logical processors
+	// macOS
+	#elif defined(__APPLE__)
+		size_t size = sizeof(num_cores);
+		sysctlbyname("hw.physicalcpu", &num_cores, &size, NULL, 0);
+	#else
+		num_cores = -1; // Unsupported platform
+	#endif
+
+	return num_cores;
+}

ndb.c

@@ -2,6 +2,7 @@
 
 #include "nostrdb.h"
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
 #include <unistd.h>
@@ -12,7 +13,7 @@ static int usage()
 	printf("usage: ndb [--skip-verification] [-d db_dir] <command>\n\n");
 	printf("commands\n\n");
 	printf("	stat\n");
-	printf("	search <fulltext query>\n");
+	printf("	search [--oldest-first] [--limit 42] <fulltext query>\n");
 	printf("	import <line-delimited json file>\n\n");
 	printf("settings\n\n");
 	printf("	--skip-verification  skip signature validation\n");
@@ -86,18 +87,58 @@ static void print_stats(struct ndb_stat *stat)
 	}
 }
 
+static void ndb_print_text_search_key(struct ndb_text_search_key *key)
+{
+	printf("K<'%.*s' %d %" PRIu64 " note_id:%" PRIu64 ">", key->str_len, key->str,
+						    key->word_index,
+						    key->timestamp,
+						    key->note_id);
+}
+
+static void print_hex(unsigned char* data, size_t size) {
+	size_t i;
+	for (i = 0; i < size; i++) {
+		printf("%02x", data[i]);
+	}
+}
+
+static void ndb_print_text_search_result(struct ndb_txn *txn,
+		struct ndb_text_search_result *r)
+{
+	size_t len;
+	struct ndb_note *note;
+
+	ndb_print_text_search_key(&r->key);
+
+	if (!(note = ndb_get_note_by_key(txn, r->key.note_id, &len))) {
+		printf(": note not found");
+		return;
+	}
+
+	printf(" ");
+	print_hex(note->id, 32);
+
+	printf("\n%s\n\n---\n", ndb_note_str(note, &note->content).str);
+}
+
+int ndb_print_search_keys(struct ndb_txn *txn);
+
 int main(int argc, char *argv[])
 {
 	struct ndb *ndb;
-	int threads = 6;
-	int i, flags;
+	int i, flags, limit;
 	struct ndb_stat stat;
 	struct ndb_txn txn;
 	struct ndb_text_search_results results;
+	struct ndb_text_search_result *result;
 	const char *dir;
 	unsigned char *data;
 	size_t data_len;
-	size_t mapsize = 1024ULL * 1024ULL * 1024ULL * 1024ULL; // 1 TiB
+	struct ndb_config config;
+	struct ndb_text_search_config search_config;
+	ndb_default_config(&config);
+	ndb_default_text_search_config(&search_config);
+	ndb_config_set_mapsize(&config, 1024ULL * 1024ULL * 1024ULL * 1024ULL /* 1 TiB */);
 
 	if (argc < 2) {
 		return usage();
@@ -118,15 +159,38 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	ndb_config_set_flags(&config, flags);
+
 	fprintf(stderr, "using db '%s'\n", dir);
 
-	if (!ndb_init(&ndb, dir, mapsize, threads, flags)) {
+	if (!ndb_init(&ndb, dir, &config)) {
 		return 2;
 	}
 
-	if (argc == 3 && !strcmp(argv[1], "search")) {
+	if (argc >= 3 && !strcmp(argv[1], "search")) {
+		for (i = 0; i < 2; i++) {
+			if (!strcmp(argv[2], "--oldest-first")) {
+				ndb_text_search_config_set_order(&search_config, NDB_ORDER_ASCENDING);
+				argv++;
+				argc--;
+			} else if (!strcmp(argv[2], "--limit")) {
+				limit = atoi(argv[3]);
+				ndb_text_search_config_set_limit(&search_config, limit);
+				argv += 2;
+				argc -= 2;
+			}
+		}
+
 		ndb_begin_query(ndb, &txn);
-		ndb_text_search(&txn, argv[2], &results);
+		ndb_text_search(&txn, argv[2], &results, &search_config);
+
+		// print results for now
+		for (i = 0; i < results.num_results; i++) {
+			result = &results.results[i];
+			printf("[%02d] ", i+1);
+			ndb_print_text_search_result(&txn, result);
+		}
+
 		ndb_end_query(&txn);
 	} else if (argc == 2 && !strcmp(argv[1], "stat")) {
 		if (!ndb_stat(ndb, &stat)) {
@@ -137,6 +201,11 @@ int main(int argc, char *argv[])
 	} else if (argc == 3 && !strcmp(argv[1], "import")) {
 		map_file(argv[2], &data, &data_len);
 		ndb_process_events(ndb, (const char *)data, data_len);
+		ndb_process_client_events(ndb, (const char *)data, data_len);
+	} else if (argc == 2 && !strcmp(argv[1], "print-search-keys")) {
+		ndb_begin_query(ndb, &txn);
+		ndb_print_search_keys(&txn);
+		ndb_end_query(&txn);
 	} else {
 		return usage();
 	}