Tools 2767 support sindex on blob bins (#75)

* fix: support backup and restore sindexes on blob bins

* build: update c-client submodule to 6.5.0

* build: pass event_lib to c client

* ci: adjust tests for server 7

.github/workflows/backup-build.yml

@@ -2,9 +2,9 @@ name: Build and Test All
 
 on:
   push:
-    branches: [ master, test-ready ]
+    branches: [ master, test-ready, ]
   pull_request:
-    branches: [ master, test-ready ]
+    branches: [ master, test-ready, "bugfix/*" ]
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -128,6 +128,8 @@ jobs:
     - name: Info
       run: make info
       working-directory: main
+    # - name: Setup tmate session
+    #   uses: mxschmitt/action-tmate@v3
     - name: Initialize codecov
       run: make coverage-init
       working-directory: main

.github/workflows/mac-artifact.yml

@@ -1,9 +1,9 @@
 name: Mac Artifact
 on:
   push:
-    branches: [ master, test-ready, "bugfix-*", actionsHub ]
+    branches: [ master, test-ready, "bugfix-*" ]
   pull_request:
-    branches: [ master, test-ready ]
+    branches: [ master, test-ready, "bugfix-*" ]
   workflow_call:
     inputs:
       submodule:

Makefile

@@ -366,6 +366,7 @@ info:
 	@echo "  ARCH:       " $(ARCH)
 	@echo "  CLIENTREPO: " $(DIR_C_CLIENT)
 	@echo "  WD:         " $(shell pwd)
+	@echo "  EVENT_LIB:  " $(EVENT_LIB)
 	@echo
 	@echo "  PATHS:"
 	@echo "      source:     " $(DIR_SRC)
@@ -414,7 +415,7 @@ $(TOML):
 	$(MAKE) -C $(DIR_TOML)
 
 $(C_CLIENT_LIB):
-	$(MAKE) -C $(DIR_C_CLIENT)
+	$(MAKE) -C $(DIR_C_CLIENT) EVENT_LIB=$(EVENT_LIB)
 
 -include $(BACKUP_DEP)
 -include $(RESTORE_DEP)

README.md

@@ -317,7 +317,7 @@ Let's look at the placeholders, there are quite a few.
 | `{name}`       | The name of the index. |
 | `{index-type}` | The type of index: `N` = index on bins, `L` = index on list elements, `K` = index on map keys, `V` = index on map values |
 | `{path}`       | The bin name |
-| `{data-type}`  | The data type of the indexed value: `N` = numeric, `S` = string |
+| `{data-type}`  | The data type of the indexed value: `N` = numeric, `S` = string, `G` = geo2dsphere, `B` = bytes/blob, `I` = invalid |
 
 The `["1"]` token is actually the number of values covered by the index. This is for future extensibility, i.e., for composite indexes that span more than one value. Right now, this token is always `["1"]`, though.
 

include/encode.h

@@ -88,7 +88,9 @@ typedef enum {
 	// The path results in an integer.
 	PATH_TYPE_NUMERIC,
 	// The path results in a geo2dsphere value.
-	PATH_TYPE_GEO2DSPHERE
+	PATH_TYPE_GEO2DSPHERE,
+	// The path results in a bytes/blob value.
+	PATH_TYPE_BLOB
 } path_type;
 
 /*

requirements.txt

@@ -1,4 +1,4 @@
-aerospike~=9.0.0
+aerospike~=14.0.0
 docker
 pytest
 pylint

src/backup_status.c

@@ -1209,7 +1209,8 @@ calc_node_list_partitions(as_cluster *clust, const as_namespace ns,
 		as_partition* part = &table->partitions[i];
 
 		for (uint32_t j = 0; j < n_node_names; j++) {
-			if (strncmp(part->master->name, (*node_names)[j], AS_NODE_NAME_SIZE) == 0) {
+			// part->nodes[0] is the master node for part
+			if (strncmp(part->nodes[0]->name, (*node_names)[j], AS_NODE_NAME_SIZE) == 0) {
 				if (last_part_included) {
 					filter->count++;
 				}

src/dec_text.c

@@ -1544,6 +1544,10 @@ text_parse_index(io_read_proxy_t *fd, as_vector *ns_vec, uint32_t *line_no,
 		case 'G':
 			path.type = PATH_TYPE_GEO2DSPHERE;
 			break;
+
+		case 'B':
+			path.type = PATH_TYPE_BLOB;
+			break;
 
 		default:
 			err("Invalid path type character %s in block (line %u, col %u)", print_char(ch),

src/enc_text.c

@@ -487,7 +487,7 @@ text_index_type_to_label(index_type type)
 static int32_t
 text_path_type_to_label(path_type type)
 {
-	return "ISNG"[(int32_t)type];
+	return "ISNGB"[(int32_t)type];
 }
 
 /*

src/restore.c

@@ -1411,6 +1411,10 @@ restore_index(aerospike *as, index_param *index, as_vector *set_vec,
 		case PATH_TYPE_GEO2DSPHERE:
 			dtype = AS_INDEX_GEO2DSPHERE;
 			break;
+
+		case PATH_TYPE_BLOB:
+			dtype = AS_INDEX_BLOB;
+			break;
 	}
 
 	as_policy_info policy;

src/utils.c

@@ -1548,6 +1548,8 @@ parse_index_info(char *ns, char *index_str, index_param *index)
 				type = PATH_TYPE_NUMERIC;
 			} else if (strcasecmp(arg, "INT SIGNED") == 0) {
 				type = PATH_TYPE_NUMERIC;
+			}else if (strcasecmp(arg, "BLOB") == 0) {
+				type = PATH_TYPE_BLOB;
 			} else if (strcasecmp(arg, "GEO2DSPHERE") == 0 || strcasecmp(arg, "GEOJSON") == 0) {
 				type = PATH_TYPE_GEO2DSPHERE;
 			} else {

test/integration/aerospike.conf

@@ -22,6 +22,7 @@
 #
 
 service {
+	cluster-name asb-cluster
 	run-as-daemon false
 	work-directory ${state_directory}
 	pidfile ${state_directory}/asd.pid
@@ -70,8 +71,9 @@ network {
 
 namespace test {
 	replication-factor 2
-	memory-size 512M
 	default-ttl 0
-	storage-engine memory
+	storage-engine memory {
+		data-size 1G
+	}
 	nsup-period 60
 }

test/integration/aerospike_servers.py

@@ -23,6 +23,7 @@
 WORK_DIRECTORY = lib.WORK_DIRECTORY
 
 SERVER_IMAGE = "aerospike/aerospike-server"
+SERVER_VERSION = ""
 
 
 STATE_DIRECTORIES = ["state-%d" % i for i in range(1, N_NODES+1)]
@@ -253,6 +254,11 @@ def start_aerospike_servers(keep_work_dir=False):
 		print("Client connected")
 		lib.safe_sleep(1)
 
+def get_aerospike_version():
+	client = get_client()
+	resp = client.info_random_node("build")
+	return resp.split("\t")[1].strip()
+
 def stop_aerospike_servers(keep_work_dir=False):
 	"""
 	Disconnects the client and stops the running asd process.

test/integration/lib.py

@@ -26,6 +26,8 @@
 SET = "test"
 CLIENT_ATTEMPTS = 20
 
+AEROSPIKE_VERSION_BLOB_SINDEX = "7.0.0.0"
+
 if sys.platform == "linux":
 	USE_VALGRIND = False
 else:
@@ -216,7 +218,7 @@ def get_key_digest(set_name, key):
 	"""
 	Calculate the digest of the given key in the given set.
 	"""
-	return get_client().get_key_digest(NAMESPACE, set_name, key)
+	return aerospike.calc_digest(NAMESPACE, set_name, key)
 
 def test_record(set_name, key):
 	"""
@@ -562,6 +564,23 @@ def create_geo_index(set_name, bin_name, index_name):
 	assert ret == 0, "Unexpected error while creating index"
 	GLOBALS["indexes"].append(index_name)
 
+def create_blob_index(set_name, bin_name, index_name):
+	"""
+	Creates a blob index.
+	"""
+	print("create blob index", index_name)
+	validate_index_creation(set_name, bin_name, index_name)
+	ret = -1
+	for _ in range(CLIENT_ATTEMPTS):
+		try:
+			ret = get_client().index_blob_create(NAMESPACE, set_name, bin_name, index_name)
+			break
+		except aerospike.exception.IndexFoundError:
+			# found the index in the database, meaning it wasn't fully deleted, pause and try again
+			safe_sleep(0.5)
+	assert ret == 0, "Unexpected error while creating index"
+	GLOBALS["indexes"].append(index_name)
+
 def create_string_list_index(set_name, bin_name, index_name):
 	"""
 	Creates a string list index.

test/integration/test_index.py

@@ -6,6 +6,7 @@
 import aerospike
 import lib
 from run_backup import backup_and_restore
+import aerospike_servers
 
 SET_NAMES = [None] + lib.index_variations(63)
 INDEX_NAMES = ["normal"] + lib.index_variations(63)
@@ -155,6 +156,25 @@ def test_geo_index():
 		restore_delay=1
 	)
 
+def test_blob_index():
+	"""
+	Tests bytes/blob indexes across all set names, index names, and index paths.
+	"""
+	# servers older than 7.0 don't support sindexes on blob bins
+	# so skip these older servers
+	aerospike_servers.start_aerospike_servers()
+	build = aerospike_servers.get_aerospike_version()
+	if build[0] < lib.AEROSPIKE_VERSION_BLOB_SINDEX[0]:
+		print("skipping test_blob_index, detected aerospike: " + build)
+		return
+
+	backup_and_restore(
+		lambda context: create_indexes(lib.create_blob_index),
+		None,
+		lambda context: check_indexes(lib.check_simple_index, bytes("testblob", "UTF-8")),
+		restore_delay=1
+	)
+
 def test_integer_list_index():
 	"""
 	Tests integer list indexes across all set names, index names, and

test/unit/test_utils.c

@@ -489,6 +489,28 @@ START_TEST(test_secondary_index_param_without_ctx)
 }
 END_TEST
 
+START_TEST(test_secondary_index_type_blob)
+{
+	char test_ns[] = "test_ns";
+	char sindex_str[] = "ns=test_ns:indexname=test_id:set=testset:bin=test_list:type=blob:indextype=list:state=RW:path=test_list";
+	index_param sindex_params;
+
+	bool res = parse_index_info(test_ns, sindex_str, &sindex_params);
+	path_param *path = as_vector_get((as_vector *)&sindex_params.path_vec, 0);
+
+	ck_assert(res); // failed parsing sindex
+	ck_assert_str_eq(sindex_params.ns, test_ns);
+	ck_assert_str_eq(sindex_params.set, "testset"); //set
+	ck_assert_str_eq(sindex_params.name, "test_id"); //indexname
+	ck_assert_int_eq(sindex_params.type, INDEX_TYPE_LIST); //indextype
+	ck_assert_ptr_null(sindex_params.ctx); //ctx should be null	
+	ck_assert_str_eq(path->path, "test_list"); //bin name
+	ck_assert_int_eq(path->type, PATH_TYPE_BLOB); //bin type
+
+	cf_free(path);
+}
+END_TEST
+
 Suite* utils_suite()
 {
 	Suite* s;