v3.0 features after crash: --initial and TLS reload , #4658 and #4659

.gitignore

@@ -51,6 +51,7 @@ binaries/*deb
 binaries/*rpm
 tools/eventslog_reader_sample
 src/proxysql-save.cfg
+src/*log*
 
 proxysql-2.0.0/
 docker/images/proxysql/rhel-compliant/rpmmacros
@@ -93,6 +94,8 @@ deps/prometheus-cpp/prometheus-cpp-*/
 deps/re2/re2-*/
 deps/sqlite3/sqlite-amalgamation-*/
 deps/coredumper/coredumper-*/
+deps/postgresql/postgresql-*/
+deps/libusual/libusual-*/
 
 test/.vagrant
 .DS_Store

Makefile

@@ -35,7 +35,7 @@ O3 := -O3 -mtune=native
 ALL_DEBUG := $(O0) -ggdb -DDEBUG
 NO_DEBUG := $(O2) -ggdb
 DEBUG := $(ALL_DEBUG)
-CURVER ?= 2.7.1
+CURVER ?= 3.0.0
 #export DEBUG
 #export EXTRALINK
 export MAKE
@@ -306,19 +306,19 @@ ubuntu: $(REL_ARCH)-ubuntu ;
 
 amd64-packages: amd64-centos amd64-ubuntu amd64-debian amd64-fedora amd64-opensuse amd64-almalinux
 amd64-almalinux: almalinux8 almalinux8-clang almalinux8-dbg almalinux9 almalinux9-clang almalinux9-dbg
-amd64-centos: centos7 centos7-dbg centos8 centos8-clang centos8-dbg centos9 centos9-clang centos9-dbg
+amd64-centos: centos8 centos8-clang centos8-dbg centos9 centos9-clang centos9-dbg
 amd64-debian: debian10 debian10-dbg debian11 debian11-clang debian11-dbg debian12 debian12-clang debian12-dbg
 amd64-fedora: fedora38 fedora38-clang fedora38-dbg fedora39 fedora39-clang fedora39-dbg fedora40 fedora40-clang fedora40-dbg fedora41 fedora41-clang fedora41-dbg
 amd64-opensuse: opensuse15 opensuse15-clang opensuse15-dbg
-amd64-ubuntu: ubuntu16 ubuntu16-dbg ubuntu18 ubuntu18-dbg ubuntu20 ubuntu20-clang ubuntu20-dbg ubuntu22 ubuntu22-clang ubuntu22-dbg ubuntu24 ubuntu24-clang ubuntu24-dbg
+amd64-ubuntu: ubuntu18 ubuntu18-dbg ubuntu20 ubuntu20-clang ubuntu20-dbg ubuntu22 ubuntu22-clang ubuntu22-dbg ubuntu24 ubuntu24-clang ubuntu24-dbg
 
 arm64-packages: arm64-centos arm64-debian arm64-ubuntu arm64-fedora arm64-opensuse arm64-almalinux
 arm64-almalinux: almalinux8 almalinux9
-arm64-centos: centos7 centos8 centos9
+arm64-centos: centos8 centos9
 arm64-debian: debian10 debian11 debian12
 arm64-fedora: fedora38 fedora39 fedora40
 arm64-opensuse: opensuse15
-arm64-ubuntu: ubuntu16 ubuntu18 ubuntu20 ubuntu22 ubuntu24
+arm64-ubuntu: ubuntu18 ubuntu20 ubuntu22 ubuntu24
 
 almalinux%: build-almalinux% ;
 centos%: build-centos% ;
@@ -350,9 +350,9 @@ binaries/proxysql%:
 	cd test/tap && ${MAKE} clean
 	cd test/deps && ${MAKE} cleanall
 	find . -not -path "./binaries/*" -not -path "./.git/*" -exec touch -h --date=@`git show -s --format=%ct HEAD` {} \;
-	@docker compose -p proxysql down -v --remove-orphans
-	@docker compose -p proxysql up $(IMG_NAME)$(IMG_TYPE)$(IMG_COMP)_build
-	@docker compose -p proxysql down -v --remove-orphans
+	@docker compose -p "${GIT_VERSION/./}" down -v --remove-orphans
+	@docker compose -p "${GIT_VERSION/./}" up $(IMG_NAME)$(IMG_TYPE)$(IMG_COMP)_build
+	@docker compose -p "${GIT_VERSION/./}" down -v --remove-orphans
 
 
 ### clean targets

deps/Makefile

@@ -38,7 +38,7 @@ endif
 STDCPP := -std=c++$(shell echo $(CPLUSPLUS) | cut -c3-4) -DCXX$(shell echo $(CPLUSPLUS) | cut -c3-4)
 
 
-targets := libconfig libdaemon jemalloc mariadb_client re2 sqlite3 pcre lz4 cityhash microhttpd curl ev libssl libhttpserver libinjection prometheus-cpp
+targets := libconfig libdaemon jemalloc mariadb_client re2 sqlite3 pcre lz4 cityhash microhttpd curl ev libssl libhttpserver libinjection prometheus-cpp postgresql libusual libscram
 ifeq ($(SYS_KERN),Linux)
 	targets += coredumper
 endif
@@ -128,7 +128,8 @@ curl/curl/lib/.libs/libcurl.a: libssl/openssl/libssl.a
 ifeq ($(SYS_KERN),Darwin)
 	cd curl/curl && patch configure < ../configure.patch
 endif
-	cd curl/curl && CFLAGS=-fPIC ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli --without-librtmp --without-libpsl --without-zstd --with-ssl=$(shell pwd)/libssl/openssl/ --enable-shared=yes && LD_LIBRARY_PATH=$(shell pwd)/libssl/openssl CC=${CC} CXX=${CXX} ${MAKE}
+	cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli --without-librtmp --without-libpsl --without-zstd --with-ssl=$(shell pwd)/libssl/openssl/ --enable-shared=yes
+	cd curl/curl && LD_LIBRARY_PATH=$(shell pwd)/libssl/openssl CFLAGS=-fPIC CC=${CC} CXX=${CXX} ${MAKE}
 
 curl: curl/curl/lib/.libs/libcurl.a
 
@@ -197,7 +198,7 @@ mariadb-client-library/mariadb_client/libmariadb/libmariadbclient.a: libssl/open
 	cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true
 	cd mariadb-client-library && tar -zxf mariadb-connector-c-3.3.8-src.tar.gz
 	cd mariadb-client-library/mariadb_client && patch -p0 < ../plugin_auth_CMakeLists.txt.patch
-	cd mariadb-client-library/mariadb_client && cmake . -Wno-dev -DOPENSSL_ROOT_DIR=$(shell pwd)/libssl/openssl/ -DOPENSSL_LIBRARIES=$(shell pwd)/libssl/openssl/ -DICONV_LIBRARIES=$(brew --prefix libiconv)/lib -DICONV_INCLUDE=$(brew --prefix libiconv)/include .
+	cd mariadb-client-library/mariadb_client && cmake . -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOPENSSL_ROOT_DIR=$(shell pwd)/libssl/openssl/ -DOPENSSL_LIBRARIES=$(shell pwd)/libssl/openssl/ -DICONV_LIBRARIES=$(brew --prefix libiconv)/lib -DICONV_INCLUDE=$(brew --prefix libiconv)/include .
 ifeq ($(PROXYDEBUG),1)
 	cd mariadb-client-library/mariadb_client && patch -p0 < ../ma_context.h.patch
 else ifeq ($(USEVALGRIND),1)
@@ -254,7 +255,6 @@ sqlite3/sqlite3/sqlite3.o:
 
 sqlite3: sqlite3/sqlite3/sqlite3.o
 
-
 libconfig/libconfig/lib/.libs/libconfig++.a:
 	cd libconfig && rm -rf libconfig-*/ || true
 	cd libconfig && tar -zxf libconfig-*.tar.gz
@@ -304,13 +304,40 @@ pcre/pcre/.libs/libpcre.a:
 
 pcre: pcre/pcre/.libs/libpcre.a
 
+postgresql/postgresql/src/interfaces/libpq/libpq.a : libssl/openssl/libssl.a
+	cd postgresql && rm -rf postgresql-*/ || true
+	cd postgresql && tar -zxf postgresql-*.tar.gz
+	cd postgresql/postgresql && patch -p0 < ../get_result_from_pgconn.patch
+	cd postgresql/postgresql && patch -p0 < ../handle_row_data.patch
+	#cd postgresql/postgresql && LD_LIBRARY_PATH="$(shell pwd)/libssl/openssl" ./configure --with-ssl=openssl --with-includes="$(shell pwd)/libssl/openssl/include/" --with-libraries="$(shell pwd)/libssl/openssl/" --without-readline --enable-debug  CFLAGS="-ggdb -O0 -fno-omit-frame-pointer" CPPFLAGS="-g -O0"
+	cd postgresql/postgresql && LD_LIBRARY_PATH="$(shell pwd)/libssl/openssl" ./configure --with-ssl=openssl --with-includes="$(shell pwd)/libssl/openssl/include/" --with-libraries="$(shell pwd)/libssl/openssl/" --without-readline
+	cd postgresql/postgresql/src/interfaces/libpq && CC=${CC} CXX=${CXX} ${MAKE} MAKELEVEL=0
+	#cd postgresql/postgresql && CC=${CC} CXX=${CXX} ${MAKE} -f src/interfaces/libpq/Makefile all
+
+postgresql: postgresql/postgresql/src/interfaces/libpq/libpq.a
+
+libusual/libusual/.libs/libusual.a: libssl/openssl/libssl.a
+	cd libusual && rm -rf libusual-*/ || true
+	cd libusual && tar -zxf libusual-*.tar.gz
+	cd libusual/libusual && ./autogen.sh
+	cd libusual/libusual && ./configure --with-openssl="$(shell pwd)/libssl/openssl/" --disable-shared
+	cd libusual/libusual && CC=${CC} CXX=${CXX} ${MAKE}
+
+libusual: libusual/libusual/.libs/libusual.a
+
+libscram/lib/libscram.a: libssl/openssl/libssl.a postgresql/postgresql/src/interfaces/libpq/libpq.a 
+	cd libscram && rm -rf lib/* || true
+	cd libscram && CC=${CC} CXX=${CXX} ${MAKE} LIBOPENSSL_DIR="$(shell pwd)/libssl/openssl" POSTGRESQL_DIR="$(shell pwd)/postgresql/postgresql/"
+
+libscram: libscram/lib/libscram.a
 
 ### clean targets
 
 cleanpart:
 	cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true
 	cd jemalloc && rm -rf jemalloc-*/ || true
 	cd sqlite3 && rm -rf sqlite-amalgamation-*/ || true
+	cd postgresql && rm -rf postgresql-*/ || true
 .PHONY: cleanpart
 
 cleanall:
@@ -333,5 +360,8 @@ cleanall:
 	cd prometheus-cpp && rm -rf prometheus-cpp-*/ || true
 	cd cityhash && rm -rf cityhash-*/ || true
 	cd coredumper && rm -rf coredumper-*/ || true
+	cd postgresql && rm -rf postgresql-*/ || true
+	cd libusual && rm -rf libusual-*/ || true
+	cd libscram && rm -rf lib/* obj/* || true
 .PHONY: cleanall
 

deps/libscram/Makefile

@@ -0,0 +1,60 @@
+CC ?= gcc
+CFLAGS_common = -Wall -Iinclude
+CFLAGS_DEBUG = -g -O0
+CFLAGS_RELEASE = -O3
+SRC_DIR = src
+OBJ_DIR_DEBUG = obj/debug
+OBJ_DIR_RELEASE = obj/release
+LIB_DIR = lib
+TARGET_DEBUG = $(LIB_DIR)/libscram.a
+TARGET_RELEASE = $(LIB_DIR)/libscram.a
+
+# Library directory passed as a parameter
+LIBUSUAL_DIR ?= /path/to/libusual
+
+# OpenSSL dir
+LIBOPENSSL_DIR ?= /path/to/openssl
+
+# PostgreSQL dir
+POSTGRESQL_DIR ?= /path/to/postgresql
+POSTGRESQL_INCLUDE_DIR = $(POSTGRESQL_DIR)/src/include/
+
+# List all source files
+SRC_FILES := $(wildcard $(SRC_DIR)/*.c)
+
+# Generate object file names
+OBJ_FILES_DEBUG := $(patsubst $(SRC_DIR)/%.c,$(OBJ_DIR_DEBUG)/%.o,$(SRC_FILES))
+OBJ_FILES_RELEASE := $(patsubst $(SRC_DIR)/%.c,$(OBJ_DIR_RELEASE)/%.o,$(SRC_FILES))
+
+# Check if ASAN is enabled
+ifdef ASAN
+CFLAGS_DEBUG += -fsanitize=address
+endif
+
+# Define default target
+all: $(TARGET_RELEASE)
+
+DEBUG: $(TARGET_DEBUG)
+
+RELEASE: $(TARGET_RELEASE)
+
+$(OBJ_DIR_DEBUG)/%.o: $(SRC_DIR)/%.c
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS_common) $(CFLAGS_DEBUG) -I$(LIBUSUAL_DIR) -I$(LIBOPENSSL_DIR) -I$(POSTGRESQL_INCLUDE_DIR) -c $< -o $@
+
+$(OBJ_DIR_RELEASE)/%.o: $(SRC_DIR)/%.c
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS_common) $(CFLAGS_RELEASE) -I$(LIBUSUAL_DIR) -I$(LIBOPENSSL_DIR) -I$(POSTGRESQL_INCLUDE_DIR) -c $< -o $@
+
+$(TARGET_DEBUG): $(OBJ_FILES_DEBUG)
+	@mkdir -p $(@D)
+	ar rcs $@ $^
+
+$(TARGET_RELEASE): $(OBJ_FILES_RELEASE)
+	@mkdir -p $(@D)
+	ar rcs $@ $^
+
+clean:
+	rm -rf $(OBJ_DIR_DEBUG)/* $(OBJ_DIR_RELEASE)/* $(LIB_DIR)/*
+
+.PHONY: all DEBUG RELEASE clean

deps/libscram/include/scram-internal.h

@@ -0,0 +1,19 @@
+
+/*
+ * Required system headers
+ */
+#include <stdint.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <assert.h>
+#include <string.h>
+
+#define int8	int8_t
+#define uint8	uint8_t
+#define uint16	uint16_t
+#define uint32	uint32_t
+#define uint64	uint64_t
+
+#define SCRAM_KEY_LEN			 SCRAM_SHA_256_KEY_LEN
+#define SCRAM_DEFAULT_ITERATIONS SCRAM_SHA_256_DEFAULT_ITERATIONS

deps/libscram/include/scram.h

@@ -0,0 +1,129 @@
+/*
+ * PgBouncer - Lightweight connection pooler for PostgreSQL.
+ *
+ * Copyright (c) 2007-2009  Marko Kreen, Skype Technologies OÜ
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+ /*
+  * Ought to match NAMEDATALEN.  Some cloud services use longer user
+  * names, so give it some extra room.
+  */
+#define MAX_USERNAME    128
+
+  /*
+   * Some cloud services use very long generated passwords, so give it
+   * plenty of room.
+   */
+#define MAX_PASSWORD    2048
+
+struct ScramState {
+	char* client_nonce;
+	char* client_first_message_bare;
+	char* client_final_message_without_proof;
+	char* server_nonce;
+	char* server_first_message;
+	uint8_t* SaltedPassword;
+	char cbind_flag;
+	bool adhoc;	/* SCRAM data made up from plain-text password */
+	int iterations;
+	char* salt;	/* base64-encoded */
+	uint8_t ClientKey[32];	/* SHA256_DIGEST_LENGTH */
+	uint8_t StoredKey[32];
+	uint8_t ServerKey[32];
+};
+
+struct PgCredentials {
+	char name[MAX_USERNAME];
+	char passwd[MAX_PASSWORD];
+	uint8_t scram_ClientKey[32];
+	uint8_t scram_ServerKey[32];
+	bool has_scram_keys;		/* true if the above two are valid */
+	bool mock_auth;
+};
+
+typedef struct ScramState ScramState;
+typedef struct PgCredentials PgCredentials;
+
+typedef enum PasswordType {
+	PASSWORD_TYPE_PLAINTEXT = 0,
+	PASSWORD_TYPE_MD5,
+	PASSWORD_TYPE_SCRAM_SHA_256
+} PasswordType;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	PasswordType get_password_type(const char* shadow_pass);
+
+	// Returns the last error message
+	const char* scram_error();
+
+	// Resets the error message
+	void scram_reset_error();
+
+	// Initializes a new ScramState object
+	ScramState* scram_state_init();
+
+	// Frees the memory allocated for a ScramState object
+	void free_scram_state(ScramState* scram_state);
+
+	/*
+	 * Functions for communicating as a client with the server
+	 */
+	char *build_client_first_message(ScramState *scram_state);
+
+	char *build_client_final_message(ScramState *scram_state,
+					 const PgCredentials *credentials,
+					 const char *server_nonce,
+					 const char *salt,
+					 int saltlen,
+					 int iterations);
+
+	bool read_server_first_message(ScramState* scram_state, char *input,
+					   char **server_nonce_p, char **salt_p, int *saltlen_p, int *iterations_p);
+
+	bool read_server_final_message(char *input, char *ServerSignature);
+
+	bool verify_server_signature(ScramState *scram_state, const PgCredentials *credentials, const char *ServerSignature);
+
+
+	/*
+	 * Functions for communicating as a server to the client
+	 */
+
+	bool read_client_first_message(char *input,
+					   char *cbind_flag_p,
+					   char **client_first_message_bare_p,
+					   char **client_nonce_p);
+
+	bool read_client_final_message(ScramState* scram_state, const uint8_t *raw_input, char *input,
+					   const char **client_final_nonce_p,
+					   char **proof_p);
+
+	char *build_server_first_message(ScramState *scram_state,
+					 const char *username, const char *stored_secret);
+
+	char *build_server_final_message(ScramState *scram_state);
+
+	bool verify_final_nonce(const ScramState *scram_state, const char *client_final_nonce);
+
+	bool verify_client_proof(ScramState *state, const char *ClientProof);
+
+	bool scram_verify_plain_password(const char *username, const char *password, const char *secret);
+
+#ifdef __cplusplus
+}
+#endif