Skip to content

Commit

Permalink
Change DB structure [POC, WIP, DANGEROUS (READ DESCRIPTION)]
Browse files Browse the repository at this point in the history 
Please, be careful while testing this commit since you can lose your DB data.
Backup your DB before performing any testing.
It's not ready for release yet and just shows PoC.
Currently this commit renames DB Table to ...V2, it's made for ease of debug.
In the final form, it will remove original DB and rename V2 to original table's name.

Introduce new DB structure and DB migration mechanism.
New structure despite being unintuitive allows better maintainability,
performance and data integrity.

Better description and links to discussion will be added near the release here.
  • Loading branch information
@H3rnand3zzz
H3rnand3zzz committed Oct 26, 2023
1 parent 0664491 commit 8b85929
Showing 1 changed file with 174 additions and 32 deletions.
206 changes: 174 additions & 32 deletions src/database.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ static void _add_to_db(ProfMessage* message, char* type, const Jid* const from_j
static char* _get_db_filename(ProfAccount* account);
static prof_msg_type_t _get_message_type_type(const char* const type);
static prof_enc_t _get_message_enc_type(const char* const encstr);
static int _get_db_version(void);
static gboolean _migrate_to_v2(void);

#define auto_sqlite __attribute__((__cleanup__(auto_free_sqlite)))

Expand Down Expand Up @@ -97,12 +99,16 @@ log_database_init(ProfAccount* account)
}

char* err_msg;

// ChatLogs Table
// Contains visual representation of chat logs
//
// id is the ID of DB the entry
// from_jid is the senders jid
// to_jid is the receivers jid
// from_resource is the senders resource
// to_jid is the receivers resource
// message is the message text
// message is the message text -- may be replaced by LMC (XEP-0308)
// timestamp the timestamp like "2020/03/24 11:12:14"
// type is there to distinguish: message (chat), MUC message (muc), muc pm (mucpm)
// stanza_id is the ID in <message>
Expand All @@ -115,16 +121,34 @@ log_database_init(ProfAccount* account)
goto out;
}

query = "CREATE TABLE IF NOT EXISTS `DbVersion` ( `dv_id` INTEGER PRIMARY KEY, `version` INTEGER UNIQUE)";
query = "CREATE TABLE IF NOT EXISTS `DbVersion` (`dv_id` INTEGER PRIMARY KEY, `version` INTEGER UNIQUE)";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
goto out;
}

query = "INSERT OR IGNORE INTO `DbVersion` (`version`) VALUES('1')";
query = "INSERT OR IGNORE INTO `DbVersion` (`version`) VALUES ('1')";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
goto out;
}

// Check && Update DB based on version

int db_version = _get_db_version();
if(db_version == -1) {
cons_show_error("DB Initialization Error: Unable to check DB version.");
goto out;
}

if(db_version < 2) {
// TODO: don't cons_show on new installation
cons_show("Updating DB to V2. Please, wait...");
if(!_migrate_to_v2()){
cons_show_error("DB Initialization Error: Unable to migrate DB to V2. Please, check error logs for details.");
goto out;
}
cons_show("DB update was successful.");
}

log_debug("Initialized SQLite database: %s", filename);
return TRUE;

Expand Down Expand Up @@ -209,9 +233,9 @@ log_database_get_limits_info(const gchar* const contact_barejid, gboolean is_las
return NULL;

if (is_last) {
query = sqlite3_mprintf("SELECT * FROM (SELECT `archive_id`, `timestamp` from `ChatLogs` WHERE (`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q') ORDER BY `timestamp` DESC LIMIT 1) ORDER BY `timestamp` ASC;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid);
query = sqlite3_mprintf("SELECT * FROM (SELECT `archive_id`, `timestamp` from `ChatLogsV2` WHERE (`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q') ORDER BY `timestamp` DESC LIMIT 1) ORDER BY `timestamp` ASC;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid);
} else {
query = sqlite3_mprintf("SELECT * FROM (SELECT `archive_id`, `timestamp` from `ChatLogs` WHERE (`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q') ORDER BY `timestamp` ASC LIMIT 1) ORDER BY `timestamp` ASC;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid);
query = sqlite3_mprintf("SELECT * FROM (SELECT `archive_id`, `timestamp` from `ChatLogsV2` WHERE (`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q') ORDER BY `timestamp` ASC LIMIT 1) ORDER BY `timestamp` ASC;", contact_barejid, myjid->barejid, myjid->barejid, contact_barejid);
}

if (!query) {
Expand Down Expand Up @@ -258,15 +282,14 @@ log_database_get_previous_chat(const gchar* const contact_barejid, const char* s
GDateTime* now = g_date_time_new_now_local();
auto_gchar gchar* end_date_fmt = end_time ? end_time : g_date_time_format_iso8601(now);
auto_sqlite gchar* query = sqlite3_mprintf("SELECT * FROM ("
"SELECT COALESCE(B.`message`, A.`message`) AS message, "
"A.`timestamp`, A.`from_jid`, A.`type`, A.`encryption` FROM `ChatLogs` AS A "
"LEFT JOIN `ChatLogs` AS B ON (A.`stanza_id` = B.`replace_id` AND A.`from_jid` = B.`from_jid`) "
"WHERE A.`replace_id` = '' "
"AND ((A.`from_jid` = '%q' AND A.`to_jid` = '%q') OR (A.`from_jid` = '%q' AND A.`to_jid` = '%q')) "
"AND A.`timestamp` < '%q' "
"AND (%Q IS NULL OR A.`timestamp` > %Q) "
"ORDER BY A.`timestamp` %s LIMIT %d) "
"ORDER BY `timestamp` %s;",
"SELECT message, `timestamp`, `from_jid`, `type`, `encryption` "
"FROM `ChatLogsV2` "
"WHERE `replaces_stanza_id` = '' "
"AND ((`from_jid` = '%q' AND `to_jid` = '%q') OR (`from_jid` = '%q' AND `to_jid` = '%q')) "
"AND `timestamp` < '%q' "
"AND (%Q IS NULL OR `timestamp` > %Q) "
"ORDER BY `timestamp` %s LIMIT %d "
") ORDER BY `timestamp` %s;",
contact_barejid, myjid->barejid, myjid->barejid, contact_barejid, end_date_fmt, start_time, start_time, sort1, MESSAGES_TO_RETRIEVE, sort2);

g_date_time_unref(now);
Expand All @@ -278,7 +301,7 @@ log_database_get_previous_chat(const gchar* const contact_barejid, const char* s

int rc = sqlite3_prepare_v2(g_chatlog_database, query, -1, &stmt, NULL);
if (rc != SQLITE_OK) {
log_error("Unknown SQLite error in log_database_get_previous_chat()");
log_error("SQLite error in log_database_get_previous_chat(): %s", sqlite3_errmsg(g_chatlog_database));
return NULL;
}

Expand Down Expand Up @@ -375,6 +398,7 @@ static void
_add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Jid* const to_jid)
{
auto_gchar gchar* pref_dblog = prefs_get_string(PREF_DBLOG);
int original_message_id = 0;

if (g_strcmp0(pref_dblog, "off") == 0) {
return;
Expand Down Expand Up @@ -409,7 +433,7 @@ _add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Ji

// Check LMC validity (XEP-0308)
if (message->replace_id) {
auto_sqlite char* replace_check_query = sqlite3_mprintf("SELECT `from_jid` FROM `ChatLogs` WHERE `stanza_id` = '%q'",
auto_sqlite char* replace_check_query = sqlite3_mprintf("SELECT `id`, `from_jid` FROM `ChatLogsV2` WHERE `stanza_id` = '%q' ORDER BY `timestamp` DESC LIMIT 1",
message->replace_id ? message->replace_id : "");

if (!replace_check_query) {
Expand All @@ -419,25 +443,31 @@ _add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Ji

sqlite3_stmt* lmc_stmt = NULL;

if (SQLITE_OK == sqlite3_prepare_v2(g_chatlog_database, replace_check_query, -1, &lmc_stmt, NULL)) {
if (sqlite3_step(lmc_stmt) == SQLITE_ROW) {
const char* from_jid_orig = (const char*)sqlite3_column_text(lmc_stmt, 0);
if (SQLITE_OK != sqlite3_prepare_v2(g_chatlog_database, replace_check_query, -1, &lmc_stmt, NULL)) {
log_error("SQLite error in _add_to_db(): %s", sqlite3_errmsg(g_chatlog_database));
return;
}

if (g_strcmp0(from_jid_orig, from_jid->barejid) != 0) {
log_error("Mismatch in sender JIDs when trying to do LMC. Corrected message sender: %s. Original message sender: %s. Replace-ID: %s. Message: %s", from_jid->barejid, from_jid_orig, message->replace_id, message->plain);
cons_show_error("%s sent message correction with mismatched sender. See log for details.", from_jid->barejid);
sqlite3_finalize(lmc_stmt);
return;
}
if (sqlite3_step(lmc_stmt) == SQLITE_ROW) {
original_message_id = sqlite3_column_int(lmc_stmt, 0);
const char* from_jid_orig = (const char*)sqlite3_column_text(lmc_stmt, 1);

if (g_strcmp0(from_jid_orig, from_jid->barejid) != 0) {
log_error("Mismatch in sender JIDs when trying to do LMC. Corrected message sender: %s. Original message sender: %s. Replace-ID: %s. Message: %s", from_jid->barejid, from_jid_orig, message->replace_id, message->plain);
cons_show_error("%s sent message correction with mismatched sender. See log for details.", from_jid->barejid);
sqlite3_finalize(lmc_stmt);
return;
}
sqlite3_finalize(lmc_stmt);
// TODO: Replace original message here
} else {
// original message not found, do something
}
sqlite3_finalize(lmc_stmt);
}

// Check for duplicate messages
auto_sqlite char* duplicate_check_query = sqlite3_mprintf("SELECT 1 FROM `ChatLogs` WHERE (`archive_id` = '%q' AND `archive_id` != '') OR (`stanza_id` = '%q' AND `stanza_id` != '')",
message->stanzaid ? message->stanzaid : "",
message->id ? message->id : "");
auto_sqlite char* duplicate_check_query = sqlite3_mprintf("SELECT 1 FROM `ChatLogsV2` WHERE (`archive_id` = '%q' AND `archive_id` != '')",
message->stanzaid ? message->stanzaid : "");

if (!duplicate_check_query) {
log_error("Could not allocate memory for SQL duplicate query in log_database_add()");
Expand All @@ -455,12 +485,13 @@ _add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Ji
}

if (duplicate_exists) {
log_warning("Duplicate stanza-id found for the message. stanza_id: %s; archive_id: %s; sender: %s; content: %s", message->id, message->stanzaid, from_jid->barejid, message->plain);
return;
log_error("Duplicate server stanza-id found for the message. stanza_id: %s; archive_id: %s; sender: %s; content: %s", message->id, message->stanzaid, from_jid->barejid, message->plain);
// cons_show // ?
// return; //?
}

// Insert the message
auto_sqlite char* query = sqlite3_mprintf("INSERT INTO `ChatLogs` (`from_jid`, `from_resource`, `to_jid`, `to_resource`, `message`, `timestamp`, `stanza_id`, `archive_id`, `replace_id`, `type`, `encryption`) VALUES ('%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q')",
auto_sqlite char* query = sqlite3_mprintf("INSERT INTO `ChatLogsV2` (`from_jid`, `from_resource`, `to_jid`, `to_resource`, `message`, `timestamp`, `stanza_id`, `archive_id`, `replaces_db_id`, `replaces_stanza_id`, `type`, `encryption`) VALUES ('%q', '%q', '%q', '%q', '%q', '%q', '%q', '%q', %d, '%q', '%q', '%q')",
from_jid->barejid,
from_jid->resourcepart ? from_jid->resourcepart : "",
to_jid->barejid,
Expand All @@ -469,6 +500,7 @@ _add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Ji
date_fmt ? date_fmt : "",
message->id ? message->id : "",
message->stanzaid ? message->stanzaid : "",
original_message_id,
message->replace_id ? message->replace_id : "",
type ? type : "",
enc ? enc : "");
Expand All @@ -493,3 +525,113 @@ _add_to_db(ProfMessage* message, char* type, const Jid* const from_jid, const Ji
}
}
}

static int
_get_db_version(void){
int current_version = -1;
const char* query = "SELECT `version` FROM `DbVersion` LIMIT 1";
sqlite3_stmt* statement;

if (sqlite3_prepare_v2(g_chatlog_database, query, -1, &statement, NULL) == SQLITE_OK) {
if (sqlite3_step(statement) == SQLITE_ROW) {
current_version = sqlite3_column_int(statement, 0);
}

sqlite3_finalize(statement);
}
return current_version;
}

static gboolean
_migrate_to_v2(void){
// `replaced_stanza_id` stanza ID of the replaced message
// `replaced_db_id` db ID of the replaced message
// `original_message` original message before replacement

char* err_msg;

// Make V2 table
char* query = "CREATE TABLE `ChatLogsV2` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `from_jid` TEXT NOT NULL, `to_jid` TEXT NOT NULL, `from_resource` TEXT, `to_resource` TEXT, `message` TEXT, `timestamp` TEXT, `type` TEXT, `stanza_id` TEXT, `archive_id` TEXT, `encryption` TEXT, `marked_read` INTEGER, `original_message` TEXT, `replaces_db_id` INTEGER, `replaces_stanza_id` TEXT)";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to create V2 table.");
return FALSE;
}

// Select all messages form V1 to V2 (and replace_id->replaces_stanza_id)
query = "INSERT INTO `ChatLogsV2` (`from_jid`, `to_jid`, `from_resource`, `to_resource`, `message`, `timestamp`, `type`, `stanza_id`, `archive_id`, `replaces_stanza_id`, `encryption`, `marked_read`) "
"SELECT `from_jid`, `to_jid`, `from_resource`, `to_resource`, `message`, `timestamp`, `type`, `stanza_id`, `archive_id`, `replace_id` AS `replaces_stanza_id`, `encryption`, `marked_read`"
"FROM `ChatLogs`;";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to copy V2 table.");
goto cleanup;
}

// Save original message and DB ID in replacement messages
// Note: sender check is required due to #1898 (24d0030) since previous messages might be affected.
query = "UPDATE `ChatLogsV2` AS A "
"SET `original_message` = B.`message`, "
"`replaces_db_id` = B.`id` "
"FROM `ChatLogs` AS B "
"WHERE A.`replaces_stanza_id` IS NOT NULL AND A.`replaces_stanza_id` != '' AND A.`replaces_stanza_id` = B.`stanza_id` "
"AND A.`from_jid` = B.`from_jid` AND A.`to_jid` = B.`to_jid`;";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to add original messages to replacement messages in V2 table.");
goto cleanup;
}


// Set original message to redacted version
// Note: sender check is required due to #1898 (24d0030) since previous messages might be affected.
query = "UPDATE `ChatLogsV2` AS A "
"SET `message` = COALESCE(B.`message`, A.`message`)"
"FROM `ChatLogsV2` AS B "
"WHERE B.`replaces_db_id` IS NOT NULL AND B.`replaces_db_id` != 0 AND A.`id` = B.`replaces_db_id` "
"AND A.`from_jid` = B.`from_jid` AND A.`to_jid` = B.`to_jid`;";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to change messages in V2 table.");
goto cleanup;
}

// Cleanup LMC (they are going to be saved in the original message)
query = "UPDATE `ChatLogsV2` "
"SET `message` = NULL "
"WHERE `replaces_stanza_id` != '';";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to change messages in V2 table.");
goto cleanup;
}

query = "UPDATE `ChatLogsV2` "
"SET `message` = NULL "
"WHERE `replaces_stanza_id` != '';";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to change messages in V2 table.");
goto cleanup;
}

// Set new DB version (V2)
query = "UPDATE `DbVersion` SET `version` = 2";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to update DB Version.");
return FALSE; // TODO: or cleanup?
}

return TRUE;

cleanup:
if (err_msg) {
log_error("SQLite error: %s", err_msg);
sqlite3_free(err_msg);
err_msg = NULL;
} else {
log_error("Unknown SQLite error.");
}

query = "DROP TABLE `ChatLogsV2`;";
if (SQLITE_OK != sqlite3_exec(g_chatlog_database, query, NULL, 0, &err_msg)) {
log_error("[DB Migration] Unable to drop V2 table: %s", err_msg);
sqlite3_free(err_msg); // TODO: test this part, looks suspicious
}

return FALSE;
}

0 comments on commit 8b85929

Please sign in to comment.