Skip to content

Commit

Permalink
Replace node-sqlite3 with better-sqlite3
Browse files Browse the repository at this point in the history 
- Which provides transaction / rollback wrapper (I heared talk on IRC that we do everyting transactional... we didn't do anything transactional)
- Is synchronous, so we can test the state of the DB beforehand (instead of after we already told everyone it's ok, like wtf why did we even check for schema version when we don't do anything about missmatchs??!)
- much much faster
- provides more functionality we might use in the future, you never know ¯\_(ツ)_/¯
- less bugs (according to better-sqlite3 team)

Also read:
https://github.com/JoshuaWise/better-sqlite3#why-should-i-use-this-instead-of-node-sqlite3
WiseLibs/better-sqlite3#262 (comment)
  • Loading branch information
@Nachtalb
Nachtalb committed May 22, 2021
1 parent 43ffe52 commit 051cdfa
Show file tree
Hide file tree
Showing 4 changed files with 192 additions and 564 deletions.
2 changes: 1 addition & 1 deletion client/components/Windows/SearchResults.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ export default {
return [];
}
return this.search.results.slice().reverse();
return this.search.results.slice();
},
chan() {
const chanId = parseInt(this.$route.params.id, 10);
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@
"yarn": "1.22.10"
},
"optionalDependencies": {
"sqlite3": "5.0.2"
"better-sqlite3": "7.4.0"
},
"devDependencies": {
"@babel/core": "7.14.0",
Expand Down
234 changes: 106 additions & 128 deletions src/plugins/messageStorage/sqlite.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,12 @@ const fs = require("fs");
const Helper = require("../../helper");
const Msg = require("../../models/msg");

let sqlite3;
let BetterSqlite3;

try {
sqlite3 = require("sqlite3");
BetterSqlite3 = require("better-sqlite3");
} catch (e) {
Helper.config.messageStorage = Helper.config.messageStorage.filter((item) => item !== "sqlite");

log.error(
"Unable to load node-sqlite3 module. See https://github.com/mapbox/node-sqlite3/wiki/Binaries"
);
log.error("Unable to load better-sqlite3 module.");
}

const currentSchemaVersion = 1520239200;
Expand All @@ -35,7 +31,7 @@ class MessageStorage {
}

enable() {
if (!sqlite3) {
if (!BetterSqlite3) {
return false;
}

Expand All @@ -50,56 +46,50 @@ class MessageStorage {
}

const sqlitePath = path.join(logsPath, `${this.client.name}.sqlite3`);
this.database = new sqlite3.Database(sqlitePath);
this.database.serialize(() => {
schema.forEach((line) => this.database.run(line));

this.database.get(
"SELECT value FROM options WHERE name = 'schema_version'",
(err, row) => {
if (err) {
this.isEnabled = false;
return log.error(`Failed to retrieve schema version: ${err}`);
}

// New table
if (row === undefined) {
this.database.serialize(() =>
this.database.run(
"INSERT INTO options (name, value) VALUES ('schema_version', ?)",
currentSchemaVersion
)
);

return;
}

const storedSchemaVersion = parseInt(row.value, 10);

if (storedSchemaVersion === currentSchemaVersion) {
return;
}

if (storedSchemaVersion > currentSchemaVersion) {
this.isEnabled = false;
return log.error(
`sqlite messages schema version is higher than expected (${storedSchemaVersion} > ${currentSchemaVersion}). Is The Lounge out of date?`
);
}

log.info(
`sqlite messages schema version is out of date (${storedSchemaVersion} < ${currentSchemaVersion}). Running migrations if any.`
);

this.database.serialize(() =>
this.database.run(
"UPDATE options SET value = ? WHERE name = 'schema_version'",
currentSchemaVersion
)
);
this.database = new BetterSqlite3(sqlitePath);

try {
this.database.transaction((queries) => {
for (const query of queries) {
this.database.prepare(query).run();
}
);
});
})(schema);

const check = this.database
.prepare("SELECT value FROM options WHERE name = 'schema_version'")
.get();

const storedSchemaVersion = check ? parseInt(check.value, 10) : null;
let stmt;

if (storedSchemaVersion === null) {
stmt = this.database.prepare(
"INSERT INTO options (name, value) VALUES ('schema_version', ?)"
);
} else if (storedSchemaVersion > currentSchemaVersion) {
log.error(
`sqlite schema version is higher than expected (${storedSchemaVersion} > ${currentSchemaVersion}). Is The Lounge out of date?`
);
return false;
} else if (storedSchemaVersion < currentSchemaVersion) {
log.info(
`sqlite schema version is out of date (${storedSchemaVersion} < ${currentSchemaVersion}). Running migrations if any.`
);

stmt = this.database.prepare(
"UPDATE options SET value = ? WHERE name = 'schema_version'"
);
}

if (stmt) {
this.database.transaction(() => {
stmt.run(currentSchemaVersion.toString());
})();
}
} catch (error) {
log.error(`Failed to initialize sqltie database: ${error}`);
return false;
}

this.isEnabled = true;
return true;
Expand All @@ -110,17 +100,14 @@ class MessageStorage {
return;
}

this.isEnabled = false;

this.database.close((err) => {
if (err) {
log.error(`Failed to close sqlite database: ${err}`);
}
try {
this.database.close();
} catch (error) {
log.error(`Failed to close sqlite database: ${error}`);
}

if (callback) {
callback(err);
}
});
this.isEnabled = false;
callback();
}

index(network, channel, msg) {
Expand All @@ -139,30 +126,32 @@ class MessageStorage {
return newMsg;
}, {});

this.database.serialize(() =>
this.database.run(
"INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)",
const index = this.database.prepare(
"INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)"
);

this.database.transaction(() => {
index.run(
network.uuid,
channel.name.toLowerCase(),
msg.time.getTime(),
msg.type,
JSON.stringify(clonedMsg)
)
);
);
})();
}

deleteChannel(network, channel) {
if (!this.isEnabled) {
return;
}

this.database.serialize(() =>
this.database.run(
"DELETE FROM messages WHERE network = ? AND channel = ?",
network.uuid,
channel.name.toLowerCase()
)
const deleteStmt = this.database.prepare(
"DELETE FROM messages WHERE network = ? AND channel = ?"
);
this.database.transaction(() => {
deleteStmt.run(network.uuid, channel.name.toLowerCase());
})();
}

/**
Expand All @@ -180,30 +169,19 @@ class MessageStorage {
const limit = Helper.config.maxHistory < 0 ? 100000 : Helper.config.maxHistory;

return new Promise((resolve, reject) => {
this.database.serialize(() =>
this.database.all(
"SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?",
[network.uuid, channel.name.toLowerCase(), limit],
(err, rows) => {
if (err) {
return reject(err);
}

resolve(
rows.reverse().map((row) => {
const msg = JSON.parse(row.msg);
msg.time = row.time;
msg.type = row.type;

const newMsg = new Msg(msg);
newMsg.id = this.client.idMsg++;

return newMsg;
})
);
}
)
const selectStmt = this.database.prepare(
"SELECT * FROM messages WHERE network = ? AND channel = ? ORDER BY time ASC LIMIT ?"
);

try {
return resolve(
selectStmt
.all(network.uuid, channel.name.toLowerCase(), limit)
.map(this._messageParser(true))
);
} catch (error) {
return reject(error);
}
});
}

Expand All @@ -213,7 +191,7 @@ class MessageStorage {
}

let select =
'SELECT msg, type, time, network, channel FROM messages WHERE type = "message" AND json_extract(msg, "$.text") LIKE ?';
"SELECT * FROM messages WHERE type = 'message' AND json_extract(msg, '$.text') LIKE ?";
const params = [`%${query.searchTerm}%`];

if (query.networkUuid) {
Expand All @@ -226,46 +204,46 @@ class MessageStorage {
params.push(query.channelName.toLowerCase());
}

const maxResults = 100;
select += " ORDER BY time ASC LIMIT ? OFFSET ? ";
params.push(100);

select += " ORDER BY time DESC LIMIT ? OFFSET ? ";
params.push(maxResults);
query.offset = parseInt(query.offset, 10) || 0;
params.push(query.offset);

return new Promise((resolve, reject) => {
this.database.all(select, params, (err, rows) => {
if (err) {
reject(err);
} else {
const response = {
searchTerm: query.searchTerm,
target: query.channelName,
networkUuid: query.networkUuid,
offset: query.offset,
results: this._parseSearchRowsToMessages(query.offset, rows),
};
resolve(response);
}
});
try {
resolve({
searchTerm: query.searchTerm,
target: query.channelName,
networkUuid: query.networkUuid,
offset: query.offset,
results: this.database
.prepare(select)
.all(params)
.map(this._messageParser(false, query.offset)),
});
} catch (error) {
return reject(error);
}
});
}

_parseSearchRowsToMessages(id, rows) {
const messages = [];

for (const row of rows) {
_messageParser(useClientId, start) {
return (row) => {
const msg = JSON.parse(row.msg);
msg.time = row.time;
msg.type = row.type;
msg.networkUuid = row.network;
msg.channelName = row.channel;
msg.id = id;
messages.push(new Msg(msg));
id += 1;
}

return messages;
if (useClientId) {
msg.id = this.client.idMsg++;
} else {
msg.id = start++;
}

return new Msg(msg);
};
}
}

Expand Down
Loading

0 comments on commit 051cdfa

Please sign in to comment.