Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OrmMigrationAndLogger #1988

Merged
merged 26 commits into from
Oct 18, 2024
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
e47d934
update migrations
PooyaRaki Sep 23, 2024
50fbd3b
registers typeorm manually, restructure datasources
PooyaRaki Sep 27, 2024
5c0a545
Add database migration lock, Add database custom query logger
PooyaRaki Oct 1, 2024
da7191b
Add tests for new database module
PooyaRaki Oct 2, 2024
474e023
Remove migration folder from root folders in package.json
PooyaRaki Oct 2, 2024
02d2403
Remove dist folder from test coverage report
PooyaRaki Oct 2, 2024
d912044
Destroys database connection in tests
PooyaRaki Oct 2, 2024
d459830
Add more tests
PooyaRaki Oct 2, 2024
2bb5427
Refactor database migrator
PooyaRaki Oct 2, 2024
c96e8f7
Add Targeted Messaging Datasource (#1975)
hectorgomezv Sep 30, 2024
af169bf
update migrations
PooyaRaki Sep 23, 2024
db940af
registers typeorm manually, restructure datasources
PooyaRaki Sep 27, 2024
521c4c8
move typeorm config values to the configuration file
PooyaRaki Sep 27, 2024
1dbff19
Add database migration lock, Add database custom query logger
PooyaRaki Oct 1, 2024
884f81c
Add Postgres Database V2 Mock to Controller Tests
PooyaRaki Oct 14, 2024
d926220
Add typeorm migration, Log only the query without parameters
PooyaRaki Oct 15, 2024
f7da627
Add unit tests
PooyaRaki Oct 15, 2024
3f9292a
Add comment about the reason we are not using pg_advisory_lock
PooyaRaki Oct 15, 2024
2ea0e5d
Refactor unit tests, initializes postgres database on module startup,…
PooyaRaki Oct 16, 2024
a294250
refactor postgres database unit tests
PooyaRaki Oct 16, 2024
ff1c5a4
remove the redundant redefinition of the migration name.
PooyaRaki Oct 16, 2024
e907e3c
refactor test, remove redundant async/await, remove redundant comment
PooyaRaki Oct 17, 2024
cfb1f8c
add comments to migration config values
PooyaRaki Oct 17, 2024
378fdd1
rename retryAfter to retryAfterMs, change runMigraiton condition, rem…
PooyaRaki Oct 17, 2024
905b0aa
Revert coverage folders
PooyaRaki Oct 17, 2024
3066835
remove a test
PooyaRaki Oct 17, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions migrations/1726752966034-notification.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import type { MigrationInterface, QueryRunner } from 'typeorm';

export class Notification1726752966034 implements MigrationInterface {
name = 'Notification1726752966034';

public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "push_notification_devices" ("id" SERIAL NOT NULL, "device_type" character varying(255) NOT NULL CHECK (device_type IN ('ANDROID', 'IOS', 'WEB')), "device_uuid" uuid NOT NULL, "cloud_messaging_token" character varying(255) NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updated_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "device_uuid" UNIQUE ("device_uuid"), CONSTRAINT "PK_e387f5cc5b4f66d63804d596c64" PRIMARY KEY ("id"))`,
`CREATE TABLE "push_notification_devices" ("id" SERIAL NOT NULL, "device_type" character varying(255) NOT NULL, "device_uuid" uuid NOT NULL, "cloud_messaging_token" character varying(255) NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updated_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "device_uuid" UNIQUE ("device_uuid"), CONSTRAINT "PK_e387f5cc5b4f66d63804d596c64" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE TABLE "notification_subscriptions" ("id" SERIAL NOT NULL, "chain_id" character varying(255) NOT NULL, "safe_address" character varying(42) NOT NULL, "signer_address" character varying(42), "created_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updated_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "pushNotificationDeviceId" integer, CONSTRAINT "UQ_3c2531929422835e4f2717ec5db" UNIQUE ("chain_id", "safe_address", "pushNotificationDeviceId", "signer_address"), CONSTRAINT "PK_8cfec5d2a549ff20d1f4e648226" PRIMARY KEY ("id"))`,
Expand Down
59 changes: 59 additions & 0 deletions migrations/1727451367471-notifications_enum.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import { MigrationInterface, QueryRunner } from 'typeorm';

export class NotificationsEnum1727451367471 implements MigrationInterface {
name = 'NotificationsEnum1727451367471';

public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "notification_types" DROP CONSTRAINT "name"`,
);
await queryRunner.query(
`ALTER TABLE "notification_types" DROP COLUMN "name"`,
);
await queryRunner.query(
`CREATE TYPE "public"."notification_types_name_enum" AS ENUM('CONFIRMATION_REQUEST', 'DELETED_MULTISIG_TRANSACTION', 'EXECUTED_MULTISIG_TRANSACTION', 'INCOMING_ETHER', 'INCOMING_TOKEN', 'MODULE_TRANSACTION', 'MESSAGE_CONFIRMATION_REQUEST')`,
);
await queryRunner.query(
`ALTER TABLE "notification_types" ADD "name" "public"."notification_types_name_enum" NOT NULL`,
);
await queryRunner.query(
`ALTER TABLE "notification_types" ADD CONSTRAINT "UQ_1d7eaa0dcf0fbfd0a8e6bdbc9c9" UNIQUE ("name")`,
hectorgomezv marked this conversation as resolved.
Show resolved Hide resolved
);
await queryRunner.query(
`ALTER TABLE "push_notification_devices" DROP COLUMN "device_type"`,
);
await queryRunner.query(
`CREATE TYPE "public"."push_notification_devices_device_type_enum" AS ENUM('ANDROID', 'IOS', 'WEB')`,
);
await queryRunner.query(
`ALTER TABLE "push_notification_devices" ADD "device_type" "public"."push_notification_devices_device_type_enum" NOT NULL`,
);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "push_notification_devices" DROP COLUMN "device_type"`,
);
await queryRunner.query(
`DROP TYPE "public"."push_notification_devices_device_type_enum"`,
);
await queryRunner.query(
`ALTER TABLE "push_notification_devices" ADD "device_type" character varying(255) NOT NULL`,
);
await queryRunner.query(
`ALTER TABLE "notification_types" DROP CONSTRAINT "UQ_1d7eaa0dcf0fbfd0a8e6bdbc9c9"`,
);
await queryRunner.query(
`ALTER TABLE "notification_types" DROP COLUMN "name"`,
);
await queryRunner.query(
`DROP TYPE "public"."notification_types_name_enum"`,
);
await queryRunner.query(
`ALTER TABLE "notification_types" ADD "name" character varying(255) NOT NULL`,
);
await queryRunner.query(
`ALTER TABLE "notification_types" ADD CONSTRAINT "name" UNIQUE ("name")`,
);
}
}
19 changes: 19 additions & 0 deletions migrations/1727701600427-update_timestamp_trigger.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { MigrationInterface, QueryRunner } from 'typeorm';

export class UpdateTimestampTrigger1727701600427 implements MigrationInterface {
name = 'UpdateTimestampTrigger1727701600427';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
CREATE FUNCTION update_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = now();
RETURN NEW;
END;
$$ language 'plpgsql';`);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP FUNCTION IF EXISTS update_updated_at;`);
}
}
33 changes: 33 additions & 0 deletions migrations/1727701873513-notification_update_updated_at.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { MigrationInterface, QueryRunner } from 'typeorm';

export class NotificationUpdateUpdatedAt1727701873513
implements MigrationInterface
{
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
CREATE TRIGGER update_push_notification_devices_updated_at
BEFORE UPDATE
ON
push_notification_devices
FOR EACH ROW
EXECUTE PROCEDURE update_updated_at();
`);
await queryRunner.query(`
CREATE TRIGGER update_notification_subscriptions_updated_at
BEFORE UPDATE
ON
notification_subscriptions
FOR EACH ROW
EXECUTE PROCEDURE update_updated_at();
`);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`DROP TRIGGER IF EXISTS update_push_notification_devices_updated_at ON push_notification_devices;`,
);
await queryRunner.query(
`DROP TRIGGER IF EXISTS update_notification_subscriptions_updated_at ON notification_subscriptions;`,
);
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { MigrationInterface, QueryRunner } from 'typeorm';

export class NotificationTypes1726754508107 implements MigrationInterface {
export class NotificationTypes1727702843994 implements MigrationInterface {
name = 'NotificationTypes1727702843994';
hectorgomezv marked this conversation as resolved.
Show resolved Hide resolved
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`INSERT INTO "notification_types" ("name") VALUES ('CONFIRMATION_REQUEST'),('DELETED_MULTISIG_TRANSACTION'),('EXECUTED_MULTISIG_TRANSACTION'),('INCOMING_ETHER'),('INCOMING_TOKEN'),('MESSAGE_CONFIRMATION_REQUEST'),('MODULE_TRANSACTION');`,
Expand Down
11 changes: 11 additions & 0 deletions migrations/tests/1727702843993-test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { MigrationInterface, QueryRunner } from 'typeorm';

export class NotificationTypes1727702843994 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`SELECT now() as current_timestamp`);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`SELECT now() as current_timestamp`);
}
}
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,7 @@
"^.+\\.(t|j)s$": "ts-jest"
},
"roots": [
"./src",
"./migrations"
iamacook marked this conversation as resolved.
Show resolved Hide resolved
"./src"
],
"collectCoverageFrom": [
"**/*.(t|j)s"
Expand All @@ -106,7 +105,8 @@
"index.ts",
".+\\/__tests__\\/.+\\.builder.(t|j)s",
".+\\/__tests__\\/.+\\.factory.(t|j)s",
".e2e-spec.ts"
".e2e-spec.ts",
"dist"
iamacook marked this conversation as resolved.
Show resolved Hide resolved
],
"testEnvironment": "node",
"moduleNameMapper": {
Expand Down
7 changes: 5 additions & 2 deletions src/__tests__/db.factory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ export class TestDbFactory {
private readonly mainConnection: postgres.Sql;

constructor() {
this.mainConnection = this.connect(this.config.db.postgres.database);
this.mainConnection = this.connect(
this.config.db.connection.postgres.database,
);
}

async createTestDatabase(dbName: string): Promise<postgres.Sql> {
Expand All @@ -38,7 +40,8 @@ export class TestDbFactory {
* @returns {@link postgres.Sql} pointing to the database
*/
private connect(dbName: string): postgres.Sql {
const { host, port, username, password, ssl } = this.config.db.postgres;
const { host, port, username, password, ssl } =
this.config.db.connection.postgres;
const sslEnabled = !this.isCIContext && ssl.enabled;
return postgres({
host,
Expand Down
16 changes: 12 additions & 4 deletions src/app.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ import { TargetedMessagingModule } from '@/routes/targeted-messaging/targeted-me
import { ConfigModule, ConfigService } from '@nestjs/config';
import { TypeOrmModule } from '@nestjs/typeorm';
import { postgresConfig } from '@/config/entities/postgres.config';
import {
LoggingService,
type ILoggingService,
} from '@/logging/logging.interface';

@Module({})
export class AppModule implements NestModule {
Expand Down Expand Up @@ -131,18 +135,22 @@ export class AppModule implements NestModule {
}),
TypeOrmModule.forRootAsync({
imports: [ConfigModule],
iamacook marked this conversation as resolved.
Show resolved Hide resolved
useFactory: async (configService: ConfigService) => {
const typeormConfig = await configService.getOrThrow('typeorm');
useFactory: async (
configService: ConfigService,
loggingService: ILoggingService,
) => {
const typeormConfig = configService.getOrThrow('db.orm');
const postgresConfigObject = postgresConfig(
await configService.getOrThrow('db.postgres'),
await configService.getOrThrow('db.connection.postgres'),
iamacook marked this conversation as resolved.
Show resolved Hide resolved
loggingService,
);

return {
...typeormConfig,
...postgresConfigObject,
};
},
inject: [ConfigService],
inject: [ConfigService, LoggingService],
}),
],
providers: [
Expand Down
40 changes: 26 additions & 14 deletions src/config/entities/__tests__/configuration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export default (): ReturnType<typeof configuration> => ({
},
application: {
isProduction: faker.datatype.boolean(),
runMigrations: true,
runMigrations: false,
iamacook marked this conversation as resolved.
Show resolved Hide resolved
port: faker.internet.port().toString(),
},
auth: {
Expand Down Expand Up @@ -84,20 +84,32 @@ export default (): ReturnType<typeof configuration> => ({
apiKey: faker.string.hexadecimal({ length: 32 }),
},
},
typeorm: { autoLoadEntities: true, manualInitialization: true },
db: {
postgres: {
schema: process.env.POSTGRES_SCHEMA || 'main',
host: process.env.POSTGRES_TEST_HOST || 'localhost',
port: process.env.POSTGRES_TEST_PORT || '5433',
database: process.env.POSTGRES_TEST_DB || 'test-db',
username: process.env.POSTGRES_TEST_USER || 'postgres',
password: process.env.POSTGRES_TEST_PASSWORD || 'postgres',
ssl: {
enabled: true,
requestCert: true,
rejectUnauthorized: true,
caPath: process.env.POSTGRES_SSL_CA_PATH,
migrator: {
migrationsExecute: true,
iamacook marked this conversation as resolved.
Show resolved Hide resolved
numberOfRetries: process.env.DB_TEST_MIGRATIONS_NUMBER_OF_RETRIES ?? 5,
retryAfter: process.env.DB_TEST_MIGRATIONS_RETRY_AFTER ?? 1000, // Milliseconds
},
orm: {
autoLoadEntities: true,
manualInitialization: true,
migrationsRun: false,
iamacook marked this conversation as resolved.
Show resolved Hide resolved
migrationsTableName: 'migrations_typeorm',
iamacook marked this conversation as resolved.
Show resolved Hide resolved
},
connection: {
postgres: {
schema: process.env.POSTGRES_SCHEMA || 'main',
host: process.env.POSTGRES_TEST_HOST || 'localhost',
port: process.env.POSTGRES_TEST_PORT || '5433',
database: process.env.POSTGRES_TEST_DB || 'test-db',
username: process.env.POSTGRES_TEST_USER || 'postgres',
password: process.env.POSTGRES_TEST_PASSWORD || 'postgres',
ssl: {
enabled: true,
requestCert: true,
rejectUnauthorized: true,
caPath: process.env.POSTGRES_SSL_CA_PATH,
},
},
},
},
Expand Down
56 changes: 35 additions & 21 deletions src/config/entities/configuration.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Custom configuration for the application

// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
export default () => ({
about: {
Expand Down Expand Up @@ -44,8 +45,8 @@ export default () => ({
application: {
isProduction: process.env.CGW_ENV === 'production',
// Enables/disables the execution of migrations on startup.
// Defaults to true.
runMigrations: process.env.RUN_MIGRATIONS?.toLowerCase() !== 'false',
// Defaults to false.
runMigrations: process.env.RUN_MIGRATIONS?.toLowerCase() === 'true',
hectorgomezv marked this conversation as resolved.
Show resolved Hide resolved
port: process.env.APPLICATION_PORT || '3000',
},
auth: {
Expand Down Expand Up @@ -131,26 +132,39 @@ export default () => ({
apiKey: process.env.INFURA_API_KEY,
},
},
typeorm: { autoLoadEntities: true, manualInitialization: true },
db: {
postgres: {
host: process.env.POSTGRES_HOST || 'localhost',
port: process.env.POSTGRES_PORT || '5432',
database: process.env.POSTGRES_DB || 'safe-client-gateway',
schema: process.env.POSTGRES_SCHEMA || 'main', //@TODO: use this schema
username: process.env.POSTGRES_USER || 'postgres',
password: process.env.POSTGRES_PASSWORD || 'postgres',
ssl: {
enabled: process.env.POSTGRES_SSL_ENABLED?.toLowerCase() === 'true',
requestCert:
process.env.POSTGRES_SSL_REQUEST_CERT?.toLowerCase() !== 'false',
// If the value is not explicitly set to false, default should be true
// If not false the server will reject any connection which is not authorized with the list of supplied CAs
// https://nodejs.org/docs/latest-v20.x/api/tls.html#tlscreateserveroptions-secureconnectionlistener
rejectUnauthorized:
process.env.POSTGRES_SSL_REJECT_UNAUTHORIZED?.toLowerCase() !==
'false',
caPath: process.env.POSTGRES_SSL_CA_PATH,
migrator: {
migrationsExecute:
process.env.DB_MIGRATIONS_EXECUTE?.toLowerCase() !== 'false',
numberOfRetries: process.env.DB_MIGRATIONS_NUMBER_OF_RETRIES ?? 5,
retryAfter: process.env.DB_MIGRATIONS_RETRY_AFTER ?? 1000, // Milliseconds
hectorgomezv marked this conversation as resolved.
Show resolved Hide resolved
},
orm: {
migrationsRun: false,
autoLoadEntities: true,
manualInitialization: true,
migrationsTableName: 'migrations_typeorm',
},
connection: {
postgres: {
host: process.env.POSTGRES_HOST || 'localhost',
port: process.env.POSTGRES_PORT || '5432',
database: process.env.POSTGRES_DB || 'safe-client-gateway',
schema: process.env.POSTGRES_SCHEMA || 'main', //@TODO: use this schema
username: process.env.POSTGRES_USER || 'postgres',
password: process.env.POSTGRES_PASSWORD || 'postgres',
ssl: {
enabled: process.env.POSTGRES_SSL_ENABLED?.toLowerCase() === 'true',
requestCert:
process.env.POSTGRES_SSL_REQUEST_CERT?.toLowerCase() !== 'false',
// If the value is not explicitly set to false, default should be true
// If not false the server will reject any connection which is not authorized with the list of supplied CAs
// https://nodejs.org/docs/latest-v20.x/api/tls.html#tlscreateserveroptions-secureconnectionlistener
rejectUnauthorized:
process.env.POSTGRES_SSL_REJECT_UNAUTHORIZED?.toLowerCase() !==
'false',
caPath: process.env.POSTGRES_SSL_CA_PATH,
},
},
},
},
Expand Down
9 changes: 6 additions & 3 deletions src/config/entities/orm.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ import { DataSource } from 'typeorm';
import configuration from '@/config/entities/configuration';
import { postgresConfig } from '@/config/entities/postgres.config';

export default new DataSource(
postgresConfig({ ...configuration().db.postgres, type: 'postgres' }),
);
export default new DataSource({
...postgresConfig({
...configuration().db.connection.postgres,
type: 'postgres',
}),
});
5 changes: 4 additions & 1 deletion src/config/entities/postgres.config.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { PostgresqlLogger } from '@/datasources/db/v2/postgresql-logger.service';
import type { ILoggingService } from '@/logging/logging.interface';
import { readFileSync } from 'fs';
import type { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions';

Expand All @@ -18,6 +20,7 @@ interface IPostgresEnvConfig {

export const postgresConfig = (
postgresEnvConfig: IPostgresEnvConfig,
logger?: ILoggingService,
): PostgresConnectionOptions => {
const isCIContext = process.env.CI?.toLowerCase() === 'true';
const isSslEnabled = !isCIContext && postgresEnvConfig.ssl.enabled;
Expand All @@ -37,7 +40,7 @@ export const postgresConfig = (
password: postgresEnvConfig.password,
database: postgresEnvConfig.database,
migrations: ['dist/migrations/*.js'],
// logging: !isCIContext,
logger: new PostgresqlLogger(undefined, logger),
ssl: isSslEnabled
? {
ca: postgresCa,
Expand Down
Loading