From 381ab35550dd03485cb6a9e930f4203b27e78e3d Mon Sep 17 00:00:00 2001 From: Mukhamediyar Kudaikulov Date: Thu, 25 Jul 2024 08:40:02 -0700 Subject: [PATCH 1/2] db only changes from mntor-3423 --- .../20240710214906_qa_custom_brokers.js | 39 +++ .../20240715110621_qa_custom_toggles.js | 26 ++ .../20240715115031_qa_custom_breaches.js | 36 +++ src/db/tables/onerep_scans.ts | 49 ++- src/db/tables/qa_customs.ts | 286 ++++++++++++++++++ 5 files changed, 422 insertions(+), 14 deletions(-) create mode 100644 src/db/migrations/20240710214906_qa_custom_brokers.js create mode 100644 src/db/migrations/20240715110621_qa_custom_toggles.js create mode 100644 src/db/migrations/20240715115031_qa_custom_breaches.js create mode 100644 src/db/tables/qa_customs.ts diff --git a/src/db/migrations/20240710214906_qa_custom_brokers.js b/src/db/migrations/20240710214906_qa_custom_brokers.js new file mode 100644 index 00000000000..f990ee12da3 --- /dev/null +++ b/src/db/migrations/20240710214906_qa_custom_brokers.js @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + /** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export function up(knex) { + return knex.schema + .createTable("qa_custom_brokers", table => { + table.increments("onerep_scan_result_id").primary(); + table.integer("onerep_profile_id").notNullable(); + table.string("link").notNullable(); + table.integer("age").nullable(); + table.string("data_broker").notNullable(); + table.jsonb("emails").notNullable(); + table.jsonb("phones").notNullable(); + table.jsonb("addresses").notNullable(); + table.jsonb("relatives").notNullable(); + table.string("first_name").notNullable(); + table.string("middle_name").nullable(); + table.string("last_name").notNullable(); + table.string("status").notNullable(); + table.boolean("manually_resolved").defaultTo(false); + table.timestamp("created_at").defaultTo(knex.fn.now()); + table.timestamp("updated_at").defaultTo(knex.fn.now()); + }); +} + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export function down(knex) { + return knex.schema.dropTableIfExists("qa_custom_brokers"); +} + + diff --git a/src/db/migrations/20240715110621_qa_custom_toggles.js b/src/db/migrations/20240715110621_qa_custom_toggles.js new file mode 100644 index 00000000000..320ea65a20f --- /dev/null +++ b/src/db/migrations/20240715110621_qa_custom_toggles.js @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export function up(knex) { + return knex.schema.createTable("qa_custom_toggles", function(table) { + table.string("email_hash").primary(); + table.integer("onerep_profile_id").notNullable(); + table.boolean("show_real_breaches").notNullable(); + table.boolean("show_custom_breaches").notNullable(); + table.boolean("show_real_brokers").notNullable(); + table.boolean("show_custom_brokers").notNullable(); + }); +} + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export function down(knex) { + return knex.schema.dropTable("qa_custom_toggles"); +} \ No newline at end of file diff --git a/src/db/migrations/20240715115031_qa_custom_breaches.js b/src/db/migrations/20240715115031_qa_custom_breaches.js new file mode 100644 index 00000000000..631bd1a1d34 --- /dev/null +++ b/src/db/migrations/20240715115031_qa_custom_breaches.js @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export function up(knex) { + // camel case for easier insertion into table + return knex.schema.createTable("qa_custom_breaches", table => { + table.string("emailHashPrefix"); + table.integer("Id").notNullable().primary(); + table.string("Name").notNullable(); + table.string("Title").notNullable(); + table.string("Domain").notNullable(); + table.timestamp("BreachDate").defaultTo(knex.fn.now()); + table.timestamp("AddedDate").defaultTo(knex.fn.now()); + table.timestamp("ModifiedDate").defaultTo(knex.fn.now()); + table.integer("PwnCount").notNullable(); + table.text("Description").notNullable(); + table.string("LogoPath").notNullable(); + table.specificType('DataClasses', 'character varying(255)[]'); + table.boolean("IsVerified").notNullable(); + table.boolean("IsFabricated").notNullable(); + table.boolean("IsSensitive").notNullable(); + table.boolean("IsRetired").notNullable(); + table.boolean("IsSpamList").notNullable(); + table.boolean("IsMalware").notNullable(); + table.string("FaviconUrl").defaultTo(null); + }); +} + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export function down(knex) { + return knex.schema.dropTableIfExists("qa_custom_breaches"); +} \ No newline at end of file diff --git a/src/db/tables/onerep_scans.ts b/src/db/tables/onerep_scans.ts index 74480c41c8f..af701826efd 100644 --- a/src/db/tables/onerep_scans.ts +++ b/src/db/tables/onerep_scans.ts @@ -16,6 +16,7 @@ import { SubscriberRow, } from "knex/types/tables"; import { RemovalStatus } from "../../app/functions/universal/scanResult.js"; +import { getQaCustomBrokers, getQaToggleRow } from "./qa_customs.ts"; const knex = createDbConnection(); @@ -139,25 +140,46 @@ async function getLatestOnerepScan( return scan ?? null; } +/* +Note: please, don't write the results of this function back to the database! +*/ async function getLatestOnerepScanResults( onerepProfileId: number | null, ): Promise { const scan = await getLatestOnerepScan(onerepProfileId); - const results = - typeof scan === "undefined" + let results: OnerepScanResultRow[] = []; + + if (typeof scan !== "undefined") { + const qaToggles = await getQaToggleRow(onerepProfileId); + let showCustomBrokers = true; + let showRealBrokers = true; + + if (qaToggles) { + showCustomBrokers = qaToggles.show_custom_brokers; + showRealBrokers = qaToggles.show_real_brokers; + } + + const qaBrokers = !showCustomBrokers ? [] - : ((await knex("onerep_scan_results") - .select("onerep_scan_results.*") - .distinctOn("link") - .where("onerep_profile_id", onerepProfileId) - .innerJoin( - "onerep_scans", - "onerep_scan_results.onerep_scan_id", - "onerep_scans.onerep_scan_id", - ) - .orderBy("link") - .orderBy("onerep_scan_result_id", "desc")) as OnerepScanResultRow[]); + : await getQaCustomBrokers(onerepProfileId, scan?.onerep_scan_id); + if (!showRealBrokers) results = qaBrokers; + else { + // Fetch initial results from onerep_scan_results + const scanResults = (await knex("onerep_scan_results") + .select("*") + .distinctOn("link") + .where("onerep_profile_id", onerepProfileId) + .innerJoin( + "onerep_scans", + "onerep_scan_results.onerep_scan_id", + "onerep_scans.onerep_scan_id", + ) + .orderBy("link") + .orderBy("onerep_scan_result_id", "desc")) as OnerepScanResultRow[]; + results = [...scanResults, ...qaBrokers]; + } + } return { scan: scan ?? null, @@ -277,7 +299,6 @@ async function markOnerepScanResultAsResolved( logger.info("scan_resolved", { onerepScanResultId, }); - await knex("onerep_scan_results") .update({ manually_resolved: true, diff --git a/src/db/tables/qa_customs.ts b/src/db/tables/qa_customs.ts new file mode 100644 index 00000000000..1905ec3780b --- /dev/null +++ b/src/db/tables/qa_customs.ts @@ -0,0 +1,286 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { OnerepScanResultRow } from "knex/types/tables"; +import { logger } from "../../app/functions/server/logging"; +import createDbConnection from "../connect"; +import { getOnerepProfileId } from "./subscribers"; +import { HibpLikeDbBreach } from "../../utils/hibp"; + +const knex = createDbConnection(); + +interface QaBrokerData { + onerep_profile_id: number; + link: string; + age?: number; + data_broker: string; + emails: string[]; + phones: string[]; + addresses: { [key: string]: string }[]; + relatives: string[]; + first_name: string; + middle_name?: string; + last_name: string; + status: string; + manually_resolved: boolean; +} + +interface QaBreachData { + emailHashPrefix: string; + Id: number; + Name: string; + Title: string; + Domain: string; + BreachDate: Date | string; + AddedDate: Date | string; + ModifiedDate: Date | string; + PwnCount: number; + Description: string; + LogoPath: string; + DataClasses: Array; + IsVerified: boolean; + IsFabricated: boolean; + IsSensitive: boolean; + IsRetired: boolean; + IsSpamList: boolean; + IsMalware: boolean; + FaviconUrl?: string | null; +} + +interface QaToggleRow { + email_hash: string; + onerep_profile_id: number; + show_real_breaches: boolean; + show_custom_breaches: boolean; + show_real_brokers: boolean; + show_custom_brokers: boolean; +} + +enum AllowedToggleColumns { + ShowRealBreaches = "show_real_breaches", + ShowCustomBreaches = "show_custom_breaches", + ShowRealBrokers = "show_real_brokers", + ShowCustomBrokers = "show_custom_brokers", +} + +async function getQaCustomBrokers( + onerepProfileId: number | null, + onerepScanId: number | undefined | null, +) { + if (!onerepProfileId) { + logger.info("getQaCustomBrokers: onerepProfileId was not provided!"); + return []; + } + if (!onerepScanId) { + logger.info("getQaCustomBrokers: onerepScanId was not provided!"); + return []; + } + + let results: OnerepScanResultRow[] = []; + + // Fetch all results from qa_custom_brokers + const brokerResults = await knex("qa_custom_brokers") + .select("*") + .where("onerep_profile_id", onerepProfileId); + + if (brokerResults.length > 0) { + /* + Since these are fake records, their corresponding scanId will be some + existing id, and broker_id will match onerep_scan_result_id for uniqueness + */ + brokerResults.forEach((brokerResult) => { + brokerResult.onerep_scan_id = onerepScanId; + brokerResult.data_broker_id = brokerResult.onerep_scan_result_id; + }); + + results = [...results, ...brokerResults]; + } + return results; +} + +/** + * Inserts a new row into the qa_custom_brokers table. + * + * @param brokerData This object conforms to QaBrokerData, which is the same as + * OnerepScanResulsRow with some fields omitted due to them being automaticallty set. + */ +async function addQaCustomBroker(brokerData: QaBrokerData): Promise { + await knex("qa_custom_brokers").insert({ + ...brokerData, + emails: JSON.stringify(brokerData.emails), + phones: JSON.stringify(brokerData.phones), + addresses: JSON.stringify(brokerData.addresses), + relatives: JSON.stringify(brokerData.relatives), + }); + logger.info(`Created a custom broker: ${brokerData.data_broker}`); +} + +async function getAllQaCustomBrokers( + onerep_profile_id: number, +): Promise { + const res = (await knex("qa_custom_brokers") + .where("onerep_profile_id", onerep_profile_id) + .select("*")) as QaBrokerData[]; + return res; +} + +async function deleteQaCustomBrokerRow(onerep_scan_result_id: number) { + await knex("qa_custom_brokers") + .where("onerep_scan_result_id", onerep_scan_result_id) + .del(); +} + +async function setQaCustomBrokerStatus( + onerep_scan_result_id: number, + newStatus: string, +) { + await knex("qa_custom_brokers") + .where("onerep_scan_result_id", onerep_scan_result_id) + .update({ status: newStatus }); +} + +async function markQaCustomBrokerAsResolved(onerepScanResultId: number) { + const rowsAffected = await knex("qa_custom_brokers") + .update({ + manually_resolved: true, + updated_at: knex.fn.now(), + }) + .where("onerep_scan_result_id", onerepScanResultId); + return rowsAffected; +} + +async function getAllQaCustomBreaches(emailHashPrefix: string) { + const res = ( + await knex("qa_custom_breaches") + .select("*") + .where("emailHashPrefix", emailHashPrefix.toLowerCase()) + ).map((b) => { + b.Id = Number(b.Id); + b.Id = Number(b.Id); + return formatQaBreach(b) as QaBreachData; + }); + return res; +} + +function formatQaBreach(breach: QaBreachData) { + const { emailHashPrefix: _, ...rest } = breach; + return rest as HibpLikeDbBreach; +} + +async function addQaCustomBreach(breach: QaBreachData): Promise { + await knex("qa_custom_breaches").insert({ + ...breach, + }); +} + +async function deleteQaCustomBreach( + emailHashPrefix: string, + Id: number, +): Promise { + await knex("qa_custom_breaches").where({ emailHashPrefix, Id }).del(); +} + +async function getQaToggleRow(emailHashOrOneRepId: string | number | null) { + if (emailHashOrOneRepId === null) { + return null; + } + if (typeof emailHashOrOneRepId === "string") { + return (await knex("qa_custom_toggles") + .select("*") + .where("email_hash", emailHashOrOneRepId) + .first()) as QaToggleRow; + } else if (typeof emailHashOrOneRepId === "number") { + return (await knex("qa_custom_toggles") + .select("*") + .where("onerep_profile_id", emailHashOrOneRepId) + .first()) as QaToggleRow; + } + return null; +} + +async function setQaToggle( + columnName: string, + isShown: boolean, + emailHash: string, +): Promise { + // List of allowed columns to toggle + const allowedColumns = [ + "show_real_breaches", + "show_custom_breaches", + "show_real_brokers", + "show_custom_brokers", + ]; + + if (!allowedColumns.includes(columnName)) { + throw new Error(`Invalid column name: ${columnName}`); + } + + // Get the current value of the specified column + const record = await knex("qa_custom_toggles") + .select(columnName) + .where("email_hash", emailHash) + .first(); + + if (!record) { + throw new Error(`No record found with given email_hash`); + } + + await knex("qa_custom_toggles") + .update({ + [columnName]: isShown, + }) + .where("email_hash", emailHash); +} + +async function createQaTogglesRow( + emailHash: string, + subscriberId: number, +): Promise { + const onerep_profile_id = await getOnerepProfileId(subscriberId); + if (onerep_profile_id === null) throw new Error("OneRep profile ID missing!"); + + const row = { + email_hash: emailHash, + onerep_profile_id, + show_real_breaches: true, + show_custom_breaches: true, + show_real_brokers: true, + show_custom_brokers: true, + }; + + // Try to insert the row + const [insertedRow] = await knex("qa_custom_toggles") + .insert(row) + .onConflict("email_hash") + .ignore() + .returning("*"); + + if (insertedRow) { + return insertedRow as QaToggleRow; + } else { + // If insert was ignored due to conflict, fetch the existing row based on email_hash + const existingRow = await knex("qa_custom_toggles") + .where({ email_hash: emailHash }) + .first(); + return existingRow as QaToggleRow; + } +} + +export { + getQaCustomBrokers, + addQaCustomBroker, + getAllQaCustomBrokers, + deleteQaCustomBrokerRow, + setQaCustomBrokerStatus, + markQaCustomBrokerAsResolved, + getAllQaCustomBreaches, + addQaCustomBreach, + deleteQaCustomBreach, + getQaToggleRow, + setQaToggle, + createQaTogglesRow, + formatQaBreach, + AllowedToggleColumns, +}; +export type { QaBrokerData, QaBreachData, QaToggleRow }; From 99f5377b520bd0d241f8aebcaf7b1c5551f5d839 Mon Sep 17 00:00:00 2001 From: Mukhamediyar Kudaikulov Date: Thu, 25 Jul 2024 13:12:32 -0700 Subject: [PATCH 2/2] added optout_attempts column --- src/db/migrations/20240710214906_qa_custom_brokers.js | 1 + src/db/tables/qa_customs.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/db/migrations/20240710214906_qa_custom_brokers.js b/src/db/migrations/20240710214906_qa_custom_brokers.js index f990ee12da3..5c71b635e63 100644 --- a/src/db/migrations/20240710214906_qa_custom_brokers.js +++ b/src/db/migrations/20240710214906_qa_custom_brokers.js @@ -25,6 +25,7 @@ export function up(knex) { table.boolean("manually_resolved").defaultTo(false); table.timestamp("created_at").defaultTo(knex.fn.now()); table.timestamp("updated_at").defaultTo(knex.fn.now()); + table.integer("optout_attempts"); }); } diff --git a/src/db/tables/qa_customs.ts b/src/db/tables/qa_customs.ts index 1905ec3780b..e4ef0dd9c55 100644 --- a/src/db/tables/qa_customs.ts +++ b/src/db/tables/qa_customs.ts @@ -24,6 +24,7 @@ interface QaBrokerData { last_name: string; status: string; manually_resolved: boolean; + optout_attempts: number; } interface QaBreachData {