Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Request metrics improvements #8420

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
+1,777 −876
Open

Request metrics improvements #8420

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
41b32a6
Network metrics
enahum Nov 30, 2024
74d3b8c
update network-client ref
enahum Dec 3, 2024
112578c
fix unit tests
enahum Dec 3, 2024
0951b33
missing catch error parsing
enahum Dec 3, 2024
dbdac89
Replace API's to fetch all teams and channels for a user
enahum Dec 10, 2024
a5d8bb2
update network library and fix tests
enahum Dec 17, 2024
9f65e90
Merge branch 'main' into request-metrics-improvements
rahimrahman Dec 23, 2024
02e41da
feat(MM-61865): send network request telemetry data to the server (#8…
rahimrahman Dec 24, 2024
6506ce8
fix: improve parallel requests calculation (#8439)
rahimrahman Jan 6, 2025
1ddf677
resolveAndFlattenModels with a capital F
rahimrahman Jan 9, 2025
b448540
Merge branch 'main' into request-metrics-improvements
mattermost-build Jan 10, 2025
641f0ca
add try..catch within resolveAndFlattenModels
rahimrahman Jan 10, 2025
d1b73a2
update groupLabel to fix failing lint
rahimrahman Jan 10, 2025
f8c0c76
fix linting issue due to unknown group label
rahimrahman Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 41 additions & 17 deletions app/actions/local/channel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import PushNotifications from '@init/push_notifications';
import {
prepareDeleteChannel, prepareMyChannelsForTeam, queryAllMyChannel,
getMyChannel, getChannelById, queryUsersOnChannel, queryUserChannelsByTypes,
prepareAllMyChannels,
} from '@queries/servers/channel';
import {queryDisplayNamePreferences} from '@queries/servers/preference';
import {prepareCommonSystemValues, type PrepareCommonSystemValuesArgs, getCommonSystemValues, getCurrentTeamId, setCurrentChannelId, getCurrentUserId, getConfig, getLicense} from '@queries/servers/system';
Expand All @@ -23,6 +24,7 @@ import {isTablet} from '@utils/helpers';
import {logError, logInfo} from '@utils/log';
import {displayGroupMessageName, displayUsername, getUserIdFromChannelName} from '@utils/user';

import type ServerDataOperator from '@database/operator/server_data_operator';
import type {Model} from '@nozbe/watermelondb';
import type ChannelModel from '@typings/database/models/servers/channel';
import type UserModel from '@typings/database/models/servers/user';
Expand Down Expand Up @@ -237,32 +239,54 @@ export async function resetMessageCount(serverUrl: string, channelId: string) {
}
}

export async function storeMyChannelsForTeam(serverUrl: string, teamId: string, channels: Channel[], memberships: ChannelMembership[], prepareRecordsOnly = false, isCRTEnabled?: boolean) {
const resolveAndFlattenModels = async (
operator: ServerDataOperator,
modelPromises: Array<Promise<Model[]>>,
description: string,
prepareRecordsOnly: boolean,
) => {
try {
const {operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl);
const modelPromises: Array<Promise<Model[]>> = [
...await prepareMyChannelsForTeam(operator, teamId, channels, memberships, isCRTEnabled),
];

const models = await Promise.all(modelPromises);
if (!models.length) {
return {models: []};
}

const flattenedModels = models.flat();

if (prepareRecordsOnly) {
return {models: flattenedModels};
if (prepareRecordsOnly || !flattenedModels.length) {
return {models: flattenedModels, error: undefined};
}

if (flattenedModels.length) {
await operator.batchRecords(flattenedModels, 'storeMyChannelsForTeam');
}
await operator.batchRecords(flattenedModels, description);
return {models: flattenedModels, error: undefined};
} catch (error) {
logError('Failed resolveAndFlattenModels', {error, description});
return {models: [], error};
}
};

return {models: flattenedModels};
export async function storeAllMyChannels(serverUrl: string, channels: Channel[], memberships: ChannelMembership[], isCRTEnabled: boolean, prepareRecordsOnly = false) {
try {
const {operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl);
const cs = new Set(channels.map((c) => c.id));
const modelPromises: Array<Promise<Model[]>> = [
...await prepareAllMyChannels(operator, channels, memberships.filter((m) => cs.has(m.channel_id)), isCRTEnabled),
];

return resolveAndFlattenModels(operator, modelPromises, 'storeAllMyChannels', prepareRecordsOnly);
} catch (error) {
logError('Failed storeAllMyChannels', {error, serverUrl, channelsCount: channels.length});
return {error, models: undefined};
}
}

export async function storeMyChannelsForTeam(serverUrl: string, teamId: string, channels: Channel[], memberships: ChannelMembership[], prepareRecordsOnly = false, isCRTEnabled?: boolean) {
try {
const {operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl);
const modelPromises: Array<Promise<Model[]>> = [
...await prepareMyChannelsForTeam(operator, teamId, channels, memberships, isCRTEnabled),
];

return resolveAndFlattenModels(operator, modelPromises, 'storeMyChannelsForTeam', prepareRecordsOnly);
} catch (error) {
logError('Failed storeMyChannelsForTeam', error);
return {error};
return {error, models: undefined};
}
}

Expand Down
86 changes: 76 additions & 10 deletions app/actions/remote/channel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import {DeviceEventEmitter} from 'react-native';

import {addChannelToDefaultCategory, handleConvertedGMCategories, storeCategories} from '@actions/local/category';
import {markChannelAsViewed, removeCurrentUserFromChannel, setChannelDeleteAt, storeMyChannelsForTeam, switchToChannel} from '@actions/local/channel';
import {markChannelAsViewed, removeCurrentUserFromChannel, setChannelDeleteAt, storeAllMyChannels, storeMyChannelsForTeam, switchToChannel} from '@actions/local/channel';
import {switchToGlobalThreads} from '@actions/local/thread';
import {loadCallForChannel} from '@calls/actions/calls';
import {DeepLink, Events, General, Preferences, Screens} from '@constants';
Expand Down Expand Up @@ -376,7 +376,7 @@ export async function fetchChannelCreator(serverUrl: string, channelId: string,
}
}

export async function fetchChannelStats(serverUrl: string, channelId: string, fetchOnly = false, groupLabel?: string) {
export async function fetchChannelStats(serverUrl: string, channelId: string, fetchOnly = false, groupLabel?: RequestGroupLabel) {
try {
const client = NetworkManager.getClient(serverUrl);
const {database, operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl);
Expand Down Expand Up @@ -404,10 +404,74 @@ export async function fetchChannelStats(serverUrl: string, channelId: string, fe
}
}

export async function fetchAllMyChannelsForAllTeams(serverUrl: string, since: number, isCRTEnabled: boolean, fetchOnly = false, groupLabel?: RequestGroupLabel) {
try {
if (!fetchOnly) {
setTeamLoading(serverUrl, true);
}
const client = NetworkManager.getClient(serverUrl);
const {operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl);
const promises: [Promise<Channel[]>, Promise<ChannelMembership[]>] = [
client.getAllChannelsFromAllTeams(since, since > 0, groupLabel),
client.getAllMyChannelMembersFromAllTeams(0, General.CHANNEL_MEMBERS_CHUNK_SIZE, groupLabel),
];

const [channels, memberships] = await Promise.all(promises);
const remaining = Math.floor(channels.length / General.CHANNEL_MEMBERS_CHUNK_SIZE);
const remainingMembershipsPromises = [];
const categoriesPromises = [];

const teamIdsSet = channels.reduce<Set<string>>((set, c) => {
if (c.team_id) {
set.add(c.team_id);
}
return set;
}, new Set());
for (const teamId of teamIdsSet) {
categoriesPromises.push(client.getCategories('me', teamId, groupLabel));
}
Comment on lines +430 to +432
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we have an absurd amount of teams, we are flooding the network layer with get categories calls. It is not common, but we have already had reports of people having 100+ teams.

Unless we create a priority queue down at the network layer, we may want to consider batching this, so other processes can access the network layer before this finishes.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. Thanks for your observation. This is something this PR/changes is trying to establish on where the bottlenecks are and learn from it. And prioritize all the issues. We have a long way to go, but first, data.


if (remaining > 0) {
for (let i = 1; i <= remaining; i++) {
remainingMembershipsPromises.push(client.getAllMyChannelMembersFromAllTeams(i, General.CHANNEL_MEMBERS_CHUNK_SIZE, groupLabel));
}
Comment on lines +435 to +437
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This for loop adds to the problem in my previous comment.

}

const results = await Promise.all([...remainingMembershipsPromises, ...categoriesPromises]);
const categories: CategoryWithChannels[] = [];
for (const result of results) {
if (Array.isArray(result)) {
memberships.push(...result);
} else if ('categories' in result) {
categories.push(...result.categories);
}
}

if (!fetchOnly) {
const {models: chModels} = await storeAllMyChannels(serverUrl, channels, memberships, isCRTEnabled, true);
const {models: catModels} = await storeCategories(serverUrl, categories, true, true); // Re-sync
const models = (chModels || []).concat(catModels || []);
if (models.length) {
await operator.batchRecords(models, 'fetchAllMyChannelsForAllTeams');
}
setTeamLoading(serverUrl, false);
}

return {channels, memberships, categories};
} catch (error) {
logDebug('error on fetchMyChannelsForTeam', getFullErrorMessage(error));
if (!fetchOnly) {
setTeamLoading(serverUrl, false);
}
forceLogoutIfNecessary(serverUrl, error);
return {error};
}
}

export async function fetchMyChannelsForTeam(
serverUrl: string, teamId: string, includeDeleted = true,
since = 0, fetchOnly = false, excludeDirect = false,
isCRTEnabled?: boolean, groupLabel?: string,
isCRTEnabled?: boolean, groupLabel?: RequestGroupLabel,
): Promise<MyChannelsRequest> {
try {
if (!fetchOnly) {
Expand Down Expand Up @@ -457,7 +521,7 @@ export async function fetchMyChannelsForTeam(
}
}

export async function fetchMyChannel(serverUrl: string, teamId: string, channelId: string, fetchOnly = false, groupLabel?: string): Promise<MyChannelsRequest> {
export async function fetchMyChannel(serverUrl: string, teamId: string, channelId: string, fetchOnly = false, groupLabel?: RequestGroupLabel): Promise<MyChannelsRequest> {
try {
const client = NetworkManager.getClient(serverUrl);

Expand All @@ -484,7 +548,7 @@ export async function fetchMyChannel(serverUrl: string, teamId: string, channelI
export async function fetchMissingDirectChannelsInfo(
serverUrl: string, directChannels: Channel[], locale?: string,
teammateDisplayNameSetting?: string, currentUserId?: string,
fetchOnly = false, groupLabel?: string,
fetchOnly = false, groupLabel?: RequestGroupLabel,
) {
try {
const {database, operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl);
Expand Down Expand Up @@ -662,7 +726,7 @@ export async function joinChannelIfNeeded(serverUrl: string, channelId: string)
}
}

export async function markChannelAsRead(serverUrl: string, channelId: string, updateLocal = false, groupLabel?: string) {
export async function markChannelAsRead(serverUrl: string, channelId: string, updateLocal = false, groupLabel?: RequestGroupLabel) {
try {
const client = NetworkManager.getClient(serverUrl);
await client.viewMyChannel(channelId, undefined, groupLabel);
Expand Down Expand Up @@ -1047,7 +1111,7 @@ export async function getChannelTimezones(serverUrl: string, channelId: string)
}
}

export async function switchToChannelById(serverUrl: string, channelId: string, teamId?: string, skipLastUnread = false, groupLabel?: string) {
export async function switchToChannelById(serverUrl: string, channelId: string, teamId?: string, skipLastUnread = false, groupLabel?: RequestGroupLabel) {
if (channelId === Screens.GLOBAL_THREADS) {
return switchToGlobalThreads(serverUrl, teamId);
}
Expand All @@ -1059,7 +1123,7 @@ export async function switchToChannelById(serverUrl: string, channelId: string,

DeviceEventEmitter.emit(Events.CHANNEL_SWITCH, true);

fetchPostsForChannel(serverUrl, channelId, false, groupLabel);
fetchPostsForChannel(serverUrl, channelId, false, false, groupLabel);
fetchChannelBookmarks(serverUrl, channelId, false, groupLabel);
await switchToChannel(serverUrl, channelId, teamId, skipLastUnread);
openChannelIfNeeded(serverUrl, channelId, groupLabel);
Expand Down Expand Up @@ -1258,8 +1322,10 @@ export const handleKickFromChannel = async (serverUrl: string, channelId: string
const currentServer = await getActiveServer();
if (currentServer?.url === serverUrl) {
const channel = await getChannelById(database, channelId);
DeviceEventEmitter.emit(event, channel?.displayName);
await dismissAllModalsAndPopToRoot();
if (channel) {
DeviceEventEmitter.emit(event, channel.displayName);
await dismissAllModalsAndPopToRoot();
}
}

const tabletDevice = isTablet();
Expand Down
2 changes: 1 addition & 1 deletion app/actions/remote/channel_bookmark.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {logError} from '@utils/log';

import {forceLogoutIfNecessary} from './session';

export async function fetchChannelBookmarks(serverUrl: string, channelId: string, fetchOnly = false, groupLabel?: string) {
export async function fetchChannelBookmarks(serverUrl: string, channelId: string, fetchOnly = false, groupLabel?: RequestGroupLabel) {
try {
const client = NetworkManager.getClient(serverUrl);
const {database, operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl);
Expand Down
5 changes: 4 additions & 1 deletion app/actions/remote/entry/app.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import {setLastServerVersionCheck} from '@actions/local/systems';
import {fetchConfigAndLicense} from '@actions/remote/systems';
import DatabaseManager from '@database/manager';
import PerformanceMetricsManager from '@managers/performance_metrics_manager';
import WebsocketManager from '@managers/websocket_manager';
import {prepareCommonSystemValues} from '@queries/servers/system';
import {deleteV1Data} from '@utils/file';
Expand All @@ -16,6 +17,8 @@ export async function appEntry(serverUrl: string, since = 0) {
return {error: `${serverUrl} database not found`};
}

PerformanceMetricsManager.startTimeToInteraction();

if (!since) {
if (Object.keys(DatabaseManager.serverDatabases).length === 1) {
await setLastServerVersionCheck(serverUrl, true);
Expand All @@ -28,7 +31,7 @@ export async function appEntry(serverUrl: string, since = 0) {
await operator.batchRecords(removeLastUnreadChannelId, 'appEntry - removeLastUnreadChannelId');
}

WebsocketManager.openAll('entry');
WebsocketManager.openAll('Cold Start');

verifyPushProxy(serverUrl);

Expand Down
Loading
Loading