diff --git a/app/actions/local/thread.ts b/app/actions/local/thread.ts index 8f25bb9aea..413341d9a1 100644 --- a/app/actions/local/thread.ts +++ b/app/actions/local/thread.ts @@ -168,7 +168,7 @@ export async function createThreadFromNewPost(serverUrl: string, post: Post, pre const {operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl); const models: Model[] = []; if (post.root_id) { - // Update the thread data: `reply_count` + // Update the thread data: `reply_count` const {model: threadModel} = await updateThread(serverUrl, post.root_id, {reply_count: post.reply_count}, true); if (threadModel) { models.push(threadModel); @@ -204,7 +204,7 @@ export async function createThreadFromNewPost(serverUrl: string, post: Post, pre } // On receiving threads, Along with the "threads" & "thread participants", extract and save "posts" & "users" -export async function processReceivedThreads(serverUrl: string, threads: Thread[], teamId: string, loadedInGlobalThreads = false, prepareRecordsOnly = false) { +export async function processReceivedThreads(serverUrl: string, threads: Thread[], teamId: string, prepareRecordsOnly = false) { try { const {database, operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl); const currentUserId = await getCurrentUserId(database); @@ -236,7 +236,6 @@ export async function processReceivedThreads(serverUrl: string, threads: Thread[ threads: threadsToHandle, teamId, prepareRecordsOnly: true, - loadedInGlobalThreads, }); const models = [...postModels, ...threadModels]; @@ -328,3 +327,17 @@ export async function updateThread(serverUrl: string, threadId: string, updatedT return {error}; } } + +export async function updateTeamThreadsSync(serverUrl: string, data: TeamThreadsSync, prepareRecordsOnly = false) { + try { + const {operator} = DatabaseManager.getServerDatabaseAndOperator(serverUrl); + const models = await operator.handleTeamThreadsSync({data: [data], prepareRecordsOnly}); + if (!prepareRecordsOnly) { + await operator.batchRecords(models); + } + return {models}; + } catch (error) { + logError('Failed updateTeamThreadsSync', error); + return {error}; + } +} diff --git a/app/actions/remote/entry/common.ts b/app/actions/remote/entry/common.ts index 72cd4a3b11..f3f1b833a6 100644 --- a/app/actions/remote/entry/common.ts +++ b/app/actions/remote/entry/common.ts @@ -10,7 +10,7 @@ import {MyPreferencesRequest, fetchMyPreferences} from '@actions/remote/preferen import {fetchRoles} from '@actions/remote/role'; import {fetchConfigAndLicense} from '@actions/remote/systems'; import {fetchAllTeams, fetchMyTeams, fetchTeamsChannelsAndUnreadPosts, MyTeamsRequest} from '@actions/remote/team'; -import {fetchNewThreads} from '@actions/remote/thread'; +import {syncTeamThreads} from '@actions/remote/thread'; import {autoUpdateTimezone, fetchMe, MyUserRequest, updateAllUsersSince} from '@actions/remote/user'; import {gqlAllChannels} from '@client/graphQL/entry'; import {General, Preferences, Screens} from '@constants'; @@ -28,6 +28,7 @@ import {prepareModels, truncateCrtRelatedTables} from '@queries/servers/entry'; import {getHasCRTChanged} from '@queries/servers/preference'; import {getConfig, getCurrentUserId, getPushVerificationStatus, getWebSocketLastDisconnected} from '@queries/servers/system'; import {deleteMyTeams, getAvailableTeamIds, getTeamChannelHistory, queryMyTeams, queryMyTeamsByIds, queryTeamsById} from '@queries/servers/team'; +import {getIsCRTEnabled} from '@queries/servers/thread'; import {isDMorGM, sortChannelsByDisplayName} from '@utils/channel'; import {getMemberChannelsFromGQLQuery, gqlToClientChannelMembership} from '@utils/graphql'; import {logDebug} from '@utils/log'; @@ -327,14 +328,14 @@ export async function restDeferredAppEntryActions( if (preferences && processIsCRTEnabled(preferences, config.CollapsedThreads, config.FeatureFlagCollapsedThreads)) { if (initialTeamId) { - await fetchNewThreads(serverUrl, initialTeamId, false); + await syncTeamThreads(serverUrl, initialTeamId); } if (teamData.teams?.length) { for await (const team of teamData.teams) { if (team.id !== initialTeamId) { // need to await here since GM/DM threads in different teams overlap - await fetchNewThreads(serverUrl, team.id, false); + await syncTeamThreads(serverUrl, team.id); } } } @@ -406,6 +407,8 @@ const graphQLSyncAllChannelMembers = async (serverUrl: string) => { return 'Server database not found'; } + const {database} = operator; + const response = await gqlAllChannels(serverUrl); if ('error' in response) { return response.error; @@ -415,7 +418,7 @@ const graphQLSyncAllChannelMembers = async (serverUrl: string) => { return response.errors[0].message; } - const userId = await getCurrentUserId(operator.database); + const userId = await getCurrentUserId(database); const channels = getMemberChannelsFromGQLQuery(response.data); const memberships = response.data.channelMembers?.map((m) => gqlToClientChannelMembership(m, userId)); @@ -424,7 +427,16 @@ const graphQLSyncAllChannelMembers = async (serverUrl: string) => { const modelPromises = await prepareMyChannelsForTeam(operator, '', channels, memberships, undefined, true); const models = (await Promise.all(modelPromises)).flat(); if (models.length) { - operator.batchRecords(models); + await operator.batchRecords(models); + } + } + + const isCRTEnabled = await getIsCRTEnabled(database); + if (isCRTEnabled) { + const myTeams = await queryMyTeams(operator.database).fetch(); + for await (const myTeam of myTeams) { + // need to await here since GM/DM threads in different teams overlap + await syncTeamThreads(serverUrl, myTeam.id); } } @@ -445,11 +457,12 @@ const restSyncAllChannelMembers = async (serverUrl: string) => { const config = await client.getClientConfigOld(); let excludeDirect = false; - for (const myTeam of myTeams) { + for await (const myTeam of myTeams) { fetchMyChannelsForTeam(serverUrl, myTeam.id, false, 0, false, excludeDirect); excludeDirect = true; if (preferences && processIsCRTEnabled(preferences, config.CollapsedThreads, config.FeatureFlagCollapsedThreads)) { - fetchNewThreads(serverUrl, myTeam.id, false); + // need to await here since GM/DM threads in different teams overlap + await syncTeamThreads(serverUrl, myTeam.id); } } } catch { diff --git a/app/actions/remote/entry/gql_common.ts b/app/actions/remote/entry/gql_common.ts index 3b70dc7116..6810997385 100644 --- a/app/actions/remote/entry/gql_common.ts +++ b/app/actions/remote/entry/gql_common.ts @@ -8,7 +8,7 @@ import {MyChannelsRequest} from '@actions/remote/channel'; import {fetchGroupsForMember} from '@actions/remote/groups'; import {fetchPostsForUnreadChannels} from '@actions/remote/post'; import {MyTeamsRequest} from '@actions/remote/team'; -import {fetchNewThreads} from '@actions/remote/thread'; +import {syncTeamThreads} from '@actions/remote/thread'; import {autoUpdateTimezone, updateAllUsersSince} from '@actions/remote/user'; import {gqlEntry, gqlEntryChannels, gqlOtherChannels} from '@client/graphQL/entry'; import {Preferences} from '@constants'; @@ -54,14 +54,14 @@ export async function deferredAppEntryGraphQLActions( if (preferences && processIsCRTEnabled(preferences, config.CollapsedThreads, config.FeatureFlagCollapsedThreads)) { if (initialTeamId) { - await fetchNewThreads(serverUrl, initialTeamId, false); + await syncTeamThreads(serverUrl, initialTeamId); } if (teamData.teams?.length) { for await (const team of teamData.teams) { if (team.id !== initialTeamId) { // need to await here since GM/DM threads in different teams overlap - await fetchNewThreads(serverUrl, team.id, false); + await syncTeamThreads(serverUrl, team.id); } } } diff --git a/app/actions/remote/post.ts b/app/actions/remote/post.ts index 9f059efcc2..ac971708fc 100644 --- a/app/actions/remote/post.ts +++ b/app/actions/remote/post.ts @@ -791,7 +791,7 @@ export async function fetchPostById(serverUrl: string, postId: string, fetchOnly if (authors?.length) { const users = await operator.handleUsers({ users: authors, - prepareRecordsOnly: false, + prepareRecordsOnly: true, }); models.push(...users); } diff --git a/app/actions/remote/thread.ts b/app/actions/remote/thread.ts index 6461ab6bfc..25f38abf10 100644 --- a/app/actions/remote/thread.ts +++ b/app/actions/remote/thread.ts @@ -1,7 +1,9 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import {markTeamThreadsAsRead, markThreadAsViewed, processReceivedThreads, switchToThread, updateThread} from '@actions/local/thread'; +import Model from '@nozbe/watermelondb/Model'; + +import {markTeamThreadsAsRead, markThreadAsViewed, processReceivedThreads, switchToThread, updateTeamThreadsSync, updateThread} from '@actions/local/thread'; import {fetchPostThread} from '@actions/remote/post'; import {General} from '@constants'; import DatabaseManager from '@database/manager'; @@ -10,19 +12,13 @@ import AppsManager from '@managers/apps_manager'; import NetworkManager from '@managers/network_manager'; import {getPostById} from '@queries/servers/post'; import {getConfigValue, getCurrentChannelId, getCurrentTeamId} from '@queries/servers/system'; -import {getIsCRTEnabled, getNewestThreadInTeam, getThreadById} from '@queries/servers/thread'; +import {getIsCRTEnabled, getThreadById, getTeamThreadsSyncData} from '@queries/servers/thread'; import {getCurrentUser} from '@queries/servers/user'; +import {getThreadsListEdges} from '@utils/thread'; import {forceLogoutIfNecessary} from './session'; import type {Client} from '@client/rest'; -import type {Model} from '@nozbe/watermelondb'; - -type FetchThreadsRequest = { - error?: unknown; -} | { - data: GetUserThreadsResponse; -}; type FetchThreadsOptions = { before?: string; @@ -34,6 +30,11 @@ type FetchThreadsOptions = { totalsOnly?: boolean; }; +enum Direction { + Up, + Down, +} + export const fetchAndSwitchToThread = async (serverUrl: string, rootId: string, isFromNotification = false) => { const database = DatabaseManager.serverDatabases[serverUrl]?.database; if (!database) { @@ -74,57 +75,6 @@ export const fetchAndSwitchToThread = async (serverUrl: string, rootId: string, return {}; }; -export const fetchThreads = async ( - serverUrl: string, - teamId: string, - { - before, - after, - perPage = General.CRT_CHUNK_SIZE, - deleted = false, - unread = false, - since, - }: FetchThreadsOptions = { - perPage: General.CRT_CHUNK_SIZE, - deleted: false, - unread: false, - since: 0, - }, -): Promise => { - const database = DatabaseManager.serverDatabases[serverUrl]?.database; - if (!database) { - return {error: `${serverUrl} database not found`}; - } - - let client; - try { - client = NetworkManager.getClient(serverUrl); - } catch (error) { - return {error}; - } - - try { - const version = await getConfigValue(database, 'Version'); - const data = await client.getThreads('me', teamId, before, after, perPage, deleted, unread, since, false, version); - - const {threads} = data; - - if (threads.length) { - // Mark all fetched threads as following - threads.forEach((thread: Thread) => { - thread.is_following = true; - }); - - await processReceivedThreads(serverUrl, threads, teamId, !unread, false); - } - - return {data}; - } catch (error) { - forceLogoutIfNecessary(serverUrl, error as ClientErrorProps); - return {error}; - } -}; - export const fetchThread = async (serverUrl: string, teamId: string, threadId: string, extended?: boolean) => { let client; try { @@ -136,7 +86,7 @@ export const fetchThread = async (serverUrl: string, teamId: string, threadId: s try { const thread = await client.getThread('me', teamId, threadId, extended); - await processReceivedThreads(serverUrl, [thread], teamId, false, false); + await processReceivedThreads(serverUrl, [thread], teamId); return {data: thread}; } catch (error) { @@ -286,17 +236,13 @@ export const updateThreadFollowing = async (serverUrl: string, teamId: string, t } }; -enum Direction { - Up, - Down, -} - -async function fetchBatchThreads( +export const fetchThreads = async ( serverUrl: string, teamId: string, options: FetchThreadsOptions, + direction?: Direction, pages?: number, -): Promise<{error: unknown; data?: Thread[]}> { +) => { const operator = DatabaseManager.serverDatabases[serverUrl]?.operator; if (!operator) { @@ -310,12 +256,7 @@ async function fetchBatchThreads( return {error}; } - // if we start from the begging of time (since = 0) we need to fetch threads from newest to oldest (Direction.Down) - // if there is another point in time, we need to fetch threads from oldest to newest (Direction.Up) - let direction = Direction.Up; - if (options.since === 0) { - direction = Direction.Down; - } + const fetchDirection = direction ?? Direction.Up; const currentUser = await getCurrentUser(operator.database); if (!currentUser) { @@ -323,34 +264,32 @@ async function fetchBatchThreads( } const version = await getConfigValue(operator.database, 'Version'); - const data: Thread[] = []; + const threadsData: Thread[] = []; + let currentPage = 0; const fetchThreadsFunc = async (opts: FetchThreadsOptions) => { - let page = 0; const {before, after, perPage = General.CRT_CHUNK_SIZE, deleted, unread, since} = opts; - page += 1; + currentPage++; const {threads} = await client.getThreads(currentUser.id, teamId, before, after, perPage, deleted, unread, since, false, version); if (threads.length) { // Mark all fetched threads as following for (const thread of threads) { - thread.is_following = true; + thread.is_following = thread.is_following ?? true; } - data.push(...threads); + threadsData.push(...threads); - if (threads.length === perPage) { + if (threads.length === perPage && (pages == null || currentPage < pages!)) { const newOptions: FetchThreadsOptions = {perPage, deleted, unread}; - if (direction === Direction.Down) { + if (fetchDirection === Direction.Down) { const last = threads[threads.length - 1]; newOptions.before = last.id; } else { const first = threads[0]; newOptions.after = first.id; } - if (pages != null && page < pages) { - fetchThreadsFunc(newOptions); - } + await fetchThreadsFunc(newOptions); } } }; @@ -361,140 +300,179 @@ async function fetchBatchThreads( if (__DEV__) { throw error; } - - return {error, data}; + return {error}; } - return {error: false, data}; -} - -export async function fetchNewThreads( - serverUrl: string, - teamId: string, - prepareRecordsOnly = false, -): Promise<{error: unknown; models?: Model[]}> { - const options: FetchThreadsOptions = { - unread: false, - deleted: true, - perPage: 60, - }; + return {error: false, threads: threadsData}; +}; +export const syncTeamThreads = async (serverUrl: string, teamId: string, prepareRecordsOnly = false) => { const operator = DatabaseManager.serverDatabases[serverUrl]?.operator; - if (!operator) { return {error: `${serverUrl} database not found`}; } - const newestThread = await getNewestThreadInTeam(operator.database, teamId, false); - options.since = newestThread ? newestThread.lastReplyAt : 0; + try { + const syncData = await getTeamThreadsSyncData(operator.database, teamId); + const syncDataUpdate = { + id: teamId, + } as TeamThreadsSync; - let response: { - error: unknown; - data?: Thread[]; - } = { - error: undefined, - data: [], - }; + const threads: Thread[] = []; - let loadedInGlobalThreads = true; - - // if we have no threads in the DB fetch all unread ones - if (options.since === 0) { - // options to fetch all unread threads - options.deleted = false; - options.unread = true; - loadedInGlobalThreads = false; - } - - response = await fetchBatchThreads(serverUrl, teamId, options); - - const {error: nErr, data} = response; - - if (nErr) { - return {error: nErr}; - } - - if (!data?.length) { - return {error: false, models: []}; - } - - const {error, models} = await processReceivedThreads(serverUrl, data, teamId, loadedInGlobalThreads, true); - - if (!error && !prepareRecordsOnly && models?.length) { - try { - await operator.batchRecords(models); - } catch (err) { - if (__DEV__) { - throw err; + /** + * If Syncing for the first time, + * - Get all unread threads to show the right badges + * - Get latest threads to show by default in the global threads screen + * Else + * - Get all threads since last sync + */ + if (!syncData || !syncData?.latest) { + const [allUnreadThreads, latestThreads] = await Promise.all([ + fetchThreads( + serverUrl, + teamId, + {unread: true}, + Direction.Down, + ), + fetchThreads( + serverUrl, + teamId, + {}, + undefined, + 1, + ), + ]); + if (allUnreadThreads.error || latestThreads.error) { + return {error: allUnreadThreads.error || latestThreads.error}; + } + if (latestThreads.threads?.length) { + // We are fetching the threads for the first time. We get "latest" and "earliest" values. + const {earliestThread, latestThread} = getThreadsListEdges(latestThreads.threads); + syncDataUpdate.latest = latestThread.last_reply_at; + syncDataUpdate.earliest = earliestThread.last_reply_at; + + threads.push(...latestThreads.threads); + } + if (allUnreadThreads.threads?.length) { + threads.push(...allUnreadThreads.threads); + } + } else { + const allNewThreads = await fetchThreads( + serverUrl, + teamId, + {deleted: true, since: syncData.latest}, + ); + if (allNewThreads.error) { + return {error: allNewThreads.error}; + } + if (allNewThreads.threads?.length) { + // As we are syncing, we get all new threads and we will update the "latest" value. + const {latestThread} = getThreadsListEdges(allNewThreads.threads); + syncDataUpdate.latest = latestThread.last_reply_at; + + threads.push(...allNewThreads.threads); } - return {error: true}; } + + const models: Model[] = []; + + if (threads.length) { + const {error, models: threadModels = []} = await processReceivedThreads(serverUrl, threads, teamId, true); + if (error) { + return {error}; + } + + if (threadModels?.length) { + models.push(...threadModels); + } + + if (syncDataUpdate.earliest || syncDataUpdate.latest) { + const {models: updateModels} = await updateTeamThreadsSync(serverUrl, syncDataUpdate, true); + if (updateModels?.length) { + models.push(...updateModels); + } + } + + if (!prepareRecordsOnly && models?.length) { + try { + await operator.batchRecords(models); + } catch (err) { + if (__DEV__) { + throw err; + } + return {error: err}; + } + } + } + + return {error: false, models}; + } catch (error) { + return {error}; } +}; - return {error: false, models}; -} - -export async function fetchRefreshThreads( - serverUrl: string, - teamId: string, - unread = false, - prepareRecordsOnly = false, -): Promise<{error: unknown; models?: Model[]}> { - const options: FetchThreadsOptions = { - unread, - deleted: true, - perPage: 60, - }; - +export const loadEarlierThreads = async (serverUrl: string, teamId: string, lastThreadId: string, prepareRecordsOnly = false) => { const operator = DatabaseManager.serverDatabases[serverUrl]?.operator; - if (!operator) { return {error: `${serverUrl} database not found`}; } - const newestThread = await getNewestThreadInTeam(operator.database, teamId, unread); - options.since = newestThread ? newestThread.lastReplyAt : 0; - - let response: { - error: unknown; - data?: Thread[]; - } = { - error: undefined, - data: [], - }; - - let pages; - - // in the case of global threads: if we have no threads in the DB fetch just one page - if (options.since === 0 && !unread) { - pages = 1; - } - - response = await fetchBatchThreads(serverUrl, teamId, options, pages); - - const {error: nErr, data} = response; - - if (nErr) { - return {error: nErr}; - } - - if (!data?.length) { - return {error: false, models: []}; - } - - const loadedInGlobalThreads = !unread; - const {error, models} = await processReceivedThreads(serverUrl, data, teamId, loadedInGlobalThreads, true); - - if (!error && !prepareRecordsOnly && models?.length) { - try { - await operator.batchRecords(models); - } catch (err) { - if (__DEV__) { - throw err; - } - return {error: true}; + try { + /* + * - We will fetch one page of old threads + * - Update the sync data with the earliest thread last_reply_at timestamp + */ + const fetchedThreads = await fetchThreads( + serverUrl, + teamId, + { + before: lastThreadId, + }, + undefined, + 1, + ); + if (fetchedThreads.error) { + return {error: fetchedThreads.error}; } - } - return {error: false, models}; -} + const models: Model[] = []; + const threads = fetchedThreads.threads || []; + + if (threads?.length) { + const {error, models: threadModels = []} = await processReceivedThreads(serverUrl, threads, teamId, true); + if (error) { + return {error}; + } + + if (threadModels?.length) { + models.push(...threadModels); + } + + const {earliestThread} = getThreadsListEdges(threads); + const syncDataUpdate = { + id: teamId, + earliest: earliestThread.last_reply_at, + } as TeamThreadsSync; + const {models: updateModels} = await updateTeamThreadsSync(serverUrl, syncDataUpdate, true); + if (updateModels?.length) { + models.push(...updateModels); + } + + if (!prepareRecordsOnly && models?.length) { + try { + await operator.batchRecords(models); + } catch (err) { + if (__DEV__) { + throw err; + } + return {error: err}; + } + } + } + + return {error: false, models, threads}; + } catch (error) { + return {error}; + } +}; diff --git a/app/constants/database.ts b/app/constants/database.ts index acf719cf08..6dd3710650 100644 --- a/app/constants/database.ts +++ b/app/constants/database.ts @@ -40,6 +40,7 @@ export const MM_TABLES = { THREAD: 'Thread', THREADS_IN_TEAM: 'ThreadsInTeam', THREAD_PARTICIPANT: 'ThreadParticipant', + TEAM_THREADS_SYNC: 'TeamThreadsSync', USER: 'User', }, }; diff --git a/app/database/manager/__mocks__/index.ts b/app/database/manager/__mocks__/index.ts index 9f84c0c17d..0d38775cc8 100644 --- a/app/database/manager/__mocks__/index.ts +++ b/app/database/manager/__mocks__/index.ts @@ -15,7 +15,7 @@ import {CategoryModel, CategoryChannelModel, ChannelModel, ChannelInfoModel, Cha GroupModel, GroupChannelModel, GroupTeamModel, GroupMembershipModel, MyChannelModel, MyChannelSettingsModel, MyTeamModel, PostModel, PostsInChannelModel, PostsInThreadModel, PreferenceModel, ReactionModel, RoleModel, SystemModel, TeamModel, TeamChannelHistoryModel, TeamMembershipModel, TeamSearchHistoryModel, - ThreadModel, ThreadParticipantModel, ThreadInTeamModel, UserModel, + ThreadModel, ThreadParticipantModel, ThreadInTeamModel, TeamThreadsSyncModel, UserModel, } from '@database/models/server'; import AppDataOperator from '@database/operator/app_data_operator'; import ServerDataOperator from '@database/operator/server_data_operator'; @@ -51,7 +51,7 @@ class DatabaseManager { GroupModel, GroupChannelModel, GroupTeamModel, GroupMembershipModel, MyChannelModel, MyChannelSettingsModel, MyTeamModel, PostModel, PostsInChannelModel, PostsInThreadModel, PreferenceModel, ReactionModel, RoleModel, SystemModel, TeamModel, TeamChannelHistoryModel, TeamMembershipModel, TeamSearchHistoryModel, - ThreadModel, ThreadParticipantModel, ThreadInTeamModel, UserModel, + ThreadModel, ThreadParticipantModel, ThreadInTeamModel, TeamThreadsSyncModel, UserModel, ]; this.databaseDirectory = ''; } diff --git a/app/database/manager/index.ts b/app/database/manager/index.ts index 6a586ea749..0f038ef733 100644 --- a/app/database/manager/index.ts +++ b/app/database/manager/index.ts @@ -16,7 +16,7 @@ import {CategoryModel, CategoryChannelModel, ChannelModel, ChannelInfoModel, Cha GroupModel, GroupChannelModel, GroupTeamModel, GroupMembershipModel, MyChannelModel, MyChannelSettingsModel, MyTeamModel, PostModel, PostsInChannelModel, PostsInThreadModel, PreferenceModel, ReactionModel, RoleModel, SystemModel, TeamModel, TeamChannelHistoryModel, TeamMembershipModel, TeamSearchHistoryModel, - ThreadModel, ThreadParticipantModel, ThreadInTeamModel, UserModel, ConfigModel, + ThreadModel, ThreadParticipantModel, ThreadInTeamModel, TeamThreadsSyncModel, UserModel, ConfigModel, } from '@database/models/server'; import AppDataOperator from '@database/operator/app_data_operator'; import ServerDataOperator from '@database/operator/server_data_operator'; @@ -50,7 +50,7 @@ class DatabaseManager { GroupModel, GroupChannelModel, GroupTeamModel, GroupMembershipModel, MyChannelModel, MyChannelSettingsModel, MyTeamModel, PostModel, PostsInChannelModel, PostsInThreadModel, PreferenceModel, ReactionModel, RoleModel, SystemModel, TeamModel, TeamChannelHistoryModel, TeamMembershipModel, TeamSearchHistoryModel, - ThreadModel, ThreadParticipantModel, ThreadInTeamModel, UserModel, + ThreadModel, ThreadParticipantModel, ThreadInTeamModel, TeamThreadsSyncModel, UserModel, ]; this.databaseDirectory = Platform.OS === 'ios' ? getIOSAppGroupDetails().appGroupDatabase : `${FileSystem.DocumentDirectoryPath}/databases/`; diff --git a/app/database/migration/server/index.ts b/app/database/migration/server/index.ts index f633a8edf7..1b7255d9ce 100644 --- a/app/database/migration/server/index.ts +++ b/app/database/migration/server/index.ts @@ -8,16 +8,46 @@ import {schemaMigrations, addColumns, createTable} from '@nozbe/watermelondb/Sch import {MM_TABLES} from '@constants/database'; import {tableSchemaSpec as configSpec} from '@database/schema/server/table_schemas/config'; +import {tableSchemaSpec as teamThreadsSyncSpec} from '@database/schema/server/table_schemas/team_threads_sync'; +import {tableSchemaSpec as threadSpec} from '@database/schema/server/table_schemas/thread'; +import {tableSchemaSpec as threadInTeamSpec} from '@database/schema/server/table_schemas/thread_in_team'; +import {tableSchemaSpec as threadParticipantSpec} from '@database/schema/server/table_schemas/thread_participant'; const {SERVER: { GROUP, MY_CHANNEL, TEAM, THREAD, + THREAD_PARTICIPANT, + THREADS_IN_TEAM, USER, }} = MM_TABLES; export default schemaMigrations({migrations: [ + { + toVersion: 7, + steps: [ + + // Along with adding the new table - TeamThreadsSync, + // We need to clear the data in thread related tables (DROP & CREATE) to fetch the fresh data from the server + createTable({ + ...teamThreadsSyncSpec, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + unsafeSql: (baseSql) => { + return ` + ${baseSql} + DROP TABLE ${THREAD}; + DROP TABLE ${THREADS_IN_TEAM}; + DROP TABLE ${THREAD_PARTICIPANT}; + `; + }, + }), + createTable(threadSpec), + createTable(threadInTeamSpec), + createTable(threadParticipantSpec), + ], + }, { toVersion: 6, steps: [ diff --git a/app/database/models/server/index.ts b/app/database/models/server/index.ts index d371e48fb1..8368b53dad 100644 --- a/app/database/models/server/index.ts +++ b/app/database/models/server/index.ts @@ -31,4 +31,5 @@ export {default as TeamSearchHistoryModel} from './team_search_history'; export {default as ThreadModel} from './thread'; export {default as ThreadInTeamModel} from './thread_in_team'; export {default as ThreadParticipantModel} from './thread_participant'; +export {default as TeamThreadsSyncModel} from './team_threads_sync'; export {default as UserModel} from './user'; diff --git a/app/database/models/server/team.ts b/app/database/models/server/team.ts index 11efd6b600..0a1d2501b8 100644 --- a/app/database/models/server/team.ts +++ b/app/database/models/server/team.ts @@ -110,10 +110,7 @@ export default class TeamModel extends Model implements TeamModelInterface { /** threads : Threads list belonging to a team */ @lazy threadsList = this.collections.get(THREAD).query( - Q.on(THREADS_IN_TEAM, Q.and( - Q.where('team_id', this.id), - Q.where('loaded_in_global_threads', true), - )), + Q.on(THREADS_IN_TEAM, 'team_id', this.id), Q.and( Q.where('reply_count', Q.gt(0)), Q.where('is_following', true), diff --git a/app/database/models/server/team_threads_sync.ts b/app/database/models/server/team_threads_sync.ts new file mode 100644 index 0000000000..9b5a8409f5 --- /dev/null +++ b/app/database/models/server/team_threads_sync.ts @@ -0,0 +1,35 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {Relation} from '@nozbe/watermelondb'; +import {field, immutableRelation} from '@nozbe/watermelondb/decorators'; +import Model, {Associations} from '@nozbe/watermelondb/Model'; + +import {MM_TABLES} from '@constants/database'; + +import type TeamModel from '@typings/database/models/servers/team'; +import type TeamThreadsSyncModelInterface from '@typings/database/models/servers/team_threads_sync'; + +const {TEAM, TEAM_THREADS_SYNC} = MM_TABLES.SERVER; + +/** + * ThreadInTeam model helps us to sync threads without creating any gaps between the threads + * by keeping track of the latest and earliest last_replied_at timestamps loaded for a team. + */ +export default class TeamThreadsSyncModel extends Model implements TeamThreadsSyncModelInterface { + /** table (name) : TeamThreadsSync */ + static table = TEAM_THREADS_SYNC; + + /** associations : Describes every relationship to this table. */ + static associations: Associations = { + [TEAM]: {type: 'belongs_to', key: 'id'}, + }; + + /** earliest: Oldest last_replied_at loaded through infinite loading */ + @field('earliest') earliest!: number; + + /** latest: Newest last_replied_at loaded during app init / navigating to global threads / pull to refresh */ + @field('latest') latest!: number; + + @immutableRelation(TEAM, 'id') team!: Relation; +} diff --git a/app/database/models/server/thread_in_team.ts b/app/database/models/server/thread_in_team.ts index d12fc0eff5..50373e0b15 100644 --- a/app/database/models/server/thread_in_team.ts +++ b/app/database/models/server/thread_in_team.ts @@ -37,9 +37,6 @@ export default class ThreadInTeamModel extends Model implements ThreadInTeamMode /** team_id: Associated team identifier */ @field('team_id') teamId!: string; - /** loaded_in_global_threads : Flag to differentiate the unread threads loaded for showing unread counts/mentions */ - @field('loaded_in_global_threads') loadedInGlobalThreads!: boolean; - @immutableRelation(THREAD, 'thread_id') thread!: Relation; @immutableRelation(TEAM, 'team_id') team!: Relation; diff --git a/app/database/operator/server_data_operator/handlers/team_threads_sync.ts b/app/database/operator/server_data_operator/handlers/team_threads_sync.ts new file mode 100644 index 0000000000..b4c096c2cf --- /dev/null +++ b/app/database/operator/server_data_operator/handlers/team_threads_sync.ts @@ -0,0 +1,71 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {Q, Database} from '@nozbe/watermelondb'; + +import {MM_TABLES} from '@constants/database'; +import {transformTeamThreadsSyncRecord} from '@database/operator/server_data_operator/transformers/thread'; +import {getRawRecordPairs, getUniqueRawsBy, getValidRecordsForUpdate} from '@database/operator/utils/general'; +import {logWarning} from '@utils/log'; + +import type {HandleTeamThreadsSyncArgs, RecordPair} from '@typings/database/database'; +import type TeamThreadsSyncModel from '@typings/database/models/servers/team_threads_sync'; + +export interface TeamThreadsSyncHandlerMix { + handleTeamThreadsSync: ({data, prepareRecordsOnly}: HandleTeamThreadsSyncArgs) => Promise; +} + +const {TEAM_THREADS_SYNC} = MM_TABLES.SERVER; + +const TeamThreadsSyncHandler = (superclass: any) => class extends superclass { + handleTeamThreadsSync = async ({data, prepareRecordsOnly = false}: HandleTeamThreadsSyncArgs): Promise => { + if (!data || !data.length) { + logWarning( + 'An empty or undefined "data" array has been passed to the handleTeamThreadsSync method', + ); + return []; + } + + const uniqueRaws = getUniqueRawsBy({raws: data, key: 'id'}) as TeamThreadsSync[]; + const ids = uniqueRaws.map((item) => item.id); + const chunks = await (this.database as Database).get(TEAM_THREADS_SYNC).query( + Q.where('id', Q.oneOf(ids)), + ).fetch(); + const chunksMap = chunks.reduce((result: Record, chunk) => { + result[chunk.id] = chunk; + return result; + }, {}); + + const create: TeamThreadsSync[] = []; + const update: RecordPair[] = []; + + for await (const item of uniqueRaws) { + const {id} = item; + const chunk = chunksMap[id]; + if (chunk) { + update.push(getValidRecordsForUpdate({ + tableName: TEAM_THREADS_SYNC, + newValue: item, + existingRecord: chunk, + })); + } else { + create.push(item); + } + } + + const models = (await this.prepareRecords({ + createRaws: getRawRecordPairs(create), + updateRaws: update, + transformer: transformTeamThreadsSyncRecord, + tableName: TEAM_THREADS_SYNC, + })) as TeamThreadsSyncModel[]; + + if (models?.length && !prepareRecordsOnly) { + await this.batchRecords(models); + } + + return models; + }; +}; + +export default TeamThreadsSyncHandler; diff --git a/app/database/operator/server_data_operator/handlers/thread.test.ts b/app/database/operator/server_data_operator/handlers/thread.test.ts index 12f471caf4..cacdf28c15 100644 --- a/app/database/operator/server_data_operator/handlers/thread.test.ts +++ b/app/database/operator/server_data_operator/handlers/thread.test.ts @@ -2,7 +2,7 @@ // See LICENSE.txt for license information. import DatabaseManager from '@database/manager'; -import {transformThreadRecord, transformThreadParticipantRecord, transformThreadInTeamRecord} from '@database/operator/server_data_operator/transformers/thread'; +import {transformThreadRecord, transformThreadParticipantRecord, transformThreadInTeamRecord, transformTeamThreadsSyncRecord} from '@database/operator/server_data_operator/transformers/thread'; import ServerDataOperator from '..'; @@ -51,7 +51,7 @@ describe('*** Operator: Thread Handlers tests ***', () => { ] as ThreadWithLastFetchedAt[]; const threadsMap = {team_id_1: threads}; - await operator.handleThreads({threads, loadedInGlobalThreads: false, prepareRecordsOnly: false, teamId: 'team_id_1'}); + await operator.handleThreads({threads, prepareRecordsOnly: false, teamId: 'team_id_1'}); expect(spyOnHandleRecords).toHaveBeenCalledWith({ fieldName: 'id', @@ -76,7 +76,6 @@ describe('*** Operator: Thread Handlers tests ***', () => { expect(spyOnHandleThreadInTeam).toHaveBeenCalledWith({ threadsMap, prepareRecordsOnly: true, - loadedInGlobalThreads: false, }); // Only one batch operation for both tables @@ -161,21 +160,77 @@ describe('*** Operator: Thread Handlers tests ***', () => { team_id_2: team2Threads, }; - await operator.handleThreadInTeam({threadsMap, loadedInGlobalThreads: true, prepareRecordsOnly: false}); + await operator.handleThreadInTeam({threadsMap, prepareRecordsOnly: false}); expect(spyOnPrepareRecords).toHaveBeenCalledWith({ createRaws: [{ - raw: {team_id: 'team_id_1', thread_id: 'thread-2', loaded_in_global_threads: true}, + raw: {team_id: 'team_id_1', thread_id: 'thread-1'}, }, { - raw: {team_id: 'team_id_2', thread_id: 'thread-2', loaded_in_global_threads: true}, + raw: {team_id: 'team_id_1', thread_id: 'thread-2'}, + }, { + raw: {team_id: 'team_id_2', thread_id: 'thread-2'}, }], transformer: transformThreadInTeamRecord, - updateRaws: [ - expect.objectContaining({ - raw: {team_id: 'team_id_1', thread_id: 'thread-1', loaded_in_global_threads: true}, - }), - ], tableName: 'ThreadsInTeam', }); }); + + it('=> HandleTeamThreadsSync: should write to the the TeamThreadsSync table', async () => { + expect.assertions(1); + + const spyOnPrepareRecords = jest.spyOn(operator, 'prepareRecords'); + + const data = [ + { + id: 'team_id_1', + earliest: 100, + latest: 200, + }, + { + id: 'team_id_2', + earliest: 100, + latest: 300, + }, + ] as TeamThreadsSync[]; + + await operator.handleTeamThreadsSync({data, prepareRecordsOnly: false}); + + expect(spyOnPrepareRecords).toHaveBeenCalledWith({ + createRaws: [{ + raw: {id: 'team_id_1', earliest: 100, latest: 200}, + }, { + raw: {id: 'team_id_2', earliest: 100, latest: 300}, + }], + updateRaws: [], + transformer: transformTeamThreadsSyncRecord, + tableName: 'TeamThreadsSync', + }); + }); + + it('=> HandleTeamThreadsSync: should update the record in TeamThreadsSync table', async () => { + expect.assertions(1); + + const spyOnPrepareRecords = jest.spyOn(operator, 'prepareRecords'); + + const data = [ + { + id: 'team_id_1', + earliest: 100, + latest: 300, + }, + ] as TeamThreadsSync[]; + + await operator.handleTeamThreadsSync({data, prepareRecordsOnly: false}); + + expect(spyOnPrepareRecords).toHaveBeenCalledWith({ + createRaws: [], + updateRaws: [ + expect.objectContaining({ + raw: {id: 'team_id_1', earliest: 100, latest: 300}, + }), + ], + transformer: transformTeamThreadsSyncRecord, + tableName: 'TeamThreadsSync', + }); + }); }); diff --git a/app/database/operator/server_data_operator/handlers/thread.ts b/app/database/operator/server_data_operator/handlers/thread.ts index d4bb3bfc78..605e71827f 100644 --- a/app/database/operator/server_data_operator/handlers/thread.ts +++ b/app/database/operator/server_data_operator/handlers/thread.ts @@ -25,7 +25,7 @@ const { } = MM_TABLES.SERVER; export interface ThreadHandlerMix { - handleThreads: ({threads, teamId, prepareRecordsOnly, loadedInGlobalThreads}: HandleThreadsArgs) => Promise; + handleThreads: ({threads, teamId, prepareRecordsOnly}: HandleThreadsArgs) => Promise; handleThreadParticipants: ({threadsParticipants, prepareRecordsOnly}: HandleThreadParticipantsArgs) => Promise; } @@ -37,7 +37,7 @@ const ThreadHandler = (superclass: any) => class extends superclass { * @param {boolean | undefined} handleThreads.prepareRecordsOnly * @returns {Promise} */ - handleThreads = async ({threads, teamId, loadedInGlobalThreads, prepareRecordsOnly = false}: HandleThreadsArgs): Promise => { + handleThreads = async ({threads, teamId, prepareRecordsOnly = false}: HandleThreadsArgs): Promise => { if (!threads?.length) { logWarning( 'An empty or undefined "threads" array has been passed to the handleThreads method', @@ -119,7 +119,6 @@ const ThreadHandler = (superclass: any) => class extends superclass { const threadsInTeam = await this.handleThreadInTeam({ threadsMap: {[teamId]: threads}, prepareRecordsOnly: true, - loadedInGlobalThreads, }) as ThreadInTeamModel[]; batch.push(...threadsInTeam); } diff --git a/app/database/operator/server_data_operator/handlers/thread_in_team.ts b/app/database/operator/server_data_operator/handlers/thread_in_team.ts index a991246ddf..f8076d16bf 100644 --- a/app/database/operator/server_data_operator/handlers/thread_in_team.ts +++ b/app/database/operator/server_data_operator/handlers/thread_in_team.ts @@ -5,20 +5,20 @@ import {Q, Database} from '@nozbe/watermelondb'; import {MM_TABLES} from '@constants/database'; import {transformThreadInTeamRecord} from '@database/operator/server_data_operator/transformers/thread'; -import {getRawRecordPairs, getValidRecordsForUpdate} from '@database/operator/utils/general'; +import {getRawRecordPairs} from '@database/operator/utils/general'; import {logWarning} from '@utils/log'; -import type {HandleThreadInTeamArgs, RecordPair} from '@typings/database/database'; +import type {HandleThreadInTeamArgs} from '@typings/database/database'; import type ThreadInTeamModel from '@typings/database/models/servers/thread_in_team'; export interface ThreadInTeamHandlerMix { - handleThreadInTeam: ({threadsMap, loadedInGlobalThreads, prepareRecordsOnly}: HandleThreadInTeamArgs) => Promise; + handleThreadInTeam: ({threadsMap, prepareRecordsOnly}: HandleThreadInTeamArgs) => Promise; } const {THREADS_IN_TEAM} = MM_TABLES.SERVER; const ThreadInTeamHandler = (superclass: any) => class extends superclass { - handleThreadInTeam = async ({threadsMap, loadedInGlobalThreads, prepareRecordsOnly = false}: HandleThreadInTeamArgs): Promise => { + handleThreadInTeam = async ({threadsMap, prepareRecordsOnly = false}: HandleThreadInTeamArgs): Promise => { if (!threadsMap || !Object.keys(threadsMap).length) { logWarning( 'An empty or undefined "threadsMap" object has been passed to the handleReceivedPostForChannel method', @@ -26,12 +26,13 @@ const ThreadInTeamHandler = (superclass: any) => class extends superclass { return []; } - const update: RecordPair[] = []; const create: ThreadInTeam[] = []; const teamIds = Object.keys(threadsMap); for await (const teamId of teamIds) { + const threadIds = threadsMap[teamId].map((thread) => thread.id); const chunks = await (this.database as Database).get(THREADS_IN_TEAM).query( Q.where('team_id', teamId), + Q.where('id', Q.oneOf(threadIds)), ).fetch(); const chunksMap = chunks.reduce((result: Record, chunk) => { result[chunk.threadId] = chunk; @@ -41,29 +42,18 @@ const ThreadInTeamHandler = (superclass: any) => class extends superclass { for (const thread of threadsMap[teamId]) { const chunk = chunksMap[thread.id]; - const newValue = { - thread_id: thread.id, - team_id: teamId, - loaded_in_global_threads: Boolean(loadedInGlobalThreads), - }; - - // update record only if loaded_in_global_threads is true - if (chunk && loadedInGlobalThreads) { - update.push(getValidRecordsForUpdate({ - tableName: THREADS_IN_TEAM, - newValue, - existingRecord: chunk, - })); - } else { - // create chunk - create.push(newValue); + // Create if the chunk is not found + if (!chunk) { + create.push({ + thread_id: thread.id, + team_id: teamId, + }); } } } const threadsInTeam = (await this.prepareRecords({ createRaws: getRawRecordPairs(create), - updateRaws: update, transformer: transformThreadInTeamRecord, tableName: THREADS_IN_TEAM, })) as ThreadInTeamModel[]; diff --git a/app/database/operator/server_data_operator/index.ts b/app/database/operator/server_data_operator/index.ts index 5ab0f1b7a8..74a8c5b39d 100644 --- a/app/database/operator/server_data_operator/index.ts +++ b/app/database/operator/server_data_operator/index.ts @@ -10,6 +10,7 @@ import PostsInChannelHandler, {PostsInChannelHandlerMix} from '@database/operato import PostsInThreadHandler, {PostsInThreadHandlerMix} from '@database/operator/server_data_operator/handlers/posts_in_thread'; import ReactionHander, {ReactionHandlerMix} from '@database/operator/server_data_operator/handlers/reaction'; import TeamHandler, {TeamHandlerMix} from '@database/operator/server_data_operator/handlers/team'; +import TeamThreadsSyncHandler, {TeamThreadsSyncHandlerMix} from '@database/operator/server_data_operator/handlers/team_threads_sync'; import ThreadHandler, {ThreadHandlerMix} from '@database/operator/server_data_operator/handlers/thread'; import ThreadInTeamHandler, {ThreadInTeamHandlerMix} from '@database/operator/server_data_operator/handlers/thread_in_team'; import UserHandler, {UserHandlerMix} from '@database/operator/server_data_operator/handlers/user'; @@ -29,6 +30,7 @@ interface ServerDataOperator extends TeamHandlerMix, ThreadHandlerMix, ThreadInTeamHandlerMix, + TeamThreadsSyncHandlerMix, UserHandlerMix {} @@ -43,6 +45,7 @@ class ServerDataOperator extends mix(ServerDataOperatorBase).with( TeamHandler, ThreadHandler, ThreadInTeamHandler, + TeamThreadsSyncHandler, UserHandler, ) { // eslint-disable-next-line no-useless-constructor diff --git a/app/database/operator/server_data_operator/transformers/thread.ts b/app/database/operator/server_data_operator/transformers/thread.ts index d143e79439..27c9c44341 100644 --- a/app/database/operator/server_data_operator/transformers/thread.ts +++ b/app/database/operator/server_data_operator/transformers/thread.ts @@ -5,6 +5,7 @@ import {MM_TABLES, OperationType} from '@constants/database'; import {prepareBaseRecord} from '@database/operator/server_data_operator/transformers/index'; import type {TransformerArgs} from '@typings/database/database'; +import type TeamThreadsSyncModel from '@typings/database/models/servers/team_threads_sync'; import type ThreadModel from '@typings/database/models/servers/thread'; import type ThreadInTeamModel from '@typings/database/models/servers/thread_in_team'; import type ThreadParticipantModel from '@typings/database/models/servers/thread_participant'; @@ -13,6 +14,7 @@ const { THREAD, THREAD_PARTICIPANT, THREADS_IN_TEAM, + TEAM_THREADS_SYNC, } = MM_TABLES.SERVER; /** @@ -30,7 +32,10 @@ export const transformThreadRecord = ({action, database, value}: TransformerArgs // If isCreateAction is true, we will use the id (API response) from the RAW, else we shall use the existing record id from the database const fieldsMapper = (thread: ThreadModel) => { thread._raw.id = isCreateAction ? (raw?.id ?? thread.id) : record.id; - thread.lastReplyAt = raw.last_reply_at; + + // When post is individually fetched, we get last_reply_at as 0, so we use the record's value + thread.lastReplyAt = raw.last_reply_at || record?.lastReplyAt; + thread.lastViewedAt = raw.last_viewed_at ?? record?.lastViewedAt ?? 0; thread.replyCount = raw.reply_count; thread.isFollowing = raw.is_following ?? record?.isFollowing; @@ -76,14 +81,10 @@ export const transformThreadParticipantRecord = ({action, database, value}: Tran export const transformThreadInTeamRecord = ({action, database, value}: TransformerArgs): Promise => { const raw = value.raw as ThreadInTeam; - const record = value.record as ThreadInTeamModel; const fieldsMapper = (threadInTeam: ThreadInTeamModel) => { threadInTeam.threadId = raw.thread_id; threadInTeam.teamId = raw.team_id; - - // if it's already loaded don't change it - threadInTeam.loadedInGlobalThreads = record?.loadedInGlobalThreads || raw.loaded_in_global_threads; }; return prepareBaseRecord({ @@ -94,3 +95,23 @@ export const transformThreadInTeamRecord = ({action, database, value}: Transform fieldsMapper, }) as Promise; }; + +export const transformTeamThreadsSyncRecord = ({action, database, value}: TransformerArgs): Promise => { + const raw = value.raw as TeamThreadsSync; + const record = value.record as TeamThreadsSyncModel; + const isCreateAction = action === OperationType.CREATE; + + const fieldsMapper = (teamThreadsSync: TeamThreadsSyncModel) => { + teamThreadsSync._raw.id = isCreateAction ? (raw?.id ?? teamThreadsSync.id) : record.id; + teamThreadsSync.earliest = raw.earliest || record?.earliest; + teamThreadsSync.latest = raw.latest || record?.latest; + }; + + return prepareBaseRecord({ + action, + database, + tableName: TEAM_THREADS_SYNC, + value, + fieldsMapper, + }) as Promise; +}; diff --git a/app/database/schema/server/index.ts b/app/database/schema/server/index.ts index 5cb7a38076..55fd6f70fc 100644 --- a/app/database/schema/server/index.ts +++ b/app/database/schema/server/index.ts @@ -34,11 +34,12 @@ import { ThreadSchema, ThreadInTeamSchema, ThreadParticipantSchema, + TeamThreadsSyncSchema, UserSchema, } from './table_schemas'; export const serverSchema: AppSchema = appSchema({ - version: 6, + version: 7, tables: [ CategorySchema, CategoryChannelSchema, @@ -67,6 +68,7 @@ export const serverSchema: AppSchema = appSchema({ TeamMembershipSchema, TeamSchema, TeamSearchHistorySchema, + TeamThreadsSyncSchema, ThreadSchema, ThreadInTeamSchema, ThreadParticipantSchema, diff --git a/app/database/schema/server/table_schemas/index.ts b/app/database/schema/server/table_schemas/index.ts index a388e51514..27676546ea 100644 --- a/app/database/schema/server/table_schemas/index.ts +++ b/app/database/schema/server/table_schemas/index.ts @@ -30,5 +30,6 @@ export {default as TeamSearchHistorySchema} from './team_search_history'; export {default as ThreadSchema} from './thread'; export {default as ThreadParticipantSchema} from './thread_participant'; export {default as ThreadInTeamSchema} from './thread_in_team'; +export {default as TeamThreadsSyncSchema} from './team_threads_sync'; export {default as UserSchema} from './user'; export {default as ConfigSchema} from './config'; diff --git a/app/database/schema/server/table_schemas/team_threads_sync.ts b/app/database/schema/server/table_schemas/team_threads_sync.ts new file mode 100644 index 0000000000..afadc1ca77 --- /dev/null +++ b/app/database/schema/server/table_schemas/team_threads_sync.ts @@ -0,0 +1,20 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {tableSchema} from '@nozbe/watermelondb'; + +import {MM_TABLES} from '@constants/database'; + +import type {TableSchemaSpec} from '@nozbe/watermelondb/Schema'; + +const {TEAM_THREADS_SYNC} = MM_TABLES.SERVER; + +export const tableSchemaSpec: TableSchemaSpec = { + name: TEAM_THREADS_SYNC, + columns: [ + {name: 'earliest', type: 'number'}, + {name: 'latest', type: 'number'}, + ], +}; + +export default tableSchema(tableSchemaSpec); diff --git a/app/database/schema/server/table_schemas/thread.ts b/app/database/schema/server/table_schemas/thread.ts index dfa05fd09a..88bb904014 100644 --- a/app/database/schema/server/table_schemas/thread.ts +++ b/app/database/schema/server/table_schemas/thread.ts @@ -5,9 +5,11 @@ import {tableSchema} from '@nozbe/watermelondb'; import {MM_TABLES} from '@constants/database'; +import type {TableSchemaSpec} from '@nozbe/watermelondb/Schema'; + const {THREAD} = MM_TABLES.SERVER; -export default tableSchema({ +export const tableSchemaSpec: TableSchemaSpec = { name: THREAD, columns: [ {name: 'is_following', type: 'boolean'}, @@ -19,5 +21,7 @@ export default tableSchema({ {name: 'viewed_at', type: 'number'}, {name: 'last_fetched_at', type: 'number', isIndexed: true}, ], -}); +}; + +export default tableSchema(tableSchemaSpec); diff --git a/app/database/schema/server/table_schemas/thread_in_team.ts b/app/database/schema/server/table_schemas/thread_in_team.ts index 329523049a..6f6ee54eff 100644 --- a/app/database/schema/server/table_schemas/thread_in_team.ts +++ b/app/database/schema/server/table_schemas/thread_in_team.ts @@ -5,13 +5,16 @@ import {tableSchema} from '@nozbe/watermelondb'; import {MM_TABLES} from '@constants/database'; +import type {TableSchemaSpec} from '@nozbe/watermelondb/Schema'; + const {THREADS_IN_TEAM} = MM_TABLES.SERVER; -export default tableSchema({ +export const tableSchemaSpec: TableSchemaSpec = { name: THREADS_IN_TEAM, columns: [ - {name: 'loaded_in_global_threads', type: 'boolean', isIndexed: true}, {name: 'team_id', type: 'string', isIndexed: true}, {name: 'thread_id', type: 'string', isIndexed: true}, ], -}); +}; + +export default tableSchema(tableSchemaSpec); diff --git a/app/database/schema/server/table_schemas/thread_participant.ts b/app/database/schema/server/table_schemas/thread_participant.ts index 5396c61af5..ce98d587c7 100644 --- a/app/database/schema/server/table_schemas/thread_participant.ts +++ b/app/database/schema/server/table_schemas/thread_participant.ts @@ -5,12 +5,16 @@ import {tableSchema} from '@nozbe/watermelondb'; import {MM_TABLES} from '@constants/database'; +import type {TableSchemaSpec} from '@nozbe/watermelondb/Schema'; + const {THREAD_PARTICIPANT} = MM_TABLES.SERVER; -export default tableSchema({ +export const tableSchemaSpec: TableSchemaSpec = { name: THREAD_PARTICIPANT, columns: [ {name: 'thread_id', type: 'string', isIndexed: true}, {name: 'user_id', type: 'string', isIndexed: true}, ], -}); +}; + +export default tableSchema(tableSchemaSpec); diff --git a/app/database/schema/server/test.ts b/app/database/schema/server/test.ts index b4e294029b..2821ea8cb1 100644 --- a/app/database/schema/server/test.ts +++ b/app/database/schema/server/test.ts @@ -38,13 +38,14 @@ const { THREAD, THREAD_PARTICIPANT, THREADS_IN_TEAM, + TEAM_THREADS_SYNC, USER, } = MM_TABLES.SERVER; describe('*** Test schema for SERVER database ***', () => { it('=> The SERVER SCHEMA should strictly match', () => { expect(serverSchema).toEqual({ - version: 6, + version: 7, unsafeSql: undefined, tables: { [CATEGORY]: { @@ -570,16 +571,26 @@ describe('*** Test schema for SERVER database ***', () => { name: THREADS_IN_TEAM, unsafeSql: undefined, columns: { - loaded_in_global_threads: {name: 'loaded_in_global_threads', type: 'boolean', isIndexed: true}, team_id: {name: 'team_id', type: 'string', isIndexed: true}, thread_id: {name: 'thread_id', type: 'string', isIndexed: true}, }, columnArray: [ - {name: 'loaded_in_global_threads', type: 'boolean', isIndexed: true}, {name: 'team_id', type: 'string', isIndexed: true}, {name: 'thread_id', type: 'string', isIndexed: true}, ], }, + [TEAM_THREADS_SYNC]: { + name: TEAM_THREADS_SYNC, + unsafeSql: undefined, + columns: { + earliest: {name: 'earliest', type: 'number'}, + latest: {name: 'latest', type: 'number'}, + }, + columnArray: [ + {name: 'earliest', type: 'number'}, + {name: 'latest', type: 'number'}, + ], + }, [USER]: { name: USER, unsafeSql: undefined, diff --git a/app/queries/servers/thread.ts b/app/queries/servers/thread.ts index 3aae03cfd1..1dfd16aaa5 100644 --- a/app/queries/servers/thread.ts +++ b/app/queries/servers/thread.ts @@ -14,10 +14,11 @@ import {getConfig, observeConfigValue} from './system'; import type ServerDataOperator from '@database/operator/server_data_operator'; import type Model from '@nozbe/watermelondb/Model'; +import type TeamThreadsSyncModel from '@typings/database/models/servers/team_threads_sync'; import type ThreadModel from '@typings/database/models/servers/thread'; import type UserModel from '@typings/database/models/servers/user'; -const {SERVER: {CHANNEL, POST, THREAD, THREADS_IN_TEAM, THREAD_PARTICIPANT, USER}} = MM_TABLES; +const {SERVER: {CHANNEL, POST, THREAD, THREADS_IN_TEAM, THREAD_PARTICIPANT, TEAM_THREADS_SYNC, USER}} = MM_TABLES; export const getIsCRTEnabled = async (database: Database): Promise => { const config = await getConfig(database); @@ -34,6 +35,11 @@ export const getThreadById = async (database: Database, threadId: string) => { } }; +export const getTeamThreadsSyncData = async (database: Database, teamId: string): Promise => { + const result = await queryTeamThreadsSync(database, teamId).fetch(); + return result?.[0]; +}; + export const observeIsCRTEnabled = (database: Database) => { const cfgValue = observeConfigValue(database, 'CollapsedThreads'); const featureFlag = observeConfigValue(database, 'FeatureFlagCollapsedThreads'); @@ -133,7 +139,7 @@ export const prepareThreadsFromReceivedPosts = async (operator: ServerDataOperat return models; }; -export const queryThreadsInTeam = (database: Database, teamId: string, onlyUnreads?: boolean, hasReplies?: boolean, isFollowing?: boolean, sort?: boolean, limit?: number): Query => { +export const queryThreadsInTeam = (database: Database, teamId: string, onlyUnreads?: boolean, hasReplies?: boolean, isFollowing?: boolean, sort?: boolean, earliest?: number): Query => { const query: Q.Clause[] = []; if (isFollowing) { @@ -152,38 +158,22 @@ export const queryThreadsInTeam = (database: Database, teamId: string, onlyUnrea query.push(Q.sortBy('last_reply_at', Q.desc)); } - let joinCondition: Q.Condition = Q.where('team_id', teamId); - - if (!onlyUnreads) { - joinCondition = Q.and( - Q.where('team_id', teamId), - Q.where('loaded_in_global_threads', true), - ); - } - query.push( - Q.on(THREADS_IN_TEAM, joinCondition), + Q.on(THREADS_IN_TEAM, Q.where('team_id', teamId)), ); - if (limit) { - query.push(Q.take(limit)); + if (earliest) { + query.push(Q.where('last_reply_at', Q.gte(earliest))); } return database.get(THREAD).query(...query); }; -export async function getNewestThreadInTeam( - database: Database, - teamId: string, - unread: boolean, -): Promise { - try { - const threads = await queryThreadsInTeam(database, teamId, unread, true, true, true, 1).fetch(); - return threads?.[0] || undefined; - } catch (e) { - return undefined; - } -} +export const queryTeamThreadsSync = (database: Database, teamId: string) => { + return database.get(TEAM_THREADS_SYNC).query( + Q.where('id', teamId), + ); +}; export function observeThreadMentionCount(database: Database, teamId?: string, includeDmGm?: boolean): Observable { return observeUnreadsAndMentionsInTeam(database, teamId, includeDmGm).pipe( diff --git a/app/screens/global_threads/threads_list/index.ts b/app/screens/global_threads/threads_list/index.ts index d3e2ef30f3..a0f8642923 100644 --- a/app/screens/global_threads/threads_list/index.ts +++ b/app/screens/global_threads/threads_list/index.ts @@ -4,9 +4,10 @@ import {withDatabase} from '@nozbe/watermelondb/DatabaseProvider'; import withObservables from '@nozbe/with-observables'; import {AppStateStatus} from 'react-native'; +import {switchMap} from 'rxjs/operators'; import {observeCurrentTeamId} from '@queries/servers/system'; -import {queryThreadsInTeam} from '@queries/servers/thread'; +import {queryTeamThreadsSync, queryThreadsInTeam} from '@queries/servers/thread'; import {observeTeammateNameDisplay} from '@queries/servers/user'; import ThreadsList from './threads_list'; @@ -26,10 +27,17 @@ const withTeamId = withObservables([], ({database}: WithDatabaseArgs) => ({ const enhanced = withObservables(['tab', 'teamId', 'forceQueryAfterAppState'], ({database, tab, teamId}: Props) => { const getOnlyUnreads = tab !== 'all'; + const teamThreadsSyncObserver = queryTeamThreadsSync(database, teamId).observeWithColumns(['earliest']); + return { unreadsCount: queryThreadsInTeam(database, teamId, true).observeCount(false), teammateNameDisplay: observeTeammateNameDisplay(database), - threads: queryThreadsInTeam(database, teamId, getOnlyUnreads, false, true, true).observe(), + threads: teamThreadsSyncObserver.pipe( + switchMap((teamThreadsSync) => { + const earliest = teamThreadsSync?.[0]?.earliest; + return queryThreadsInTeam(database, teamId, getOnlyUnreads, false, true, true, earliest).observe(); + }), + ), }; }); diff --git a/app/screens/global_threads/threads_list/threads_list.tsx b/app/screens/global_threads/threads_list/threads_list.tsx index 303a07fb2d..dda1999a94 100644 --- a/app/screens/global_threads/threads_list/threads_list.tsx +++ b/app/screens/global_threads/threads_list/threads_list.tsx @@ -4,7 +4,7 @@ import React, {useCallback, useEffect, useMemo, useState, useRef} from 'react'; import {FlatList, ListRenderItemInfo, StyleSheet} from 'react-native'; -import {fetchNewThreads, fetchRefreshThreads, fetchThreads} from '@actions/remote/thread'; +import {loadEarlierThreads, syncTeamThreads} from '@actions/remote/thread'; import Loading from '@components/loading'; import {General, Screens} from '@constants'; import {useServerUrl} from '@context/server'; @@ -57,6 +57,7 @@ const ThreadsList = ({ const serverUrl = useServerUrl(); const theme = useTheme(); + const flatListRef = useRef>(null); const hasFetchedOnce = useRef(false); const [isLoading, setIsLoading] = useState(false); const [endReached, setEndReached] = useState(false); @@ -66,22 +67,21 @@ const ThreadsList = ({ const lastThread = threads?.length > 0 ? threads[threads.length - 1] : null; useEffect(() => { - // This is to be called only when there are no threads - if (tab === 'all' && noThreads && !hasFetchedOnce.current) { - setIsLoading(true); - fetchThreads(serverUrl, teamId).finally(() => { - hasFetchedOnce.current = true; - setIsLoading(false); - }); + if (hasFetchedOnce.current || tab !== 'all') { + return; } - }, [noThreads, serverUrl, tab]); - useEffect(() => { - // This is to be called when threads already exist locally and to fetch the latest threads + // Display loading only when there are no threads if (!noThreads) { - fetchNewThreads(serverUrl, teamId); + setIsLoading(true); } - }, [noThreads, serverUrl, teamId]); + syncTeamThreads(serverUrl, teamId).then(() => { + hasFetchedOnce.current = true; + }); + if (!noThreads) { + setIsLoading(false); + } + }, [noThreads, serverUrl, tab, teamId]); const listEmptyComponent = useMemo(() => { if (isLoading) { @@ -118,33 +118,33 @@ const ThreadsList = ({ return null; }, [isLoading, tab, theme, endReached]); + const handleTabChange = useCallback((value: GlobalThreadsTab) => { + setTab(value); + flatListRef.current?.scrollToOffset({animated: true, offset: 0}); + }, [setTab]); + const handleRefresh = useCallback(() => { setRefreshing(true); - fetchRefreshThreads(serverUrl, teamId, tab === 'unreads').finally(() => { + syncTeamThreads(serverUrl, teamId).finally(() => { setRefreshing(false); }); }, [serverUrl, teamId]); const handleEndReached = useCallback(() => { - if (!lastThread || tab === 'unreads' || endReached) { + if (tab === 'unreads' || endReached || !lastThread) { return; } - const options = { - before: lastThread.id, - perPage: General.CRT_CHUNK_SIZE, - }; - setIsLoading(true); - fetchThreads(serverUrl, teamId, options).then((response) => { - if ('data' in response) { - setEndReached(response.data.threads.length < General.CRT_CHUNK_SIZE); + loadEarlierThreads(serverUrl, teamId, lastThread.id).then((response) => { + if (response.threads) { + setEndReached(response.threads.length < General.CRT_CHUNK_SIZE); } }).finally(() => { setIsLoading(false); }); - }, [endReached, lastThread?.id, serverUrl, tab, teamId]); + }, [endReached, serverUrl, tab, teamId, lastThread]); const renderItem = useCallback(({item}: ListRenderItemInfo) => (
{ + // Sort a clone of 'threads' array by last_reply_at + const sortedThreads = [...threads].sort((a, b) => { + return a.last_reply_at - b.last_reply_at; + }); + + const earliestThread = sortedThreads[0]; + const latestThread = sortedThreads[sortedThreads.length - 1]; + + return {earliestThread, latestThread}; +}; diff --git a/types/database/database.ts b/types/database/database.ts index 24deead0fe..3041c2b068 100644 --- a/types/database/database.ts +++ b/types/database/database.ts @@ -90,7 +90,6 @@ export type HandleThreadsArgs = { threads?: ThreadWithLastFetchedAt[]; prepareRecordsOnly?: boolean; teamId?: string; - loadedInGlobalThreads?: boolean; }; export type HandleThreadParticipantsArgs = { @@ -102,7 +101,11 @@ export type HandleThreadParticipantsArgs = { export type HandleThreadInTeamArgs = { threadsMap?: Record; prepareRecordsOnly?: boolean; - loadedInGlobalThreads?: boolean; +}; + +export type HandleTeamThreadsSyncArgs = { + data: TeamThreadsSync[]; + prepareRecordsOnly?: boolean; }; export type SanitizeReactionsArgs = { diff --git a/types/database/models/servers/team_threads_sync.ts b/types/database/models/servers/team_threads_sync.ts new file mode 100644 index 0000000000..f40d6d6715 --- /dev/null +++ b/types/database/models/servers/team_threads_sync.ts @@ -0,0 +1,29 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import type TeamModel from './team'; +import type {Relation, Model} from '@nozbe/watermelondb'; +import type {Associations} from '@nozbe/watermelondb/Model'; + +/** + * ThreadInTeam model helps us to sync threads without creating any gaps between the threads + * by keeping track of the latest and earliest last_replied_at timestamps loaded for a team. + */ +declare class TeamThreadsSyncModel extends Model { + /** table (name) : TeamThreadsSync */ + static table: string; + + /** associations : Describes every relationship to this table. */ + static associations: Associations; + + /** earliest: Oldest last_replied_at loaded through infinite loading */ + earliest: number; + + /** latest: Newest last_replied_at loaded during app init / navigating to global threads / pull to refresh */ + latest: number; + + /** team : The related record to the parent Team model */ + team: Relation; +} + +export default TeamThreadsSyncModel; diff --git a/types/database/models/servers/thread_in_team.ts b/types/database/models/servers/thread_in_team.ts index 601fa09c05..d8a0325772 100644 --- a/types/database/models/servers/thread_in_team.ts +++ b/types/database/models/servers/thread_in_team.ts @@ -24,9 +24,6 @@ declare class ThreadInTeamModel extends Model { /** teamId: Associated thread identifier */ teamId: string; - /** loaded_in_global_threads : Flag to differentiate the unread threads loaded for showing unread counts/mentions */ - loadedInGlobalThreads: boolean; - /** thread : The related record to the parent Thread model */ thread: Relation; diff --git a/types/database/raw_values.d.ts b/types/database/raw_values.d.ts index 5a335a5b47..3d6e40a186 100644 --- a/types/database/raw_values.d.ts +++ b/types/database/raw_values.d.ts @@ -91,7 +91,12 @@ type TermsOfService = { type ThreadInTeam = { thread_id: string; team_id: string; - loaded_in_global_threads: boolean; +}; + +type TeamThreadsSync = { + id: string; + earliest: number; + latest: number; }; type RawValue = @@ -125,5 +130,6 @@ type RawValue = | ThreadWithLastFetchedAt | ThreadInTeam | ThreadParticipant + | TeamThreadsSync | UserProfile | Pick