From 7ab7edeefd344487964ddfb6d4f7d71a6601a519 Mon Sep 17 00:00:00 2001 From: Laura Hausmann Date: Sun, 26 Nov 2023 22:21:47 +0100 Subject: [PATCH] [mastodon-client] Improve html cache performance --- .../src/remote/activitypub/models/person.ts | 32 ++++++++------- .../server/api/mastodon/converters/note.ts | 4 +- .../api/mastodon/converters/notification.ts | 14 ++++++- .../server/api/mastodon/converters/user.ts | 39 ++++++++++++------- .../api/mastodon/helpers/notification.ts | 5 ++- 5 files changed, 61 insertions(+), 33 deletions(-) diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index c021dd679..fe2887d02 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -617,20 +617,22 @@ export async function updatePerson( ); } - await UserProfiles.update( - { userId: user.id }, - { - url: url, - fields, - description: person._misskey_summary - ? truncate(person._misskey_summary, summaryLength) - : person.summary - ? await htmlToMfm(truncate(person.summary, summaryLength), person.tag) - : null, - birthday: bday ? bday[0] : null, - location: person["vcard:Address"] || null, - }, - ); + // Get old profile to see if we need to update any matching html cache entries + const oldProfile = await UserProfiles.findOneBy({ userId: user.id }); + + const newProfile = { + url: url, + fields, + description: person._misskey_summary + ? truncate(person._misskey_summary, summaryLength) + : person.summary + ? await htmlToMfm(truncate(person.summary, summaryLength), person.tag) + : null, + birthday: bday ? bday[0] : null, + location: person["vcard:Address"] || null + } as Partial; + + await UserProfiles.update({ userId: user.id }, newProfile); publishInternalEvent("remoteUserUpdated", { id: user.id }); @@ -639,7 +641,7 @@ export async function updatePerson( // Mentions update, then prewarm html cache UserProfiles.updateMentions(user!.id) - .then(_ => UserConverter.prewarmCacheById(user!.id)); + .then(_ => UserConverter.prewarmCacheById(user!.id, oldProfile)); // If the user in question is a follower, followers will also be updated. await Followings.update( diff --git a/packages/backend/src/server/api/mastodon/converters/note.ts b/packages/backend/src/server/api/mastodon/converters/note.ts index a02249b97..1813cf954 100644 --- a/packages/backend/src/server/api/mastodon/converters/note.ts +++ b/packages/backend/src/server/api/mastodon/converters/note.ts @@ -182,7 +182,7 @@ export class NoteConverter { return Promise.all(encoded); } - private static async aggregateData(notes: Note[], ctx: MastoContext): Promise { + public static async aggregateData(notes: Note[], ctx: MastoContext): Promise { if (notes.length === 0) return; const user = ctx.user as ILocalUser | null; @@ -307,7 +307,7 @@ export class NoteConverter { return Promise.resolve(dbHit) .then(res => { - if (res === null || (res.updatedAt !== note.updatedAt)) { + if (res === null || (res.updatedAt?.getTime() !== note.updatedAt?.getTime())) { this.prewarmCache(note); return null; } diff --git a/packages/backend/src/server/api/mastodon/converters/notification.ts b/packages/backend/src/server/api/mastodon/converters/notification.ts index 80868ecb9..e0c600cf5 100644 --- a/packages/backend/src/server/api/mastodon/converters/notification.ts +++ b/packages/backend/src/server/api/mastodon/converters/notification.ts @@ -1,4 +1,4 @@ -import { ILocalUser } from "@/models/entities/user.js"; +import { ILocalUser, User } from "@/models/entities/user.js"; import { Notification } from "@/models/entities/notification.js"; import { notificationTypes } from "@/types.js"; import { UserConverter } from "@/server/api/mastodon/converters/user.js"; @@ -9,6 +9,8 @@ import { getNote } from "@/server/api/common/getters.js"; import { getStubMastoContext, MastoContext } from "@/server/api/mastodon/index.js"; import { Notifications } from "@/models/index.js"; import isQuote from "@/misc/is-quote.js"; +import { unique } from "@/prelude/array.js"; +import { Note } from "@/models/entities/note.js"; type NotificationType = typeof notificationTypes[number]; @@ -51,11 +53,21 @@ export class NotificationConverter { } public static async encodeMany(notifications: Notification[], ctx: MastoContext): Promise { + await this.aggregateData(notifications, ctx); const encoded = notifications.map(u => this.encode(u, ctx)); return Promise.all(encoded) .then(p => p.filter(n => n !== null) as MastodonEntity.Notification[]); } + private static async aggregateData(notifications: Notification[], ctx: MastoContext): Promise { + if (notifications.length === 0) return; + const notes = unique(notifications.filter(p => p.note != null).map((n) => n.note as Note)); + const users = unique(notifications.filter(p => p.notifier != null).map(n => n.notifier as User) + .concat(notifications.filter(p => p.notifiee != null).map(n => n.notifiee as User))); + await NoteConverter.aggregateData(notes, ctx); + await UserConverter.aggregateData(users, ctx); + } + private static encodeNotificationType(t: NotificationType): MastodonEntity.NotificationType { //FIXME: Implement custom notification for followRequestAccepted //FIXME: Implement mastodon notification type 'update' on misskey side diff --git a/packages/backend/src/server/api/mastodon/converters/user.ts b/packages/backend/src/server/api/mastodon/converters/user.ts index 5cf5e559d..7c0849d1f 100644 --- a/packages/backend/src/server/api/mastodon/converters/user.ts +++ b/packages/backend/src/server/api/mastodon/converters/user.ts @@ -1,6 +1,6 @@ import { ILocalUser, User } from "@/models/entities/user.js"; import config from "@/config/index.js"; -import {DriveFiles, Followings, HtmlUserCacheEntries, UserProfiles, Users} from "@/models/index.js"; +import { DriveFiles, Followings, HtmlUserCacheEntries, UserProfiles, Users } from "@/models/index.js"; import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js"; import { populateEmojis } from "@/misc/populate-emojis.js"; import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js"; @@ -9,7 +9,7 @@ import { awaitAll } from "@/prelude/await-all.js"; import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js"; import { MfmHelpers } from "@/server/api/mastodon/helpers/mfm.js"; import { MastoContext } from "@/server/api/mastodon/index.js"; -import {IMentionedRemoteUsers, Note} from "@/models/entities/note.js"; +import { IMentionedRemoteUsers, Note } from "@/models/entities/note.js"; import { UserProfile } from "@/models/entities/user-profile.js"; import { In } from "typeorm"; import { unique } from "@/prelude/array.js"; @@ -35,7 +35,7 @@ export class UserConverter { const cacheHit = cache.accounts.find(p => p.id == u.id); if (cacheHit) return cacheHit; - const identifier = `${u.id}:${(u.updatedAt ?? u.createdAt).getTime()}`; + const identifier = `${u.id}:${(u.lastFetchedAt ?? u.createdAt).getTime()}`; let fqn = `${u.username}@${u.host ?? config.domain}`; let acct = u.username; let acctUrl = `https://${u.host || config.host}/@${u.username}`; @@ -243,7 +243,7 @@ export class UserConverter { return Promise.resolve(dbHit) .then(res => { - if (res === null || (res.updatedAt !== user.updatedAt ?? user.createdAt)) { + if (res === null || (res.updatedAt.getTime() !== (user.lastFetchedAt ?? user.createdAt).getTime())) { this.prewarmCache(user, profile); return null; } @@ -251,13 +251,24 @@ export class UserConverter { }); } - public static async prewarmCache(user: User, profile?: UserProfile | null): Promise { - if (!config.htmlCache?.prewarm) return; - const identifier = `${user.id}:${(user.updatedAt ?? user.createdAt).getTime()}`; + public static async prewarmCache(user: User, profile?: UserProfile | null, oldProfile?: UserProfile | null): Promise { + const identifier = `${user.id}:${(user.lastFetchedAt ?? user.createdAt).getTime()}`; if (profile !== null) { - if (profile === undefined) { - profile = await UserProfiles.findOneBy({userId: user.id}); - } + if (config.htmlCache?.dbFallback) { + if (profile === undefined) { + profile = await UserProfiles.findOneBy({ userId: user.id }); + } + if (oldProfile !== undefined && profile?.fields === oldProfile?.fields && profile?.description === oldProfile?.description) { + HtmlUserCacheEntries.update({ userId: user.id }, { updatedAt: user.lastFetchedAt ?? user.createdAt }); + return; + } + } + + if (!config.htmlCache?.prewarm) return; + + if (profile === undefined) { + profile = await UserProfiles.findOneBy({ userId: user.id }); + } if (await this.userBioHtmlCache.get(identifier) === undefined) { const bio = MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, user.host) @@ -267,7 +278,7 @@ export class UserConverter { this.userBioHtmlCache.set(identifier, await bio); if (config.htmlCache?.dbFallback) - HtmlUserCacheEntries.upsert({ userId: user.id, bio: await bio }, ["userId"]); + HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.lastFetchedAt ?? user.createdAt, bio: await bio }, ["userId"]); } if (await this.userFieldsHtmlCache.get(identifier) === undefined) { @@ -275,12 +286,12 @@ export class UserConverter { this.userFieldsHtmlCache.set(identifier, fields); if (config.htmlCache?.dbFallback) - HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.updatedAt ?? user.createdAt, fields: fields }, ["userId"]); + HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.lastFetchedAt ?? user.createdAt, fields: fields }, ["userId"]); } } } - public static async prewarmCacheById(userId: string): Promise { - await this.prewarmCache(await getUser(userId)); + public static async prewarmCacheById(userId: string, oldProfile?: UserProfile | null): Promise { + await this.prewarmCache(await getUser(userId), undefined, oldProfile); } } diff --git a/packages/backend/src/server/api/mastodon/helpers/notification.ts b/packages/backend/src/server/api/mastodon/helpers/notification.ts index 1d1647b66..7397a2dea 100644 --- a/packages/backend/src/server/api/mastodon/helpers/notification.ts +++ b/packages/backend/src/server/api/mastodon/helpers/notification.ts @@ -31,7 +31,10 @@ export class NotificationHelpers { if (accountId !== undefined) query.andWhere("notification.notifierId = :notifierId", { notifierId: accountId }); - query.leftJoinAndSelect("notification.note", "note"); + query + .leftJoinAndSelect("notification.note", "note") + .leftJoinAndSelect("notification.notifier", "notifier") + .leftJoinAndSelect("notification.notifiee", "notifiee"); return PaginationHelpers.execQueryLinkPagination(query, limit, minId !== undefined, ctx); }