e2net/iceshrimp
3
0
Fork 0
forked from mirrors/iceshrimp
Code Pull requests Activity

[mastodon-client] Improve html cache performance

Browse source
This commit is contained in:
Laura Hausmann 2023-11-26 22:21:47 +01:00
parent 61c532a854
commit 7ab7edeefd
Signed by: zotan
GPG key ID: D044E84C5BE01605
5 changed files with 61 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
packages/backend/src/remote/activitypub/models/person.ts
View file

@ -617,20 +617,22 @@ export async function updatePerson(
);
}
await UserProfiles.update(
{ userId: user.id },
{
url: url,
fields,
description: person._misskey_summary
? truncate(person._misskey_summary, summaryLength)
: person.summary
? await htmlToMfm(truncate(person.summary, summaryLength), person.tag)
: null,
birthday: bday ? bday[0] : null,
location: person["vcard:Address"] || null,
},
);
// Get old profile to see if we need to update any matching html cache entries
const oldProfile = await UserProfiles.findOneBy({ userId: user.id });
const newProfile = {
url: url,
fields,
description: person._misskey_summary
? truncate(person._misskey_summary, summaryLength)
: person.summary
? await htmlToMfm(truncate(person.summary, summaryLength), person.tag)
: null,
birthday: bday ? bday[0] : null,
location: person["vcard:Address"] || null
} as Partial<UserProfile>;
await UserProfiles.update({ userId: user.id }, newProfile);
publishInternalEvent("remoteUserUpdated", { id: user.id });
@ -639,7 +641,7 @@ export async function updatePerson(
// Mentions update, then prewarm html cache
UserProfiles.updateMentions(user!.id)
.then(_ => UserConverter.prewarmCacheById(user!.id));
.then(_ => UserConverter.prewarmCacheById(user!.id, oldProfile));
// If the user in question is a follower, followers will also be updated.
await Followings.update(

4
packages/backend/src/server/api/mastodon/converters/note.ts
View file

@ -182,7 +182,7 @@ export class NoteConverter {
return Promise.all(encoded);
}
private static async aggregateData(notes: Note[], ctx: MastoContext): Promise<void> {
public static async aggregateData(notes: Note[], ctx: MastoContext): Promise<void> {
if (notes.length === 0) return;
const user = ctx.user as ILocalUser | null;
@ -307,7 +307,7 @@ export class NoteConverter {
return Promise.resolve(dbHit)
.then(res => {
if (res === null || (res.updatedAt !== note.updatedAt)) {
if (res === null || (res.updatedAt?.getTime() !== note.updatedAt?.getTime())) {
this.prewarmCache(note);
return null;
}

14
packages/backend/src/server/api/mastodon/converters/notification.ts
View file

@ -1,4 +1,4 @@
import { ILocalUser } from "@/models/entities/user.js";
import { ILocalUser, User } from "@/models/entities/user.js";
import { Notification } from "@/models/entities/notification.js";
import { notificationTypes } from "@/types.js";
import { UserConverter } from "@/server/api/mastodon/converters/user.js";
@ -9,6 +9,8 @@ import { getNote } from "@/server/api/common/getters.js";
import { getStubMastoContext, MastoContext } from "@/server/api/mastodon/index.js";
import { Notifications } from "@/models/index.js";
import isQuote from "@/misc/is-quote.js";
import { unique } from "@/prelude/array.js";
import { Note } from "@/models/entities/note.js";
type NotificationType = typeof notificationTypes[number];
@ -51,11 +53,21 @@ export class NotificationConverter {
}
public static async encodeMany(notifications: Notification[], ctx: MastoContext): Promise<MastodonEntity.Notification[]> {
await this.aggregateData(notifications, ctx);
const encoded = notifications.map(u => this.encode(u, ctx));
return Promise.all(encoded)
.then(p => p.filter(n => n !== null) as MastodonEntity.Notification[]);
}
private static async aggregateData(notifications: Notification[], ctx: MastoContext): Promise<void> {
if (notifications.length === 0) return;
const notes = unique(notifications.filter(p => p.note != null).map((n) => n.note as Note));
const users = unique(notifications.filter(p => p.notifier != null).map(n => n.notifier as User)
.concat(notifications.filter(p => p.notifiee != null).map(n => n.notifiee as User)));
await NoteConverter.aggregateData(notes, ctx);
await UserConverter.aggregateData(users, ctx);
}
private static encodeNotificationType(t: NotificationType): MastodonEntity.NotificationType {
//FIXME: Implement custom notification for followRequestAccepted
//FIXME: Implement mastodon notification type 'update' on misskey side

39
packages/backend/src/server/api/mastodon/converters/user.ts
View file

@ -1,6 +1,6 @@
import { ILocalUser, User } from "@/models/entities/user.js";
import config from "@/config/index.js";
import {DriveFiles, Followings, HtmlUserCacheEntries, UserProfiles, Users} from "@/models/index.js";
import { DriveFiles, Followings, HtmlUserCacheEntries, UserProfiles, Users } from "@/models/index.js";
import { EmojiConverter } from "@/server/api/mastodon/converters/emoji.js";
import { populateEmojis } from "@/misc/populate-emojis.js";
import { escapeMFM } from "@/server/api/mastodon/converters/mfm.js";
@ -9,7 +9,7 @@ import { awaitAll } from "@/prelude/await-all.js";
import { AccountCache, UserHelpers } from "@/server/api/mastodon/helpers/user.js";
import { MfmHelpers } from "@/server/api/mastodon/helpers/mfm.js";
import { MastoContext } from "@/server/api/mastodon/index.js";
import {IMentionedRemoteUsers, Note} from "@/models/entities/note.js";
import { IMentionedRemoteUsers, Note } from "@/models/entities/note.js";
import { UserProfile } from "@/models/entities/user-profile.js";
import { In } from "typeorm";
import { unique } from "@/prelude/array.js";
@ -35,7 +35,7 @@ export class UserConverter {
const cacheHit = cache.accounts.find(p => p.id == u.id);
if (cacheHit) return cacheHit;
const identifier = `${u.id}:${(u.updatedAt ?? u.createdAt).getTime()}`;
const identifier = `${u.id}:${(u.lastFetchedAt ?? u.createdAt).getTime()}`;
let fqn = `${u.username}@${u.host ?? config.domain}`;
let acct = u.username;
let acctUrl = `https://${u.host || config.host}/@${u.username}`;
@ -243,7 +243,7 @@ export class UserConverter {
return Promise.resolve(dbHit)
.then(res => {
if (res === null || (res.updatedAt !== user.updatedAt ?? user.createdAt)) {
if (res === null || (res.updatedAt.getTime() !== (user.lastFetchedAt ?? user.createdAt).getTime())) {
this.prewarmCache(user, profile);
return null;
}
@ -251,13 +251,24 @@ export class UserConverter {
});
}
public static async prewarmCache(user: User, profile?: UserProfile | null): Promise<void> {
if (!config.htmlCache?.prewarm) return;
const identifier = `${user.id}:${(user.updatedAt ?? user.createdAt).getTime()}`;
public static async prewarmCache(user: User, profile?: UserProfile | null, oldProfile?: UserProfile | null): Promise<void> {
const identifier = `${user.id}:${(user.lastFetchedAt ?? user.createdAt).getTime()}`;
if (profile !== null) {
if (profile === undefined) {
profile = await UserProfiles.findOneBy({userId: user.id});
}
if (config.htmlCache?.dbFallback) {
if (profile === undefined) {
profile = await UserProfiles.findOneBy({ userId: user.id });
}
if (oldProfile !== undefined && profile?.fields === oldProfile?.fields && profile?.description === oldProfile?.description) {
HtmlUserCacheEntries.update({ userId: user.id }, { updatedAt: user.lastFetchedAt ?? user.createdAt });
return;
}
}
if (!config.htmlCache?.prewarm) return;
if (profile === undefined) {
profile = await UserProfiles.findOneBy({ userId: user.id });
}
if (await this.userBioHtmlCache.get(identifier) === undefined) {
const bio = MfmHelpers.toHtml(mfm.parse(profile?.description ?? ""), profile?.mentions, user.host)
@ -267,7 +278,7 @@ export class UserConverter {
this.userBioHtmlCache.set(identifier, await bio);
if (config.htmlCache?.dbFallback)
HtmlUserCacheEntries.upsert({ userId: user.id, bio: await bio }, ["userId"]);
HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.lastFetchedAt ?? user.createdAt, bio: await bio }, ["userId"]);
}
if (await this.userFieldsHtmlCache.get(identifier) === undefined) {
@ -275,12 +286,12 @@ export class UserConverter {
this.userFieldsHtmlCache.set(identifier, fields);
if (config.htmlCache?.dbFallback)
HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.updatedAt ?? user.createdAt, fields: fields }, ["userId"]);
HtmlUserCacheEntries.upsert({ userId: user.id, updatedAt: user.lastFetchedAt ?? user.createdAt, fields: fields }, ["userId"]);
}
}
}
public static async prewarmCacheById(userId: string): Promise<void> {
await this.prewarmCache(await getUser(userId));
public static async prewarmCacheById(userId: string, oldProfile?: UserProfile | null): Promise<void> {
await this.prewarmCache(await getUser(userId), undefined, oldProfile);
}
}

5
packages/backend/src/server/api/mastodon/helpers/notification.ts
View file

@ -31,7 +31,10 @@ export class NotificationHelpers {
if (accountId !== undefined)
query.andWhere("notification.notifierId = :notifierId", { notifierId: accountId });
query.leftJoinAndSelect("notification.note", "note");
query
.leftJoinAndSelect("notification.note", "note")
.leftJoinAndSelect("notification.notifier", "notifier")
.leftJoinAndSelect("notification.notifiee", "notifiee");
return PaginationHelpers.execQueryLinkPagination(query, limit, minId !== undefined, ctx);
}