e2net/iceshrimp
3
0
Fork 0
forked from mirrors/iceshrimp
Code Pull requests Activity

[backend] Use a recursion limiter for user profile mentions instead of disabling recursion altogether

Browse source
This commit is contained in:
Laura Hausmann 2023-10-25 13:16:54 +02:00
parent 9b13ec9c0c
commit 2575588fa3
Signed by: zotan
GPG key ID: D044E84C5BE01605
5 changed files with 47 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
packages/backend/src/models/repositories/user-profile.ts
View file

@ -6,13 +6,13 @@ import { resolveMentionToUserAndProfile } from "@/remote/resolve-user.js";
import { IMentionedRemoteUsers } from "@/models/entities/note.js";
import { unique } from "@/prelude/array.js";
import config from "@/config/index.js";
import { Semaphore } from "async-mutex";
import { Mutex, Semaphore } from "async-mutex";
const queue = new Semaphore(5);
export const UserProfileRepository = db.getRepository(UserProfile).extend({
// We must never await this without promiseEarlyReturn, otherwise giant webring-style profile mention trees will cause the queue to stop working
async updateMentions(id: UserProfile["userId"]){
async updateMentions(id: UserProfile["userId"], limiter: RecursionLimiter = new RecursionLimiter(20)){
const profile = await this.findOneBy({ userId: id });
if (!profile) return;
const tokens: mfm.MfmNode[] = [];
@ -24,16 +24,16 @@ export const UserProfileRepository = db.getRepository(UserProfile).extend({
return queue.runExclusive(async () => {
const partial = {
mentions: await populateMentions(tokens, profile.userHost)
mentions: await populateMentions(tokens, profile.userHost, limiter)
};
return UserProfileRepository.update(profile.userId, partial);
});
},
});
async function populateMentions(tokens: mfm.MfmNode[], objectHost: string | null): Promise<IMentionedRemoteUsers> {
async function populateMentions(tokens: mfm.MfmNode[], objectHost: string | null, limiter: RecursionLimiter): Promise<IMentionedRemoteUsers> {
const mentions = extractMentions(tokens);
const resolved = await Promise.all(mentions.map(m => resolveMentionToUserAndProfile(m.username, m.host, objectHost)));
const resolved = await Promise.all(mentions.map(m => resolveMentionToUserAndProfile(m.username, m.host, objectHost, limiter)));
const remote = resolved.filter(p => p && p.data.host !== config.domain && (p.data.host !== null || objectHost !== null))
.map(p => p!);
const res = remote.map(m => {
@ -47,3 +47,17 @@ async function populateMentions(tokens: mfm.MfmNode[], objectHost: string | null
return unique(res);
}
export class RecursionLimiter {
private counter;
private mutex = new Mutex();
constructor(count: number = 20) {
this.counter = count;
}
public shouldContinue(): Promise<boolean> {
return this.mutex.runExclusive(() => {
return this.counter-- > 0;
});
}
}

4
packages/backend/src/remote/activitypub/models/mention.ts
View file

@ -6,9 +6,11 @@ import type { IObject, IApMention } from "../type.js";
import { isMention } from "../type.js";
import Resolver from "../resolver.js";
import { resolvePerson } from "./person.js";
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
export async function extractApMentions(
tags: IObject | IObject[] | null | undefined,
limiter: RecursionLimiter = new RecursionLimiter(20)
) {
const hrefs = unique(
extractApMentionObjects(tags).map((x) => x.href as string),
@ -20,7 +22,7 @@ export async function extractApMentions(
const mentionedUsers = (
await Promise.all(
hrefs.map((x) =>
limit(() => resolvePerson(x, resolver).catch(() => null)),
limit(() => resolvePerson(x, resolver, limiter).catch(() => null)),
),
)
).filter((x): x is CacheableUser => x != null);

14
packages/backend/src/remote/activitypub/models/note.ts
View file

@ -53,6 +53,7 @@ import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
import { truncate } from "@/misc/truncate.js";
import { type Size, getEmojiSize } from "@/misc/emoji-meta.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
const logger = apLogger;
@ -108,6 +109,7 @@ export async function createNote(
value: string | IObject,
resolver?: Resolver,
silent = false,
limiter: RecursionLimiter = new RecursionLimiter(20)
): Promise<Note | null> {
if (resolver == null) resolver = new Resolver();
@ -163,6 +165,7 @@ export async function createNote(
const actor = (await resolvePerson(
getOneApId(note.attributedTo),
resolver,
limiter
)) as CacheableRemoteUser;
// Skip if author is suspended.
@ -188,8 +191,8 @@ export async function createNote(
let isTalk = note._misskey_talk && visibility === "specified";
const apMentions = await extractApMentions(note.tag);
const apHashtags = await extractApHashtags(note.tag);
const apMentions = await extractApMentions(note.tag, limiter);
const apHashtags = extractApHashtags(note.tag);
// Attachments
// TODO: attachmentは必ずしもImageではない
@ -216,7 +219,7 @@ export async function createNote(
// Reply
const reply: Note | null = note.inReplyTo
? await resolveNote(note.inReplyTo, resolver)
? await resolveNote(note.inReplyTo, resolver, limiter)
.then((x) => {
if (x == null) {
logger.warn("Specified inReplyTo, but nout found");
@ -262,7 +265,7 @@ export async function createNote(
if (typeof uri !== "string" || !uri.match(/^https?:/))
return { status: "permerror" };
try {
const res = await resolveNote(uri);
const res = await resolveNote(uri, undefined, limiter);
if (res) {
return {
status: "ok",
@ -403,6 +406,7 @@ export async function createNote(
export async function resolveNote(
value: string | IObject,
resolver?: Resolver,
limiter: RecursionLimiter = new RecursionLimiter(20)
): Promise<Note | null> {
const uri = typeof value === "string" ? value : value.id;
if (uri == null) throw new Error("missing uri");
@ -437,7 +441,7 @@ export async function resolveNote(
// Fetch from remote server and register
// If the attached `Note` Object is specified here instead of the uri, the note will be generated without going through the server fetch.
// Since the attached Note Object may be disguised, always specify the uri and fetch it from the server.
return await createNote(uri, resolver, true);
return await createNote(uri, resolver, true, limiter);
} finally {
unlock();
}

16
packages/backend/src/remote/activitypub/models/person.ts
View file

@ -53,6 +53,7 @@ import {
getSubjectHostFromRemoteUser,
getSubjectHostFromAcctParts
} from "@/remote/resolve-user.js"
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
const logger = apLogger;
@ -170,7 +171,7 @@ export async function createPerson(
uri: string,
resolver?: Resolver,
subjectHost?: string,
skipMentions: boolean = false
limiter: RecursionLimiter = new RecursionLimiter(20)
): Promise<User> {
if (typeof uri !== "string") throw new Error("uri is not string");
@ -400,7 +401,7 @@ export async function createPerson(
updateUsertags(user!, tags);
// Mentions update
if (!skipMentions) UserProfiles.updateMentions(user!.id);
if (await limiter.shouldContinue()) UserProfiles.updateMentions(user!.id, limiter);
//#region Fetch avatar and header image
const [avatar, banner] = await Promise.all(
@ -436,7 +437,7 @@ export async function createPerson(
});
//#endregion
await updateFeatured(user!.id, resolver).catch((err) => logger.error(err));
await updateFeatured(user!.id, resolver, limiter).catch((err) => logger.error(err));
return user!;
}
@ -646,6 +647,7 @@ export async function updatePerson(
export async function resolvePerson(
uri: string,
resolver?: Resolver,
limiter: RecursionLimiter = new RecursionLimiter(20)
): Promise<CacheableUser> {
if (typeof uri !== "string") throw new Error("uri is not string");
@ -659,7 +661,7 @@ export async function resolvePerson(
// Fetched from remote server and registered
if (resolver == null) resolver = new Resolver();
return await createPerson(uri, resolver);
return await createPerson(uri, resolver, undefined, limiter);
}
const services: {
@ -720,7 +722,7 @@ export async function analyzeAttachments(
return { fields, services };
}
export async function updateFeatured(userId: User["id"], resolver?: Resolver) {
export async function updateFeatured(userId: User["id"], resolver?: Resolver, limiter: RecursionLimiter = new RecursionLimiter(20)) {
const user = await Users.findOneByOrFail({ id: userId });
if (!Users.isRemoteUser(user)) return;
if (!user.featured) return;
@ -742,14 +744,14 @@ export async function updateFeatured(userId: User["id"], resolver?: Resolver) {
toArray(unresolvedItems).map((x) => resolver.resolve(x)),
);
// Resolve and regist Notes
// Resolve and register Notes
resolver.reset();
const limit = promiseLimit<Note | null>(2);
const featuredNotes = await Promise.all(
items
.filter((item) => getApType(item) === "Note") // TODO: Maybe it doesn't have to be a Note.
.slice(0, 5)
.map((item) => limit(() => resolveNote(item, resolver))),
.map((item) => limit(() => resolveNote(item, resolver, limiter))),
);
await db.transaction(async (transactionalEntityManager) => {

13
packages/backend/src/remote/resolve-user.ts
View file

@ -11,6 +11,7 @@ import { remoteLogger } from "./logger.js";
import { Cache } from "@/misc/cache.js";
import { IMentionedRemoteUsers } from "@/models/entities/note.js";
import { UserProfile } from "@/models/entities/user-profile.js";
import { RecursionLimiter } from "@/models/repositories/user-profile.js";
const logger = remoteLogger.createSubLogger("resolve-user");
const uriHostCache = new Cache<string>("resolveUserUriHost", 60 * 60 * 24);
@ -31,7 +32,7 @@ export async function resolveUser(
host: string | null,
refresh: boolean = true,
awaitRefresh: boolean = true,
skipMentionsOnCreate: boolean = false
limiter: RecursionLimiter = new RecursionLimiter(20)
): Promise<User> {
const usernameLower = username.toLowerCase();
@ -105,14 +106,14 @@ export async function resolveUser(
// Otherwise create and return new user
else {
logger.succ(`return new remote user: ${chalk.magenta(finalAcctLower)}`);
return await createPerson(fingerRes.self.href, undefined, subjectHost, skipMentionsOnCreate);
return await createPerson(fingerRes.self.href, undefined, subjectHost, limiter);
}
}
}
// Not a split domain setup, so we can simply create and return the new user
logger.succ(`return new remote user: ${chalk.magenta(finalAcctLower)}`);
return await createPerson(fingerRes.self.href, undefined, subjectHost, skipMentionsOnCreate);
return await createPerson(fingerRes.self.href, undefined, subjectHost, limiter);
}
// If user information is out of date, return it by starting over from WebFinger
@ -188,17 +189,17 @@ export async function resolveUser(
} else if (refresh && !awaitRefresh && (user.lastFetchedAt == null || Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24)) {
// Run the refresh in the background
// noinspection ES6MissingAwait
resolveUser(username, host, true, true, skipMentionsOnCreate);
resolveUser(username, host, true, true, limiter);
}
logger.info(`return existing remote user: ${acctLower}`);
return user;
}
export async function resolveMentionToUserAndProfile(username: string, host: string | null, objectHost: string | null) {
export async function resolveMentionToUserAndProfile(username: string, host: string | null, objectHost: string | null, limiter: RecursionLimiter) {
return profileMentionCache.fetch(`${username}@${host ?? objectHost}`, async () => {
try {
const user = await resolveUser(username, host ?? objectHost, false, false, true);
const user = await resolveUser(username, host ?? objectHost, false, false, limiter);
const profile = await UserProfiles.findOneBy({ userId: user.id });
const data = { username, host: host ?? objectHost };