e2net/iceshrimp
3
0
Fork 0
forked from mirrors/iceshrimp
Code Pull requests Activity

[mastodon-client] Add basic support for filters

Browse source 
Currently you have to configure these in the web ui, but this will eventually be implemented as well
This commit is contained in:
Laura Hausmann 2023-11-27 20:47:42 +01:00
parent ef3463e8dc
commit 03cdf4ec4a
Signed by: zotan
GPG key ID: D044E84C5BE01605
20 changed files with 99 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
packages/backend/src/misc/is-filtered.ts
View file

@ -5,12 +5,20 @@ import { getWordHardMute } from "@/misc/check-word-mute.js";
import { Cache } from "@/misc/cache.js";
import { unique } from "@/prelude/array.js";
import config from "@/config/index.js";
import { UserProfiles } from "@/models/index.js";
const filteredNoteCache = new Cache<boolean>("filteredNote", config.wordMuteCache?.ttlSeconds ?? 60 * 60 * 24);
const mutedWordsCache = new Cache<UserProfile["mutedWords"]>("mutedWords", 60 * 5);
export function isFiltered(note: Note, user: { id: User["id"] } | null | undefined, profile: { mutedWords: UserProfile["mutedWords"] } | null): boolean | Promise<boolean> {
if (!user || !profile) return false;
if (profile.mutedWords.length < 1) return false;
return filteredNoteCache.fetch(`${note.id}:${(note.updatedAt ?? note.createdAt).getTime()}:${user.id}`,
() => getWordHardMute(note, user, unique(profile.mutedWords)));
export async function isFiltered(note: Note, user: { id: User["id"] } | null | undefined, profile?: { mutedWords: UserProfile["mutedWords"] } | null): Promise<boolean> {
if (!user) return false;
if (profile === undefined)
profile = { mutedWords: await mutedWordsCache.fetch(user.id, async () =>
UserProfiles.findOneBy({ userId: user.id }).then(p => p?.mutedWords ?? [])) };
if (!profile || profile.mutedWords.length < 1) return false;
const ts = (note.updatedAt ?? note.createdAt) as Date | string;
const identifier = (typeof ts === "string" ? new Date(ts) : ts)?.getTime() ?? '0';
return filteredNoteCache.fetch(`${note.id}:${ts}:${user.id}`,
() => getWordHardMute(note, user, unique(profile!.mutedWords)));
}

17
packages/backend/src/models/repositories/note.ts
View file

@ -33,8 +33,6 @@ import { isFiltered } from "@/misc/is-filtered.js";
import { UserProfile } from "@/models/entities/user-profile.js";
import { Cache } from "@/misc/cache.js";
const mutedWordsCache = new Cache<UserProfile["mutedWords"]>("mutedWords", 60 * 5);
export async function populatePoll(note: Note, meId: User["id"] | null) {
const poll = await Polls.findOneByOrFail({ noteId: note.id });
const choices = poll.choices.map((c) => ({
@ -180,7 +178,6 @@ export const NoteRepository = db.getRepository(Note).extend({
};
},
userCache: PackedUserCache = Users.getFreshPackedUserCache(),
profile?: { mutedWords: UserProfile["mutedWords"] } | null,
): Promise<Packed<"Note">> {
const opts = Object.assign(
{
@ -193,11 +190,6 @@ export const NoteRepository = db.getRepository(Note).extend({
const note =
typeof src === "object" ? src : await this.findOneByOrFail({ id: src });
const host = note.userHost;
const meProfile = profile !== undefined
? profile
: meId !== null
? { mutedWords: await mutedWordsCache.fetch(meId, async () => UserProfiles.findOneBy({ userId: meId }).then(p => p?.mutedWords ?? [])) }
: null;
if (!(await this.isVisibleForMe(note, meId))) {
throw new IdentifiableError(
@ -269,7 +261,7 @@ export const NoteRepository = db.getRepository(Note).extend({
? {
myReaction: populateMyReaction(note, meId, options?._hint_),
isRenoted: populateIsRenoted(note, meId, options?._hint_),
isFiltered: isFiltered(note, me, await meProfile),
isFiltered: isFiltered(note, me),
}
: {}),
@ -338,7 +330,6 @@ export const NoteRepository = db.getRepository(Note).extend({
options?: {
detail?: boolean;
},
profile?: { mutedWords: UserProfile["mutedWords"] } | null,
) {
if (notes.length === 0) return [];
@ -374,10 +365,6 @@ export const NoteRepository = db.getRepository(Note).extend({
!!myRenotes.find(p => p.renoteId == target),
);
}
profile = profile !== undefined
? profile
: { mutedWords: await mutedWordsCache.fetch(meId, async () => UserProfiles.findOneBy({ userId: meId }).then(p => p?.mutedWords ?? [])) };
}
await prefetchEmojis(aggregateNoteEmojis(notes));
@ -390,7 +377,7 @@ export const NoteRepository = db.getRepository(Note).extend({
myReactions: myReactionsMap,
myRenotes: myRenotesMap
},
}, undefined, profile),
}),
),
);

23
packages/backend/src/server/api/mastodon/converters/note.ts
View file

@ -33,6 +33,7 @@ import { unique } from "@/prelude/array.js";
import { NoteReaction } from "@/models/entities/note-reaction.js";
import { Cache } from "@/misc/cache.js";
import { HtmlNoteCacheEntry } from "@/models/entities/html-note-cache-entry.js";
import { isFiltered } from "@/misc/is-filtered.js";
export class NoteConverter {
private static noteContentHtmlCache = new Cache<string | null>('html:note:content', config.htmlCache?.ttlSeconds ?? 60 * 60);
@ -138,6 +139,21 @@ export class NoteConverter {
const reblog = Promise.resolve(renote).then(renote => recurseCounter > 0 && renote ? this.encode(renote, ctx, isQuote(renote) && !isQuote(note) ? --recurseCounter : 0) : null);
const filtered = isFiltered(note, user).then(res => {
if (!res || ctx.filterContext == null || !['home', 'public'].includes(ctx.filterContext)) return null;
return [{
filter: {
id: '0',
title: 'Hard word mutes',
context: ['home', 'public'],
expires_at: null,
filter_action: 'hide',
keywords: [],
statuses: [],
}
} as MastodonEntity.FilterResult];
});
// noinspection ES6MissingAwait
return await awaitAll({
id: note.id,
@ -172,7 +188,8 @@ export class NoteConverter {
reactions: populated.then(populated => Promise.resolve(reaction).then(reaction => this.encodeReactions(note.reactions, reaction?.reaction, populated))),
bookmarked: isBookmarked,
quote: reblog.then(reblog => isQuote(note) ? reblog : null),
edited_at: note.updatedAt?.toISOString() ?? null
edited_at: note.updatedAt?.toISOString() ?? null,
filtered: filtered,
});
}
@ -293,8 +310,8 @@ export class NoteConverter {
}).filter(r => r.count > 0);
}
public static async encodeEvent(note: Note, user: ILocalUser | undefined): Promise<MastodonEntity.Status> {
const ctx = getStubMastoContext(user);
public static async encodeEvent(note: Note, user: ILocalUser | undefined, filterContext?: string): Promise<MastodonEntity.Status> {
const ctx = getStubMastoContext(user, filterContext);
NoteHelpers.fixupEventNote(note);
return NoteConverter.encode(note, ctx);
}

4
packages/backend/src/server/api/mastodon/converters/notification.ts
View file

@ -95,8 +95,8 @@ export class NotificationConverter {
}
}
public static async encodeEvent(target: Notification["id"], user: ILocalUser): Promise<MastodonEntity.Notification | null> {
const ctx = getStubMastoContext(user);
public static async encodeEvent(target: Notification["id"], user: ILocalUser, filterContext?: string): Promise<MastodonEntity.Notification | null> {
const ctx = getStubMastoContext(user, filterContext);
const notification = await Notifications.findOneByOrFail({ id: target });
return this.encode(notification, ctx).catch(_ => null);
}

2
packages/backend/src/server/api/mastodon/endpoints/account.ts
View file

@ -6,6 +6,7 @@ import { UserHelpers } from "@/server/api/mastodon/helpers/user.js";
import { ListHelpers } from "@/server/api/mastodon/helpers/list.js";
import { auth } from "@/server/api/mastodon/middleware/auth.js";
import { SearchHelpers } from "@/server/api/mastodon/helpers/search.js";
import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
export function setupEndpointsAccount(router: Router): void {
router.get("/v1/accounts/verify_credentials",
@ -52,6 +53,7 @@ export function setupEndpointsAccount(router: Router): void {
router.get<{ Params: { id: string } }>(
"/v1/accounts/:id/statuses",
auth(false, ["read:statuses"]),
filterContext('account'),
async (ctx) => {
const query = await UserHelpers.getUserCachedOr404(ctx.params.id, ctx);
const args = normalizeUrlQuery(argsToBools(limitToInt(ctx.query)));

2
packages/backend/src/server/api/mastodon/endpoints/misc.ts
View file

@ -4,6 +4,7 @@ import { argsToBools, limitToInt } from "@/server/api/mastodon/endpoints/timelin
import { Announcements } from "@/models/index.js";
import { auth } from "@/server/api/mastodon/middleware/auth.js";
import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
export function setupEndpointsMisc(router: Router): void {
router.get("/v1/custom_emojis",
@ -48,6 +49,7 @@ export function setupEndpointsMisc(router: Router): void {
);
router.get("/v1/trends/statuses",
filterContext('public'),
async (ctx) => {
const args = limitToInt(ctx.query);
ctx.body = await MiscHelpers.getTrendingStatuses(args.limit, args.offset, ctx);

3
packages/backend/src/server/api/mastodon/endpoints/notifications.ts
View file

@ -3,10 +3,12 @@ import { limitToInt, normalizeUrlQuery } from "./timeline.js";
import { NotificationHelpers } from "@/server/api/mastodon/helpers/notification.js";
import { NotificationConverter } from "@/server/api/mastodon/converters/notification.js";
import { auth } from "@/server/api/mastodon/middleware/auth.js";
import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
export function setupEndpointsNotifications(router: Router): void {
router.get("/v1/notifications",
auth(true, ['read:notifications']),
filterContext('notifications'),
async (ctx) => {
const args = normalizeUrlQuery(limitToInt(ctx.query), ['types[]', 'exclude_types[]']);
const res = await NotificationHelpers.getNotifications(args.max_id, args.since_id, args.min_id, args.limit, args['types[]'], args['exclude_types[]'], args.account_id, ctx);
@ -16,6 +18,7 @@ export function setupEndpointsNotifications(router: Router): void {
router.get("/v1/notifications/:id",
auth(true, ['read:notifications']),
filterContext('notifications'),
async (ctx) => {
const notification = await NotificationHelpers.getNotificationOr404(ctx.params.id, ctx);
ctx.body = await NotificationConverter.encode(notification, ctx);

3
packages/backend/src/server/api/mastodon/endpoints/status.ts
View file

@ -7,6 +7,7 @@ import { PollHelpers } from "@/server/api/mastodon/helpers/poll.js";
import { toArray } from "@/prelude/array.js";
import { auth } from "@/server/api/mastodon/middleware/auth.js";
import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
export function setupEndpointsStatus(router: Router): void {
router.post("/v1/statuses",
@ -40,6 +41,7 @@ export function setupEndpointsStatus(router: Router): void {
);
router.get<{ Params: { id: string } }>("/v1/statuses/:id",
auth(false, ["read:statuses"]),
filterContext('thread'),
async (ctx) => {
const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);
@ -57,6 +59,7 @@ export function setupEndpointsStatus(router: Router): void {
router.get<{ Params: { id: string } }>(
"/v1/statuses/:id/context",
auth(false, ["read:statuses"]),
filterContext('thread'),
async (ctx) => {
//FIXME: determine final limits within helper functions instead of here
const note = await NoteHelpers.getNoteOr404(ctx.params.id, ctx);

5
packages/backend/src/server/api/mastodon/endpoints/timeline.ts
View file

@ -5,6 +5,7 @@ import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
import { UserLists } from "@/models/index.js";
import { auth } from "@/server/api/mastodon/middleware/auth.js";
import { MastoApiError } from "@/server/api/mastodon/middleware/catch-errors.js";
import { filterContext } from "@/server/api/mastodon/middleware/filter-context.js";
//TODO: Move helper functions to a helper class
export function limitToInt(q: ParsedUrlQuery, additional: string[] = []) {
@ -53,6 +54,7 @@ export function normalizeUrlQuery(q: ParsedUrlQuery, arrayKeys: string[] = []):
export function setupEndpointsTimeline(router: Router): void {
router.get("/v1/timelines/public",
auth(true, ['read:statuses']),
filterContext('public'),
async (ctx, reply) => {
const args = normalizeUrlQuery(argsToBools(limitToInt(ctx.query)));
const res = await TimelineHelpers.getPublicTimeline(args.max_id, args.since_id, args.min_id, args.limit, args.only_media, args.local, args.remote, ctx);
@ -61,6 +63,7 @@ export function setupEndpointsTimeline(router: Router): void {
router.get<{ Params: { hashtag: string } }>(
"/v1/timelines/tag/:hashtag",
auth(false, ['read:statuses']),
filterContext('public'),
async (ctx, reply) => {
const tag = (ctx.params.hashtag ?? '').trim().toLowerCase();
const args = normalizeUrlQuery(argsToBools(limitToInt(ctx.query)), ['any[]', 'all[]', 'none[]']);
@ -70,6 +73,7 @@ export function setupEndpointsTimeline(router: Router): void {
);
router.get("/v1/timelines/home",
auth(true, ['read:statuses']),
filterContext('home'),
async (ctx, reply) => {
const args = normalizeUrlQuery(limitToInt(ctx.query));
const res = await TimelineHelpers.getHomeTimeline(args.max_id, args.since_id, args.min_id, args.limit, ctx);
@ -78,6 +82,7 @@ export function setupEndpointsTimeline(router: Router): void {
router.get<{ Params: { listId: string } }>(
"/v1/timelines/list/:listId",
auth(true, ['read:lists']),
filterContext('home'),
async (ctx, reply) => {
const list = await UserLists.findOneBy({ userId: ctx.user.id, id: ctx.params.listId });
if (!list) throw new MastoApiError(404);

9
packages/backend/src/server/api/mastodon/entities/filter.ts
View file

@ -1,12 +1,13 @@
namespace MastodonEntity {
export type Filter = {
id: string;
phrase: string;
title: string;
context: Array<FilterContext>;
expires_at: string | null;
irreversible: boolean;
whole_word: boolean;
filter_action: 'warn' | 'hide';
keywords: FilterKeyword[];
statuses: FilterStatus[];
};
export type FilterContext = string;
export type FilterContext = 'home' | 'notifications' | 'public' | 'thread' | 'account';
}

7
packages/backend/src/server/api/mastodon/entities/filter_keyword.ts Normal file
View file

@ -0,0 +1,7 @@
namespace MastodonEntity {
export type FilterKeyword = {
id: string;
keyword: string;
whole_word: boolean;
};
}

7
packages/backend/src/server/api/mastodon/entities/filter_result.ts Normal file
View file

@ -0,0 +1,7 @@
namespace MastodonEntity {
export type FilterResult = {
filter: Filter;
keyword_matches?: string[];
status_matches?: string[];
};
}

6
packages/backend/src/server/api/mastodon/entities/filter_status.ts Normal file
View file

@ -0,0 +1,6 @@
namespace MastodonEntity {
export type FilterStatus = {
id: string;
status_id: string;
};
}

1
packages/backend/src/server/api/mastodon/entities/status.ts
View file

@ -43,6 +43,7 @@ namespace MastodonEntity {
quote: Status | null;
bookmarked: boolean;
edited_at: string | null;
filtered: Array<FilterResult> | null;
};
export type StatusCreationRequest = {

7
packages/backend/src/server/api/mastodon/index.ts
View file

@ -50,9 +50,10 @@ function setupMiddleware(router: Router): void {
router.use(CacheMiddleware);
}
export function getStubMastoContext(user: ILocalUser | null | undefined): any {
export function getStubMastoContext(user: ILocalUser | null | undefined, filterContext?: string): any {
return {
user: user ?? null,
cache: UserHelpers.getFreshAccountCache()
cache: UserHelpers.getFreshAccountCache(),
filterContext: filterContext,
};
}
}

8
packages/backend/src/server/api/mastodon/middleware/filter-context.ts Normal file
View file

@ -0,0 +1,8 @@
import { MastoContext } from "@/server/api/mastodon/index.js";
export function filterContext(context: string) {
return async function filterContext(ctx: MastoContext, next: () => Promise<any>) {
ctx.filterContext = context;
await next();
};
}

6
packages/backend/src/server/api/mastodon/streaming/channels/list.ts
View file

@ -5,7 +5,6 @@ import { StreamMessages } from "@/server/api/stream/types.js";
import { Packed } from "@/misc/schema.js";
import { User } from "@/models/entities/user.js";
import { UserListJoinings } from "@/models/index.js";
import { isFiltered } from "@/misc/is-filtered.js";
export class MastodonStreamList extends MastodonStream {
public static shouldShare = false;
@ -50,7 +49,7 @@ export class MastodonStreamList extends MastodonStream {
private async onNote(note: Note) {
if (!await this.shouldProcessNote(note)) return;
const encoded = await NoteConverter.encodeEvent(note, this.user)
const encoded = await NoteConverter.encodeEvent(note, this.user, 'home')
this.connection.send(this.chName, "update", encoded);
}
@ -60,7 +59,7 @@ export class MastodonStreamList extends MastodonStream {
switch (data.type) {
case "updated":
const encoded = await NoteConverter.encodeEvent(note, this.user);
const encoded = await NoteConverter.encodeEvent(note, this.user, 'home');
this.connection.send(this.chName, "status.update", encoded);
break;
case "deleted":
@ -75,7 +74,6 @@ export class MastodonStreamList extends MastodonStream {
if (!this.listUsers.includes(note.userId)) return false;
if (note.channelId) return false;
if (note.renoteId !== null && !note.text && this.renoteMuting.has(note.userId)) return false;
if (this.userProfile && (await isFiltered(note as Note, this.user, this.userProfile))) return false;
if (note.visibility === "specified") return !!note.visibleUserIds?.includes(this.user.id);
if (note.visibility === "followers") return this.following.has(note.userId);
return true;

6
packages/backend/src/server/api/mastodon/streaming/channels/public.ts
View file

@ -6,7 +6,6 @@ import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
import { StreamMessages } from "@/server/api/stream/types.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
import isQuote from "@/misc/is-quote.js";
import { isFiltered } from "@/misc/is-filtered.js";
export class MastodonStreamPublic extends MastodonStream {
public static shouldShare = true;
@ -40,7 +39,7 @@ export class MastodonStreamPublic extends MastodonStream {
private async onNote(note: Note) {
if (!await this.shouldProcessNote(note)) return;
const encoded = await NoteConverter.encodeEvent(note, this.user)
const encoded = await NoteConverter.encodeEvent(note, this.user, 'public')
this.connection.send(this.chName, "update", encoded);
}
@ -50,7 +49,7 @@ export class MastodonStreamPublic extends MastodonStream {
switch (data.type) {
case "updated":
const encoded = await NoteConverter.encodeEvent(note, this.user);
const encoded = await NoteConverter.encodeEvent(note, this.user, 'public');
this.connection.send(this.chName, "status.update", encoded);
break;
case "deleted":
@ -72,7 +71,6 @@ export class MastodonStreamPublic extends MastodonStream {
if (isUserRelated(note, this.muting)) return false;
if (isUserRelated(note, this.blocking)) return false;
if (note.renoteId !== null && !isQuote(note) && this.renoteMuting.has(note.userId)) return false;
if (this.userProfile && (await isFiltered(note as Note, this.user, this.userProfile))) return false;
return true;
}

6
packages/backend/src/server/api/mastodon/streaming/channels/tag.ts
View file

@ -5,7 +5,6 @@ import { Note } from "@/models/entities/note.js";
import { NoteConverter } from "@/server/api/mastodon/converters/note.js";
import { StreamMessages } from "@/server/api/stream/types.js";
import isQuote from "@/misc/is-quote.js";
import { isFiltered } from "@/misc/is-filtered.js";
export class MastodonStreamTag extends MastodonStream {
public static shouldShare = false;
@ -34,7 +33,7 @@ export class MastodonStreamTag extends MastodonStream {
private async onNote(note: Note) {
if (!await this.shouldProcessNote(note)) return;
const encoded = await NoteConverter.encodeEvent(note, this.user)
const encoded = await NoteConverter.encodeEvent(note, this.user, 'public')
this.connection.send(this.chName, "update", encoded);
}
@ -44,7 +43,7 @@ export class MastodonStreamTag extends MastodonStream {
switch (data.type) {
case "updated":
const encoded = await NoteConverter.encodeEvent(note, this.user);
const encoded = await NoteConverter.encodeEvent(note, this.user, 'public');
this.connection.send(this.chName, "status.update", encoded);
break;
case "deleted":
@ -64,7 +63,6 @@ export class MastodonStreamTag extends MastodonStream {
if (isUserRelated(note, this.muting)) return false;
if (isUserRelated(note, this.blocking)) return false;
if (note.renoteId !== null && !isQuote(note) && this.renoteMuting.has(note.userId)) return false;
if (this.userProfile && (await isFiltered(note as Note, this.user, this.userProfile))) return false;
return true;
}

8
packages/backend/src/server/api/mastodon/streaming/channels/user.ts
View file

@ -7,7 +7,6 @@ import { StreamMessages } from "@/server/api/stream/types.js";
import { NotificationConverter } from "@/server/api/mastodon/converters/notification.js";
import { AnnouncementConverter } from "@/server/api/mastodon/converters/announcement.js";
import isQuote from "@/misc/is-quote.js";
import { isFiltered } from "@/misc/is-filtered.js";
export class MastodonStreamUser extends MastodonStream {
public static shouldShare = true;
@ -40,7 +39,7 @@ export class MastodonStreamUser extends MastodonStream {
private async onNote(note: Note) {
if (!await this.shouldProcessNote(note)) return;
const encoded = await NoteConverter.encodeEvent(note, this.user)
const encoded = await NoteConverter.encodeEvent(note, this.user, 'home')
this.connection.send(this.chName, "update", encoded);
}
@ -50,7 +49,7 @@ export class MastodonStreamUser extends MastodonStream {
switch (data.type) {
case "updated":
const encoded = await NoteConverter.encodeEvent(note, this.user);
const encoded = await NoteConverter.encodeEvent(note, this.user, 'home');
this.connection.send(this.chName, "status.update", encoded);
break;
case "deleted":
@ -64,7 +63,7 @@ export class MastodonStreamUser extends MastodonStream {
private async onUserEvent(data: StreamMessages["main"]["payload"]) {
switch (data.type) {
case "notification":
const encoded = await NotificationConverter.encodeEvent(data.body.id, this.user);
const encoded = await NotificationConverter.encodeEvent(data.body.id, this.user, 'notifications');
if (encoded) this.connection.send(this.chName, "notification", encoded);
break;
default:
@ -99,7 +98,6 @@ export class MastodonStreamUser extends MastodonStream {
if (isUserRelated(note, this.blocking)) return false;
if (isUserRelated(note, this.hidden)) return false;
if (note.renoteId !== null && !isQuote(note) && this.renoteMuting.has(note.userId)) return false;
if (this.userProfile && (await isFiltered(note as Note, this.user, this.userProfile))) return false;
return true;
}