diff options
Diffstat (limited to 'packages/backend/src/misc')
| -rw-r--r-- | packages/backend/src/misc/FileWriterStream.ts | 2 | ||||
| -rw-r--r-- | packages/backend/src/misc/QuantumKVCache.ts | 385 | ||||
| -rw-r--r-- | packages/backend/src/misc/cache.ts | 53 | ||||
| -rw-r--r-- | packages/backend/src/misc/diff-arrays.ts | 102 | ||||
| -rw-r--r-- | packages/backend/src/misc/extract-custom-emojis-from-mfm.ts | 2 | ||||
| -rw-r--r-- | packages/backend/src/misc/extract-hashtags.ts | 2 | ||||
| -rw-r--r-- | packages/backend/src/misc/extract-mentions.ts | 2 | ||||
| -rw-r--r-- | packages/backend/src/misc/fastify-reply-error.ts | 4 | ||||
| -rw-r--r-- | packages/backend/src/misc/id/aid.ts | 3 | ||||
| -rw-r--r-- | packages/backend/src/misc/id/aidx.ts | 3 | ||||
| -rw-r--r-- | packages/backend/src/misc/identifiable-error.ts | 4 | ||||
| -rw-r--r-- | packages/backend/src/misc/is-retryable-error.ts | 16 | ||||
| -rw-r--r-- | packages/backend/src/misc/render-full-error.ts | 60 | ||||
| -rw-r--r-- | packages/backend/src/misc/render-inline-error.ts | 75 | ||||
| -rw-r--r-- | packages/backend/src/misc/status-error.ts | 4 | ||||
| -rw-r--r-- | packages/backend/src/misc/truncate.ts | 4 | ||||
| -rw-r--r-- | packages/backend/src/misc/verify-field-link.ts | 2 |
17 files changed, 694 insertions, 29 deletions
diff --git a/packages/backend/src/misc/FileWriterStream.ts b/packages/backend/src/misc/FileWriterStream.ts index 27c67cb5df..a61d949ef4 100644 --- a/packages/backend/src/misc/FileWriterStream.ts +++ b/packages/backend/src/misc/FileWriterStream.ts @@ -21,7 +21,7 @@ export class FileWriterStream extends WritableStream<Uint8Array> { write: async (chunk, controller) => { if (file === null) { controller.error(); - throw new Error(); + throw new Error('file is null'); } await file.write(chunk); diff --git a/packages/backend/src/misc/QuantumKVCache.ts b/packages/backend/src/misc/QuantumKVCache.ts new file mode 100644 index 0000000000..b96937d6f2 --- /dev/null +++ b/packages/backend/src/misc/QuantumKVCache.ts @@ -0,0 +1,385 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { InternalEventService } from '@/core/InternalEventService.js'; +import { bindThis } from '@/decorators.js'; +import { InternalEventTypes } from '@/core/GlobalEventService.js'; +import { MemoryKVCache } from '@/misc/cache.js'; + +export interface QuantumKVOpts<T> { + /** + * Memory cache lifetime in milliseconds. + */ + lifetime: number; + + /** + * Callback to fetch the value for a key that wasn't found in the cache. + * May be synchronous or async. + */ + fetcher: (key: string, cache: QuantumKVCache<T>) => T | Promise<T>; + + /** + * Optional callback to fetch the value for multiple keys that weren't found in the cache. + * May be synchronous or async. + * If not provided, then the implementation will fall back on repeated calls to fetcher(). + */ + bulkFetcher?: (keys: string[], cache: QuantumKVCache<T>) => Iterable<[key: string, value: T]> | Promise<Iterable<[key: string, value: T]>>; + + /** + * Optional callback when one or more values are changed (created, updated, or deleted) in the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * Implementations may be synchronous or async. + */ + onChanged?: (keys: string[], cache: QuantumKVCache<T>) => void | Promise<void>; +} + +/** + * QuantumKVCache is a lifetime-bounded memory cache (like MemoryKVCache) with automatic cross-cluster synchronization via Redis. + * All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache. + * This ensures that a call to get() will never return stale data. + */ +export class QuantumKVCache<T> implements Iterable<[key: string, value: T]> { + private readonly memoryCache: MemoryKVCache<T>; + + public readonly fetcher: QuantumKVOpts<T>['fetcher']; + public readonly bulkFetcher: QuantumKVOpts<T>['bulkFetcher']; + public readonly onChanged: QuantumKVOpts<T>['onChanged']; + + /** + * @param internalEventService Service bus to synchronize events. + * @param name Unique name of the cache - must be the same in all processes. + * @param opts Cache options + */ + constructor( + private readonly internalEventService: InternalEventService, + private readonly name: string, + opts: QuantumKVOpts<T>, + ) { + this.memoryCache = new MemoryKVCache(opts.lifetime); + this.fetcher = opts.fetcher; + this.bulkFetcher = opts.bulkFetcher; + this.onChanged = opts.onChanged; + + this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, { + // Ignore our own events, otherwise we'll immediately erase any set value. + ignoreLocal: true, + }); + } + + /** + * The number of items currently in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + public get size() { + return this.memoryCache.size; + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *entries(): Generator<[key: string, value: T]> { + for (const entry of this.memoryCache.entries) { + yield [entry[0], entry[1].value]; + } + } + + /** + * Iterates all keys in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *keys() { + for (const entry of this.memoryCache.entries) { + yield entry[0]; + } + } + + /** + * Iterates all values pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public *values() { + for (const entry of this.memoryCache.entries) { + yield entry[1].value; + } + } + + /** + * Creates or updates a value in the cache, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + * Skips if the value is unchanged. + */ + @bindThis + public async set(key: string, value: T): Promise<void> { + if (this.memoryCache.get(key) === value) { + return; + } + + this.memoryCache.set(key, value); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); + + if (this.onChanged) { + await this.onChanged([key], this); + } + } + + /** + * Creates or updates multiple value in the cache, and erases any stale caches across the cluster. + * Fires an onSet for each changed item event after the cache has been updated in all processes. + * Skips if all values are unchanged. + */ + @bindThis + public async setMany(items: Iterable<[key: string, value: T]>): Promise<void> { + const changedKeys: string[] = []; + + for (const item of items) { + if (this.memoryCache.get(item[0]) !== item[1]) { + changedKeys.push(item[0]); + this.memoryCache.set(item[0], item[1]); + } + } + + if (changedKeys.length > 0) { + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: changedKeys }); + + if (this.onChanged) { + await this.onChanged(changedKeys, this); + } + } + } + + /** + * Adds a value to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public add(key: string, value: T): void { + this.memoryCache.set(key, value); + } + + /** + * Adds multiple values to the local memory cache without notifying other process. + * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * This should only be used when the value is known to be current, like after fetching from the database. + */ + @bindThis + public addMany(items: Iterable<[key: string, value: T]>): void { + for (const [key, value] of items) { + this.memoryCache.set(key, value); + } + } + + /** + * Gets a value from the local memory cache, or returns undefined if not found. + * Returns cached data only - does not make any fetches. + */ + @bindThis + public get(key: string): T | undefined { + return this.memoryCache.get(key); + } + + /** + * Gets multiple values from the local memory cache; returning undefined for any missing keys. + * Returns cached data only - does not make any fetches. + */ + @bindThis + public getMany(keys: Iterable<string>): [key: string, value: T | undefined][] { + const results: [key: string, value: T | undefined][] = []; + for (const key of keys) { + results.push([key, this.get(key)]); + } + return results; + } + + /** + * Gets or fetches a value from the cache. + * Fires an onSet event, but does not emit an update event to other processes. + */ + @bindThis + public async fetch(key: string): Promise<T> { + let value = this.memoryCache.get(key); + if (value === undefined) { + value = await this.fetcher(key, this); + this.memoryCache.set(key, value); + + if (this.onChanged) { + await this.onChanged([key], this); + } + } + return value; + } + + /** + * Gets or fetches multiple values from the cache. + * Fires onSet events, but does not emit any update events to other processes. + */ + @bindThis + public async fetchMany(keys: Iterable<string>): Promise<[key: string, value: T][]> { + const results: [key: string, value: T][] = []; + const toFetch: string[] = []; + + // Spliterate into cached results / uncached keys. + for (const key of keys) { + const fromCache = this.get(key); + if (fromCache) { + results.push([key, fromCache]); + } else { + toFetch.push(key); + } + } + + // Fetch any uncached keys + if (toFetch.length > 0) { + const fetched = await this.bulkFetch(toFetch); + + // Add to cache and return set + this.addMany(fetched); + results.push(...fetched); + + // Emit event + if (this.onChanged) { + await this.onChanged(toFetch, this); + } + } + + return results; + } + + /** + * Returns true is a key exists in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + @bindThis + public has(key: string): boolean { + return this.memoryCache.get(key) !== undefined; + } + + /** + * Deletes a value from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event after the cache has been updated in all processes. + */ + @bindThis + public async delete(key: string): Promise<void> { + this.memoryCache.delete(key); + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); + + if (this.onChanged) { + await this.onChanged([key], this); + } + } + /** + * Deletes multiple values from the cache, and erases any stale caches across the cluster. + * Fires an onDelete event for each key after the cache has been updated in all processes. + * Skips if the input is empty. + */ + @bindThis + public async deleteMany(keys: Iterable<string>): Promise<void> { + const deleted: string[] = []; + + for (const key of keys) { + this.memoryCache.delete(key); + deleted.push(key); + } + + if (deleted.length === 0) { + return; + } + + await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: deleted }); + + if (this.onChanged) { + await this.onChanged(deleted, this); + } + } + + /** + * Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster. + * Fires an onSet event after the cache has been updated in all processes. + */ + @bindThis + public async refresh(key: string): Promise<T> { + const value = await this.fetcher(key, this); + await this.set(key, value); + return value; + } + + @bindThis + public async refreshMany(keys: Iterable<string>): Promise<[key: string, value: T][]> { + const values = await this.bulkFetch(keys); + await this.setMany(values); + return values; + } + + /** + * Erases all entries from the local memory cache. + * Does not send any events or update other processes. + */ + @bindThis + public clear() { + this.memoryCache.clear(); + } + + /** + * Removes expired cache entries from the local view. + * Does not send any events or update other processes. + */ + @bindThis + public gc() { + this.memoryCache.gc(); + } + + /** + * Erases all data and disconnects from the cluster. + * This *must* be called when shutting down to prevent memory leaks! + */ + @bindThis + public dispose() { + this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated); + + this.memoryCache.dispose(); + } + + @bindThis + private async bulkFetch(keys: Iterable<string>): Promise<[key: string, value: T][]> { + if (this.bulkFetcher) { + const results = await this.bulkFetcher(Array.from(keys), this); + return Array.from(results); + } + + const results: [key: string, value: T][] = []; + for (const key of keys) { + const value = await this.fetcher(key, this); + results.push([key, value]); + } + return results; + } + + @bindThis + private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise<void> { + if (data.name === this.name) { + for (const key of data.keys) { + this.memoryCache.delete(key); + } + + if (this.onChanged) { + await this.onChanged(data.keys, this); + } + } + } + + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + [Symbol.iterator](): Iterator<[key: string, value: T]> { + return this.entries(); + } +} diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 48b8f43678..666e684c1c 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -9,9 +9,9 @@ import { bindThis } from '@/decorators.js'; export class RedisKVCache<T> { private readonly lifetime: number; private readonly memoryCache: MemoryKVCache<T>; - private readonly fetcher: (key: string) => Promise<T>; - private readonly toRedisConverter: (value: T) => string; - private readonly fromRedisConverter: (value: string) => T | undefined; + public readonly fetcher: (key: string) => Promise<T>; + public readonly toRedisConverter: (value: T) => string; + public readonly fromRedisConverter: (value: string) => T | undefined; constructor( private redisClient: Redis.Redis, @@ -100,6 +100,11 @@ export class RedisKVCache<T> { } @bindThis + public clear() { + this.memoryCache.clear(); + } + + @bindThis public gc() { this.memoryCache.gc(); } @@ -113,9 +118,9 @@ export class RedisKVCache<T> { export class RedisSingleCache<T> { private readonly lifetime: number; private readonly memoryCache: MemorySingleCache<T>; - private readonly fetcher: () => Promise<T>; - private readonly toRedisConverter: (value: T) => string; - private readonly fromRedisConverter: (value: string) => T | undefined; + public readonly fetcher: () => Promise<T>; + public readonly toRedisConverter: (value: T) => string; + public readonly fromRedisConverter: (value: string) => T | undefined; constructor( private redisClient: Redis.Redis, @@ -123,16 +128,17 @@ export class RedisSingleCache<T> { opts: { lifetime: number; memoryCacheLifetime: number; - fetcher: RedisSingleCache<T>['fetcher']; - toRedisConverter: RedisSingleCache<T>['toRedisConverter']; - fromRedisConverter: RedisSingleCache<T>['fromRedisConverter']; + fetcher?: RedisSingleCache<T>['fetcher']; + toRedisConverter?: RedisSingleCache<T>['toRedisConverter']; + fromRedisConverter?: RedisSingleCache<T>['fromRedisConverter']; }, ) { this.lifetime = opts.lifetime; this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); - this.fetcher = opts.fetcher; - this.toRedisConverter = opts.toRedisConverter; - this.fromRedisConverter = opts.fromRedisConverter; + + this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); }); + this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value)); + this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value)); } @bindThis @@ -237,6 +243,16 @@ export class MemoryKVCache<T> { return cached.value; } + public has(key: string): boolean { + const cached = this.cache.get(key); + if (cached == null) return false; + if ((Date.now() - cached.date) > this.lifetime) { + this.cache.delete(key); + return false; + } + return true; + } + @bindThis public delete(key: string): void { this.cache.delete(key); @@ -308,11 +324,24 @@ export class MemoryKVCache<T> { } } + /** + * Removes all entries from the cache, but does not dispose it. + */ + @bindThis + public clear(): void { + this.cache.clear(); + } + @bindThis public dispose(): void { + this.clear(); clearInterval(this.gcIntervalHandle); } + public get size() { + return this.cache.size; + } + public get entries() { return this.cache.entries(); } diff --git a/packages/backend/src/misc/diff-arrays.ts b/packages/backend/src/misc/diff-arrays.ts new file mode 100644 index 0000000000..b50ca1d4f7 --- /dev/null +++ b/packages/backend/src/misc/diff-arrays.ts @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export interface DiffResult<T> { + added: T[]; + removed: T[]; +} + +/** + * Calculates the difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. + * Result sets are de-duplicated, and will be empty if no data was added or removed (respectively). + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArrays<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): DiffResult<T> { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + // data before AND after => changed + if (before?.size && after?.size) { + const added: T[] = []; + const removed: T[] = []; + + for (const host of before) { + // before and NOT after => removed + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { + removed.push(host); + } + } + + for (const host of after) { + // after and NOT before => added + if (!before.has(host)) { + added.push(host); + } + } + + return { added, removed }; + } + + // data ONLY before => all removed + if (before?.size) { + return { added: [], removed: Array.from(before) }; + } + + // data ONLY after => all added + if (after?.size) { + return { added: Array.from(after), removed: [] }; + } + + // data NEITHER before nor after => no change + return { added: [], removed: [] }; +} + +/** + * Checks for any difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArraysSimple<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): boolean { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + if (before?.size && after?.size) { + // different size => changed + if (before.size !== after.size) return true; + + // removed => changed + for (const host of before) { + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { + return true; + } + } + + // added => changed + for (const host of after) { + if (!before.has(host)) { + return true; + } + } + + // identical values => no change + return false; + } + + // before and NOT after => change + if (before?.size) return true; + + // after and NOT before => change + if (after?.size) return true; + + // NEITHER before nor after => no change + return false; +} diff --git a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts index 36a9b8e1f4..73ae9abb54 100644 --- a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts +++ b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { unique } from '@/misc/prelude/array.js'; export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] { diff --git a/packages/backend/src/misc/extract-hashtags.ts b/packages/backend/src/misc/extract-hashtags.ts index ed7606d995..d3d245d414 100644 --- a/packages/backend/src/misc/extract-hashtags.ts +++ b/packages/backend/src/misc/extract-hashtags.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { unique } from '@/misc/prelude/array.js'; export function extractHashtags(nodes: mfm.MfmNode[]): string[] { diff --git a/packages/backend/src/misc/extract-mentions.ts b/packages/backend/src/misc/extract-mentions.ts index bb21c32ffb..2ec9349718 100644 --- a/packages/backend/src/misc/extract-mentions.ts +++ b/packages/backend/src/misc/extract-mentions.ts @@ -5,7 +5,7 @@ // test is located in test/extract-mentions -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] { // TODO: 重複を削除 diff --git a/packages/backend/src/misc/fastify-reply-error.ts b/packages/backend/src/misc/fastify-reply-error.ts index e6c4e78d2f..03109e8b96 100644 --- a/packages/backend/src/misc/fastify-reply-error.ts +++ b/packages/backend/src/misc/fastify-reply-error.ts @@ -8,8 +8,8 @@ export class FastifyReplyError extends Error { public message: string; public statusCode: number; - constructor(statusCode: number, message: string) { - super(message); + constructor(statusCode: number, message: string, cause?: unknown) { + super(message, cause ? { cause } : undefined); this.message = message; this.statusCode = statusCode; } diff --git a/packages/backend/src/misc/id/aid.ts b/packages/backend/src/misc/id/aid.ts index c0e8478db5..f0eba2d99c 100644 --- a/packages/backend/src/misc/id/aid.ts +++ b/packages/backend/src/misc/id/aid.ts @@ -8,6 +8,7 @@ import * as crypto from 'node:crypto'; import { parseBigInt36 } from '@/misc/bigint.js'; +import { IdentifiableError } from '../identifiable-error.js'; export const aidRegExp = /^[0-9a-z]{10}$/; @@ -26,7 +27,7 @@ function getNoise(): string { } export function genAid(t: number): string { - if (isNaN(t)) throw new Error('Failed to create AID: Invalid Date'); + if (isNaN(t)) throw new IdentifiableError('6b73b7d5-9d2b-48b4-821c-ef955efe80ad', 'Failed to create AID: Invalid Date'); counter++; return getTime(t) + getNoise(); } diff --git a/packages/backend/src/misc/id/aidx.ts b/packages/backend/src/misc/id/aidx.ts index 006673a6d0..d2bb566e35 100644 --- a/packages/backend/src/misc/id/aidx.ts +++ b/packages/backend/src/misc/id/aidx.ts @@ -10,6 +10,7 @@ import { customAlphabet } from 'nanoid'; import { parseBigInt36 } from '@/misc/bigint.js'; +import { IdentifiableError } from '../identifiable-error.js'; export const aidxRegExp = /^[0-9a-z]{16}$/; @@ -34,7 +35,7 @@ function getNoise(): string { } export function genAidx(t: number): string { - if (isNaN(t)) throw new Error('Failed to create AIDX: Invalid Date'); + if (isNaN(t)) throw new IdentifiableError('6b73b7d5-9d2b-48b4-821c-ef955efe80ad', 'Failed to create AIDX: Invalid Date'); counter++; return getTime(t) + nodeId + getNoise(); } diff --git a/packages/backend/src/misc/identifiable-error.ts b/packages/backend/src/misc/identifiable-error.ts index f5c3fcd6cb..56e13f2622 100644 --- a/packages/backend/src/misc/identifiable-error.ts +++ b/packages/backend/src/misc/identifiable-error.ts @@ -15,8 +15,8 @@ export class IdentifiableError extends Error { */ public readonly isRetryable: boolean; - constructor(id: string, message?: string, isRetryable = false) { - super(message); + constructor(id: string, message?: string, isRetryable = false, cause?: unknown) { + super(message, cause ? { cause } : undefined); this.message = message ?? ''; this.id = id; this.isRetryable = isRetryable; diff --git a/packages/backend/src/misc/is-retryable-error.ts b/packages/backend/src/misc/is-retryable-error.ts index 9bb8700c7a..63b561b280 100644 --- a/packages/backend/src/misc/is-retryable-error.ts +++ b/packages/backend/src/misc/is-retryable-error.ts @@ -3,20 +3,34 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { AbortError } from 'node-fetch'; +import { AbortError, FetchError } from 'node-fetch'; import { UnrecoverableError } from 'bullmq'; import { StatusError } from '@/misc/status-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; +import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; +import { ConflictError } from '@/server/SkRateLimiterService.js'; /** * Returns false if the provided value represents a "permanent" error that cannot be retried. * Returns true if the error is retryable, unknown (as all errors are retryable by default), or not an error object. */ export function isRetryableError(e: unknown): boolean { + if (e instanceof AggregateError) return e.errors.every(inner => isRetryableError(inner)); if (e instanceof StatusError) return e.isRetryable; if (e instanceof IdentifiableError) return e.isRetryable; + if (e instanceof CaptchaError) { + if (e.code === captchaErrorCodes.verificationFailed) return false; + if (e.code === captchaErrorCodes.invalidParameters) return false; + if (e.code === captchaErrorCodes.invalidProvider) return false; + return true; + } + if (e instanceof FastifyReplyError) return false; + if (e instanceof ConflictError) return true; if (e instanceof UnrecoverableError) return false; if (e instanceof AbortError) return true; + if (e instanceof FetchError) return true; + if (e instanceof SyntaxError) return false; if (e instanceof Error) return e.name === 'AbortError'; return true; } diff --git a/packages/backend/src/misc/render-full-error.ts b/packages/backend/src/misc/render-full-error.ts new file mode 100644 index 0000000000..5f0a09bba9 --- /dev/null +++ b/packages/backend/src/misc/render-full-error.ts @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as Bull from 'bullmq'; +import { AbortError, FetchError } from 'node-fetch'; +import { StatusError } from '@/misc/status-error.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; +import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; + +export function renderFullError(e?: unknown): unknown { + if (e === undefined) return 'undefined'; + if (e === null) return 'null'; + + if (e instanceof Error) { + if (isSimpleError(e)) { + return renderInlineError(e); + } + + const data: ErrorData = {}; + if (e.stack) data.stack = e.stack; + if (e.message) data.message = e.message; + if (e.name) data.name = e.name; + + // mix "cause" and "errors" + if (e instanceof AggregateError && e.errors.length > 0) { + const causes = e.errors.map(inner => renderFullError(inner)); + if (e.cause) { + causes.push(renderFullError(e.cause)); + } + data.cause = causes; + } else if (e.cause) { + data.cause = renderFullError(e.cause); + } + + return data; + } + + return e; +} + +function isSimpleError(e: Error): boolean { + if (e instanceof Bull.UnrecoverableError) return true; + if (e instanceof AbortError || e.name === 'AbortError') return true; + if (e instanceof FetchError || e.name === 'FetchError') return true; + if (e instanceof StatusError) return true; + if (e instanceof IdentifiableError) return true; + if (e instanceof FetchError) return true; + if (e instanceof CaptchaError && e.code !== captchaErrorCodes.unknown) return true; + return false; +} + +interface ErrorData { + stack?: Error['stack']; + message?: Error['message']; + name?: Error['name']; + cause?: Error['cause'] | Error['cause'][]; +} diff --git a/packages/backend/src/misc/render-inline-error.ts b/packages/backend/src/misc/render-inline-error.ts new file mode 100644 index 0000000000..07f9f3068e --- /dev/null +++ b/packages/backend/src/misc/render-inline-error.ts @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { StatusError } from '@/misc/status-error.js'; +import { CaptchaError } from '@/core/CaptchaService.js'; + +export function renderInlineError(err: unknown): string { + const parts: string[] = []; + renderTo(err, parts); + return parts.join(''); +} + +function renderTo(err: unknown, parts: string[]): void { + parts.push(printError(err)); + + if (err instanceof AggregateError) { + for (let i = 0; i < err.errors.length; i++) { + parts.push(` [${i + 1}/${err.errors.length}]: `); + renderTo(err.errors[i], parts); + } + } + + if (err instanceof Error) { + if (err.cause) { + parts.push(' [caused by]: '); + renderTo(err.cause, parts); + // const cause = renderInlineError(err.cause); + // parts.push(' [caused by]: ', cause); + } + } +} + +function printError(err: unknown): string { + if (err === undefined) return 'undefined'; + if (err === null) return 'null'; + + if (err instanceof IdentifiableError) { + if (err.message) { + return `${err.name} ${err.id}: ${err.message}`; + } else { + return `${err.name} ${err.id}`; + } + } + + if (err instanceof StatusError) { + if (err.message) { + return `${err.name} ${err.statusCode}: ${err.message}`; + } else if (err.statusMessage) { + return `${err.name} ${err.statusCode}: ${err.statusMessage}`; + } else { + return `${err.name} ${err.statusCode}`; + } + } + + if (err instanceof CaptchaError) { + if (err.code.description) { + return `${err.name} ${err.code.description}: ${err.message}`; + } else { + return `${err.name}: ${err.message}`; + } + } + + if (err instanceof Error) { + if (err.message) { + return `${err.name}: ${err.message}`; + } else { + return err.name; + } + } + + return String(err); +} diff --git a/packages/backend/src/misc/status-error.ts b/packages/backend/src/misc/status-error.ts index c3533db607..4fd3bfcafb 100644 --- a/packages/backend/src/misc/status-error.ts +++ b/packages/backend/src/misc/status-error.ts @@ -9,8 +9,8 @@ export class StatusError extends Error { public isClientError: boolean; public isRetryable: boolean; - constructor(message: string, statusCode: number, statusMessage?: string) { - super(message); + constructor(message: string, statusCode: number, statusMessage?: string, cause?: unknown) { + super(message, cause ? { cause } : undefined); this.name = 'StatusError'; this.statusCode = statusCode; this.statusMessage = statusMessage; diff --git a/packages/backend/src/misc/truncate.ts b/packages/backend/src/misc/truncate.ts index 1c8a274609..a313ab7854 100644 --- a/packages/backend/src/misc/truncate.ts +++ b/packages/backend/src/misc/truncate.ts @@ -3,14 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { substring } from 'stringz'; - export function truncate(input: string, size: number): string; export function truncate(input: string | undefined, size: number): string | undefined; export function truncate(input: string | undefined, size: number): string | undefined { if (!input) { return input; } else { - return substring(input, 0, size); + return input.slice(0, size); } } diff --git a/packages/backend/src/misc/verify-field-link.ts b/packages/backend/src/misc/verify-field-link.ts index 62542eaaa0..f9fc352806 100644 --- a/packages/backend/src/misc/verify-field-link.ts +++ b/packages/backend/src/misc/verify-field-link.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { load as cheerio } from 'cheerio'; +import { load as cheerio } from 'cheerio/slim'; import type { HttpRequestService } from '@/core/HttpRequestService.js'; type Field = { name: string, value: string }; |