From 0dead4637e650285bbec75f4d541f0ab76f2a116 Mon Sep 17 00:00:00 2001 From: Marie Date: Fri, 28 Mar 2025 00:21:44 +0100 Subject: add: bunnycdn storage support --- packages/backend/src/core/DriveService.ts | 41 +++++++++++++++++++------------ 1 file changed, 25 insertions(+), 16 deletions(-) (limited to 'packages/backend/src/core/DriveService.ts') diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index a65059b417..3d53a94b43 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -44,6 +44,7 @@ import { correctFilename } from '@/misc/correct-filename.js'; import { isMimeImage } from '@/misc/is-mime-image.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { BunnyService } from '@/core/BunnyService.js'; type AddFileArgs = { /** User who wish to add file */ @@ -121,6 +122,7 @@ export class DriveService { private downloadService: DownloadService, private internalStorageService: InternalStorageService, private s3Service: S3Service, + private bunnyService: BunnyService, private imageProcessingService: ImageProcessingService, private videoProcessingService: VideoProcessingService, private globalEventService: GlobalEventService, @@ -405,20 +407,24 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - await this.s3Service.upload(this.meta, params) - .then( - result => { - if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput - this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); - } else { // AbortMultipartUploadCommandOutput - this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`); - } - }) - .catch( - err => { - this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); - }, - ); + if (this.meta.objectStorageAccessKey) { + await this.s3Service.upload(this.meta, params) + .then( + result => { + if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput + this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); + } else { // AbortMultipartUploadCommandOutput + this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`); + } + }) + .catch( + err => { + this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); + }, + ); + } else { + await this.bunnyService.upload(this.meta, key, stream); + } } // Expire oldest file (without avatar or banner) of remote user @@ -814,8 +820,11 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - - await this.s3Service.delete(this.meta, param); + if (this.meta.objectStorageAccessKey) { + await this.s3Service.delete(this.meta, param); + } else { + await this.bunnyService.delete(this.meta, key); + } } catch (err: any) { if (err.name === 'NoSuchKey') { this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error); -- cgit v1.2.3-freya From 3bdac95bfd95c8f0845b4ea80ed98cd820833c06 Mon Sep 17 00:00:00 2001 From: Marie Date: Fri, 28 Mar 2025 00:41:37 +0100 Subject: upd: check if endpoint includes bunnycdn.com --- packages/backend/src/core/DriveService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/core/DriveService.ts') diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 3d53a94b43..83791f7e8d 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,7 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (this.meta.objectStorageAccessKey) { + if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.upload(this.meta, params) .then( result => { @@ -820,7 +820,7 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (this.meta.objectStorageAccessKey) { + if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.delete(this.meta, param); } else { await this.bunnyService.delete(this.meta, key); -- cgit v1.2.3-freya From 6dae5c916599d595215c82a22a120e998a77fd11 Mon Sep 17 00:00:00 2001 From: Marie Date: Fri, 28 Mar 2025 00:42:28 +0100 Subject: upd: remove old check --- packages/backend/src/core/DriveService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/core/DriveService.ts') diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 83791f7e8d..616deb0221 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,7 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { + if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.upload(this.meta, params) .then( result => { @@ -820,7 +820,7 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (this.meta.objectStorageAccessKey && !this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { + if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { await this.s3Service.delete(this.meta, param); } else { await this.bunnyService.delete(this.meta, key); -- cgit v1.2.3-freya From a35bfa9f1a4f426c1c7bdfead9f7554d11f31427 Mon Sep 17 00:00:00 2001 From: Marie Date: Fri, 28 Mar 2025 00:46:23 +0100 Subject: upd flip check --- packages/backend/src/core/DriveService.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'packages/backend/src/core/DriveService.ts') diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 616deb0221..53bc4e553f 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,9 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { + if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { + await this.bunnyService.upload(this.meta, key, stream); + } else { await this.s3Service.upload(this.meta, params) .then( result => { @@ -422,8 +424,6 @@ export class DriveService { this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); }, ); - } else { - await this.bunnyService.upload(this.meta, key, stream); } } @@ -820,10 +820,10 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (!this.meta.objectStorageEndpoint?.includes('bunnycdn.com')) { - await this.s3Service.delete(this.meta, param); - } else { + if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { await this.bunnyService.delete(this.meta, key); + } else { + await this.s3Service.delete(this.meta, param); } } catch (err: any) { if (err.name === 'NoSuchKey') { -- cgit v1.2.3-freya From 0481b25a62457f9ae44f1ef60e71dbab1944bde1 Mon Sep 17 00:00:00 2001 From: Marie Date: Fri, 28 Mar 2025 14:36:44 +0100 Subject: upd: create usingBunnyCDN --- packages/backend/src/core/BunnyService.ts | 7 ++++++- packages/backend/src/core/DriveService.ts | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'packages/backend/src/core/DriveService.ts') diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index 1563bc2711..3931ccfdd1 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -29,6 +29,11 @@ export class BunnyService { }; } + @bindThis + public usingBunnyCDN(meta: MiMeta) { + return meta.objectStorageEndpoint && meta.objectStorageEndpoint.includes('bunnycdn.com') ? true : false; + } + @bindThis public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) { const client = this.getBunnyInfo(meta); @@ -41,7 +46,7 @@ export class BunnyService { const data = Buffer.isBuffer(input) ? Readable.from(input) : input; const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !meta.objectStorageUseProxy, true); - + // Seperation of path and host/domain is required here const options = { method: 'PUT', diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 53bc4e553f..31c065d4b4 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -407,7 +407,7 @@ export class DriveService { ); if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { + if (this.bunnyService.usingBunnyCDN(this.meta)) { await this.bunnyService.upload(this.meta, key, stream); } else { await this.s3Service.upload(this.meta, params) @@ -820,7 +820,7 @@ export class DriveService { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.includes('bunnycdn.com')) { + if (this.bunnyService.usingBunnyCDN(this.meta)) { await this.bunnyService.delete(this.meta, key); } else { await this.s3Service.delete(this.meta, param); -- cgit v1.2.3-freya From 621f2e2ee42074c6aa5f3b20e33525e4a13fe169 Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 6 May 2025 01:02:27 +0200 Subject: upd: add logger, check hostname, add catch --- packages/backend/src/core/BunnyService.ts | 20 ++++++++++++++++++-- packages/backend/src/core/DriveService.ts | 6 +++++- 2 files changed, 23 insertions(+), 3 deletions(-) (limited to 'packages/backend/src/core/DriveService.ts') diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index 09212fffbf..9e606423b6 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -12,12 +12,16 @@ import type { MiMeta } from '@/models/Meta.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import Logger from '@/logger.js'; @Injectable() export class BunnyService { + private bunnyCdnLogger: Logger; + constructor( private httpRequestService: HttpRequestService, ) { + this.bunnyCdnLogger = new Logger('bunnycdn', 'blue'); } @bindThis @@ -40,7 +44,8 @@ export class BunnyService { @bindThis public usingBunnyCDN(meta: MiMeta) { - return meta.objectStorageEndpoint && meta.objectStorageEndpoint.includes('bunnycdn.com') ? true : false; + const client = this.getBunnyInfo(meta); + return new URL(client.fullUrl).hostname.endsWith('bunnycdn.com'); } @bindThis @@ -66,8 +71,19 @@ export class BunnyService { const req = https.request(options); + // Log and throw error if BunnyCDN detects wrong data and return to prevent console spam as this event occurs multiple times + req.on('response', (res) => { + if (res.statusCode === 401) { + this.bunnyCdnLogger.error('Invalid AccessKey or region hostname'); + data.destroy(); + return; + } + }); + req.on('error', (error) => { - console.error(error); + this.bunnyCdnLogger.error(error); + data.destroy(); + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occured during the connectiong to BunnyCDN'); }); data.pipe(req).on('finish', () => { diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 31c065d4b4..bca8c576f0 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -408,7 +408,11 @@ export class DriveService { if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; if (this.bunnyService.usingBunnyCDN(this.meta)) { - await this.bunnyService.upload(this.meta, key, stream); + await this.bunnyService.upload(this.meta, key, stream).catch( + err => { + this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); + }, + ); } else { await this.s3Service.upload(this.meta, params) .then( -- cgit v1.2.3-freya