From 658901a47f3171941a66fa3e0a4906a9d30acbb5 Mon Sep 17 00:00:00 2001 From: YS <47836716+yszkst@users.noreply.github.com> Date: Thu, 23 Mar 2023 13:48:14 +0900 Subject: bump aws-sdk to v3 for s3 (#10363) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * indent * aws-sdk v3移行 --- packages/backend/src/core/DriveService.ts | 53 +++++++++++++-------------- packages/backend/src/core/S3Service.ts | 61 ++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 45 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index f1e93d6dd9..97d03d5f52 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -4,6 +4,7 @@ import { v4 as uuid } from 'uuid'; import sharp from 'sharp'; import { sharpBmp } from 'sharp-read-bmp'; import { IsNull } from 'typeorm'; +import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -36,7 +37,6 @@ import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import { correctFilename } from '@/misc/correct-filename.js'; import { isMimeImage } from '@/misc/is-mime-image.js'; -import type S3 from 'aws-sdk/clients/s3.js'; type AddFileArgs = { /** User who wish to add file */ @@ -81,6 +81,7 @@ type UploadFromUrlArgs = { export class DriveService { private registerLogger: Logger; private downloaderLogger: Logger; + private deleteLogger: Logger; constructor( @Inject(DI.config) @@ -118,6 +119,7 @@ export class DriveService { const logger = new Logger('drive', 'blue'); this.registerLogger = logger.createSubLogger('register', 'yellow'); this.downloaderLogger = logger.createSubLogger('downloader'); + this.deleteLogger = logger.createSubLogger('delete'); } /*** @@ -368,7 +370,7 @@ export class DriveService { Body: stream, ContentType: type, CacheControl: 'max-age=31536000, immutable', - } as S3.PutObjectRequest; + } as PutObjectCommandInput; if (filename) params.ContentDisposition = contentDisposition( 'inline', @@ -378,21 +380,16 @@ export class DriveService { ); if (meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - const s3 = this.s3Service.getS3(meta); - - const upload = s3.upload(params, { - partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024, - }); - - await upload.promise() + await this.s3Service.upload(meta, params) .then( result => { - if (result) { + if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); - } else { - this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`); + } else { // AbortMultipartUploadCommandOutput + this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`); } - }, + }) + .catch( err => { this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); }, @@ -528,10 +525,10 @@ export class DriveService { }; const properties: { - width?: number; - height?: number; - orientation?: number; - } = {}; + width?: number; + height?: number; + orientation?: number; + } = {}; if (info.width) { properties['width'] = info.width; @@ -720,22 +717,22 @@ export class DriveService { @bindThis public async deleteObjectStorageFile(key: string) { const meta = await this.metaService.fetch(); - - const s3 = this.s3Service.getS3(meta); - try { - await s3.deleteObject({ - Bucket: meta.objectStorageBucket!, + const param = { + Bucket: meta.objectStorageBucket, Key: key, - }).promise(); + } as DeleteObjectCommandInput; + + await this.s3Service.delete(meta, param); } catch (err: any) { - if (err.code === 'NoSuchKey') { - console.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err); + if (err.name === 'NoSuchKey') { + this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error); return; + } else { + throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, { + cause: err, + }); } - throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, { - cause: err, - }); } } diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index cc8f950813..629278d915 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -1,11 +1,16 @@ import { URL } from 'node:url'; +import * as http from 'node:http'; +import * as https from 'node:https'; import { Inject, Injectable } from '@nestjs/common'; -import S3 from 'aws-sdk/clients/s3.js'; +import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { Upload } from '@aws-sdk/lib-storage'; +import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { Meta } from '@/models/entities/Meta.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3'; @Injectable() export class S3Service { @@ -18,25 +23,47 @@ export class S3Service { } @bindThis - public getS3(meta: Meta) { + public getS3Client(meta: Meta): S3Client { const u = meta.objectStorageEndpoint - ? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}` - : `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`; + ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` + : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent - return new S3({ - endpoint: meta.objectStorageEndpoint && meta.objectStorageEndpoint.length > 0 - ? meta.objectStorageEndpoint - : undefined, - accessKeyId: meta.objectStorageAccessKey!, - secretAccessKey: meta.objectStorageSecretKey!, + const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy); + const handlerOption: NodeHttpHandlerOptions = {}; + if (meta.objectStorageUseSSL) { + handlerOption.httpsAgent = agent as https.Agent; + } else { + handlerOption.httpAgent = agent as http.Agent; + } + + return new S3Client({ + endpoint: meta.objectStorageEndpoint ? u : undefined, + credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? { + accessKeyId: meta.objectStorageAccessKey, + secretAccessKey: meta.objectStorageSecretKey, + } : undefined, region: meta.objectStorageRegion ?? undefined, - sslEnabled: meta.objectStorageUseSSL, - s3ForcePathStyle: !meta.objectStorageEndpoint // AWS with endPoint omitted - ? false - : meta.objectStorageS3ForcePathStyle, - httpOptions: { - agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy), - }, + tls: meta.objectStorageUseSSL, + forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted + requestHandler: new NodeHttpHandler(handlerOption), }); } + + @bindThis + public async upload(meta: Meta, input: PutObjectCommandInput) { + const client = this.getS3Client(meta); + return new Upload({ + client, + params: input, + partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com') + ? 500 * 1024 * 1024 + : 8 * 1024 * 1024, + }).done(); + } + + @bindThis + public delete(meta: Meta, input: DeleteObjectCommandInput) { + const client = this.getS3Client(meta); + return client.send(new DeleteObjectCommand(input)); + } } -- cgit v1.2.3-freya