summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/misc/get-drive-file-url.ts15
-rw-r--r--src/models/drive-file-webpublic.ts29
-rw-r--r--src/models/drive-file.ts67
-rw-r--r--src/server/api/endpoints/drive/files.ts2
-rw-r--r--src/server/api/endpoints/drive/files/check_existence.ts2
-rw-r--r--src/server/api/endpoints/drive/files/create.ts2
-rw-r--r--src/server/api/endpoints/drive/files/find.ts2
-rw-r--r--src/server/api/endpoints/drive/files/show.ts3
-rw-r--r--src/server/api/endpoints/drive/files/update.ts2
-rw-r--r--src/server/api/endpoints/drive/files/upload_from_url.ts2
-rw-r--r--src/server/api/endpoints/drive/stream.ts2
-rw-r--r--src/server/file/send-drive-file.ts19
-rw-r--r--src/services/drive/add-file.ts155
-rw-r--r--src/services/drive/delete-file.ts20
14 files changed, 281 insertions, 41 deletions
diff --git a/src/misc/get-drive-file-url.ts b/src/misc/get-drive-file-url.ts
index 0fe467261e..6ab7bfdb1b 100644
--- a/src/misc/get-drive-file-url.ts
+++ b/src/misc/get-drive-file-url.ts
@@ -6,15 +6,24 @@ export default function(file: IDriveFile, thumbnail = false): string {
if (file.metadata.withoutChunks) {
if (thumbnail) {
- return file.metadata.thumbnailUrl || file.metadata.url;
+ return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || file.metadata.url;
} else {
- return file.metadata.url;
+ return file.metadata.webpublicUrl || file.metadata.url;
}
} else {
if (thumbnail) {
return `${config.drive_url}/${file._id}?thumbnail`;
} else {
- return `${config.drive_url}/${file._id}`;
+ return `${config.drive_url}/${file._id}?web`;
}
}
}
+
+export function getOriginalUrl(file: IDriveFile) {
+ if (file.metadata && file.metadata.url) {
+ return file.metadata.url;
+ }
+
+ const accessKey = file.metadata ? file.metadata.accessKey : null;
+ return `${config.drive_url}/${file._id}${accessKey ? '?original=' + accessKey : ''}`;
+}
diff --git a/src/models/drive-file-webpublic.ts b/src/models/drive-file-webpublic.ts
new file mode 100644
index 0000000000..d087c355d3
--- /dev/null
+++ b/src/models/drive-file-webpublic.ts
@@ -0,0 +1,29 @@
+import * as mongo from 'mongodb';
+import monkDb, { nativeDbConn } from '../db/mongodb';
+
+const DriveFileWebpublic = monkDb.get<IDriveFileWebpublic>('driveFileWebpublics.files');
+DriveFileWebpublic.createIndex('metadata.originalId', { sparse: true, unique: true });
+export default DriveFileWebpublic;
+
+export const DriveFileWebpublicChunk = monkDb.get('driveFileWebpublics.chunks');
+
+export const getDriveFileWebpublicBucket = async (): Promise<mongo.GridFSBucket> => {
+ const db = await nativeDbConn();
+ const bucket = new mongo.GridFSBucket(db, {
+ bucketName: 'driveFileWebpublics'
+ });
+ return bucket;
+};
+
+export type IMetadata = {
+ originalId: mongo.ObjectID;
+};
+
+export type IDriveFileWebpublic = {
+ _id: mongo.ObjectID;
+ uploadDate: Date;
+ md5: string;
+ filename: string;
+ contentType: string;
+ metadata: IMetadata;
+};
diff --git a/src/models/drive-file.ts b/src/models/drive-file.ts
index d0c0905fc2..e4c1598049 100644
--- a/src/models/drive-file.ts
+++ b/src/models/drive-file.ts
@@ -3,7 +3,7 @@ const deepcopy = require('deepcopy');
import { pack as packFolder } from './drive-folder';
import monkDb, { nativeDbConn } from '../db/mongodb';
import isObjectId from '../misc/is-objectid';
-import getDriveFileUrl from '../misc/get-drive-file-url';
+import getDriveFileUrl, { getOriginalUrl } from '../misc/get-drive-file-url';
const DriveFile = monkDb.get<IDriveFile>('driveFiles.files');
DriveFile.createIndex('md5');
@@ -28,21 +28,48 @@ export type IMetadata = {
_user: any;
folderId: mongo.ObjectID;
comment: string;
+
+ /**
+ * リモートインスタンスから取得した場合の元URL
+ */
uri?: string;
+
+ /**
+ * URL for web(生成されている場合) or original
+ * * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
+ */
url?: string;
+
+ /**
+ * URL for thumbnail (thumbnailがなければなし)
+ * * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
+ */
thumbnailUrl?: string;
+
+ /**
+ * URL for original (web用が生成されてない場合はurlがoriginalを指す)
+ * * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
+ */
+ webpublicUrl?: string;
+
+ accessKey?: string;
+
src?: string;
deletedAt?: Date;
/**
- * このファイルの中身データがMongoDB内に保存されているのか否か
+ * このファイルの中身データがMongoDB内に保存されていないか否か
* オブジェクトストレージを利用している or リモートサーバーへの直リンクである
- * な場合は false になります
+ * な場合は true になります
*/
withoutChunks?: boolean;
storage?: string;
- storageProps?: any;
+
+ /***
+ * ObjectStorage の格納先の情報
+ */
+ storageProps?: IStorageProps;
isSensitive?: boolean;
/**
@@ -56,6 +83,25 @@ export type IMetadata = {
isRemote?: boolean;
};
+export type IStorageProps = {
+ /**
+ * ObjectStorage key for original
+ */
+ key: string;
+
+ /***
+ * ObjectStorage key for thumbnail (thumbnailがなければなし)
+ */
+ thumbnailKey?: string;
+
+ /***
+ * ObjectStorage key for webpublic (webpublicがなければなし)
+ */
+ webpublicKey?: string;
+
+ id?: string;
+};
+
export type IDriveFile = {
_id: mongo.ObjectID;
uploadDate: Date;
@@ -83,7 +129,8 @@ export function validateFileName(name: string): boolean {
export const packMany = (
files: any[],
options?: {
- detail: boolean
+ detail?: boolean
+ self?: boolean,
}
) => {
return Promise.all(files.map(f => pack(f, options)));
@@ -95,11 +142,13 @@ export const packMany = (
export const pack = (
file: any,
options?: {
- detail: boolean
+ detail?: boolean,
+ self?: boolean,
}
) => new Promise<any>(async (resolve, reject) => {
const opts = Object.assign({
- detail: false
+ detail: false,
+ self: false
}, options);
let _file: any;
@@ -165,5 +214,9 @@ export const pack = (
delete _target.isRemote;
delete _target._user;
+ if (opts.self) {
+ _target.url = getOriginalUrl(_file);
+ }
+
resolve(_target);
});
diff --git a/src/server/api/endpoints/drive/files.ts b/src/server/api/endpoints/drive/files.ts
index 27f101562d..20955e0e4e 100644
--- a/src/server/api/endpoints/drive/files.ts
+++ b/src/server/api/endpoints/drive/files.ts
@@ -77,5 +77,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
sort: sort
});
- res(await packMany(files));
+ res(await packMany(files, { detail: false, self: true }));
}));
diff --git a/src/server/api/endpoints/drive/files/check_existence.ts b/src/server/api/endpoints/drive/files/check_existence.ts
index d3ba4b386d..6e986d4170 100644
--- a/src/server/api/endpoints/drive/files/check_existence.ts
+++ b/src/server/api/endpoints/drive/files/check_existence.ts
@@ -32,6 +32,6 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
if (file === null) {
res({ file: null });
} else {
- res({ file: await pack(file) });
+ res({ file: await pack(file, { self: true }) });
}
}));
diff --git a/src/server/api/endpoints/drive/files/create.ts b/src/server/api/endpoints/drive/files/create.ts
index 53c62dd868..0660627f08 100644
--- a/src/server/api/endpoints/drive/files/create.ts
+++ b/src/server/api/endpoints/drive/files/create.ts
@@ -74,7 +74,7 @@ export default define(meta, (ps, user, app, file, cleanup) => new Promise(async
cleanup();
- res(pack(driveFile));
+ res(pack(driveFile, { self: true }));
} catch (e) {
console.error(e);
diff --git a/src/server/api/endpoints/drive/files/find.ts b/src/server/api/endpoints/drive/files/find.ts
index 8bc392fefe..25135e83a2 100644
--- a/src/server/api/endpoints/drive/files/find.ts
+++ b/src/server/api/endpoints/drive/files/find.ts
@@ -31,5 +31,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
'metadata.folderId': ps.folderId
});
- res(await Promise.all(files.map(file => pack(file))));
+ res(await Promise.all(files.map(file => pack(file, { self: true }))));
}));
diff --git a/src/server/api/endpoints/drive/files/show.ts b/src/server/api/endpoints/drive/files/show.ts
index 450a97065b..95c3323fbb 100644
--- a/src/server/api/endpoints/drive/files/show.ts
+++ b/src/server/api/endpoints/drive/files/show.ts
@@ -41,7 +41,8 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
// Serialize
const _file = await pack(file, {
- detail: true
+ detail: true,
+ self: true
});
res(_file);
diff --git a/src/server/api/endpoints/drive/files/update.ts b/src/server/api/endpoints/drive/files/update.ts
index 4efec3dc2a..a5835c6d65 100644
--- a/src/server/api/endpoints/drive/files/update.ts
+++ b/src/server/api/endpoints/drive/files/update.ts
@@ -111,7 +111,7 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
});
// Serialize
- const fileObj = await pack(file);
+ const fileObj = await pack(file, { self: true });
// Response
res(fileObj);
diff --git a/src/server/api/endpoints/drive/files/upload_from_url.ts b/src/server/api/endpoints/drive/files/upload_from_url.ts
index b7b9cb41c4..fc386e1638 100644
--- a/src/server/api/endpoints/drive/files/upload_from_url.ts
+++ b/src/server/api/endpoints/drive/files/upload_from_url.ts
@@ -50,5 +50,5 @@ export const meta = {
};
export default define(meta, (ps, user) => new Promise(async (res, rej) => {
- res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force)));
+ res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force), { self: true }));
}));
diff --git a/src/server/api/endpoints/drive/stream.ts b/src/server/api/endpoints/drive/stream.ts
index 804ecf50d9..c8342c66b5 100644
--- a/src/server/api/endpoints/drive/stream.ts
+++ b/src/server/api/endpoints/drive/stream.ts
@@ -65,5 +65,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
sort: sort
});
- res(await packMany(files));
+ res(await packMany(files, { self: true }));
}));
diff --git a/src/server/file/send-drive-file.ts b/src/server/file/send-drive-file.ts
index b904bda91b..c64177d4ee 100644
--- a/src/server/file/send-drive-file.ts
+++ b/src/server/file/send-drive-file.ts
@@ -3,6 +3,7 @@ import * as send from 'koa-send';
import * as mongodb from 'mongodb';
import DriveFile, { getDriveFileBucket } from '../../models/drive-file';
import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
+import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
const assets = `${__dirname}/../../server/file/assets/`;
@@ -41,6 +42,11 @@ export default async function(ctx: Koa.Context) {
}
const sendRaw = async () => {
+ if (file.metadata && file.metadata.accessKey && file.metadata.accessKey != ctx.query['original']) {
+ ctx.status = 403;
+ return;
+ }
+
const bucket = await getDriveFileBucket();
const readable = bucket.openDownloadStream(fileId);
readable.on('error', commonReadableHandlerGenerator(ctx));
@@ -60,6 +66,19 @@ export default async function(ctx: Koa.Context) {
} else {
await sendRaw();
}
+ } else if ('web' in ctx.query) {
+ const web = await DriveFileWebpublic.findOne({
+ 'metadata.originalId': fileId
+ });
+
+ if (web != null) {
+ ctx.set('Content-Type', file.contentType);
+
+ const bucket = await getDriveFileWebpublicBucket();
+ ctx.body = bucket.openDownloadStream(web._id);
+ } else {
+ await sendRaw();
+ }
} else {
if ('download' in ctx.query) {
ctx.set('Content-Disposition', 'attachment');
diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts
index d5156de6c4..2ea8cdc3bd 100644
--- a/src/services/drive/add-file.ts
+++ b/src/services/drive/add-file.ts
@@ -16,6 +16,7 @@ import { publishMainStream, publishDriveStream } from '../../stream';
import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import delFile from './delete-file';
import config from '../../config';
+import { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import driveChart from '../../chart/drive';
import perUserDriveChart from '../../chart/per-user-drive';
@@ -23,7 +24,71 @@ import fetchMeta from '../../misc/fetch-meta';
const log = debug('misskey:drive:add-file');
-async function save(path: string, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> {
+/***
+ * Save file
+ * @param path Path for original
+ * @param name Name for original
+ * @param type Content-Type for original
+ * @param hash Hash for original
+ * @param size Size for original
+ * @param metadata
+ */
+async function save(path: string, name: string, type: string, hash: string, size: number, metadata: IMetadata): Promise<IDriveFile> {
+ // #region webpublic
+ let webpublic: Buffer;
+ let webpublicExt = 'jpg';
+ let webpublicType = 'image/jpeg';
+
+ if (!metadata.uri) { // from local instance
+ log(`creating web image`);
+
+ if (['image/jpeg'].includes(type)) {
+ webpublic = await sharp(path)
+ .resize(2048, 2048, {
+ fit: 'inside',
+ withoutEnlargement: true
+ })
+ .rotate()
+ .jpeg({
+ quality: 85,
+ progressive: true
+ })
+ .toBuffer();
+ } else if (['image/webp'].includes(type)) {
+ webpublic = await sharp(path)
+ .resize(2048, 2048, {
+ fit: 'inside',
+ withoutEnlargement: true
+ })
+ .rotate()
+ .webp({
+ quality: 85
+ })
+ .toBuffer();
+
+ webpublicExt = 'webp';
+ webpublicType = 'image/webp';
+ } else if (['image/png'].includes(type)) {
+ webpublic = await sharp(path)
+ .resize(2048, 2048, {
+ fit: 'inside',
+ withoutEnlargement: true
+ })
+ .rotate()
+ .png()
+ .toBuffer();
+
+ webpublicExt = 'png';
+ webpublicType = 'image/png';
+ } else {
+ log(`web image not created (not an image)`);
+ }
+ } else {
+ log(`web image not created (from remote)`);
+ }
+ // #endregion webpublic
+
+ // #region thumbnail
let thumbnail: Buffer;
let thumbnailExt = 'jpg';
let thumbnailType = 'image/jpeg';
@@ -53,10 +118,9 @@ async function save(path: string, name: string, type: string, hash: string, size
thumbnailExt = 'png';
thumbnailType = 'image/png';
}
+ // #endregion thumbnail
if (config.drive && config.drive.storage == 'minio') {
- const minio = new Minio.Client(config.drive.config);
-
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
if (ext === '') {
@@ -66,33 +130,41 @@ async function save(path: string, name: string, type: string, hash: string, size
}
const key = `${config.drive.prefix}/${uuid.v4()}${ext}`;
+ const webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${webpublicExt}`;
const thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${thumbnailExt}`;
- const baseUrl = config.drive.baseUrl
- || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
+ log(`uploading original: ${key}`);
+ const uploads = [
+ upload(key, fs.createReadStream(path), type)
+ ];
- await minio.putObject(config.drive.bucket, key, fs.createReadStream(path), size, {
- 'Content-Type': type,
- 'Cache-Control': 'max-age=31536000, immutable'
- });
+ if (webpublic) {
+ log(`uploading webpublic: ${webpublicKey}`);
+ uploads.push(upload(webpublicKey, webpublic, webpublicType));
+ }
if (thumbnail) {
- await minio.putObject(config.drive.bucket, thumbnailKey, thumbnail, size, {
- 'Content-Type': thumbnailType,
- 'Cache-Control': 'max-age=31536000, immutable'
- });
+ log(`uploading thumbnail: ${thumbnailKey}`);
+ uploads.push(upload(thumbnailKey, thumbnail, thumbnailType));
}
+ await Promise.all(uploads);
+
+ const baseUrl = config.drive.baseUrl
+ || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
+
Object.assign(metadata, {
withoutChunks: true,
storage: 'minio',
storageProps: {
key: key,
- thumbnailKey: thumbnailKey
+ webpublicKey: webpublic ? webpublicKey : null,
+ thumbnailKey: thumbnail ? thumbnailKey : null,
},
url: `${ baseUrl }/${ key }`,
+ webpublicUrl: webpublic ? `${ baseUrl }/${ webpublicKey }` : null,
thumbnailUrl: thumbnail ? `${ baseUrl }/${ thumbnailKey }` : null
- });
+ } as IMetadata);
const file = await DriveFile.insert({
length: size,
@@ -105,29 +177,55 @@ async function save(path: string, name: string, type: string, hash: string, size
return file;
} else {
- // Get MongoDB GridFS bucket
- const bucket = await getDriveFileBucket();
+ // #region store original
+ const originalDst = await getDriveFileBucket();
- const file = await new Promise<IDriveFile>((resolve, reject) => {
- const writeStream = bucket.openUploadStream(name, {
+ // web用(Exif削除済み)がある場合はオリジナルにアクセス制限
+ if (webpublic) metadata.accessKey = uuid.v4();
+
+ const originalFile = await new Promise<IDriveFile>((resolve, reject) => {
+ const writeStream = originalDst.openUploadStream(name, {
contentType: type,
metadata
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
-
fs.createReadStream(path).pipe(writeStream);
});
+ log(`original stored to ${originalFile._id}`);
+ // #endregion store original
+
+ // #region store webpublic
+ if (webpublic) {
+ const webDst = await getDriveFileWebpublicBucket();
+
+ const webFile = await new Promise<IDriveFile>((resolve, reject) => {
+ const writeStream = webDst.openUploadStream(name, {
+ contentType: webpublicType,
+ metadata: {
+ originalId: originalFile._id
+ }
+ });
+
+ writeStream.once('finish', resolve);
+ writeStream.on('error', reject);
+ writeStream.end(webpublic);
+ });
+
+ log(`web stored ${webFile._id}`);
+ }
+ // #endregion store webpublic
+
if (thumbnail) {
const thumbnailBucket = await getDriveFileThumbnailBucket();
- await new Promise<IDriveFile>((resolve, reject) => {
+ const tuhmFile = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = thumbnailBucket.openUploadStream(name, {
contentType: thumbnailType,
metadata: {
- originalId: file._id
+ originalId: originalFile._id
}
});
@@ -135,12 +233,23 @@ async function save(path: string, name: string, type: string, hash: string, size
writeStream.on('error', reject);
writeStream.end(thumbnail);
});
+
+ log(`thumbnail stored ${tuhmFile._id}`);
}
- return file;
+ return originalFile;
}
}
+async function upload(key: string, stream: fs.ReadStream | Buffer, type: string) {
+ const minio = new Minio.Client(config.drive.config);
+
+ await minio.putObject(config.drive.bucket, key, stream, null, {
+ 'Content-Type': type,
+ 'Cache-Control': 'max-age=31536000, immutable'
+ });
+}
+
async function deleteOldFile(user: IRemoteUser) {
const oldFile = await DriveFile.findOne({
_id: {
diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts
index 3e2f42003b..92d0010bcf 100644
--- a/src/services/drive/delete-file.ts
+++ b/src/services/drive/delete-file.ts
@@ -4,6 +4,7 @@ import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-
import config from '../../config';
import driveChart from '../../chart/drive';
import perUserDriveChart from '../../chart/per-user-drive';
+import DriveFileWebpublic, { DriveFileWebpublicChunk } from '../../models/drive-file-webpublic';
export default async function(file: IDriveFile, isExpired = false) {
if (file.metadata.storage == 'minio') {
@@ -20,6 +21,11 @@ export default async function(file: IDriveFile, isExpired = false) {
const thumbnailObj = file.metadata.storageProps.thumbnailKey ? file.metadata.storageProps.thumbnailKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-thumbnail`;
await minio.removeObject(config.drive.bucket, thumbnailObj);
}
+
+ if (file.metadata.webpublicUrl) {
+ const webpublicObj = file.metadata.storageProps.webpublicKey ? file.metadata.storageProps.webpublicKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-original`;
+ await minio.removeObject(config.drive.bucket, webpublicObj);
+ }
}
// チャンクをすべて削除
@@ -48,6 +54,20 @@ export default async function(file: IDriveFile, isExpired = false) {
}
//#endregion
+ //#region Web公開用もあれば削除
+ const webpublic = await DriveFileWebpublic.findOne({
+ 'metadata.originalId': file._id
+ });
+
+ if (webpublic) {
+ await DriveFileWebpublicChunk.remove({
+ files_id: webpublic._id
+ });
+
+ await DriveFileWebpublic.remove({ _id: webpublic._id });
+ }
+ //#endregion
+
// 統計を更新
driveChart.update(file, false);
perUserDriveChart.update(file, false);
generated by cgit v1.2.3-freya (git 2.53.0.84.g453e7b744a) at 2026-03-03 23:29:06 +0000