import type { Readable } from 'stream'; import type { GetObjectCommandInput, HeadObjectCommandInput } from '@aws-sdk/client-s3'; import { S3Client, HeadObjectCommand, GetObjectCommand, DeleteObjectsCommand, PutObjectCommand, DeleteObjectCommand, ListObjectsCommand, ObjectCannedACL, AbortMultipartUploadCommand, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import urljoin from 'url-join'; import { AttachmentType, FilePathOnStoragePrefix, ResponseMode, type RespondOptions, } from '~/server/interfaces/attachment'; import type { IAttachmentDocument } from '~/server/models/attachment'; import loggerFactory from '~/utils/logger'; import { configManager } from '../../config-manager'; import { AbstractFileUploader, type TemporaryUrl, type SaveFileParam, } from '../file-uploader'; import { ContentHeaders } from '../utils'; import { AwsMultipartUploader } from './multipart-uploader'; const logger = loggerFactory('growi:service:fileUploaderAws'); /** * File metadata in storage * TODO: mv this to "./uploader" */ interface FileMeta { name: string; size: number; } const isFileExists = async(s3: S3Client, params: HeadObjectCommandInput) => { try { await s3.send(new HeadObjectCommand(params)); } catch (err) { if (err != null && err.code === 'NotFound') { return false; } throw err; } return true; }; const ObjectCannedACLs = [ ObjectCannedACL.authenticated_read, ObjectCannedACL.aws_exec_read, ObjectCannedACL.bucket_owner_full_control, ObjectCannedACL.bucket_owner_read, ObjectCannedACL.private, ObjectCannedACL.public_read, ObjectCannedACL.public_read_write, ]; const isValidObjectCannedACL = (acl: string | null): acl is ObjectCannedACL => { return ObjectCannedACLs.includes(acl as ObjectCannedACL); }; /** * @see: https://dev.growi.org/5d091f611fe336003eec5bfd * @returns ObjectCannedACL */ const getS3PutObjectCannedAcl = (): ObjectCannedACL | undefined => { const s3ObjectCannedACL = configManager.getConfig('crowi', 'aws:s3ObjectCannedACL'); if (isValidObjectCannedACL(s3ObjectCannedACL)) { return s3ObjectCannedACL; } return undefined; }; const getS3Bucket = (): string | undefined => { return configManager.getConfig('crowi', 'aws:s3Bucket') ?? undefined; // return undefined when getConfig() returns null }; const S3Factory = (): S3Client => { return new S3Client({ credentials: { accessKeyId: configManager.getConfig('crowi', 'aws:s3AccessKeyId'), secretAccessKey: configManager.getConfig('crowi', 'aws:s3SecretAccessKey'), }, region: configManager.getConfig('crowi', 'aws:s3Region'), endpoint: configManager.getConfig('crowi', 'aws:s3CustomEndpoint'), forcePathStyle: configManager.getConfig('crowi', 'aws:s3CustomEndpoint') != null, // s3ForcePathStyle renamed to forcePathStyle in v3 }); }; const getFilePathOnStorage = (attachment: IAttachmentDocument) => { if (attachment.filePath != null) { // DEPRECATED: remains for backward compatibility for v3.3.x or below return attachment.filePath; } let dirName: string; if (attachment.attachmentType === AttachmentType.PAGE_BULK_EXPORT) { dirName = FilePathOnStoragePrefix.pageBulkExport; } else if (attachment.page != null) { dirName = FilePathOnStoragePrefix.attachment; } else { dirName = FilePathOnStoragePrefix.user; } const filePath = urljoin(dirName, attachment.fileName); return filePath; }; // TODO: rewrite this module to be a type-safe implementation class AwsFileUploader extends AbstractFileUploader { /** * @inheritdoc */ override isValidUploadSettings(): boolean { throw new Error('Method not implemented.'); } /** * @inheritdoc */ override listFiles() { throw new Error('Method not implemented.'); } /** * @inheritdoc */ override saveFile(param: SaveFileParam) { throw new Error('Method not implemented.'); } /** * @inheritdoc */ override deleteFiles() { throw new Error('Method not implemented.'); } /** * @inheritdoc */ override determineResponseMode() { return configManager.getConfig('crowi', 'aws:referenceFileWithRelayMode') ? ResponseMode.RELAY : ResponseMode.REDIRECT; } /** * @inheritdoc */ override async uploadAttachment(readable: Readable, attachment: IAttachmentDocument): Promise { if (!this.getIsUploadable()) { throw new Error('AWS is not configured.'); } logger.debug(`File uploading: fileName=${attachment.fileName}`); const s3 = S3Factory(); const filePath = getFilePathOnStorage(attachment); const contentHeaders = new ContentHeaders(attachment); await s3.send(new PutObjectCommand({ Bucket: getS3Bucket(), Key: filePath, Body: readable, ACL: getS3PutObjectCannedAcl(), // put type and the file name for reference information when uploading ContentType: contentHeaders.contentType?.value.toString(), ContentDisposition: contentHeaders.contentDisposition?.value.toString(), })); } /** * @inheritdoc */ override respond(): void { throw new Error('AwsFileUploader does not support ResponseMode.DELEGATE.'); } /** * @inheritdoc */ override async findDeliveryFile(attachment: IAttachmentDocument): Promise { if (!this.getIsReadable()) { throw new Error('AWS is not configured.'); } const s3 = S3Factory(); const filePath = getFilePathOnStorage(attachment); const params = { Bucket: getS3Bucket(), Key: filePath, }; // check file exists const isExists = await isFileExists(s3, params); if (!isExists) { throw new Error(`Any object that relate to the Attachment (${filePath}) does not exist in AWS S3`); } try { const body = (await s3.send(new GetObjectCommand(params))).Body; if (body == null) { throw new Error(`S3 returned null for the Attachment (${filePath})`); } // eslint-disable-next-line no-nested-ternary return 'stream' in body ? body.stream() as unknown as NodeJS.ReadableStream // get stream from Blob and cast force : body as unknown as NodeJS.ReadableStream; // cast force } catch (err) { logger.error(err); throw new Error(`Coudn't get file from AWS for the Attachment (${attachment._id.toString()})`); } } /** * @inheritDoc */ override async generateTemporaryUrl(attachment: IAttachmentDocument, opts?: RespondOptions): Promise { if (!this.getIsUploadable()) { throw new Error('AWS is not configured.'); } const s3 = S3Factory(); const filePath = getFilePathOnStorage(attachment); const lifetimeSecForTemporaryUrl = configManager.getConfig('crowi', 'aws:lifetimeSecForTemporaryUrl'); // issue signed url (default: expires 120 seconds) // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getSignedUrl-property const isDownload = opts?.download ?? false; const contentHeaders = new ContentHeaders(attachment, { inline: !isDownload }); const params: GetObjectCommandInput = { Bucket: getS3Bucket(), Key: filePath, ResponseContentType: contentHeaders.contentType?.value.toString(), ResponseContentDisposition: contentHeaders.contentDisposition?.value.toString(), }; const signedUrl = await getSignedUrl(s3, new GetObjectCommand(params), { expiresIn: lifetimeSecForTemporaryUrl, }); return { url: signedUrl, lifetimeSec: lifetimeSecForTemporaryUrl, }; } override createMultipartUploader(uploadKey: string, maxPartSize: number) { const s3 = S3Factory(); return new AwsMultipartUploader(s3, getS3Bucket(), uploadKey, maxPartSize); } override async abortPreviousMultipartUpload(uploadKey: string, uploadId: string) { try { await S3Factory().send(new AbortMultipartUploadCommand({ Bucket: getS3Bucket(), Key: uploadKey, UploadId: uploadId, })); } catch (e) { // allow duplicate abort requests to ensure abortion if (e.response?.status !== 404) { throw e; } } } } module.exports = (crowi) => { const lib = new AwsFileUploader(crowi); lib.isValidUploadSettings = function() { return configManager.getConfig('crowi', 'aws:s3AccessKeyId') != null && configManager.getConfig('crowi', 'aws:s3SecretAccessKey') != null && ( configManager.getConfig('crowi', 'aws:s3Region') != null || configManager.getConfig('crowi', 'aws:s3CustomEndpoint') != null ) && configManager.getConfig('crowi', 'aws:s3Bucket') != null; }; (lib as any).deleteFile = async function(attachment) { const filePath = getFilePathOnStorage(attachment); return (lib as any).deleteFileByFilePath(filePath); }; (lib as any).deleteFiles = async function(attachments) { if (!lib.getIsUploadable()) { throw new Error('AWS is not configured.'); } const s3 = S3Factory(); const filePaths = attachments.map((attachment) => { return { Key: getFilePathOnStorage(attachment) }; }); const totalParams = { Bucket: getS3Bucket(), Delete: { Objects: filePaths }, }; return s3.send(new DeleteObjectsCommand(totalParams)); }; (lib as any).deleteFileByFilePath = async function(filePath) { if (!lib.getIsUploadable()) { throw new Error('AWS is not configured.'); } const s3 = S3Factory(); const params = { Bucket: getS3Bucket(), Key: filePath, }; // check file exists const isExists = await isFileExists(s3, params); if (!isExists) { logger.warn(`Any object that relate to the Attachment (${filePath}) does not exist in AWS S3`); return; } return s3.send(new DeleteObjectCommand(params)); }; lib.saveFile = async function({ filePath, contentType, data }) { const s3 = S3Factory(); return s3.send(new PutObjectCommand({ Bucket: getS3Bucket(), ContentType: contentType, Key: filePath, Body: data, ACL: getS3PutObjectCannedAcl(), })); }; (lib as any).checkLimit = async function(uploadFileSize) { const maxFileSize = configManager.getConfig('crowi', 'app:maxFileSize'); const totalLimit = configManager.getConfig('crowi', 'app:fileUploadTotalLimit'); return lib.doCheckLimit(uploadFileSize, maxFileSize, totalLimit); }; /** * List files in storage */ (lib as any).listFiles = async function() { if (!lib.getIsReadable()) { throw new Error('AWS is not configured.'); } const files: FileMeta[] = []; const s3 = S3Factory(); const params = { Bucket: getS3Bucket(), }; let shouldContinue = true; let nextMarker: string | undefined; // handle pagination while (shouldContinue) { // eslint-disable-next-line no-await-in-loop const { Contents = [], IsTruncated, NextMarker } = await s3.send(new ListObjectsCommand({ ...params, Marker: nextMarker, })); files.push(...( Contents.map(({ Key, Size }) => ({ name: Key as string, size: Size as number, })) )); if (!IsTruncated) { shouldContinue = false; nextMarker = undefined; } else { nextMarker = NextMarker; } } return files; }; return lib; };