create attachment for multipart upload

apps/app/src/features/page-bulk-export/server/service/page-bulk-export.ts

@@ -10,7 +10,11 @@ import type { QueueObject } from 'async';
 import gc from 'expose-gc/function';
 import mongoose from 'mongoose';
 
+import type { SupportedActionType } from '~/interfaces/activity';
 import { SupportedAction, SupportedTargetModel } from '~/interfaces/activity';
+import { AttachmentType, FilePathOnStoragePrefix } from '~/server/interfaces/attachment';
+import type { IAttachmentDocument } from '~/server/models';
+import { Attachment } from '~/server/models';
 import type { PageModel, PageDocument } from '~/server/models/page';
 import Subscription from '~/server/models/subscription';
 import type { IAwsMultipartUploader } from '~/server/service/file-uploader/aws/multipart-upload';
@@ -59,7 +63,9 @@ class PageBulkExportService {
     }
 
     const timeStamp = (new Date()).getTime();
-    const uploadKey = `page-bulk-export-${timeStamp}.zip`;
+    const originalName = `page-bulk-export-${timeStamp}.zip`;
+    const attachment = Attachment.createWithoutSave(null, currentUser, originalName, 'zip', 0, AttachmentType.PAGE_BULK_EXPORT);
+    const uploadKey = `${FilePathOnStoragePrefix.pageBulkExport}/${attachment.fileName}`;
 
     const pagesReadable = this.getPageReadable(basePagePath);
     const zipArchiver = this.setUpZipArchiver();
@@ -84,21 +90,22 @@ class PageBulkExportService {
       await Subscription.upsertSubscription(currentUser, SupportedTargetModel.MODEL_PAGE_BULK_EXPORT_JOB, pageBulkExportJob, SubscriptionStatusType.SUBSCRIBE);
     }
     catch (err) {
+      logger.error(err);
       await multipartUploader.abortUpload();
       throw err;
     }
 
-    const multipartUploadWritable = this.getMultipartUploadWritable(multipartUploader, pageBulkExportJob, currentUser, activityParameters);
+    const multipartUploadWritable = this.getMultipartUploadWritable(multipartUploader, pageBulkExportJob, attachment, activityParameters);
 
     // Cannot directly pipe from pagesWritable to zipArchiver due to how the 'append' method works.
     // Hence, execution of two pipelines is required.
-    pipeline(pagesReadable, pagesWritable, err => this.handleExportError(err, activityParameters, currentUser, pageBulkExportJob, multipartUploader));
+    pipeline(pagesReadable, pagesWritable, err => this.handleExportError(err, activityParameters, pageBulkExportJob, multipartUploader));
     pipeline(zipArchiver, bufferToPartSizeTransform, multipartUploadWritable,
-      err => this.handleExportError(err, activityParameters, currentUser, pageBulkExportJob, multipartUploader));
+      err => this.handleExportError(err, activityParameters, pageBulkExportJob, multipartUploader));
   }
 
   private async handleExportError(
-      err: Error | null, activityParameters: ActivityParameters, user, pageBulkExportJob: PageBulkExportJobDocument, multipartUploader?: IAwsMultipartUploader,
+      err: Error | null, activityParameters: ActivityParameters, pageBulkExportJob: PageBulkExportJobDocument, multipartUploader?: IAwsMultipartUploader,
   ): Promise<void> {
     if (err != null) {
       logger.error(err);
@@ -106,14 +113,7 @@ class PageBulkExportService {
         await multipartUploader.abortUpload();
       }
 
-      const activity = await this.crowi.activityService.createActivity({
-        ...activityParameters,
-        action: SupportedAction.ACTION_PAGE_BULK_EXPORT_FAILED,
-        targetModel: SupportedTargetModel.MODEL_PAGE_BULK_EXPORT_JOB,
-        target: pageBulkExportJob,
-      });
-      const preNotify = preNotifyService.generatePreNotify(activity);
-      this.activityEvent.emit('updated', activity, pageBulkExportJob, preNotify);
+      await this.notifyExportResult(activityParameters, pageBulkExportJob, SupportedAction.ACTION_PAGE_BULK_EXPORT_FAILED);
     }
   }
 
@@ -186,7 +186,10 @@ class PageBulkExportService {
   }
 
   private getMultipartUploadWritable(
-      multipartUploader: IAwsMultipartUploader, pageBulkExportJob: PageBulkExportJobDocument, user, activityParameters: ActivityParameters,
+      multipartUploader: IAwsMultipartUploader,
+      pageBulkExportJob: PageBulkExportJobDocument,
+      attachment: IAttachmentDocument,
+      activityParameters: ActivityParameters,
   ): Writable {
     let partNumber = 1;
 
@@ -208,17 +211,16 @@ class PageBulkExportService {
       final: async(callback) => {
         try {
           await multipartUploader.completeUpload();
+
+          const fileSize = await multipartUploader.getUploadedFileSize();
+          attachment.fileSize = fileSize;
+          await attachment.save();
+
           pageBulkExportJob.completedAt = new Date();
+          pageBulkExportJob.attachment = attachment._id;
           await pageBulkExportJob.save();
 
-          const activity = await this.crowi.activityService.createActivity({
-            ...activityParameters,
-            action: SupportedAction.ACTION_PAGE_BULK_EXPORT_COMPLETED,
-            targetModel: SupportedTargetModel.MODEL_PAGE_BULK_EXPORT_JOB,
-            target: pageBulkExportJob,
-          });
-          const preNotify = preNotifyService.generatePreNotify(activity);
-          this.activityEvent.emit('updated', activity, pageBulkExportJob, preNotify);
+          await this.notifyExportResult(activityParameters, pageBulkExportJob, SupportedAction.ACTION_PAGE_BULK_EXPORT_COMPLETED);
         }
         catch (err) {
           callback(err);
@@ -229,6 +231,19 @@ class PageBulkExportService {
     });
   }
 
+  private async notifyExportResult(
+      activityParameters: ActivityParameters, pageBulkExportJob: PageBulkExportJobDocument, action: SupportedActionType,
+  ) {
+    const activity = await this.crowi.activityService.createActivity({
+      ...activityParameters,
+      action,
+      targetModel: SupportedTargetModel.MODEL_PAGE_BULK_EXPORT_JOB,
+      target: pageBulkExportJob,
+    });
+    const preNotify = preNotifyService.generatePreNotify(activity);
+    this.activityEvent.emit('updated', activity, pageBulkExportJob, preNotify);
+  }
+
 }
 
 // eslint-disable-next-line import/no-mutable-exports

apps/app/src/server/interfaces/attachment.ts

@@ -2,6 +2,7 @@ export const AttachmentType = {
   BRAND_LOGO: 'BRAND_LOGO',
   WIKI_PAGE: 'WIKI_PAGE',
   PROFILE_IMAGE: 'PROFILE_IMAGE',
+  PAGE_BULK_EXPORT: 'PAGE_BULK_EXPORT',
 } as const;
 
 export type AttachmentType = typeof AttachmentType[keyof typeof AttachmentType];
@@ -29,3 +30,9 @@ export const ResponseMode = {
   DELEGATE: 'delegate',
 } as const;
 export type ResponseMode = typeof ResponseMode[keyof typeof ResponseMode];
+
+export const FilePathOnStoragePrefix = {
+  attachment: 'attachment',
+  user: 'user',
+  pageBulkExport: 'page-bulk-export',
+} as const;

apps/app/src/server/models/attachment.ts

@@ -32,7 +32,9 @@ export interface IAttachmentDocument extends IAttachment, Document {
   cashTemporaryUrlByProvideSec: CashTemporaryUrlByProvideSec,
 }
 export interface IAttachmentModel extends Model<IAttachmentDocument> {
-  createWithoutSave
+  createWithoutSave: (
+    pageId, user, originalName: string, fileFormat: string, fileSize: number, attachmentType: AttachmentType,
+  ) => IAttachmentDocument;
 }
 
 const attachmentSchema = new Schema({
@@ -69,7 +71,9 @@ attachmentSchema.set('toObject', { virtuals: true });
 attachmentSchema.set('toJSON', { virtuals: true });
 
 
-attachmentSchema.statics.createWithoutSave = function(pageId, user, fileStream, originalName, fileFormat, fileSize, attachmentType) {
+attachmentSchema.statics.createWithoutSave = function(
+    pageId, user, originalName: string, fileFormat: string, fileSize: number, attachmentType: AttachmentType,
+) {
   // eslint-disable-next-line @typescript-eslint/no-this-alias
   const Attachment = this;
 

apps/app/src/server/service/attachment.js

@@ -35,7 +35,7 @@ class AttachmentService {
     // create an Attachment document and upload file
     let attachment;
     try {
-      attachment = Attachment.createWithoutSave(pageId, user, fileStream, file.originalname, file.mimetype, file.size, attachmentType);
+      attachment = Attachment.createWithoutSave(pageId, user, file.originalname, file.mimetype, file.size, attachmentType);
       await fileUploadService.uploadAttachment(fileStream, attachment);
       await attachment.save();
     }

apps/app/src/server/service/file-uploader/aws/index.ts

@@ -1,5 +1,3 @@
-import { Writable } from 'stream';
-
 import {
   S3Client,
   HeadObjectCommand,
@@ -10,14 +8,11 @@ import {
   ListObjectsCommand,
   type GetObjectCommandInput,
   ObjectCannedACL,
-  CreateMultipartUploadCommand,
-  UploadPartCommand,
-  CompleteMultipartUploadCommand,
 } from '@aws-sdk/client-s3';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import urljoin from 'url-join';
 
-import { ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
+import { FilePathOnStoragePrefix, ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
 import type { IAttachmentDocument } from '~/server/models';
 import loggerFactory from '~/utils/logger';
 
@@ -90,8 +85,8 @@ const getFilePathOnStorage = (attachment) => {
   }
 
   const dirName = (attachment.page != null)
-    ? 'attachment'
-    : 'user';
+    ? FilePathOnStoragePrefix.attachment
+    : FilePathOnStoragePrefix.user;
   const filePath = urljoin(dirName, attachment.fileName);
 
   return filePath;

apps/app/src/server/service/file-uploader/aws/multipart-upload.ts

@@ -1,5 +1,6 @@
 import {
   CreateMultipartUploadCommand, UploadPartCommand, type S3Client, CompleteMultipartUploadCommand, AbortMultipartUploadCommand,
+  HeadObjectCommand,
 } from '@aws-sdk/client-s3';
 
 import loggerFactory from '~/utils/logger';
@@ -20,6 +21,7 @@ export interface IAwsMultipartUploader {
   completeUpload(): Promise<void>;
   abortUpload(): Promise<void>;
   uploadId: string | undefined;
+  getUploadedFileSize(): Promise<number>;
 }
 
 /**
@@ -42,6 +44,8 @@ export class AwsMultipartUploader implements IAwsMultipartUploader {
 
   private currentStatus: UploadStatus = UploadStatus.BEFORE_INIT;
 
+  private _uploadedFileSize: number | undefined;
+
   constructor(s3Client: S3Client, bucket: string, uploadKey: string) {
     this.s3Client = s3Client;
     this.bucket = bucket;
@@ -108,6 +112,18 @@ export class AwsMultipartUploader implements IAwsMultipartUploader {
     logger.info(`Multipart upload aborted. Upload key: ${this.uploadKey}`);
   }
 
+  async getUploadedFileSize(): Promise<number> {
+    if (this._uploadedFileSize != null) return this._uploadedFileSize;
+
+    this.validateUploadStatus(UploadStatus.COMPLETED);
+    const headData = await this.s3Client.send(new HeadObjectCommand({
+      Bucket: this.bucket,
+      Key: this.uploadKey,
+    }));
+    this._uploadedFileSize = headData.ContentLength;
+    return this._uploadedFileSize ?? 0;
+  }
+
   private validateUploadStatus(desiredStatus: UploadStatus): void {
     if (desiredStatus === this.currentStatus) return;
 

apps/app/src/server/service/file-uploader/azure.ts

@@ -1,11 +1,14 @@
-import { ClientSecretCredential, TokenCredential } from '@azure/identity';
-import {
-  generateBlobSASQueryParameters,
-  BlobServiceClient,
+import type { TokenCredential } from '@azure/identity';
+import { ClientSecretCredential } from '@azure/identity';
+import type {
   BlobClient,
   BlockBlobClient,
   BlobDeleteOptions,
   ContainerClient,
+} from '@azure/storage-blob';
+import {
+  generateBlobSASQueryParameters,
+  BlobServiceClient,
   ContainerSASPermissions,
   SASProtocol,
   type BlobDeleteIfExistsResponse,
@@ -13,7 +16,7 @@ import {
   type BlockBlobParallelUploadOptions,
 } from '@azure/storage-blob';
 
-import { ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
+import { FilePathOnStoragePrefix, ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
 import type { IAttachmentDocument } from '~/server/models';
 import loggerFactory from '~/utils/logger';
 
@@ -60,7 +63,7 @@ async function getContainerClient(): Promise<ContainerClient> {
 }
 
 function getFilePathOnStorage(attachment) {
-  const dirName = (attachment.page != null) ? 'attachment' : 'user';
+  const dirName = (attachment.page != null) ? FilePathOnStoragePrefix.attachment : FilePathOnStoragePrefix.user;
   return urljoin(dirName, attachment.fileName);
 }
 

apps/app/src/server/service/file-uploader/gcs.ts

@@ -2,7 +2,7 @@ import { Storage } from '@google-cloud/storage';
 import urljoin from 'url-join';
 
 import type Crowi from '~/server/crowi';
-import { ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
+import { FilePathOnStoragePrefix, ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
 import type { IAttachmentDocument } from '~/server/models';
 import loggerFactory from '~/utils/logger';
 
@@ -36,8 +36,8 @@ function getFilePathOnStorage(attachment) {
   const namespace = configManager.getConfig('crowi', 'gcs:uploadNamespace');
   // const namespace = null;
   const dirName = (attachment.page != null)
-    ? 'attachment'
-    : 'user';
+    ? FilePathOnStoragePrefix.attachment
+    : FilePathOnStoragePrefix.user;
   const filePath = urljoin(namespace || '', dirName, attachment.fileName);
 
   return filePath;

apps/app/src/server/service/file-uploader/local.ts

@@ -2,7 +2,7 @@ import { Readable } from 'stream';
 
 import type { Response } from 'express';
 
-import { ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
+import { FilePathOnStoragePrefix, ResponseMode, type RespondOptions } from '~/server/interfaces/attachment';
 import type { IAttachmentDocument } from '~/server/models';
 import loggerFactory from '~/utils/logger';
 
@@ -101,8 +101,8 @@ module.exports = function(crowi) {
 
   function getFilePathOnStorage(attachment) {
     const dirName = (attachment.page != null)
-      ? 'attachment'
-      : 'user';
+      ? FilePathOnStoragePrefix.attachment
+      : FilePathOnStoragePrefix.user;
     const filePath = path.posix.join(basePath, dirName, attachment.fileName);
 
     return filePath;