reuse existing upload on duplicate request

apps/app/src/features/page-bulk-export/interfaces/page-bulk-export.ts

@@ -30,6 +30,7 @@ export interface IPageBulkExportJob {
   completedAt?: Date, // the date at which job was completed
   attachment?: Ref<IAttachment>,
   status: PageBulkExportJobStatus,
+  revisionListHash?: string, // Hash created from the list of revision IDs. Used to detect existing duplicate uploads.
 }
 
 export interface IPageBulkExportJobHasId extends IPageBulkExportJob, HasObjectId {}

apps/app/src/features/page-bulk-export/server/models/page-bulk-export-job.ts

@@ -21,6 +21,7 @@ const pageBulkExportJobSchema = new Schema<PageBulkExportJobDocument>({
   status: {
     type: String, enum: Object.values(PageBulkExportJobStatus), required: true, default: PageBulkExportJobStatus.initializing,
   },
+  revisionListHash: { type: String },
 }, { timestamps: true });
 
 export default getOrCreateModel<PageBulkExportJobDocument, PageBulkExportJobModel>('PageBulkExportJob', pageBulkExportJobSchema);

apps/app/src/features/page-bulk-export/server/service/page-bulk-export-job-cron.ts

@@ -47,7 +47,16 @@ class PageBulkExportJobCronService extends CronService {
     });
     for (const downloadExpiredExportJob of downloadExpiredExportJobs) {
       try {
-        this.crowi.attachmentService?.removeAttachment(downloadExpiredExportJob.attachment);
+        // eslint-disable-next-line no-await-in-loop
+        const hasSameAttachmentAndDownloadNotExpired = await PageBulkExportJob.findOne({
+          attachment: downloadExpiredExportJob.attachment,
+          _id: { $ne: downloadExpiredExportJob._id },
+          completedAt: { $gte: new Date(Date.now() - downloadExpirationSeconds * 1000) },
+        });
+        if (hasSameAttachmentAndDownloadNotExpired == null) {
+          // delete attachment if no other export job (which download has not expired) has re-used it
+          this.crowi.attachmentService?.removeAttachment(downloadExpiredExportJob.attachment);
+        }
       }
       catch (err) {
         logger.error(err);

apps/app/src/features/page-bulk-export/server/service/page-bulk-export.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import type { Readable } from 'stream';
@@ -128,6 +129,7 @@ class PageBulkExportService {
     if (duplicatePageBulkExportJobInProgress != null) {
       throw new DuplicateBulkExportJobError();
     }
+
     const pageBulkExportJob: PageBulkExportJobDocument & HasObjectId = await PageBulkExportJob.create({
       user: currentUser, page: basePage, format, status: PageBulkExportJobStatus.initializing,
     });
@@ -147,7 +149,22 @@ class PageBulkExportService {
 
       if (pageBulkExportJob.status === PageBulkExportJobStatus.initializing) {
         await this.createPageSnapshots(user, pageBulkExportJob);
-        pageBulkExportJob.status = PageBulkExportJobStatus.exporting;
+
+        const duplicateExportJob = await PageBulkExportJob.findOne({
+          user: pageBulkExportJob.user,
+          page: pageBulkExportJob.page,
+          format: pageBulkExportJob.format,
+          status: PageBulkExportJobStatus.completed,
+          revisionListHash: pageBulkExportJob.revisionListHash,
+        });
+        if (duplicateExportJob != null) {
+          // if an upload with the exact same contents exists, re-use the same attachment of that upload
+          pageBulkExportJob.attachment = duplicateExportJob.attachment;
+          pageBulkExportJob.status = PageBulkExportJobStatus.completed;
+        }
+        else {
+          pageBulkExportJob.status = PageBulkExportJobStatus.exporting;
+        }
         await pageBulkExportJob.save();
       }
       if (pageBulkExportJob.status === PageBulkExportJobStatus.exporting) {
@@ -197,7 +214,8 @@ class PageBulkExportService {
   }
 
   /**
-   * Create a snapshot for each page that is to be exported in the pageBulkExportJob
+   * Create a snapshot for each page that is to be exported in the pageBulkExportJob.
+   * Also calulate revisionListHash and save it to the pageBulkExportJob.
    */
   private async createPageSnapshots(user, pageBulkExportJob: PageBulkExportJobDocument): Promise<void> {
     // if the process of creating snapshots was interrupted, delete the snapshots and create from the start
@@ -208,6 +226,8 @@ class PageBulkExportService {
       throw new Error('Base page not found');
     }
 
+    const revisionListHash = createHash('sha256');
+
     // create a Readable for pages to be exported
     const { PageQueryBuilder } = this.pageModel;
     const builder = await new PageQueryBuilder(this.pageModel.find())
@@ -223,6 +243,9 @@ class PageBulkExportService {
       objectMode: true,
       write: async(page: PageDocument, encoding, callback) => {
         try {
+          if (page.revision != null) {
+            revisionListHash.update(getIdForRef(page.revision).toString());
+          }
           await PageBulkExportPageSnapshot.create({
             pageBulkExportJob,
             path: page.path,
@@ -240,6 +263,9 @@ class PageBulkExportService {
     this.pageBulkExportJobStreamManager.addJobStream(pageBulkExportJob._id, pagesReadable);
 
     await pipelinePromise(pagesReadable, pageSnapshotsWritable);
+
+    pageBulkExportJob.revisionListHash = revisionListHash.digest('hex');
+    await pageBulkExportJob.save();
   }
 
   /**
@@ -302,7 +328,8 @@ class PageBulkExportService {
     const pageArchiver = this.setUpPageArchiver();
     const bufferToPartSizeTransform = getBufferToFixedSizeTransform(this.maxPartSize);
 
-    const originalName = `${pageBulkExportJob._id}.${this.compressExtension}`;
+    if (pageBulkExportJob.revisionListHash == null) throw new Error('revisionListHash is not set');
+    const originalName = `${pageBulkExportJob.revisionListHash}.${this.compressExtension}`;
     const attachment = Attachment.createWithoutSave(null, user, originalName, this.compressExtension, 0, AttachmentType.PAGE_BULK_EXPORT);
     const uploadKey = `${FilePathOnStoragePrefix.pageBulkExport}/${attachment.fileName}`;