Create page snapshots and use them for export

apps/app/public/static/locales/en_US/translation.json

@@ -640,7 +640,8 @@
     "markdown": "Markdown",
     "choose_export_format": "Select export format",
     "bulk_export_started": "Please wait a moment...",
-    "bulk_export_download_expired": "Download period has expired"
+    "bulk_export_download_expired": "Download period has expired",
+    "duplicate_bulk_export_job_error": "Export for the same page and its children is in progress"
   },
   "message": {
     "successfully_connected": "Successfully Connected!",

apps/app/public/static/locales/fr_FR/translation.json

@@ -634,7 +634,8 @@
     "markdown": "Markdown",
     "choose_export_format": "Sélectionnez le format d'exportation",
     "bulk_export_started": "Patientez s'il-vous-plait...",
-    "bulk_export_download_expired": "La période de téléchargement a expiré"
+    "bulk_export_download_expired": "La période de téléchargement a expiré",
+    "duplicate_bulk_export_job_error": "L'export pour la même page et ses enfants est en cours"
   },
   "message": {
     "successfully_connected": "Connecté!",

apps/app/public/static/locales/ja_JP/translation.json

@@ -673,7 +673,8 @@
     "markdown": "マークダウン",
     "choose_export_format": "エクスポート形式を選択してください",
     "bulk_export_started": "ただいま準備中です...",
-    "bulk_export_download_expired": "ダウンロード期限が切れました"
+    "bulk_export_download_expired": "ダウンロード期限が切れました",
+    "duplicate_bulk_export_job_error": "既に同じページとその配下のエクスポートが進行中です"
   },
   "message": {
     "successfully_connected": "接続に成功しました!",

apps/app/public/static/locales/zh_CN/translation.json

@@ -643,7 +643,8 @@
     "markdown": "Markdown",
     "choose_export_format": "选择导出格式",
     "bulk_export_started": "目前我们正在准备...",
-    "bulk_export_download_expired": "下载期限已过"
+    "bulk_export_download_expired": "下载期限已过",
+    "duplicate_bulk_export_job_error": "正在导出同一页面及其子页面"
   },
   "message": {
     "successfully_connected": "连接成功！",

apps/app/src/features/page-bulk-export/client/components/PageBulkExportSelectModal.tsx

@@ -18,7 +18,9 @@ const PageBulkExportSelectModal = (): JSX.Element => {
       toastSuccess(t('page_export.bulk_export_started'));
     }
     catch (e) {
-      toastError(t('page_export.failed_to_export'));
+      // TODO: Enable cancel and restart of export if duplicate export exists (https://redmine.weseek.co.jp/issues/150418)
+      const errorCode = e?.[0].code ?? 'page_export.failed_to_export';
+      toastError(t(errorCode));
     }
     close();
   };
@@ -38,7 +40,7 @@ const PageBulkExportSelectModal = (): JSX.Element => {
               </small>
             </div>
             <div className="d-flex justify-content-center mt-2">
-              <button className="btn btn-primary" type="button" onClick={() => startBulkExport(PageBulkExportFormat.markdown)}>
+              <button className="btn btn-primary" type="button" onClick={() => startBulkExport(PageBulkExportFormat.md)}>
                 {t('page_export.markdown')}
               </button>
               <button className="btn btn-primary ms-2" type="button" onClick={() => startBulkExport(PageBulkExportFormat.pdf)}>PDF</button>

apps/app/src/features/page-bulk-export/interfaces/page-bulk-export.ts

@@ -4,12 +4,20 @@ import type {
 } from '@growi/core';
 
 export const PageBulkExportFormat = {
-  markdown: 'markdown',
+  md: 'md',
   pdf: 'pdf',
 } as const;
 
 export type PageBulkExportFormat = typeof PageBulkExportFormat[keyof typeof PageBulkExportFormat]
 
+export const PageBulkExportJobStatus = {
+  inProgress: 'inProgress',
+  completed: 'completed',
+  failed: 'failed',
+} as const;
+
+export type PageBulkExportJobStatus = typeof PageBulkExportJobStatus[keyof typeof PageBulkExportJobStatus]
+
 export interface IPageBulkExportJob {
   user: Ref<IUser>, // user that started export job
   page: Ref<IPage>, // the root page of page tree to export
@@ -18,12 +26,13 @@ export interface IPageBulkExportJob {
   format: PageBulkExportFormat,
   completedAt: Date | null, // the date at which job was completed
   attachment?: Ref<IAttachment>,
+  status: PageBulkExportJobStatus,
 }
 
 export interface IPageBulkExportJobHasId extends IPageBulkExportJob, HasObjectId {}
 
 // snapshot of page info to upload
-export interface IPageBulkExportPageInfo {
+export interface IPageBulkExportPageSnapshot {
   pageBulkExportJob: Ref<IPageBulkExportJob>,
   path: string, // page path when export was stared
   revision: Ref<IRevision>, // page revision when export was stared

apps/app/src/features/page-bulk-export/server/models/page-bulk-export-job.ts

@@ -3,7 +3,7 @@ import { type Document, type Model, Schema } from 'mongoose';
 import { getOrCreateModel } from '~/server/util/mongoose-utils';
 
 import type { IPageBulkExportJob } from '../../interfaces/page-bulk-export';
-import { PageBulkExportFormat } from '../../interfaces/page-bulk-export';
+import { PageBulkExportFormat, PageBulkExportJobStatus } from '../../interfaces/page-bulk-export';
 
 export interface PageBulkExportJobDocument extends IPageBulkExportJob, Document {}
 
@@ -17,6 +17,9 @@ const pageBulkExportJobSchema = new Schema<PageBulkExportJobDocument>({
   format: { type: String, enum: Object.values(PageBulkExportFormat), required: true },
   completedAt: { type: Date },
   attachment: { type: Schema.Types.ObjectId, ref: 'Attachment' },
+  status: {
+    type: String, enum: Object.values(PageBulkExportJobStatus), required: true, default: PageBulkExportJobStatus.inProgress,
+  },
 }, { timestamps: true });
 
 export default getOrCreateModel<PageBulkExportJobDocument, PageBulkExportJobModel>('PageBulkExportJob', pageBulkExportJobSchema);

apps/app/src/features/page-bulk-export/server/models/page-bulk-export-page-info.ts

@@ -1,17 +0,0 @@
-import { type Document, type Model, Schema } from 'mongoose';
-
-import { getOrCreateModel } from '~/server/util/mongoose-utils';
-
-import { IPageBulkExportPageInfo } from '../../interfaces/page-bulk-export';
-
-export interface PageBulkExportPageInfoDocument extends IPageBulkExportPageInfo, Document {}
-
-export type PageBulkExportPageInfoModel = Model<PageBulkExportPageInfoDocument>
-
-const pageBulkExportPageInfoSchema = new Schema<PageBulkExportPageInfoDocument>({
-  pageBulkExportJob: { type: Schema.Types.ObjectId, ref: 'PageBulkExportJob', required: true },
-  path: { type: String, required: true },
-  revision: { type: Schema.Types.ObjectId, ref: 'Revision', required: true },
-}, { timestamps: true });
-
-export default getOrCreateModel<PageBulkExportPageInfoDocument, PageBulkExportPageInfoModel>('PageBulkExportPageInfo', pageBulkExportPageInfoSchema);

apps/app/src/features/page-bulk-export/server/models/page-bulk-export-page-snapshot.ts

@@ -0,0 +1,19 @@
+import { type Document, type Model, Schema } from 'mongoose';
+
+import { getOrCreateModel } from '~/server/util/mongoose-utils';
+
+import type { IPageBulkExportPageSnapshot } from '../../interfaces/page-bulk-export';
+
+export interface PageBulkExportPageSnapshotDocument extends IPageBulkExportPageSnapshot, Document {}
+
+export type PageBulkExportPageSnapshotModel = Model<PageBulkExportPageSnapshotDocument>
+
+const pageBulkExportPageInfoSchema = new Schema<PageBulkExportPageSnapshotDocument>({
+  pageBulkExportJob: { type: Schema.Types.ObjectId, ref: 'PageBulkExportJob', required: true },
+  path: { type: String, required: true },
+  revision: { type: Schema.Types.ObjectId, ref: 'Revision', required: true },
+}, { timestamps: true });
+
+export default getOrCreateModel<PageBulkExportPageSnapshotDocument, PageBulkExportPageSnapshotModel>(
+  'PageBulkExportPageSnapshot', pageBulkExportPageInfoSchema,
+);

apps/app/src/features/page-bulk-export/server/routes/apiv3/page-bulk-export.ts

@@ -7,7 +7,7 @@ import type Crowi from '~/server/crowi';
 import type { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response';
 import loggerFactory from '~/utils/logger';
 
-import { pageBulkExportService } from '../../service/page-bulk-export';
+import { DuplicateBulkExportJobError, pageBulkExportService } from '../../service/page-bulk-export';
 
 const logger = loggerFactory('growi:routes:apiv3:page-bulk-export');
 
@@ -45,7 +45,10 @@ module.exports = (crowi: Crowi): Router => {
     }
     catch (err) {
       logger.error(err);
-      return res.apiv3Err(new ErrorV3('Failed to start bulk export'));
+      if (err instanceof DuplicateBulkExportJobError) {
+        return res.apiv3Err(new ErrorV3('Duplicate bulk export job is in progress', 'page_export.duplicate_bulk_export_job_error'), 409);
+      }
+      return res.apiv3Err(new ErrorV3('Failed to start bulk export', 'page_export.failed_to_export'));
     }
   });
 

apps/app/src/features/page-bulk-export/server/service/page-bulk-export.ts

@@ -27,18 +27,27 @@ import { preNotifyService } from '~/server/service/pre-notify';
 import { getBufferToFixedSizeTransform } from '~/server/util/stream';
 import loggerFactory from '~/utils/logger';
 
-import { PageBulkExportFormat } from '../../interfaces/page-bulk-export';
+import { PageBulkExportFormat, PageBulkExportJobStatus } from '../../interfaces/page-bulk-export';
 import type { PageBulkExportJobDocument } from '../models/page-bulk-export-job';
 import PageBulkExportJob from '../models/page-bulk-export-job';
+import PageBulkExportPageSnapshot from '../models/page-bulk-export-page-snapshot';
 
 
 const logger = loggerFactory('growi:services:PageBulkExportService');
 
 type ActivityParameters ={
-  ip: string;
+  ip: string | undefined;
   endpoint: string;
 }
 
+export class DuplicateBulkExportJobError extends Error {
+
+  constructor() {
+    super('Duplicate bulk export job is in progress');
+  }
+
+}
+
 class PageBulkExportService {
 
   crowi: any;
@@ -69,18 +78,22 @@ class PageBulkExportService {
       throw new Error('Base page not found or not accessible');
     }
 
-    const pageBulkExportJob: PageBulkExportJobDocument & HasObjectId = await PageBulkExportJob.create({
-      user: currentUser,
-      page: basePage,
-      format: PageBulkExportFormat.markdown,
-    });
+    const format = PageBulkExportFormat.md;
+    const pageBulkExportJobProperties = {
+      user: currentUser, page: basePage, format, status: PageBulkExportJobStatus.inProgress,
+    };
+    const duplicatePageBulkExportJobInProgress: PageBulkExportJobDocument & HasObjectId | null = await PageBulkExportJob.findOne(pageBulkExportJobProperties);
+    if (duplicatePageBulkExportJobInProgress != null) {
+      throw new DuplicateBulkExportJobError();
+    }
+    const pageBulkExportJob: PageBulkExportJobDocument & HasObjectId = await PageBulkExportJob.create(pageBulkExportJobProperties);
 
     await Subscription.upsertSubscription(currentUser, SupportedTargetModel.MODEL_PAGE_BULK_EXPORT_JOB, pageBulkExportJob, SubscriptionStatusType.SUBSCRIBE);
 
     this.bulkExportWithBasePagePath(basePagePath, currentUser, activityParameters, pageBulkExportJob);
   }
 
-  async bulkExportWithBasePagePath(
+  private async bulkExportWithBasePagePath(
       basePagePath: string, currentUser, activityParameters: ActivityParameters, pageBulkExportJob: PageBulkExportJobDocument & HasObjectId,
   ): Promise<void> {
     const timeStamp = (new Date()).getTime();
@@ -153,7 +166,7 @@ class PageBulkExportService {
   }
 
   private async exportPagesToFS(basePagePath: string, outputDir: string, currentUser): Promise<void> {
-    const pagesReadable = await this.getPageReadable(basePagePath, currentUser);
+    const pagesReadable = await this.getPageReadable(basePagePath, currentUser, true);
     const pagesWritable = this.getPageWritable(outputDir);
 
     return pipelinePromise(pagesReadable, pagesWritable);
@@ -162,7 +175,7 @@ class PageBulkExportService {
   /**
    * Get a Readable of all the pages under the specified path, including the root page.
    */
-  private async getPageReadable(basePagePath: string, currentUser): Promise<Readable> {
+  private async getPageReadable(basePagePath: string, currentUser, populateRevision = false): Promise<Readable> {
     const Page = mongoose.model<IPage, PageModel>('Page');
     const { PageQueryBuilder } = Page;
 
@@ -170,13 +183,38 @@ class PageBulkExportService {
       .addConditionToListWithDescendants(basePagePath)
       .addViewerCondition(currentUser);
 
+    if (populateRevision) {
+      builder.query = builder.query.populate('revision');
+    }
+
     return builder
       .query
-      .populate('revision')
       .lean()
       .cursor({ batchSize: this.pageBatchSize });
   }
 
+  private async createPageSnapshots(basePagePath: string, currentUser, pageBulkExportJob: PageBulkExportJobDocument) {
+    const pagesReadable = await this.getPageReadable(basePagePath, currentUser);
+    const pageSnapshotwritable = new Writable({
+      objectMode: true,
+      write: async(page: PageDocument, encoding, callback) => {
+        try {
+          await PageBulkExportPageSnapshot.create({
+            pageBulkExportJob: pageBulkExportJob._id,
+            path: page.path,
+            revision: page.revision,
+          });
+        }
+        catch (err) {
+          callback(err);
+          return;
+        }
+        callback();
+      },
+    });
+    await pipelinePromise(pagesReadable, pageSnapshotwritable);
+  }
+
   /**
    * Get a Writable that writes the page body temporarily to fs
    */