add pdf convert process to PageBulkExportJobCronService

apps/app/src/features/page-bulk-export/server/service/page-bulk-export-job-cron/index.ts

@@ -17,13 +17,14 @@ import type { FileUploader } from '~/server/service/file-uploader';
 import { preNotifyService } from '~/server/service/pre-notify';
 import loggerFactory from '~/utils/logger';
 
-import { PageBulkExportJobInProgressStatus, PageBulkExportJobStatus } from '../../../interfaces/page-bulk-export';
+import { PageBulkExportFormat, PageBulkExportJobInProgressStatus, PageBulkExportJobStatus } from '../../../interfaces/page-bulk-export';
 import type { PageBulkExportJobDocument } from '../../models/page-bulk-export-job';
 import PageBulkExportJob from '../../models/page-bulk-export-job';
 import PageBulkExportPageSnapshot from '../../models/page-bulk-export-page-snapshot';
 
 
 import { BulkExportJobExpiredError, BulkExportJobRestartedError } from './errors';
+import { requestPdfConverter } from './request-pdf-converter';
 import { compressAndUploadAsync } from './steps/compress-and-upload-async';
 import { createPageSnapshotsAsync } from './steps/create-page-snapshots-async';
 import { exportPagesToFsAsync } from './steps/export-pages-to-fs-async';
@@ -130,20 +131,25 @@ class PageBulkExportJobCronService extends CronService implements IPageBulkExpor
    * @param pageBulkExportJob PageBulkExportJob in progress
    */
   async proceedBulkExportJob(pageBulkExportJob: PageBulkExportJobDocument) {
-    if (pageBulkExportJob.restartFlag) {
-      await this.cleanUpExportJobResources(pageBulkExportJob, true);
-      pageBulkExportJob.restartFlag = false;
-      pageBulkExportJob.status = PageBulkExportJobStatus.initializing;
-      pageBulkExportJob.statusOnPreviousCronExec = undefined;
-      await pageBulkExportJob.save();
-    }
-
-    // return if job is still the same status as the previous cron exec
-    if (pageBulkExportJob.status === pageBulkExportJob.statusOnPreviousCronExec) {
-      return;
-    }
-    const User = mongoose.model<IUser>('User');
     try {
+      if (pageBulkExportJob.restartFlag) {
+        await this.cleanUpExportJobResources(pageBulkExportJob, true);
+        pageBulkExportJob.restartFlag = false;
+        pageBulkExportJob.status = PageBulkExportJobStatus.initializing;
+        pageBulkExportJob.statusOnPreviousCronExec = undefined;
+        await pageBulkExportJob.save();
+      }
+
+      if (pageBulkExportJob.status === PageBulkExportJobStatus.exporting && pageBulkExportJob.format === PageBulkExportFormat.pdf) {
+        await requestPdfConverter(pageBulkExportJob);
+      }
+
+      // return if job is still the same status as the previous cron exec
+      if (pageBulkExportJob.status === pageBulkExportJob.statusOnPreviousCronExec) {
+        return;
+      }
+
+      const User = mongoose.model<IUser>('User');
       const user = await User.findById(getIdForRef(pageBulkExportJob.user));
 
       // update statusOnPreviousCronExec before starting processes that updates status

apps/app/src/features/page-bulk-export/server/service/page-bulk-export-job-cron/request-pdf-converter.ts

@@ -0,0 +1,57 @@
+import { configManager } from '~/server/service/config-manager';
+
+import { PageBulkExportJobStatus } from '../../../interfaces/page-bulk-export';
+import type { PageBulkExportJobDocument } from '../../models/page-bulk-export-job';
+import PageBulkExportPageSnapshot from '../../models/page-bulk-export-page-snapshot';
+
+import { BulkExportJobExpiredError } from './errors';
+
+import { PdfCtrlSyncJobStatus202Status, PdfCtrlSyncJobStatusBodyStatus, pdfCtrlSyncJobStatus } from '^/../pdf-converter/dist/client-library';
+
+export async function requestPdfConverter(pageBulkExportJob: PageBulkExportJobDocument): Promise<void> {
+  const jobCreatedAt = pageBulkExportJob.createdAt;
+  if (jobCreatedAt == null) {
+    throw new Error('createdAt is not set');
+  }
+
+  const exportJobExpirationSeconds = configManager.getConfig('crowi', 'app:bulkExportJobExpirationSeconds');
+  const bulkExportJobExpirationDate = new Date(jobCreatedAt.getTime() + exportJobExpirationSeconds * 1000);
+  let pdfConvertStatus: PdfCtrlSyncJobStatusBodyStatus = PdfCtrlSyncJobStatusBodyStatus.HTML_EXPORT_IN_PROGRESS;
+
+  const lastExportPagePath = (await PageBulkExportPageSnapshot.findOne({ pageBulkExportJob }).sort({ path: -1 }))?.path;
+  if (lastExportPagePath == null) {
+    throw new Error('lastExportPagePath is missing');
+  }
+
+  if (new Date() > bulkExportJobExpirationDate) {
+    throw new BulkExportJobExpiredError();
+  }
+  try {
+    if (pageBulkExportJob.lastExportedPagePath === lastExportPagePath) {
+      pdfConvertStatus = PdfCtrlSyncJobStatusBodyStatus.HTML_EXPORT_DONE;
+    }
+
+    if (pageBulkExportJob.status === PageBulkExportJobStatus.failed) {
+      pdfConvertStatus = PdfCtrlSyncJobStatusBodyStatus.FAILED;
+    }
+
+    const res = await pdfCtrlSyncJobStatus({
+      jobId: pageBulkExportJob._id.toString(), expirationDate: bulkExportJobExpirationDate.toISOString(), status: pdfConvertStatus,
+    }, { baseURL: configManager.getConfig('crowi', 'app:pageBulkExportPdfConverterUrl') });
+
+    if (res.data.status === PdfCtrlSyncJobStatus202Status.PDF_EXPORT_DONE) {
+      pageBulkExportJob.status = PageBulkExportJobStatus.uploading;
+      await pageBulkExportJob.save();
+    }
+    else if (res.data.status === PdfCtrlSyncJobStatus202Status.FAILED) {
+      throw new Error('PDF export failed');
+    }
+  }
+  catch (err) {
+    // Only set as failure when host is ready but failed.
+    // If host is not ready, the request should be retried on the next cron execution.
+    if (!['ENOTFOUND', 'ECONNREFUSED'].includes(err.code)) {
+      throw err;
+    }
+  }
+}

apps/app/src/features/page-bulk-export/server/service/page-bulk-export-job-cron/steps/export-pages-to-fs-async.ts

@@ -43,8 +43,12 @@ function getPageWritable(this: IPageBulkExportJobCronService, pageBulkExportJob:
     },
     final: async(callback) => {
       try {
-        pageBulkExportJob.status = PageBulkExportJobStatus.uploading;
-        await pageBulkExportJob.save();
+        // If the format is md, the export process ends here.
+        // If the format is pdf, pdf conversion in pdf-converter has to finish.
+        if (pageBulkExportJob.format === PageBulkExportFormat.md) {
+          pageBulkExportJob.status = PageBulkExportJobStatus.uploading;
+          await pageBulkExportJob.save();
+        }
       }
       catch (err) {
         callback(err);

apps/app/src/features/page-bulk-export/server/service/page-bulk-export-pdf-convert-cron.ts

@@ -1,117 +0,0 @@
-import type { HydratedDocument } from 'mongoose';
-import EventEmitter from 'node:events';
-
-import { configManager } from '~/server/service/config-manager';
-import CronService from '~/server/service/cron';
-import loggerFactory from '~/utils/logger';
-
-import { PageBulkExportFormat, PageBulkExportJobStatus } from '../../interfaces/page-bulk-export';
-import type { PageBulkExportJobDocument } from '../models/page-bulk-export-job';
-import PageBulkExportJob from '../models/page-bulk-export-job';
-
-import { BulkExportJobExpiredError } from './page-bulk-export/errors';
-import { PdfCtrlSyncJobStatus202Status, PdfCtrlSyncJobStatusBodyStatus, pdfCtrlSyncJobStatus } from '^/../pdf-converter/dist/client-library';
-import PageBulkExportPageSnapshot from '../models/page-bulk-export-page-snapshot';
-
-const logger = loggerFactory('growi:service:page-bulk-export-pdf-convert-cron');
-
-const eventEmitter = new EventEmitter();
-
-/**
- * Start pdf export by requesting pdf-converter and keep updating/checking the status until the export is done
- * ref) https://dev.growi.org/66ee8495830566b31e02c953#growi
- * @param pageBulkExportJob page bulk export job in execution
- */
-class PageBulkExportPdfConvertCronService extends CronService {
-
-  override getCronSchedule(): string {
-    return configManager.getConfig('crowi', 'app:pageBulkExportPdfConvertCronSchedule');
-  }
-
-  override async executeJob(): Promise<void> {
-    const pdfExportingJobs = await PageBulkExportJob.find({
-      format: PageBulkExportFormat.pdf,
-      status: PageBulkExportJobStatus.exporting,
-    });
-
-    const results = await Promise.allSettled(
-      pdfExportingJobs.map(async (pageBulkExportJob) => {
-        await this.requestPdfConverter(pageBulkExportJob);
-      })
-    )
-
-    results.forEach((result) => {
-      if (result.status === 'rejected') logger.error(result.reason);
-    });
-  }
-
-  async requestPdfConverter(pageBulkExportJob: HydratedDocument<PageBulkExportJobDocument>): Promise<void> {
-    const jobCreatedAt = pageBulkExportJob.createdAt;
-    if (jobCreatedAt == null) {
-      eventEmitter.emit('pdfExportFailed');
-      throw new Error('createdAt is not set');
-    }
-
-    const exportJobExpirationSeconds = configManager.getConfig('crowi', 'app:bulkExportJobExpirationSeconds');
-    const bulkExportJobExpirationDate = new Date(jobCreatedAt.getTime() + exportJobExpirationSeconds * 1000);
-    let pdfConvertStatus: PdfCtrlSyncJobStatusBodyStatus = PdfCtrlSyncJobStatusBodyStatus.HTML_EXPORT_IN_PROGRESS;
-
-    const lastExportPagePath = (await PageBulkExportPageSnapshot.findOne({ pageBulkExportJob }).sort({ path: -1 }))?.path;
-    if (lastExportPagePath == null) {
-      eventEmitter.emit('pdfExportFailed id:' + pageBulkExportJob._id.toString());
-      throw new Error('lastExportPagePath is missing');
-    }
-
-    if (new Date() > bulkExportJobExpirationDate) {
-      eventEmitter.emit('bulkExportJobExpired id:' + pageBulkExportJob._id.toString());
-    }
-    try {
-      if (pageBulkExportJob.lastExportedPagePath === lastExportPagePath) {
-        pdfConvertStatus = PdfCtrlSyncJobStatusBodyStatus.HTML_EXPORT_DONE;
-      }
-
-      if (pageBulkExportJob.status === PageBulkExportJobStatus.failed) {
-        pdfConvertStatus = PdfCtrlSyncJobStatusBodyStatus.FAILED;
-      }
-
-      const res = await pdfCtrlSyncJobStatus({
-        jobId: pageBulkExportJob._id.toString(), expirationDate: bulkExportJobExpirationDate.toISOString(), status: pdfConvertStatus,
-      }, { baseURL: configManager.getConfig('crowi', 'app:pageBulkExportPdfConverterUrl') });
-
-      if (res.data.status === PdfCtrlSyncJobStatus202Status.PDF_EXPORT_DONE) {
-        eventEmitter.emit('pdfExportDone id:' + pageBulkExportJob._id.toString());
-      }
-      else if (res.data.status === PdfCtrlSyncJobStatus202Status.FAILED) {
-        eventEmitter.emit('pdfExportFailed id:' + pageBulkExportJob._id.toString());
-      }
-    }
-    catch (err) {
-      // Only set as failure when host is ready but failed.
-      // If host is not ready, the request should be retried on the next cron execution.
-      if (!['ENOTFOUND', 'ECONNREFUSED'].includes(err.code)) {
-        eventEmitter.emit('pdfExportFailed id:' + pageBulkExportJob._id.toString());
-        throw err;
-      }
-    }
-  };
-}
-
-export const pageBulkExportPdfConvertCronService = Object.freeze(new PageBulkExportPdfConvertCronService());
-
-export function waitPdfExportToFs(pageBulkExportJob: PageBulkExportJobDocument): Promise<void> {
-  return new Promise<void>((resolve, reject) => {
-    eventEmitter.on('pdfExportDone id:' + pageBulkExportJob._id.toString(), () => {
-      resolve();
-      return;
-    });
-
-    eventEmitter.on('bulkExportJobExpired id:' + pageBulkExportJob._id.toString(), () => {
-      reject(new BulkExportJobExpiredError());
-      return;
-    });
-    eventEmitter.on('pdfExportFailed id:' + pageBulkExportJob._id.toString(), () => {
-      reject(new Error('PDF export failed'));
-      return;
-    });
-  });
-}

apps/app/src/server/service/config-loader.ts

@@ -781,12 +781,6 @@ const ENV_VAR_NAME_TO_CONFIG_INFO: Record<string, EnvConfig> = {
     type: ValueType.STRING,
     default: '*/10 * * * *', // every 10 minutes
   },
-  BULK_EXPORT_PDF_CONVERT_CRON_SCHEDULE: {
-    ns: 'crowi',
-    key: 'app:pageBulkExportPdfConvertCronSchedule',
-    type: ValueType.STRING,
-    default: '* * * * *', // every 1 minute
-  },
   BULK_EXPORT_PARALLEL_EXEC_LIMIT: {
     ns: 'crowi',
     key: 'app:pageBulkExportParallelExecLimit',