bulk export page to local fs in zip format

apps/app/src/features/page-bulk-export/client/components/PageBulkExportSelectModal.tsx

@@ -1,23 +1,33 @@
 import { useTranslation } from 'next-i18next';
 import { Modal, ModalHeader, ModalBody } from 'reactstrap';
 
-import { toastSuccess } from '~/client/util/toastr';
+import { apiv3Post } from '~/client/util/apiv3-client';
+import { toastError, toastSuccess } from '~/client/util/toastr';
 import { usePageBulkExportSelectModal } from '~/features/page-bulk-export/client/stores/modal';
+import { PageBulkExportFormat } from '~/features/page-bulk-export/interfaces/page-bulk-export';
+import { useCurrentPagePath } from '~/stores/page';
 
 const PageBulkExportSelectModal = (): JSX.Element => {
   const { t } = useTranslation();
   const { data: status, close } = usePageBulkExportSelectModal();
+  const { data: currentPagePath } = useCurrentPagePath();
 
-  const startBulkExport = () => {
+  const startBulkExport = async(format: PageBulkExportFormat) => {
+    try {
+      await apiv3Post('/page-bulk-export', { path: currentPagePath, format });
+      toastSuccess(t('page_export.bulk_export_started'));
+    }
+    catch (e) {
+      toastError(t('page_export.failed_to_export'));
+    }
     close();
-    toastSuccess(t('page_export.bulk_export_started'));
   };
 
   return (
     <>
       {status != null && (
         <Modal isOpen={status.isOpened} toggle={close}>
-          <ModalHeader tag="h4" toggle={close} className="bg-primary text-light">
+          <ModalHeader tag="h5" toggle={close} className="bg-primary text-light">
             {t('page_export.bulk_export')}
           </ModalHeader>
           <ModalBody>
@@ -28,8 +38,10 @@ const PageBulkExportSelectModal = (): JSX.Element => {
               </small>
             </div>
             <div className="d-flex justify-content-center mt-2">
-              <button className="btn btn-primary" type="button" onClick={startBulkExport}>{t('page_export.markdown')}</button>
-              <button className="btn btn-primary ml-2" type="button" onClick={startBulkExport}>PDF</button>
+              <button className="btn btn-primary" type="button" onClick={() => startBulkExport(PageBulkExportFormat.markdown)}>
+                {t('page_export.markdown')}
+              </button>
+              <button className="btn btn-primary ms-2" type="button" onClick={() => startBulkExport(PageBulkExportFormat.pdf)}>PDF</button>
             </div>
           </ModalBody>
         </Modal>

apps/app/src/features/page-bulk-export/interfaces/page-bulk-export.ts

@@ -7,7 +7,7 @@ export const PageBulkExportFormat = {
   pdf: 'pdf',
 } as const;
 
-type PageBulkExportFormat = typeof PageBulkExportFormat[keyof typeof PageBulkExportFormat]
+export type PageBulkExportFormat = typeof PageBulkExportFormat[keyof typeof PageBulkExportFormat]
 
 export interface IPageBulkExportJob {
   user: Ref<IUser>, // user that started export job

apps/app/src/features/page-bulk-export/server/routes/apiv3/page-bulk-export.ts

@@ -1,12 +1,14 @@
 import { ErrorV3 } from '@growi/core/dist/models';
 import { Router, Request } from 'express';
-import { param, validationResult } from 'express-validator';
+import { body, validationResult } from 'express-validator';
 
 import Crowi from '~/server/crowi';
 import { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response';
 import loggerFactory from '~/utils/logger';
 
-const logger = loggerFactory('growi:routes:apiv3:external-user-group');
+import { pageBulkExportService } from '../../service/page-bulk-export';
+
+const logger = loggerFactory('growi:routes:apiv3:page-bulk-export');
 
 const router = Router();
 
@@ -19,8 +21,8 @@ module.exports = (crowi: Crowi): Router => {
 
   const validators = {
     pageBulkExport: [
-      param('path').exists({ checkFalsy: true }).isString(),
-      param('format').exists({ checkFalsy: true }).isString(),
+      body('path').exists({ checkFalsy: true }).isString(),
+      body('format').exists({ checkFalsy: true }).isString(),
     ],
   };
 
@@ -30,17 +32,16 @@ module.exports = (crowi: Crowi): Router => {
       return res.status(400).json({ errors: errors.array() });
     }
 
-    const { path, format } = req.params;
+    const { path, format } = req.body;
 
     try {
-      await crowi.exportService?.bulkExportWithBasePagePath(path);
+      await pageBulkExportService?.bulkExportWithBasePagePath(path);
 
-      return res.apiv3(204);
+      return res.apiv3({}, 204);
     }
     catch (err) {
-      const msg = 'Error occurred in fetching external user group list';
-      logger.error('Error', err);
-      return res.apiv3Err(new ErrorV3(msg));
+      logger.error(err);
+      return res.apiv3Err(new ErrorV3('Error occurred in exporting page tree'));
     }
   });
 

apps/app/src/features/page-bulk-export/server/service/page-bulk-export.ts

@@ -0,0 +1,105 @@
+import fs from 'fs';
+import path from 'path';
+import { Writable } from 'stream';
+
+import { isPopulated } from '@growi/core';
+import { normalizePath } from '@growi/core/dist/utils/path-utils';
+import archiver, { Archiver } from 'archiver';
+
+import { PageModel, PageDocument } from '~/server/models/page';
+import loggerFactory from '~/utils/logger';
+
+
+const logger = loggerFactory('growi:services:PageBulkExportService'); // eslint-disable-line no-unused-vars
+
+const streamToPromise = require('stream-to-promise');
+
+class PageBulkExportService {
+
+  crowi: any;
+
+  constructor(crowi) {
+    this.crowi = crowi;
+  }
+
+  getPageReadableStream(basePagePath: string) {
+    const Page = this.crowi.model('Page') as PageModel;
+    const { PageQueryBuilder } = Page;
+
+    const builder = new PageQueryBuilder(Page.find())
+      .addConditionToListOnlyDescendants(basePagePath);
+
+    return builder
+      .query
+      .populate('revision')
+      .lean()
+      .cursor({ batchSize: 100 }); // convert to stream
+  }
+
+  setUpZipArchiver(): Archiver {
+    const timeStamp = (new Date()).getTime();
+    const zipFilePath = path.join(__dirname, `${timeStamp}.md.zip`);
+
+    const archive = archiver('zip', {
+      zlib: { level: 9 }, // maximum compression
+    });
+
+    // good practice to catch warnings (ie stat failures and other non-blocking errors)
+    archive.on('warning', (err) => {
+      if (err.code === 'ENOENT') logger.error(err);
+      else throw err;
+    });
+    // good practice to catch this error explicitly
+    archive.on('error', (err) => { throw err });
+
+    // pipe archive data to the file
+    const output = fs.createWriteStream(zipFilePath);
+    archive.pipe(output);
+
+    return archive;
+  }
+
+  async bulkExportWithBasePagePath(basePagePath: string): Promise<void> {
+    // get pages with descendants as stream
+    const pageReadableStream = this.getPageReadableStream(basePagePath);
+
+    const archive = this.setUpZipArchiver();
+
+    const pagesWritable = new Writable({
+      objectMode: true,
+      async write(page: PageDocument, encoding, callback) {
+        try {
+          const revision = page.revision;
+
+          if (revision != null && isPopulated(revision)) {
+            const markdownBody = revision.body;
+            // write to zip
+            const pathNormalized = normalizePath(page.path);
+            archive.append(markdownBody, { name: `${pathNormalized}.md` });
+          }
+        }
+        catch (err) {
+          logger.error(err);
+          throw Error('Failed to export page tree');
+        }
+
+        callback();
+      },
+      final(callback) {
+        archive.finalize();
+        callback();
+      },
+    });
+
+    pageReadableStream.pipe(pagesWritable);
+
+    await streamToPromise(archive);
+  }
+
+}
+
+// eslint-disable-next-line import/no-mutable-exports
+export let pageBulkExportService: PageBulkExportService | undefined; // singleton instance
+export default function instanciate(crowi): void {
+  pageBulkExportService = new PageBulkExportService(crowi);
+}

apps/app/src/server/crowi/index.js

@@ -12,6 +12,7 @@ import pkg from '^/package.json';
 
 import { KeycloakUserGroupSyncService } from '~/features/external-user-group/server/service/keycloak-user-group-sync';
 import { LdapUserGroupSyncService } from '~/features/external-user-group/server/service/ldap-user-group-sync';
+import instanciatePageBulkExportService from '~/features/page-bulk-export/server/service/page-bulk-export';
 import QuestionnaireService from '~/features/questionnaire/server/service/questionnaire';
 import QuestionnaireCronService from '~/features/questionnaire/server/service/questionnaire-cron';
 import CdnResourcesService from '~/services/cdn-resources-service';
@@ -153,6 +154,7 @@ Crowi.prototype.init = async function() {
     this.setupUserGroupService(),
     this.setupExport(),
     this.setupImport(),
+    this.setupPageBulkExportService(),
     this.setupGrowiPluginService(),
     this.setupPageService(),
     this.setupInAppNotificationService(),
@@ -693,6 +695,10 @@ Crowi.prototype.setupExport = async function() {
   instanciateExportService(this);
 };
 
+Crowi.prototype.setupPageBulkExportService = async function() {
+  instanciatePageBulkExportService(this);
+};
+
 Crowi.prototype.setupImport = async function() {
   const ImportService = require('../service/import');
   if (this.importService == null) {

apps/app/src/server/routes/apiv3/index.js

@@ -117,6 +117,7 @@ module.exports = (crowi, app) => {
   router.use('/bookmark-folder', require('./bookmark-folder')(crowi));
   router.use('/questionnaire', require('~/features/questionnaire/server/routes/apiv3/questionnaire')(crowi));
   router.use('/templates', require('~/features/templates/server/routes/apiv3')(crowi));
+  router.use('/page-bulk-export', require('~/features/page-bulk-export/server/routes/apiv3/page-bulk-export')(crowi));
 
   router.use('/me', require('./me')(crowi));
 

apps/app/src/server/service/export.ts

@@ -454,24 +454,21 @@ class ExportService {
         try {
           const revision = page.revision;
 
-          let markdownBody = 'This page does not have any content.';
           if (revision != null && isPopulated(revision)) {
-            markdownBody = revision.body;
+            const markdownBody = revision.body;
+            // write to zip
+            const pathNormalized = normalizePath(page.path);
+            archive.append(markdownBody, { name: `${pathNormalized}.md` });
           }
-
-          // write to zip
-          const pathNormalized = normalizePath(page.path);
-          archive.append(markdownBody, { name: `${pathNormalized}.md` });
         }
         catch (err) {
-          logger.error('Error occurred while converting data to readable: ', err);
+          logger.error(err);
           throw Error('だめ');
         }
 
         callback();
       },
       final(callback) {
-        // TODO: multi-part upload instead of calling finalize() 78070
         archive.finalize();
         callback();
       },