Merge pull request #8426 from weseek/feat/78028-135784-bulk-export-page-to-local-fs-in-zip

Feat/78028 135784 bulk export page to local fs in zip

apps/app/package.json

@@ -68,8 +68,8 @@
     "@elastic/elasticsearch8": "npm:@elastic/elasticsearch@^8.7.0",
     "@godaddy/terminus": "^4.9.0",
     "@google-cloud/storage": "^5.8.5",
-    "@growi/custom-icons": "link:../../packages/custom-icons",
     "@growi/core": "link:../../packages/core",
+    "@growi/custom-icons": "link:../../packages/custom-icons",
     "@growi/pluginkit": "link:../../packages/pluginkit",
     "@growi/preset-templates": "link:../../packages/preset-templates",
     "@growi/preset-themes": "link:../../packages/preset-themes",
@@ -226,12 +226,13 @@
     "@next/bundle-analyzer": "^13.2.3",
     "@swc-node/jest": "^1.6.2",
     "@swc/jest": "^0.2.24",
+    "@types/archiver": "^6.0.2",
     "@types/express": "^4.17.11",
     "@types/jest": "^29.5.2",
     "@types/react-scroll": "^1.8.4",
     "@types/throttle-debounce": "^5.0.1",
-    "@types/url-join": "^4.0.2",
     "@types/unzip-stream": "^0.3.4",
+    "@types/url-join": "^4.0.2",
     "@vitest/coverage-v8": "^0.34.6",
     "autoprefixer": "^9.0.0",
     "babel-loader": "^8.2.5",

apps/app/src/features/page-bulk-export/client/components/PageBulkExportSelectModal.tsx

@@ -1,23 +1,33 @@
 import { useTranslation } from 'next-i18next';
 import { Modal, ModalHeader, ModalBody } from 'reactstrap';
 
-import { toastSuccess } from '~/client/util/toastr';
+import { apiv3Post } from '~/client/util/apiv3-client';
+import { toastError, toastSuccess } from '~/client/util/toastr';
 import { usePageBulkExportSelectModal } from '~/features/page-bulk-export/client/stores/modal';
+import { PageBulkExportFormat } from '~/features/page-bulk-export/interfaces/page-bulk-export';
+import { useCurrentPagePath } from '~/stores/page';
 
 const PageBulkExportSelectModal = (): JSX.Element => {
   const { t } = useTranslation();
   const { data: status, close } = usePageBulkExportSelectModal();
+  const { data: currentPagePath } = useCurrentPagePath();
 
-  const startBulkExport = () => {
+  const startBulkExport = async(format: PageBulkExportFormat) => {
+    try {
+      await apiv3Post('/page-bulk-export', { path: currentPagePath, format });
+      toastSuccess(t('page_export.bulk_export_started'));
+    }
+    catch (e) {
+      toastError(t('page_export.failed_to_export'));
+    }
     close();
-    toastSuccess(t('page_export.bulk_export_started'));
   };
 
   return (
     <>
       {status != null && (
         <Modal isOpen={status.isOpened} toggle={close}>
-          <ModalHeader tag="h4" toggle={close} className="bg-primary text-light">
+          <ModalHeader tag="h5" toggle={close} className="bg-primary text-light">
             {t('page_export.bulk_export')}
           </ModalHeader>
           <ModalBody>
@@ -28,8 +38,10 @@ const PageBulkExportSelectModal = (): JSX.Element => {
               </small>
             </div>
             <div className="d-flex justify-content-center mt-2">
-              <button className="btn btn-primary" type="button" onClick={startBulkExport}>{t('page_export.markdown')}</button>
-              <button className="btn btn-primary ml-2" type="button" onClick={startBulkExport}>PDF</button>
+              <button className="btn btn-primary" type="button" onClick={() => startBulkExport(PageBulkExportFormat.markdown)}>
+                {t('page_export.markdown')}
+              </button>
+              <button className="btn btn-primary ms-2" type="button" onClick={() => startBulkExport(PageBulkExportFormat.pdf)}>PDF</button>
             </div>
           </ModalBody>
         </Modal>

apps/app/src/features/page-bulk-export/interfaces/page-bulk-export.ts

@@ -7,7 +7,7 @@ export const PageBulkExportFormat = {
   pdf: 'pdf',
 } as const;
 
-type PageBulkExportFormat = typeof PageBulkExportFormat[keyof typeof PageBulkExportFormat]
+export type PageBulkExportFormat = typeof PageBulkExportFormat[keyof typeof PageBulkExportFormat]
 
 export interface IPageBulkExportJob {
   user: Ref<IUser>, // user that started export job

apps/app/src/features/page-bulk-export/server/routes/apiv3/page-bulk-export.ts

@@ -0,0 +1,51 @@
+import { ErrorV3 } from '@growi/core/dist/models';
+import { Router, Request } from 'express';
+import { body, validationResult } from 'express-validator';
+
+import Crowi from '~/server/crowi';
+import { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response';
+import loggerFactory from '~/utils/logger';
+
+import { pageBulkExportService } from '../../service/page-bulk-export';
+
+const logger = loggerFactory('growi:routes:apiv3:page-bulk-export');
+
+const router = Router();
+
+interface AuthorizedRequest extends Request {
+  user?: any
+}
+
+module.exports = (crowi: Crowi): Router => {
+  const loginRequiredStrictly = require('~/server/middlewares/login-required')(crowi);
+
+  const validators = {
+    pageBulkExport: [
+      body('path').exists({ checkFalsy: true }).isString(),
+      body('format').exists({ checkFalsy: true }).isString(),
+    ],
+  };
+
+  router.post('/', loginRequiredStrictly, validators.pageBulkExport, async(req: AuthorizedRequest, res: ApiV3Response) => {
+    const errors = validationResult(req);
+    if (!errors.isEmpty()) {
+      return res.status(400).json({ errors: errors.array() });
+    }
+
+    const { path, format } = req.body;
+
+    try {
+      // temporal await, remove it after multi-part upload is implemented in https://redmine.weseek.co.jp/issues/78038
+      await pageBulkExportService?.bulkExportWithBasePagePath(path);
+
+      return res.apiv3({}, 204);
+    }
+    catch (err) {
+      logger.error(err);
+      return res.apiv3Err(new ErrorV3('Error occurred in exporting page tree'));
+    }
+  });
+
+  return router;
+
+};

apps/app/src/features/page-bulk-export/server/service/page-bulk-export.ts

@@ -0,0 +1,106 @@
+import fs from 'fs';
+import path from 'path';
+import { Writable } from 'stream';
+
+import { type IPage, isPopulated } from '@growi/core';
+import { normalizePath } from '@growi/core/dist/utils/path-utils';
+import archiver, { Archiver } from 'archiver';
+import mongoose from 'mongoose';
+
+import { PageModel, PageDocument } from '~/server/models/page';
+import loggerFactory from '~/utils/logger';
+
+
+const logger = loggerFactory('growi:services:PageBulkExportService');
+
+const streamToPromise = require('stream-to-promise');
+
+class PageBulkExportService {
+
+  crowi: any;
+
+  constructor(crowi) {
+    this.crowi = crowi;
+  }
+
+  getPageReadableStream(basePagePath: string) {
+    const Page = mongoose.model<IPage, PageModel>('Page');
+    const { PageQueryBuilder } = Page;
+
+    const builder = new PageQueryBuilder(Page.find())
+      .addConditionToListOnlyDescendants(basePagePath);
+
+    return builder
+      .query
+      .populate('revision')
+      .lean()
+      .cursor({ batchSize: 100 }); // convert to stream
+  }
+
+  setUpZipArchiver(): Archiver {
+    const timeStamp = (new Date()).getTime();
+    const zipFilePath = path.join(__dirname, `${timeStamp}.md.zip`);
+
+    const archive = archiver('zip', {
+      zlib: { level: 9 }, // maximum compression
+    });
+
+    // good practice to catch warnings (ie stat failures and other non-blocking errors)
+    archive.on('warning', (err) => {
+      if (err.code === 'ENOENT') logger.error(err);
+      else throw err;
+    });
+    // good practice to catch this error explicitly
+    archive.on('error', (err) => { throw err });
+
+    // pipe archive data to the file
+    const output = fs.createWriteStream(zipFilePath);
+    archive.pipe(output);
+
+    return archive;
+  }
+
+  async bulkExportWithBasePagePath(basePagePath: string): Promise<void> {
+    // get pages with descendants as stream
+    const pageReadableStream = this.getPageReadableStream(basePagePath);
+
+    const archive = this.setUpZipArchiver();
+
+    const pagesWritable = new Writable({
+      objectMode: true,
+      async write(page: PageDocument, encoding, callback) {
+        try {
+          const revision = page.revision;
+
+          if (revision != null && isPopulated(revision)) {
+            const markdownBody = revision.body;
+            // write to zip
+            const pathNormalized = normalizePath(page.path);
+            archive.append(markdownBody, { name: `${pathNormalized}.md` });
+          }
+        }
+        catch (err) {
+          logger.error(err);
+          throw Error('Failed to export page tree');
+        }
+
+        callback();
+      },
+      final(callback) {
+        archive.finalize();
+        callback();
+      },
+    });
+
+    pageReadableStream.pipe(pagesWritable);
+
+    await streamToPromise(archive);
+  }
+
+}
+
+// eslint-disable-next-line import/no-mutable-exports
+export let pageBulkExportService: PageBulkExportService | undefined; // singleton instance
+export default function instanciate(crowi): void {
+  pageBulkExportService = new PageBulkExportService(crowi);
+}

apps/app/src/server/crowi/index.js

@@ -12,6 +12,7 @@ import pkg from '^/package.json';
 
 import { KeycloakUserGroupSyncService } from '~/features/external-user-group/server/service/keycloak-user-group-sync';
 import { LdapUserGroupSyncService } from '~/features/external-user-group/server/service/ldap-user-group-sync';
+import instanciatePageBulkExportService from '~/features/page-bulk-export/server/service/page-bulk-export';
 import QuestionnaireService from '~/features/questionnaire/server/service/questionnaire';
 import QuestionnaireCronService from '~/features/questionnaire/server/service/questionnaire-cron';
 import CdnResourcesService from '~/services/cdn-resources-service';
@@ -153,6 +154,7 @@ Crowi.prototype.init = async function() {
     this.setupUserGroupService(),
     this.setupExport(),
     this.setupImport(),
+    this.setupPageBulkExportService(),
     this.setupGrowiPluginService(),
     this.setupPageService(),
     this.setupInAppNotificationService(),
@@ -693,6 +695,10 @@ Crowi.prototype.setupExport = async function() {
   instanciateExportService(this);
 };
 
+Crowi.prototype.setupPageBulkExportService = async function() {
+  instanciatePageBulkExportService(this);
+};
+
 Crowi.prototype.setupImport = async function() {
   const ImportService = require('../service/import');
   if (this.importService == null) {

apps/app/src/server/models/page.ts

@@ -226,7 +226,7 @@ export class PageQueryBuilder {
   /**
    * generate the query to find the pages '{path}/*' (exclude '{path}' self).
    */
-  addConditionToListOnlyDescendants(path: string, option): PageQueryBuilder {
+  addConditionToListOnlyDescendants(path: string): PageQueryBuilder {
     // exclude the target page
     this.query = this.query.and({ path: { $ne: path } });
 

apps/app/src/server/routes/apiv3/index.js

@@ -117,6 +117,7 @@ module.exports = (crowi, app) => {
   router.use('/bookmark-folder', require('./bookmark-folder')(crowi));
   router.use('/questionnaire', require('~/features/questionnaire/server/routes/apiv3/questionnaire')(crowi));
   router.use('/templates', require('~/features/templates/server/routes/apiv3')(crowi));
+  router.use('/page-bulk-export', require('~/features/page-bulk-export/server/routes/apiv3/page-bulk-export')(crowi));
 
   router.use('/me', require('./me')(crowi));
 

apps/app/src/server/service/export.ts

@@ -2,6 +2,8 @@ import fs from 'fs';
 import path from 'path';
 import { Readable, Transform } from 'stream';
 
+import archiver from 'archiver';
+
 import { toArrayIfNot } from '~/utils/array-utils';
 import loggerFactory from '~/utils/logger';
 
@@ -14,9 +16,8 @@ import GrowiBridgeService from './growi-bridge';
 import { ZipFileStat } from './interfaces/export';
 
 
-const logger = loggerFactory('growi:services:ExportService'); // eslint-disable-line no-unused-vars
+const logger = loggerFactory('growi:services:ExportService');
 
-const archiver = require('archiver');
 const mongoose = require('mongoose');
 const streamToPromise = require('stream-to-promise');
 

apps/app/src/server/service/page/delete-completely-user-home-by-system.ts

@@ -77,7 +77,7 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string,
     // Find descendant pages with system deletion condition
     const builder = new PageQueryBuilder(Page.find(), true)
       .addConditionForSystemDeletion()
-      .addConditionToListOnlyDescendants(userHomepage.path, {});
+      .addConditionToListOnlyDescendants(userHomepage.path);
 
     // Stream processing to delete descendant pages
     // ────────┤ start │─────────

yarn.lock

@@ -3752,6 +3752,13 @@
   dependencies:
     tslib "2.1.0"
 
+"@types/archiver@^6.0.2":
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2"
+  integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==
+  dependencies:
+    "@types/readdir-glob" "*"
+
 "@types/argparse@1.0.38":
   version "1.0.38"
   resolved "https://registry.yarnpkg.com/@types/argparse/-/argparse-1.0.38.tgz#a81fd8606d481f873a3800c6ebae4f1d768a56a9"
@@ -4190,6 +4197,13 @@
     "@types/scheduler" "*"
     csstype "^3.0.2"
 
+"@types/readdir-glob@*":
+  version "1.1.5"
+  resolved "https://registry.yarnpkg.com/@types/readdir-glob/-/readdir-glob-1.1.5.tgz#21a4a98898fc606cb568ad815f2a0eedc24d412a"
+  integrity sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/retry@^0.12.0":
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"