refactor export API and ExportService

src/client/js/components/Admin/Export/ExportZipFormModal.jsx

@@ -54,13 +54,15 @@ class ExportZipFormModal extends React.Component {
 
     try {
       // TODO: use appContainer.apiv3.post
-      const { zipFileStat } = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
+      const result = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
       // TODO: toastSuccess, toastError
-      this.props.onZipFileStatAdd(zipFileStat);
-      this.props.onClose();
+
+      if (!result.ok) {
+        throw new Error('Error occured.');
+      }
 
       // TODO: toastSuccess, toastError
-      toastr.success(undefined, `Generated ${zipFileStat.fileName}`, {
+      toastr.success(undefined, 'Export process has requested.', {
         closeButton: true,
         progressBar: true,
         newestOnTop: false,
@@ -69,6 +71,8 @@ class ExportZipFormModal extends React.Component {
         timeOut: '1200',
         extendedTimeOut: '150',
       });
+      this.props.onClose();
+
     }
     catch (err) {
       // TODO: toastSuccess, toastError

src/server/routes/apiv3/export.js

@@ -1,7 +1,6 @@
 const loggerFactory = require('@alias/logger');
 
 const logger = loggerFactory('growi:routes:apiv3:export');
-const path = require('path');
 const fs = require('fs');
 
 const express = require('express');
@@ -74,24 +73,11 @@ module.exports = (crowi) => {
       // get model for collection
       const models = collections.map(collectionName => growiBridgeService.getModelFromCollectionName(collectionName));
 
-      const [metaJson, jsonFiles] = await Promise.all([
-        exportService.createMetaJson(),
-        exportService.exportMultipleCollectionsToJsons(models),
-      ]);
-
-      // zip json
-      const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
-      // add meta.json in zip
-      configs.push({ from: metaJson, as: path.basename(metaJson) });
-      // exec zip
-      const zipFile = await exportService.zipFiles(configs);
-      // get stats for the zip file
-      const zipFileStat = await growiBridgeService.parseZipFile(zipFile);
+      exportService.exportCollectionsToZippedJson(models);
 
       // TODO: use res.apiv3
       return res.status(200).json({
         ok: true,
-        zipFileStat,
       });
     }
     catch (err) {

src/server/service/export.js

@@ -69,11 +69,12 @@ class ExportService {
    * @param {function} [getLogText] (n, total) => { ... }
    * @return {string} path to the exported json file
    */
-  async export(writeStream, readStream, total, getLogText) {
+  generateTransformStream(total, getLogText) {
     let n = 0;
 
     const logProgress = this.logProgress.bind(this);
-    const transformStream = new Transform({
+    return new Transform({
+      // highWaterMark: 16 * 1024 * 1024,
       transform(chunk, encoding, callback) {
         // write beginning brace
         if (n === 0) {
@@ -91,20 +92,12 @@ class ExportService {
 
         callback();
       },
-      flush(callback) {
+      final(callback) {
         // write ending brace
         this.push(']');
         callback();
       },
     });
-
-    readStream
-      .pipe(transformStream)
-      .pipe(writeStream);
-
-    await streamToPromise(readStream);
-
-    return writeStream.path;
   }
 
   /**
@@ -116,29 +109,58 @@ class ExportService {
    */
   async exportCollectionToJson(Model) {
     const { collectionName } = Model.collection;
+
+    // get native Cursor instance
+    //  cz: Mongoose cursor might cause memory leak
+    const nativeCursor = Model.collection.find();
+    const readStream = nativeCursor
+      .snapshot()
+      // .batchSize(10)
+      .stream({
+        // highWaterMark: 16 * 1024 * 1024,
+        transform: JSON.stringify,
+      });
+
+    // get TransformStream
+    const total = await Model.countDocuments();
+    const getLogText = (n, total) => `${collectionName}: ${n}/${total} written`;
+    const transformStream = this.generateTransformStream(total, getLogText);
+
+    // create WritableStream
     const jsonFileToWrite = path.join(this.baseDir, `${collectionName}.json`);
     const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() });
-    const readStream = Model.find().snapshot().stream({ transform: JSON.stringify });
-    const total = await Model.countDocuments();
-    // const getLogText = (n, total) => `${collectionName}: ${n}/${total} written`;
-    const getLogText = undefined;
 
-    const jsonFileWritten = await this.export(writeStream, readStream, total, getLogText);
+    readStream
+      .pipe(transformStream)
+      .pipe(writeStream);
+
+    await streamToPromise(readStream);
 
-    return jsonFileWritten;
+    return writeStream.path;
   }
 
   /**
-   * export multiple collections
+   * export multiple Collections into json and Zip
    *
    * @memberOf ExportService
    * @param {Array.<object>} models array of instances of mongoose model
    * @return {Array.<string>} paths to json files created
    */
-  async exportMultipleCollectionsToJsons(models) {
-    const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));
+  async exportCollectionsToZippedJson(models) {
+    const metaJson = await this.createMetaJson();
+
+    const promisesForModels = models.map(Model => this.exportCollectionToJson(Model));
+    const jsonFiles = await Promise.all(promisesForModels);
+
+    // zip json
+    const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
+    // add meta.json in zip
+    configs.push({ from: metaJson, as: path.basename(metaJson) });
+    // exec zip
+    const zipFile = await this.zipFiles(configs);
 
-    return jsonFiles;
+    // get stats for the zip file
+    return this.growiBridgeService.parseZipFile(zipFile);
   }
 
   /**