Merge pull request #1270 from weseek/imprv/export-services

Imprv/export services

src/client/js/app.jsx

@@ -251,7 +251,7 @@ if (adminUserGroupPageElem != null) {
 const adminExportPageElem = document.getElementById('admin-export-page');
 if (adminExportPageElem != null) {
   ReactDOM.render(
-    <Provider inject={[]}>
+    <Provider inject={[appContainer, websocketContainer]}>
       <I18nextProvider i18n={i18n}>
         <ExportPage
           crowi={appContainer}

src/client/js/components/Admin/Export/ExportPage.jsx

@@ -24,6 +24,7 @@ class ExportPage extends React.Component {
       progressList: [],
       isExportModalOpen: false,
       isExporting: false,
+      isZipping: false,
       isExported: false,
     };
 
@@ -59,8 +60,17 @@ class ExportPage extends React.Component {
 
     // websocket event
     socket.on('admin:onProgressForExport', ({ currentCount, totalCount, progressList }) => {
-      const isExporting = currentCount !== totalCount;
-      this.setState({ isExporting, progressList });
+      this.setState({
+        isExporting: true,
+        progressList,
+      });
+    });
+
+    // websocket event
+    socket.on('admin:onStartZippingForExport', () => {
+      this.setState({
+        isZipping: true,
+      });
     });
 
     // websocket event
@@ -69,6 +79,7 @@ class ExportPage extends React.Component {
 
       this.setState({
         isExporting: false,
+        isZipping: false,
         isExported: true,
         zipFileStats,
       });
@@ -137,20 +148,18 @@ class ExportPage extends React.Component {
   }
 
   /**
-   * @params {object} export status data
+   * event handler invoked when export process was requested successfully
    */
-  exportingRequestedHandler(status) {
-    const { zipFileStats, isExporting, progressList } = status;
-    this.setState({ zipFileStats, isExporting, progressList });
+  exportingRequestedHandler() {
   }
 
-  renderProgressBars() {
+  renderProgressBarsForCollections() {
     const cols = this.state.progressList.map((progressData) => {
       const { collectionName, currentCount, totalCount } = progressData;
       return (
         <div className="col-md-6" key={collectionName}>
           <ExportingProgressBar
-            collectionName={collectionName}
+            header={collectionName}
             currentCount={currentCount}
             totalCount={totalCount}
           />
@@ -161,21 +170,47 @@ class ExportPage extends React.Component {
     return <div className="row px-3">{cols}</div>;
   }
 
+  renderProgressBarForZipping() {
+    const { isZipping, isExported } = this.state;
+    const showZippingBar = isZipping || isExported;
+
+    if (!showZippingBar) {
+      return <></>;
+    }
+
+    return (
+      <div className="row px-3">
+        <div className="col-md-12" key="progressBarForZipping">
+          <ExportingProgressBar
+            header="Zip Files"
+            currentCount={1}
+            totalCount={1}
+            isInProgress={isZipping}
+          />
+        </div>
+      </div>
+    );
+  }
+
   render() {
     const { t } = this.props;
+    const { isExporting, isExported, progressList } = this.state;
 
-    const showExportingData = (this.state.isExported || this.state.isExporting) && (this.state.progressList != null);
+    const showExportingData = (isExported || isExporting) && (progressList != null);
 
     return (
       <Fragment>
         <h2>{t('Export Data')}</h2>
 
-        <button type="button" className="btn btn-default" onClick={this.openExportModal}>{t('export_management.create_new_exported_data')}</button>
+        <button type="button" className="btn btn-default" disabled={isExporting} onClick={this.openExportModal}>
+          {t('export_management.create_new_exported_data')}
+        </button>
 
         { showExportingData && (
           <div className="mt-5">
             <h3>{t('export_management.exporting_data_list')}</h3>
-            { this.renderProgressBars() }
+            { this.renderProgressBarsForCollections() }
+            { this.renderProgressBarForZipping() }
           </div>
         ) }
 

src/client/js/components/Admin/Export/ExportZipFormModal.jsx

@@ -84,9 +84,11 @@ class ExportZipFormModal extends React.Component {
         extendedTimeOut: '150',
       });
 
-      this.props.onExportingRequested(result.status);
+      this.props.onExportingRequested();
       this.props.onClose();
 
+      this.setState({ selectedCollections: new Set() });
+
     }
     catch (err) {
       // TODO: toastSuccess, toastError

src/client/js/components/Admin/Export/ExportingProgressBar.jsx

@@ -6,15 +6,19 @@ class ExportingProgressBar extends React.Component {
 
 
   render() {
-    const { collectionName, currentCount, totalCount } = this.props;
+    const {
+      header, currentCount, totalCount, isInProgress,
+    } = this.props;
 
     const percentage = currentCount / totalCount * 100;
-    const isActive = currentCount !== totalCount;
+    const isActive = (isInProgress != null)
+      ? isInProgress //                         apply props.isInProgress if set
+      : (currentCount !== totalCount); //       otherwise, set true when currentCount does not equal totalCount
 
     return (
       <>
-        <h5>
-          {collectionName}
+        <h5 className="my-1">
+          {header}
           <div className="pull-right">{currentCount} / {totalCount}</div>
         </h5>
         <div className="progress progress-sm">
@@ -32,9 +36,10 @@ class ExportingProgressBar extends React.Component {
 }
 
 ExportingProgressBar.propTypes = {
-  collectionName: PropTypes.string.isRequired,
+  header: PropTypes.string.isRequired,
   currentCount: PropTypes.number.isRequired,
   totalCount: PropTypes.number.isRequired,
+  isInProgress: PropTypes.bool,
 };
 
 export default withTranslation()(ExportingProgressBar);

src/server/routes/apiv3/export.js

@@ -42,7 +42,7 @@ module.exports = (crowi) => {
   const adminRequired = require('../../middleware/admin-required')(crowi);
   const csrf = require('../../middleware/csrf')(crowi);
 
-  const { growiBridgeService, exportService } = crowi;
+  const { exportService } = crowi;
 
   this.adminEvent = crowi.event('admin');
 
@@ -50,6 +50,9 @@ module.exports = (crowi) => {
   this.adminEvent.on('onProgressForExport', (data) => {
     crowi.getIo().sockets.emit('admin:onProgressForExport', data);
   });
+  this.adminEvent.on('onStartZippingForExport', (data) => {
+    crowi.getIo().sockets.emit('admin:onStartZippingForExport', data);
+  });
   this.adminEvent.on('onTerminateForExport', (data) => {
     crowi.getIo().sockets.emit('admin:onTerminateForExport', data);
   });
@@ -103,17 +106,12 @@ module.exports = (crowi) => {
     // TODO: add express validator
     try {
       const { collections } = req.body;
-      // get model for collection
-      const models = collections.map(collectionName => growiBridgeService.getModelFromCollectionName(collectionName));
-
-      exportService.export(models);
 
-      const status = await exportService.getStatus();
+      exportService.export(collections);
 
       // TODO: use res.apiv3
       return res.status(200).json({
         ok: true,
-        status,
       });
     }
     catch (err) {

src/server/routes/apiv3/mongo.js

@@ -3,6 +3,7 @@ const loggerFactory = require('@alias/logger');
 const logger = loggerFactory('growi:routes:apiv3:mongo'); // eslint-disable-line no-unused-vars
 
 const express = require('express');
+const mongoose = require('mongoose');
 
 const router = express.Router();
 
@@ -33,12 +34,12 @@ module.exports = (crowi) => {
    *                      type: string
    */
   router.get('/collections', async(req, res) => {
-    const collections = Object.values(crowi.models).map(model => model.collection.name);
+    const collections = Object.keys(mongoose.connection.collections);
 
     // TODO: use res.apiv3
     return res.json({
       ok: true,
-      collections: [...new Set(collections)], // remove duplicates
+      collections,
     });
   });
 

src/server/service/export.js

@@ -1,9 +1,13 @@
 const logger = require('@alias/logger')('growi:services:ExportService'); // eslint-disable-line no-unused-vars
+
 const fs = require('fs');
 const path = require('path');
+const mongoose = require('mongoose');
 const { Transform } = require('stream');
 const streamToPromise = require('stream-to-promise');
 const archiver = require('archiver');
+
+
 const toArrayIfNot = require('../../lib/util/toArrayIfNot');
 
 
@@ -26,10 +30,10 @@ class ExportingStatus {
     this.progressMap = {};
   }
 
-  async init(models) {
-    const promisesForCreatingInstance = models.map(async(Model) => {
-      const collectionName = Model.collection.name;
-      const totalCount = await Model.countDocuments();
+  async init(collections) {
+    const promisesForCreatingInstance = collections.map(async(collectionName) => {
+      const collection = mongoose.connection.collection(collectionName);
+      const totalCount = await collection.count();
       return new ExportingProgress(collectionName, totalCount);
     });
     this.progressList = await Promise.all(promisesForCreatingInstance);
@@ -177,15 +181,13 @@ class ExportService {
    * dump a mongodb collection into json
    *
    * @memberOf ExportService
-   * @param {object} Model instance of mongoose model
+   * @param {string} collectionName collection name
    * @return {string} path to zip file
    */
-  async exportCollectionToJson(Model) {
-    const collectionName = Model.collection.name;
+  async exportCollectionToJson(collectionName) {
+    const collection = mongoose.connection.collection(collectionName);
 
-    // get native Cursor instance
-    //  cz: Mongoose cursor might cause memory leak
-    const nativeCursor = Model.collection.find();
+    const nativeCursor = collection.find();
     const readStream = nativeCursor
       .snapshot()
       .stream({ transform: JSON.stringify });
@@ -215,14 +217,17 @@ class ExportService {
    * export multiple Collections into json and Zip
    *
    * @memberOf ExportService
-   * @param {Array.<object>} models array of instances of mongoose model
+   * @param {Array.<string>} collections array of collection name
    * @return {Array.<string>} paths to json files created
    */
-  async exportCollectionsToZippedJson(models) {
+  async exportCollectionsToZippedJson(collections) {
     const metaJson = await this.createMetaJson();
 
-    const promisesForModels = models.map(Model => this.exportCollectionToJson(Model));
-    const jsonFiles = await Promise.all(promisesForModels);
+    const promises = collections.map(collectionName => this.exportCollectionToJson(collectionName));
+    const jsonFiles = await Promise.all(promises);
+
+    // send terminate event
+    this.emitStartZippingEvent();
 
     // zip json
     const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
@@ -240,16 +245,16 @@ class ExportService {
     // TODO: remove broken zip file
   }
 
-  async export(models) {
+  async export(collections) {
     if (this.currentExportingStatus != null) {
       throw new Error('There is an exporting process running.');
     }
 
     this.currentExportingStatus = new ExportingStatus();
-    await this.currentExportingStatus.init(models);
+    await this.currentExportingStatus.init(collections);
 
     try {
-      await this.exportCollectionsToZippedJson(models);
+      await this.exportCollectionsToZippedJson(collections);
     }
     finally {
       this.currentExportingStatus = null;
@@ -299,6 +304,13 @@ class ExportService {
     this.adminEvent.emit('onProgressForExport', data);
   }
 
+  /**
+   * emit start zipping event
+   */
+  emitStartZippingEvent() {
+    this.adminEvent.emit('onStartZippingForExport', {});
+  }
+
   /**
    * emit terminate event
    * @param {object} zipFileStat added zip file status data