Merge branch 'master' into reactify-admin/external-account

CHANGES.md

@@ -2,8 +2,13 @@
 
 ## 3.5.17-RC
 
+* Feature: Upload to GCS (Google Cloud Storage)
+* Feature: Statistics API
+* Improvement: Export progress bar
+* Improvement: Reactify admin pages
 * Fix: Use HTTP PlantUML URL in default
     * Introduced by 3.5.12
+* Support: REPL with `console` npm scripts
 
 ## 3.5.16
 

package.json

@@ -32,6 +32,7 @@
     "clean:app": "rimraf -- public/js public/styles",
     "clean:report": "rimraf -- report",
     "clean": "npm-run-all -p clean:*",
+    "console": "env-cmd -f config/env.dev.js node --experimental-repl-await src/server/console.js",
     "heroku-postbuild": "sh bin/heroku/install-plugins.sh && npm run build:prod",
     "lint:js:fix": "eslint \"**/*.{js,jsx}\" --fix",
     "lint:js": "eslint \"**/*.{js,jsx}\"",

resource/locales/en-US/translation.json

@@ -784,10 +784,12 @@
   },
   "export_management": {
     "beta_warning": "This function is Beta.",
+    "exporting_data_list": "Exporting Data List",
     "exported_data_list": "Exported Data List",
     "export_collections": "Export Collections",
     "check_all": "Check All",
     "uncheck_all": "Uncheck All",
+    "desc_password_seed": "DO NOT FORGET to set current <code>PASSWORD_SEED</code> to your new GROWI system when restoring user data, or users will NOT be able to login with their password.<br><br><strong>HINT:</strong><br>The current <code>PASSWORD_SEED</code> will be stored in <code>meta.json</code> in exported ZIP.",
     "create_new_exported_data": "Create New Exported Data",
     "export": "Export",
     "cancel": "Cancel",

resource/locales/ja/translation.json

@@ -769,10 +769,12 @@
   },
   "export_management": {
     "beta_warning": "この機能はベータ版です",
+    "exporting_data_list": "エクスポート中のデータ",
     "exported_data_list": "エクスポートデータリスト",
     "export_collections": "コレクションのエクスポート",
     "check_all": "全てにチェックを付ける",
     "uncheck_all": "全てからチェックを外す",
+    "desc_password_seed": "ユーザーデータをバックアップ/リストアする場合、現在の <code>PASSWORD_SEED</code> を新しい GROWI システムにセットすることを忘れないでください。さもなくば、ユーザーがパスワードでログインできなくなります。<br><br><strong>ヒント:</strong><br>現在の <code>PASSWORD_SEED</code> は、エクスポートされる ZIP 中の <code>meta.json</code> に保存されます。",
     "create_new_exported_data": "エクスポートデータの新規作成",
     "export": "エクスポート",
     "cancel": "キャンセル",

src/client/js/app.jsx

@@ -236,7 +236,7 @@ if (adminUserGroupPageElem != null) {
   const isAclEnabled = adminUserGroupPageElem.getAttribute('data-isAclEnabled') === 'true';
 
   ReactDOM.render(
-    <Provider inject={[]}>
+    <Provider inject={[websocketContainer]}>
       <I18nextProvider i18n={i18n}>
         <UserGroupPage
           crowi={appContainer}
@@ -251,7 +251,7 @@ if (adminUserGroupPageElem != null) {
 const adminExportPageElem = document.getElementById('admin-export-page');
 if (adminExportPageElem != null) {
   ReactDOM.render(
-    <Provider inject={[]}>
+    <Provider inject={[appContainer, websocketContainer]}>
       <I18nextProvider i18n={i18n}>
         <ExportPage
           crowi={appContainer}

src/client/js/components/Admin/Export/ExportPage.jsx

@@ -3,12 +3,16 @@ import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 import * as toastr from 'toastr';
 
-import ExportZipFormModal from './ExportZipFormModal';
-import ZipFileTable from './ZipFileTable';
+
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
+import WebsocketContainer from '../../../services/WebsocketContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
+import ExportZipFormModal from './ExportZipFormModal';
+import ZipFileTable from './ZipFileTable';
+import ExportingProgressBar from './ExportingProgressBar';
+
 class ExportPage extends React.Component {
 
   constructor(props) {
@@ -17,26 +21,80 @@ class ExportPage extends React.Component {
     this.state = {
       collections: [],
       zipFileStats: [],
+      progressList: [],
       isExportModalOpen: false,
+      isExporting: false,
+      isZipping: false,
+      isExported: false,
     };
 
     this.onZipFileStatAdd = this.onZipFileStatAdd.bind(this);
     this.onZipFileStatRemove = this.onZipFileStatRemove.bind(this);
     this.openExportModal = this.openExportModal.bind(this);
     this.closeExportModal = this.closeExportModal.bind(this);
+    this.exportingRequestedHandler = this.exportingRequestedHandler.bind(this);
   }
 
-  async componentDidMount() {
+  async componentWillMount() {
     // TODO:: use apiv3.get
     // eslint-disable-next-line no-unused-vars
-    const [{ collections }, { zipFileStats }] = await Promise.all([
+    const [{ collections }, { status }] = await Promise.all([
       this.props.appContainer.apiGet('/v3/mongo/collections', {}),
       this.props.appContainer.apiGet('/v3/export/status', {}),
     ]);
     // TODO: toastSuccess, toastError
 
-    this.setState({ collections: ['pages', 'revisions'], zipFileStats }); // FIXME: delete this line and uncomment the line below
-    // this.setState({ collections, zipFileStats });
+    const { zipFileStats, isExporting, progressList } = status;
+    this.setState({
+      collections,
+      zipFileStats,
+      isExporting,
+      progressList,
+    });
+
+    this.setupWebsocketEventHandler();
+  }
+
+  setupWebsocketEventHandler() {
+    const socket = this.props.websocketContainer.getWebSocket();
+
+    // websocket event
+    socket.on('admin:onProgressForExport', ({ currentCount, totalCount, progressList }) => {
+      this.setState({
+        isExporting: true,
+        progressList,
+      });
+    });
+
+    // websocket event
+    socket.on('admin:onStartZippingForExport', () => {
+      this.setState({
+        isZipping: true,
+      });
+    });
+
+    // websocket event
+    socket.on('admin:onTerminateForExport', ({ addedZipFileStat }) => {
+      const zipFileStats = this.state.zipFileStats.concat([addedZipFileStat]);
+
+      this.setState({
+        isExporting: false,
+        isZipping: false,
+        isExported: true,
+        zipFileStats,
+      });
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, `New Exported Data '${addedZipFileStat.fileName}' is added`, {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+    });
   }
 
   onZipFileStatAdd(newStat) {
@@ -89,18 +147,72 @@ class ExportPage extends React.Component {
     this.setState({ isExportModalOpen: false });
   }
 
+  /**
+   * event handler invoked when export process was requested successfully
+   */
+  exportingRequestedHandler() {
+  }
+
+  renderProgressBarsForCollections() {
+    const cols = this.state.progressList.map((progressData) => {
+      const { collectionName, currentCount, totalCount } = progressData;
+      return (
+        <div className="col-md-6" key={collectionName}>
+          <ExportingProgressBar
+            header={collectionName}
+            currentCount={currentCount}
+            totalCount={totalCount}
+          />
+        </div>
+      );
+    });
+
+    return <div className="row px-3">{cols}</div>;
+  }
+
+  renderProgressBarForZipping() {
+    const { isZipping, isExported } = this.state;
+    const showZippingBar = isZipping || isExported;
+
+    if (!showZippingBar) {
+      return <></>;
+    }
+
+    return (
+      <div className="row px-3">
+        <div className="col-md-12" key="progressBarForZipping">
+          <ExportingProgressBar
+            header="Zip Files"
+            currentCount={1}
+            totalCount={1}
+            isInProgress={isZipping}
+          />
+        </div>
+      </div>
+    );
+  }
+
   render() {
     const { t } = this.props;
+    const { isExporting, isExported, progressList } = this.state;
+
+    const showExportingData = (isExported || isExporting) && (progressList != null);
 
     return (
       <Fragment>
-        <div className="alert alert-warning">
-          <i className="icon-exclamation"></i> { t('export_management.beta_warning') }
-        </div>
-
         <h2>{t('Export Data')}</h2>
 
-        <button type="button" className="btn btn-default" onClick={this.openExportModal}>{t('export_management.create_new_exported_data')}</button>
+        <button type="button" className="btn btn-default" disabled={isExporting} onClick={this.openExportModal}>
+          {t('export_management.create_new_exported_data')}
+        </button>
+
+        { showExportingData && (
+          <div className="mt-5">
+            <h3>{t('export_management.exporting_data_list')}</h3>
+            { this.renderProgressBarsForCollections() }
+            { this.renderProgressBarForZipping() }
+          </div>
+        ) }
 
         <div className="mt-5">
           <h3>{t('export_management.exported_data_list')}</h3>
@@ -112,10 +224,9 @@ class ExportPage extends React.Component {
 
         <ExportZipFormModal
           isOpen={this.state.isExportModalOpen}
+          onExportingRequested={this.exportingRequestedHandler}
           onClose={this.closeExportModal}
           collections={this.state.collections}
-          zipFileStats={this.state.zipFileStats}
-          onZipFileStatAdd={this.onZipFileStatAdd}
         />
       </Fragment>
     );
@@ -126,13 +237,14 @@ class ExportPage extends React.Component {
 ExportPage.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+  websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
 };
 
 /**
  * Wrapper component for using unstated
  */
 const ExportPageFormWrapper = (props) => {
-  return createSubscribedElement(ExportPage, props, [AppContainer]);
+  return createSubscribedElement(ExportPage, props, [AppContainer, WebsocketContainer]);
 };
 
 export default withTranslation()(ExportPageFormWrapper);

src/client/js/components/Admin/Export/ExportZipFormModal.jsx

@@ -8,13 +8,25 @@ import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
+
+const GROUPS_PAGE = [
+  'pages', 'revisions', 'tags', 'pagetagrelations',
+];
+const GROUPS_USER = [
+  'users', 'externalaccounts', 'usergroups', 'usergrouprelations',
+];
+const GROUPS_CONFIG = [
+  'configs', 'updateposts', 'globalnotificationsettings',
+];
+const ALL_GROUPED_COLLECTIONS = GROUPS_PAGE.concat(GROUPS_USER).concat(GROUPS_CONFIG);
+
 class ExportZipFormModal extends React.Component {
 
   constructor(props) {
     super(props);
 
     this.state = {
-      collections: new Set(),
+      selectedCollections: new Set(),
     };
 
     this.toggleCheckbox = this.toggleCheckbox.bind(this);
@@ -29,24 +41,24 @@ class ExportZipFormModal extends React.Component {
     const { name, checked } = target;
 
     this.setState((prevState) => {
-      const collections = new Set(prevState.collections);
+      const selectedCollections = new Set(prevState.selectedCollections);
       if (checked) {
-        collections.add(name);
+        selectedCollections.add(name);
       }
       else {
-        collections.delete(name);
+        selectedCollections.delete(name);
       }
 
-      return { collections };
+      return { selectedCollections };
     });
   }
 
   checkAll() {
-    this.setState({ collections: new Set(this.props.collections) });
+    this.setState({ selectedCollections: new Set(this.props.collections) });
   }
 
   uncheckAll() {
-    this.setState({ collections: new Set() });
+    this.setState({ selectedCollections: new Set() });
   }
 
   async export(e) {
@@ -54,13 +66,15 @@ class ExportZipFormModal extends React.Component {
 
     try {
       // TODO: use appContainer.apiv3.post
-      const { zipFileStat } = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
+      const result = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.selectedCollections) });
       // TODO: toastSuccess, toastError
-      this.props.onZipFileStatAdd(zipFileStat);
-      this.props.onClose();
+
+      if (!result.ok) {
+        throw new Error('Error occured.');
+      }
 
       // TODO: toastSuccess, toastError
-      toastr.success(undefined, `Generated ${zipFileStat.fileName}`, {
+      toastr.success(undefined, 'Export process has requested.', {
         closeButton: true,
         progressBar: true,
         newestOnTop: false,
@@ -69,6 +83,12 @@ class ExportZipFormModal extends React.Component {
         timeOut: '1200',
         extendedTimeOut: '150',
       });
+
+      this.props.onExportingRequested();
+      this.props.onClose();
+
+      this.setState({ selectedCollections: new Set() });
+
     }
     catch (err) {
       // TODO: toastSuccess, toastError
@@ -84,7 +104,66 @@ class ExportZipFormModal extends React.Component {
   }
 
   validateForm() {
-    return this.state.collections.size > 0;
+    return this.state.selectedCollections.size > 0;
+  }
+
+  renderWarnForUser() {
+    // whether this.state.selectedCollections includes one of GROUPS_USER
+    const isUserRelatedDataSelected = GROUPS_USER.some((collectionName) => {
+      return this.state.selectedCollections.has(collectionName);
+    });
+
+    if (!isUserRelatedDataSelected) {
+      return <></>;
+    }
+
+    const html = this.props.t('export_management.desc_password_seed');
+
+    // eslint-disable-next-line react/no-danger
+    return <div className="well well-sm" dangerouslySetInnerHTML={{ __html: html }}></div>;
+  }
+
+  renderGroups(groupList, color) {
+    const collectionNames = groupList.filter((collectionName) => {
+      return this.props.collections.includes(collectionName);
+    });
+
+    return this.renderCheckboxes(collectionNames, color);
+  }
+
+  renderOthers() {
+    const collectionNames = this.props.collections.filter((collectionName) => {
+      return !ALL_GROUPED_COLLECTIONS.includes(collectionName);
+    });
+
+    return this.renderCheckboxes(collectionNames);
+  }
+
+  renderCheckboxes(collectionNames, color) {
+    const checkboxColor = color ? `checkbox-${color}` : 'checkbox-info';
+
+    return (
+      <div className={`row checkbox ${checkboxColor}`}>
+        {collectionNames.map((collectionName) => {
+          return (
+            <div className="col-xs-6 my-1" key={collectionName}>
+              <input
+                type="checkbox"
+                id={collectionName}
+                name={collectionName}
+                className="form-check-input"
+                value={collectionName}
+                checked={this.state.selectedCollections.has(collectionName)}
+                onChange={this.toggleCheckbox}
+              />
+              <label className="text-capitalize form-check-label ml-3" htmlFor={collectionName}>
+                {collectionName}
+              </label>
+            </div>
+          );
+        })}
+      </div>
+    );
   }
 
   render() {
@@ -108,25 +187,30 @@ class ExportZipFormModal extends React.Component {
                 </button>
               </div>
             </div>
-            <div className="checkbox checkbox-info">
-              {this.props.collections.map((collectionName) => {
-                return (
-                  <div className="my-1" key={collectionName}>
-                    <input
-                      type="checkbox"
-                      id={collectionName}
-                      name={collectionName}
-                      className="form-check-input"
-                      value={collectionName}
-                      checked={this.state.collections.has(collectionName)}
-                      onChange={this.toggleCheckbox}
-                    />
-                    <label className="text-capitalize form-check-label ml-3" htmlFor={collectionName}>
-                      {collectionName}
-                    </label>
-                  </div>
-                );
-              })}
+            <div className="row mt-4">
+              <div className="col-xs-12">
+                <legend>Page Collections</legend>
+                { this.renderGroups(GROUPS_PAGE) }
+              </div>
+            </div>
+            <div className="row mt-4">
+              <div className="col-xs-12">
+                <legend>User Collections</legend>
+                { this.renderGroups(GROUPS_USER, 'danger') }
+                { this.renderWarnForUser() }
+              </div>
+            </div>
+            <div className="row mt-4">
+              <div className="col-xs-12">
+                <legend>Config Collections</legend>
+                { this.renderGroups(GROUPS_CONFIG) }
+              </div>
+            </div>
+            <div className="row mt-4">
+              <div className="col-xs-12">
+                <legend>Other Collections</legend>
+                { this.renderOthers() }
+              </div>
             </div>
           </Modal.Body>
 
@@ -146,10 +230,9 @@ ExportZipFormModal.propTypes = {
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
 
   isOpen: PropTypes.bool.isRequired,
+  onExportingRequested: PropTypes.func.isRequired,
   onClose: PropTypes.func.isRequired,
   collections: PropTypes.arrayOf(PropTypes.string).isRequired,
-  zipFileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
-  onZipFileStatAdd: PropTypes.func.isRequired,
 };
 
 /**

src/client/js/components/Admin/Export/ExportingProgressBar.jsx

@@ -0,0 +1,45 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+class ExportingProgressBar extends React.Component {
+
+
+  render() {
+    const {
+      header, currentCount, totalCount, isInProgress,
+    } = this.props;
+
+    const percentage = currentCount / totalCount * 100;
+    const isActive = (isInProgress != null)
+      ? isInProgress //                         apply props.isInProgress if set
+      : (currentCount !== totalCount); //       otherwise, set true when currentCount does not equal totalCount
+
+    return (
+      <>
+        <h5 className="my-1">
+          {header}
+          <div className="pull-right">{currentCount} / {totalCount}</div>
+        </h5>
+        <div className="progress progress-sm">
+          <div
+            className={`progress-bar ${isActive ? 'progress-bar-info progress-bar-striped active' : 'progress-bar-success'}`}
+            style={{ width: `${percentage}%` }}
+          >
+            <span className="sr-only">{percentage.toFixed(0)}% Complete</span>
+          </div>
+        </div>
+      </>
+    );
+  }
+
+}
+
+ExportingProgressBar.propTypes = {
+  header: PropTypes.string.isRequired,
+  currentCount: PropTypes.number.isRequired,
+  totalCount: PropTypes.number.isRequired,
+  isInProgress: PropTypes.bool,
+};
+
+export default withTranslation()(ExportingProgressBar);

src/server/console.js

@@ -0,0 +1,47 @@
+require('module-alias/register');
+
+const repl = require('repl');
+const fs = require('fs');
+const path = require('path');
+const mongoose = require('mongoose');
+const { getMongoUri } = require('@commons/util/mongoose-utils');
+
+const models = require('./models');
+
+Object.keys(models).forEach((modelName) => {
+  global[modelName] = models[modelName];
+});
+
+mongoose.Promise = global.Promise;
+
+const replServer = repl.start({
+  prompt: `${process.env.NODE_ENV} > `,
+  ignoreUndefined: true,
+});
+
+// add history function into repl
+// see: https://qiita.com/acro5piano/items/dc62b94d7b04505a4aca
+// see: https://qiita.com/potato4d/items/7131028497de53ceb48e
+const userHome = process.env[process.platform === 'win32' ? 'USERPROFILE' : 'HOME'];
+const replHistoryPath = path.join(userHome, '.node_repl_history');
+fs.readFile(replHistoryPath, 'utf8', (err, data) => {
+  if (err != null) {
+    return;
+  }
+  return data.split('\n').forEach((command) => { return replServer.history.push(command) });
+});
+
+replServer.context.mongoose = mongoose;
+replServer.context.models = models;
+
+mongoose.connect(getMongoUri(), { useNewUrlParser: true })
+  .then(() => {
+    replServer.context.db = mongoose.connection.db;
+  });
+
+replServer.on('exit', () => {
+  fs.writeFile(replHistoryPath, replServer.history.join('\n'), (err) => {
+    console.log(err); // eslint-disable-line no-console
+    process.exit();
+  });
+});

src/server/crowi/index.js

@@ -70,6 +70,7 @@ function Crowi(rootdir) {
     search: new (require(`${self.eventsDir}search`))(this),
     bookmark: new (require(`${self.eventsDir}bookmark`))(this),
     tag: new (require(`${self.eventsDir}tag`))(this),
+    admin: new (require(`${self.eventsDir}admin`))(this),
   };
 }
 

src/server/events/admin.js

@@ -0,0 +1,11 @@
+const util = require('util');
+const events = require('events');
+
+function AdminEvent(crowi) {
+  this.crowi = crowi;
+
+  events.EventEmitter.call(this);
+}
+util.inherits(AdminEvent, events.EventEmitter);
+
+module.exports = AdminEvent;

src/server/routes/apiv3/export.js

@@ -1,7 +1,6 @@
 const loggerFactory = require('@alias/logger');
 
 const logger = loggerFactory('growi:routes:apiv3:export');
-const path = require('path');
 const fs = require('fs');
 
 const express = require('express');
@@ -14,13 +13,50 @@ const router = express.Router();
  *    name: Export
  */
 
+/**
+ * @swagger
+ *
+ *  components:
+ *    schemas:
+ *      ExportStatus:
+ *        type: object
+ *        properties:
+ *          zipFileStats:
+ *            type: array
+ *            items:
+ *              type: object
+ *              description: the property of each file
+ *          progressList:
+ *            type: array
+ *            items:
+ *              type: object
+ *              description: progress data for each exporting collections
+ *          isExporting:
+ *            type: boolean
+ *            description: whether the current exporting job exists or not
+ */
+
 module.exports = (crowi) => {
   const accessTokenParser = require('../../middleware/access-token-parser')(crowi);
   const loginRequired = require('../../middleware/login-required')(crowi);
   const adminRequired = require('../../middleware/admin-required')(crowi);
   const csrf = require('../../middleware/csrf')(crowi);
 
-  const { growiBridgeService, exportService } = crowi;
+  const { exportService } = crowi;
+
+  this.adminEvent = crowi.event('admin');
+
+  // setup event
+  this.adminEvent.on('onProgressForExport', (data) => {
+    crowi.getIo().sockets.emit('admin:onProgressForExport', data);
+  });
+  this.adminEvent.on('onStartZippingForExport', (data) => {
+    crowi.getIo().sockets.emit('admin:onStartZippingForExport', data);
+  });
+  this.adminEvent.on('onTerminateForExport', (data) => {
+    crowi.getIo().sockets.emit('admin:onTerminateForExport', data);
+  });
+
 
   /**
    * @swagger
@@ -36,17 +72,17 @@ module.exports = (crowi) => {
    *            application/json:
    *              schema:
    *                properties:
-   *                  zipFileStats:
-   *                    type: array
-   *                    items:
-   *                      type: object
-   *                      description: the property of each file
+   *                  status:
+   *                    $ref: '#/components/schemas/ExportStatus'
    */
   router.get('/status', accessTokenParser, loginRequired, adminRequired, async(req, res) => {
-    const zipFileStats = await exportService.getStatus();
+    const status = await exportService.getStatus();
 
     // TODO: use res.apiv3
-    return res.json({ ok: true, zipFileStats });
+    return res.json({
+      ok: true,
+      status,
+    });
   });
 
   /**
@@ -63,35 +99,19 @@ module.exports = (crowi) => {
    *            application/json:
    *              schema:
    *                properties:
-   *                  zipFileStat:
-   *                    type: object
-   *                    description: the property of the zip file
+   *                  status:
+   *                    $ref: '#/components/schemas/ExportStatus'
    */
   router.post('/', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
     // TODO: add express validator
     try {
       const { collections } = req.body;
-      // get model for collection
-      const models = collections.map(collectionName => growiBridgeService.getModelFromCollectionName(collectionName));
-
-      const [metaJson, jsonFiles] = await Promise.all([
-        exportService.createMetaJson(),
-        exportService.exportMultipleCollectionsToJsons(models),
-      ]);
-
-      // zip json
-      const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
-      // add meta.json in zip
-      configs.push({ from: metaJson, as: path.basename(metaJson) });
-      // exec zip
-      const zipFile = await exportService.zipFiles(configs);
-      // get stats for the zip file
-      const zipFileStat = await growiBridgeService.parseZipFile(zipFile);
+
+      exportService.export(collections);
 
       // TODO: use res.apiv3
       return res.status(200).json({
         ok: true,
-        zipFileStat,
       });
     }
     catch (err) {

src/server/routes/apiv3/mongo.js

@@ -3,6 +3,7 @@ const loggerFactory = require('@alias/logger');
 const logger = loggerFactory('growi:routes:apiv3:mongo'); // eslint-disable-line no-unused-vars
 
 const express = require('express');
+const mongoose = require('mongoose');
 
 const router = express.Router();
 
@@ -33,12 +34,12 @@ module.exports = (crowi) => {
    *                      type: string
    */
   router.get('/collections', async(req, res) => {
-    const collections = Object.values(crowi.models).map(model => model.collection.name);
+    const collections = Object.keys(mongoose.connection.collections);
 
     // TODO: use res.apiv3
     return res.json({
       ok: true,
-      collections: [...new Set(collections)], // remove duplicates
+      collections,
     });
   });
 

src/server/service/export.js

@@ -1,10 +1,60 @@
 const logger = require('@alias/logger')('growi:services:ExportService'); // eslint-disable-line no-unused-vars
+
 const fs = require('fs');
 const path = require('path');
+const mongoose = require('mongoose');
+const { Transform } = require('stream');
 const streamToPromise = require('stream-to-promise');
 const archiver = require('archiver');
+
+
 const toArrayIfNot = require('../../lib/util/toArrayIfNot');
 
+
+class ExportingProgress {
+
+  constructor(collectionName, totalCount) {
+    this.collectionName = collectionName;
+    this.currentCount = 0;
+    this.totalCount = totalCount;
+  }
+
+}
+
+class ExportingStatus {
+
+  constructor() {
+    this.totalCount = 0;
+
+    this.progressList = null;
+    this.progressMap = {};
+  }
+
+  async init(collections) {
+    const promisesForCreatingInstance = collections.map(async(collectionName) => {
+      const collection = mongoose.connection.collection(collectionName);
+      const totalCount = await collection.count();
+      return new ExportingProgress(collectionName, totalCount);
+    });
+    this.progressList = await Promise.all(promisesForCreatingInstance);
+
+    // collection name to instance mapping
+    this.progressList.forEach((p) => {
+      this.progressMap[p.collectionName] = p;
+      this.totalCount += p.totalCount;
+    });
+  }
+
+  get currentCount() {
+    return this.progressList.reduce(
+      (acc, crr) => acc + crr.currentCount,
+      0,
+    );
+  }
+
+}
+
+
 class ExportService {
 
   constructor(crowi) {
@@ -15,13 +65,17 @@ class ExportService {
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
     this.per = 100;
     this.zlibLevel = 9; // 0(min) - 9(max)
+
+    this.adminEvent = crowi.event('admin');
+
+    this.currentExportingStatus = null;
   }
 
   /**
    * parse all zip files in downloads dir
    *
    * @memberOf ExportService
-   * @return {Array.<object>} info for zip files
+   * @return {object} info for zip files and whether currentExportingStatus exists
    */
   async getStatus() {
     const zipFiles = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' });
@@ -30,7 +84,16 @@ class ExportService {
       return this.growiBridgeService.parseZipFile(zipFile);
     }));
 
-    return zipFileStats;
+    // filter null object (broken zip)
+    const filtered = zipFileStats.filter(element => element != null);
+
+    const isExporting = this.currentExportingStatus != null;
+
+    return {
+      zipFileStats: filtered,
+      isExporting,
+      progressList: isExporting ? this.currentExportingStatus.progressList : null,
+    };
   }
 
   /**
@@ -59,87 +122,201 @@ class ExportService {
   }
 
   /**
-   * dump a collection into json
    *
-   * @memberOf ExportService
-   * @param {string} file path to json file to be written
-   * @param {readStream} readStream  read stream
-   * @param {number} total number of target items (optional)
-   * @param {function} [getLogText] (n, total) => { ... }
-   * @return {string} path to the exported json file
+   * @param {ExportProguress} exportProgress
+   * @return {Transform}
    */
-  async export(writeStream, readStream, total, getLogText) {
-    let n = 0;
+  generateLogStream(exportProgress) {
+    const logProgress = this.logProgress.bind(this);
 
-    // open an array
-    writeStream.write('[');
+    let count = 0;
 
-    readStream.on('data', (chunk) => {
-      if (n !== 0) writeStream.write(',');
-      writeStream.write(JSON.stringify(chunk));
-      n++;
-      this.logProgress(n, total, getLogText);
-    });
+    return new Transform({
+      transform(chunk, encoding, callback) {
+        count++;
+        logProgress(exportProgress, count);
 
-    readStream.on('end', () => {
-      // close the array
-      writeStream.write(']');
-      writeStream.close();
+        this.push(chunk);
+
+        callback();
+      },
     });
+  }
 
-    await streamToPromise(readStream);
+  /**
+   * insert beginning/ending brackets and comma separator for Json Array
+   *
+   * @memberOf ExportService
+   * @return {TransformStream}
+   */
+  generateTransformStream() {
+    let isFirst = true;
+
+    const transformStream = new Transform({
+      transform(chunk, encoding, callback) {
+        // write beginning brace
+        if (isFirst) {
+          this.push('[');
+          isFirst = false;
+        }
+        // write separator
+        else {
+          this.push(',');
+        }
+
+        this.push(chunk);
+        callback();
+      },
+      final(callback) {
+        // write ending brace
+        this.push(']');
+        callback();
+      },
+    });
 
-    return writeStream.path;
+    return transformStream;
   }
 
   /**
    * dump a mongodb collection into json
    *
    * @memberOf ExportService
-   * @param {object} Model instance of mongoose model
+   * @param {string} collectionName collection name
    * @return {string} path to zip file
    */
-  async exportCollectionToJson(Model) {
-    const collectionName = Model.collection.name;
+  async exportCollectionToJson(collectionName) {
+    const collection = mongoose.connection.collection(collectionName);
+
+    const nativeCursor = collection.find();
+    const readStream = nativeCursor
+      .snapshot()
+      .stream({ transform: JSON.stringify });
+
+    // get TransformStream
+    const transformStream = this.generateTransformStream();
+
+    // log configuration
+    const exportProgress = this.currentExportingStatus.progressMap[collectionName];
+    const logStream = this.generateLogStream(exportProgress);
+
+    // create WritableStream
     const jsonFileToWrite = path.join(this.baseDir, `${collectionName}.json`);
     const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() });
-    const readStream = Model.find().cursor();
-    const total = await Model.countDocuments();
-    const getLogText = (n, total) => `${collectionName}: ${n}/${total} written`;
 
-    const jsonFileWritten = await this.export(writeStream, readStream, total, getLogText);
+    readStream
+      .pipe(logStream)
+      .pipe(transformStream)
+      .pipe(writeStream);
+
+    await streamToPromise(readStream);
 
-    return jsonFileWritten;
+    return writeStream.path;
   }
 
   /**
-   * export multiple collections
+   * export multiple Collections into json and Zip
    *
    * @memberOf ExportService
-   * @param {Array.<object>} models array of instances of mongoose model
+   * @param {Array.<string>} collections array of collection name
    * @return {Array.<string>} paths to json files created
    */
-  async exportMultipleCollectionsToJsons(models) {
-    const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));
+  async exportCollectionsToZippedJson(collections) {
+    const metaJson = await this.createMetaJson();
+
+    const promises = collections.map(collectionName => this.exportCollectionToJson(collectionName));
+    const jsonFiles = await Promise.all(promises);
+
+    // send terminate event
+    this.emitStartZippingEvent();
+
+    // zip json
+    const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
+    // add meta.json in zip
+    configs.push({ from: metaJson, as: path.basename(metaJson) });
+    // exec zip
+    const zipFile = await this.zipFiles(configs);
+
+    // get stats for the zip file
+    const addedZipFileStat = await this.growiBridgeService.parseZipFile(zipFile);
+
+    // send terminate event
+    this.emitTerminateEvent(addedZipFileStat);
+
+    // TODO: remove broken zip file
+  }
+
+  async export(collections) {
+    if (this.currentExportingStatus != null) {
+      throw new Error('There is an exporting process running.');
+    }
+
+    this.currentExportingStatus = new ExportingStatus();
+    await this.currentExportingStatus.init(collections);
+
+    try {
+      await this.exportCollectionsToZippedJson(collections);
+    }
+    finally {
+      this.currentExportingStatus = null;
+    }
 
-    return jsonFiles;
   }
 
   /**
    * log export progress
    *
    * @memberOf ExportService
-   * @param {number} n number of items exported
-   * @param {number} total number of target items (optional)
-   * @param {function} [getLogText] (n, total) => { ... }
+   *
+   * @param {ExportProgress} exportProgress
+   * @param {number} currentCount number of items exported
    */
-  logProgress(n, total, getLogText) {
-    const output = getLogText ? getLogText(n, total) : `${n}/${total} items written`;
+  logProgress(exportProgress, currentCount) {
+    const output = `${exportProgress.collectionName}: ${currentCount}/${exportProgress.totalCount} written`;
+
+    // update exportProgress.currentCount
+    exportProgress.currentCount = currentCount;
 
     // output every this.per items
-    if (n % this.per === 0) logger.debug(output);
+    if (currentCount % this.per === 0) {
+      logger.debug(output);
+      this.emitProgressEvent();
+    }
     // output last item
-    else if (n === total) logger.info(output);
+    else if (currentCount === exportProgress.totalCount) {
+      logger.info(output);
+      this.emitProgressEvent();
+    }
+  }
+
+  /**
+   * emit progress event
+   * @param {ExportProgress} exportProgress
+   */
+  emitProgressEvent(exportProgress) {
+    const { currentCount, totalCount, progressList } = this.currentExportingStatus;
+    const data = {
+      currentCount,
+      totalCount,
+      progressList,
+    };
+
+    // send event (in progress in global)
+    this.adminEvent.emit('onProgressForExport', data);
+  }
+
+  /**
+   * emit start zipping event
+   */
+  emitStartZippingEvent() {
+    this.adminEvent.emit('onStartZippingForExport', {});
+  }
+
+  /**
+   * emit terminate event
+   * @param {object} zipFileStat added zip file status data
+   */
+  emitTerminateEvent(zipFileStat) {
+    this.adminEvent.emit('onTerminateForExport', { addedZipFileStat: zipFileStat });
   }
 
   /**

src/server/service/growi-bridge.js

@@ -99,11 +99,12 @@ class GrowiBridgeService {
    * @return {object} meta{object} and files{Array.<object>}
    */
   async parseZipFile(zipFile) {
-    const readStream = fs.createReadStream(zipFile);
-    const unzipStream = readStream.pipe(unzipper.Parse());
     const fileStats = [];
     let meta = {};
 
+    const readStream = fs.createReadStream(zipFile);
+    const unzipStream = readStream.pipe(unzipper.Parse());
+
     unzipStream.on('entry', async(entry) => {
       const fileName = entry.path;
       const size = entry.vars.uncompressedSize; // There is also compressedSize;
@@ -122,7 +123,14 @@ class GrowiBridgeService {
       entry.autodrain();
     });
 
-    await streamToPromise(unzipStream);
+    try {
+      await streamToPromise(unzipStream);
+    }
+    // if zip is broken
+    catch (err) {
+      logger.error(err);
+      return null;
+    }
 
     return {
       meta,