Merge pull request #1215 from weseek/feat/growi-export-n-import

Feat/growi export n import

package.json

@@ -68,6 +68,7 @@
       "mongoose: somehow GlobalNotificationSetting CRUD does not work with mongoose v5.6.0",
       "openid-client: Node.js 12 or higher is required for openid-client@3 and above."
     ],
+    "JSONStream": "^1.3.5",
     "archiver": "^3.1.1",
     "async": "^3.0.1",
     "aws-sdk": "^2.88.0",
@@ -133,6 +134,7 @@
     "string-width": "^4.1.0",
     "swig-templates": "^2.0.2",
     "uglifycss": "^0.0.29",
+    "unzipper": "^0.10.5",
     "url-join": "^4.0.0",
     "xss": "^1.0.6"
   },

resource/locales/en-US/translation.json

@@ -737,7 +737,18 @@
   },
 
   "importer_management": {
+    "beta_warning": "This function is Beta.",
     "import_from": "Import from %s",
+    "import_form_growi": "Import from GROWI",
+    "growi_settings": {
+      "overwrite_documents": "Imported documents will overwrite existing documents",
+      "zip_file": "Zip File",
+      "uploaded_data": "Uploaded Data",
+      "extracted_file": "Extracted File",
+      "collection": "Collection",
+      "upload": "Upload",
+      "discard": "Discard Uploaded Data"
+    },
     "esa_settings": {
       "team_name": "Team name",
       "access_token": "Access token",
@@ -751,5 +762,22 @@
     "import": "Import",
     "page_skip": "Pages with a name that already exists on GROWI are not imported",
     "Directory_hierarchy_tag": "Directory Hierarchy Tag"
+  },
+
+  "export_management": {
+    "beta_warning": "This function is Beta.",
+    "export_as_zip": "Export Data as Zip",
+    "export_collections": "Export Collections",
+    "check_all": "Check All",
+    "uncheck_all": "Uncheck All",
+    "export": "Export",
+    "cancel": "Cancel",
+    "file": "File",
+    "growi_version": "Growi Version",
+    "collections": "Collections",
+    "exported_at": "Exported At",
+    "export_menu": "Export Menu",
+    "download": "Download",
+    "delete": "Delete"
   }
 }

resource/locales/ja/translation.json

@@ -720,7 +720,18 @@
   },
 
   "importer_management": {
+    "beta_warning": "この機能はベータ版です",
     "import_from": "%s からインポート",
+    "import_form_growi": "GROWIからインポート",
+    "growi_settings": {
+      "overwrite_documents": "インポートされたドキュメントは既存のドキュメントを上書きします",
+      "zip_file": "Zip ファイル",
+      "uploaded_data": "アップロードされたデータ",
+      "extracted_file": "展開されたファイル",
+      "collection": "コレクション",
+      "upload": "アップロード",
+      "discard": "アップロードしたデータを破棄する"
+    },
     "esa_settings": {
       "team_name": "チーム名",
       "access_token": "アクセストークン",
@@ -734,5 +745,22 @@
     "import": "インポート",
     "page_skip": "既に GROWI 側に同名のページが存在する場合、そのページはスキップされます",
     "Directory_hierarchy_tag": "ディレクトリ階層タグ"
+  },
+
+  "export_management": {
+    "beta_warning": "この機能はベータ版です",
+    "export_as_zip": "Zipファイルでエクスポート",
+    "export_collections": "コレクションのエクスポート",
+    "check_all": "全てにチェックを付ける",
+    "uncheck_all": "全てからチェックを外す",
+    "export": "エクスポート",
+    "cancel": "キャンセル",
+    "file": "ファイル名",
+    "growi_version": "Growi バージョン",
+    "collections": "コレクション",
+    "exported_at": "エクスポートされた時間",
+    "export_menu": "エクスポートメニュー",
+    "download": "ダウンロード",
+    "delete": "削除"
   }
 }

src/client/js/app.jsx

@@ -39,6 +39,7 @@ import CustomScriptEditor from './components/Admin/CustomScriptEditor';
 import CustomHeaderEditor from './components/Admin/CustomHeaderEditor';
 import AdminRebuildSearch from './components/Admin/AdminRebuildSearch';
 import ExportPage from './components/Admin/Export/ExportPage';
+import GrowiZipImportSection from './components/Admin/Import/GrowiZipImportSection';
 import GroupDeleteModal from './components/GroupDeleteModal/GroupDeleteModal';
 
 import AppContainer from './services/AppContainer';
@@ -202,6 +203,19 @@ if (adminExportPageElem != null) {
   );
 }
 
+// TODO: move to /imponents/Admin/Importer.jsx
+const growiImportElem = document.getElementById('growi-import');
+if (growiImportElem != null) {
+  ReactDOM.render(
+    <Provider inject={[]}>
+      <I18nextProvider i18n={i18n}>
+        <GrowiZipImportSection />
+      </I18nextProvider>
+    </Provider>,
+    growiImportElem,
+  );
+}
+
 // うわーもうー (commented by Crowi team -- 2018.03.23 Yuki Takei)
 $('a[data-toggle="tab"][href="#revision-history"]').on('show.bs.tab', () => {
   ReactDOM.render(

src/client/js/components/Admin/Export/ExportAsZip.jsx

@@ -1,100 +0,0 @@
-import React, { Fragment } from 'react';
-import PropTypes from 'prop-types';
-import { withTranslation } from 'react-i18next';
-
-import { createSubscribedElement } from '../../UnstatedUtils';
-import AppContainer from '../../../services/AppContainer';
-// import { toastSuccess, toastError } from '../../../util/apiNotification';
-
-class ExportPage extends React.Component {
-
-  constructor(props) {
-    super(props);
-
-    this.state = {
-      files: {},
-    };
-
-    this.createZipFile = this.createZipFile.bind(this);
-    this.deleteZipFile = this.deleteZipFile.bind(this);
-  }
-
-  async componentDidMount() {
-    const res = await this.props.appContainer.apiGet('/v3/export', {});
-
-    this.setState({ files: res.files });
-  }
-
-  async createZipFile() {
-    // TODO use appContainer.apiv3.post
-    const res = await this.props.appContainer.apiPost('/v3/export/pages', {});
-    // TODO toastSuccess, toastError
-    this.setState((prevState) => {
-      return {
-        files: {
-          ...prevState.files,
-          [res.collection]: res.file,
-        },
-      };
-    });
-  }
-
-  async deleteZipFile() {
-    // TODO use appContainer.apiv3.delete
-    // TODO toastSuccess, toastError
-  }
-
-  render() {
-    // const { t } = this.props;
-
-    return (
-      <Fragment>
-        <h2>Export Data as Zip</h2>
-        <form className="my-5">
-          {Object.keys(this.state.files).map((file) => {
-            const disabled = file !== 'pages';
-            return (
-              <div className="form-check" key={file}>
-                <input
-                  type="radio"
-                  id={file}
-                  name="collection"
-                  className="form-check-input"
-                  value={file}
-                  disabled={disabled}
-                  checked={!disabled}
-                  onChange={() => {}}
-                />
-                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={file}>
-                  {file} ({this.state.files[file] || 'not found'})
-                </label>
-              </div>
-            );
-          })}
-        </form>
-        <button type="button" className="btn btn-sm btn-default" onClick={this.createZipFile}>Generate</button>
-        <a href="/_api/v3/export/pages">
-          <button type="button" className="btn btn-sm btn-primary ml-2">Download</button>
-        </a>
-        {/* <button type="button" className="btn btn-sm btn-danger ml-2" onClick={this.deleteZipFile}>Clear</button> */}
-      </Fragment>
-    );
-  }
-
-}
-
-ExportPage.propTypes = {
-  t: PropTypes.func.isRequired, // i18next
-  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
-
-  isAclEnabled: PropTypes.bool,
-};
-
-/**
- * Wrapper component for using unstated
- */
-const ExportPageWrapper = (props) => {
-  return createSubscribedElement(ExportPage, props, [AppContainer]);
-};
-
-export default withTranslation()(ExportPageWrapper);

src/client/js/components/Admin/Export/ExportPage.jsx

@@ -1,17 +1,137 @@
 import React, { Fragment } from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import * as toastr from 'toastr';
 
-import ExportAsZip from './ExportAsZip';
+import ExportZipFormModal from './ExportZipFormModal';
+import ZipFileTable from './ZipFileTable';
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
 
 class ExportPage extends React.Component {
 
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      collections: [],
+      zipFileStats: [],
+      isExportModalOpen: false,
+    };
+
+    this.onZipFileStatAdd = this.onZipFileStatAdd.bind(this);
+    this.onZipFileStatRemove = this.onZipFileStatRemove.bind(this);
+    this.openExportModal = this.openExportModal.bind(this);
+    this.closeExportModal = this.closeExportModal.bind(this);
+  }
+
+  async componentDidMount() {
+    // TODO:: use apiv3.get
+    // eslint-disable-next-line no-unused-vars
+    const [{ collections }, { zipFileStats }] = await Promise.all([
+      this.props.appContainer.apiGet('/v3/mongo/collections', {}),
+      this.props.appContainer.apiGet('/v3/export/status', {}),
+    ]);
+    // TODO: toastSuccess, toastError
+
+    this.setState({ collections: ['pages', 'revisions'], zipFileStats }); // FIXME: delete this line and uncomment the line below
+    // this.setState({ collections, zipFileStats });
+  }
+
+  onZipFileStatAdd(newStat) {
+    this.setState((prevState) => {
+      return {
+        zipFileStats: [...prevState.zipFileStats, newStat],
+      };
+    });
+  }
+
+  async onZipFileStatRemove(fileName) {
+    try {
+      await this.props.appContainer.apiRequest('delete', `/v3/export/${fileName}`, {});
+
+      this.setState((prevState) => {
+        return {
+          zipFileStats: prevState.zipFileStats.filter(stat => stat.fileName !== fileName),
+        };
+      });
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, `Deleted ${fileName}`, {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
+  }
+
+  openExportModal() {
+    this.setState({ isExportModalOpen: true });
+  }
+
+  closeExportModal() {
+    this.setState({ isExportModalOpen: false });
+  }
+
   render() {
+    const { t } = this.props;
+
     return (
       <Fragment>
-        <ExportAsZip />
+        <h2>{t('export_management.export_as_zip')}</h2>
+        <div className="alert alert-warning">
+          <i className="icon-exclamation"></i> { t('export_management.beta_warning') }
+        </div>
+        <div className="row my-5">
+          <div className="col-xs-offset-3 col-xs-6">
+            <button type="submit" className="btn btn-sm btn-primary" onClick={this.openExportModal}>{t('export_management.export')}</button>
+          </div>
+        </div>
+        <ExportZipFormModal
+          isOpen={this.state.isExportModalOpen}
+          onClose={this.closeExportModal}
+          collections={this.state.collections}
+          zipFileStats={this.state.zipFileStats}
+          onZipFileStatAdd={this.onZipFileStatAdd}
+        />
+        {this.state.zipFileStats.length > 0 && (
+          <ZipFileTable
+            zipFileStats={this.state.zipFileStats}
+            onZipFileStatRemove={this.onZipFileStatRemove}
+          />
+        )}
       </Fragment>
     );
   }
 
 }
 
-export default ExportPage;
+ExportPage.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ExportPageFormWrapper = (props) => {
+  return createSubscribedElement(ExportPage, props, [AppContainer]);
+};
+
+export default withTranslation()(ExportPageFormWrapper);

src/client/js/components/Admin/Export/ExportTableMenu.jsx

@@ -0,0 +1,52 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+class ExportTableMenu extends React.Component {
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <div className="btn-group admin-user-menu">
+        <button type="button" className="btn btn-sm btn-default dropdown-toggle" data-toggle="dropdown">
+          <i className="icon-settings"></i> <span className="caret"></span>
+        </button>
+        <ul className="dropdown-menu" role="menu">
+          <li className="dropdown-header">{t('export_management.export_menu')}</li>
+          <li>
+            <a href={`/admin/export/${this.props.fileName}`}>
+              <i className="icon-cloud-download" /> {t('export_management.download')}
+            </a>
+          </li>
+          <li onClick={() => this.props.onZipFileStatRemove(this.props.fileName)}>
+            <a>
+              <span className="text-danger"><i className="icon-trash" /> {t('export_management.delete')}</span>
+            </a>
+          </li>
+        </ul>
+      </div>
+    );
+  }
+
+}
+
+ExportTableMenu.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+  fileName: PropTypes.string.isRequired,
+  onZipFileStatRemove: PropTypes.func.isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ExportTableMenuWrapper = (props) => {
+  return createSubscribedElement(ExportTableMenu, props, [AppContainer]);
+};
+
+export default withTranslation()(ExportTableMenuWrapper);

src/client/js/components/Admin/Export/ExportZipFormModal.jsx

@@ -0,0 +1,162 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import Modal from 'react-bootstrap/es/Modal';
+import * as toastr from 'toastr';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+class ExportZipFormModal extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      collections: new Set(),
+    };
+
+    this.toggleCheckbox = this.toggleCheckbox.bind(this);
+    this.checkAll = this.checkAll.bind(this);
+    this.uncheckAll = this.uncheckAll.bind(this);
+    this.export = this.export.bind(this);
+    this.validateForm = this.validateForm.bind(this);
+  }
+
+  toggleCheckbox(e) {
+    const { target } = e;
+    const { name, checked } = target;
+
+    this.setState((prevState) => {
+      const collections = new Set(prevState.collections);
+      if (checked) {
+        collections.add(name);
+      }
+      else {
+        collections.delete(name);
+      }
+
+      return { collections };
+    });
+  }
+
+  checkAll() {
+    this.setState({ collections: new Set(this.props.collections) });
+  }
+
+  uncheckAll() {
+    this.setState({ collections: new Set() });
+  }
+
+  async export(e) {
+    e.preventDefault();
+
+    try {
+      // TODO: use appContainer.apiv3.post
+      const { zipFileStat } = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
+      // TODO: toastSuccess, toastError
+      this.props.onZipFileStatAdd(zipFileStat);
+      this.props.onClose();
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, `Generated ${zipFileStat.fileName}`, {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
+  }
+
+  validateForm() {
+    return this.state.collections.size > 0;
+  }
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <Modal show={this.props.isOpen} onHide={this.props.onClose}>
+        <Modal.Header closeButton>
+          <Modal.Title>{t('export_management.export_collections')}</Modal.Title>
+        </Modal.Header>
+
+        <form onSubmit={this.export}>
+          <Modal.Body>
+            <div className="row">
+              <div className="col-sm-12">
+                <button type="button" className="btn btn-sm btn-default mr-2" onClick={this.checkAll}>
+                  <i className="fa fa-check-square-o"></i> {t('export_management.check_all')}
+                </button>
+                <button type="button" className="btn btn-sm btn-default mr-2" onClick={this.uncheckAll}>
+                  <i className="fa fa-square-o"></i> {t('export_management.uncheck_all')}
+                </button>
+              </div>
+            </div>
+            <div className="checkbox checkbox-info">
+              {this.props.collections.map((collectionName) => {
+                return (
+                  <div className="my-1" key={collectionName}>
+                    <input
+                      type="checkbox"
+                      id={collectionName}
+                      name={collectionName}
+                      className="form-check-input"
+                      value={collectionName}
+                      checked={this.state.collections.has(collectionName)}
+                      onChange={this.toggleCheckbox}
+                    />
+                    <label className="text-capitalize form-check-label ml-3" htmlFor={collectionName}>
+                      {collectionName}
+                    </label>
+                  </div>
+                );
+              })}
+            </div>
+          </Modal.Body>
+
+          <Modal.Footer>
+            <button type="button" className="btn btn-sm btn-default" onClick={this.props.onClose}>{t('export_management.cancel')}</button>
+            <button type="submit" className="btn btn-sm btn-primary" disabled={!this.validateForm()}>{t('export_management.export')}</button>
+          </Modal.Footer>
+        </form>
+      </Modal>
+    );
+  }
+
+}
+
+ExportZipFormModal.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+
+  isOpen: PropTypes.bool.isRequired,
+  onClose: PropTypes.func.isRequired,
+  collections: PropTypes.arrayOf(PropTypes.string).isRequired,
+  zipFileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
+  onZipFileStatAdd: PropTypes.func.isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ExportZipFormModalWrapper = (props) => {
+  return createSubscribedElement(ExportZipFormModal, props, [AppContainer]);
+};
+
+export default withTranslation()(ExportZipFormModalWrapper);

src/client/js/components/Admin/Export/ZipFileTable.jsx

@@ -0,0 +1,67 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import { format } from 'date-fns';
+
+import ExportTableMenu from './ExportTableMenu';
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+class ZipFileTable extends React.Component {
+
+  render() {
+    // eslint-disable-next-line no-unused-vars
+    const { t } = this.props;
+
+    return (
+      <table className="table table-bordered">
+        <thead>
+          <tr>
+            <th>{t('export_management.file')}</th>
+            <th>{t('export_management.growi_version')}</th>
+            <th>{t('export_management.collections')}</th>
+            <th>{t('export_management.exported_at')}</th>
+            <th></th>
+          </tr>
+        </thead>
+        <tbody>
+          {this.props.zipFileStats.map(({ meta, fileName, fileStats }) => {
+            return (
+              <tr key={fileName}>
+                <th>{fileName}</th>
+                <td>{meta.version}</td>
+                <td className="text-capitalize">{fileStats.map(fileStat => fileStat.collectionName).join(', ')}</td>
+                <td>{meta.exportedAt ? format(new Date(meta.exportedAt), 'yyyy/MM/dd HH:mm:ss') : ''}</td>
+                <td>
+                  <ExportTableMenu
+                    fileName={fileName}
+                    onZipFileStatRemove={this.props.onZipFileStatRemove}
+                  />
+                </td>
+              </tr>
+            );
+          })}
+        </tbody>
+      </table>
+    );
+  }
+
+}
+
+ZipFileTable.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+
+  zipFileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
+  onZipFileStatRemove: PropTypes.func.isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ZipFileTableWrapper = (props) => {
+  return createSubscribedElement(ZipFileTable, props, [AppContainer]);
+};
+
+export default withTranslation()(ZipFileTableWrapper);

src/client/js/components/Admin/Import/GrowiZipImportForm.jsx

@@ -0,0 +1,181 @@
+import React, { Fragment } from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import * as toastr from 'toastr';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+class GrowiImportForm extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.initialState = {
+      collections: new Set(),
+      schema: {
+        pages: {},
+        revisions: {},
+        // ...
+      },
+    };
+
+    this.state = this.initialState;
+
+    this.toggleCheckbox = this.toggleCheckbox.bind(this);
+    this.import = this.import.bind(this);
+    this.validateForm = this.validateForm.bind(this);
+  }
+
+  toggleCheckbox(e) {
+    const { target } = e;
+    const { name, checked } = target;
+
+    this.setState((prevState) => {
+      const collections = new Set(prevState.collections);
+      if (checked) {
+        collections.add(name);
+      }
+      else {
+        collections.delete(name);
+      }
+      return { collections };
+    });
+  }
+
+  async import(e) {
+    e.preventDefault();
+
+    try {
+      // TODO: use appContainer.apiv3.post
+      const { results } = await this.props.appContainer.apiPost('/v3/import', {
+        fileName: this.props.fileName,
+        collections: Array.from(this.state.collections),
+        schema: this.state.schema,
+      });
+
+      this.setState(this.initialState);
+      this.props.onPostImport();
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, 'Imported documents', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+
+      for (const { collectionName, failedIds } of results) {
+        if (failedIds.length > 0) {
+          toastr.error(`failed to insert ${failedIds.join(', ')}`, collectionName, {
+            closeButton: true,
+            progressBar: true,
+            newestOnTop: false,
+            timeOut: '30000',
+          });
+        }
+      }
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
+  }
+
+  validateForm() {
+    return this.state.collections.size > 0;
+  }
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <form className="row" onSubmit={this.import}>
+        <div className="col-xs-12">
+          <table className="table table-bordered table-mapping">
+            <caption>{t('importer_management.growi_settings.uploaded_data')}</caption>
+            <thead>
+              <tr>
+                <th></th>
+                <th>{t('importer_management.growi_settings.extracted_file')}</th>
+                <th>{t('importer_management.growi_settings.collection')}</th>
+              </tr>
+            </thead>
+            <tbody>
+              {this.props.fileStats.map((fileStat) => {
+                  const { fileName, collectionName } = fileStat;
+                  const checked = this.state.collections.has(collectionName);
+                  return (
+                    <Fragment key={collectionName}>
+                      <tr>
+                        <td>
+                          <input
+                            type="checkbox"
+                            id={collectionName}
+                            name={collectionName}
+                            className="form-check-input"
+                            value={collectionName}
+                            checked={checked}
+                            onChange={this.toggleCheckbox}
+                          />
+                        </td>
+                        <td>{fileName}</td>
+                        <td className="text-capitalize">{collectionName}</td>
+                      </tr>
+                      {checked && (
+                        <tr>
+                          <td className="text-muted" colSpan="3">
+                            TBD: define how {collectionName} are imported
+                            {/* TODO: create a component for each collection to modify this.state.schema */}
+                          </td>
+                        </tr>
+                      )}
+                    </Fragment>
+                  );
+                })}
+            </tbody>
+          </table>
+        </div>
+        <div className="col-xs-12 text-center">
+          <button type="submit" className="btn btn-primary mx-1" disabled={!this.validateForm()}>
+            { t('importer_management.import') }
+          </button>
+          <button type="button" className="btn btn-default mx-1" onClick={this.props.onDiscard}>
+            { t('importer_management.growi_settings.discard') }
+          </button>
+        </div>
+      </form>
+    );
+  }
+
+}
+
+GrowiImportForm.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+
+  fileName: PropTypes.string,
+  fileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
+  onDiscard: PropTypes.func.isRequired,
+  onPostImport: PropTypes.func.isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const GrowiImportFormWrapper = (props) => {
+  return createSubscribedElement(GrowiImportForm, props, [AppContainer]);
+};
+
+export default withTranslation()(GrowiImportFormWrapper);

src/client/js/components/Admin/Import/GrowiZipImportSection.jsx

@@ -0,0 +1,119 @@
+import React, { Fragment } from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import * as toastr from 'toastr';
+
+import GrowiZipUploadForm from './GrowiZipUploadForm';
+import GrowiZipImportForm from './GrowiZipImportForm';
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+class GrowiZipImportSection extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.initialState = {
+      fileName: '',
+      fileStats: [],
+    };
+
+    this.state = this.initialState;
+
+    this.handleUpload = this.handleUpload.bind(this);
+    this.discardData = this.discardData.bind(this);
+    this.resetState = this.resetState.bind(this);
+  }
+
+  handleUpload({ meta, fileName, fileStats }) {
+    this.setState({
+      fileName,
+      fileStats,
+    });
+  }
+
+  async discardData() {
+    try {
+      const { fileName } = this.state;
+      await this.props.appContainer.apiRequest('delete', `/v3/import/${this.state.fileName}`, {});
+      this.resetState();
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, `Deleted ${fileName}`, {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
+  }
+
+  resetState() {
+    this.setState(this.initialState);
+  }
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <Fragment>
+        <legend>{t('importer_management.import_form_growi')}</legend>
+
+        <div className="alert alert-warning">
+          <i className="icon-exclamation"></i> { t('importer_management.beta_warning') }
+        </div>
+
+        <div className="well well-sm small">
+          <ul>
+            <li>{t('importer_management.growi_settings.overwrite_documents')}</li>
+          </ul>
+        </div>
+
+        {this.state.fileName ? (
+          <Fragment>
+            <GrowiZipImportForm
+              fileName={this.state.fileName}
+              fileStats={this.state.fileStats}
+              onDiscard={this.discardData}
+              onPostImport={this.resetState}
+            />
+          </Fragment>
+        ) : (
+          <GrowiZipUploadForm
+            onUpload={this.handleUpload}
+          />
+        )}
+      </Fragment>
+    );
+  }
+
+}
+
+GrowiZipImportSection.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const GrowiZipImportSectionWrapper = (props) => {
+  return createSubscribedElement(GrowiZipImportSection, props, [AppContainer]);
+};
+
+export default withTranslation()(GrowiZipImportSectionWrapper);

src/client/js/components/Admin/Import/GrowiZipUploadForm.jsx

@@ -0,0 +1,93 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+class GrowiZipUploadForm extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.inputRef = React.createRef();
+
+    this.changeFileName = this.changeFileName.bind(this);
+    this.uploadZipFile = this.uploadZipFile.bind(this);
+    this.validateForm = this.validateForm.bind(this);
+  }
+
+  changeFileName(e) {
+    // to trigger rerender at onChange event
+    // eslint-disable-next-line react/no-unused-state
+    this.setState({ dummy: e.target.files[0].name });
+  }
+
+  async uploadZipFile(e) {
+    e.preventDefault();
+
+    const formData = new FormData();
+    formData.append('_csrf', this.props.appContainer.csrfToken);
+    formData.append('file', this.inputRef.current.files[0]);
+
+    // TODO: use appContainer.apiv3.post
+    const { data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
+    this.props.onUpload(data);
+    // TODO: toastSuccess, toastError
+  }
+
+  validateForm() {
+    return (
+      this.inputRef.current // null check
+      && this.inputRef.current.files[0] // null check
+      && /\.zip$/.test(this.inputRef.current.files[0].name) // validate extension
+    );
+  }
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <form className="form-horizontal" onSubmit={this.uploadZipFile}>
+        <fieldset>
+          <div className="form-group d-flex align-items-center">
+            <label htmlFor="file" className="col-xs-3 control-label">{t('importer_management.growi_settings.zip_file')}</label>
+            <div className="col-xs-6">
+              <input
+                type="file"
+                name="file"
+                className="form-control-file"
+                ref={this.inputRef}
+                onChange={this.changeFileName}
+              />
+            </div>
+          </div>
+          <div className="form-group">
+            <div className="col-xs-offset-3 col-xs-6">
+              <button type="submit" className="btn btn-primary" disabled={!this.validateForm()}>
+                {t('importer_management.growi_settings.upload')}
+              </button>
+            </div>
+          </div>
+        </fieldset>
+      </form>
+    );
+  }
+
+}
+
+GrowiZipUploadForm.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+  onUpload: PropTypes.func.isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const GrowiZipUploadFormWrapper = (props) => {
+  return createSubscribedElement(GrowiZipUploadForm, props, [AppContainer]);
+};
+
+export default withTranslation()(GrowiZipUploadFormWrapper);

src/lib/util/toArrayIfNot.js

@@ -0,0 +1,15 @@
+// converts non-array item to array
+
+const toArrayIfNot = (item) => {
+  if (item == null) {
+    return [];
+  }
+
+  if (Array.isArray(item)) {
+    return item;
+  }
+
+  return [item];
+};
+
+module.exports = toArrayIfNot;

src/server/crowi/index.js

@@ -46,7 +46,9 @@ function Crowi(rootdir) {
   this.appService = null;
   this.fileUploadService = null;
   this.restQiitaAPIService = null;
+  this.growiBridgeService = null;
   this.exportService = null;
+  this.importService = null;
   this.cdnResourcesService = new CdnResourcesService();
   this.interceptorManager = new InterceptorManager();
   this.xss = new Xss();
@@ -86,10 +88,12 @@ Crowi.prototype.init = async function() {
   // customizeService depends on AppService and XssService
   // passportService depends on appService
   // slack depends on setUpSlacklNotification
+  // export and import depends on setUpGrowiBridge
   await Promise.all([
     this.setUpApp(),
     this.setUpXss(),
     this.setUpSlacklNotification(),
+    this.setUpGrowiBridge(),
   ]);
 
   await Promise.all([
@@ -105,6 +109,7 @@ Crowi.prototype.init = async function() {
     this.setUpRestQiitaAPI(),
     this.setupUserGroup(),
     this.setupExport(),
+    this.setupImport(),
   ]);
 
   // globalNotification depends on slack and mailer
@@ -124,6 +129,7 @@ Crowi.prototype.initForTest = async function() {
     this.setUpApp(),
     // this.setUpXss(),
     // this.setUpSlacklNotification(),
+    // this.setUpGrowiBridge(),
   ]);
 
   await Promise.all([
@@ -137,6 +143,9 @@ Crowi.prototype.initForTest = async function() {
     this.setUpAcl(),
   //   this.setUpCustomize(),
   //   this.setUpRestQiitaAPI(),
+  //   this.setupUserGroup(),
+  //   this.setupExport(),
+  //   this.setupImport(),
   ]);
 
   // globalNotification depends on slack and mailer
@@ -534,6 +543,13 @@ Crowi.prototype.setupUserGroup = async function() {
   }
 };
 
+Crowi.prototype.setUpGrowiBridge = async function() {
+  const GrowiBridgeService = require('../service/growi-bridge');
+  if (this.growiBridgeService == null) {
+    this.growiBridgeService = new GrowiBridgeService(this);
+  }
+};
+
 Crowi.prototype.setupExport = async function() {
   const ExportService = require('../service/export');
   if (this.exportService == null) {
@@ -541,4 +557,11 @@ Crowi.prototype.setupExport = async function() {
   }
 };
 
+Crowi.prototype.setupImport = async function() {
+  const ImportService = require('../service/import');
+  if (this.importService == null) {
+    this.importService = new ImportService(this);
+  }
+};
+
 module.exports = Crowi;

src/server/routes/admin.js

@@ -18,6 +18,7 @@ module.exports = function(crowi, app) {
     aclService,
     slackNotificationService,
     customizeService,
+    exportService,
   } = crowi;
 
   const recommendedWhitelist = require('@commons/service/xss/recommended-whitelist');
@@ -874,6 +875,21 @@ module.exports = function(crowi, app) {
     return res.render('admin/export');
   };
 
+  actions.export.download = (req, res) => {
+    // TODO: add express validator
+    const { fileName } = req.params;
+
+    try {
+      const zipFile = exportService.getFile(fileName);
+      return res.download(zipFile);
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.json(ApiResponse.error());
+    }
+  };
+
   actions.api = {};
   actions.api.appSetting = async function(req, res) {
     const form = req.form.settingForm;

src/server/routes/apiv3/export.js

@@ -2,6 +2,7 @@ const loggerFactory = require('@alias/logger');
 
 const logger = loggerFactory('growi:routes:apiv3:export'); // eslint-disable-line no-unused-vars
 const path = require('path');
+const fs = require('fs');
 
 const express = require('express');
 
@@ -14,48 +15,79 @@ const router = express.Router();
  */
 
 module.exports = (crowi) => {
-  const { exportService } = crowi;
-  const { Page } = crowi.models;
+  const { growiBridgeService, exportService } = crowi;
 
   /**
    * @swagger
    *
-   *  /export:
+   *  /export/status:
    *    get:
    *      tags: [Export]
-   *      description: get mongodb collections names and zip files for them
+   *      description: get properties of stored zip files for export
    *      responses:
    *        200:
-   *          description: export cache info
+   *          description: the zip file statuses
+   *          content:
+   *            application/json:
+   *              schema:
+   *                properties:
+   *                  zipFileStats:
+   *                    type: array
+   *                    items:
+   *                      type: object
+   *                      description: the property of each file
    */
-  router.get('/', async(req, res) => {
-    const files = exportService.getStatus();
+  router.get('/status', async(req, res) => {
+    const zipFileStats = await exportService.getStatus();
 
     // TODO: use res.apiv3
-    return res.json({ ok: true, files });
+    return res.json({ ok: true, zipFileStats });
   });
 
   /**
    * @swagger
    *
-   *  /export/pages:
-   *    get:
+   *  /export:
+   *    post:
    *      tags: [Export]
-   *      description: download a zipped json for page collection
+   *      description: generate zipped jsons for collections
    *      responses:
    *        200:
-   *          description: a zip file
+   *          description: a zip file is generated
+   *          content:
+   *            application/json:
+   *              schema:
+   *                properties:
+   *                  zipFileStat:
+   *                    type: object
+   *                    description: the property of the zip file
    */
-  router.get('/pages', async(req, res) => {
-    // TODO: rename path to "/:collection" and add express validator
+  router.post('/', async(req, res) => {
+    // TODO: add express validator
     try {
-      const file = exportService.getZipFile(Page);
+      const { collections } = req.body;
+      // get model for collection
+      const models = collections.map(collectionName => growiBridgeService.getModelFromCollectionName(collectionName));
+
+      const [metaJson, jsonFiles] = await Promise.all([
+        exportService.createMetaJson(),
+        exportService.exportMultipleCollectionsToJsons(models),
+      ]);
 
-      if (file == null) {
-        throw new Error('the target file does not exist');
-      }
+      // zip json
+      const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
+      // add meta.json in zip
+      configs.push({ from: metaJson, as: path.basename(metaJson) });
+      // exec zip
+      const zipFile = await exportService.zipFiles(configs);
+      // get stats for the zip file
+      const zipFileStat = await growiBridgeService.parseZipFile(zipFile);
 
-      return res.download(file);
+      // TODO: use res.apiv3
+      return res.status(200).json({
+        ok: true,
+        zipFileStat,
+      });
     }
     catch (err) {
       // TODO: use ApiV3Error
@@ -67,56 +99,42 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
-   *    post:
+   *  /export/{fileName}:
+   *    delete:
    *      tags: [Export]
-   *      description: generate a zipped json for page collection
+   *      description: delete the file
+   *      parameters:
+   *        - name: fileName
+   *          in: path
+   *          description: the file name of zip file
+   *          required: true
+   *          schema:
+   *            type: string
    *      responses:
    *        200:
-   *          description: a zip file is generated
+   *          description: the file is deleted
+   *          content:
+   *            application/json:
+   *              schema:
+   *                type: object
    */
-  router.post('/pages', async(req, res) => {
-    // TODO: rename path to "/:collection" and add express validator
+  router.delete('/:fileName', async(req, res) => {
+    // TODO: add express validator
+    const { fileName } = req.params;
+
     try {
-      const file = await exportService.exportCollection(Page);
+      const zipFile = exportService.getFile(fileName);
+      fs.unlinkSync(zipFile);
+
       // TODO: use res.apiv3
-      return res.status(200).json({
-        ok: true,
-        collection: [Page.collection.collectionName],
-        file: path.basename(file),
-      });
+      return res.status(200).send({ ok: true });
     }
     catch (err) {
       // TODO: use ApiV3Error
       logger.error(err);
-      return res.status(500).send({ status: 'ERROR' });
+      return res.status(500).send({ ok: false });
     }
   });
 
-  /**
-   * @swagger
-   *
-   *  /export/pages:
-   *    delete:
-   *      tags: [Export]
-   *      description: unlink a json and zip file for page collection
-   *      responses:
-   *        200:
-   *          description: the json and zip file are removed
-   */
-  // router.delete('/pages', async(req, res) => {
-  //   // TODO: rename path to "/:collection" and add express validator
-  //   try {
-  //     // remove .json and .zip for collection
-  //     // TODO: use res.apiv3
-  //     return res.status(200).send({ status: 'DONE' });
-  //   }
-  //   catch (err) {
-  //     // TODO: use ApiV3Error
-  //     logger.error(err);
-  //     return res.status(500).send({ status: 'ERROR' });
-  //   }
-  // });
-
   return router;
 };

src/server/routes/apiv3/import.js

@@ -0,0 +1,243 @@
+const loggerFactory = require('@alias/logger');
+
+const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-line no-unused-vars
+const path = require('path');
+const fs = require('fs');
+const multer = require('multer');
+const { ObjectId } = require('mongoose').Types;
+
+const express = require('express');
+
+const router = express.Router();
+
+/**
+ * @swagger
+ *  tags:
+ *    name: Import
+ */
+
+module.exports = (crowi) => {
+  const { growiBridgeService, importService } = crowi;
+  const uploads = multer({
+    storage: multer.diskStorage({
+      destination: (req, file, cb) => {
+        cb(null, importService.baseDir);
+      },
+      filename(req, file, cb) {
+        // to prevent hashing the file name. files with same name will be overwritten.
+        cb(null, file.originalname);
+      },
+    }),
+    fileFilter: (req, file, cb) => {
+      if (path.extname(file.originalname) === '.zip') {
+        return cb(null, true);
+      }
+      cb(new Error('Only ".zip" is allowed'));
+    },
+  });
+
+  /**
+   * defined overwrite params for each collection
+   * all imported documents are overwriten by this value
+   * each value can be any value or a function (_value, { _document, key, schema }) { return newValue }
+   *
+   * @param {object} Model instance of mongoose model
+   * @param {object} req request object
+   * @return {object} document to be persisted
+   */
+  const overwriteParamsFn = async(Model, schema, req) => {
+    const { collectionName } = Model.collection;
+
+    /* eslint-disable no-case-declarations */
+    switch (Model.collection.collectionName) {
+      case 'pages':
+        // TODO: use schema and req to generate overwriteParams
+        // e.g. { creator: schema.creator === 'me' ? ObjectId(req.user._id) : importService.keepOriginal }
+        return {
+          status: 'published', // FIXME when importing users and user groups
+          grant: 1, // FIXME when importing users and user groups
+          grantedUsers: [], // FIXME when importing users and user groups
+          grantedGroup: null, // FIXME when importing users and user groups
+          creator: ObjectId(req.user._id), // FIXME when importing users
+          lastUpdateUser: ObjectId(req.user._id), // FIXME when importing users
+          liker: [], // FIXME when importing users
+          seenUsers: [], // FIXME when importing users
+          commentCount: 0, // FIXME when importing comments
+          extended: {}, // FIXME when ?
+          pageIdOnHackmd: undefined, // FIXME when importing hackmd?
+          revisionHackmdSynced: undefined, // FIXME when importing hackmd?
+          hasDraftOnHackmd: undefined, // FIXME when importing hackmd?
+        };
+      // case 'revisoins':
+      //   return {};
+      // case 'users':
+      //   return {};
+      // ... add more cases
+      default:
+        throw new Error(`cannot find a model for collection name "${collectionName}"`);
+    }
+    /* eslint-enable no-case-declarations */
+  };
+
+  /**
+   * @swagger
+   *
+   *  /import:
+   *    post:
+   *      tags: [Import]
+   *      description: import a collection from a zipped json
+   *      responses:
+   *        200:
+   *          description: the data is successfully imported
+   *          content:
+   *            application/json:
+   *              schema:
+   *                properties:
+   *                  results:
+   *                    type: array
+   *                    items:
+   *                      type: object
+   *                      description: collectionName, insertedIds, failedIds
+   */
+  router.post('/', async(req, res) => {
+    // TODO: add express validator
+
+    const { fileName, collections, schema } = req.body;
+    const zipFile = importService.getFile(fileName);
+
+    // unzip
+    await importService.unzip(zipFile);
+    // eslint-disable-next-line no-unused-vars
+    const { meta, fileStats } = await growiBridgeService.parseZipFile(zipFile);
+
+    // delete zip file after unzipping and parsing it
+    fs.unlinkSync(zipFile);
+
+    // filter fileStats
+    const filteredFileStats = fileStats.filter(({ fileName, collectionName, size }) => { return collections.includes(collectionName) });
+
+    try {
+      // validate with meta.json
+      importService.validate(meta);
+
+      const results = await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
+        const Model = growiBridgeService.getModelFromCollectionName(collectionName);
+        const jsonFile = importService.getFile(fileName);
+
+        let overwriteParams;
+        if (overwriteParamsFn[collectionName] != null) {
+          // await in case overwriteParamsFn[collection] is a Promise
+          overwriteParams = await overwriteParamsFn(Model, schema[collectionName], req);
+        }
+
+        const { insertedIds, failedIds } = await importService.import(Model, jsonFile, overwriteParams);
+
+        return {
+          collectionName,
+          insertedIds,
+          failedIds,
+        };
+      }));
+
+      // TODO: use res.apiv3
+      return res.send({ ok: true, results });
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
+
+  /**
+   * @swagger
+   *
+   *  /import/upload:
+   *    post:
+   *      tags: [Import]
+   *      description: upload a zip file
+   *      responses:
+   *        200:
+   *          description: the file is uploaded
+   *          content:
+   *            application/json:
+   *              schema:
+   *                properties:
+   *                  meta:
+   *                    type: object
+   *                    description: the meta data of the uploaded file
+   *                  fileName:
+   *                    type: string
+   *                    description: the base name of the uploaded file
+   *                  fileStats:
+   *                    type: array
+   *                    items:
+   *                      type: object
+   *                      description: the property of each extracted file
+   */
+  router.post('/upload', uploads.single('file'), async(req, res) => {
+    const { file } = req;
+    const zipFile = importService.getFile(file.filename);
+
+    try {
+      const data = await growiBridgeService.parseZipFile(zipFile);
+
+      // validate with meta.json
+      importService.validate(data.meta);
+
+      // TODO: use res.apiv3
+      return res.send({
+        ok: true,
+        data,
+      });
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
+
+  /**
+   * @swagger
+   *
+   *  /import/{fileName}:
+   *    post:
+   *      tags: [Import]
+   *      description: delete a zip file
+   *      parameters:
+   *        - name: fileName
+   *          in: path
+   *          description: the file name of zip file
+   *          required: true
+   *          schema:
+   *            type: string
+   *      responses:
+   *        200:
+   *          description: the file is deleted
+   *          content:
+   *            application/json:
+   *              schema:
+   *                type: object
+   */
+  router.delete('/:fileName', async(req, res) => {
+    const { fileName } = req.params;
+
+    try {
+      const zipFile = importService.getFile(fileName);
+      fs.unlinkSync(zipFile);
+
+      // TODO: use res.apiv3
+      return res.send({
+        ok: true,
+      });
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
+
+  return router;
+};

src/server/routes/apiv3/index.js

@@ -9,7 +9,11 @@ const router = express.Router();
 module.exports = (crowi) => {
   router.use('/healthcheck', require('./healthcheck')(crowi));
 
+  router.use('/mongo', require('./mongo')(crowi));
+
   router.use('/export', require('./export')(crowi));
 
+  router.use('/import', require('./import')(crowi));
+
   return router;
 };

src/server/routes/apiv3/mongo.js

@@ -0,0 +1,46 @@
+const loggerFactory = require('@alias/logger');
+
+const logger = loggerFactory('growi:routes:apiv3:mongo'); // eslint-disable-line no-unused-vars
+
+const express = require('express');
+
+const router = express.Router();
+
+/**
+ * @swagger
+ *  tags:
+ *    name: Mongo
+ */
+
+module.exports = (crowi) => {
+  /**
+   * @swagger
+   *
+   *  /mongo/collections:
+   *    get:
+   *      tags: [Mongo]
+   *      description: get mongodb collections names
+   *      responses:
+   *        200:
+   *          description: a list of collections in mongoDB
+   *          content:
+   *            application/json:
+   *              schema:
+   *                properties:
+   *                  collections:
+   *                    type: array
+   *                    items:
+   *                      type: string
+   */
+  router.get('/collections', async(req, res) => {
+    const collections = Object.values(crowi.models).map(model => model.collection.collectionName);
+
+    // TODO: use res.apiv3
+    return res.json({
+      ok: true,
+      collections: [...new Set(collections)], // remove duplicates
+    });
+  });
+
+  return router;
+};

src/server/routes/index.js

@@ -165,6 +165,7 @@ module.exports = function(crowi, app) {
 
   // export management for admin
   app.get('/admin/export' , loginRequired() , adminRequired ,admin.export.index);
+  app.get('/admin/export/:fileName' , loginRequired() , adminRequired ,admin.export.download);
 
   app.get('/me'                       , loginRequired() , me.index);
   app.get('/me/password'              , loginRequired() , me.password);

src/server/service/export.js

@@ -3,53 +3,59 @@ const fs = require('fs');
 const path = require('path');
 const streamToPromise = require('stream-to-promise');
 const archiver = require('archiver');
+const toArrayIfNot = require('../../lib/util/toArrayIfNot');
 
 class ExportService {
 
   constructor(crowi) {
+    this.crowi = crowi;
+    this.appService = crowi.appService;
+    this.growiBridgeService = crowi.growiBridgeService;
+    this.getFile = this.growiBridgeService.getFile.bind(this);
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
-    this.extension = 'json';
-    this.encoding = 'utf-8';
     this.per = 100;
     this.zlibLevel = 9; // 0(min) - 9(max)
+  }
 
-    this.files = {};
-    // populate this.files
-    // this.files = {
-    //   configs: path.join(this.baseDir, 'configs.json'),
-    //   pages: path.join(this.baseDir, 'pages.json'),
-    //   pagetagrelations: path.join(this.baseDir, 'pagetagrelations.json'),
-    //   ...
-    // };
-    // TODO: handle 3 globalnotificationsettings collection properly
-    // see Object.values(crowi.models).forEach((m) => { return console.log(m.collection.collectionName) });
-    Object.values(crowi.models).forEach((m) => {
-      const name = m.collection.collectionName;
-      this.files[name] = path.join(this.baseDir, `${name}.${this.extension}`);
-    });
+  /**
+   * parse all zip files in downloads dir
+   *
+   * @memberOf ExportService
+   * @return {Array.<object>} info for zip files
+   */
+  async getStatus() {
+    const zipFiles = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' });
+    const zipFileStats = await Promise.all(zipFiles.map((file) => {
+      const zipFile = this.getFile(file);
+      return this.growiBridgeService.parseZipFile(zipFile);
+    }));
+
+    return zipFileStats;
   }
 
   /**
-   * dump a collection into json
+   * create meta.json
    *
    * @memberOf ExportService
-   * @return {object} cache info for exported zip files
+   * @return {string} path to meta.json
    */
-  getStatus() {
-    const status = {};
-    const collections = Object.keys(this.files);
-    collections.forEach((file) => {
-      status[path.basename(file, '.zip')] = null;
-    });
+  async createMetaJson() {
+    const metaJson = path.join(this.baseDir, this.growiBridgeService.getMetaFileName());
+    const writeStream = fs.createWriteStream(metaJson, { encoding: this.growiBridgeService.getEncoding() });
+
+    const metaData = {
+      version: this.crowi.version,
+      url: this.appService.getSiteUrl(),
+      passwordSeed: this.crowi.env.PASSWORD_SEED,
+      exportedAt: new Date(),
+    };
 
-    // extract ${collectionName}.zip
-    const files = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' && collections.includes(path.basename(file, '.zip')) });
+    writeStream.write(JSON.stringify(metaData));
+    writeStream.close();
 
-    files.forEach((file) => {
-      status[path.basename(file, '.zip')] = file;
-    });
+    await streamToPromise(writeStream);
 
-    return status;
+    return metaJson;
   }
 
   /**
@@ -58,29 +64,32 @@ class ExportService {
    * @memberOf ExportService
    * @param {string} file path to json file to be written
    * @param {readStream} readStream  read stream
-   * @param {number} [total] number of target items (optional)
+   * @param {number} total number of target items (optional)
+   * @param {function} [getLogText] (n, total) => { ... }
+   * @return {string} path to the exported json file
    */
-  async export(file, readStream, total) {
+  async export(writeStream, readStream, total, getLogText) {
     let n = 0;
-    const ws = fs.createWriteStream(file, { encoding: this.encoding });
 
     // open an array
-    ws.write('[');
+    writeStream.write('[');
 
     readStream.on('data', (chunk) => {
-      if (n !== 0) ws.write(',');
-      ws.write(JSON.stringify(chunk));
+      if (n !== 0) writeStream.write(',');
+      writeStream.write(JSON.stringify(chunk));
       n++;
-      this.logProgress(n, total);
+      this.logProgress(n, total, getLogText);
     });
 
     readStream.on('end', () => {
       // close the array
-      ws.write(']');
-      ws.close();
+      writeStream.write(']');
+      writeStream.close();
     });
 
     await streamToPromise(readStream);
+
+    return writeStream.path;
   }
 
   /**
@@ -90,19 +99,30 @@ class ExportService {
    * @param {object} Model instance of mongoose model
    * @return {string} path to zip file
    */
-  async exportCollection(Model) {
-    const modelName = Model.collection.collectionName;
-    const file = this.files[modelName];
-    const total = await Model.countDocuments();
+  async exportCollectionToJson(Model) {
+    const { collectionName } = Model.collection;
+    const jsonFileToWrite = path.join(this.baseDir, `${collectionName}.json`);
+    const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() });
     const readStream = Model.find().cursor();
+    const total = await Model.countDocuments();
+    const getLogText = (n, total) => `${collectionName}: ${n}/${total} written`;
 
-    await this.export(file, readStream, total);
+    const jsonFileWritten = await this.export(writeStream, readStream, total, getLogText);
 
-    const { file: zipFile, size } = await this.zipSingleFile(file);
+    return jsonFileWritten;
+  }
 
-    logger.info(`exported ${modelName} collection into ${zipFile} (${size} bytes)`);
+  /**
+   * export multiple collections
+   *
+   * @memberOf ExportService
+   * @param {Array.<object>} models array of instances of mongoose model
+   * @return {Array.<string>} paths to json files created
+   */
+  async exportMultipleCollectionsToJsons(models) {
+    const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));
 
-    return zipFile;
+    return jsonFiles;
   }
 
   /**
@@ -110,16 +130,11 @@ class ExportService {
    *
    * @memberOf ExportService
    * @param {number} n number of items exported
-   * @param {number} [total] number of target items (optional)
+   * @param {number} total number of target items (optional)
+   * @param {function} [getLogText] (n, total) => { ... }
    */
-  logProgress(n, total) {
-    let output;
-    if (total) {
-      output = `${n}/${total} written`;
-    }
-    else {
-      output = `${n} items written`;
-    }
+  logProgress(n, total, getLogText) {
+    const output = getLogText ? getLogText(n, total) : `${n}/${total} items written`;
 
     // output every this.per items
     if (n % this.per === 0) logger.debug(output);
@@ -128,21 +143,21 @@ class ExportService {
   }
 
   /**
-   * zip a file
+   * zip files into one zip file
    *
    * @memberOf ExportService
-   * @param {string} from path to input file
-   * @param {string} [to=`${path.join(path.dirname(from), `${path.basename(from, path.extname(from))}.zip`)}`] path to output file
-   * @param {string} [as=path.basename(from)] file name after unzipped
-   * @return {object} file path and file size
+   * @param {object|array<object>} configs object or array of object { from: "path to source file", as: "file name after unzipped" }
+   * @return {string} absolute path to the zip file
    * @see https://www.archiverjs.com/#quick-start
    */
-  async zipSingleFile(from, to = this.replaceExtension(from, 'zip'), as = path.basename(from)) {
+  async zipFiles(_configs) {
+    const configs = toArrayIfNot(_configs);
+    const appTitle = this.appService.getAppTitle();
+    const timeStamp = (new Date()).getTime();
+    const zipFile = path.join(this.baseDir, `${appTitle}-${timeStamp}.zip`);
     const archive = archiver('zip', {
       zlib: { level: this.zlibLevel },
     });
-    const input = fs.createReadStream(from);
-    const output = fs.createWriteStream(to);
 
     // good practice to catch warnings (ie stat failures and other non-blocking errors)
     archive.on('warning', (err) => {
@@ -153,8 +168,14 @@ class ExportService {
     // good practice to catch this error explicitly
     archive.on('error', (err) => { throw err });
 
-    // append a file from stream
-    archive.append(input, { name: as });
+    for (const { from, as } of configs) {
+      const input = fs.createReadStream(from);
+
+      // append a file from stream
+      archive.append(input, { name: as });
+    }
+
+    const output = fs.createWriteStream(zipFile);
 
     // pipe archive data to the file
     archive.pipe(output);
@@ -165,39 +186,14 @@ class ExportService {
 
     await streamToPromise(archive);
 
-    return {
-      file: to,
-      size: archive.pointer(),
-    };
-  }
+    logger.info(`zipped growi data into ${zipFile} (${archive.pointer()} bytes)`);
 
-  /**
-   * replace a file extension
-   *
-   * @memberOf ExportService
-   * @param {string} file file path
-   * @param {string} extension new extension
-   * @return {string} path to file with new extension
-   */
-  replaceExtension(file, extension) {
-    return `${path.join(path.dirname(file), `${path.basename(file, path.extname(file))}.${extension}`)}`;
-  }
-
-  /**
-   * get the path to the zipped file for a collection
-   *
-   * @memberOf ExportService
-   * @param {object} Model instance of mongoose model
-   * @return {string} path to zip file
-   */
-  getZipFile(Model) {
-    const json = this.files[Model.collection.collectionName];
-    const zip = this.replaceExtension(json, 'zip');
-    if (!fs.existsSync(zip)) {
-      return null;
+    // delete json files
+    for (const { from } of configs) {
+      fs.unlinkSync(from);
     }
 
-    return zip;
+    return zipFile;
   }
 
 }

src/server/service/growi-bridge.js

@@ -0,0 +1,134 @@
+const logger = require('@alias/logger')('growi:services:GrowiBridgeService'); // eslint-disable-line no-unused-vars
+const fs = require('fs');
+const path = require('path');
+const streamToPromise = require('stream-to-promise');
+const unzipper = require('unzipper');
+
+/**
+ * the service class for bridging GROWIs (export and import)
+ * common properties and methods between export service and import service are defined in this service
+ */
+class GrowiBridgeService {
+
+  constructor(crowi) {
+    this.encoding = 'utf-8';
+    this.metaFileName = 'meta.json';
+
+    // { pages: Page, users: User, ... }
+    this.collectionMap = {};
+    this.initCollectionMap(crowi.models);
+  }
+
+  /**
+   * initialize collection map
+   *
+   * @memberOf GrowiBridgeService
+   * @param {object} models from models/index.js
+   */
+  initCollectionMap(models) {
+    for (const model of Object.values(models)) {
+      this.collectionMap[model.collection.collectionName] = model;
+    }
+  }
+
+  /**
+   * getter for encoding
+   *
+   * @memberOf GrowiBridgeService
+   * @return {string} encoding
+   */
+  getEncoding() {
+    return this.encoding;
+  }
+
+  /**
+   * getter for metaFileName
+   *
+   * @memberOf GrowiBridgeService
+   * @return {string} base name of meta file
+   */
+  getMetaFileName() {
+    return this.metaFileName;
+  }
+
+  /**
+   * get a model from collection name
+   *
+   * @memberOf GrowiBridgeService
+   * @param {string} collectionName collection name
+   * @return {object} instance of mongoose model
+   */
+  getModelFromCollectionName(collectionName) {
+    const Model = this.collectionMap[collectionName];
+
+    if (Model == null) {
+      throw new Error(`cannot find a model for collection name "${collectionName}"`);
+    }
+
+    return Model;
+  }
+
+  /**
+   * get the absolute path to a file
+   * this method must must be bound to the caller (this.baseDir is undefined in this service)
+   *
+   * @memberOf GrowiBridgeService
+   * @param {string} fileName base name of file
+   * @return {string} absolute path to the file
+   */
+  getFile(fileName) {
+    if (this.baseDir == null) {
+      throw new Error('baseDir is not defined');
+    }
+
+    const jsonFile = path.join(this.baseDir, fileName);
+
+    // throws err if the file does not exist
+    fs.accessSync(jsonFile);
+
+    return jsonFile;
+  }
+
+  /**
+   * parse a zip file
+   *
+   * @memberOf GrowiBridgeService
+   * @param {string} zipFile path to zip file
+   * @return {object} meta{object} and files{Array.<object>}
+   */
+  async parseZipFile(zipFile) {
+    const readStream = fs.createReadStream(zipFile);
+    const unzipStream = readStream.pipe(unzipper.Parse());
+    const fileStats = [];
+    let meta = {};
+
+    unzipStream.on('entry', async(entry) => {
+      const fileName = entry.path;
+      const size = entry.vars.uncompressedSize; // There is also compressedSize;
+
+      if (fileName === this.getMetaFileName()) {
+        meta = JSON.parse((await entry.buffer()).toString());
+      }
+      else {
+        fileStats.push({
+          fileName,
+          collectionName: path.basename(fileName, '.json'),
+          size,
+        });
+      }
+
+      entry.autodrain();
+    });
+
+    await streamToPromise(unzipStream);
+
+    return {
+      meta,
+      fileName: path.basename(zipFile),
+      fileStats,
+    };
+  }
+
+}
+
+module.exports = GrowiBridgeService;

src/server/service/import.js

@@ -0,0 +1,269 @@
+const logger = require('@alias/logger')('growi:services:ImportService'); // eslint-disable-line no-unused-vars
+const fs = require('fs');
+const path = require('path');
+const JSONStream = require('JSONStream');
+const streamToPromise = require('stream-to-promise');
+const unzipper = require('unzipper');
+const { ObjectId } = require('mongoose').Types;
+
+class ImportService {
+
+  constructor(crowi) {
+    this.crowi = crowi;
+    this.growiBridgeService = crowi.growiBridgeService;
+    this.getFile = this.growiBridgeService.getFile.bind(this);
+    this.baseDir = path.join(crowi.tmpDir, 'imports');
+    this.per = 100;
+    this.keepOriginal = this.keepOriginal.bind(this);
+
+    // { pages: { _id: ..., path: ..., ...}, users: { _id: ..., username: ..., }, ... }
+    this.convertMap = {};
+    this.initConvertMap(crowi.models);
+  }
+
+  /**
+   * initialize convert map. set keepOriginal as default
+   *
+   * @memberOf ImportService
+   * @param {object} models from models/index.js
+   */
+  initConvertMap(models) {
+    // by default, original value is used for imported documents
+    for (const model of Object.values(models)) {
+      const { collectionName } = model.collection;
+      this.convertMap[collectionName] = {};
+
+      for (const key of Object.keys(model.schema.paths)) {
+        this.convertMap[collectionName][key] = this.keepOriginal;
+      }
+    }
+  }
+
+  /**
+   * keep original value
+   * automatically convert ObjectId
+   *
+   * @memberOf ImportService
+   * @param {any} _value value from imported document
+   * @param {{ _document: object, schema: object, key: string }}
+   * @return {any} new value for the document
+   */
+  keepOriginal(_value, { _document, schema, key }) {
+    let value;
+    if (schema[key].instance === 'ObjectID' && ObjectId.isValid(_value)) {
+      value = ObjectId(_value);
+    }
+    else {
+      value = _value;
+    }
+
+    return value;
+  }
+
+  /**
+   * import a collection from json
+   *
+   * @memberOf ImportService
+   * @param {object} Model instance of mongoose model
+   * @param {string} jsonFile absolute path to the jsonFile being imported
+   * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
+   * @return {insertedIds: Array.<string>, failedIds: Array.<string>}
+   */
+  async import(Model, jsonFile, overwriteParams = {}) {
+    // streamToPromise(jsonStream) throws an error, use new Promise instead
+    return new Promise((resolve, reject) => {
+      const { collectionName } = Model.collection;
+
+      let counter = 0;
+      let insertedIds = [];
+      let failedIds = [];
+      let unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+
+      const readStream = fs.createReadStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
+      const jsonStream = readStream.pipe(JSONStream.parse('*'));
+
+      jsonStream.on('data', async(document) => {
+        // documents are not persisted until unorderedBulkOp.execute()
+        unorderedBulkOp.insert(this.convertDocuments(Model, document, overwriteParams));
+
+        counter++;
+
+        if (counter % this.per === 0) {
+          // puase jsonStream to prevent more items to be added to unorderedBulkOp
+          jsonStream.pause();
+
+          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
+          insertedIds = [...insertedIds, ..._insertedIds];
+          failedIds = [...failedIds, ..._failedIds];
+
+          // reset initializeUnorderedBulkOp
+          unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+
+          // resume jsonStream
+          jsonStream.resume();
+        }
+      });
+
+      jsonStream.on('end', async(data) => {
+        // insert the rest. avoid errors when unorderedBulkOp has no items
+        if (unorderedBulkOp.s.currentBatch !== null) {
+          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
+          insertedIds = [...insertedIds, ..._insertedIds];
+          failedIds = [...failedIds, ..._failedIds];
+        }
+
+        logger.info(`Done. Inserted ${insertedIds.length} ${collectionName}.`);
+
+        if (failedIds.length > 0) {
+          logger.error(`Failed to insert ${failedIds.length} ${collectionName}: ${failedIds.join(', ')}.`);
+        }
+
+        // clean up tmp directory
+        fs.unlinkSync(jsonFile);
+
+        return resolve({
+          insertedIds,
+          failedIds,
+        });
+      });
+    });
+  }
+
+  /**
+   * extract a zip file
+   *
+   * @memberOf ImportService
+   * @param {string} zipFile absolute path to zip file
+   * @return {Array.<string>} array of absolute paths to extracted files
+   */
+  async unzip(zipFile) {
+    const readStream = fs.createReadStream(zipFile);
+    const unzipStream = readStream.pipe(unzipper.Parse());
+    const files = [];
+
+    unzipStream.on('entry', (entry) => {
+      const fileName = entry.path;
+
+      if (fileName === this.growiBridgeService.getMetaFileName()) {
+        // skip meta.json
+        entry.autodrain();
+      }
+      else {
+        const jsonFile = path.join(this.baseDir, fileName);
+        const writeStream = fs.createWriteStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
+        entry.pipe(writeStream);
+        files.push(jsonFile);
+      }
+    });
+
+    await streamToPromise(unzipStream);
+
+    return files;
+  }
+
+  /**
+   * execute unorderedBulkOp and ignore errors
+   *
+   * @memberOf ImportService
+   * @param {object} unorderedBulkOp result of Model.collection.initializeUnorderedBulkOp()
+   * @return {{nInserted: number, failed: Array.<string>}} number of docuemnts inserted and failed
+   */
+  async execUnorderedBulkOpSafely(unorderedBulkOp) {
+    // keep the number of documents inserted and failed for logger
+    let insertedIds = [];
+    let failedIds = [];
+
+    // try catch to skip errors
+    try {
+      const log = await unorderedBulkOp.execute();
+      const _insertedIds = log.result.insertedIds.map(op => op._id);
+      insertedIds = [...insertedIds, ..._insertedIds];
+    }
+    catch (err) {
+      const collectionName = unorderedBulkOp.s.namespace;
+
+      for (const error of err.result.result.writeErrors) {
+        logger.error(`${collectionName}: ${error.errmsg}`);
+      }
+
+      const _failedIds = err.result.result.writeErrors.map(err => err.err.op._id);
+      const _insertedIds = err.result.result.insertedIds.filter(op => !_failedIds.includes(op._id)).map(op => op._id);
+
+      failedIds = [...failedIds, ..._failedIds];
+      insertedIds = [...insertedIds, ..._insertedIds];
+    }
+
+    logger.debug(`Importing ${unorderedBulkOp.s.collection.s.name}. Inserted: ${insertedIds.length}. Failed: ${failedIds.length}.`);
+
+    return {
+      insertedIds,
+      failedIds,
+    };
+  }
+
+  /**
+   * execute unorderedBulkOp and ignore errors
+   *
+   * @memberOf ImportService
+   * @param {object} Model instance of mongoose model
+   * @param {object} _document document being imported
+   * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
+   * @return {object} document to be persisted
+   */
+  convertDocuments(Model, _document, overwriteParams) {
+    const collectionName = Model.collection.collectionName;
+    const schema = Model.schema.paths;
+    const convertMap = this.convertMap[collectionName];
+
+    if (convertMap == null) {
+      throw new Error(`attribute map is not defined for ${collectionName}`);
+    }
+
+    const document = {};
+
+    // assign value from documents being imported
+    for (const entry of Object.entries(convertMap)) {
+      const [key, value] = entry;
+
+      // distinguish between null and undefined
+      if (_document[key] === undefined) {
+        continue; // next entry
+      }
+
+      document[key] = (typeof value === 'function') ? value(_document[key], { _document, key, schema }) : value;
+    }
+
+    // overwrite documents with custom values
+    for (const entry of Object.entries(overwriteParams)) {
+      const [key, value] = entry;
+
+      // distinguish between null and undefined
+      if (_document[key] !== undefined) {
+        document[key] = (typeof value === 'function') ? value(_document[key], { _document, key, schema }) : value;
+      }
+    }
+
+    return document;
+  }
+
+  /**
+   * validate using meta.json
+   * to pass validation, all the criteria must be met
+   *   - ${version of this growi} === ${version of growi that exported data}
+   *
+   * @memberOf ImportService
+   * @param {object} meta meta data from meta.json
+   */
+  validate(meta) {
+    if (meta.version !== this.crowi.version) {
+      throw new Error('the version of this growi and the growi that exported the data are not met');
+    }
+
+    // TODO: check if all migrations are completed
+    // - export: throw err if there are pending migrations
+    // - import: throw err if there are pending migrations
+  }
+
+}
+
+module.exports = ImportService;

src/server/views/admin/importer.html

@@ -39,6 +39,9 @@
       </div>
       {% endif %}
 
+      <!-- TODO: move to /imponents/Admin/Importer.jsx -->
+      <div id="growi-import"></div>
+
       <!-- esa Importer management forms -->
       <form action="/_api/admin/settings/importerEsa" method="post" class="form-horizontal" id="importerSettingFormEsa" role="form"
           data-success-messaage="{{ ('Updated') }}">

tmp/downloads/.keep

@@ -0,0 +1 @@
+

tmp/imports/.keep

@@ -0,0 +1 @@
+

yarn.lock

@@ -1263,6 +1263,14 @@
   resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
   integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==
 
+JSONStream@^1.3.5:
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0"
+  integrity sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==
+  dependencies:
+    jsonparse "^1.2.0"
+    through ">=2.2.7 <3"
+
 abab@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.0.tgz#aba0ab4c5eee2d4c79d3487d85450fb2376ebb0f"
@@ -1965,6 +1973,11 @@ bfj@^6.1.1:
     hoopy "^0.1.2"
     tryer "^1.0.0"
 
+big-integer@^1.6.17:
+  version "1.6.44"
+  resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.44.tgz#4ee9ae5f5839fc11ade338fea216b4513454a539"
+  integrity sha512-7MzElZPTyJ2fNvBkPxtFQ2fWIkVmuzw41+BZHSzpEq3ymB2MfeKp1+yXl/tS75xCx+WnyV+yb0kp+K1C3UNwmQ==
+
 big.js@^3.1.3:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e"
@@ -1983,6 +1996,14 @@ binary-extensions@^1.0.0:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205"
 
+binary@~0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79"
+  integrity sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=
+  dependencies:
+    buffers "~0.1.1"
+    chainsaw "~0.1.0"
+
 bl@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.0.tgz#3611ec00579fd18561754360b21e9f784500ff88"
@@ -2013,6 +2034,11 @@ bluebird@^3.5.5:
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.5.tgz#a8d0afd73251effbbd5fe384a77d73003c17a71f"
   integrity sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==
 
+bluebird@~3.4.1:
+  version "3.4.7"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3"
+  integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=
+
 bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.4.0:
   version "4.11.8"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f"
@@ -2288,6 +2314,11 @@ buffer-from@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
 
+buffer-indexof-polyfill@~1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz#a9fb806ce8145d5428510ce72f278bb363a638bf"
+  integrity sha1-qfuAbOgUXVQoUQznLyeLs2OmOL8=
+
 buffer-xor@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9"
@@ -2308,6 +2339,11 @@ buffer@^5.1.0:
     base64-js "^1.0.2"
     ieee754 "^1.1.4"
 
+buffers@~0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb"
+  integrity sha1-skV5w77U1tOWru5tmorn9Ugqt7s=
+
 builtin-modules@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f"
@@ -2508,6 +2544,13 @@ chain-function@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/chain-function/-/chain-function-1.0.0.tgz#0d4ab37e7e18ead0bdc47b920764118ce58733dc"
 
+chainsaw@~0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98"
+  integrity sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=
+  dependencies:
+    traverse ">=0.3.0 <0.4"
+
 chalk@2.4.2, chalk@^2.0.1, chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -3805,6 +3848,13 @@ dtrace-provider@~0.8:
   dependencies:
     nan "^2.3.3"
 
+duplexer2@~0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1"
+  integrity sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=
+  dependencies:
+    readable-stream "^2.0.2"
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@@ -5059,6 +5109,16 @@ fstream@^1.0.0, fstream@^1.0.2:
     mkdirp ">=0.5 0"
     rimraf "2"
 
+fstream@^1.0.12:
+  version "1.0.12"
+  resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045"
+  integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==
+  dependencies:
+    graceful-fs "^4.1.2"
+    inherits "~2.0.0"
+    mkdirp ">=0.5 0"
+    rimraf "2"
+
 function-bind@^1.0.2, function-bind@^1.1.0, function-bind@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
@@ -5363,7 +5423,7 @@ graceful-fs@^4.1.15:
   version "4.1.15"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
 
-graceful-fs@^4.2.0:
+graceful-fs@^4.2.0, graceful-fs@^4.2.2:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.2.tgz#6f0952605d0140c1cfdb138ed005775b92d67b02"
   integrity sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==
@@ -6998,6 +7058,11 @@ jsonify@~0.0.0:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
 
+jsonparse@^1.2.0:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"
+  integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=
+
 jsonpointer@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-4.0.1.tgz#4fd92cb34e0e9db3c89c8622ecf51f9b978c6cb9"
@@ -7207,6 +7272,11 @@ linkify-it@^2.0.0:
   dependencies:
     uc.micro "^1.0.1"
 
+listenercount@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/listenercount/-/listenercount-1.0.1.tgz#84c8a72ab59c4725321480c975e6508342e70937"
+  integrity sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=
+
 load-css-file@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/load-css-file/-/load-css-file-1.0.0.tgz#dac097ead6470f4c3f23d4bc5b9ff2c3decb212f"
@@ -10291,7 +10361,7 @@ read-pkg@^3.0.0:
     normalize-package-data "^2.3.2"
     path-type "^3.0.0"
 
-"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.1.5, readable-stream@^2.3.6:
+"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.1.5, readable-stream@^2.3.6, readable-stream@~2.3.6:
   version "2.3.6"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
   dependencies:
@@ -11172,7 +11242,7 @@ set-value@^2.0.0:
     is-plain-object "^2.0.3"
     split-string "^3.0.1"
 
-setimmediate@^1.0.4, setimmediate@^1.0.5:
+setimmediate@^1.0.4, setimmediate@^1.0.5, setimmediate@~1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
 
@@ -12250,7 +12320,7 @@ through2@^2.0.0:
     readable-stream "^2.1.5"
     xtend "~4.0.1"
 
-through@2, through@^2.3.6, through@~2.3, through@~2.3.1:
+through@2, "through@>=2.2.7 <3", through@^2.3.6, through@~2.3, through@~2.3.1:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
 
@@ -12353,6 +12423,11 @@ tr46@^1.0.1:
   dependencies:
     punycode "^2.1.0"
 
+"traverse@>=0.3.0 <0.4":
+  version "0.3.9"
+  resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9"
+  integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=
+
 trim-newlines@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"
@@ -12649,6 +12724,22 @@ unzip-response@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97"
 
+unzipper@^0.10.5:
+  version "0.10.5"
+  resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.10.5.tgz#4d189ae6f8af634b26efe1a1817c399e0dd4a1a0"
+  integrity sha512-i5ufkXNjWZYxU/0nKKf6LkvW8kn9YzRvfwuPWjXP+JTFce/8bqeR0gEfbiN2IDdJa6ZU6/2IzFRLK0z1v0uptw==
+  dependencies:
+    big-integer "^1.6.17"
+    binary "~0.3.0"
+    bluebird "~3.4.1"
+    buffer-indexof-polyfill "~1.0.0"
+    duplexer2 "~0.1.4"
+    fstream "^1.0.12"
+    graceful-fs "^4.2.2"
+    listenercount "~1.0.1"
+    readable-stream "~2.3.6"
+    setimmediate "~1.0.4"
+
 upath@^1.0.5, upath@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/upath/-/upath-1.1.0.tgz#35256597e46a581db4793d0ce47fa9aebfc9fabd"