Merge pull request #1317 from weseek/imprv/refactor-import

Imprv/refactor import

resource/locales/en-US/translation.json

@@ -763,6 +763,32 @@
         "at_least_one": "Select one or more collections.",
         "page_and_revision": "'Pages' and 'Revisions' must be imported both.",
         "depends": "'{{target}}' must be selected when '{{condition}}' is selected."
+      },
+      "configuration": {
+        "pages": {
+          "overwrite_author": {
+            "label": "Overwrite page's author with the current user",
+            "desc": "Recommended <span class=\"text-danger\">NOT</span> to check this when users will also be restored."
+          },
+          "set_public_to_page": {
+            "label": "Set 'Public' to the pages that is '{{from}}'",
+            "desc": "Make sure that this configuration makes all <b>'{{from}}'</b> pages readable from <span class=\"text-danger\">ANY users</span>."
+          },
+          "initialize_meta_datas": {
+            "label": "Initialize page's like, read users and comment count",
+            "desc": "Recommended <span class=\"text-danger\">NOT</span> to check this when users will also be restored."
+          },
+          "initialize_hackmd_related_datas": {
+            "label": "Initialize HackMD related data",
+            "desc": "Recommended to check this unless there is important drafts on HackMD."
+          }
+        },
+        "revisions": {
+          "overwrite_author": {
+            "label": "Overwrite revision's author with the current user",
+            "desc": "Recommended <span class=\"text-danger\">NOT</span> to check this when users will also be restored."
+          }
+        }
       }
     },
     "esa_settings": {

resource/locales/ja/translation.json

@@ -748,6 +748,32 @@
         "at_least_one": "コレクションが選択されていません",
         "page_and_revision": "'Pages' と 'Revisions' はセットでインポートする必要があります",
         "depends": "'{{condition}}' をインポートする場合は、'{{target}}' を一緒に選択する必要があります"
+      },
+      "configuration": {
+        "pages": {
+          "overwrite_author": {
+            "label": "ページ作成者を現在のユーザーで上書きする",
+            "desc": "users を同時に復元しない場合、このオプションは<span class=\"text-danger\">非推奨</span>です。"
+          },
+          "set_public_to_page": {
+            "label": "'{{from}}' 設定のページを '公開' 設定にする",
+            "desc": "全ての <b>'{{from}}'</b> 設定のページが<span class=\"text-danger\">全ユーザーから</span>読み取り可能になることに注意してください。"
+          },
+          "initialize_meta_datas": {
+            "label": "「いいね」「閲覧したユーザー」「コメント数」を初期化する",
+            "desc": "users を同時に復元しない場合、このオプションは<span class=\"text-danger\">非推奨</span>です。"
+          },
+          "initialize_hackmd_related_datas": {
+            "label": "HackMD 関連データを初期化する",
+            "desc": "HackMD に重要な下書きデータがない限りはこのオプションをチェックすることを推奨します。"
+          }
+        },
+        "revisions": {
+          "overwrite_author": {
+            "label": "リビジョン作成者を現在のユーザーで上書きする",
+            "desc": "users を同時に復元しない場合、このオプションは<span class=\"text-danger\">非推奨</span>です。"
+          }
+        }
       }
     },
     "esa_settings": {

src/client/js/components/Admin/ExportData/ZipFileTable.jsx

@@ -26,12 +26,12 @@ class ZipFileTable extends React.Component {
           </tr>
         </thead>
         <tbody>
-          {this.props.zipFileStats.map(({ meta, fileName, fileStats }) => {
+          {this.props.zipFileStats.map(({ meta, fileName, innerFileStats }) => {
             return (
               <tr key={fileName}>
                 <th>{fileName}</th>
                 <td>{meta.version}</td>
-                <td className="text-capitalize">{fileStats.map(fileStat => fileStat.collectionName).join(', ')}</td>
+                <td className="text-capitalize">{innerFileStats.map(fileStat => fileStat.collectionName).join(', ')}</td>
                 <td>{meta.exportedAt ? format(new Date(meta.exportedAt), 'yyyy/MM/dd HH:mm:ss') : ''}</td>
                 <td>
                   <ExportTableMenu

src/client/js/components/Admin/ExportDataPage.jsx

@@ -4,12 +4,13 @@ import { withTranslation } from 'react-i18next';
 import * as toastr from 'toastr';
 
 
-import ProgressBar from './Common/ProgressBar';
 import { createSubscribedElement } from '../UnstatedUtils';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
 import AppContainer from '../../services/AppContainer';
 import WebsocketContainer from '../../services/WebsocketContainer';
-// import { toastSuccess, toastError } from '../../../util/apiNotification';
 
+import ProgressBar from './Common/ProgressBar';
 
 import ExportZipFormModal from './ExportData/ExportZipFormModal';
 import ZipFileTable from './ExportData/ZipFileTable';

src/client/js/components/Admin/ImportData/ErrorViewer.jsx

@@ -0,0 +1,52 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import Modal from 'react-bootstrap/es/Modal';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+
+
+class ErrorViewer extends React.Component {
+
+  render() {
+    const { errors } = this.props;
+
+    let value = '(no errors)';
+    if (errors != null && errors.length > 0) {
+      const lines = errors.map((obj) => {
+        return JSON.stringify(obj);
+      });
+      value = lines.join('\n');
+    }
+
+    return (
+      <Modal show={this.props.isOpen} onHide={this.props.onClose}>
+        <Modal.Header closeButton className="bg-danger">
+          <Modal.Title className="text-white">Errors</Modal.Title>
+        </Modal.Header>
+        <Modal.Body>
+          <textarea className="form-control" rows="8" readOnly wrap="off" defaultValue={value}></textarea>
+        </Modal.Body>
+      </Modal>
+    );
+  }
+
+}
+
+ErrorViewer.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+
+  isOpen: PropTypes.bool.isRequired,
+  onClose: PropTypes.func.isRequired,
+
+  errors: PropTypes.arrayOf(PropTypes.object),
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ErrorViewerWrapper = (props) => {
+  return createSubscribedElement(ErrorViewer, props, []);
+};
+
+export default withTranslation()(ErrorViewerWrapper);

src/client/js/components/Admin/ImportData/GrowiZipImportConfigurationModal.jsx

@@ -0,0 +1,228 @@
+/* eslint-disable react/no-danger */
+
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import Modal from 'react-bootstrap/es/Modal';
+
+import GrowiArchiveImportOption from '@commons/models/admin/growi-archive-import-option';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+
+class GrowiZipImportConfigurationModal extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      option: null,
+    };
+
+    this.initialize = this.initialize.bind(this);
+    this.updateOption = this.updateOption.bind(this);
+  }
+
+  async initialize() {
+    await this.setState({
+      option: Object.assign({}, this.props.option), // clone
+    });
+  }
+
+  /**
+   * invoked when the value of control is changed
+   * @param {object} updateObj
+   */
+  changeHandler(updateObj) {
+    const { option } = this.state;
+    const newOption = Object.assign(option, updateObj);
+    this.setState({ option: newOption });
+  }
+
+  updateOption() {
+    const {
+      collectionName, onOptionChange, onClose,
+    } = this.props;
+
+    if (onOptionChange != null) {
+      onOptionChange(collectionName, this.state.option);
+    }
+
+    onClose();
+  }
+
+  renderPagesContents() {
+    const { t } = this.props;
+    const { option } = this.state;
+
+    const translationBase = 'importer_management.growi_settings.configuration.pages';
+
+    /* eslint-disable react/no-unescaped-entities */
+    return (
+      <>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt4"
+            type="checkbox"
+            checked={option.isOverwriteAuthorWithCurrentUser || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ isOverwriteAuthorWithCurrentUser: !option.isOverwriteAuthorWithCurrentUser })}
+          />
+          <label htmlFor="cbOpt4">
+            {t(`${translationBase}.overwrite_author.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.overwrite_author.desc`) }} />
+          </label>
+        </div>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt1"
+            type="checkbox"
+            checked={option.makePublicForGrant2 || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ makePublicForGrant2: !option.makePublicForGrant2 })}
+          />
+          <label htmlFor="cbOpt1">
+            {t(`${translationBase}.set_public_to_page.label`, { from: t('Anyone with the link') })}
+            <p
+              className="help-block mt-0"
+              dangerouslySetInnerHTML={{ __html: t(`${translationBase}.set_public_to_page.desc`, { from: t('Anyone with the link') }) }}
+            />
+          </label>
+        </div>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt2"
+            type="checkbox"
+            checked={option.makePublicForGrant4 || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ makePublicForGrant4: !option.makePublicForGrant4 })}
+          />
+          <label htmlFor="cbOpt2">
+            {t(`${translationBase}.set_public_to_page.label`, { from: t('Just me') })}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.set_public_to_page.desc`, { from: t('Just me') }) }} />
+          </label>
+        </div>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt3"
+            type="checkbox"
+            checked={option.makePublicForGrant5 || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ makePublicForGrant5: !option.makePublicForGrant5 })}
+          />
+          <label htmlFor="cbOpt3">
+            {t(`${translationBase}.set_public_to_page.label`, { from: t('Only inside the group') })}
+            <p
+              className="help-block mt-0"
+              dangerouslySetInnerHTML={{ __html: t(`${translationBase}.set_public_to_page.desc`, { from: t('Only inside the group') }) }}
+            />
+          </label>
+        </div>
+        <div className="checkbox checkbox-default">
+          <input
+            id="cbOpt5"
+            type="checkbox"
+            checked={option.initPageMetadatas || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ initPageMetadatas: !option.initPageMetadatas })}
+          />
+          <label htmlFor="cbOpt5">
+            {t(`${translationBase}.initialize_meta_datas.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.initialize_meta_datas.desc`) }} />
+          </label>
+        </div>
+        <div className="checkbox checkbox-default">
+          <input
+            id="cbOpt6"
+            type="checkbox"
+            checked={option.initHackmdDatas || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ initHackmdDatas: !option.initHackmdDatas })}
+          />
+          <label htmlFor="cbOpt6">
+            {t(`${translationBase}.initialize_hackmd_related_datas.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.initialize_hackmd_related_datas.desc`) }} />
+          </label>
+        </div>
+      </>
+    );
+    /* eslint-enable react/no-unescaped-entities */
+  }
+
+  renderRevisionsContents() {
+    const { t } = this.props;
+    const { option } = this.state;
+
+    const translationBase = 'importer_management.growi_settings.configuration.revisions';
+
+    /* eslint-disable react/no-unescaped-entities */
+    return (
+      <>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt1"
+            type="checkbox"
+            checked={option.isOverwriteAuthorWithCurrentUser || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ isOverwriteAuthorWithCurrentUser: !option.isOverwriteAuthorWithCurrentUser })}
+          />
+          <label htmlFor="cbOpt1">
+            {t(`${translationBase}.overwrite_author.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.overwrite_author.desc`) }} />
+          </label>
+        </div>
+      </>
+    );
+    /* eslint-enable react/no-unescaped-entities */
+  }
+
+  render() {
+    const { t, collectionName } = this.props;
+    const { option } = this.state;
+
+    let contents = null;
+    if (option != null) {
+      switch (collectionName) {
+        case 'pages':
+          contents = this.renderPagesContents();
+          break;
+        case 'revisions':
+          contents = this.renderRevisionsContents();
+          break;
+      }
+    }
+
+    return (
+      <Modal show={this.props.isOpen} onHide={this.props.onClose} onEnter={this.initialize}>
+        <Modal.Header closeButton>
+          <Modal.Title>{`'${collectionName}'`} Configuration</Modal.Title>
+        </Modal.Header>
+
+        <Modal.Body>
+          {contents}
+        </Modal.Body>
+
+        <Modal.Footer>
+          <button type="button" className="btn btn-sm btn-default" onClick={this.props.onClose}>{t('Cancel')}</button>
+          <button type="button" className="btn btn-sm btn-primary" onClick={this.updateOption}>{t('Update')}</button>
+        </Modal.Footer>
+      </Modal>
+    );
+  }
+
+}
+
+GrowiZipImportConfigurationModal.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+
+  isOpen: PropTypes.bool.isRequired,
+  onClose: PropTypes.func.isRequired,
+  onOptionChange: PropTypes.func,
+
+  collectionName: PropTypes.string,
+  option: PropTypes.instanceOf(GrowiArchiveImportOption).isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const GrowiZipImportConfigurationModalWrapper = (props) => {
+  return createSubscribedElement(GrowiZipImportConfigurationModal, props, [AppContainer]);
+};
+
+export default withTranslation()(GrowiZipImportConfigurationModalWrapper);

src/client/js/components/Admin/ImportData/GrowiZipImportForm.jsx

@@ -1,11 +1,21 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
-import * as toastr from 'toastr';
+
+import GrowiArchiveImportOption from '@commons/models/admin/growi-archive-import-option';
+import ImportOptionForPages from '@commons/models/admin/import-option-for-pages';
+import ImportOptionForRevisions from '@commons/models/admin/import-option-for-revisions';
 
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
-// import { toastSuccess, toastError } from '../../../util/apiNotification';
+import WebsocketContainer from '../../../services/WebsocketContainer';
+import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+
+import GrowiZipImportItem, { DEFAULT_MODE, MODE_RESTRICTED_COLLECTION } from './GrowiZipImportItem';
+import GrowiZipImportConfigurationModal from './GrowiZipImportConfigurationModal';
+import ErrorViewer from './ErrorViewer';
+
 
 const GROUPS_PAGE = [
   'pages', 'revisions', 'tags', 'pagetagrelations',
@@ -18,6 +28,10 @@ const GROUPS_CONFIG = [
 ];
 const ALL_GROUPED_COLLECTIONS = GROUPS_PAGE.concat(GROUPS_USER).concat(GROUPS_CONFIG);
 
+const IMPORT_OPTION_CLASS_MAPPING = {
+  pages: ImportOptionForPages,
+  revisions: ImportOptionForRevisions,
+};
 
 class GrowiImportForm extends React.Component {
 
@@ -25,24 +39,42 @@ class GrowiImportForm extends React.Component {
     super(props);
 
     this.initialState = {
-      collectionNameToFileNameMap: {},
+      isImporting: false,
+      isImported: false,
+      progressMap: [],
+      errorsMap: [],
+
       selectedCollections: new Set(),
-      schema: {
-        pages: {},
-        revisions: {},
-        // ...
-      },
+
+      // store relations from collection name to file name
+      collectionNameToFileNameMap: {},
+      // store relations from collection name to GrowiArchiveImportOption instance
+      optionsMap: {},
+
+      isConfigurationModalOpen: false,
+      collectionNameForConfiguration: null,
+
+      isErrorsViewerOpen: false,
+      collectionNameForErrorsViewer: null,
 
       canImport: false,
-      errorsForPageGroups: [],
-      errorsForUserGroups: [],
-      errorsForConfigGroups: [],
-      errorsForOtherGroups: [],
+      warnForPageGroups: [],
+      warnForUserGroups: [],
+      warnForConfigGroups: [],
+      warnForOtherGroups: [],
     };
 
-    this.props.fileStats.forEach((fileStat) => {
+    this.props.innerFileStats.forEach((fileStat) => {
       const { fileName, collectionName } = fileStat;
       this.initialState.collectionNameToFileNameMap[collectionName] = fileName;
+
+      // determine initial mode
+      const initialMode = (MODE_RESTRICTED_COLLECTION[collectionName] != null)
+        ? MODE_RESTRICTED_COLLECTION[collectionName][0]
+        : DEFAULT_MODE;
+      // create GrowiArchiveImportOption instance
+      const ImportOption = IMPORT_OPTION_CLASS_MAPPING[collectionName] || GrowiArchiveImportOption;
+      this.initialState.optionsMap[collectionName] = new ImportOption(initialMode);
     });
 
     this.state = this.initialState;
@@ -50,6 +82,9 @@ class GrowiImportForm extends React.Component {
     this.toggleCheckbox = this.toggleCheckbox.bind(this);
     this.checkAll = this.checkAll.bind(this);
     this.uncheckAll = this.uncheckAll.bind(this);
+    this.updateOption = this.updateOption.bind(this);
+    this.openConfigurationModal = this.openConfigurationModal.bind(this);
+    this.showErrorsViewer = this.showErrorsViewer.bind(this);
     this.validate = this.validate.bind(this);
     this.import = this.import.bind(this);
   }
@@ -58,20 +93,71 @@ class GrowiImportForm extends React.Component {
     return Object.keys(this.state.collectionNameToFileNameMap);
   }
 
-  async toggleCheckbox(e) {
-    const { target } = e;
-    const { name, checked } = target;
+  componentWillMount() {
+    this.setupWebsocketEventHandler();
+  }
 
-    await this.setState((prevState) => {
-      const selectedCollections = new Set(prevState.selectedCollections);
-      if (checked) {
-        selectedCollections.add(name);
-      }
-      else {
-        selectedCollections.delete(name);
-      }
-      return { selectedCollections };
+  componentWillUnmount() {
+    this.teardownWebsocketEventHandler();
+  }
+
+  setupWebsocketEventHandler() {
+    const socket = this.props.websocketContainer.getWebSocket();
+
+    // websocket event
+    // eslint-disable-next-line object-curly-newline
+    socket.on('admin:onProgressForImport', ({ collectionName, collectionProgress, appendedErrors }) => {
+      const { progressMap, errorsMap } = this.state;
+      progressMap[collectionName] = collectionProgress;
+
+      const errors = errorsMap[collectionName] || [];
+      errorsMap[collectionName] = errors.concat(appendedErrors);
+
+      this.setState({
+        isImporting: true,
+        progressMap,
+        errorsMap,
+      });
+    });
+
+    // websocket event
+    socket.on('admin:onTerminateForImport', () => {
+      this.setState({
+        isImporting: false,
+        isImported: true,
+      });
+
+      toastSuccess(undefined, 'Import process has terminated.');
+    });
+
+    // websocket event
+    socket.on('admin:onErrorForImport', (err) => {
+      this.setState({
+        isImporting: false,
+        isImported: false,
+      });
+
+      toastError(err, 'Import process has failed.');
     });
+  }
+
+  teardownWebsocketEventHandler() {
+    const socket = this.props.websocketContainer.getWebSocket();
+
+    socket.removeAllListeners('admin:onProgressForImport');
+    socket.removeAllListeners('admin:onTerminateForImport');
+  }
+
+  async toggleCheckbox(collectionName, bool) {
+    const selectedCollections = new Set(this.state.selectedCollections);
+    if (bool) {
+      selectedCollections.add(collectionName);
+    }
+    else {
+      selectedCollections.delete(collectionName);
+    }
+
+    await this.setState({ selectedCollections });
 
     this.validate();
   }
@@ -86,13 +172,32 @@ class GrowiImportForm extends React.Component {
     this.validate();
   }
 
+  updateOption(collectionName, data) {
+    const { optionsMap } = this.state;
+    const options = optionsMap[collectionName];
+
+    // merge
+    Object.assign(options, data);
+
+    optionsMap[collectionName] = options;
+    this.setState({ optionsMap });
+  }
+
+  openConfigurationModal(collectionName) {
+    this.setState({ isConfigurationModalOpen: true, collectionNameForConfiguration: collectionName });
+  }
+
+  showErrorsViewer(collectionName) {
+    this.setState({ isErrorsViewerOpen: true, collectionNameForErrorsViewer: collectionName });
+  }
+
   async validate() {
     // init errors
     await this.setState({
-      errorsForPageGroups: [],
-      errorsForUserGroups: [],
-      errorsForConfigGroups: [],
-      errorsForOtherGroups: [],
+      warnForPageGroups: [],
+      warnForUserGroups: [],
+      warnForConfigGroups: [],
+      warnForOtherGroups: [],
     });
 
     await this.validateCollectionSize();
@@ -102,10 +207,10 @@ class GrowiImportForm extends React.Component {
     await this.validateUserGroupRelations();
 
     const errors = [
-      ...this.state.errorsForPageGroups,
-      ...this.state.errorsForUserGroups,
-      ...this.state.errorsForConfigGroups,
-      ...this.state.errorsForOtherGroups,
+      ...this.state.warnForPageGroups,
+      ...this.state.warnForUserGroups,
+      ...this.state.warnForConfigGroups,
+      ...this.state.warnForOtherGroups,
     ];
     const canImport = errors.length === 0;
 
@@ -114,18 +219,18 @@ class GrowiImportForm extends React.Component {
 
   async validateCollectionSize(validationErrors) {
     const { t } = this.props;
-    const { errorsForOtherGroups, selectedCollections } = this.state;
+    const { warnForOtherGroups, selectedCollections } = this.state;
 
     if (selectedCollections.size === 0) {
-      errorsForOtherGroups.push(t('importer_management.growi_settings.errors.at_least_one'));
+      warnForOtherGroups.push(t('importer_management.growi_settings.errors.at_least_one'));
     }
 
-    this.setState({ errorsForOtherGroups });
+    this.setState({ warnForOtherGroups });
   }
 
   async validatePagesCollectionPairs() {
     const { t } = this.props;
-    const { errorsForPageGroups, selectedCollections } = this.state;
+    const { warnForPageGroups, selectedCollections } = this.state;
 
     const pageRelatedCollectionsLength = ['pages', 'revisions'].filter((collectionName) => {
       return selectedCollections.has(collectionName);
@@ -133,98 +238,81 @@ class GrowiImportForm extends React.Component {
 
     // MUST be included both or neither when importing
     if (pageRelatedCollectionsLength !== 0 && pageRelatedCollectionsLength !== 2) {
-      errorsForPageGroups.push(t('importer_management.growi_settings.errors.page_and_revision'));
+      warnForPageGroups.push(t('importer_management.growi_settings.errors.page_and_revision'));
     }
 
-    this.setState({ errorsForPageGroups });
+    this.setState({ warnForPageGroups });
   }
 
   async validateExternalAccounts() {
     const { t } = this.props;
-    const { errorsForUserGroups, selectedCollections } = this.state;
+    const { warnForUserGroups, selectedCollections } = this.state;
 
     // MUST include also 'users' if 'externalaccounts' is selected
     if (selectedCollections.has('externalaccounts')) {
       if (!selectedCollections.has('users')) {
-        errorsForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Users', condition: 'Externalaccounts' }));
+        warnForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Users', condition: 'Externalaccounts' }));
       }
     }
 
-    this.setState({ errorsForUserGroups });
+    this.setState({ warnForUserGroups });
   }
 
   async validateUserGroups() {
     const { t } = this.props;
-    const { errorsForUserGroups, selectedCollections } = this.state;
+    const { warnForUserGroups, selectedCollections } = this.state;
 
     // MUST include also 'users' if 'usergroups' is selected
     if (selectedCollections.has('usergroups')) {
       if (!selectedCollections.has('users')) {
-        errorsForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Users', condition: 'Usergroups' }));
+        warnForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Users', condition: 'Usergroups' }));
       }
     }
 
-    this.setState({ errorsForUserGroups });
+    this.setState({ warnForUserGroups });
   }
 
   async validateUserGroupRelations() {
     const { t } = this.props;
-    const { errorsForUserGroups, selectedCollections } = this.state;
+    const { warnForUserGroups, selectedCollections } = this.state;
 
     // MUST include also 'usergroups' if 'usergrouprelations' is selected
     if (selectedCollections.has('usergrouprelations')) {
       if (!selectedCollections.has('usergroups')) {
-        errorsForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Usergroups', condition: 'Usergrouprelations' }));
+        warnForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Usergroups', condition: 'Usergrouprelations' }));
       }
     }
 
-    this.setState({ errorsForUserGroups });
+    this.setState({ warnForUserGroups });
   }
 
   async import() {
+    const { appContainer, fileName, onPostImport } = this.props;
+    const { selectedCollections, optionsMap } = this.state;
+
+    // init progress data
+    await this.setState({
+      isImporting: true,
+      progressMap: [],
+      errorsMap: [],
+    });
+
     try {
       // TODO: use appContainer.apiv3.post
-      const { results } = await this.props.appContainer.apiPost('/v3/import', {
-        fileName: this.props.fileName,
-        collections: Array.from(this.state.selectedCollections),
-        schema: this.state.schema,
+      await appContainer.apiv3Post('/import', {
+        fileName,
+        collections: Array.from(selectedCollections),
+        optionsMap,
       });
 
-      this.setState(this.initialState);
-      this.props.onPostImport();
-
-      // TODO: toastSuccess, toastError
-      toastr.success(undefined, 'Imported documents', {
-        closeButton: true,
-        progressBar: true,
-        newestOnTop: false,
-        showDuration: '100',
-        hideDuration: '100',
-        timeOut: '1200',
-        extendedTimeOut: '150',
-      });
-
-      for (const { collectionName, failedIds } of results) {
-        if (failedIds.length > 0) {
-          toastr.error(`failed to insert ${failedIds.join(', ')}`, collectionName, {
-            closeButton: true,
-            progressBar: true,
-            newestOnTop: false,
-            timeOut: '30000',
-          });
-        }
+      if (onPostImport != null) {
+        onPostImport();
       }
+
+      toastSuccess(undefined, 'Import process has requested.');
     }
     catch (err) {
-      // TODO: toastSuccess, toastError
-      toastr.error(err, 'Error', {
-        closeButton: true,
-        progressBar: true,
-        newestOnTop: false,
-        showDuration: '100',
-        hideDuration: '100',
-        timeOut: '3000',
-      });
+      toastError(err, 'Import request failed.');
     }
   }
 
@@ -245,7 +333,7 @@ class GrowiImportForm extends React.Component {
     );
   }
 
-  renderGroups(groupList, groupName, errors, { wellContent, color } = {}) {
+  renderGroups(groupList, groupName, errors, { wellContent } = {}) {
     const collectionNames = groupList.filter((collectionName) => {
       return this.allCollectionNames.includes(collectionName);
     });
@@ -264,7 +352,7 @@ class GrowiImportForm extends React.Component {
             </ul>
           </div>
         ) }
-        { this.renderCheckboxes(collectionNames, color) }
+        { this.renderImportItems(collectionNames) }
         { this.renderWarnForGroups(errors, `warnFor${groupName}`) }
       </div>
     );
@@ -275,29 +363,47 @@ class GrowiImportForm extends React.Component {
       return !ALL_GROUPED_COLLECTIONS.includes(collectionName);
     });
 
-    return this.renderGroups(collectionNames, 'Other', this.state.errorsForOtherGroups);
+    return this.renderGroups(collectionNames, 'Other', this.state.warnForOtherGroups);
   }
 
-  renderCheckboxes(collectionNames, color) {
-    const checkboxColor = color ? `checkbox-${color}` : 'checkbox-info';
+  renderImportItems(collectionNames) {
+    const {
+      isImporting,
+      isImported,
+      progressMap,
+      errorsMap,
+
+      selectedCollections,
+      optionsMap,
+    } = this.state;
 
     return (
-      <div className={`row checkbox ${checkboxColor}`}>
+      <div className="row">
         {collectionNames.map((collectionName) => {
+          const collectionProgress = progressMap[collectionName];
+          const errors = errorsMap[collectionName];
+          const isConfigButtonAvailable = Object.keys(IMPORT_OPTION_CLASS_MAPPING).includes(collectionName);
+
           return (
             <div className="col-xs-6 my-1" key={collectionName}>
-              <input
-                type="checkbox"
-                id={collectionName}
-                name={collectionName}
-                className="form-check-input"
-                value={collectionName}
-                checked={this.state.selectedCollections.has(collectionName)}
+              <GrowiZipImportItem
+                isImporting={isImporting}
+                isImported={collectionProgress ? isImported : false}
+                insertedCount={collectionProgress ? collectionProgress.insertedCount : 0}
+                modifiedCount={collectionProgress ? collectionProgress.modifiedCount : 0}
+                errorsCount={errors ? errors.length : 0}
+
+                collectionName={collectionName}
+                isSelected={selectedCollections.has(collectionName)}
+                option={optionsMap[collectionName]}
+
+                isConfigButtonAvailable={isConfigButtonAvailable}
+
                 onChange={this.toggleCheckbox}
+                onOptionChange={this.updateOption}
+                onConfigButtonClicked={this.openConfigurationModal}
+                onErrorLinkClicked={this.showErrorsViewer}
               />
-              <label className="text-capitalize form-check-label ml-3" htmlFor={collectionName}>
-                {collectionName}
-              </label>
             </div>
           );
         })}
@@ -305,10 +411,43 @@ class GrowiImportForm extends React.Component {
     );
   }
 
+  renderConfigurationModal() {
+    const { isConfigurationModalOpen, collectionNameForConfiguration: collectionName, optionsMap } = this.state;
+
+    if (collectionName == null) {
+      return null;
+    }
+
+    return (
+      <GrowiZipImportConfigurationModal
+        isOpen={isConfigurationModalOpen}
+        onClose={() => this.setState({ isConfigurationModalOpen: false })}
+        onOptionChange={this.updateOption}
+        collectionName={collectionName}
+        option={optionsMap[collectionName]}
+      />
+    );
+  }
+
+  renderErrorsViewer() {
+    const { isErrorsViewerOpen, errorsMap, collectionNameForErrorsViewer } = this.state;
+    const errors = errorsMap[collectionNameForErrorsViewer];
+
+    return (
+      <ErrorViewer
+        isOpen={isErrorsViewerOpen}
+        onClose={() => this.setState({ isErrorsViewerOpen: false })}
+        errors={errors}
+      />
+    );
+  }
 
   render() {
     const { t } = this.props;
-    const { errorsForPageGroups, errorsForUserGroups, errorsForConfigGroups } = this.state;
+    const {
+      canImport, isImporting,
+      warnForPageGroups, warnForUserGroups, warnForConfigGroups,
+    } = this.state;
 
     return (
       <>
@@ -325,19 +464,22 @@ class GrowiImportForm extends React.Component {
           </div>
         </form>
 
-        { this.renderGroups(GROUPS_PAGE, 'Page', errorsForPageGroups, { wellContent: t('importer_management.growi_settings.overwrite_documents') }) }
-        { this.renderGroups(GROUPS_USER, 'User', errorsForUserGroups) }
-        { this.renderGroups(GROUPS_CONFIG, 'Config', errorsForConfigGroups) }
+        { this.renderGroups(GROUPS_PAGE, 'Page', warnForPageGroups, { wellContent: t('importer_management.growi_settings.overwrite_documents') }) }
+        { this.renderGroups(GROUPS_USER, 'User', warnForUserGroups) }
+        { this.renderGroups(GROUPS_CONFIG, 'Config', warnForConfigGroups) }
         { this.renderOthers() }
 
-        <div className="mt-5 text-center">
+        <div className="mt-4 text-center">
           <button type="button" className="btn btn-default mx-1" onClick={this.props.onDiscard}>
             { t('importer_management.growi_settings.discard') }
           </button>
-          <button type="button" className="btn btn-primary mx-1" onClick={this.import} disabled={!this.state.canImport}>
+          <button type="button" className="btn btn-primary mx-1" onClick={this.import} disabled={!canImport || isImporting}>
             { t('importer_management.import') }
           </button>
         </div>
+
+        { this.renderConfigurationModal() }
+        { this.renderErrorsViewer() }
       </>
     );
   }
@@ -347,18 +489,19 @@ class GrowiImportForm extends React.Component {
 GrowiImportForm.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+  websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
 
   fileName: PropTypes.string,
-  fileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
+  innerFileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
   onDiscard: PropTypes.func.isRequired,
-  onPostImport: PropTypes.func.isRequired,
+  onPostImport: PropTypes.func,
 };
 
 /**
  * Wrapper component for using unstated
  */
 const GrowiImportFormWrapper = (props) => {
-  return createSubscribedElement(GrowiImportForm, props, [AppContainer]);
+  return createSubscribedElement(GrowiImportForm, props, [AppContainer, WebsocketContainer]);
 };
 
 export default withTranslation()(GrowiImportFormWrapper);

src/client/js/components/Admin/ImportData/GrowiZipImportItem.jsx

@@ -0,0 +1,253 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+
+// eslint-disable-next-line no-unused-vars
+import { withTranslation } from 'react-i18next';
+
+import ProgressBar from 'react-bootstrap/es/ProgressBar';
+
+import GrowiArchiveImportOption from '@commons/models/admin/growi-archive-import-option';
+
+
+const MODE_ATTR_MAP = {
+  insert: { color: 'info', icon: 'icon-plus', label: 'Insert' },
+  upsert: { color: 'success', icon: 'icon-plus', label: 'Upsert' },
+  flushAndInsert: { color: 'danger', icon: 'icon-refresh', label: 'Flush and Insert' },
+};
+
+export const DEFAULT_MODE = 'insert';
+
+export const MODE_RESTRICTED_COLLECTION = {
+  configs: ['flushAndInsert'],
+  users: ['insert', 'upsert'],
+};
+
+export default class GrowiZipImportItem extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.changeHandler = this.changeHandler.bind(this);
+    this.modeSelectedHandler = this.modeSelectedHandler.bind(this);
+    this.configButtonClickedHandler = this.configButtonClickedHandler.bind(this);
+    this.errorLinkClickedHandler = this.errorLinkClickedHandler.bind(this);
+  }
+
+  changeHandler(e) {
+    const { collectionName, onChange } = this.props;
+
+    if (onChange != null) {
+      onChange(collectionName, e.target.checked);
+    }
+  }
+
+  modeSelectedHandler(mode) {
+    const { collectionName, onOptionChange } = this.props;
+
+    if (onOptionChange == null) {
+      return;
+    }
+
+    onOptionChange(collectionName, { mode });
+  }
+
+  configButtonClickedHandler() {
+    const { collectionName, onConfigButtonClicked } = this.props;
+
+    if (onConfigButtonClicked == null) {
+      return;
+    }
+
+    onConfigButtonClicked(collectionName);
+  }
+
+  errorLinkClickedHandler() {
+    const { collectionName, onErrorLinkClicked } = this.props;
+
+    if (onErrorLinkClicked == null) {
+      return;
+    }
+
+    onErrorLinkClicked(collectionName);
+  }
+
+  renderModeLabel(mode, isColorized = false) {
+    const attrMap = MODE_ATTR_MAP[mode];
+    const className = isColorized ? `text-${attrMap.color}` : '';
+    return <span className={className}><i className={attrMap.icon}></i> {attrMap.label}</span>;
+  }
+
+  renderCheckbox() {
+    const {
+      collectionName, isSelected, isImporting,
+    } = this.props;
+
+    return (
+      <div className="checkbox checkbox-info my-0">
+        <input
+          type="checkbox"
+          id={collectionName}
+          name={collectionName}
+          className="form-check-input"
+          value={collectionName}
+          checked={isSelected}
+          disabled={isImporting}
+          onChange={this.changeHandler}
+        />
+        <label className="text-capitalize form-check-label" htmlFor={collectionName}>
+          {collectionName}
+        </label>
+      </div>
+    );
+  }
+
+  renderModeSelector() {
+    const {
+      collectionName, option, isImporting,
+    } = this.props;
+
+    const attrMap = MODE_ATTR_MAP[option.mode];
+    const btnColor = `btn-${attrMap.color}`;
+
+    const modes = MODE_RESTRICTED_COLLECTION[collectionName] || Object.keys(MODE_ATTR_MAP);
+
+    return (
+      <span className="d-inline-flex align-items-center">
+        Mode:&nbsp;
+        <div className="dropdown d-inline-block">
+          <button
+            className={`btn ${btnColor} btn-xs dropdown-toggle`}
+            type="button"
+            id="ddmMode"
+            disabled={isImporting}
+            data-toggle="dropdown"
+            aria-haspopup="true"
+            aria-expanded="true"
+          >
+            {this.renderModeLabel(option.mode)}
+            <span className="caret ml-2"></span>
+          </button>
+          <ul className="dropdown-menu" aria-labelledby="ddmMode">
+            { modes.map((mode) => {
+              return (
+                <li key={`buttonMode_${mode}`}>
+                  <a type="button" role="button" onClick={() => this.modeSelectedHandler(mode)}>
+                    {this.renderModeLabel(mode, true)}
+                  </a>
+                </li>
+              );
+            }) }
+          </ul>
+        </div>
+      </span>
+    );
+  }
+
+  renderConfigButton() {
+    const { isImporting, isConfigButtonAvailable } = this.props;
+
+    return (
+      <button
+        type="button"
+        className="btn btn-default btn-xs ml-2"
+        disabled={isImporting || !isConfigButtonAvailable}
+        onClick={isConfigButtonAvailable ? this.configButtonClickedHandler : null}
+      >
+        <i className="icon-settings"></i>
+      </button>
+    );
+  }
+
+  renderProgressBar() {
+    const {
+      isImporting, insertedCount, modifiedCount, errorsCount,
+    } = this.props;
+
+    const total = insertedCount + modifiedCount + errorsCount;
+
+    return (
+      <ProgressBar className="mb-0">
+        <ProgressBar max={total} striped={isImporting} active={isImporting} now={insertedCount} bsStyle="info" />
+        <ProgressBar max={total} striped={isImporting} active={isImporting} now={modifiedCount} bsStyle="success" />
+        <ProgressBar max={total} striped={isImporting} active={isImporting} now={errorsCount} bsStyle="danger" />
+      </ProgressBar>
+    );
+  }
+
+  renderBody() {
+    const { isImporting, isImported } = this.props;
+
+    if (!isImporting && !isImported) {
+      return 'Ready';
+    }
+
+    const { insertedCount, modifiedCount, errorsCount } = this.props;
+    return (
+      <div className="w-100 text-center">
+        <span className="text-info"><strong>{insertedCount}</strong> Inserted</span>,&nbsp;
+        <span className="text-success"><strong>{modifiedCount}</strong> Modified</span>,&nbsp;
+        { errorsCount > 0
+          ? <a className="text-danger" role="button" onClick={this.errorLinkClickedHandler}><u><strong>{errorsCount}</strong> Failed</u></a>
+          : <span className="text-muted"><strong>0</strong> Failed</span>
+        }
+      </div>
+    );
+
+  }
+
+  render() {
+    const {
+      isSelected,
+    } = this.props;
+
+    return (
+      <div className="panel panel-default">
+        <div className="panel-heading">
+          <div className="d-flex justify-content-between align-items-center">
+            {/* left */}
+            {this.renderCheckbox()}
+            {/* right */}
+            <span className="d-flex align-items-center">
+              {this.renderModeSelector()}
+              {this.renderConfigButton()}
+            </span>
+          </div>
+        </div>
+        { isSelected && (
+          <>
+            {this.renderProgressBar()}
+            <div className="panel-body">
+              {this.renderBody()}
+            </div>
+          </>
+        ) }
+      </div>
+    );
+  }
+
+}
+
+GrowiZipImportItem.propTypes = {
+  collectionName: PropTypes.string.isRequired,
+  isSelected: PropTypes.bool.isRequired,
+  option: PropTypes.instanceOf(GrowiArchiveImportOption).isRequired,
+
+  isImporting: PropTypes.bool.isRequired,
+  isImported: PropTypes.bool.isRequired,
+  insertedCount: PropTypes.number,
+  modifiedCount: PropTypes.number,
+  errorsCount: PropTypes.number,
+
+  isConfigButtonAvailable: PropTypes.bool,
+
+  onChange: PropTypes.func,
+  onOptionChange: PropTypes.func,
+  onConfigButtonClicked: PropTypes.func,
+  onErrorLinkClicked: PropTypes.func,
+};
+
+GrowiZipImportItem.defaultProps = {
+  insertedCount: 0,
+  modifiedCount: 0,
+  errorsCount: 0,
+};

src/client/js/components/Admin/ImportData/GrowiZipImportSection.jsx

@@ -16,8 +16,8 @@ class GrowiZipImportSection extends React.Component {
     super(props);
 
     this.initialState = {
-      fileName: '',
-      fileStats: [],
+      fileName: null,
+      innerFileStats: null,
     };
 
     this.state = this.initialState;
@@ -27,17 +27,27 @@ class GrowiZipImportSection extends React.Component {
     this.resetState = this.resetState.bind(this);
   }
 
-  handleUpload({ meta, fileName, fileStats }) {
+  async componentWillMount() {
+    // get uploaded file status
+    const res = await this.props.appContainer.apiv3Get('/import/status');
+
+    if (res.data.zipFileStat != null) {
+      const { fileName, innerFileStats } = res.data.zipFileStat;
+      this.setState({ fileName, innerFileStats });
+    }
+  }
+
+  handleUpload({ meta, fileName, innerFileStats }) {
     this.setState({
       fileName,
-      fileStats,
+      innerFileStats,
     });
   }
 
   async discardData() {
     try {
       const { fileName } = this.state;
-      await this.props.appContainer.apiDelete(`/v3/import/${this.state.fileName}`, {});
+      await this.props.appContainer.apiv3Delete('/import/all');
       this.resetState();
 
       // TODO: toastSuccess, toastError
@@ -79,13 +89,12 @@ class GrowiZipImportSection extends React.Component {
           <i className="icon-exclamation"></i> { t('importer_management.beta_warning') }
         </div>
 
-        {this.state.fileName ? (
+        { this.state.fileName != null ? (
           <div className="px-4">
             <GrowiZipImportForm
               fileName={this.state.fileName}
-              fileStats={this.state.fileStats}
+              innerFileStats={this.state.innerFileStats}
               onDiscard={this.discardData}
-              onPostImport={this.resetState}
             />
           </div>
         ) : (

src/client/js/components/Admin/ImportData/GrowiZipUploadForm.jsx

@@ -31,8 +31,7 @@ class GrowiZipUploadForm extends React.Component {
     formData.append('_csrf', this.props.appContainer.csrfToken);
     formData.append('file', this.inputRef.current.files[0]);
 
-    // TODO: use appContainer.apiv3.post
-    const { data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
+    const { data } = await this.props.appContainer.apiv3Post('/import/upload', formData);
     this.props.onUpload(data);
     // TODO: toastSuccess, toastError
   }

src/client/js/components/Admin/ImportDataPage.jsx

@@ -22,6 +22,7 @@ class ImportDataPage extends React.Component {
       qiitaTeamName: '',
       qiitaAccessToken: '',
     };
+
     this.esaHandleSubmit = this.esaHandleSubmit.bind(this);
     this.esaHandleSubmitTest = this.esaHandleSubmitTest.bind(this);
     this.esaHandleSubmitUpdate = this.esaHandleSubmitUpdate.bind(this);
@@ -137,7 +138,7 @@ class ImportDataPage extends React.Component {
         <GrowiZipImportSection />
 
         <form
-          className="form-horizontal"
+          className="form-horizontal mt-5"
           id="importerSettingFormEsa"
           role="form"
         >
@@ -329,6 +330,12 @@ class ImportDataPage extends React.Component {
 
 }
 
+ImportDataPage.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+};
+
+
 /**
  * Wrapper component for using unstated
  */
@@ -336,9 +343,5 @@ const ImportDataPageWrapper = (props) => {
   return createSubscribedElement(ImportDataPage, props, [AppContainer]);
 };
 
-ImportDataPage.propTypes = {
-  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
-  t: PropTypes.func.isRequired, // i18next
-};
 
 export default withTranslation()(ImportDataPageWrapper);

src/client/js/util/apiNotification.js

@@ -10,7 +10,7 @@ const toastrOption = {
     newestOnTop: false,
     showDuration: '100',
     hideDuration: '100',
-    timeOut: '3000',
+    timeOut: '0',
   },
   success: {
     closeButton: true,

src/lib/models/admin/growi-archive-import-option.js

@@ -0,0 +1,13 @@
+class GrowiArchiveImportOption {
+
+  constructor(mode, initProps = {}) {
+    this.mode = mode;
+
+    Object.entries(initProps).forEach(([key, value]) => {
+      this[key] = value;
+    });
+  }
+
+}
+
+module.exports = GrowiArchiveImportOption;

src/lib/models/admin/import-option-for-pages.js

@@ -0,0 +1,20 @@
+const GrowiArchiveImportOption = require('./growi-archive-import-option');
+
+const DEFAULT_PROPS = {
+  isOverwriteAuthorWithCurrentUser: false,
+  makePublicForGrant2: false,
+  makePublicForGrant4: false,
+  makePublicForGrant5: false,
+  initPageMetadatas: false,
+  initHackmdDatas: false,
+};
+
+class ImportOptionForPages extends GrowiArchiveImportOption {
+
+  constructor(mode, initProps) {
+    super(mode, initProps || DEFAULT_PROPS);
+  }
+
+}
+
+module.exports = ImportOptionForPages;

src/lib/models/admin/import-option-for-revisions.js

@@ -0,0 +1,15 @@
+const GrowiArchiveImportOption = require('./growi-archive-import-option');
+
+const DEFAULT_PROPS = {
+  isOverwriteAuthorWithCurrentUser: false,
+};
+
+class ImportOptionForRevisions extends GrowiArchiveImportOption {
+
+  constructor(mode, initProps) {
+    super(mode, initProps || DEFAULT_PROPS);
+  }
+
+}
+
+module.exports = ImportOptionForRevisions;

src/server/models/ErrorV3.js

@@ -1,9 +1,10 @@
 class ErrorV3 extends Error {
 
-  constructor(message = '', code = '') {
+  constructor(message = '', code = '', stack = undefined) {
     super(); // do not provide message to the super constructor
     this.message = message;
     this.code = code;
+    this.stack = stack;
   }
 
 }

src/server/models/vo/collection-progress.js

@@ -0,0 +1,13 @@
+class CollectionProgress {
+
+  constructor(collectionName, totalCount) {
+    this.collectionName = collectionName;
+    this.currentCount = 0;
+    this.insertedCount = 0;
+    this.modifiedCount = 0;
+    this.totalCount = totalCount;
+  }
+
+}
+
+module.exports = CollectionProgress;

src/server/models/vo/collection-progressing-status.js

@@ -0,0 +1,35 @@
+const CollectionProgress = require('./collection-progress');
+
+class CollectionProgressingStatus {
+
+  constructor(collections) {
+    this.totalCount = 0;
+    this.progressMap = {};
+
+    this.progressList = collections.map((collectionName) => {
+      return new CollectionProgress(collectionName, 0);
+    });
+
+    // collection name to instance mapping
+    this.progressList.forEach((p) => {
+      this.progressMap[p.collectionName] = p;
+    });
+  }
+
+  recalculateTotalCount() {
+    this.progressList.forEach((p) => {
+      this.progressMap[p.collectionName] = p;
+      this.totalCount += p.totalCount;
+    });
+  }
+
+  get currentCount() {
+    return this.progressList.reduce(
+      (acc, crr) => acc + crr.currentCount,
+      0,
+    );
+  }
+
+}
+
+module.exports = CollectionProgressingStatus;

src/server/routes/apiv3/import.js

@@ -3,13 +3,16 @@ const loggerFactory = require('@alias/logger');
 const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-line no-unused-vars
 
 const path = require('path');
-const fs = require('fs');
 const multer = require('multer');
 
+// eslint-disable-next-line no-unused-vars
 const { ObjectId } = require('mongoose').Types;
 
 const express = require('express');
 
+const GrowiArchiveImportOption = require('@commons/models/admin/growi-archive-import-option');
+
+
 const router = express.Router();
 
 /**
@@ -18,6 +21,44 @@ const router = express.Router();
  *    name: Import
  */
 
+/**
+ * @swagger
+ *
+ *  components:
+ *    schemas:
+ *      ImportStatus:
+ *        type: object
+ *        properties:
+ *          zipFileStat:
+ *            type: object
+ *            description: the property object
+ *          progressList:
+ *            type: array
+ *            items:
+ *              type: object
+ *              description: progress data for each exporting collections
+ *          isImporting:
+ *            type: boolean
+ *            description: whether the current importing job exists or not
+ */
+
+/**
+ * generate overwrite params with overwrite-params/* modules
+ * @param {string} collectionName
+ * @param {object} req Request Object
+ * @param {GrowiArchiveImportOption} options GrowiArchiveImportOption instance
+ */
+const generateOverwriteParams = (collectionName, req, options) => {
+  switch (collectionName) {
+    case 'pages':
+      return require('./overwrite-params/pages')(req, options);
+    case 'revisions':
+      return require('./overwrite-params/revisions')(req, options);
+    default:
+      return {};
+  }
+};
+
 module.exports = (crowi) => {
   const { growiBridgeService, importService } = crowi;
   const accessTokenParser = require('../../middleware/access-token-parser')(crowi);
@@ -25,6 +66,19 @@ module.exports = (crowi) => {
   const adminRequired = require('../../middleware/admin-required')(crowi);
   const csrf = require('../../middleware/csrf')(crowi);
 
+  this.adminEvent = crowi.event('admin');
+
+  // setup event
+  this.adminEvent.on('onProgressForImport', (data) => {
+    crowi.getIo().sockets.emit('admin:onProgressForImport', data);
+  });
+  this.adminEvent.on('onTerminateForImport', (data) => {
+    crowi.getIo().sockets.emit('admin:onTerminateForImport', data);
+  });
+  this.adminEvent.on('onErrorForImport', (data) => {
+    crowi.getIo().sockets.emit('admin:onErrorForImport', data);
+  });
+
   const uploads = multer({
     storage: multer.diskStorage({
       destination: (req, file, cb) => {
@@ -43,48 +97,33 @@ module.exports = (crowi) => {
     },
   });
 
+
   /**
-   * defined overwrite params for each collection
-   * all imported documents are overwriten by this value
-   * each value can be any value or a function (_value, { _document, key, schema }) { return newValue }
+   * @swagger
    *
-   * @param {object} Model instance of mongoose model
-   * @param {object} req request object
-   * @return {object} document to be persisted
+   *  /import/status:
+   *    get:
+   *      tags: [Import]
+   *      description: Get properties of stored zip files for import
+   *      responses:
+   *        200:
+   *          description: the zip file statuses
+   *          content:
+   *            application/json:
+   *              schema:
+   *                properties:
+   *                  status:
+   *                    $ref: '#/components/schemas/ImportStatus'
    */
-  const overwriteParamsFn = async(Model, schema, req) => {
-    const collectionName = Model.collection.name;
-
-    /* eslint-disable no-case-declarations */
-    switch (Model.collection.collectionName) {
-      case 'pages':
-        // TODO: use schema and req to generate overwriteParams
-        // e.g. { creator: schema.creator === 'me' ? ObjectId(req.user._id) : importService.keepOriginal }
-        return {
-          status: 'published', // FIXME when importing users and user groups
-          grant: 1, // FIXME when importing users and user groups
-          grantedUsers: [], // FIXME when importing users and user groups
-          grantedGroup: null, // FIXME when importing users and user groups
-          creator: ObjectId(req.user._id), // FIXME when importing users
-          lastUpdateUser: ObjectId(req.user._id), // FIXME when importing users
-          liker: [], // FIXME when importing users
-          seenUsers: [], // FIXME when importing users
-          commentCount: 0, // FIXME when importing comments
-          extended: {}, // FIXME when ?
-          pageIdOnHackmd: undefined, // FIXME when importing hackmd?
-          revisionHackmdSynced: undefined, // FIXME when importing hackmd?
-          hasDraftOnHackmd: undefined, // FIXME when importing hackmd?
-        };
-      // case 'revisoins':
-      //   return {};
-      // case 'users':
-      //   return {};
-      // ... add more cases
-      default:
-        throw new Error(`cannot find a model for collection name "${collectionName}"`);
+  router.get('/status', accessTokenParser, loginRequired, adminRequired, async(req, res) => {
+    try {
+      const status = await importService.getStatus();
+      return res.apiv3(status);
+    }
+    catch (err) {
+      return res.apiv3Err(err, 500);
     }
-    /* eslint-enable no-case-declarations */
-  };
+  });
 
   /**
    * @swagger
@@ -93,66 +132,105 @@ module.exports = (crowi) => {
    *    post:
    *      tags: [Import]
    *      description: import a collection from a zipped json
+   *      requestBody:
+   *        required: true
+   *        content:
+   *          application/json:
+   *            schema:
+   *              type: object
+   *              properties:
+   *                fileName:
+   *                  description: the file name of zip file
+   *                  type: string
+   *                collections:
+   *                  description: collection names to import
+   *                  type: array
+   *                  items:
+   *                    type: string
+   *                optionsMap:
+   *                  description: |
+   *                    the map object of importing option that have collection name as the key
+   *                  additionalProperties:
+   *                    type: object
+   *                    properties:
+   *                      mode:
+   *                        description: Import mode
+   *                        type: string
+   *                        enum: [insert, upsert, flushAndInsert]
    *      responses:
    *        200:
-   *          description: the data is successfully imported
-   *          content:
-   *            application/json:
-   *              schema:
-   *                properties:
-   *                  results:
-   *                    type: array
-   *                    items:
-   *                      type: object
-   *                      description: collectionName, insertedIds, failedIds
+   *          description: Import process has requested
    */
   router.post('/', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
     // TODO: add express validator
 
-    const { fileName, collections, schema } = req.body;
+    const { fileName, collections, optionsMap } = req.body;
     const zipFile = importService.getFile(fileName);
 
-    // unzip
-    await importService.unzip(zipFile);
-    // eslint-disable-next-line no-unused-vars
-    const { meta, fileStats } = await growiBridgeService.parseZipFile(zipFile);
+    // return response first
+    res.apiv3();
 
-    // delete zip file after unzipping and parsing it
-    fs.unlinkSync(zipFile);
+    /*
+     * unzip, parse
+     */
+    let meta = null;
+    let fileStatsToImport = null;
+    try {
+      // unzip
+      await importService.unzip(zipFile);
 
-    // filter fileStats
-    const filteredFileStats = fileStats.filter(({ fileName, collectionName, size }) => { return collections.includes(collectionName) });
+      // eslint-disable-next-line no-unused-vars
+      const { meta: parsedMeta, fileStats, innerFileStats } = await growiBridgeService.parseZipFile(zipFile);
+      meta = parsedMeta;
 
+      // filter innerFileStats
+      fileStatsToImport = innerFileStats.filter(({ fileName, collectionName, size }) => {
+        return collections.includes(collectionName);
+      });
+    }
+    catch (err) {
+      logger.error(err);
+      this.adminEvent.emit('onErrorForImport', { message: err.message });
+      return;
+    }
+
+    /*
+     * validate with meta.json
+     */
     try {
-      // validate with meta.json
       importService.validate(meta);
+    }
+    catch (err) {
+      logger.error(err);
+      this.adminEvent.emit('onErrorForImport', { message: err.message });
+      return;
+    }
 
-      const results = await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
-        const Model = growiBridgeService.getModelFromCollectionName(collectionName);
-        const jsonFile = importService.getFile(fileName);
+    // generate maps of ImportSettings to import
+    const importSettingsMap = {};
+    fileStatsToImport.forEach(({ fileName, collectionName }) => {
+      // instanciate GrowiArchiveImportOption
+      const options = new GrowiArchiveImportOption(null, optionsMap[collectionName]);
 
-        let overwriteParams;
-        if (overwriteParamsFn[collectionName] != null) {
-          // await in case overwriteParamsFn[collection] is a Promise
-          overwriteParams = await overwriteParamsFn(Model, schema[collectionName], req);
-        }
+      // generate options
+      const importSettings = importService.generateImportSettings(options.mode);
+      importSettings.jsonFileName = fileName;
 
-        const { insertedIds, failedIds } = await importService.import(Model, jsonFile, overwriteParams);
+      // generate overwrite params
+      importSettings.overwriteParams = generateOverwriteParams(collectionName, req, options);
 
-        return {
-          collectionName,
-          insertedIds,
-          failedIds,
-        };
-      }));
+      importSettingsMap[collectionName] = importSettings;
+    });
 
-      // TODO: use res.apiv3
-      return res.send({ ok: true, results });
+    /*
+     * import
+     */
+    try {
+      importService.import(collections, importSettingsMap);
     }
     catch (err) {
-      // TODO: use ApiV3Error
       logger.error(err);
-      return res.status(500).send({ status: 'ERROR' });
+      this.adminEvent.emit('onErrorForImport', { message: err.message });
     }
   });
 
@@ -192,11 +270,7 @@ module.exports = (crowi) => {
       // validate with meta.json
       importService.validate(data.meta);
 
-      // TODO: use res.apiv3
-      return res.send({
-        ok: true,
-        data,
-      });
+      return res.apiv3(data);
     }
     catch (err) {
       // TODO: use ApiV3Error
@@ -208,41 +282,23 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /import/{fileName}:
-   *    post:
+   *  /import/all:
+   *    delete:
    *      tags: [Import]
-   *      description: delete a zip file
-   *      parameters:
-   *        - name: fileName
-   *          in: path
-   *          description: the file name of zip file
-   *          required: true
-   *          schema:
-   *            type: string
+   *      description: Delete all zip files
    *      responses:
    *        200:
-   *          description: the file is deleted
-   *          content:
-   *            application/json:
-   *              schema:
-   *                type: object
+   *          description: all files are deleted
    */
-  router.delete('/:fileName', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
-    const { fileName } = req.params;
-
+  router.delete('/all', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
     try {
-      const zipFile = importService.getFile(fileName);
-      fs.unlinkSync(zipFile);
+      importService.deleteAllZipFiles();
 
-      // TODO: use res.apiv3
-      return res.send({
-        ok: true,
-      });
+      return res.apiv3();
     }
     catch (err) {
-      // TODO: use ApiV3Error
       logger.error(err);
-      return res.status(500).send({ status: 'ERROR' });
+      return res.apiv3Err(err, 500);
     }
   });
 

src/server/routes/apiv3/overwrite-params/pages.js

@@ -0,0 +1,62 @@
+const mongoose = require('mongoose');
+
+// eslint-disable-next-line no-unused-vars
+const ImportOptionForPages = require('@commons/models/admin/import-option-for-pages');
+
+const { ObjectId } = mongoose.Types;
+
+const {
+  GRANT_PUBLIC,
+} = mongoose.model('Page');
+
+class PageOverwriteParamsFactory {
+
+  /**
+   * generate overwrite params object
+   * @param {object} req
+   * @param {ImportOptionForPages} option
+   * @return object
+   *  key: property name
+   *  value: any value or a function `(value, { document, schema, propertyName }) => { return newValue }`
+   */
+  static generate(req, option) {
+    const params = {};
+
+    if (option.isOverwriteAuthorWithCurrentUser) {
+      const userId = ObjectId(req.user._id);
+      params.creator = userId;
+      params.lastUpdateUser = userId;
+    }
+
+    params.grant = (value, { document, schema, propertyName }) => {
+      if (option.makePublicForGrant2 && value === 2) {
+        return GRANT_PUBLIC;
+      }
+      if (option.makePublicForGrant4 && value === 4) {
+        return GRANT_PUBLIC;
+      }
+      if (option.makePublicForGrant5 && value === 5) {
+        return GRANT_PUBLIC;
+      }
+      return value;
+    };
+
+    if (option.initPageMetadatas) {
+      params.liker = [];
+      params.seenUsers = [];
+      params.commentCount = 0;
+      params.extended = {};
+    }
+
+    if (option.initHackmdDatas) {
+      params.pageIdOnHackmd = undefined;
+      params.revisionHackmdSynced = undefined;
+      params.hasDraftOnHackmd = undefined;
+    }
+
+    return params;
+  }
+
+}
+
+module.exports = (req, option) => PageOverwriteParamsFactory.generate(req, option);

src/server/routes/apiv3/overwrite-params/revisions.js

@@ -0,0 +1,31 @@
+const mongoose = require('mongoose');
+
+// eslint-disable-next-line no-unused-vars
+const ImportOptionForPages = require('@commons/models/admin/import-option-for-pages');
+
+const { ObjectId } = mongoose.Types;
+
+class RevisionOverwriteParamsFactory {
+
+  /**
+   * generate overwrite params object
+   * @param {object} req
+   * @param {ImportOptionForPages} option
+   * @return object
+   *  key: property name
+   *  value: any value or a function `(value, { document, schema, propertyName }) => { return newValue }`
+   */
+  static generate(req, option) {
+    const params = {};
+
+    if (option.isOverwriteAuthorWithCurrentUser) {
+      const userId = ObjectId(req.user._id);
+      params.author = userId;
+    }
+
+    return params;
+  }
+
+}
+
+module.exports = (req, option) => RevisionOverwriteParamsFactory.generate(req, option);

src/server/routes/apiv3/response.js

@@ -3,7 +3,7 @@ const toArrayIfNot = require('../../../lib/util/toArrayIfNot');
 const addCustomFunctionToResponse = (express, crowi) => {
   const { ErrorV3 } = crowi.models;
 
-  express.response.apiv3 = function(obj) { // not arrow function
+  express.response.apiv3 = function(obj = {}) { // not arrow function
     // obj must be object
     if (typeof obj !== 'object' || obj instanceof Array) {
       throw new Error('invalid value supplied to res.apiv3');
@@ -22,6 +22,9 @@ const addCustomFunctionToResponse = (express, crowi) => {
       if (e instanceof ErrorV3) {
         return e;
       }
+      if (e instanceof Error) {
+        return new ErrorV3(e.message, null, e.stack);
+      }
       if (typeof e === 'string') {
         return { message: e };
       }

src/server/service/export.js

@@ -7,54 +7,25 @@ const { Transform } = require('stream');
 const streamToPromise = require('stream-to-promise');
 const archiver = require('archiver');
 
-
 const toArrayIfNot = require('../../lib/util/toArrayIfNot');
 
+const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
 
-class ExportingProgress {
-
-  constructor(collectionName, totalCount) {
-    this.collectionName = collectionName;
-    this.currentCount = 0;
-    this.totalCount = totalCount;
-  }
-
-}
-
-class ExportingStatus {
-
-  constructor() {
-    this.totalCount = 0;
-
-    this.progressList = null;
-    this.progressMap = {};
-  }
-
-  async init(collections) {
-    const promisesForCreatingInstance = collections.map(async(collectionName) => {
-      const collection = mongoose.connection.collection(collectionName);
-      const totalCount = await collection.count();
-      return new ExportingProgress(collectionName, totalCount);
-    });
-    this.progressList = await Promise.all(promisesForCreatingInstance);
+class ExportProgressingStatus extends CollectionProgressingStatus {
 
-    // collection name to instance mapping
-    this.progressList.forEach((p) => {
-      this.progressMap[p.collectionName] = p;
-      this.totalCount += p.totalCount;
+  async init() {
+    // retrieve total document count from each collections
+    const promises = this.progressList.map(async(collectionProgress) => {
+      const collection = mongoose.connection.collection(collectionProgress.collectionName);
+      collectionProgress.totalCount = await collection.count();
     });
-  }
+    await Promise.all(promises);
 
-  get currentCount() {
-    return this.progressList.reduce(
-      (acc, crr) => acc + crr.currentCount,
-      0,
-    );
+    this.recalculateTotalCount();
   }
 
 }
 
-
 class ExportService {
 
   constructor(crowi) {
@@ -68,14 +39,14 @@ class ExportService {
 
     this.adminEvent = crowi.event('admin');
 
-    this.currentExportingStatus = null;
+    this.currentProgressingStatus = null;
   }
 
   /**
    * parse all zip files in downloads dir
    *
    * @memberOf ExportService
-   * @return {object} info for zip files and whether currentExportingStatus exists
+   * @return {object} info for zip files and whether currentProgressingStatus exists
    */
   async getStatus() {
     const zipFiles = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' });
@@ -87,12 +58,12 @@ class ExportService {
     // filter null object (broken zip)
     const filtered = zipFileStats.filter(element => element != null);
 
-    const isExporting = this.currentExportingStatus != null;
+    const isExporting = this.currentProgressingStatus != null;
 
     return {
       zipFileStats: filtered,
       isExporting,
-      progressList: isExporting ? this.currentExportingStatus.progressList : null,
+      progressList: isExporting ? this.currentProgressingStatus.progressList : null,
     };
   }
 
@@ -123,7 +94,7 @@ class ExportService {
 
   /**
    *
-   * @param {ExportProguress} exportProgress
+   * @param {ExportProgress} exportProgress
    * @return {Transform}
    */
   generateLogStream(exportProgress) {
@@ -196,7 +167,7 @@ class ExportService {
     const transformStream = this.generateTransformStream();
 
     // log configuration
-    const exportProgress = this.currentExportingStatus.progressMap[collectionName];
+    const exportProgress = this.currentProgressingStatus.progressMap[collectionName];
     const logStream = this.generateLogStream(exportProgress);
 
     // create WritableStream
@@ -246,18 +217,18 @@ class ExportService {
   }
 
   async export(collections) {
-    if (this.currentExportingStatus != null) {
+    if (this.currentProgressingStatus != null) {
       throw new Error('There is an exporting process running.');
     }
 
-    this.currentExportingStatus = new ExportingStatus();
-    await this.currentExportingStatus.init(collections);
+    this.currentProgressingStatus = new ExportProgressingStatus(collections);
+    await this.currentProgressingStatus.init();
 
     try {
       await this.exportCollectionsToZippedJson(collections);
     }
     finally {
-      this.currentExportingStatus = null;
+      this.currentProgressingStatus = null;
     }
 
   }
@@ -267,14 +238,14 @@ class ExportService {
    *
    * @memberOf ExportService
    *
-   * @param {ExportProgress} exportProgress
+   * @param {CollectionProgress} collectionProgress
    * @param {number} currentCount number of items exported
    */
-  logProgress(exportProgress, currentCount) {
-    const output = `${exportProgress.collectionName}: ${currentCount}/${exportProgress.totalCount} written`;
+  logProgress(collectionProgress, currentCount) {
+    const output = `${collectionProgress.collectionName}: ${currentCount}/${collectionProgress.totalCount} written`;
 
     // update exportProgress.currentCount
-    exportProgress.currentCount = currentCount;
+    collectionProgress.currentCount = currentCount;
 
     // output every this.per items
     if (currentCount % this.per === 0) {
@@ -282,7 +253,7 @@ class ExportService {
       this.emitProgressEvent();
     }
     // output last item
-    else if (currentCount === exportProgress.totalCount) {
+    else if (currentCount === collectionProgress.totalCount) {
       logger.info(output);
       this.emitProgressEvent();
     }
@@ -290,10 +261,9 @@ class ExportService {
 
   /**
    * emit progress event
-   * @param {ExportProgress} exportProgress
    */
-  emitProgressEvent(exportProgress) {
-    const { currentCount, totalCount, progressList } = this.currentExportingStatus;
+  emitProgressEvent() {
+    const { currentCount, totalCount, progressList } = this.currentProgressingStatus;
     const data = {
       currentCount,
       totalCount,

src/server/service/growi-bridge.js

@@ -99,7 +99,8 @@ class GrowiBridgeService {
    * @return {object} meta{object} and files{Array.<object>}
    */
   async parseZipFile(zipFile) {
-    const fileStats = [];
+    const fileStat = fs.statSync(zipFile);
+    const innerFileStats = [];
     let meta = {};
 
     const readStream = fs.createReadStream(zipFile);
@@ -113,7 +114,7 @@ class GrowiBridgeService {
         meta = JSON.parse((await entry.buffer()).toString());
       }
       else {
-        fileStats.push({
+        innerFileStats.push({
           fileName,
           collectionName: path.basename(fileName, '.json'),
           size,
@@ -135,7 +136,8 @@ class GrowiBridgeService {
     return {
       meta,
       fileName: path.basename(zipFile),
-      fileStats,
+      fileStat,
+      innerFileStats,
     };
   }
 

src/server/service/import.js

@@ -1,11 +1,41 @@
 const logger = require('@alias/logger')('growi:services:ImportService'); // eslint-disable-line no-unused-vars
 const fs = require('fs');
 const path = require('path');
+
+const { Writable, Transform } = require('stream');
 const JSONStream = require('JSONStream');
 const streamToPromise = require('stream-to-promise');
 const unzipper = require('unzipper');
+
 const { ObjectId } = require('mongoose').Types;
 
+const { createBatchStream } = require('../util/batch-stream');
+const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
+
+
+const BULK_IMPORT_SIZE = 100;
+
+
+class ImportSettings {
+
+  constructor(mode) {
+    this.mode = mode || 'insert';
+    this.jsonFileName = null;
+    this.overwriteParams = null;
+  }
+
+}
+
+class ImportingCollectionError extends Error {
+
+  constructor(collectionProgress, error) {
+    super(error);
+    this.collectionProgress = collectionProgress;
+  }
+
+}
+
+
 class ImportService {
 
   constructor(crowi) {
@@ -13,12 +43,23 @@ class ImportService {
     this.growiBridgeService = crowi.growiBridgeService;
     this.getFile = this.growiBridgeService.getFile.bind(this);
     this.baseDir = path.join(crowi.tmpDir, 'imports');
-    this.per = 100;
     this.keepOriginal = this.keepOriginal.bind(this);
 
+    this.adminEvent = crowi.event('admin');
+
     // { pages: { _id: ..., path: ..., ...}, users: { _id: ..., username: ..., }, ... }
     this.convertMap = {};
     this.initConvertMap(crowi.models);
+
+    this.currentProgressingStatus = null;
+  }
+
+  /**
+   * generate ImportSettings instance
+   * @param {string} mode bulk operation mode (insert | upsert | flushAndInsert)
+   */
+  generateImportSettings(mode) {
+    return new ImportSettings(mode);
   }
 
   /**
@@ -48,89 +89,229 @@ class ImportService {
    * automatically convert ObjectId
    *
    * @memberOf ImportService
-   * @param {any} _value value from imported document
-   * @param {{ _document: object, schema: object, key: string }}
+   * @param {any} value value from imported document
+   * @param {{ document: object, schema: object, key: string }}
    * @return {any} new value for the document
    */
-  keepOriginal(_value, { _document, schema, key }) {
-    let value;
-    if (schema[key].instance === 'ObjectID' && ObjectId.isValid(_value)) {
-      value = ObjectId(_value);
+  keepOriginal(value, { document, schema, propertyName }) {
+    let _value;
+    if (schema[propertyName].instance === 'ObjectID' && ObjectId.isValid(value)) {
+      _value = ObjectId(value);
     }
     else {
-      value = _value;
+      _value = value;
     }
 
-    return value;
+    return _value;
+  }
+
+  /**
+   * parse all zip files in downloads dir
+   *
+   * @memberOf ExportService
+   * @return {object} info for zip files and whether currentProgressingStatus exists
+   */
+  async getStatus() {
+    const zipFiles = fs.readdirSync(this.baseDir).filter(file => path.extname(file) === '.zip');
+    const zipFileStats = await Promise.all(zipFiles.map((file) => {
+      const zipFile = this.getFile(file);
+      return this.growiBridgeService.parseZipFile(zipFile);
+    }));
+
+    // filter null object (broken zip)
+    const filtered = zipFileStats
+      .filter(zipFileStat => zipFileStat != null);
+    // sort with ctime("Change Time" - Time when file status was last changed (inode data modification).)
+    filtered.sort((a, b) => { return a.fileStat.ctime - b.fileStat.ctime });
+
+    const isImporting = this.currentProgressingStatus != null;
+
+    return {
+      zipFileStat: filtered.pop(),
+      isImporting,
+      progressList: isImporting ? this.currentProgressingStatus.progressList : null,
+    };
+  }
+
+  /**
+   * import collections from json
+   *
+   * @param {string} collections MongoDB collection name
+   * @param {array} importSettingsMap key: collection name, value: ImportSettings instance
+   */
+  async import(collections, importSettingsMap) {
+    // init status object
+    this.currentProgressingStatus = new CollectionProgressingStatus(collections);
+
+    try {
+      const promises = collections.map((collectionName) => {
+        const importSettings = importSettingsMap[collectionName];
+        return this.importCollection(collectionName, importSettings);
+      });
+      await Promise.all(promises);
+    }
+    // catch ImportingCollectionError
+    catch (err) {
+      const { collectionProgress } = err;
+      logger.error(`failed to import to ${collectionProgress.collectionName}`, err);
+      this.emitProgressEvent(collectionProgress, { message: err.message });
+    }
+    finally {
+      this.currentProgressingStatus = null;
+      this.emitTerminateEvent();
+    }
   }
 
   /**
    * import a collection from json
    *
    * @memberOf ImportService
-   * @param {object} Model instance of mongoose model
-   * @param {string} jsonFile absolute path to the jsonFile being imported
-   * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
+   * @param {string} collectionName MongoDB collection name
+   * @param {ImportSettings} importSettings
    * @return {insertedIds: Array.<string>, failedIds: Array.<string>}
    */
-  async import(Model, jsonFile, overwriteParams = {}) {
-    // streamToPromise(jsonStream) throws an error, use new Promise instead
-    return new Promise((resolve, reject) => {
-      const collectionName = Model.collection.name;
+  async importCollection(collectionName, importSettings) {
+    // prepare functions invoked from custom streams
+    const convertDocuments = this.convertDocuments.bind(this);
+    const bulkOperate = this.bulkOperate.bind(this);
+    const execUnorderedBulkOpSafely = this.execUnorderedBulkOpSafely.bind(this);
+    const emitProgressEvent = this.emitProgressEvent.bind(this);
+
+    const { mode, jsonFileName, overwriteParams } = importSettings;
+    const Model = this.growiBridgeService.getModelFromCollectionName(collectionName);
+    const jsonFile = this.getFile(jsonFileName);
+    const collectionProgress = this.currentProgressingStatus.progressMap[collectionName];
+
+    try {
+      // validate options
+      this.validateImportSettings(collectionName, importSettings);
 
-      let counter = 0;
-      let insertedIds = [];
-      let failedIds = [];
-      let unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+      // flush
+      if (mode === 'flushAndInsert') {
+        await Model.remove({});
+      }
 
+      // stream 1
       const readStream = fs.createReadStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
-      const jsonStream = readStream.pipe(JSONStream.parse('*'));
 
-      jsonStream.on('data', async(document) => {
-        // documents are not persisted until unorderedBulkOp.execute()
-        unorderedBulkOp.insert(this.convertDocuments(Model, document, overwriteParams));
+      // stream 2
+      const jsonStream = JSONStream.parse('*');
+
+      // stream 3
+      const convertStream = new Transform({
+        objectMode: true,
+        transform(doc, encoding, callback) {
+          const converted = convertDocuments(collectionName, doc, overwriteParams);
+          this.push(converted);
+          callback();
+        },
+      });
 
-        counter++;
+      // stream 4
+      const batchStream = createBatchStream(BULK_IMPORT_SIZE);
+
+      // stream 5
+      const writeStream = new Writable({
+        objectMode: true,
+        async write(batch, encoding, callback) {
+          const unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+
+          // documents are not persisted until unorderedBulkOp.execute()
+          batch.forEach((document) => {
+            bulkOperate(unorderedBulkOp, collectionName, document, importSettings);
+          });
+
+          // exec
+          const { insertedCount, modifiedCount, errors } = await execUnorderedBulkOpSafely(unorderedBulkOp);
+          logger.debug(`Importing ${collectionName}. Inserted: ${insertedCount}. Modified: ${modifiedCount}. Failed: ${errors.length}.`);
+
+          const increment = insertedCount + modifiedCount + errors.length;
+          collectionProgress.currentCount += increment;
+          collectionProgress.totalCount += increment;
+          collectionProgress.insertedCount += insertedCount;
+          collectionProgress.modifiedCount += modifiedCount;
+
+          emitProgressEvent(collectionProgress, errors);
+
+          callback();
+        },
+        final(callback) {
+          logger.info(`Importing ${collectionName} has terminated.`);
+          callback();
+        },
+      });
 
-        if (counter % this.per === 0) {
-          // puase jsonStream to prevent more items to be added to unorderedBulkOp
-          jsonStream.pause();
+      readStream
+        .pipe(jsonStream)
+        .pipe(convertStream)
+        .pipe(batchStream)
+        .pipe(writeStream);
 
-          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
-          insertedIds = [...insertedIds, ..._insertedIds];
-          failedIds = [...failedIds, ..._failedIds];
+      await streamToPromise(writeStream);
 
-          // reset initializeUnorderedBulkOp
-          unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+      // clean up tmp directory
+      fs.unlinkSync(jsonFile);
+    }
+    catch (err) {
+      throw new ImportingCollectionError(collectionProgress, err);
+    }
 
-          // resume jsonStream
-          jsonStream.resume();
-        }
-      });
+  }
 
-      jsonStream.on('end', async(data) => {
-        // insert the rest. avoid errors when unorderedBulkOp has no items
-        if (unorderedBulkOp.s.currentBatch !== null) {
-          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
-          insertedIds = [...insertedIds, ..._insertedIds];
-          failedIds = [...failedIds, ..._failedIds];
+  /**
+   *
+   * @param {string} collectionName
+   * @param {importSettings} importSettings
+   */
+  validateImportSettings(collectionName, importSettings) {
+    const { mode } = importSettings;
+
+    switch (collectionName) {
+      case 'configs':
+        if (mode !== 'flushAndInsert') {
+          throw new Error(`The specified mode '${mode}' is not allowed when importing to 'configs' collection.`);
         }
+        break;
+    }
+  }
 
-        logger.info(`Done. Inserted ${insertedIds.length} ${collectionName}.`);
+  /**
+   * process bulk operation
+   * @param {object} bulk MongoDB Bulk instance
+   * @param {string} collectionName collection name
+   * @param {object} document
+   * @param {ImportSettings} importSettings
+   */
+  bulkOperate(bulk, collectionName, document, importSettings) {
+    // insert
+    if (importSettings.mode !== 'upsert') {
+      return bulk.insert(document);
+    }
 
-        if (failedIds.length > 0) {
-          logger.error(`Failed to insert ${failedIds.length} ${collectionName}: ${failedIds.join(', ')}.`);
-        }
+    // upsert
+    switch (collectionName) {
+      default:
+        return bulk.find({ _id: document._id }).upsert().replaceOne(document);
+    }
+  }
 
-        // clean up tmp directory
-        fs.unlinkSync(jsonFile);
+  /**
+   * emit progress event
+   * @param {CollectionProgress} collectionProgress
+   * @param {object} appendedErrors key: collection name, value: array of error object
+   */
+  emitProgressEvent(collectionProgress, appendedErrors) {
+    const { collectionName } = collectionProgress;
 
-        return resolve({
-          insertedIds,
-          failedIds,
-        });
-      });
-    });
+    // send event (in progress in global)
+    this.adminEvent.emit('onProgressForImport', { collectionName, collectionProgress, appendedErrors });
+  }
+
+  /**
+   * emit terminate event
+   */
+  emitTerminateEvent() {
+    this.adminEvent.emit('onTerminateForImport');
   }
 
   /**
@@ -170,38 +351,31 @@ class ImportService {
    *
    * @memberOf ImportService
    * @param {object} unorderedBulkOp result of Model.collection.initializeUnorderedBulkOp()
-   * @return {{nInserted: number, failed: Array.<string>}} number of docuemnts inserted and failed
+   * @return {object} e.g. { insertedCount: 10, errors: [...] }
    */
   async execUnorderedBulkOpSafely(unorderedBulkOp) {
-    // keep the number of documents inserted and failed for logger
-    let insertedIds = [];
-    let failedIds = [];
+    let errors = [];
+    let result = null;
 
-    // try catch to skip errors
     try {
       const log = await unorderedBulkOp.execute();
-      const _insertedIds = log.result.insertedIds.map(op => op._id);
-      insertedIds = [...insertedIds, ..._insertedIds];
+      result = log.result;
     }
     catch (err) {
-      const collectionName = unorderedBulkOp.s.namespace;
-
-      for (const error of err.result.result.writeErrors) {
-        logger.error(`${collectionName}: ${error.errmsg}`);
-      }
-
-      const _failedIds = err.result.result.writeErrors.map(err => err.err.op._id);
-      const _insertedIds = err.result.result.insertedIds.filter(op => !_failedIds.includes(op._id)).map(op => op._id);
-
-      failedIds = [...failedIds, ..._failedIds];
-      insertedIds = [...insertedIds, ..._insertedIds];
+      result = err.result;
+      errors = err.writeErrors.map((err) => {
+        const moreDetailErr = err.err;
+        return { _id: moreDetailErr.op._id, message: err.errmsg };
+      });
     }
 
-    logger.debug(`Importing ${unorderedBulkOp.s.collection.s.name}. Inserted: ${insertedIds.length}. Failed: ${failedIds.length}.`);
+    const insertedCount = result.nInserted + result.nUpserted;
+    const modifiedCount = result.nModified;
 
     return {
-      insertedIds,
-      failedIds,
+      insertedCount,
+      modifiedCount,
+      errors,
     };
   }
 
@@ -209,13 +383,13 @@ class ImportService {
    * execute unorderedBulkOp and ignore errors
    *
    * @memberOf ImportService
-   * @param {object} Model instance of mongoose model
-   * @param {object} _document document being imported
+   * @param {string} collectionName
+   * @param {object} document document being imported
    * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
    * @return {object} document to be persisted
    */
-  convertDocuments(Model, _document, overwriteParams) {
-    const collectionName = Model.collection.name;
+  convertDocuments(collectionName, document, overwriteParams) {
+    const Model = this.growiBridgeService.getModelFromCollectionName(collectionName);
     const schema = Model.schema.paths;
     const convertMap = this.convertMap[collectionName];
 
@@ -223,31 +397,33 @@ class ImportService {
       throw new Error(`attribute map is not defined for ${collectionName}`);
     }
 
-    const document = {};
+    const _document = {};
 
     // assign value from documents being imported
-    for (const entry of Object.entries(convertMap)) {
-      const [key, value] = entry;
+    Object.entries(convertMap).forEach(([propertyName, convertedValue]) => {
+      const value = document[propertyName];
 
       // distinguish between null and undefined
-      if (_document[key] === undefined) {
-        continue; // next entry
+      if (value === undefined) {
+        return; // next entry
       }
 
-      document[key] = (typeof value === 'function') ? value(_document[key], { _document, key, schema }) : value;
-    }
+      const convertFunc = (typeof convertedValue === 'function') ? convertedValue : null;
+      _document[propertyName] = (convertFunc != null) ? convertFunc(value, { document, propertyName, schema }) : convertedValue;
+    });
 
     // overwrite documents with custom values
-    for (const entry of Object.entries(overwriteParams)) {
-      const [key, value] = entry;
+    Object.entries(overwriteParams).forEach(([propertyName, overwriteValue]) => {
+      const value = document[propertyName];
 
       // distinguish between null and undefined
-      if (_document[key] !== undefined) {
-        document[key] = (typeof value === 'function') ? value(_document[key], { _document, key, schema }) : value;
+      if (value !== undefined) {
+        const overwriteFunc = (typeof overwriteValue === 'function') ? overwriteValue : null;
+        _document[propertyName] = (overwriteFunc != null) ? overwriteFunc(value, { document: _document, propertyName, schema }) : overwriteValue;
       }
-    }
+    });
 
-    return document;
+    return _document;
   }
 
   /**
@@ -268,6 +444,15 @@ class ImportService {
     // - import: throw err if there are pending migrations
   }
 
+  /**
+   * Delete all uploaded files
+   */
+  deleteAllZipFiles() {
+    fs.readdirSync(this.baseDir)
+      .filter(file => path.extname(file) === '.zip')
+      .forEach(file => fs.unlinkSync(path.join(this.baseDir, file)));
+  }
+
 }
 
 module.exports = ImportService;

src/server/util/batch-stream.js

@@ -0,0 +1,35 @@
+const { Transform } = require('stream');
+
+function createBatchStream(batchSize) {
+  let batchBuffer = [];
+
+  return new Transform({
+    // object mode
+    objectMode: true,
+
+    transform(doc, encoding, callback) {
+      batchBuffer.push(doc);
+
+      if (batchBuffer.length >= batchSize) {
+        this.push(batchBuffer);
+
+        // reset buffer
+        batchBuffer = [];
+      }
+
+      callback();
+    },
+
+    final(callback) {
+      if (batchBuffer.length > 0) {
+        this.push(batchBuffer);
+      }
+      callback();
+    },
+
+  });
+}
+
+module.exports = {
+  createBatchStream,
+};

src/server/util/search.js

@@ -11,6 +11,8 @@ const {
 } = require('stream');
 const streamToPromise = require('stream-to-promise');
 
+const { createBatchStream } = require('./batch-stream');
+
 const BULK_REINDEX_SIZE = 100;
 
 function SearchClient(crowi, esUri) {
@@ -301,6 +303,7 @@ SearchClient.prototype.updateOrInsertPages = async function(queryFactory, isEmit
 
   const searchEvent = this.searchEvent;
 
+  // prepare functions invoked from custom streams
   const prepareBodyForCreate = this.prepareBodyForCreate.bind(this);
   const shouldIndexed = this.shouldIndexed.bind(this);
   const bulkWrite = this.client.bulk.bind(this.client);
@@ -334,26 +337,7 @@ SearchClient.prototype.updateOrInsertPages = async function(queryFactory, isEmit
     },
   });
 
-  let batchBuffer = [];
-  const batchingStream = new Transform({
-    objectMode: true,
-    transform(doc, encoding, callback) {
-      batchBuffer.push(doc);
-
-      if (batchBuffer.length >= BULK_REINDEX_SIZE) {
-        this.push(batchBuffer);
-        batchBuffer = [];
-      }
-
-      callback();
-    },
-    final(callback) {
-      if (batchBuffer.length > 0) {
-        this.push(batchBuffer);
-      }
-      callback();
-    },
-  });
+  const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
   const appendBookmarkCountStream = new Transform({
     objectMode: true,
@@ -436,7 +420,7 @@ SearchClient.prototype.updateOrInsertPages = async function(queryFactory, isEmit
 
   readStream
     .pipe(thinOutStream)
-    .pipe(batchingStream)
+    .pipe(batchStream)
     .pipe(appendBookmarkCountStream)
     .pipe(appendTagNamesStream)
     .pipe(writeStream);