Merge remote-tracking branch 'origin/reactify-admin/markDownSettings' into rewritte-swagger

# Conflicts:
#	src/server/routes/apiv3/markdown-setting.js
#	src/server/views/admin/markdown.html

CHANGES.md

@@ -1,13 +1,33 @@
 # CHANGES
 
-## 3.5.17-RC
+## 3.5.19-RC
+
+* 
+
+## 3.5.18
+
+* Improvement: Import GROWI Archive
+    * Process asynchronously
+    * Collection configurations
+    * Selectable mode (insert/upsert/flush and insert)
+    * Safely mode settings for configs and users collections
+    * Show errors view
+* Improvement: Optimize handling promise of stream when exporting archive
+* Improvement: Optimize handling promise of stream when building indices
+* Improvement: Add link to [docs.growi.org](https://docs.growi.org)
+* Fix: Monospace font code is broken when printing on Mac
+
+## 3.5.17
 
 * Feature: Upload to GCS (Google Cloud Storage)
 * Feature: Statistics API
-* Improvement: Export progress bar
+* Improvement: Optimize exporting
+* Improvement: Show progress bar when exporting
+* Improvement: Validate collection combinations when importing
 * Improvement: Reactify admin pages
 * Fix: Use HTTP PlantUML URL in default
     * Introduced by 3.5.12
+* Fix: Config default values
 * Support: REPL with `console` npm scripts
 
 ## 3.5.16

README.md

@@ -163,22 +163,26 @@ Environment Variables
     * PASSWORD_SEED: A password seed used by password hash generator.
     * SECRET_TOKEN: A secret key for verifying the integrity of signed cookies.
     * SESSION_NAME: The name of the session ID cookie to set in the response by Express. default: `connect.sid`
-    * FILE_UPLOAD: Attached files storage. default: `aws`
-      * `aws` : AWS S3 (needs AWS settings on Admin page)
-      * `mongodb` : MongoDB GridFS (Setting-less)
-      * `local` : Server's Local file system (Setting-less)
-      * `none` : Disable file uploading
-    * MAX_FILE_SIZE: The maximum file size limit for uploads (bytes). default: `Infinity`
-    * FILE_UPLOAD_TOTAL_LIMIT: Total capacity limit for uploads (bytes). default: `Infinity`
-    * MONGO_GRIDFS_TOTAL_LIMIT: Total capacity limit of MongoDB GridFS (bytes). default: `Infinity`
-      * MONGO_GRIDFS_TOTAL_LIMIT setting  takes precedence over FILE_UPLOAD_TOTAL_LIMIT.
     * SAML_USES_ONLY_ENV_VARS_FOR_SOME_OPTIONS: If `true`, the system uses only the value of the environment variable as the value of the SAML option that can be set via the environment variable.
     * PUBLISH_OPEN_API: Publish GROWI OpenAPI resources with [ReDoc](https://github.com/Rebilly/ReDoc). Visit `/api-docs`.
     * FORCE_WIKI_MODE: Forces wiki mode. default: undefined
-      * `public`  : Forces all pages to become public
-      * `private` : Forces all pages to become private
-      * undefined : Publicity will be configured by the admin security page settings
+        * `public`  : Forces all pages to become public
+        * `private` : Forces all pages to become private
+        * undefined : Publicity will be configured by the admin security page settings
     * FORMAT_NODE_LOG: If `false`, Output server log as JSON. defautl: `true` (Enabled only when `NODE_ENV=production`)
+* **Option for file uploading**
+    * FILE_UPLOAD: Attached files storage. default: `aws`
+        * `aws` : Amazon Web Service S3 (needs AWS settings on Admin page)
+        * `gcs` : Google Cloud Storage (needs settings with environment variables)
+        * `mongodb` : MongoDB GridFS (Setting-less)
+        * `local` : Server's Local file system (Setting-less)
+        * `none` : Disable file uploading
+    * MAX_FILE_SIZE: The maximum file size limit for uploads (bytes). default: `Infinity`
+    * FILE_UPLOAD_TOTAL_LIMIT: Total capacity limit for uploads (bytes). default: `Infinity`
+    * GCS_API_KEY_JSON_PATH: Path of the JSON file that contains [service account key to authenticate to GCP API](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)
+    * GCS_BUCKET: Name of the GCS bucket
+    * MONGO_GRIDFS_TOTAL_LIMIT: Total capacity limit of MongoDB GridFS (bytes). default: `Infinity`
+        * MONGO_GRIDFS_TOTAL_LIMIT setting takes precedence over FILE_UPLOAD_TOTAL_LIMIT.
 * **Option to integrate with external systems**
     * HACKMD_URI: URI to connect to [HackMD(CodiMD)](https://hackmd.io/) server.
         * **This server must load the GROWI agent. [Here's how to prepare it](https://docs.growi.org/guide/admin-cookbook/integrate-with-hackmd.html).**

config/migrate.js

@@ -7,15 +7,9 @@
 
 require('module-alias/register');
 
-function getMongoUri(env) {
-  return env.MONGOLAB_URI // for B.C.
-    || env.MONGODB_URI // MONGOLAB changes their env name
-    || env.MONGOHQ_URL
-    || env.MONGO_URI
-    || ((env.NODE_ENV === 'test') ? 'mongodb://localhost/growi_test' : 'mongodb://localhost/growi');
-}
+const { getMongoUri } = require('@commons/util/mongoose-utils');
 
-const mongoUri = getMongoUri(process.env);
+const mongoUri = getMongoUri();
 const match = mongoUri.match(/^(.+)\/([^/]+)$/);
 
 module.exports = {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "growi",
-  "version": "3.5.17-RC",
+  "version": "3.5.19-RC",
   "description": "Team collaboration software using markdown",
   "tags": [
     "wiki",
@@ -20,7 +20,7 @@
     "url": "https://github.com/weseek/growi/issues"
   },
   "scripts": {
-    "build:apiv3:jsdoc": "swagger-jsdoc -o tmp/swagger.json -d config/swagger-definition.js src/server/routes/apiv3/**/*.js",
+    "build:apiv3:jsdoc": "swagger-jsdoc -o tmp/swagger.json -d config/swagger-definition.js src/server/**/*.js",
     "build:dev:app:watch": "npm run build:dev:app -- --watch",
     "build:dev:app": "env-cmd -f config/env.dev.js webpack --config config/webpack.dev.js --progress",
     "build:dev:dll": "webpack --config config/webpack.dev.dll.js",
@@ -72,6 +72,7 @@
     "@google-cloud/storage": "^3.3.0",
     "JSONStream": "^1.3.5",
     "archiver": "^3.1.1",
+    "array.prototype.flatmap": "^1.2.2",
     "async": "^3.0.1",
     "aws-sdk": "^2.88.0",
     "axios": "^0.19.0",
@@ -113,7 +114,7 @@
     "module-alias": "^2.0.6",
     "mongoose": "5.4.4",
     "mongoose-gridfs": "^1.2.2",
-    "mongoose-paginate": "^5.0.3",
+    "mongoose-paginate-v2": "^1.3.2",
     "mongoose-unique-validator": "^2.0.3",
     "multer": "~1.4.0",
     "multer-autoreap": "^1.0.3",

resource/locales/en-US/translation.json

@@ -113,7 +113,7 @@
   "UserGroup Management": "UserGroup Management",
   "Full Text Search Management": "Full Text Search Management",
   "Import Data": "Import Data",
-  "Export Data": "Export Data",
+  "Export Archive Data": "Export Archive Data",
   "Basic Settings": "Basic Settings",
   "Basic authentication": "Basic authentication",
   "Register limitation": "Register limitation",
@@ -749,19 +749,48 @@
   },
 
   "importer_management": {
-    "import_form_esa": "Import from esa.io",
-    "import_form_qiita": "import_from Qiita:Team",
     "beta_warning": "This function is Beta.",
-    "import_from": "Import from %s",
-    "import_form_growi": "Import from GROWI",
+    "import_from": "Import from {{from}}",
+    "import_growi_archive": "Import GROWI Archive",
     "growi_settings": {
       "overwrite_documents": "Imported documents will overwrite existing documents",
-      "zip_file": "Zip File",
+      "growi_archive_file": "GROWI Archive File",
       "uploaded_data": "Uploaded Data",
       "extracted_file": "Extracted File",
       "collection": "Collection",
       "upload": "Upload",
-      "discard": "Discard Uploaded Data"
+      "discard": "Discard Uploaded Data",
+      "errors": {
+        "at_least_one": "Select one or more collections.",
+        "page_and_revision": "'Pages' and 'Revisions' must be imported both.",
+        "depends": "'{{target}}' must be selected when '{{condition}}' is selected."
+      },
+      "configuration": {
+        "pages": {
+          "overwrite_author": {
+            "label": "Overwrite page's author with the current user",
+            "desc": "Recommended <span class=\"text-danger\">NOT</span> to check this when users will also be restored."
+          },
+          "set_public_to_page": {
+            "label": "Set 'Public' to the pages that is '{{from}}'",
+            "desc": "Make sure that this configuration makes all <b>'{{from}}'</b> pages readable from <span class=\"text-danger\">ANY users</span>."
+          },
+          "initialize_meta_datas": {
+            "label": "Initialize page's like, read users and comment count",
+            "desc": "Recommended <span class=\"text-danger\">NOT</span> to check this when users will also be restored."
+          },
+          "initialize_hackmd_related_datas": {
+            "label": "Initialize HackMD related data",
+            "desc": "Recommended to check this unless there is important drafts on HackMD."
+          }
+        },
+        "revisions": {
+          "overwrite_author": {
+            "label": "Overwrite revision's author with the current user",
+            "desc": "Recommended <span class=\"text-danger\">NOT</span> to check this when users will also be restored."
+          }
+        }
+      }
     },
     "esa_settings": {
       "team_name": "Team name",
@@ -786,14 +815,13 @@
     "rebuild_description_3":"This may take a while."
   },
   "export_management": {
-    "beta_warning": "This function is Beta.",
-    "exporting_data_list": "Exporting Data List",
-    "exported_data_list": "Exported Data List",
+    "exporting_collection_list": "Exporting Collection List",
+    "exported_data_list": "Exported Archive Data List",
     "export_collections": "Export Collections",
     "check_all": "Check All",
     "uncheck_all": "Uncheck All",
     "desc_password_seed": "DO NOT FORGET to set current <code>PASSWORD_SEED</code> to your new GROWI system when restoring user data, or users will NOT be able to login with their password.<br><br><strong>HINT:</strong><br>The current <code>PASSWORD_SEED</code> will be stored in <code>meta.json</code> in exported ZIP.",
-    "create_new_exported_data": "Create New Exported Data",
+    "create_new_archive_data": "Create New Archive Data",
     "export": "Export",
     "cancel": "Cancel",
     "file": "File",

resource/locales/ja/translation.json

@@ -113,7 +113,7 @@
   "UserGroup Management": "グループ管理",
   "Full Text Search Management": "全文検索管理",
   "Import Data": "データインポート",
-  "Export Data": "データエクスポート",
+  "Export Archive Data": "データアーカイブ",
   "Basic Settings": "基本設定",
   "Register limitation": "登録の制限",
   "The contents entered here will be shown in the header etc": "ここに入力した内容は、ヘッダー等に表示されます。",
@@ -734,19 +734,48 @@
   },
 
   "importer_management": {
-    "import_form_esa": "esa.ioからインポート",
-    "import_form_qiita": "Qiita:Teamからインポート",
     "beta_warning": "この機能はベータ版です",
-    "import_from": "%s からインポート",
-    "import_form_growi": "GROWIからインポート",
+    "import_from": "{{from}} からインポート",
+    "import_growi_archive": "GROWI アーカイブをインポート",
     "growi_settings": {
       "overwrite_documents": "インポートされたドキュメントは既存のドキュメントを上書きします",
-      "zip_file": "Zip ファイル",
+      "growi_archive_file": "GROWI アーカイブファイル",
       "uploaded_data": "アップロードされたデータ",
       "extracted_file": "展開されたファイル",
       "collection": "コレクション",
       "upload": "アップロード",
-      "discard": "アップロードしたデータを破棄する"
+      "discard": "アップロードしたデータを破棄する",
+      "errors": {
+        "at_least_one": "コレクションが選択されていません",
+        "page_and_revision": "'Pages' と 'Revisions' はセットでインポートする必要があります",
+        "depends": "'{{condition}}' をインポートする場合は、'{{target}}' を一緒に選択する必要があります"
+      },
+      "configuration": {
+        "pages": {
+          "overwrite_author": {
+            "label": "ページ作成者を現在のユーザーで上書きする",
+            "desc": "users を同時に復元しない場合、このオプションは<span class=\"text-danger\">非推奨</span>です。"
+          },
+          "set_public_to_page": {
+            "label": "'{{from}}' 設定のページを '公開' 設定にする",
+            "desc": "全ての <b>'{{from}}'</b> 設定のページが<span class=\"text-danger\">全ユーザーから</span>読み取り可能になることに注意してください。"
+          },
+          "initialize_meta_datas": {
+            "label": "「いいね」「閲覧したユーザー」「コメント数」を初期化する",
+            "desc": "users を同時に復元しない場合、このオプションは<span class=\"text-danger\">非推奨</span>です。"
+          },
+          "initialize_hackmd_related_datas": {
+            "label": "HackMD 関連データを初期化する",
+            "desc": "HackMD に重要な下書きデータがない限りはこのオプションをチェックすることを推奨します。"
+          }
+        },
+        "revisions": {
+          "overwrite_author": {
+            "label": "リビジョン作成者を現在のユーザーで上書きする",
+            "desc": "users を同時に復元しない場合、このオプションは<span class=\"text-danger\">非推奨</span>です。"
+          }
+        }
+      }
     },
     "esa_settings": {
       "team_name": "チーム名",
@@ -771,14 +800,13 @@
     "rebuild_description_3":""
   },
   "export_management": {
-    "beta_warning": "この機能はベータ版です",
-    "exporting_data_list": "エクスポート中のデータ",
-    "exported_data_list": "エクスポートデータリスト",
+    "exporting_collection_list": "エクスポート中のコレクション",
+    "exported_data_list": "エクスポートされたアーカイブリスト",
     "export_collections": "コレクションのエクスポート",
     "check_all": "全てにチェックを付ける",
     "uncheck_all": "全てからチェックを外す",
     "desc_password_seed": "ユーザーデータをバックアップ/リストアする場合、現在の <code>PASSWORD_SEED</code> を新しい GROWI システムにセットすることを忘れないでください。さもなくば、ユーザーがパスワードでログインできなくなります。<br><br><strong>ヒント:</strong><br>現在の <code>PASSWORD_SEED</code> は、エクスポートされる ZIP 中の <code>meta.json</code> に保存されます。",
-    "create_new_exported_data": "エクスポートデータの新規作成",
+    "create_new_archive_data": "アーカイブデータの新規作成",
     "export": "エクスポート",
     "cancel": "キャンセル",
     "file": "ファイル名",

src/client/js/app.jsx

@@ -38,15 +38,14 @@ import UserGroupDetailPage from './components/Admin/UserGroupDetail/UserGroupDet
 import CustomCssEditor from './components/Admin/CustomCssEditor';
 import CustomScriptEditor from './components/Admin/CustomScriptEditor';
 import CustomHeaderEditor from './components/Admin/CustomHeaderEditor';
-import AdminRebuildSearch from './components/Admin/AdminRebuildSearch';
 import MarkdownSetting from './components/Admin/MarkdownSetting/MarkDownSetting';
-import Users from './components/Admin/Users/Users';
+import UserManagement from './components/Admin/UserManagement';
 import ManageExternalAccount from './components/Admin/Users/ManageExternalAccount';
 import UserGroupPage from './components/Admin/UserGroup/UserGroupPage';
 import Customize from './components/Admin/Customize/Customize';
-import Importer from './components/Admin/Importer';
-import FullTextSearchManagement from './components/Admin/FullTextSearchManagement/FullTextSearchPage';
-import ExportPage from './components/Admin/Export/ExportPage';
+import ImportDataPage from './components/Admin/ImportDataPage';
+import ExportArchiveDataPage from './components/Admin/ExportArchiveDataPage';
+import FullTextSearchManagement from './components/Admin/FullTextSearchManagement';
 
 import AppContainer from './services/AppContainer';
 import PageContainer from './services/PageContainer';
@@ -114,7 +113,7 @@ let componentMappings = {
   'admin-external-account-setting': <ManageExternalAccount />,
 
   'staff-credit': <StaffCredit />,
-  'admin-importer': <Importer />,
+  'admin-importer': <ImportDataPage />,
 };
 
 // additional definitions if data exists
@@ -133,8 +132,6 @@ if (pageContainer.state.pageId != null) {
     'bookmark-button-lg':  <BookmarkButton pageId={pageContainer.state.pageId} crowi={appContainer} size="lg" />,
     'rename-page-name-input':  <PagePathAutoComplete crowi={appContainer} initializedPath={pageContainer.state.path} />,
     'duplicate-page-name-input':  <PagePathAutoComplete crowi={appContainer} initializedPath={pageContainer.state.path} />,
-
-    'admin-rebuild-search': <AdminRebuildSearch crowi={appContainer} />,
   }, componentMappings);
 }
 if (pageContainer.state.path != null) {
@@ -167,7 +164,7 @@ if (adminUsersElem != null) {
   ReactDOM.render(
     <Provider inject={[injectableContainers, adminUsersContainer]}>
       <I18nextProvider i18n={i18n}>
-        <Users />
+        <UserManagement />
       </I18nextProvider>
     </Provider>,
     adminUsersElem,
@@ -253,7 +250,7 @@ if (adminExportPageElem != null) {
   ReactDOM.render(
     <Provider inject={[appContainer, websocketContainer]}>
       <I18nextProvider i18n={i18n}>
-        <ExportPage
+        <ExportArchiveDataPage
           crowi={appContainer}
         />
       </I18nextProvider>

src/client/js/components/Admin/AdminRebuildSearch.jsx

@@ -1,82 +0,0 @@
-import React from 'react';
-import PropTypes from 'prop-types';
-
-import { createSubscribedElement } from '../UnstatedUtils';
-import WebsocketContainer from '../../services/WebsocketContainer';
-
-class AdminRebuildSearch extends React.Component {
-
-  constructor(props) {
-    super(props);
-
-    this.state = {
-      isCompleted: false,
-      total: 0,
-      current: 0,
-      skip: 0,
-    };
-  }
-
-  componentDidMount() {
-    const socket = this.props.websocketContainer.getWebSocket();
-
-    socket.on('admin:addPageProgress', (data) => {
-      const newStates = Object.assign(data, { isCompleted: false });
-      this.setState(newStates);
-    });
-
-    socket.on('admin:finishAddPage', (data) => {
-      const newStates = Object.assign(data, { isCompleted: true });
-      this.setState(newStates);
-    });
-  }
-
-  render() {
-    const {
-      total, current, skip, isCompleted,
-    } = this.state;
-    if (total === 0) {
-      return null;
-    }
-
-    const progressBarLabel = isCompleted ? 'Completed' : `Processing.. ${current}/${total} (${skip} skips)`;
-    const progressBarWidth = isCompleted ? '100%' : `${(current / total) * 100}%`;
-    const progressBarClassNames = isCompleted
-      ? 'progress-bar progress-bar-success'
-      : 'progress-bar progress-bar-striped progress-bar-animated active';
-
-    return (
-      <div>
-        <h5>
-          {progressBarLabel}
-          <span className="pull-right">{progressBarWidth}</span>
-        </h5>
-        <div className="progress progress-sm">
-          <div
-            className={progressBarClassNames}
-            role="progressbar"
-            aria-valuemin="0"
-            aria-valuenow={current}
-            aria-valuemax={total}
-            style={{ width: progressBarWidth }}
-          >
-          </div>
-        </div>
-      </div>
-    );
-  }
-
-}
-
-/**
- * Wrapper component for using unstated
- */
-const AdminRebuildSearchWrapper = (props) => {
-  return createSubscribedElement(AdminRebuildSearch, props, [WebsocketContainer]);
-};
-
-AdminRebuildSearch.propTypes = {
-  websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
-};
-
-export default AdminRebuildSearchWrapper;

src/client/js/components/Admin/Export/ExportingProgressBar.jsx → src/client/js/components/Admin/Common/ProgressBar.jsx

@@ -2,7 +2,7 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 
-class ExportingProgressBar extends React.Component {
+class ProgressBar extends React.Component {
 
 
   render() {
@@ -35,11 +35,11 @@ class ExportingProgressBar extends React.Component {
 
 }
 
-ExportingProgressBar.propTypes = {
+ProgressBar.propTypes = {
   header: PropTypes.string.isRequired,
   currentCount: PropTypes.number.isRequired,
   totalCount: PropTypes.number.isRequired,
   isInProgress: PropTypes.bool,
 };
 
-export default withTranslation()(ExportingProgressBar);
+export default withTranslation()(ProgressBar);

src/client/js/components/Admin/Export/ZipFileTable.jsx → src/client/js/components/Admin/ExportArchiveData/ArchiveFilesTable.jsx

@@ -3,12 +3,13 @@ import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 import { format } from 'date-fns';
 
-import ExportTableMenu from './ExportTableMenu';
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
-class ZipFileTable extends React.Component {
+import ArchiveFilesTableMenu from './ArchiveFilesTableMenu';
+
+class ArchiveFilesTable extends React.Component {
 
   render() {
     // eslint-disable-next-line no-unused-vars
@@ -26,15 +27,15 @@ class ZipFileTable extends React.Component {
           </tr>
         </thead>
         <tbody>
-          {this.props.zipFileStats.map(({ meta, fileName, fileStats }) => {
+          {this.props.zipFileStats.map(({ meta, fileName, innerFileStats }) => {
             return (
               <tr key={fileName}>
                 <th>{fileName}</th>
                 <td>{meta.version}</td>
-                <td className="text-capitalize">{fileStats.map(fileStat => fileStat.collectionName).join(', ')}</td>
+                <td className="text-capitalize">{innerFileStats.map(fileStat => fileStat.collectionName).join(', ')}</td>
                 <td>{meta.exportedAt ? format(new Date(meta.exportedAt), 'yyyy/MM/dd HH:mm:ss') : ''}</td>
                 <td>
-                  <ExportTableMenu
+                  <ArchiveFilesTableMenu
                     fileName={fileName}
                     onZipFileStatRemove={this.props.onZipFileStatRemove}
                   />
@@ -49,7 +50,7 @@ class ZipFileTable extends React.Component {
 
 }
 
-ZipFileTable.propTypes = {
+ArchiveFilesTable.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
 
@@ -60,8 +61,8 @@ ZipFileTable.propTypes = {
 /**
  * Wrapper component for using unstated
  */
-const ZipFileTableWrapper = (props) => {
-  return createSubscribedElement(ZipFileTable, props, [AppContainer]);
+const ArchiveFilesTableWrapper = (props) => {
+  return createSubscribedElement(ArchiveFilesTable, props, [AppContainer]);
 };
 
-export default withTranslation()(ZipFileTableWrapper);
+export default withTranslation()(ArchiveFilesTableWrapper);

src/client/js/components/Admin/Export/ExportTableMenu.jsx → src/client/js/components/Admin/ExportArchiveData/ArchiveFilesTableMenu.jsx

@@ -6,7 +6,7 @@ import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
-class ExportTableMenu extends React.Component {
+class ArchiveFilesTableMenu extends React.Component {
 
   render() {
     const { t } = this.props;
@@ -35,7 +35,7 @@ class ExportTableMenu extends React.Component {
 
 }
 
-ExportTableMenu.propTypes = {
+ArchiveFilesTableMenu.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
   fileName: PropTypes.string.isRequired,
@@ -45,8 +45,8 @@ ExportTableMenu.propTypes = {
 /**
  * Wrapper component for using unstated
  */
-const ExportTableMenuWrapper = (props) => {
-  return createSubscribedElement(ExportTableMenu, props, [AppContainer]);
+const ArchiveFilesTableMenuWrapper = (props) => {
+  return createSubscribedElement(ArchiveFilesTableMenu, props, [AppContainer]);
 };
 
-export default withTranslation()(ExportTableMenuWrapper);
+export default withTranslation()(ArchiveFilesTableMenuWrapper);

src/client/js/components/Admin/Export/ExportZipFormModal.jsx → src/client/js/components/Admin/ExportArchiveData/SelectCollectionsModal.jsx

@@ -20,7 +20,7 @@ const GROUPS_CONFIG = [
 ];
 const ALL_GROUPED_COLLECTIONS = GROUPS_PAGE.concat(GROUPS_USER).concat(GROUPS_CONFIG);
 
-class ExportZipFormModal extends React.Component {
+class SelectCollectionsModal extends React.Component {
 
   constructor(props) {
     super(props);
@@ -225,7 +225,7 @@ class ExportZipFormModal extends React.Component {
 
 }
 
-ExportZipFormModal.propTypes = {
+SelectCollectionsModal.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
 
@@ -238,8 +238,8 @@ ExportZipFormModal.propTypes = {
 /**
  * Wrapper component for using unstated
  */
-const ExportZipFormModalWrapper = (props) => {
-  return createSubscribedElement(ExportZipFormModal, props, [AppContainer]);
+const SelectCollectionsModalWrapper = (props) => {
+  return createSubscribedElement(SelectCollectionsModal, props, [AppContainer]);
 };
 
-export default withTranslation()(ExportZipFormModalWrapper);
+export default withTranslation()(SelectCollectionsModalWrapper);

src/client/js/components/Admin/Export/ExportPage.jsx → src/client/js/components/Admin/ExportArchiveDataPage.jsx

@@ -4,16 +4,18 @@ import { withTranslation } from 'react-i18next';
 import * as toastr from 'toastr';
 
 
-import { createSubscribedElement } from '../../UnstatedUtils';
-import AppContainer from '../../../services/AppContainer';
-import WebsocketContainer from '../../../services/WebsocketContainer';
+import { createSubscribedElement } from '../UnstatedUtils';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
-import ExportZipFormModal from './ExportZipFormModal';
-import ZipFileTable from './ZipFileTable';
-import ExportingProgressBar from './ExportingProgressBar';
+import AppContainer from '../../services/AppContainer';
+import WebsocketContainer from '../../services/WebsocketContainer';
 
-class ExportPage extends React.Component {
+import ProgressBar from './Common/ProgressBar';
+
+import SelectCollectionsModal from './ExportArchiveData/SelectCollectionsModal';
+import ArchiveFilesTable from './ExportArchiveData/ArchiveFilesTable';
+
+class ExportArchiveDataPage extends React.Component {
 
   constructor(props) {
     super(props);
@@ -85,7 +87,7 @@ class ExportPage extends React.Component {
       });
 
       // TODO: toastSuccess, toastError
-      toastr.success(undefined, `New Exported Data '${addedZipFileStat.fileName}' is added`, {
+      toastr.success(undefined, `New Archive Data '${addedZipFileStat.fileName}' is added`, {
         closeButton: true,
         progressBar: true,
         newestOnTop: false,
@@ -158,7 +160,7 @@ class ExportPage extends React.Component {
       const { collectionName, currentCount, totalCount } = progressData;
       return (
         <div className="col-md-6" key={collectionName}>
-          <ExportingProgressBar
+          <ProgressBar
             header={collectionName}
             currentCount={currentCount}
             totalCount={totalCount}
@@ -181,7 +183,7 @@ class ExportPage extends React.Component {
     return (
       <div className="row px-3">
         <div className="col-md-12" key="progressBarForZipping">
-          <ExportingProgressBar
+          <ProgressBar
             header="Zip Files"
             currentCount={1}
             totalCount={1}
@@ -200,15 +202,15 @@ class ExportPage extends React.Component {
 
     return (
       <Fragment>
-        <h2>{t('Export Data')}</h2>
+        <h2>{t('Export Archive Data')}</h2>
 
         <button type="button" className="btn btn-default" disabled={isExporting} onClick={this.openExportModal}>
-          {t('export_management.create_new_exported_data')}
+          {t('export_management.create_new_archive_data')}
         </button>
 
         { showExportingData && (
           <div className="mt-5">
-            <h3>{t('export_management.exporting_data_list')}</h3>
+            <h3>{t('export_management.exporting_collection_list')}</h3>
             { this.renderProgressBarsForCollections() }
             { this.renderProgressBarForZipping() }
           </div>
@@ -216,13 +218,13 @@ class ExportPage extends React.Component {
 
         <div className="mt-5">
           <h3>{t('export_management.exported_data_list')}</h3>
-          <ZipFileTable
+          <ArchiveFilesTable
             zipFileStats={this.state.zipFileStats}
             onZipFileStatRemove={this.onZipFileStatRemove}
           />
         </div>
 
-        <ExportZipFormModal
+        <SelectCollectionsModal
           isOpen={this.state.isExportModalOpen}
           onExportingRequested={this.exportingRequestedHandler}
           onClose={this.closeExportModal}
@@ -234,7 +236,7 @@ class ExportPage extends React.Component {
 
 }
 
-ExportPage.propTypes = {
+ExportArchiveDataPage.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
   websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
@@ -243,8 +245,8 @@ ExportPage.propTypes = {
 /**
  * Wrapper component for using unstated
  */
-const ExportPageFormWrapper = (props) => {
-  return createSubscribedElement(ExportPage, props, [AppContainer, WebsocketContainer]);
+const ExportArchiveDataPageWrapper = (props) => {
+  return createSubscribedElement(ExportArchiveDataPage, props, [AppContainer, WebsocketContainer]);
 };
 
-export default withTranslation()(ExportPageFormWrapper);
+export default withTranslation()(ExportArchiveDataPageWrapper);

src/client/js/components/Admin/FullTextSearchManagement.jsx

@@ -0,0 +1,35 @@
+import React, { Fragment } from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+import { createSubscribedElement } from '../UnstatedUtils';
+import AppContainer from '../../services/AppContainer';
+
+import RebuildIndex from './FullTextSearchManagement/RebuildIndex';
+
+
+class FullTextSearchManagement extends React.Component {
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <Fragment>
+        <h2> { t('full_text_search_management.elasticsearch_management') } </h2>
+        <RebuildIndex />
+      </Fragment>
+    );
+  }
+
+}
+
+const FullTextSearchManagementWrapper = (props) => {
+  return createSubscribedElement(FullTextSearchManagement, props, [AppContainer]);
+};
+
+FullTextSearchManagement.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+};
+
+export default withTranslation()(FullTextSearchManagementWrapper);

src/client/js/components/Admin/FullTextSearchManagement/FullTextSearchPage.jsx

@@ -1,75 +0,0 @@
-import React, { Fragment } from 'react';
-import PropTypes from 'prop-types';
-import { withTranslation } from 'react-i18next';
-
-import AdminRebuildSearch from '../AdminRebuildSearch';
-import AppContainer from '../../../services/AppContainer';
-
-import { createSubscribedElement } from '../../UnstatedUtils';
-import { toastSuccess, toastError } from '../../../util/apiNotification';
-
-
-class FullTextSearchManagement extends React.Component {
-
-  constructor(props) {
-    super(props);
-
-    this.buildIndex = this.buildIndex.bind(this);
-  }
-
-  async buildIndex() {
-
-    const { appContainer } = this.props;
-    const pageId = this.pageId;
-
-    try {
-      const res = await appContainer.apiPost('/admin/search/build', { page_id: pageId });
-      if (!res.ok) {
-        throw new Error(res.message);
-      }
-      else {
-        toastSuccess('Building request is successfully posted.');
-      }
-    }
-    catch (e) {
-      toastError(e, (new Error('エラーが発生しました')));
-    }
-  }
-
-  render() {
-    const { t } = this.props;
-
-    return (
-      <Fragment>
-        <fieldset>
-          <legend> { t('full_text_search_management.elasticsearch_management') } </legend>
-          <div className="form-group form-horizontal">
-            <div className="col-xs-3 control-label"></div>
-            <div className="col-xs-7">
-              <button type="submit" className="btn btn-inverse" onClick={this.buildIndex}>{ t('full_text_search_management.build_button') }</button>
-              <p className="help-block">
-                { t('full_text_search_management.rebuild_description_1') }<br />
-                { t('full_text_search_management.rebuild_description_2') }<br />
-                { t('full_text_search_management.rebuild_description_3') }<br />
-              </p>
-            </div>
-          </div>
-        </fieldset>
-
-        <AdminRebuildSearch />
-      </Fragment>
-    );
-  }
-
-}
-
-const FullTextSearchManagementWrapper = (props) => {
-  return createSubscribedElement(FullTextSearchManagement, props, [AppContainer]);
-};
-
-FullTextSearchManagement.propTypes = {
-  t: PropTypes.func.isRequired, // i18next
-  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
-};
-
-export default withTranslation()(FullTextSearchManagementWrapper);

src/client/js/components/Admin/FullTextSearchManagement/RebuildIndex.jsx

@@ -0,0 +1,133 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+import WebsocketContainer from '../../../services/WebsocketContainer';
+import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+import ProgressBar from '../Common/ProgressBar';
+
+class RebuildIndex extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      isProcessing: false,
+      isCompleted: false,
+
+      total: 0,
+      current: 0,
+      skip: 0,
+    };
+
+    this.buildIndex = this.buildIndex.bind(this);
+  }
+
+  componentDidMount() {
+    const socket = this.props.websocketContainer.getWebSocket();
+
+    socket.on('admin:addPageProgress', (data) => {
+      this.setState({
+        isProcessing: true,
+        ...data,
+      });
+    });
+
+    socket.on('admin:finishAddPage', (data) => {
+      this.setState({
+        isProcessing: false,
+        isCompleted: true,
+        ...data,
+      });
+    });
+  }
+
+  async buildIndex() {
+
+    const { appContainer } = this.props;
+    const pageId = this.pageId;
+
+    try {
+      const res = await appContainer.apiPost('/admin/search/build', { page_id: pageId });
+      if (!res.ok) {
+        throw new Error(res.message);
+      }
+
+      this.setState({ isProcessing: true });
+      toastSuccess('Rebuilding is requested');
+    }
+    catch (e) {
+      toastError(e);
+    }
+  }
+
+  renderProgressBar() {
+    const {
+      total, current, skip, isProcessing, isCompleted,
+    } = this.state;
+    const showProgressBar = isProcessing || isCompleted;
+
+    if (!showProgressBar) {
+      return null;
+    }
+
+    const header = isCompleted ? 'Completed' : `Processing.. (${skip} skips)`;
+
+    return (
+      <ProgressBar
+        header={header}
+        currentCount={current}
+        totalCount={total}
+      />
+    );
+  }
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <>
+        <div className="row">
+          <div className="col-xs-3 control-label"></div>
+          <div className="col-xs-9">
+            { this.renderProgressBar() }
+
+            <button
+              type="submit"
+              className="btn btn-inverse"
+              onClick={this.buildIndex}
+              disabled={this.state.isProcessing}
+            >
+              { t('full_text_search_management.build_button') }
+            </button>
+
+            <p className="help-block">
+              { t('full_text_search_management.rebuild_description_1') }<br />
+              { t('full_text_search_management.rebuild_description_2') }<br />
+              { t('full_text_search_management.rebuild_description_3') }<br />
+            </p>
+          </div>
+        </div>
+      </>
+    );
+  }
+
+}
+
+/**
+ * Wrapper component for using unstated
+ */
+const RebuildIndexWrapper = (props) => {
+  return createSubscribedElement(RebuildIndex, props, [AppContainer, WebsocketContainer]);
+};
+
+RebuildIndex.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+  websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
+};
+
+export default withTranslation()(RebuildIndexWrapper);

src/client/js/components/Admin/Import/GrowiZipImportForm.jsx

@@ -1,181 +0,0 @@
-import React, { Fragment } from 'react';
-import PropTypes from 'prop-types';
-import { withTranslation } from 'react-i18next';
-import * as toastr from 'toastr';
-
-import { createSubscribedElement } from '../../UnstatedUtils';
-import AppContainer from '../../../services/AppContainer';
-// import { toastSuccess, toastError } from '../../../util/apiNotification';
-
-class GrowiImportForm extends React.Component {
-
-  constructor(props) {
-    super(props);
-
-    this.initialState = {
-      collections: new Set(),
-      schema: {
-        pages: {},
-        revisions: {},
-        // ...
-      },
-    };
-
-    this.state = this.initialState;
-
-    this.toggleCheckbox = this.toggleCheckbox.bind(this);
-    this.import = this.import.bind(this);
-    this.validateForm = this.validateForm.bind(this);
-  }
-
-  toggleCheckbox(e) {
-    const { target } = e;
-    const { name, checked } = target;
-
-    this.setState((prevState) => {
-      const collections = new Set(prevState.collections);
-      if (checked) {
-        collections.add(name);
-      }
-      else {
-        collections.delete(name);
-      }
-      return { collections };
-    });
-  }
-
-  async import(e) {
-    e.preventDefault();
-
-    try {
-      // TODO: use appContainer.apiv3.post
-      const { results } = await this.props.appContainer.apiPost('/v3/import', {
-        fileName: this.props.fileName,
-        collections: Array.from(this.state.collections),
-        schema: this.state.schema,
-      });
-
-      this.setState(this.initialState);
-      this.props.onPostImport();
-
-      // TODO: toastSuccess, toastError
-      toastr.success(undefined, 'Imported documents', {
-        closeButton: true,
-        progressBar: true,
-        newestOnTop: false,
-        showDuration: '100',
-        hideDuration: '100',
-        timeOut: '1200',
-        extendedTimeOut: '150',
-      });
-
-      for (const { collectionName, failedIds } of results) {
-        if (failedIds.length > 0) {
-          toastr.error(`failed to insert ${failedIds.join(', ')}`, collectionName, {
-            closeButton: true,
-            progressBar: true,
-            newestOnTop: false,
-            timeOut: '30000',
-          });
-        }
-      }
-    }
-    catch (err) {
-      // TODO: toastSuccess, toastError
-      toastr.error(err, 'Error', {
-        closeButton: true,
-        progressBar: true,
-        newestOnTop: false,
-        showDuration: '100',
-        hideDuration: '100',
-        timeOut: '3000',
-      });
-    }
-  }
-
-  validateForm() {
-    return this.state.collections.size > 0;
-  }
-
-  render() {
-    const { t } = this.props;
-
-    return (
-      <form className="row" onSubmit={this.import}>
-        <div className="col-xs-12">
-          <table className="table table-bordered table-mapping">
-            <caption>{t('importer_management.growi_settings.uploaded_data')}</caption>
-            <thead>
-              <tr>
-                <th></th>
-                <th>{t('importer_management.growi_settings.extracted_file')}</th>
-                <th>{t('importer_management.growi_settings.collection')}</th>
-              </tr>
-            </thead>
-            <tbody>
-              {this.props.fileStats.map((fileStat) => {
-                  const { fileName, collectionName } = fileStat;
-                  const checked = this.state.collections.has(collectionName);
-                  return (
-                    <Fragment key={collectionName}>
-                      <tr>
-                        <td>
-                          <input
-                            type="checkbox"
-                            id={collectionName}
-                            name={collectionName}
-                            className="form-check-input"
-                            value={collectionName}
-                            checked={checked}
-                            onChange={this.toggleCheckbox}
-                          />
-                        </td>
-                        <td>{fileName}</td>
-                        <td className="text-capitalize">{collectionName}</td>
-                      </tr>
-                      {checked && (
-                        <tr>
-                          <td className="text-muted" colSpan="3">
-                            TBD: define how {collectionName} are imported
-                            {/* TODO: create a component for each collection to modify this.state.schema */}
-                          </td>
-                        </tr>
-                      )}
-                    </Fragment>
-                  );
-                })}
-            </tbody>
-          </table>
-        </div>
-        <div className="col-xs-12 text-center">
-          <button type="submit" className="btn btn-primary mx-1" disabled={!this.validateForm()}>
-            { t('importer_management.import') }
-          </button>
-          <button type="button" className="btn btn-default mx-1" onClick={this.props.onDiscard}>
-            { t('importer_management.growi_settings.discard') }
-          </button>
-        </div>
-      </form>
-    );
-  }
-
-}
-
-GrowiImportForm.propTypes = {
-  t: PropTypes.func.isRequired, // i18next
-  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
-
-  fileName: PropTypes.string,
-  fileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
-  onDiscard: PropTypes.func.isRequired,
-  onPostImport: PropTypes.func.isRequired,
-};
-
-/**
- * Wrapper component for using unstated
- */
-const GrowiImportFormWrapper = (props) => {
-  return createSubscribedElement(GrowiImportForm, props, [AppContainer]);
-};
-
-export default withTranslation()(GrowiImportFormWrapper);

src/client/js/components/Admin/ImportData/GrowiArchive/ErrorViewer.jsx

@@ -0,0 +1,52 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import Modal from 'react-bootstrap/es/Modal';
+
+import { createSubscribedElement } from '../../../UnstatedUtils';
+
+
+class ErrorViewer extends React.Component {
+
+  render() {
+    const { errors } = this.props;
+
+    let value = '(no errors)';
+    if (errors != null && errors.length > 0) {
+      const lines = errors.map((obj) => {
+        return JSON.stringify(obj);
+      });
+      value = lines.join('\n');
+    }
+
+    return (
+      <Modal show={this.props.isOpen} onHide={this.props.onClose}>
+        <Modal.Header closeButton className="bg-danger">
+          <Modal.Title className="text-white">Errors</Modal.Title>
+        </Modal.Header>
+        <Modal.Body>
+          <textarea className="form-control" rows="8" readOnly wrap="off" defaultValue={value}></textarea>
+        </Modal.Body>
+      </Modal>
+    );
+  }
+
+}
+
+ErrorViewer.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+
+  isOpen: PropTypes.bool.isRequired,
+  onClose: PropTypes.func.isRequired,
+
+  errors: PropTypes.arrayOf(PropTypes.object),
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ErrorViewerWrapper = (props) => {
+  return createSubscribedElement(ErrorViewer, props, []);
+};
+
+export default withTranslation()(ErrorViewerWrapper);

src/client/js/components/Admin/ImportData/GrowiArchive/ImportCollectionConfigurationModal.jsx

@@ -0,0 +1,228 @@
+/* eslint-disable react/no-danger */
+
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+import Modal from 'react-bootstrap/es/Modal';
+
+import GrowiArchiveImportOption from '@commons/models/admin/growi-archive-import-option';
+
+import { createSubscribedElement } from '../../../UnstatedUtils';
+import AppContainer from '../../../../services/AppContainer';
+// import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+
+class ImportCollectionConfigurationModal extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      option: null,
+    };
+
+    this.initialize = this.initialize.bind(this);
+    this.updateOption = this.updateOption.bind(this);
+  }
+
+  async initialize() {
+    await this.setState({
+      option: Object.assign({}, this.props.option), // clone
+    });
+  }
+
+  /**
+   * invoked when the value of control is changed
+   * @param {object} updateObj
+   */
+  changeHandler(updateObj) {
+    const { option } = this.state;
+    const newOption = Object.assign(option, updateObj);
+    this.setState({ option: newOption });
+  }
+
+  updateOption() {
+    const {
+      collectionName, onOptionChange, onClose,
+    } = this.props;
+
+    if (onOptionChange != null) {
+      onOptionChange(collectionName, this.state.option);
+    }
+
+    onClose();
+  }
+
+  renderPagesContents() {
+    const { t } = this.props;
+    const { option } = this.state;
+
+    const translationBase = 'importer_management.growi_settings.configuration.pages';
+
+    /* eslint-disable react/no-unescaped-entities */
+    return (
+      <>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt4"
+            type="checkbox"
+            checked={option.isOverwriteAuthorWithCurrentUser || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ isOverwriteAuthorWithCurrentUser: !option.isOverwriteAuthorWithCurrentUser })}
+          />
+          <label htmlFor="cbOpt4">
+            {t(`${translationBase}.overwrite_author.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.overwrite_author.desc`) }} />
+          </label>
+        </div>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt1"
+            type="checkbox"
+            checked={option.makePublicForGrant2 || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ makePublicForGrant2: !option.makePublicForGrant2 })}
+          />
+          <label htmlFor="cbOpt1">
+            {t(`${translationBase}.set_public_to_page.label`, { from: t('Anyone with the link') })}
+            <p
+              className="help-block mt-0"
+              dangerouslySetInnerHTML={{ __html: t(`${translationBase}.set_public_to_page.desc`, { from: t('Anyone with the link') }) }}
+            />
+          </label>
+        </div>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt2"
+            type="checkbox"
+            checked={option.makePublicForGrant4 || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ makePublicForGrant4: !option.makePublicForGrant4 })}
+          />
+          <label htmlFor="cbOpt2">
+            {t(`${translationBase}.set_public_to_page.label`, { from: t('Just me') })}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.set_public_to_page.desc`, { from: t('Just me') }) }} />
+          </label>
+        </div>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt3"
+            type="checkbox"
+            checked={option.makePublicForGrant5 || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ makePublicForGrant5: !option.makePublicForGrant5 })}
+          />
+          <label htmlFor="cbOpt3">
+            {t(`${translationBase}.set_public_to_page.label`, { from: t('Only inside the group') })}
+            <p
+              className="help-block mt-0"
+              dangerouslySetInnerHTML={{ __html: t(`${translationBase}.set_public_to_page.desc`, { from: t('Only inside the group') }) }}
+            />
+          </label>
+        </div>
+        <div className="checkbox checkbox-default">
+          <input
+            id="cbOpt5"
+            type="checkbox"
+            checked={option.initPageMetadatas || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ initPageMetadatas: !option.initPageMetadatas })}
+          />
+          <label htmlFor="cbOpt5">
+            {t(`${translationBase}.initialize_meta_datas.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.initialize_meta_datas.desc`) }} />
+          </label>
+        </div>
+        <div className="checkbox checkbox-default">
+          <input
+            id="cbOpt6"
+            type="checkbox"
+            checked={option.initHackmdDatas || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ initHackmdDatas: !option.initHackmdDatas })}
+          />
+          <label htmlFor="cbOpt6">
+            {t(`${translationBase}.initialize_hackmd_related_datas.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.initialize_hackmd_related_datas.desc`) }} />
+          </label>
+        </div>
+      </>
+    );
+    /* eslint-enable react/no-unescaped-entities */
+  }
+
+  renderRevisionsContents() {
+    const { t } = this.props;
+    const { option } = this.state;
+
+    const translationBase = 'importer_management.growi_settings.configuration.revisions';
+
+    /* eslint-disable react/no-unescaped-entities */
+    return (
+      <>
+        <div className="checkbox checkbox-warning">
+          <input
+            id="cbOpt1"
+            type="checkbox"
+            checked={option.isOverwriteAuthorWithCurrentUser || false} // add ' || false' to avoid uncontrolled input warning
+            onChange={() => this.changeHandler({ isOverwriteAuthorWithCurrentUser: !option.isOverwriteAuthorWithCurrentUser })}
+          />
+          <label htmlFor="cbOpt1">
+            {t(`${translationBase}.overwrite_author.label`)}
+            <p className="help-block mt-0" dangerouslySetInnerHTML={{ __html: t(`${translationBase}.overwrite_author.desc`) }} />
+          </label>
+        </div>
+      </>
+    );
+    /* eslint-enable react/no-unescaped-entities */
+  }
+
+  render() {
+    const { t, collectionName } = this.props;
+    const { option } = this.state;
+
+    let contents = null;
+    if (option != null) {
+      switch (collectionName) {
+        case 'pages':
+          contents = this.renderPagesContents();
+          break;
+        case 'revisions':
+          contents = this.renderRevisionsContents();
+          break;
+      }
+    }
+
+    return (
+      <Modal show={this.props.isOpen} onHide={this.props.onClose} onEnter={this.initialize}>
+        <Modal.Header closeButton>
+          <Modal.Title>{`'${collectionName}'`} Configuration</Modal.Title>
+        </Modal.Header>
+
+        <Modal.Body>
+          {contents}
+        </Modal.Body>
+
+        <Modal.Footer>
+          <button type="button" className="btn btn-sm btn-default" onClick={this.props.onClose}>{t('Cancel')}</button>
+          <button type="button" className="btn btn-sm btn-primary" onClick={this.updateOption}>{t('Update')}</button>
+        </Modal.Footer>
+      </Modal>
+    );
+  }
+
+}
+
+ImportCollectionConfigurationModal.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+
+  isOpen: PropTypes.bool.isRequired,
+  onClose: PropTypes.func.isRequired,
+  onOptionChange: PropTypes.func,
+
+  collectionName: PropTypes.string,
+  option: PropTypes.instanceOf(GrowiArchiveImportOption).isRequired,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ImportCollectionConfigurationModalWrapper = (props) => {
+  return createSubscribedElement(ImportCollectionConfigurationModal, props, [AppContainer]);
+};
+
+export default withTranslation()(ImportCollectionConfigurationModalWrapper);

src/client/js/components/Admin/ImportData/GrowiArchive/ImportCollectionItem.jsx

@@ -0,0 +1,253 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+
+// eslint-disable-next-line no-unused-vars
+import { withTranslation } from 'react-i18next';
+
+import ProgressBar from 'react-bootstrap/es/ProgressBar';
+
+import GrowiArchiveImportOption from '@commons/models/admin/growi-archive-import-option';
+
+
+const MODE_ATTR_MAP = {
+  insert: { color: 'info', icon: 'icon-plus', label: 'Insert' },
+  upsert: { color: 'success', icon: 'icon-plus', label: 'Upsert' },
+  flushAndInsert: { color: 'danger', icon: 'icon-refresh', label: 'Flush and Insert' },
+};
+
+export const DEFAULT_MODE = 'insert';
+
+export const MODE_RESTRICTED_COLLECTION = {
+  configs: ['flushAndInsert'],
+  users: ['insert', 'upsert'],
+};
+
+export default class ImportCollectionItem extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.changeHandler = this.changeHandler.bind(this);
+    this.modeSelectedHandler = this.modeSelectedHandler.bind(this);
+    this.configButtonClickedHandler = this.configButtonClickedHandler.bind(this);
+    this.errorLinkClickedHandler = this.errorLinkClickedHandler.bind(this);
+  }
+
+  changeHandler(e) {
+    const { collectionName, onChange } = this.props;
+
+    if (onChange != null) {
+      onChange(collectionName, e.target.checked);
+    }
+  }
+
+  modeSelectedHandler(mode) {
+    const { collectionName, onOptionChange } = this.props;
+
+    if (onOptionChange == null) {
+      return;
+    }
+
+    onOptionChange(collectionName, { mode });
+  }
+
+  configButtonClickedHandler() {
+    const { collectionName, onConfigButtonClicked } = this.props;
+
+    if (onConfigButtonClicked == null) {
+      return;
+    }
+
+    onConfigButtonClicked(collectionName);
+  }
+
+  errorLinkClickedHandler() {
+    const { collectionName, onErrorLinkClicked } = this.props;
+
+    if (onErrorLinkClicked == null) {
+      return;
+    }
+
+    onErrorLinkClicked(collectionName);
+  }
+
+  renderModeLabel(mode, isColorized = false) {
+    const attrMap = MODE_ATTR_MAP[mode];
+    const className = isColorized ? `text-${attrMap.color}` : '';
+    return <span className={className}><i className={attrMap.icon}></i> {attrMap.label}</span>;
+  }
+
+  renderCheckbox() {
+    const {
+      collectionName, isSelected, isImporting,
+    } = this.props;
+
+    return (
+      <div className="checkbox checkbox-info my-0">
+        <input
+          type="checkbox"
+          id={collectionName}
+          name={collectionName}
+          className="form-check-input"
+          value={collectionName}
+          checked={isSelected}
+          disabled={isImporting}
+          onChange={this.changeHandler}
+        />
+        <label className="text-capitalize form-check-label" htmlFor={collectionName}>
+          {collectionName}
+        </label>
+      </div>
+    );
+  }
+
+  renderModeSelector() {
+    const {
+      collectionName, option, isImporting,
+    } = this.props;
+
+    const attrMap = MODE_ATTR_MAP[option.mode];
+    const btnColor = `btn-${attrMap.color}`;
+
+    const modes = MODE_RESTRICTED_COLLECTION[collectionName] || Object.keys(MODE_ATTR_MAP);
+
+    return (
+      <span className="d-inline-flex align-items-center">
+        Mode:&nbsp;
+        <div className="dropdown d-inline-block">
+          <button
+            className={`btn ${btnColor} btn-xs dropdown-toggle`}
+            type="button"
+            id="ddmMode"
+            disabled={isImporting}
+            data-toggle="dropdown"
+            aria-haspopup="true"
+            aria-expanded="true"
+          >
+            {this.renderModeLabel(option.mode)}
+            <span className="caret ml-2"></span>
+          </button>
+          <ul className="dropdown-menu" aria-labelledby="ddmMode">
+            { modes.map((mode) => {
+              return (
+                <li key={`buttonMode_${mode}`}>
+                  <a type="button" role="button" onClick={() => this.modeSelectedHandler(mode)}>
+                    {this.renderModeLabel(mode, true)}
+                  </a>
+                </li>
+              );
+            }) }
+          </ul>
+        </div>
+      </span>
+    );
+  }
+
+  renderConfigButton() {
+    const { isImporting, isConfigButtonAvailable } = this.props;
+
+    return (
+      <button
+        type="button"
+        className="btn btn-default btn-xs ml-2"
+        disabled={isImporting || !isConfigButtonAvailable}
+        onClick={isConfigButtonAvailable ? this.configButtonClickedHandler : null}
+      >
+        <i className="icon-settings"></i>
+      </button>
+    );
+  }
+
+  renderProgressBar() {
+    const {
+      isImporting, insertedCount, modifiedCount, errorsCount,
+    } = this.props;
+
+    const total = insertedCount + modifiedCount + errorsCount;
+
+    return (
+      <ProgressBar className="mb-0">
+        <ProgressBar max={total} striped={isImporting} active={isImporting} now={insertedCount} bsStyle="info" />
+        <ProgressBar max={total} striped={isImporting} active={isImporting} now={modifiedCount} bsStyle="success" />
+        <ProgressBar max={total} striped={isImporting} active={isImporting} now={errorsCount} bsStyle="danger" />
+      </ProgressBar>
+    );
+  }
+
+  renderBody() {
+    const { isImporting, isImported } = this.props;
+
+    if (!isImporting && !isImported) {
+      return 'Ready';
+    }
+
+    const { insertedCount, modifiedCount, errorsCount } = this.props;
+    return (
+      <div className="w-100 text-center">
+        <span className="text-info"><strong>{insertedCount}</strong> Inserted</span>,&nbsp;
+        <span className="text-success"><strong>{modifiedCount}</strong> Modified</span>,&nbsp;
+        { errorsCount > 0
+          ? <a className="text-danger" role="button" onClick={this.errorLinkClickedHandler}><u><strong>{errorsCount}</strong> Failed</u></a>
+          : <span className="text-muted"><strong>0</strong> Failed</span>
+        }
+      </div>
+    );
+
+  }
+
+  render() {
+    const {
+      isSelected,
+    } = this.props;
+
+    return (
+      <div className="panel panel-default">
+        <div className="panel-heading">
+          <div className="d-flex justify-content-between align-items-center">
+            {/* left */}
+            {this.renderCheckbox()}
+            {/* right */}
+            <span className="d-flex align-items-center">
+              {this.renderModeSelector()}
+              {this.renderConfigButton()}
+            </span>
+          </div>
+        </div>
+        { isSelected && (
+          <>
+            {this.renderProgressBar()}
+            <div className="panel-body">
+              {this.renderBody()}
+            </div>
+          </>
+        ) }
+      </div>
+    );
+  }
+
+}
+
+ImportCollectionItem.propTypes = {
+  collectionName: PropTypes.string.isRequired,
+  isSelected: PropTypes.bool.isRequired,
+  option: PropTypes.instanceOf(GrowiArchiveImportOption).isRequired,
+
+  isImporting: PropTypes.bool.isRequired,
+  isImported: PropTypes.bool.isRequired,
+  insertedCount: PropTypes.number,
+  modifiedCount: PropTypes.number,
+  errorsCount: PropTypes.number,
+
+  isConfigButtonAvailable: PropTypes.bool,
+
+  onChange: PropTypes.func,
+  onOptionChange: PropTypes.func,
+  onConfigButtonClicked: PropTypes.func,
+  onErrorLinkClicked: PropTypes.func,
+};
+
+ImportCollectionItem.defaultProps = {
+  insertedCount: 0,
+  modifiedCount: 0,
+  errorsCount: 0,
+};

src/client/js/components/Admin/ImportData/GrowiArchive/ImportForm.jsx

@@ -0,0 +1,507 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+import GrowiArchiveImportOption from '@commons/models/admin/growi-archive-import-option';
+import ImportOptionForPages from '@commons/models/admin/import-option-for-pages';
+import ImportOptionForRevisions from '@commons/models/admin/import-option-for-revisions';
+
+import { createSubscribedElement } from '../../../UnstatedUtils';
+import AppContainer from '../../../../services/AppContainer';
+import WebsocketContainer from '../../../../services/WebsocketContainer';
+import { toastSuccess, toastError } from '../../../../util/apiNotification';
+
+
+import ImportCollectionItem, { DEFAULT_MODE, MODE_RESTRICTED_COLLECTION } from './ImportCollectionItem';
+import ImportCollectionConfigurationModal from './ImportCollectionConfigurationModal';
+import ErrorViewer from './ErrorViewer';
+
+
+const GROUPS_PAGE = [
+  'pages', 'revisions', 'tags', 'pagetagrelations',
+];
+const GROUPS_USER = [
+  'users', 'externalaccounts', 'usergroups', 'usergrouprelations',
+];
+const GROUPS_CONFIG = [
+  'configs', 'updateposts', 'globalnotificationsettings',
+];
+const ALL_GROUPED_COLLECTIONS = GROUPS_PAGE.concat(GROUPS_USER).concat(GROUPS_CONFIG);
+
+const IMPORT_OPTION_CLASS_MAPPING = {
+  pages: ImportOptionForPages,
+  revisions: ImportOptionForRevisions,
+};
+
+class ImportForm extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.initialState = {
+      isImporting: false,
+      isImported: false,
+      progressMap: [],
+      errorsMap: [],
+
+      selectedCollections: new Set(),
+
+      // store relations from collection name to file name
+      collectionNameToFileNameMap: {},
+      // store relations from collection name to GrowiArchiveImportOption instance
+      optionsMap: {},
+
+      isConfigurationModalOpen: false,
+      collectionNameForConfiguration: null,
+
+      isErrorsViewerOpen: false,
+      collectionNameForErrorsViewer: null,
+
+      canImport: false,
+      warnForPageGroups: [],
+      warnForUserGroups: [],
+      warnForConfigGroups: [],
+      warnForOtherGroups: [],
+    };
+
+    this.props.innerFileStats.forEach((fileStat) => {
+      const { fileName, collectionName } = fileStat;
+      this.initialState.collectionNameToFileNameMap[collectionName] = fileName;
+
+      // determine initial mode
+      const initialMode = (MODE_RESTRICTED_COLLECTION[collectionName] != null)
+        ? MODE_RESTRICTED_COLLECTION[collectionName][0]
+        : DEFAULT_MODE;
+      // create GrowiArchiveImportOption instance
+      const ImportOption = IMPORT_OPTION_CLASS_MAPPING[collectionName] || GrowiArchiveImportOption;
+      this.initialState.optionsMap[collectionName] = new ImportOption(initialMode);
+    });
+
+    this.state = this.initialState;
+
+    this.toggleCheckbox = this.toggleCheckbox.bind(this);
+    this.checkAll = this.checkAll.bind(this);
+    this.uncheckAll = this.uncheckAll.bind(this);
+    this.updateOption = this.updateOption.bind(this);
+    this.openConfigurationModal = this.openConfigurationModal.bind(this);
+    this.showErrorsViewer = this.showErrorsViewer.bind(this);
+    this.validate = this.validate.bind(this);
+    this.import = this.import.bind(this);
+  }
+
+  get allCollectionNames() {
+    return Object.keys(this.state.collectionNameToFileNameMap);
+  }
+
+  componentWillMount() {
+    this.setupWebsocketEventHandler();
+  }
+
+  componentWillUnmount() {
+    this.teardownWebsocketEventHandler();
+  }
+
+  setupWebsocketEventHandler() {
+    const socket = this.props.websocketContainer.getWebSocket();
+
+    // websocket event
+    // eslint-disable-next-line object-curly-newline
+    socket.on('admin:onProgressForImport', ({ collectionName, collectionProgress, appendedErrors }) => {
+      const { progressMap, errorsMap } = this.state;
+      progressMap[collectionName] = collectionProgress;
+
+      const errors = errorsMap[collectionName] || [];
+      errorsMap[collectionName] = errors.concat(appendedErrors);
+
+      this.setState({
+        isImporting: true,
+        progressMap,
+        errorsMap,
+      });
+    });
+
+    // websocket event
+    socket.on('admin:onTerminateForImport', () => {
+      this.setState({
+        isImporting: false,
+        isImported: true,
+      });
+
+      toastSuccess(undefined, 'Import process has terminated.');
+    });
+
+    // websocket event
+    socket.on('admin:onErrorForImport', (err) => {
+      this.setState({
+        isImporting: false,
+        isImported: false,
+      });
+
+      toastError(err, 'Import process has failed.');
+    });
+  }
+
+  teardownWebsocketEventHandler() {
+    const socket = this.props.websocketContainer.getWebSocket();
+
+    socket.removeAllListeners('admin:onProgressForImport');
+    socket.removeAllListeners('admin:onTerminateForImport');
+  }
+
+  async toggleCheckbox(collectionName, bool) {
+    const selectedCollections = new Set(this.state.selectedCollections);
+    if (bool) {
+      selectedCollections.add(collectionName);
+    }
+    else {
+      selectedCollections.delete(collectionName);
+    }
+
+    await this.setState({ selectedCollections });
+
+    this.validate();
+  }
+
+  async checkAll() {
+    await this.setState({ selectedCollections: new Set(this.allCollectionNames) });
+    this.validate();
+  }
+
+  async uncheckAll() {
+    await this.setState({ selectedCollections: new Set() });
+    this.validate();
+  }
+
+  updateOption(collectionName, data) {
+    const { optionsMap } = this.state;
+    const options = optionsMap[collectionName];
+
+    // merge
+    Object.assign(options, data);
+
+    optionsMap[collectionName] = options;
+    this.setState({ optionsMap });
+  }
+
+  openConfigurationModal(collectionName) {
+    this.setState({ isConfigurationModalOpen: true, collectionNameForConfiguration: collectionName });
+  }
+
+  showErrorsViewer(collectionName) {
+    this.setState({ isErrorsViewerOpen: true, collectionNameForErrorsViewer: collectionName });
+  }
+
+  async validate() {
+    // init errors
+    await this.setState({
+      warnForPageGroups: [],
+      warnForUserGroups: [],
+      warnForConfigGroups: [],
+      warnForOtherGroups: [],
+    });
+
+    await this.validateCollectionSize();
+    await this.validatePagesCollectionPairs();
+    await this.validateExternalAccounts();
+    await this.validateUserGroups();
+    await this.validateUserGroupRelations();
+
+    const errors = [
+      ...this.state.warnForPageGroups,
+      ...this.state.warnForUserGroups,
+      ...this.state.warnForConfigGroups,
+      ...this.state.warnForOtherGroups,
+    ];
+    const canImport = errors.length === 0;
+
+    this.setState({ canImport });
+  }
+
+  async validateCollectionSize(validationErrors) {
+    const { t } = this.props;
+    const { warnForOtherGroups, selectedCollections } = this.state;
+
+    if (selectedCollections.size === 0) {
+      warnForOtherGroups.push(t('importer_management.growi_settings.errors.at_least_one'));
+    }
+
+    this.setState({ warnForOtherGroups });
+  }
+
+  async validatePagesCollectionPairs() {
+    const { t } = this.props;
+    const { warnForPageGroups, selectedCollections } = this.state;
+
+    const pageRelatedCollectionsLength = ['pages', 'revisions'].filter((collectionName) => {
+      return selectedCollections.has(collectionName);
+    }).length;
+
+    // MUST be included both or neither when importing
+    if (pageRelatedCollectionsLength !== 0 && pageRelatedCollectionsLength !== 2) {
+      warnForPageGroups.push(t('importer_management.growi_settings.errors.page_and_revision'));
+    }
+
+    this.setState({ warnForPageGroups });
+  }
+
+  async validateExternalAccounts() {
+    const { t } = this.props;
+    const { warnForUserGroups, selectedCollections } = this.state;
+
+    // MUST include also 'users' if 'externalaccounts' is selected
+    if (selectedCollections.has('externalaccounts')) {
+      if (!selectedCollections.has('users')) {
+        warnForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Users', condition: 'Externalaccounts' }));
+      }
+    }
+
+    this.setState({ warnForUserGroups });
+  }
+
+  async validateUserGroups() {
+    const { t } = this.props;
+    const { warnForUserGroups, selectedCollections } = this.state;
+
+    // MUST include also 'users' if 'usergroups' is selected
+    if (selectedCollections.has('usergroups')) {
+      if (!selectedCollections.has('users')) {
+        warnForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Users', condition: 'Usergroups' }));
+      }
+    }
+
+    this.setState({ warnForUserGroups });
+  }
+
+  async validateUserGroupRelations() {
+    const { t } = this.props;
+    const { warnForUserGroups, selectedCollections } = this.state;
+
+    // MUST include also 'usergroups' if 'usergrouprelations' is selected
+    if (selectedCollections.has('usergrouprelations')) {
+      if (!selectedCollections.has('usergroups')) {
+        warnForUserGroups.push(t('importer_management.growi_settings.errors.depends', { target: 'Usergroups', condition: 'Usergrouprelations' }));
+      }
+    }
+
+    this.setState({ warnForUserGroups });
+  }
+
+  async import() {
+    const { appContainer, fileName, onPostImport } = this.props;
+    const { selectedCollections, optionsMap } = this.state;
+
+    // init progress data
+    await this.setState({
+      isImporting: true,
+      progressMap: [],
+      errorsMap: [],
+    });
+
+    try {
+      // TODO: use appContainer.apiv3.post
+      await appContainer.apiv3Post('/import', {
+        fileName,
+        collections: Array.from(selectedCollections),
+        optionsMap,
+      });
+
+      if (onPostImport != null) {
+        onPostImport();
+      }
+
+      toastSuccess(undefined, 'Import process has requested.');
+    }
+    catch (err) {
+      toastError(err, 'Import request failed.');
+    }
+  }
+
+  renderWarnForGroups(errors, key) {
+    if (errors.length === 0) {
+      return null;
+    }
+
+    return (
+      <div key={key} className="alert alert-warning">
+        <ul>
+          { errors.map((error, index) => {
+            // eslint-disable-next-line react/no-array-index-key
+            return <li key={`${key}-${index}`}>{error}</li>;
+          }) }
+        </ul>
+      </div>
+    );
+  }
+
+  renderGroups(groupList, groupName, errors, { wellContent } = {}) {
+    const collectionNames = groupList.filter((collectionName) => {
+      return this.allCollectionNames.includes(collectionName);
+    });
+
+    if (collectionNames.length === 0) {
+      return null;
+    }
+
+    return (
+      <div className="mt-4">
+        <legend>{groupName} Collections</legend>
+        { wellContent != null && (
+          <div className="well well-sm small">
+            <ul>
+              <li>{wellContent}</li>
+            </ul>
+          </div>
+        ) }
+        { this.renderImportItems(collectionNames) }
+        { this.renderWarnForGroups(errors, `warnFor${groupName}`) }
+      </div>
+    );
+  }
+
+  renderOthers() {
+    const collectionNames = this.allCollectionNames.filter((collectionName) => {
+      return !ALL_GROUPED_COLLECTIONS.includes(collectionName);
+    });
+
+    return this.renderGroups(collectionNames, 'Other', this.state.warnForOtherGroups);
+  }
+
+  renderImportItems(collectionNames) {
+    const {
+      isImporting,
+      isImported,
+      progressMap,
+      errorsMap,
+
+      selectedCollections,
+      optionsMap,
+    } = this.state;
+
+    return (
+      <div className="row">
+        {collectionNames.map((collectionName) => {
+          const collectionProgress = progressMap[collectionName];
+          const errors = errorsMap[collectionName];
+          const isConfigButtonAvailable = Object.keys(IMPORT_OPTION_CLASS_MAPPING).includes(collectionName);
+
+          return (
+            <div className="col-xs-6 my-1" key={collectionName}>
+              <ImportCollectionItem
+                isImporting={isImporting}
+                isImported={collectionProgress ? isImported : false}
+                insertedCount={collectionProgress ? collectionProgress.insertedCount : 0}
+                modifiedCount={collectionProgress ? collectionProgress.modifiedCount : 0}
+                errorsCount={errors ? errors.length : 0}
+
+                collectionName={collectionName}
+                isSelected={selectedCollections.has(collectionName)}
+                option={optionsMap[collectionName]}
+
+                isConfigButtonAvailable={isConfigButtonAvailable}
+
+                onChange={this.toggleCheckbox}
+                onOptionChange={this.updateOption}
+                onConfigButtonClicked={this.openConfigurationModal}
+                onErrorLinkClicked={this.showErrorsViewer}
+              />
+            </div>
+          );
+        })}
+      </div>
+    );
+  }
+
+  renderConfigurationModal() {
+    const { isConfigurationModalOpen, collectionNameForConfiguration: collectionName, optionsMap } = this.state;
+
+    if (collectionName == null) {
+      return null;
+    }
+
+    return (
+      <ImportCollectionConfigurationModal
+        isOpen={isConfigurationModalOpen}
+        onClose={() => this.setState({ isConfigurationModalOpen: false })}
+        onOptionChange={this.updateOption}
+        collectionName={collectionName}
+        option={optionsMap[collectionName]}
+      />
+    );
+  }
+
+  renderErrorsViewer() {
+    const { isErrorsViewerOpen, errorsMap, collectionNameForErrorsViewer } = this.state;
+    const errors = errorsMap[collectionNameForErrorsViewer];
+
+    return (
+      <ErrorViewer
+        isOpen={isErrorsViewerOpen}
+        onClose={() => this.setState({ isErrorsViewerOpen: false })}
+        errors={errors}
+      />
+    );
+  }
+
+  render() {
+    const { t } = this.props;
+    const {
+      canImport, isImporting,
+      warnForPageGroups, warnForUserGroups, warnForConfigGroups,
+    } = this.state;
+
+    return (
+      <>
+        <form className="form-inline">
+          <div className="form-group">
+            <button type="button" className="btn btn-sm btn-default mr-2" onClick={this.checkAll}>
+              <i className="fa fa-check-square-o"></i> {t('export_management.check_all')}
+            </button>
+          </div>
+          <div className="form-group">
+            <button type="button" className="btn btn-sm btn-default mr-2" onClick={this.uncheckAll}>
+              <i className="fa fa-square-o"></i> {t('export_management.uncheck_all')}
+            </button>
+          </div>
+        </form>
+
+        { this.renderGroups(GROUPS_PAGE, 'Page', warnForPageGroups, { wellContent: t('importer_management.growi_settings.overwrite_documents') }) }
+        { this.renderGroups(GROUPS_USER, 'User', warnForUserGroups) }
+        { this.renderGroups(GROUPS_CONFIG, 'Config', warnForConfigGroups) }
+        { this.renderOthers() }
+
+        <div className="mt-4 text-center">
+          <button type="button" className="btn btn-default mx-1" onClick={this.props.onDiscard}>
+            { t('importer_management.growi_settings.discard') }
+          </button>
+          <button type="button" className="btn btn-primary mx-1" onClick={this.import} disabled={!canImport || isImporting}>
+            { t('importer_management.import') }
+          </button>
+        </div>
+
+        { this.renderConfigurationModal() }
+        { this.renderErrorsViewer() }
+      </>
+    );
+  }
+
+}
+
+ImportForm.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+  websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
+
+  fileName: PropTypes.string,
+  innerFileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
+  onDiscard: PropTypes.func.isRequired,
+  onPostImport: PropTypes.func,
+};
+
+/**
+ * Wrapper component for using unstated
+ */
+const ImportFormWrapper = (props) => {
+  return createSubscribedElement(ImportForm, props, [AppContainer, WebsocketContainer]);
+};
+
+export default withTranslation()(ImportFormWrapper);

src/client/js/components/Admin/Import/GrowiZipUploadForm.jsx → src/client/js/components/Admin/ImportData/GrowiArchive/UploadForm.jsx

@@ -2,11 +2,11 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 
-import { createSubscribedElement } from '../../UnstatedUtils';
-import AppContainer from '../../../services/AppContainer';
+import { createSubscribedElement } from '../../../UnstatedUtils';
+import AppContainer from '../../../../services/AppContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
-class GrowiZipUploadForm extends React.Component {
+class UploadForm extends React.Component {
 
   constructor(props) {
     super(props);
@@ -31,8 +31,7 @@ class GrowiZipUploadForm extends React.Component {
     formData.append('_csrf', this.props.appContainer.csrfToken);
     formData.append('file', this.inputRef.current.files[0]);
 
-    // TODO: use appContainer.apiv3.post
-    const { data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
+    const { data } = await this.props.appContainer.apiv3Post('/import/upload', formData);
     this.props.onUpload(data);
     // TODO: toastSuccess, toastError
   }
@@ -51,13 +50,14 @@ class GrowiZipUploadForm extends React.Component {
     return (
       <form className="form-horizontal" onSubmit={this.uploadZipFile}>
         <fieldset>
-          <div className="form-group d-flex align-items-center">
-            <label htmlFor="file" className="col-xs-3 control-label">{t('importer_management.growi_settings.zip_file')}</label>
+          <div className="form-group">
+            <label htmlFor="file" className="col-xs-3 control-label">{t('importer_management.growi_settings.growi_archive_file')}</label>
             <div className="col-xs-6">
               <input
                 type="file"
                 name="file"
                 className="form-control-file"
+                accept=".growi.zip"
                 ref={this.inputRef}
                 onChange={this.changeFileName}
               />
@@ -77,7 +77,7 @@ class GrowiZipUploadForm extends React.Component {
 
 }
 
-GrowiZipUploadForm.propTypes = {
+UploadForm.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
   onUpload: PropTypes.func.isRequired,
@@ -86,8 +86,8 @@ GrowiZipUploadForm.propTypes = {
 /**
  * Wrapper component for using unstated
  */
-const GrowiZipUploadFormWrapper = (props) => {
-  return createSubscribedElement(GrowiZipUploadForm, props, [AppContainer]);
+const UploadFormWrapper = (props) => {
+  return createSubscribedElement(UploadForm, props, [AppContainer]);
 };
 
-export default withTranslation()(GrowiZipUploadFormWrapper);
+export default withTranslation()(UploadFormWrapper);

src/client/js/components/Admin/Import/GrowiZipImportSection.jsx → src/client/js/components/Admin/ImportData/GrowiArchiveSection.jsx

@@ -3,20 +3,21 @@ import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 import * as toastr from 'toastr';
 
-import GrowiZipUploadForm from './GrowiZipUploadForm';
-import GrowiZipImportForm from './GrowiZipImportForm';
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
-class GrowiZipImportSection extends React.Component {
+import UploadForm from './GrowiArchive/UploadForm';
+import ImportForm from './GrowiArchive/ImportForm';
+
+class GrowiArchiveSection extends React.Component {
 
   constructor(props) {
     super(props);
 
     this.initialState = {
-      fileName: '',
-      fileStats: [],
+      fileName: null,
+      innerFileStats: null,
     };
 
     this.state = this.initialState;
@@ -26,17 +27,27 @@ class GrowiZipImportSection extends React.Component {
     this.resetState = this.resetState.bind(this);
   }
 
-  handleUpload({ meta, fileName, fileStats }) {
+  async componentWillMount() {
+    // get uploaded file status
+    const res = await this.props.appContainer.apiv3Get('/import/status');
+
+    if (res.data.zipFileStat != null) {
+      const { fileName, innerFileStats } = res.data.zipFileStat;
+      this.setState({ fileName, innerFileStats });
+    }
+  }
+
+  handleUpload({ meta, fileName, innerFileStats }) {
     this.setState({
       fileName,
-      fileStats,
+      innerFileStats,
     });
   }
 
   async discardData() {
     try {
       const { fileName } = this.state;
-      await this.props.appContainer.apiDelete(`/v3/import/${this.state.fileName}`, {});
+      await this.props.appContainer.apiv3Delete('/import/all');
       this.resetState();
 
       // TODO: toastSuccess, toastError
@@ -72,29 +83,18 @@ class GrowiZipImportSection extends React.Component {
 
     return (
       <Fragment>
-        <legend>{t('importer_management.import_form_growi')}</legend>
-
-        <div className="alert alert-warning">
-          <i className="icon-exclamation"></i> { t('importer_management.beta_warning') }
-        </div>
-
-        <div className="well well-sm small">
-          <ul>
-            <li>{t('importer_management.growi_settings.overwrite_documents')}</li>
-          </ul>
-        </div>
+        <h2>{t('importer_management.import_growi_archive')}</h2>
 
-        {this.state.fileName ? (
-          <Fragment>
-            <GrowiZipImportForm
+        { this.state.fileName != null ? (
+          <div className="px-4">
+            <ImportForm
               fileName={this.state.fileName}
-              fileStats={this.state.fileStats}
+              innerFileStats={this.state.innerFileStats}
               onDiscard={this.discardData}
-              onPostImport={this.resetState}
             />
-          </Fragment>
+          </div>
         ) : (
-          <GrowiZipUploadForm
+          <UploadForm
             onUpload={this.handleUpload}
           />
         )}
@@ -104,7 +104,7 @@ class GrowiZipImportSection extends React.Component {
 
 }
 
-GrowiZipImportSection.propTypes = {
+GrowiArchiveSection.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
 };
@@ -112,8 +112,8 @@ GrowiZipImportSection.propTypes = {
 /**
  * Wrapper component for using unstated
  */
-const GrowiZipImportSectionWrapper = (props) => {
-  return createSubscribedElement(GrowiZipImportSection, props, [AppContainer]);
+const GrowiArchiveSectionWrapper = (props) => {
+  return createSubscribedElement(GrowiArchiveSection, props, [AppContainer]);
 };
 
-export default withTranslation()(GrowiZipImportSectionWrapper);
+export default withTranslation()(GrowiArchiveSectionWrapper);

src/client/js/components/Admin/Importer.jsx → src/client/js/components/Admin/ImportDataPage.jsx

@@ -8,11 +8,11 @@ import { toastSuccess, toastError } from '../../util/apiNotification';
 
 import AppContainer from '../../services/AppContainer';
 
-import GrowiZipImportSection from './Import/GrowiZipImportSection';
+import GrowiArchiveSection from './ImportData/GrowiArchiveSection';
 
 const logger = loggerFactory('growi:importer');
 
-class Importer extends React.Component {
+class ImportDataPage extends React.Component {
 
   constructor(props) {
     super(props);
@@ -22,6 +22,7 @@ class Importer extends React.Component {
       qiitaTeamName: '',
       qiitaAccessToken: '',
     };
+
     this.esaHandleSubmit = this.esaHandleSubmit.bind(this);
     this.esaHandleSubmitTest = this.esaHandleSubmitTest.bind(this);
     this.esaHandleSubmitUpdate = this.esaHandleSubmitUpdate.bind(this);
@@ -134,15 +135,15 @@ class Importer extends React.Component {
     const { t } = this.props;
     return (
       <Fragment>
-        <GrowiZipImportSection />
+        <GrowiArchiveSection />
 
         <form
-          className="form-horizontal"
+          className="form-horizontal mt-5"
           id="importerSettingFormEsa"
           role="form"
         >
           <fieldset>
-            <legend>{ t('importer_management.import_form_esa') }</legend>
+            <legend>{ t('importer_management.import_from', { from: 'esa.io' }) }</legend>
             <table className="table table-bordered table-mapping">
               <thead>
                 <tr>
@@ -232,7 +233,7 @@ class Importer extends React.Component {
           role="form"
         >
           <fieldset>
-            <legend>{ t('importer_management.import_form_qiita', 'Qiita:Team') }</legend>
+            <legend>{ t('importer_management.import_from', { from: 'Qiita:Team' }) }</legend>
             <table className="table table-bordered table-mapping">
               <thead>
                 <tr>
@@ -329,16 +330,18 @@ class Importer extends React.Component {
 
 }
 
+ImportDataPage.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+};
+
+
 /**
  * Wrapper component for using unstated
  */
-const ImporterWrapper = (props) => {
-  return createSubscribedElement(Importer, props, [AppContainer]);
+const ImportDataPageWrapper = (props) => {
+  return createSubscribedElement(ImportDataPage, props, [AppContainer]);
 };
 
-Importer.propTypes = {
-  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
-  t: PropTypes.func.isRequired, // i18next
-};
 
-export default withTranslation()(ImporterWrapper);
+export default withTranslation()(ImportDataPageWrapper);

src/client/js/components/Admin/Users/Users.jsx → src/client/js/components/Admin/UserManagement.jsx

@@ -2,18 +2,20 @@ import React, { Fragment } from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 
-import PasswordResetModal from './PasswordResetModal';
-import PaginationWrapper from '../../PaginationWrapper';
-import InviteUserControl from './InviteUserControl';
-import UserTable from './UserTable';
+import PaginationWrapper from '../PaginationWrapper';
 
-import { createSubscribedElement } from '../../UnstatedUtils';
-import { toastError } from '../../../util/apiNotification';
 
-import AppContainer from '../../../services/AppContainer';
-import AdminUsersContainer from '../../../services/AdminUsersContainer';
+import { createSubscribedElement } from '../UnstatedUtils';
+import { toastError } from '../../util/apiNotification';
 
-class UserPage extends React.Component {
+import AppContainer from '../../services/AppContainer';
+import AdminUsersContainer from '../../services/AdminUsersContainer';
+
+import PasswordResetModal from './Users/PasswordResetModal';
+import InviteUserControl from './Users/InviteUserControl';
+import UserTable from './Users/UserTable';
+
+class UserManagement extends React.Component {
 
   constructor(props) {
     super();
@@ -21,6 +23,10 @@ class UserPage extends React.Component {
     this.handlePage = this.handlePage.bind(this);
   }
 
+  componentWillMount() {
+    this.handlePage(1);
+  }
+
   async handlePage(selectedPage) {
     try {
       await this.props.adminUsersContainer.retrieveUsersByPagingNum(selectedPage);
@@ -33,6 +39,17 @@ class UserPage extends React.Component {
   render() {
     const { t, adminUsersContainer } = this.props;
 
+    const pager = (
+      <div className="pull-right">
+        <PaginationWrapper
+          activePage={adminUsersContainer.state.activePage}
+          changePage={this.handlePage}
+          totalItemsCount={adminUsersContainer.state.totalUsers}
+          pagingLimit={adminUsersContainer.state.pagingLimit}
+        />
+      </div>
+    );
+
     return (
       <Fragment>
         {adminUsersContainer.state.userForPasswordResetModal && <PasswordResetModal />}
@@ -43,28 +60,28 @@ class UserPage extends React.Component {
             { t('user_management.external_account') }
           </a>
         </p>
+
+        <h2>{ t('User_Management') }</h2>
+
+        {pager}
         <UserTable />
-        <PaginationWrapper
-          activePage={adminUsersContainer.state.activePage}
-          changePage={this.handlePage}
-          totalItemsCount={adminUsersContainer.state.totalUsers}
-          pagingLimit={adminUsersContainer.state.pagingLimit}
-        />
+        {pager}
+
       </Fragment>
     );
   }
 
 }
 
-const UserPageWrapper = (props) => {
-  return createSubscribedElement(UserPage, props, [AppContainer, AdminUsersContainer]);
-};
 
-UserPage.propTypes = {
+UserManagement.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
   adminUsersContainer: PropTypes.instanceOf(AdminUsersContainer).isRequired,
+};
 
+const UserManagementWrapper = (props) => {
+  return createSubscribedElement(UserManagement, props, [AppContainer, AdminUsersContainer]);
 };
 
-export default withTranslation()(UserPageWrapper);
+export default withTranslation()(UserManagementWrapper);

src/client/js/components/Admin/Users/UserTable.jsx

@@ -66,14 +66,12 @@ class UserTable extends React.Component {
 
     return (
       <Fragment>
-        <h2>{ t('User_Management') }</h2>
-
         <table className="table table-default table-bordered table-user-list">
           <thead>
             <tr>
               <th width="100px">#</th>
               <th>{ t('status') }</th>
-              <th><code>{ t('User') }</code></th>
+              <th><code>username</code></th>
               <th>{ t('Name') }</th>
               <th>{ t('Email') }</th>
               <th width="100px">{ t('Created') }</th>

src/client/js/components/Page/RevisionBody.jsx

@@ -53,6 +53,7 @@ export default class RevisionBody extends React.Component {
           }
         }}
         className={`wiki ${additionalClassName}`}
+        // eslint-disable-next-line react/no-danger
         dangerouslySetInnerHTML={this.generateInnerHtml(this.props.html)}
       />
     );

src/client/js/components/PageHistory/RevisionDiff.jsx

@@ -34,6 +34,7 @@ export default class RevisionDiff extends React.Component {
     }
 
     const diffView = { __html: diffViewHTML };
+    // eslint-disable-next-line react/no-danger
     return <div className="revision-history-diff" dangerouslySetInnerHTML={diffView} />;
   }
 

src/client/js/components/SearchPage/SearchResultList.jsx

@@ -15,12 +15,16 @@ class SearchResultList extends React.Component {
 
   render() {
     const resultList = this.props.pages.map((page) => {
+      const showTags = (page.tags != null) && (page.tags.length > 0);
+
       return (
         <div id={page._id} key={page._id} className="search-result-page mb-5">
-          <h2><a href={page.path}>{page.path}</a></h2>
-          { page.tags.length > 0 && (
-            <span><i className="tag-icon icon-tag"></i> {page.tags.join(', ')}</span>
-          )}
+          <h2>
+            <a href={page.path}>{page.path}</a>
+            { showTags && (
+              <div className="mt-1 small"><i className="tag-icon icon-tag"></i> {page.tags.join(', ')}</div>
+            )}
+          </h2>
           <RevisionLoader
             growiRenderer={this.growiRenderer}
             pageId={page._id}

src/client/js/services/AdminUsersContainer.js

@@ -17,7 +17,7 @@ export default class AdminUsersContainer extends Container {
     this.appContainer = appContainer;
 
     this.state = {
-      users: JSON.parse(document.getElementById('admin-user-page').getAttribute('users')) || [],
+      users: [],
       isPasswordResetModalShown: false,
       isUserInviteModalShown: false,
       userForPasswordResetModal: null,
@@ -46,11 +46,13 @@ export default class AdminUsersContainer extends Container {
   async retrieveUsersByPagingNum(selectedPage) {
 
     const params = { page: selectedPage };
-    const response = await this.appContainer.apiv3.get('/users', params);
+    const { data } = await this.appContainer.apiv3.get('/users', params);
 
-    const users = response.data.users;
-    const totalUsers = response.data.totalUsers;
-    const pagingLimit = response.data.pagingLimit;
+    if (data.paginateResult == null) {
+      throw new Error('data must conclude \'paginateResult\' property.');
+    }
+
+    const { docs: users, totalDocs: totalUsers, limit: pagingLimit } = data.paginateResult;
 
     this.setState({
       users,

src/client/js/util/apiNotification.js

@@ -10,7 +10,7 @@ const toastrOption = {
     newestOnTop: false,
     showDuration: '100',
     hideDuration: '100',
-    timeOut: '3000',
+    timeOut: '0',
   },
   success: {
     closeButton: true,

src/client/styles/scss/_layout.scss

@@ -17,7 +17,8 @@
     }
 
     .nav-item-admin,
-    .nav-item-create-page {
+    .nav-item-create-page,
+    .nav-item-help {
       span {
         margin-left: 0.5em;
         @media (max-width: $screen-xs-min) {

src/client/styles/scss/_override-bootstrap-variables.scss

@@ -26,7 +26,7 @@ $brand-danger:          $danger;
 //## Font, line-height, and color for body text, headings, and more.
 $font-family-sans-serif:  Lato, -apple-system, BlinkMacSystemFont, 'Hiragino Kaku Gothic ProN', Meiryo, sans-serif;
 $font-family-serif:       Georgia, "Times New Roman", Times, serif;
-$font-family-monospace:   Osaka-Mono, "MS Gothic", Monaco, Menlo, Consolas, "Courier New", monospace;
+$font-family-monospace:   SFMono-Regular,Consolas,Liberation Mono,Menlo,monospace;
 $font-family-base:        $font-family-sans-serif;
 
 

src/lib/models/admin/growi-archive-import-option.js

@@ -0,0 +1,13 @@
+class GrowiArchiveImportOption {
+
+  constructor(mode, initProps = {}) {
+    this.mode = mode;
+
+    Object.entries(initProps).forEach(([key, value]) => {
+      this[key] = value;
+    });
+  }
+
+}
+
+module.exports = GrowiArchiveImportOption;

src/lib/models/admin/import-option-for-pages.js

@@ -0,0 +1,20 @@
+const GrowiArchiveImportOption = require('./growi-archive-import-option');
+
+const DEFAULT_PROPS = {
+  isOverwriteAuthorWithCurrentUser: false,
+  makePublicForGrant2: false,
+  makePublicForGrant4: false,
+  makePublicForGrant5: false,
+  initPageMetadatas: false,
+  initHackmdDatas: false,
+};
+
+class ImportOptionForPages extends GrowiArchiveImportOption {
+
+  constructor(mode, initProps) {
+    super(mode, initProps || DEFAULT_PROPS);
+  }
+
+}
+
+module.exports = ImportOptionForPages;

src/lib/models/admin/import-option-for-revisions.js

@@ -0,0 +1,15 @@
+const GrowiArchiveImportOption = require('./growi-archive-import-option');
+
+const DEFAULT_PROPS = {
+  isOverwriteAuthorWithCurrentUser: false,
+};
+
+class ImportOptionForRevisions extends GrowiArchiveImportOption {
+
+  constructor(mode, initProps) {
+    super(mode, initProps || DEFAULT_PROPS);
+  }
+
+}
+
+module.exports = ImportOptionForRevisions;

src/lib/util/mongoose-utils.js

@@ -1,5 +1,15 @@
 const mongoose = require('mongoose');
 
+const getMongoUri = () => {
+  const { env } = process;
+
+  return env.MONGOLAB_URI // for B.C.
+    || env.MONGODB_URI // MONGOLAB changes their env name
+    || env.MONGOHQ_URL
+    || env.MONGO_URI
+    || ((env.NODE_ENV === 'test') ? 'mongodb://localhost/growi_test' : 'mongodb://localhost/growi');
+};
+
 const getModelSafely = (modelName) => {
   if (mongoose.modelNames().includes(modelName)) {
     return mongoose.model(modelName);
@@ -8,5 +18,6 @@ const getModelSafely = (modelName) => {
 };
 
 module.exports = {
+  getMongoUri,
   getModelSafely,
 };

src/server/crowi/index.js

@@ -6,6 +6,7 @@ const pkg = require('@root/package.json');
 const InterceptorManager = require('@commons/service/interceptor-manager');
 const CdnResourcesService = require('@commons/service/cdn-resources-service');
 const Xss = require('@commons/service/xss');
+const { getMongoUri } = require('@commons/util/mongoose-utils');
 
 const path = require('path');
 
@@ -74,14 +75,6 @@ function Crowi(rootdir) {
   };
 }
 
-function getMongoUrl(env) {
-  return env.MONGOLAB_URI // for B.C.
-    || env.MONGODB_URI // MONGOLAB changes their env name
-    || env.MONGOHQ_URL
-    || env.MONGO_URI
-    || ((process.env.NODE_ENV === 'test') ? 'mongodb://localhost/growi_test' : 'mongodb://localhost/growi');
-}
-
 Crowi.prototype.init = async function() {
   await this.setupDatabase();
   await this.setupModels();
@@ -202,10 +195,10 @@ Crowi.prototype.event = function(name, event) {
 };
 
 Crowi.prototype.setupDatabase = function() {
-  // mongoUri = mongodb://user:password@host/dbname
   mongoose.Promise = global.Promise;
 
-  const mongoUri = getMongoUrl(this.env);
+  // mongoUri = mongodb://user:password@host/dbname
+  const mongoUri = getMongoUri();
 
   return mongoose.connect(mongoUri, { useNewUrlParser: true });
 };
@@ -216,7 +209,7 @@ Crowi.prototype.setupSessionConfig = function() {
   const sessionAge = (1000 * 3600 * 24 * 30);
   const redisUrl = this.env.REDISTOGO_URL || this.env.REDIS_URI || this.env.REDIS_URL || null;
 
-  const mongoUrl = getMongoUrl(this.env);
+  const mongoUrl = getMongoUri();
   let sessionConfig;
 
   return new Promise(((resolve, reject) => {

src/server/middlewares/ApiV3FormValidator.js

@@ -1,11 +1,11 @@
 const logger = require('@alias/logger')('growi:middlewares:ApiV3FormValidator');
 const { validationResult } = require('express-validator/check');
 
+const ErrorV3 = require('../models/vo/error-apiv3');
+
 class ApiV3FormValidator {
 
   constructor(crowi) {
-    const { ErrorV3 } = crowi.models;
-
     return (req, res, next) => {
       logger.debug('req.query', req.query);
       logger.debug('req.params', req.params);

src/server/models/bookmark.js

@@ -21,6 +21,23 @@ module.exports = function(crowi) {
     return await this.count({ page: pageId });
   };
 
+  /**
+   * @return {object} key: page._id, value: bookmark count
+   */
+  bookmarkSchema.statics.getPageIdToCountMap = async function(pageIds) {
+    const results = await this.aggregate()
+      .match({ page: { $in: pageIds } })
+      .group({ _id: '$page', count: { $sum: 1 } });
+
+    // convert to map
+    const idToCountMap = {};
+    results.forEach((result) => {
+      idToCountMap[result._id] = result.count;
+    });
+
+    return idToCountMap;
+  };
+
   bookmarkSchema.statics.populatePage = async function(bookmarks) {
     const Bookmark = this;
     const User = crowi.model('User');
@@ -122,12 +139,12 @@ module.exports = function(crowi) {
     }
   };
 
-  bookmarkSchema.statics.removeBookmark = async function(page, user) {
+  bookmarkSchema.statics.removeBookmark = async function(pageId, user) {
     const Bookmark = this;
 
     try {
-      const data = await Bookmark.findOneAndRemove({ page, user });
-      bookmarkEvent.emit('delete', page);
+      const data = await Bookmark.findOneAndRemove({ page: pageId, user });
+      bookmarkEvent.emit('delete', pageId);
       return data;
     }
     catch (err) {

src/server/models/external-account.js

@@ -3,7 +3,7 @@
 
 const debug = require('debug')('growi:models:external-account');
 const mongoose = require('mongoose');
-const mongoosePaginate = require('mongoose-paginate');
+const mongoosePaginate = require('mongoose-paginate-v2');
 const uniqueValidator = require('mongoose-unique-validator');
 
 const ObjectId = mongoose.Schema.Types.ObjectId;

src/server/models/index.js

@@ -15,7 +15,4 @@ module.exports = {
   GlobalNotificationSetting: require('./GlobalNotificationSetting'),
   GlobalNotificationMailSetting: require('./GlobalNotificationSetting/GlobalNotificationMailSetting'),
   GlobalNotificationSlackSetting: require('./GlobalNotificationSetting/GlobalNotificationSlackSetting'),
-
-  // non-persistent models
-  ErrorV3: require('./ErrorV3'),
 };

src/server/models/openapi/paginate-result.js

@@ -0,0 +1,48 @@
+/**
+ * @see https://www.npmjs.com/package/mongoose-paginate-v2
+ * @swagger
+ *
+ *  components:
+ *    schemas:
+ *      PaginateResult:
+ *        type: object
+ *        properties:
+ *          docs:
+ *            type: array
+ *            description: Array of documents
+ *            items:
+ *              type: object
+ *          totalDocs:
+ *            type: number
+ *            description: Total number of documents in collection that match a query
+ *          limit:
+ *            type: number
+ *            description: Limit that was used
+ *          hasPrevPage:
+ *            type: number
+ *            description: Availability of prev page.
+ *          hasNextPage:
+ *            type: number
+ *            description: Availability of next page.
+ *          page:
+ *            type: number
+ *            description: Current page number
+ *          totalPages:
+ *            type: number
+ *            description: Total number of pages.
+ *          offset:
+ *            type: number
+ *            description: Only if specified or default page/offset values were used
+ *          prefPage:
+ *            type: number
+ *            description: Previous page number if available or NULL
+ *          nextPage:
+ *            type: number
+ *            description: Next page number if available or NULL
+ *          pagingCounter:
+ *            type: number
+ *            description: The starting sl. number of first document.
+ *          meta:
+ *            type: number
+ *            description: Object of pagination meta data (Default false).
+ */

src/server/models/page-tag-relation.js

@@ -1,8 +1,10 @@
 // disable no-return-await for model functions
 /* eslint-disable no-return-await */
 
+const flatMap = require('array.prototype.flatmap');
+
 const mongoose = require('mongoose');
-const mongoosePaginate = require('mongoose-paginate');
+const mongoosePaginate = require('mongoose-paginate-v2');
 
 const ObjectId = mongoose.Schema.Types.ObjectId;
 
@@ -60,6 +62,52 @@ class PageTagRelation {
     return relations.map((relation) => { return relation.relatedTag.name });
   }
 
+  /**
+   * @return {object} key: Page._id, value: array of tag names
+   */
+  static async getIdToTagNamesMap(pageIds) {
+    /**
+     * @see https://docs.mongodb.com/manual/reference/operator/aggregation/group/#pivot-data
+     *
+     * results will be:
+     * [
+     *   { _id: 58dca7b2c435b3480098dbbc, tagIds: [ 5da630f71a677515601e36d7, 5da77163ec786e4fe43e0e3e ]},
+     *   { _id: 58dca7b2c435b3480098dbbd, tagIds: [ ... ]},
+     *   ...
+     * ]
+     */
+    const results = await this.aggregate()
+      .match({ relatedPage: { $in: pageIds } })
+      .group({ _id: '$relatedPage', tagIds: { $push: '$relatedTag' } });
+
+    if (results.length === 0) {
+      return {};
+    }
+
+    results.flatMap = flatMap.shim(); // TODO: remove after upgrading to node v12
+
+    // extract distinct tag ids
+    const allTagIds = results
+      .flatMap(result => result.tagIds); // map + flatten
+    const distinctTagIds = Array.from(new Set(allTagIds));
+
+    // retrieve tag documents
+    const Tag = mongoose.model('Tag');
+    const tagIdToNameMap = await Tag.getIdToNameMap(distinctTagIds);
+
+    // convert to map
+    const idToTagNamesMap = {};
+    results.forEach((result) => {
+      const tagNames = result.tagIds
+        .map(tagId => tagIdToNameMap[tagId])
+        .filter(tagName => tagName != null); // filter null object
+
+      idToTagNamesMap[result._id] = tagNames;
+    });
+
+    return idToTagNamesMap;
+  }
+
   static async updatePageTags(pageId, tags) {
     if (pageId == null || tags == null) {
       throw new Error('args \'pageId\' and \'tags\' are required.');

src/server/models/page.js

@@ -7,7 +7,7 @@ const debug = require('debug')('growi:models:page');
 const nodePath = require('path');
 const urljoin = require('url-join');
 const mongoose = require('mongoose');
-const mongoosePaginate = require('mongoose-paginate');
+const mongoosePaginate = require('mongoose-paginate-v2');
 const uniqueValidator = require('mongoose-unique-validator');
 
 const { pathUtils } = require('growi-commons');
@@ -923,21 +923,6 @@ module.exports = function(crowi) {
     return { templateBody, templateTags };
   };
 
-  /**
-   * Bulk get (for internal only)
-   */
-  pageSchema.statics.getStreamOfFindAll = function(options) {
-    const criteria = { redirectTo: null };
-
-    return this.find(criteria)
-      .populate([
-        { path: 'creator', model: 'User' },
-        { path: 'revision', model: 'Revision' },
-      ])
-      .lean()
-      .cursor();
-  };
-
   async function pushRevision(pageData, newRevision, user) {
     await newRevision.save();
     debug('Successfully saved new revision', newRevision);
@@ -1384,10 +1369,6 @@ module.exports = function(crowi) {
     return addSlashOfEnd(path);
   };
 
-  pageSchema.statics.allPageCount = function() {
-    return this.count({ redirectTo: null });
-  };
-
   pageSchema.statics.GRANT_PUBLIC = GRANT_PUBLIC;
   pageSchema.statics.GRANT_RESTRICTED = GRANT_RESTRICTED;
   pageSchema.statics.GRANT_SPECIFIED = GRANT_SPECIFIED;

src/server/models/tag.js

@@ -2,7 +2,7 @@
 /* eslint-disable no-return-await */
 
 const mongoose = require('mongoose');
-const mongoosePaginate = require('mongoose-paginate');
+const mongoosePaginate = require('mongoose-paginate-v2');
 
 /*
  * define schema
@@ -23,6 +23,17 @@ schema.plugin(mongoosePaginate);
  */
 class Tag {
 
+  static async getIdToNameMap(tagIds) {
+    const tags = await this.find({ _id: { $in: tagIds } });
+
+    const idToNameMap = {};
+    tags.forEach((tag) => {
+      idToNameMap[tag._id.toString()] = tag.name;
+    });
+
+    return idToNameMap;
+  }
+
   static async findOrCreate(tagName) {
     const tag = await this.findOne({ name: tagName });
     if (!tag) {

src/server/models/user-group-relation.js

@@ -1,6 +1,6 @@
 const debug = require('debug')('growi:models:userGroupRelation');
 const mongoose = require('mongoose');
-const mongoosePaginate = require('mongoose-paginate');
+const mongoosePaginate = require('mongoose-paginate-v2');
 
 const ObjectId = mongoose.Schema.Types.ObjectId;
 

src/server/models/user-group.js

@@ -1,6 +1,6 @@
 const debug = require('debug')('growi:models:userGroup');
 const mongoose = require('mongoose');
-const mongoosePaginate = require('mongoose-paginate');
+const mongoosePaginate = require('mongoose-paginate-v2');
 
 
 /*

src/server/models/user.js

@@ -3,9 +3,9 @@
 const debug = require('debug')('growi:models:user');
 const logger = require('@alias/logger')('growi:models:user');
 const mongoose = require('mongoose');
+const mongoosePaginate = require('mongoose-paginate-v2');
 const path = require('path');
 const uniqueValidator = require('mongoose-unique-validator');
-const mongoosePaginate = require('mongoose-paginate');
 
 const ObjectId = mongoose.Schema.Types.ObjectId;
 const crypto = require('crypto');
@@ -426,17 +426,6 @@ module.exports = function(crowi) {
       });
   };
 
-  userSchema.statics.findUsersWithPagination = async function(options) {
-    const defaultOptions = {
-      sort: { status: 1, username: 1, createdAt: 1 },
-      page: 1,
-      limit: PAGE_ITEMS,
-    };
-    const mergedOptions = Object.assign(defaultOptions, options);
-
-    return this.paginate({ status: { $ne: STATUS_DELETED } }, mergedOptions);
-  };
-
   userSchema.statics.findUsersByPartOfEmail = function(emailPart, options) {
     const status = options.status || null;
     const emailPartRegExp = new RegExp(emailPart.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'));
@@ -504,15 +493,12 @@ module.exports = function(crowi) {
   };
 
   userSchema.statics.isUserCountExceedsUpperLimit = async function() {
-    const { aclService } = crowi;
+    const { configManager } = crowi;
 
-    const userUpperLimit = aclService.userUpperLimit();
-    if (userUpperLimit === 0) {
-      return false;
-    }
+    const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
 
     const activeUsers = await this.countListByStatus(STATUS_ACTIVE);
-    if (userUpperLimit !== 0 && userUpperLimit <= activeUsers) {
+    if (userUpperLimit <= activeUsers) {
       return true;
     }
 

src/server/models/vo/collection-progress.js

@@ -0,0 +1,13 @@
+class CollectionProgress {
+
+  constructor(collectionName, totalCount) {
+    this.collectionName = collectionName;
+    this.currentCount = 0;
+    this.insertedCount = 0;
+    this.modifiedCount = 0;
+    this.totalCount = totalCount;
+  }
+
+}
+
+module.exports = CollectionProgress;

src/server/models/vo/collection-progressing-status.js

@@ -0,0 +1,35 @@
+const CollectionProgress = require('./collection-progress');
+
+class CollectionProgressingStatus {
+
+  constructor(collections) {
+    this.totalCount = 0;
+    this.progressMap = {};
+
+    this.progressList = collections.map((collectionName) => {
+      return new CollectionProgress(collectionName, 0);
+    });
+
+    // collection name to instance mapping
+    this.progressList.forEach((p) => {
+      this.progressMap[p.collectionName] = p;
+    });
+  }
+
+  recalculateTotalCount() {
+    this.progressList.forEach((p) => {
+      this.progressMap[p.collectionName] = p;
+      this.totalCount += p.totalCount;
+    });
+  }
+
+  get currentCount() {
+    return this.progressList.reduce(
+      (acc, crr) => acc + crr.currentCount,
+      0,
+    );
+  }
+
+}
+
+module.exports = CollectionProgressingStatus;

src/server/models/ErrorV3.js → src/server/models/vo/error-apiv3.js

@@ -1,13 +1,12 @@
 class ErrorV3 extends Error {
 
-  constructor(message = '', code = '') {
+  constructor(message = '', code = '', stack = undefined) {
     super(); // do not provide message to the super constructor
     this.message = message;
     this.code = code;
+    this.stack = stack;
   }
 
 }
 
-module.exports = function(crowi) {
-  return ErrorV3;
-};
+module.exports = ErrorV3;

src/server/routes/admin.js

@@ -91,6 +91,14 @@ module.exports = function(crowi, app) {
     return pager;
   }
 
+  // setup websocket event for rebuild index
+  searchEvent.on('addPageProgress', (total, current, skip) => {
+    crowi.getIo().sockets.emit('admin:addPageProgress', { total, current, skip });
+  });
+  searchEvent.on('finishAddPage', (total, current, skip) => {
+    crowi.getIo().sockets.emit('admin:finishAddPage', { total, current, skip });
+  });
+
   actions.index = function(req, res) {
     return res.render('admin/index', {
       plugins: pluginUtils.listPlugins(crowi.rootDir),
@@ -398,27 +406,7 @@ module.exports = function(crowi, app) {
 
   actions.user = {};
   actions.user.index = async function(req, res) {
-    const activeUsers = await User.countListByStatus(User.STATUS_ACTIVE);
-    const userUpperLimit = aclService.userUpperLimit();
-    const isUserCountExceedsUpperLimit = await User.isUserCountExceedsUpperLimit();
-
-    const page = parseInt(req.query.page) || 1;
-
-    const result = await User.findUsersWithPagination({
-      page,
-      select: `${User.USER_PUBLIC_FIELDS} lastLoginAt`,
-      populate: User.IMAGE_POPULATION,
-    });
-
-    const pager = createPager(result.total, result.limit, result.page, result.pages, MAX_PAGE_LIST);
-
-    return res.render('admin/users', {
-      users: result.docs,
-      pager,
-      activeUsers,
-      userUpperLimit,
-      isUserCountExceedsUpperLimit,
-    });
+    return res.render('admin/users');
   };
 
   // これやったときの relation の挙動未確認
@@ -1010,7 +998,6 @@ module.exports = function(crowi, app) {
     const { validationResult } = require('express-validator');
     const errors = validationResult(req);
     if (!errors.isEmpty()) {
-      console.log('validator', errors);
       return res.json(ApiResponse.error('Qiita form is blank'));
     }
 
@@ -1105,14 +1092,12 @@ module.exports = function(crowi, app) {
       return res.json(ApiResponse.error('ElasticSearch Integration is not set up.'));
     }
 
-    searchEvent.on('addPageProgress', (total, current, skip) => {
-      crowi.getIo().sockets.emit('admin:addPageProgress', { total, current, skip });
-    });
-    searchEvent.on('finishAddPage', (total, current, skip) => {
-      crowi.getIo().sockets.emit('admin:finishAddPage', { total, current, skip });
-    });
-
-    await search.buildIndex();
+    try {
+      search.buildIndex();
+    }
+    catch (err) {
+      return res.json(ApiResponse.error(err));
+    }
 
     return res.json(ApiResponse.success());
   };

src/server/routes/apiv3/import.js

@@ -3,13 +3,16 @@ const loggerFactory = require('@alias/logger');
 const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-line no-unused-vars
 
 const path = require('path');
-const fs = require('fs');
 const multer = require('multer');
 
+// eslint-disable-next-line no-unused-vars
 const { ObjectId } = require('mongoose').Types;
 
 const express = require('express');
 
+const GrowiArchiveImportOption = require('@commons/models/admin/growi-archive-import-option');
+
+
 const router = express.Router();
 
 /**
@@ -18,6 +21,44 @@ const router = express.Router();
  *    name: Import
  */
 
+/**
+ * @swagger
+ *
+ *  components:
+ *    schemas:
+ *      ImportStatus:
+ *        type: object
+ *        properties:
+ *          zipFileStat:
+ *            type: object
+ *            description: the property object
+ *          progressList:
+ *            type: array
+ *            items:
+ *              type: object
+ *              description: progress data for each exporting collections
+ *          isImporting:
+ *            type: boolean
+ *            description: whether the current importing job exists or not
+ */
+
+/**
+ * generate overwrite params with overwrite-params/* modules
+ * @param {string} collectionName
+ * @param {object} req Request Object
+ * @param {GrowiArchiveImportOption} options GrowiArchiveImportOption instance
+ */
+const generateOverwriteParams = (collectionName, req, options) => {
+  switch (collectionName) {
+    case 'pages':
+      return require('./overwrite-params/pages')(req, options);
+    case 'revisions':
+      return require('./overwrite-params/revisions')(req, options);
+    default:
+      return {};
+  }
+};
+
 module.exports = (crowi) => {
   const { growiBridgeService, importService } = crowi;
   const accessTokenParser = require('../../middleware/access-token-parser')(crowi);
@@ -25,6 +66,19 @@ module.exports = (crowi) => {
   const adminRequired = require('../../middleware/admin-required')(crowi);
   const csrf = require('../../middleware/csrf')(crowi);
 
+  this.adminEvent = crowi.event('admin');
+
+  // setup event
+  this.adminEvent.on('onProgressForImport', (data) => {
+    crowi.getIo().sockets.emit('admin:onProgressForImport', data);
+  });
+  this.adminEvent.on('onTerminateForImport', (data) => {
+    crowi.getIo().sockets.emit('admin:onTerminateForImport', data);
+  });
+  this.adminEvent.on('onErrorForImport', (data) => {
+    crowi.getIo().sockets.emit('admin:onErrorForImport', data);
+  });
+
   const uploads = multer({
     storage: multer.diskStorage({
       destination: (req, file, cb) => {
@@ -43,48 +97,33 @@ module.exports = (crowi) => {
     },
   });
 
+
   /**
-   * defined overwrite params for each collection
-   * all imported documents are overwriten by this value
-   * each value can be any value or a function (_value, { _document, key, schema }) { return newValue }
+   * @swagger
    *
-   * @param {object} Model instance of mongoose model
-   * @param {object} req request object
-   * @return {object} document to be persisted
+   *  /import/status:
+   *    get:
+   *      tags: [Import]
+   *      description: Get properties of stored zip files for import
+   *      responses:
+   *        200:
+   *          description: the zip file statuses
+   *          content:
+   *            application/json:
+   *              schema:
+   *                properties:
+   *                  status:
+   *                    $ref: '#/components/schemas/ImportStatus'
    */
-  const overwriteParamsFn = async(Model, schema, req) => {
-    const collectionName = Model.collection.name;
-
-    /* eslint-disable no-case-declarations */
-    switch (Model.collection.collectionName) {
-      case 'pages':
-        // TODO: use schema and req to generate overwriteParams
-        // e.g. { creator: schema.creator === 'me' ? ObjectId(req.user._id) : importService.keepOriginal }
-        return {
-          status: 'published', // FIXME when importing users and user groups
-          grant: 1, // FIXME when importing users and user groups
-          grantedUsers: [], // FIXME when importing users and user groups
-          grantedGroup: null, // FIXME when importing users and user groups
-          creator: ObjectId(req.user._id), // FIXME when importing users
-          lastUpdateUser: ObjectId(req.user._id), // FIXME when importing users
-          liker: [], // FIXME when importing users
-          seenUsers: [], // FIXME when importing users
-          commentCount: 0, // FIXME when importing comments
-          extended: {}, // FIXME when ?
-          pageIdOnHackmd: undefined, // FIXME when importing hackmd?
-          revisionHackmdSynced: undefined, // FIXME when importing hackmd?
-          hasDraftOnHackmd: undefined, // FIXME when importing hackmd?
-        };
-      // case 'revisoins':
-      //   return {};
-      // case 'users':
-      //   return {};
-      // ... add more cases
-      default:
-        throw new Error(`cannot find a model for collection name "${collectionName}"`);
+  router.get('/status', accessTokenParser, loginRequired, adminRequired, async(req, res) => {
+    try {
+      const status = await importService.getStatus();
+      return res.apiv3(status);
+    }
+    catch (err) {
+      return res.apiv3Err(err, 500);
     }
-    /* eslint-enable no-case-declarations */
-  };
+  });
 
   /**
    * @swagger
@@ -93,66 +132,105 @@ module.exports = (crowi) => {
    *    post:
    *      tags: [Import]
    *      description: import a collection from a zipped json
+   *      requestBody:
+   *        required: true
+   *        content:
+   *          application/json:
+   *            schema:
+   *              type: object
+   *              properties:
+   *                fileName:
+   *                  description: the file name of zip file
+   *                  type: string
+   *                collections:
+   *                  description: collection names to import
+   *                  type: array
+   *                  items:
+   *                    type: string
+   *                optionsMap:
+   *                  description: |
+   *                    the map object of importing option that have collection name as the key
+   *                  additionalProperties:
+   *                    type: object
+   *                    properties:
+   *                      mode:
+   *                        description: Import mode
+   *                        type: string
+   *                        enum: [insert, upsert, flushAndInsert]
    *      responses:
    *        200:
-   *          description: the data is successfully imported
-   *          content:
-   *            application/json:
-   *              schema:
-   *                properties:
-   *                  results:
-   *                    type: array
-   *                    items:
-   *                      type: object
-   *                      description: collectionName, insertedIds, failedIds
+   *          description: Import process has requested
    */
   router.post('/', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
     // TODO: add express validator
 
-    const { fileName, collections, schema } = req.body;
+    const { fileName, collections, optionsMap } = req.body;
     const zipFile = importService.getFile(fileName);
 
-    // unzip
-    await importService.unzip(zipFile);
-    // eslint-disable-next-line no-unused-vars
-    const { meta, fileStats } = await growiBridgeService.parseZipFile(zipFile);
+    // return response first
+    res.apiv3();
 
-    // delete zip file after unzipping and parsing it
-    fs.unlinkSync(zipFile);
+    /*
+     * unzip, parse
+     */
+    let meta = null;
+    let fileStatsToImport = null;
+    try {
+      // unzip
+      await importService.unzip(zipFile);
 
-    // filter fileStats
-    const filteredFileStats = fileStats.filter(({ fileName, collectionName, size }) => { return collections.includes(collectionName) });
+      // eslint-disable-next-line no-unused-vars
+      const { meta: parsedMeta, fileStats, innerFileStats } = await growiBridgeService.parseZipFile(zipFile);
+      meta = parsedMeta;
 
+      // filter innerFileStats
+      fileStatsToImport = innerFileStats.filter(({ fileName, collectionName, size }) => {
+        return collections.includes(collectionName);
+      });
+    }
+    catch (err) {
+      logger.error(err);
+      this.adminEvent.emit('onErrorForImport', { message: err.message });
+      return;
+    }
+
+    /*
+     * validate with meta.json
+     */
     try {
-      // validate with meta.json
       importService.validate(meta);
+    }
+    catch (err) {
+      logger.error(err);
+      this.adminEvent.emit('onErrorForImport', { message: err.message });
+      return;
+    }
 
-      const results = await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
-        const Model = growiBridgeService.getModelFromCollectionName(collectionName);
-        const jsonFile = importService.getFile(fileName);
+    // generate maps of ImportSettings to import
+    const importSettingsMap = {};
+    fileStatsToImport.forEach(({ fileName, collectionName }) => {
+      // instanciate GrowiArchiveImportOption
+      const options = new GrowiArchiveImportOption(null, optionsMap[collectionName]);
 
-        let overwriteParams;
-        if (overwriteParamsFn[collectionName] != null) {
-          // await in case overwriteParamsFn[collection] is a Promise
-          overwriteParams = await overwriteParamsFn(Model, schema[collectionName], req);
-        }
+      // generate options
+      const importSettings = importService.generateImportSettings(options.mode);
+      importSettings.jsonFileName = fileName;
 
-        const { insertedIds, failedIds } = await importService.import(Model, jsonFile, overwriteParams);
+      // generate overwrite params
+      importSettings.overwriteParams = generateOverwriteParams(collectionName, req, options);
 
-        return {
-          collectionName,
-          insertedIds,
-          failedIds,
-        };
-      }));
+      importSettingsMap[collectionName] = importSettings;
+    });
 
-      // TODO: use res.apiv3
-      return res.send({ ok: true, results });
+    /*
+     * import
+     */
+    try {
+      importService.import(collections, importSettingsMap);
     }
     catch (err) {
-      // TODO: use ApiV3Error
       logger.error(err);
-      return res.status(500).send({ status: 'ERROR' });
+      this.adminEvent.emit('onErrorForImport', { message: err.message });
     }
   });
 
@@ -192,11 +270,7 @@ module.exports = (crowi) => {
       // validate with meta.json
       importService.validate(data.meta);
 
-      // TODO: use res.apiv3
-      return res.send({
-        ok: true,
-        data,
-      });
+      return res.apiv3(data);
     }
     catch (err) {
       // TODO: use ApiV3Error
@@ -208,41 +282,23 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /import/{fileName}:
-   *    post:
+   *  /import/all:
+   *    delete:
    *      tags: [Import]
-   *      description: delete a zip file
-   *      parameters:
-   *        - name: fileName
-   *          in: path
-   *          description: the file name of zip file
-   *          required: true
-   *          schema:
-   *            type: string
+   *      description: Delete all zip files
    *      responses:
    *        200:
-   *          description: the file is deleted
-   *          content:
-   *            application/json:
-   *              schema:
-   *                type: object
+   *          description: all files are deleted
    */
-  router.delete('/:fileName', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
-    const { fileName } = req.params;
-
+  router.delete('/all', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
     try {
-      const zipFile = importService.getFile(fileName);
-      fs.unlinkSync(zipFile);
+      importService.deleteAllZipFiles();
 
-      // TODO: use res.apiv3
-      return res.send({
-        ok: true,
-      });
+      return res.apiv3();
     }
     catch (err) {
-      // TODO: use ApiV3Error
       logger.error(err);
-      return res.status(500).send({ status: 'ERROR' });
+      return res.apiv3Err(err, 500);
     }
   });
 

src/server/routes/apiv3/markdown-setting.js

@@ -1,6 +1,6 @@
-/* eslint-disable no-unused-vars */
 const loggerFactory = require('@alias/logger');
 
+// eslint-disable-next-line no-unused-vars
 const logger = loggerFactory('growi:routes:apiv3:user-group');
 
 const express = require('express');
@@ -9,6 +9,8 @@ const router = express.Router();
 
 const { body } = require('express-validator/check');
 
+const ErrorV3 = require('../../models/vo/error-apiv3');
+
 const validator = {
   lineBreak: [
     body('isEnabledLinebreaks').isBoolean(),
@@ -24,6 +26,7 @@ const validator = {
   ],
 };
 
+
 /**
  * @swagger
  *  tags:
@@ -35,10 +38,9 @@ module.exports = (crowi) => {
   const adminRequired = require('../../middleware/admin-required')(crowi);
   const csrf = require('../../middleware/csrf')(crowi);
 
-  const {
-    ErrorV3,
-    Config,
-  } = crowi.models;
+  // const {
+  //   Config,
+  // } = crowi.models;
 
   const { ApiV3FormValidator } = crowi.middlewares;
 

src/server/routes/apiv3/overwrite-params/pages.js

@@ -0,0 +1,62 @@
+const mongoose = require('mongoose');
+
+// eslint-disable-next-line no-unused-vars
+const ImportOptionForPages = require('@commons/models/admin/import-option-for-pages');
+
+const { ObjectId } = mongoose.Types;
+
+const {
+  GRANT_PUBLIC,
+} = mongoose.model('Page');
+
+class PageOverwriteParamsFactory {
+
+  /**
+   * generate overwrite params object
+   * @param {object} req
+   * @param {ImportOptionForPages} option
+   * @return object
+   *  key: property name
+   *  value: any value or a function `(value, { document, schema, propertyName }) => { return newValue }`
+   */
+  static generate(req, option) {
+    const params = {};
+
+    if (option.isOverwriteAuthorWithCurrentUser) {
+      const userId = ObjectId(req.user._id);
+      params.creator = userId;
+      params.lastUpdateUser = userId;
+    }
+
+    params.grant = (value, { document, schema, propertyName }) => {
+      if (option.makePublicForGrant2 && value === 2) {
+        return GRANT_PUBLIC;
+      }
+      if (option.makePublicForGrant4 && value === 4) {
+        return GRANT_PUBLIC;
+      }
+      if (option.makePublicForGrant5 && value === 5) {
+        return GRANT_PUBLIC;
+      }
+      return value;
+    };
+
+    if (option.initPageMetadatas) {
+      params.liker = [];
+      params.seenUsers = [];
+      params.commentCount = 0;
+      params.extended = {};
+    }
+
+    if (option.initHackmdDatas) {
+      params.pageIdOnHackmd = undefined;
+      params.revisionHackmdSynced = undefined;
+      params.hasDraftOnHackmd = undefined;
+    }
+
+    return params;
+  }
+
+}
+
+module.exports = (req, option) => PageOverwriteParamsFactory.generate(req, option);

src/server/routes/apiv3/overwrite-params/revisions.js

@@ -0,0 +1,31 @@
+const mongoose = require('mongoose');
+
+// eslint-disable-next-line no-unused-vars
+const ImportOptionForPages = require('@commons/models/admin/import-option-for-pages');
+
+const { ObjectId } = mongoose.Types;
+
+class RevisionOverwriteParamsFactory {
+
+  /**
+   * generate overwrite params object
+   * @param {object} req
+   * @param {ImportOptionForPages} option
+   * @return object
+   *  key: property name
+   *  value: any value or a function `(value, { document, schema, propertyName }) => { return newValue }`
+   */
+  static generate(req, option) {
+    const params = {};
+
+    if (option.isOverwriteAuthorWithCurrentUser) {
+      const userId = ObjectId(req.user._id);
+      params.author = userId;
+    }
+
+    return params;
+  }
+
+}
+
+module.exports = (req, option) => RevisionOverwriteParamsFactory.generate(req, option);

src/server/routes/apiv3/response.js

@@ -1,9 +1,10 @@
 const toArrayIfNot = require('../../../lib/util/toArrayIfNot');
 
+const ErrorV3 = require('../../models/vo/error-apiv3');
+
 const addCustomFunctionToResponse = (express, crowi) => {
-  const { ErrorV3 } = crowi.models;
 
-  express.response.apiv3 = function(obj) { // not arrow function
+  express.response.apiv3 = function(obj = {}) { // not arrow function
     // obj must be object
     if (typeof obj !== 'object' || obj instanceof Array) {
       throw new Error('invalid value supplied to res.apiv3');
@@ -22,6 +23,9 @@ const addCustomFunctionToResponse = (express, crowi) => {
       if (e instanceof ErrorV3) {
         return e;
       }
+      if (e instanceof Error) {
+        return new ErrorV3(e.message, null, e.stack);
+      }
       if (typeof e === 'string') {
         return { message: e };
       }

src/server/routes/apiv3/user-group-relation.js

@@ -4,6 +4,8 @@ const logger = loggerFactory('growi:routes:apiv3:user-group-relation'); // eslin
 
 const express = require('express');
 
+const ErrorV3 = require('../../models/vo/error-apiv3');
+
 const router = express.Router();
 
 /**
@@ -16,12 +18,12 @@ module.exports = (crowi) => {
   const loginRequiredStrictly = require('../../middleware/login-required')(crowi);
   const adminRequired = require('../../middleware/admin-required')(crowi);
 
-  const { ErrorV3, UserGroup, UserGroupRelation } = crowi.models;
+  const { UserGroup, UserGroupRelation } = crowi.models;
 
   /**
    * @swagger
    *  paths:
-   *    /_api/v3/user-group-relations:
+   *    /user-group-relations:
    *      get:
    *        tags: [UserGroupRelation]
    *        description: Gets the user group relations

src/server/routes/apiv3/user-group.js

@@ -9,12 +9,17 @@ const router = express.Router();
 const { body, param, query } = require('express-validator/check');
 const { sanitizeQuery } = require('express-validator/filter');
 
-const validator = {};
+const mongoose = require('mongoose');
 
-const { ObjectId } = require('mongoose').Types;
+const ErrorV3 = require('../../models/vo/error-apiv3');
 
 const { toPagingLimit, toPagingOffset } = require('../../util/express-validator/sanitizer');
 
+const validator = {};
+
+const { ObjectId } = mongoose.Types;
+
+
 /**
  * @swagger
  *  tags:
@@ -27,7 +32,6 @@ module.exports = (crowi) => {
   const csrf = require('../../middleware/csrf')(crowi);
 
   const {
-    ErrorV3,
     UserGroup,
     UserGroupRelation,
     User,
@@ -39,7 +43,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups:
+   *    /user-groups:
    *      get:
    *        tags: [UserGroup]
    *        description: Get usergroups
@@ -77,7 +81,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups:
+   *    /user-groups:
    *      post:
    *        tags: [UserGroup]
    *        description: Adds userGroup
@@ -127,7 +131,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}:
+   *    /user-groups/{id}:
    *      delete:
    *        tags: [UserGroup]
    *        description: Deletes userGroup
@@ -187,7 +191,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}:
+   *    /user-groups/{id}:
    *      put:
    *        tags: [UserGroup]
    *        description: Update userGroup
@@ -242,7 +246,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}/users:
+   *    /user-groups/{id}/users:
    *      get:
    *        tags: [UserGroup]
    *        description: Get users related to the userGroup
@@ -290,7 +294,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}/unrelated-users:
+   *    /user-groups/{id}/unrelated-users:
    *      get:
    *        tags: [UserGroup]
    *        description: Get users unrelated to the userGroup
@@ -339,7 +343,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}/users:
+   *    /user-groups/{id}/users:
    *      post:
    *        tags: [UserGroup]
    *        description: Add a user to the userGroup
@@ -398,7 +402,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}/users:
+   *    /user-groups/{id}/users:
    *      delete:
    *        tags: [UserGroup]
    *        description: remove a user from the userGroup
@@ -458,7 +462,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}/user-group-relations:
+   *    /user-groups/{id}/user-group-relations:
    *      get:
    *        tags: [UserGroup]
    *        description: Get the user group relations for the userGroup
@@ -510,7 +514,7 @@ module.exports = (crowi) => {
    * @swagger
    *
    *  paths:
-   *    /_api/v3/user-groups/{id}/pages:
+   *    /user-groups/{id}/pages:
    *      get:
    *        tags: [UserGroup]
    *        description: Get closed pages for the userGroup

src/server/routes/apiv3/users.js

@@ -9,6 +9,10 @@ const router = express.Router();
 const { body } = require('express-validator/check');
 const { isEmail } = require('validator');
 
+const ErrorV3 = require('../../models/vo/error-apiv3');
+
+const PAGE_ITEMS = 50;
+
 const validator = {};
 
 /**
@@ -17,14 +21,12 @@ const validator = {};
  *    name: Users
  */
 
-
 module.exports = (crowi) => {
   const loginRequiredStrictly = require('../../middleware/login-required')(crowi);
   const adminRequired = require('../../middleware/admin-required')(crowi);
   const csrf = require('../../middleware/csrf')(crowi);
 
   const {
-    ErrorV3,
     User,
     Page,
     ExternalAccount,
@@ -47,21 +49,27 @@ module.exports = (crowi) => {
    *              application/json:
    *                schema:
    *                  properties:
-   *                    users:
-   *                      type: object
-   *                      description: a result of `Users.find`
+   *                    paginateResult:
+   *                      $ref: '#/components/schemas/PaginateResult'
    */
   router.get('/', loginRequiredStrictly, adminRequired, async(req, res) => {
+    const page = parseInt(req.query.page) || 1;
+
     try {
-      const page = parseInt(req.query.page) || 1;
-      const result = await User.findUsersWithPagination({ page });
-      const { docs: users, total: totalUsers, limit: pagingLimit } = result;
-      return res.apiv3({ users, totalUsers, pagingLimit });
+      const paginateResult = await User.paginate(
+        { status: { $ne: User.STATUS_DELETED } },
+        {
+          sort: { status: 1, username: 1, createdAt: 1 },
+          page,
+          limit: PAGE_ITEMS,
+        },
+      );
+      return res.apiv3({ paginateResult });
     }
     catch (err) {
       const msg = 'Error occurred in fetching user group list';
       logger.error('Error', err);
-      return res.apiv3Err(new ErrorV3(msg, 'user-group-list-fetch-failed'));
+      return res.apiv3Err(new ErrorV3(msg, 'user-group-list-fetch-failed'), 500);
     }
   });
 

src/server/service/acl.js

@@ -79,17 +79,6 @@ class AclService {
     return labels;
   }
 
-  userUpperLimit() {
-    // const limit = this.configManager.getConfig('crowi', 'USER_UPPER_LIMIT');
-    const limit = process.env.USER_UPPER_LIMIT;
-
-    if (limit) {
-      return Number(limit);
-    }
-
-    return 0;
-  }
-
 }
 
 module.exports = AclService;

src/server/service/config-loader.js

@@ -140,7 +140,10 @@ const ENV_VAR_NAME_TO_CONFIG_INFO = {
     ns:      'crowi',
     key:     'gridfs:totalLimit',
     type:    TYPES.NUMBER,
-    default: null,
+    default: null, // set null in default for backward compatibility
+    //                cz: Newer system respects FILE_UPLOAD_TOTAL_LIMIT.
+    //                    If the default value of MONGO_GRIDFS_TOTAL_LIMIT is Infinity,
+    //                      the system can't distinguish between "not specified" and "Infinity is specified".
   },
   FORCE_WIKI_MODE: {
     ns:      'crowi',
@@ -148,6 +151,12 @@ const ENV_VAR_NAME_TO_CONFIG_INFO = {
     type:    TYPES.STRING,
     default: undefined,
   },
+  USER_UPPER_LIMIT: {
+    ns:      'crowi',
+    key:     'security:userUpperLimit',
+    type:    TYPES.NUMBER,
+    default: Infinity,
+  },
   LOCAL_STRATEGY_ENABLED: {
     ns:      'crowi',
     key:     'security:passport-local:isEnabled',
@@ -259,10 +268,12 @@ class ConfigLoader {
     const configFromDB = await this.loadFromDB();
     const configFromEnvVars = this.loadFromEnvVars();
 
-    // merge defaults
-    let mergedConfigFromDB = Object.assign({ crowi: this.configModel.getDefaultCrowiConfigsObject() }, configFromDB);
-    mergedConfigFromDB = Object.assign({ markdown: this.configModel.getDefaultMarkdownConfigsObject() }, mergedConfigFromDB);
-    mergedConfigFromDB = Object.assign({ notification: this.configModel.getDefaultNotificationConfigsObject() }, mergedConfigFromDB);
+    // merge defaults per ns
+    const mergedConfigFromDB = {
+      crowi: Object.assign(this.configModel.getDefaultCrowiConfigsObject(), configFromDB.crowi),
+      markdown: Object.assign(this.configModel.getDefaultMarkdownConfigsObject(), configFromDB.markdown),
+      notification: Object.assign(this.configModel.getDefaultNotificationConfigsObject(), configFromDB.notification),
+    };
 
     // In getConfig API, only null is used as a value to indicate that a config is not set.
     // So, if a value loaded from the database is emtpy,

src/server/service/export.js

@@ -7,54 +7,25 @@ const { Transform } = require('stream');
 const streamToPromise = require('stream-to-promise');
 const archiver = require('archiver');
 
-
 const toArrayIfNot = require('../../lib/util/toArrayIfNot');
 
+const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
 
-class ExportingProgress {
-
-  constructor(collectionName, totalCount) {
-    this.collectionName = collectionName;
-    this.currentCount = 0;
-    this.totalCount = totalCount;
-  }
-
-}
-
-class ExportingStatus {
-
-  constructor() {
-    this.totalCount = 0;
-
-    this.progressList = null;
-    this.progressMap = {};
-  }
+class ExportProgressingStatus extends CollectionProgressingStatus {
 
-  async init(collections) {
-    const promisesForCreatingInstance = collections.map(async(collectionName) => {
-      const collection = mongoose.connection.collection(collectionName);
-      const totalCount = await collection.count();
-      return new ExportingProgress(collectionName, totalCount);
+  async init() {
+    // retrieve total document count from each collections
+    const promises = this.progressList.map(async(collectionProgress) => {
+      const collection = mongoose.connection.collection(collectionProgress.collectionName);
+      collectionProgress.totalCount = await collection.count();
     });
-    this.progressList = await Promise.all(promisesForCreatingInstance);
+    await Promise.all(promises);
 
-    // collection name to instance mapping
-    this.progressList.forEach((p) => {
-      this.progressMap[p.collectionName] = p;
-      this.totalCount += p.totalCount;
-    });
-  }
-
-  get currentCount() {
-    return this.progressList.reduce(
-      (acc, crr) => acc + crr.currentCount,
-      0,
-    );
+    this.recalculateTotalCount();
   }
 
 }
 
-
 class ExportService {
 
   constructor(crowi) {
@@ -68,14 +39,14 @@ class ExportService {
 
     this.adminEvent = crowi.event('admin');
 
-    this.currentExportingStatus = null;
+    this.currentProgressingStatus = null;
   }
 
   /**
    * parse all zip files in downloads dir
    *
    * @memberOf ExportService
-   * @return {object} info for zip files and whether currentExportingStatus exists
+   * @return {object} info for zip files and whether currentProgressingStatus exists
    */
   async getStatus() {
     const zipFiles = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' });
@@ -87,12 +58,12 @@ class ExportService {
     // filter null object (broken zip)
     const filtered = zipFileStats.filter(element => element != null);
 
-    const isExporting = this.currentExportingStatus != null;
+    const isExporting = this.currentProgressingStatus != null;
 
     return {
       zipFileStats: filtered,
       isExporting,
-      progressList: isExporting ? this.currentExportingStatus.progressList : null,
+      progressList: isExporting ? this.currentProgressingStatus.progressList : null,
     };
   }
 
@@ -123,7 +94,7 @@ class ExportService {
 
   /**
    *
-   * @param {ExportProguress} exportProgress
+   * @param {ExportProgress} exportProgress
    * @return {Transform}
    */
   generateLogStream(exportProgress) {
@@ -196,7 +167,7 @@ class ExportService {
     const transformStream = this.generateTransformStream();
 
     // log configuration
-    const exportProgress = this.currentExportingStatus.progressMap[collectionName];
+    const exportProgress = this.currentProgressingStatus.progressMap[collectionName];
     const logStream = this.generateLogStream(exportProgress);
 
     // create WritableStream
@@ -208,7 +179,7 @@ class ExportService {
       .pipe(transformStream)
       .pipe(writeStream);
 
-    await streamToPromise(readStream);
+    await streamToPromise(writeStream);
 
     return writeStream.path;
   }
@@ -246,18 +217,18 @@ class ExportService {
   }
 
   async export(collections) {
-    if (this.currentExportingStatus != null) {
+    if (this.currentProgressingStatus != null) {
       throw new Error('There is an exporting process running.');
     }
 
-    this.currentExportingStatus = new ExportingStatus();
-    await this.currentExportingStatus.init(collections);
+    this.currentProgressingStatus = new ExportProgressingStatus(collections);
+    await this.currentProgressingStatus.init();
 
     try {
       await this.exportCollectionsToZippedJson(collections);
     }
     finally {
-      this.currentExportingStatus = null;
+      this.currentProgressingStatus = null;
     }
 
   }
@@ -267,14 +238,14 @@ class ExportService {
    *
    * @memberOf ExportService
    *
-   * @param {ExportProgress} exportProgress
+   * @param {CollectionProgress} collectionProgress
    * @param {number} currentCount number of items exported
    */
-  logProgress(exportProgress, currentCount) {
-    const output = `${exportProgress.collectionName}: ${currentCount}/${exportProgress.totalCount} written`;
+  logProgress(collectionProgress, currentCount) {
+    const output = `${collectionProgress.collectionName}: ${currentCount}/${collectionProgress.totalCount} written`;
 
     // update exportProgress.currentCount
-    exportProgress.currentCount = currentCount;
+    collectionProgress.currentCount = currentCount;
 
     // output every this.per items
     if (currentCount % this.per === 0) {
@@ -282,7 +253,7 @@ class ExportService {
       this.emitProgressEvent();
     }
     // output last item
-    else if (currentCount === exportProgress.totalCount) {
+    else if (currentCount === collectionProgress.totalCount) {
       logger.info(output);
       this.emitProgressEvent();
     }
@@ -290,10 +261,9 @@ class ExportService {
 
   /**
    * emit progress event
-   * @param {ExportProgress} exportProgress
    */
-  emitProgressEvent(exportProgress) {
-    const { currentCount, totalCount, progressList } = this.currentExportingStatus;
+  emitProgressEvent() {
+    const { currentCount, totalCount, progressList } = this.currentProgressingStatus;
     const data = {
       currentCount,
       totalCount,
@@ -331,7 +301,7 @@ class ExportService {
     const configs = toArrayIfNot(_configs);
     const appTitle = this.appService.getAppTitle();
     const timeStamp = (new Date()).getTime();
-    const zipFile = path.join(this.baseDir, `${appTitle}-${timeStamp}.zip`);
+    const zipFile = path.join(this.baseDir, `${appTitle}-${timeStamp}.growi.zip`);
     const archive = archiver('zip', {
       zlib: { level: this.zlibLevel },
     });

src/server/service/growi-bridge.js

@@ -99,7 +99,8 @@ class GrowiBridgeService {
    * @return {object} meta{object} and files{Array.<object>}
    */
   async parseZipFile(zipFile) {
-    const fileStats = [];
+    const fileStat = fs.statSync(zipFile);
+    const innerFileStats = [];
     let meta = {};
 
     const readStream = fs.createReadStream(zipFile);
@@ -113,7 +114,7 @@ class GrowiBridgeService {
         meta = JSON.parse((await entry.buffer()).toString());
       }
       else {
-        fileStats.push({
+        innerFileStats.push({
           fileName,
           collectionName: path.basename(fileName, '.json'),
           size,
@@ -135,7 +136,8 @@ class GrowiBridgeService {
     return {
       meta,
       fileName: path.basename(zipFile),
-      fileStats,
+      fileStat,
+      innerFileStats,
     };
   }
 

src/server/service/import.js

@@ -1,11 +1,41 @@
 const logger = require('@alias/logger')('growi:services:ImportService'); // eslint-disable-line no-unused-vars
 const fs = require('fs');
 const path = require('path');
+
+const { Writable, Transform } = require('stream');
 const JSONStream = require('JSONStream');
 const streamToPromise = require('stream-to-promise');
 const unzipper = require('unzipper');
+
 const { ObjectId } = require('mongoose').Types;
 
+const { createBatchStream } = require('../util/batch-stream');
+const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
+
+
+const BULK_IMPORT_SIZE = 100;
+
+
+class ImportSettings {
+
+  constructor(mode) {
+    this.mode = mode || 'insert';
+    this.jsonFileName = null;
+    this.overwriteParams = null;
+  }
+
+}
+
+class ImportingCollectionError extends Error {
+
+  constructor(collectionProgress, error) {
+    super(error);
+    this.collectionProgress = collectionProgress;
+  }
+
+}
+
+
 class ImportService {
 
   constructor(crowi) {
@@ -13,12 +43,23 @@ class ImportService {
     this.growiBridgeService = crowi.growiBridgeService;
     this.getFile = this.growiBridgeService.getFile.bind(this);
     this.baseDir = path.join(crowi.tmpDir, 'imports');
-    this.per = 100;
     this.keepOriginal = this.keepOriginal.bind(this);
 
+    this.adminEvent = crowi.event('admin');
+
     // { pages: { _id: ..., path: ..., ...}, users: { _id: ..., username: ..., }, ... }
     this.convertMap = {};
     this.initConvertMap(crowi.models);
+
+    this.currentProgressingStatus = null;
+  }
+
+  /**
+   * generate ImportSettings instance
+   * @param {string} mode bulk operation mode (insert | upsert | flushAndInsert)
+   */
+  generateImportSettings(mode) {
+    return new ImportSettings(mode);
   }
 
   /**
@@ -48,89 +89,229 @@ class ImportService {
    * automatically convert ObjectId
    *
    * @memberOf ImportService
-   * @param {any} _value value from imported document
-   * @param {{ _document: object, schema: object, key: string }}
+   * @param {any} value value from imported document
+   * @param {{ document: object, schema: object, key: string }}
    * @return {any} new value for the document
    */
-  keepOriginal(_value, { _document, schema, key }) {
-    let value;
-    if (schema[key].instance === 'ObjectID' && ObjectId.isValid(_value)) {
-      value = ObjectId(_value);
+  keepOriginal(value, { document, schema, propertyName }) {
+    let _value;
+    if (schema[propertyName].instance === 'ObjectID' && ObjectId.isValid(value)) {
+      _value = ObjectId(value);
     }
     else {
-      value = _value;
+      _value = value;
     }
 
-    return value;
+    return _value;
+  }
+
+  /**
+   * parse all zip files in downloads dir
+   *
+   * @memberOf ExportService
+   * @return {object} info for zip files and whether currentProgressingStatus exists
+   */
+  async getStatus() {
+    const zipFiles = fs.readdirSync(this.baseDir).filter(file => path.extname(file) === '.zip');
+    const zipFileStats = await Promise.all(zipFiles.map((file) => {
+      const zipFile = this.getFile(file);
+      return this.growiBridgeService.parseZipFile(zipFile);
+    }));
+
+    // filter null object (broken zip)
+    const filtered = zipFileStats
+      .filter(zipFileStat => zipFileStat != null);
+    // sort with ctime("Change Time" - Time when file status was last changed (inode data modification).)
+    filtered.sort((a, b) => { return a.fileStat.ctime - b.fileStat.ctime });
+
+    const isImporting = this.currentProgressingStatus != null;
+
+    return {
+      zipFileStat: filtered.pop(),
+      isImporting,
+      progressList: isImporting ? this.currentProgressingStatus.progressList : null,
+    };
+  }
+
+  /**
+   * import collections from json
+   *
+   * @param {string} collections MongoDB collection name
+   * @param {array} importSettingsMap key: collection name, value: ImportSettings instance
+   */
+  async import(collections, importSettingsMap) {
+    // init status object
+    this.currentProgressingStatus = new CollectionProgressingStatus(collections);
+
+    try {
+      const promises = collections.map((collectionName) => {
+        const importSettings = importSettingsMap[collectionName];
+        return this.importCollection(collectionName, importSettings);
+      });
+      await Promise.all(promises);
+    }
+    // catch ImportingCollectionError
+    catch (err) {
+      const { collectionProgress } = err;
+      logger.error(`failed to import to ${collectionProgress.collectionName}`, err);
+      this.emitProgressEvent(collectionProgress, { message: err.message });
+    }
+    finally {
+      this.currentProgressingStatus = null;
+      this.emitTerminateEvent();
+    }
   }
 
   /**
    * import a collection from json
    *
    * @memberOf ImportService
-   * @param {object} Model instance of mongoose model
-   * @param {string} jsonFile absolute path to the jsonFile being imported
-   * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
+   * @param {string} collectionName MongoDB collection name
+   * @param {ImportSettings} importSettings
    * @return {insertedIds: Array.<string>, failedIds: Array.<string>}
    */
-  async import(Model, jsonFile, overwriteParams = {}) {
-    // streamToPromise(jsonStream) throws an error, use new Promise instead
-    return new Promise((resolve, reject) => {
-      const collectionName = Model.collection.name;
+  async importCollection(collectionName, importSettings) {
+    // prepare functions invoked from custom streams
+    const convertDocuments = this.convertDocuments.bind(this);
+    const bulkOperate = this.bulkOperate.bind(this);
+    const execUnorderedBulkOpSafely = this.execUnorderedBulkOpSafely.bind(this);
+    const emitProgressEvent = this.emitProgressEvent.bind(this);
+
+    const { mode, jsonFileName, overwriteParams } = importSettings;
+    const Model = this.growiBridgeService.getModelFromCollectionName(collectionName);
+    const jsonFile = this.getFile(jsonFileName);
+    const collectionProgress = this.currentProgressingStatus.progressMap[collectionName];
+
+    try {
+      // validate options
+      this.validateImportSettings(collectionName, importSettings);
 
-      let counter = 0;
-      let insertedIds = [];
-      let failedIds = [];
-      let unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+      // flush
+      if (mode === 'flushAndInsert') {
+        await Model.remove({});
+      }
 
+      // stream 1
       const readStream = fs.createReadStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
-      const jsonStream = readStream.pipe(JSONStream.parse('*'));
 
-      jsonStream.on('data', async(document) => {
-        // documents are not persisted until unorderedBulkOp.execute()
-        unorderedBulkOp.insert(this.convertDocuments(Model, document, overwriteParams));
+      // stream 2
+      const jsonStream = JSONStream.parse('*');
+
+      // stream 3
+      const convertStream = new Transform({
+        objectMode: true,
+        transform(doc, encoding, callback) {
+          const converted = convertDocuments(collectionName, doc, overwriteParams);
+          this.push(converted);
+          callback();
+        },
+      });
 
-        counter++;
+      // stream 4
+      const batchStream = createBatchStream(BULK_IMPORT_SIZE);
+
+      // stream 5
+      const writeStream = new Writable({
+        objectMode: true,
+        async write(batch, encoding, callback) {
+          const unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+
+          // documents are not persisted until unorderedBulkOp.execute()
+          batch.forEach((document) => {
+            bulkOperate(unorderedBulkOp, collectionName, document, importSettings);
+          });
+
+          // exec
+          const { insertedCount, modifiedCount, errors } = await execUnorderedBulkOpSafely(unorderedBulkOp);
+          logger.debug(`Importing ${collectionName}. Inserted: ${insertedCount}. Modified: ${modifiedCount}. Failed: ${errors.length}.`);
+
+          const increment = insertedCount + modifiedCount + errors.length;
+          collectionProgress.currentCount += increment;
+          collectionProgress.totalCount += increment;
+          collectionProgress.insertedCount += insertedCount;
+          collectionProgress.modifiedCount += modifiedCount;
+
+          emitProgressEvent(collectionProgress, errors);
+
+          callback();
+        },
+        final(callback) {
+          logger.info(`Importing ${collectionName} has terminated.`);
+          callback();
+        },
+      });
 
-        if (counter % this.per === 0) {
-          // puase jsonStream to prevent more items to be added to unorderedBulkOp
-          jsonStream.pause();
+      readStream
+        .pipe(jsonStream)
+        .pipe(convertStream)
+        .pipe(batchStream)
+        .pipe(writeStream);
 
-          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
-          insertedIds = [...insertedIds, ..._insertedIds];
-          failedIds = [...failedIds, ..._failedIds];
+      await streamToPromise(writeStream);
 
-          // reset initializeUnorderedBulkOp
-          unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+      // clean up tmp directory
+      fs.unlinkSync(jsonFile);
+    }
+    catch (err) {
+      throw new ImportingCollectionError(collectionProgress, err);
+    }
 
-          // resume jsonStream
-          jsonStream.resume();
-        }
-      });
+  }
 
-      jsonStream.on('end', async(data) => {
-        // insert the rest. avoid errors when unorderedBulkOp has no items
-        if (unorderedBulkOp.s.currentBatch !== null) {
-          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
-          insertedIds = [...insertedIds, ..._insertedIds];
-          failedIds = [...failedIds, ..._failedIds];
+  /**
+   *
+   * @param {string} collectionName
+   * @param {importSettings} importSettings
+   */
+  validateImportSettings(collectionName, importSettings) {
+    const { mode } = importSettings;
+
+    switch (collectionName) {
+      case 'configs':
+        if (mode !== 'flushAndInsert') {
+          throw new Error(`The specified mode '${mode}' is not allowed when importing to 'configs' collection.`);
         }
+        break;
+    }
+  }
 
-        logger.info(`Done. Inserted ${insertedIds.length} ${collectionName}.`);
+  /**
+   * process bulk operation
+   * @param {object} bulk MongoDB Bulk instance
+   * @param {string} collectionName collection name
+   * @param {object} document
+   * @param {ImportSettings} importSettings
+   */
+  bulkOperate(bulk, collectionName, document, importSettings) {
+    // insert
+    if (importSettings.mode !== 'upsert') {
+      return bulk.insert(document);
+    }
 
-        if (failedIds.length > 0) {
-          logger.error(`Failed to insert ${failedIds.length} ${collectionName}: ${failedIds.join(', ')}.`);
-        }
+    // upsert
+    switch (collectionName) {
+      default:
+        return bulk.find({ _id: document._id }).upsert().replaceOne(document);
+    }
+  }
 
-        // clean up tmp directory
-        fs.unlinkSync(jsonFile);
+  /**
+   * emit progress event
+   * @param {CollectionProgress} collectionProgress
+   * @param {object} appendedErrors key: collection name, value: array of error object
+   */
+  emitProgressEvent(collectionProgress, appendedErrors) {
+    const { collectionName } = collectionProgress;
 
-        return resolve({
-          insertedIds,
-          failedIds,
-        });
-      });
-    });
+    // send event (in progress in global)
+    this.adminEvent.emit('onProgressForImport', { collectionName, collectionProgress, appendedErrors });
+  }
+
+  /**
+   * emit terminate event
+   */
+  emitTerminateEvent() {
+    this.adminEvent.emit('onTerminateForImport');
   }
 
   /**
@@ -170,38 +351,31 @@ class ImportService {
    *
    * @memberOf ImportService
    * @param {object} unorderedBulkOp result of Model.collection.initializeUnorderedBulkOp()
-   * @return {{nInserted: number, failed: Array.<string>}} number of docuemnts inserted and failed
+   * @return {object} e.g. { insertedCount: 10, errors: [...] }
    */
   async execUnorderedBulkOpSafely(unorderedBulkOp) {
-    // keep the number of documents inserted and failed for logger
-    let insertedIds = [];
-    let failedIds = [];
+    let errors = [];
+    let result = null;
 
-    // try catch to skip errors
     try {
       const log = await unorderedBulkOp.execute();
-      const _insertedIds = log.result.insertedIds.map(op => op._id);
-      insertedIds = [...insertedIds, ..._insertedIds];
+      result = log.result;
     }
     catch (err) {
-      const collectionName = unorderedBulkOp.s.namespace;
-
-      for (const error of err.result.result.writeErrors) {
-        logger.error(`${collectionName}: ${error.errmsg}`);
-      }
-
-      const _failedIds = err.result.result.writeErrors.map(err => err.err.op._id);
-      const _insertedIds = err.result.result.insertedIds.filter(op => !_failedIds.includes(op._id)).map(op => op._id);
-
-      failedIds = [...failedIds, ..._failedIds];
-      insertedIds = [...insertedIds, ..._insertedIds];
+      result = err.result;
+      errors = err.writeErrors.map((err) => {
+        const moreDetailErr = err.err;
+        return { _id: moreDetailErr.op._id, message: err.errmsg };
+      });
     }
 
-    logger.debug(`Importing ${unorderedBulkOp.s.collection.s.name}. Inserted: ${insertedIds.length}. Failed: ${failedIds.length}.`);
+    const insertedCount = result.nInserted + result.nUpserted;
+    const modifiedCount = result.nModified;
 
     return {
-      insertedIds,
-      failedIds,
+      insertedCount,
+      modifiedCount,
+      errors,
     };
   }
 
@@ -209,13 +383,13 @@ class ImportService {
    * execute unorderedBulkOp and ignore errors
    *
    * @memberOf ImportService
-   * @param {object} Model instance of mongoose model
-   * @param {object} _document document being imported
+   * @param {string} collectionName
+   * @param {object} document document being imported
    * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
    * @return {object} document to be persisted
    */
-  convertDocuments(Model, _document, overwriteParams) {
-    const collectionName = Model.collection.name;
+  convertDocuments(collectionName, document, overwriteParams) {
+    const Model = this.growiBridgeService.getModelFromCollectionName(collectionName);
     const schema = Model.schema.paths;
     const convertMap = this.convertMap[collectionName];
 
@@ -223,31 +397,33 @@ class ImportService {
       throw new Error(`attribute map is not defined for ${collectionName}`);
     }
 
-    const document = {};
+    const _document = {};
 
     // assign value from documents being imported
-    for (const entry of Object.entries(convertMap)) {
-      const [key, value] = entry;
+    Object.entries(convertMap).forEach(([propertyName, convertedValue]) => {
+      const value = document[propertyName];
 
       // distinguish between null and undefined
-      if (_document[key] === undefined) {
-        continue; // next entry
+      if (value === undefined) {
+        return; // next entry
       }
 
-      document[key] = (typeof value === 'function') ? value(_document[key], { _document, key, schema }) : value;
-    }
+      const convertFunc = (typeof convertedValue === 'function') ? convertedValue : null;
+      _document[propertyName] = (convertFunc != null) ? convertFunc(value, { document, propertyName, schema }) : convertedValue;
+    });
 
     // overwrite documents with custom values
-    for (const entry of Object.entries(overwriteParams)) {
-      const [key, value] = entry;
+    Object.entries(overwriteParams).forEach(([propertyName, overwriteValue]) => {
+      const value = document[propertyName];
 
       // distinguish between null and undefined
-      if (_document[key] !== undefined) {
-        document[key] = (typeof value === 'function') ? value(_document[key], { _document, key, schema }) : value;
+      if (value !== undefined) {
+        const overwriteFunc = (typeof overwriteValue === 'function') ? overwriteValue : null;
+        _document[propertyName] = (overwriteFunc != null) ? overwriteFunc(value, { document: _document, propertyName, schema }) : overwriteValue;
       }
-    }
+    });
 
-    return document;
+    return _document;
   }
 
   /**
@@ -268,6 +444,15 @@ class ImportService {
     // - import: throw err if there are pending migrations
   }
 
+  /**
+   * Delete all uploaded files
+   */
+  deleteAllZipFiles() {
+    fs.readdirSync(this.baseDir)
+      .filter(file => path.extname(file) === '.zip')
+      .forEach(file => fs.unlinkSync(path.join(this.baseDir, file)));
+  }
+
 }
 
 module.exports = ImportService;

src/server/util/batch-stream.js

@@ -0,0 +1,35 @@
+const { Transform } = require('stream');
+
+function createBatchStream(batchSize) {
+  let batchBuffer = [];
+
+  return new Transform({
+    // object mode
+    objectMode: true,
+
+    transform(doc, encoding, callback) {
+      batchBuffer.push(doc);
+
+      if (batchBuffer.length >= batchSize) {
+        this.push(batchBuffer);
+
+        // reset buffer
+        batchBuffer = [];
+      }
+
+      callback();
+    },
+
+    final(callback) {
+      if (batchBuffer.length > 0) {
+        this.push(batchBuffer);
+      }
+      callback();
+    },
+
+  });
+}
+
+module.exports = {
+  createBatchStream,
+};

src/server/util/search.js

@@ -6,6 +6,15 @@ const elasticsearch = require('elasticsearch');
 const debug = require('debug')('growi:lib:search');
 const logger = require('@alias/logger')('growi:lib:search');
 
+const {
+  Writable, Transform,
+} = require('stream');
+const streamToPromise = require('stream-to-promise');
+
+const { createBatchStream } = require('./batch-stream');
+
+const BULK_REINDEX_SIZE = 100;
+
 function SearchClient(crowi, esUri) {
   this.DEFAULT_OFFSET = 0;
   this.DEFAULT_LIMIT = 50;
@@ -84,9 +93,8 @@ SearchClient.prototype.checkESVersion = async function() {
 
 SearchClient.prototype.registerUpdateEvent = function() {
   const pageEvent = this.crowi.event('page');
-  pageEvent.on('create', this.syncPageCreated.bind(this));
+  pageEvent.on('create', this.syncPageUpdated.bind(this));
   pageEvent.on('update', this.syncPageUpdated.bind(this));
-  pageEvent.on('updateTag', this.syncPageUpdated.bind(this));
   pageEvent.on('delete', this.syncPageDeleted.bind(this));
 
   const bookmarkEvent = this.crowi.event('bookmark');
@@ -98,7 +106,7 @@ SearchClient.prototype.registerUpdateEvent = function() {
 };
 
 SearchClient.prototype.shouldIndexed = function(page) {
-  return (page.redirectTo == null);
+  return page.creator != null && page.revision != null && page.redirectTo == null;
 };
 
 // BONSAI_URL is following format:
@@ -163,6 +171,7 @@ SearchClient.prototype.buildIndex = async function(uri) {
   // reindex to tmp index
   await this.createIndex(tmpIndexName);
   await client.reindex({
+    waitForCompletion: false,
     body: {
       source: { index: indexName },
       dest: { index: tmpIndexName },
@@ -225,38 +234,6 @@ function generateDocContentsRelatedToRestriction(page) {
   };
 }
 
-SearchClient.prototype.prepareBodyForUpdate = function(body, page) {
-  if (!Array.isArray(body)) {
-    throw new Error('Body must be an array.');
-  }
-
-  const command = {
-    update: {
-      _index: this.aliasName,
-      _type: 'pages',
-      _id: page._id.toString(),
-    },
-  };
-
-  let document = {
-    path: page.path,
-    body: page.revision.body,
-    comment_count: page.commentCount,
-    bookmark_count: page.bookmarkCount || 0,
-    like_count: page.liker.length || 0,
-    updated_at: page.updatedAt,
-    tag_names: page.tagNames,
-  };
-
-  document = Object.assign(document, generateDocContentsRelatedToRestriction(page));
-
-  body.push(command);
-  body.push({
-    doc: document,
-    doc_as_upsert: true,
-  });
-};
-
 SearchClient.prototype.prepareBodyForCreate = function(body, page) {
   if (!Array.isArray(body)) {
     throw new Error('Body must be an array.');
@@ -296,7 +273,7 @@ SearchClient.prototype.prepareBodyForDelete = function(body, page) {
 
   const command = {
     delete: {
-      _index: this.aliasName,
+      _index: this.indexName,
       _type: 'pages',
       _id: page._id.toString(),
     },
@@ -305,122 +282,165 @@ SearchClient.prototype.prepareBodyForDelete = function(body, page) {
   body.push(command);
 };
 
-SearchClient.prototype.addPages = async function(pages) {
+SearchClient.prototype.addAllPages = async function() {
+  const Page = this.crowi.model('Page');
+  return this.updateOrInsertPages(() => Page.find(), true);
+};
+
+SearchClient.prototype.updateOrInsertPageById = async function(pageId) {
+  const Page = this.crowi.model('Page');
+  return this.updateOrInsertPages(() => Page.findById(pageId));
+};
+
+/**
+ * @param {function} queryFactory factory method to generate a Mongoose Query instance
+ */
+SearchClient.prototype.updateOrInsertPages = async function(queryFactory, isEmittingProgressEvent = false) {
+  const Page = this.crowi.model('Page');
+  const { PageQueryBuilder } = Page;
   const Bookmark = this.crowi.model('Bookmark');
   const PageTagRelation = this.crowi.model('PageTagRelation');
-  const body = [];
 
-  /* eslint-disable no-await-in-loop */
-  for (const page of pages) {
-    page.bookmarkCount = await Bookmark.countByPageId(page._id);
-    const tagRelations = await PageTagRelation.find({ relatedPage: page._id }).populate('relatedTag');
-    page.tagNames = tagRelations.map((relation) => { return relation.relatedTag.name });
-    this.prepareBodyForCreate(body, page);
-  }
-  /* eslint-enable no-await-in-loop */
+  const searchEvent = this.searchEvent;
 
-  logger.debug('addPages(): Sending Request to ES', body);
-  return this.client.bulk({
-    body,
-  });
-};
+  // prepare functions invoked from custom streams
+  const prepareBodyForCreate = this.prepareBodyForCreate.bind(this);
+  const shouldIndexed = this.shouldIndexed.bind(this);
+  const bulkWrite = this.client.bulk.bind(this.client);
 
-SearchClient.prototype.updatePages = async function(pages) {
-  const self = this;
-  const PageTagRelation = this.crowi.model('PageTagRelation');
-  const body = [];
+  const findQuery = new PageQueryBuilder(queryFactory()).addConditionToExcludeRedirect().query;
+  const countQuery = new PageQueryBuilder(queryFactory()).addConditionToExcludeRedirect().query;
 
-  /* eslint-disable no-await-in-loop */
-  for (const page of pages) {
-    const tagRelations = await PageTagRelation.find({ relatedPage: page._id }).populate('relatedTag');
-    page.tagNames = tagRelations.map((relation) => { return relation.relatedTag.name });
-    self.prepareBodyForUpdate(body, page);
-  }
+  const totalCount = await countQuery.count();
 
-  logger.debug('updatePages(): Sending Request to ES', body);
-  return this.client.bulk({
-    body,
+  const readStream = findQuery
+    // populate data which will be referenced by prepareBodyForCreate()
+    .populate([
+      { path: 'creator', model: 'User', select: 'username' },
+      { path: 'revision', model: 'Revision', select: 'body' },
+    ])
+    .snapshot()
+    .lean()
+    .cursor();
+
+  let skipped = 0;
+  const thinOutStream = new Transform({
+    objectMode: true,
+    async transform(doc, encoding, callback) {
+      if (shouldIndexed(doc)) {
+        this.push(doc);
+      }
+      else {
+        skipped++;
+      }
+      callback();
+    },
   });
-};
 
-SearchClient.prototype.deletePages = function(pages) {
-  const self = this;
-  const body = [];
+  const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
-  pages.map((page) => {
-    self.prepareBodyForDelete(body, page);
-    return;
+  const appendBookmarkCountStream = new Transform({
+    objectMode: true,
+    async transform(chunk, encoding, callback) {
+      const pageIds = chunk.map(doc => doc._id);
+
+      const idToCountMap = await Bookmark.getPageIdToCountMap(pageIds);
+      const idsHavingCount = Object.keys(idToCountMap);
+
+      // append count
+      chunk
+        .filter(doc => idsHavingCount.includes(doc._id.toString()))
+        .forEach((doc) => {
+          // append count from idToCountMap
+          doc.bookmarkCount = idToCountMap[doc._id.toString()];
+        });
+
+      this.push(chunk);
+      callback();
+    },
   });
 
-  logger.debug('deletePages(): Sending Request to ES', body);
-  return this.client.bulk({
-    body,
+  const appendTagNamesStream = new Transform({
+    objectMode: true,
+    async transform(chunk, encoding, callback) {
+      const pageIds = chunk.map(doc => doc._id);
+
+      const idToTagNamesMap = await PageTagRelation.getIdToTagNamesMap(pageIds);
+      const idsHavingTagNames = Object.keys(idToTagNamesMap);
+
+      // append tagNames
+      chunk
+        .filter(doc => idsHavingTagNames.includes(doc._id.toString()))
+        .forEach((doc) => {
+          // append tagName from idToTagNamesMap
+          doc.tagNames = idToTagNamesMap[doc._id.toString()];
+        });
+
+      this.push(chunk);
+      callback();
+    },
   });
-};
 
-SearchClient.prototype.addAllPages = async function() {
-  const self = this;
-  const Page = this.crowi.model('Page');
-  const allPageCount = await Page.allPageCount();
-  const Bookmark = this.crowi.model('Bookmark');
-  const PageTagRelation = this.crowi.model('PageTagRelation');
-  const cursor = Page.getStreamOfFindAll();
-  let body = [];
-  let sent = 0;
-  let skipped = 0;
-  let total = 0;
+  let count = 0;
+  const writeStream = new Writable({
+    objectMode: true,
+    async write(batch, encoding, callback) {
+      const body = [];
+      batch.forEach(doc => prepareBodyForCreate(body, doc));
 
-  return new Promise((resolve, reject) => {
-    const bulkSend = (body) => {
-      self.client
-        .bulk({
+      try {
+        const res = await bulkWrite({
           body,
           requestTimeout: Infinity,
-        })
-        .then((res) => {
-          logger.info('addAllPages add anyway (items, errors, took): ', (res.items || []).length, res.errors, res.took, 'ms');
-        })
-        .catch((err) => {
-          logger.error('addAllPages error on add anyway: ', err);
         });
-    };
 
-    cursor
-      .eachAsync(async(doc) => {
-        if (!doc.creator || !doc.revision || !self.shouldIndexed(doc)) {
-          // debug('Skipped', doc.path);
-          skipped++;
-          return;
+        count += (res.items || []).length;
+
+        logger.info(`Adding pages progressing: (count=${count}, errors=${res.errors}, took=${res.took}ms)`);
+
+        if (isEmittingProgressEvent) {
+          searchEvent.emit('addPageProgress', totalCount, count, skipped);
         }
-        total++;
+      }
+      catch (err) {
+        logger.error('addAllPages error on add anyway: ', err);
+      }
+
+      callback();
+    },
+    final(callback) {
+      logger.info(`Adding pages has terminated: (totalCount=${totalCount}, skipped=${skipped})`);
 
-        const bookmarkCount = await Bookmark.countByPageId(doc._id);
-        const tagRelations = await PageTagRelation.find({ relatedPage: doc._id }).populate('relatedTag');
-        const page = { ...doc, bookmarkCount, tagNames: tagRelations.map((relation) => { return relation.relatedTag.name }) };
-        self.prepareBodyForCreate(body, page);
+      if (isEmittingProgressEvent) {
+        searchEvent.emit('finishAddPage', totalCount, count, skipped);
+      }
+      callback();
+    },
+  });
 
-        if (body.length >= 4000) {
-          // send each 2000 docs. (body has 2 elements for each data)
-          sent++;
-          logger.debug('Sending request (seq, total, skipped)', sent, total, skipped);
-          bulkSend(body);
-          this.searchEvent.emit('addPageProgress', allPageCount, total, skipped);
+  readStream
+    .pipe(thinOutStream)
+    .pipe(batchStream)
+    .pipe(appendBookmarkCountStream)
+    .pipe(appendTagNamesStream)
+    .pipe(writeStream);
 
-          body = [];
-        }
-      })
-      .then(() => {
-        // send all remaining data on body[]
-        logger.debug('Sending last body of bulk operation:', body.length);
-        bulkSend(body);
-        this.searchEvent.emit('finishAddPage', allPageCount, total, skipped);
-
-        resolve();
-      })
-      .catch((e) => {
-        logger.error('Error wile iterating cursor.eachAsync()', e);
-        reject(e);
-      });
+  return streamToPromise(writeStream);
+
+};
+
+SearchClient.prototype.deletePages = function(pages) {
+  const self = this;
+  const body = [];
+
+  pages.map((page) => {
+    self.prepareBodyForDelete(body, page);
+    return;
+  });
+
+  logger.debug('deletePages(): Sending Request to ES', body);
+  return this.client.bulk({
+    body,
   });
 };
 
@@ -855,76 +875,44 @@ SearchClient.prototype.parseQueryString = function(queryString) {
   };
 };
 
-SearchClient.prototype.syncPageCreated = function(page, user, bookmarkCount = 0) {
-  debug('SearchClient.syncPageCreated', page.path);
+SearchClient.prototype.syncPageUpdated = async function(page, user) {
+  logger.debug('SearchClient.syncPageUpdated', page.path);
 
+  // delete if page should not indexed
   if (!this.shouldIndexed(page)) {
+    try {
+      await this.deletePages([page]);
+    }
+    catch (err) {
+      logger.error('deletePages:ES Error', err);
+    }
     return;
   }
 
-  page.bookmarkCount = bookmarkCount;
-  this.addPages([page])
-    .then((res) => {
-      debug('ES Response', res);
-    })
-    .catch((err) => {
-      logger.error('ES Error', err);
-    });
-};
-
-SearchClient.prototype.syncPageUpdated = function(page, user, bookmarkCount = 0) {
-  debug('SearchClient.syncPageUpdated', page.path);
-  // TODO delete
-  if (!this.shouldIndexed(page)) {
-    this.deletePages([page])
-      .then((res) => {
-        debug('deletePages: ES Response', res);
-      })
-      .catch((err) => {
-        logger.error('deletePages:ES Error', err);
-      });
-
-    return;
-  }
-
-  page.bookmarkCount = bookmarkCount;
-  this.updatePages([page])
-    .then((res) => {
-      debug('ES Response', res);
-    })
-    .catch((err) => {
-      logger.error('ES Error', err);
-    });
+  return this.updateOrInsertPageById(page._id);
 };
 
-SearchClient.prototype.syncPageDeleted = function(page, user) {
+SearchClient.prototype.syncPageDeleted = async function(page, user) {
   debug('SearchClient.syncPageDeleted', page.path);
 
-  this.deletePages([page])
-    .then((res) => {
-      debug('deletePages: ES Response', res);
-    })
-    .catch((err) => {
-      logger.error('deletePages:ES Error', err);
-    });
+  try {
+    return await this.deletePages([page]);
+  }
+  catch (err) {
+    logger.error('deletePages:ES Error', err);
+  }
 };
 
 SearchClient.prototype.syncBookmarkChanged = async function(pageId) {
-  const Page = this.crowi.model('Page');
-  const Bookmark = this.crowi.model('Bookmark');
-  const page = await Page.findById(pageId);
-  const bookmarkCount = await Bookmark.countByPageId(pageId);
+  logger.debug('SearchClient.syncBookmarkChanged', pageId);
 
-  page.bookmarkCount = bookmarkCount;
-  this.updatePages([page])
-    .then((res) => { return debug('ES Response', res) })
-    .catch((err) => { return logger.error('ES Error', err) });
+  return this.updateOrInsertPageById(pageId);
 };
 
 SearchClient.prototype.syncTagChanged = async function(page) {
-  this.updatePages([page])
-    .then((res) => { return debug('ES Response', res) })
-    .catch((err) => { return logger.error('ES Error', err) });
+  logger.debug('SearchClient.syncTagChanged', page.path);
+
+  return this.updateOrInsertPageById(page._id);
 };
 
 

src/server/views/admin/Users_reserve.html

@@ -1,321 +0,0 @@
-{% extends '../layout/admin.html' %}
-
-{% block html_title %}{{ customizeService.generateCustomTitle(t('User_Management')) }}{% endblock %}
-
-{% block content_header %}
-<div class="header-wrap">
-  <header id="page-header">
-    <h1 id="admin-title" class="title">{{ t('User_Management') }}</h1>
-  </header>
-</div>
-{% endblock %}
-
-{% block content_main %}
-<div class="content-main">
-  {% set smessage = req.flash('successMessage') %}
-  {% if smessage.length %}
-  <div class="alert alert-success">
-    {{ smessage }}
-  </div>
-  {% endif %}
-
-  {% set emessage = req.flash('errorMessage') %}
-  {% if emessage.length %}
-  <div class="alert alert-danger">
-    {{ emessage }}
-  </div>
-  {% endif %}
-
-  <div class="row">
-    <div class="col-md-3">
-      {% include './widget/menu.html' with {current: 'user'} %}
-    </div>
-
-    <div class="col-md-9">
-      <p>
-        <button data-toggle="collapse" class="btn btn-default" href="#inviteUserForm" {% if isUserCountExceedsUpperLimit %}disabled{% endif %}>
-          {{ t("user_management.invite_users") }}
-        </button>
-        <a class="btn btn-default btn-outline" href="/admin/users/external-accounts">
-          <i class="icon-user-follow" aria-hidden="true"></i>
-          {{ t("user_management.external_account") }}
-        </a>
-      </p>
-      <form role="form" action="/admin/user/invite" method="post">
-        <div id="inviteUserForm" class="collapse">
-          <div class="form-group">
-            <label for="inviteForm[emailList]">{{ t('user_management.emails') }}</label>
-            <textarea class="form-control" name="inviteForm[emailList]" placeholder="{{ t('eg') }} user@growi.org"></textarea>
-          </div>
-          <div class="checkbox checkbox-info">
-            <input type="checkbox" id="inviteWithEmail" name="inviteForm[sendEmail]" checked>
-            <label for="inviteWithEmail">{{ t('user_management.invite_thru_email') }}</label>
-          </div>
-          <button type="submit" class="btn btn-primary">{{ t('user_management.invite') }}</button>
-        </div>
-        <input type="hidden" name="_csrf" value="{{ csrf() }}">
-      </form>
-
-      {% if isUserCountExceedsUpperLimit === true %}
-      <label>{{ t('user_management.cannot_invite_maximum_users') }}</label>
-      {% endif %}
-      {% if userUpperLimit !== 0 %}
-      <label>{{ t('user_management.current_users') }}{{ activeUsers }}</label>
-      {% endif %}
-
-      {% set createdUser = req.flash('createdUser') %}
-      {% if createdUser.length %}
-      <div class="modal fade in" id="createdUserModal">
-        <div class="modal-dialog">
-          <div class="modal-content">
-
-            <div class="modal-header">
-              <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
-              <div class="modal-title">{{ t('user_management.invited') }}</div>
-            </div>
-
-            <div class="modal-body">
-              <p>
-                {{ t('user_management.temporary_password') }}<br>
-                {{ t('user_management.password_never_seen') }}<span class="text-danger">{{ t('user_management.send_temporary_password') }}</span>
-              </p>
-
-              <pre>{% for cUser in createdUser %}{% if cUser.user %}{{ cUser.email }} {{ cUser.password }}<br>{% else %}{{ cUser.email }} 作成失敗<br>{% endif %}{% endfor %}</pre>
-            </div>
-
-          </div><!-- /.modal-content -->
-        </div><!-- /.modal-dialog -->
-      </div><!-- /.modal -->
-      {% endif %}
-
-      {# FIXME とりあえずクソ実装。React化はやくしたいなー(チラッチラッ #}
-      <div class="modal fade" id="admin-password-reset-modal">
-        <div class="modal-dialog">
-          <div class="modal-content">
-            <div class="modal-header">
-              <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
-              <div class="modal-title">{{ t('user_management.reset_password')}}</div>
-            </div>
-
-            <div class="modal-body">
-              <p>
-                {{ t('user_management.password_never_seen') }}<br>
-              <span class="text-danger">{{ t('user_management.send_new_password') }}</span>
-              </p>
-              <p>
-              {{ t('user_management.target_user') }}: <code id="admin-password-reset-user"></code>
-              </p>
-
-              <form method="post" id="admin-users-reset-password">
-                <input type="hidden" name="user_id" value="">
-                <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                <button type="submit" value="" class="btn btn-primary">
-                  {{ t('user_management.reset_password')}}
-                </button>
-              </form>
-
-            </div>
-
-          </div><!-- /.modal-content -->
-        </div>/.modal-dialog
-      </div>
-      <div class="modal fade" id="admin-password-reset-modal-done">
-        <div class="modal-dialog">
-          <div class="modal-content">
-
-            <div class="modal-header">
-              <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
-              <div class="modal-title">{{ t('user_management.reset_password') }}</div>
-            </div>
-
-            <div class="modal-body">
-              <p class="alert alert-danger">Let the user know the new password below and strongly recommend to change another one immediately. </p>
-              <p>
-              Reset user: <code id="admin-password-reset-done-user"></code>
-              </p>
-              <p>
-              New password: <code id="admin-password-reset-done-password"></code>
-              </p>
-            </div>
-            <div class="modal-footer">
-              <button class="btn btn-primary" data-dismiss="modal">OK</button>
-            </div>
-          </div><!-- /.modal-content -->
-        </div><!-- /.modal-dialog -->
-      </div>
-
-      <h2>{{ t("User_Management") }}</h2>
-
-      <table class="table table-default table-bordered table-user-list">
-        <thead>
-          <tr>
-            <th width="100px">#</th>
-            <th>{{ t('status') }}</th>
-            <th><code>{{ t('User') }}</code></th>
-            <th>{{ t('Name') }}</th>
-            <th>{{ t('Email') }}</th>
-            <th width="100px">{{ t('Created') }}</th>
-            <th width="150px">{{ t('Last_Login') }}</th>
-            <th width="70px"></th>
-          </tr>
-        </thead>
-        <tbody>
-          {% for sUser in users %}
-          {% set sUserId = sUser._id.toString() %}
-          <tr>
-            <td>
-              <img src="{{ sUser|picture }}" class="picture img-circle" />
-              {% if sUser.admin %}
-              <span class="label label-inverse label-admin">
-              {{ t('administrator') }}
-              </span>
-              {% endif %}
-            </td>
-            <td>
-              <span class="label {{ css.userStatus(sUser) }}">
-                {{ consts.userStatus[sUser.status] }}
-              </span>
-            </td>
-            <td>
-              <strong>{{ sUser.username }}</strong>
-            </td>
-            <td>{{ sUser.name }}</td>
-            <td>{{ sUser.email }}</td>
-            <td>{{ sUser.createdAt|date('Y-m-d', sUser.createdAt.getTimezoneOffset()) }}</td>
-            <td>
-              {% if sUser.lastLoginAt %}
-                {{ sUser.lastLoginAt|date('Y-m-d H:i', sUser.createdAt.getTimezoneOffset()) }}
-              {% endif %}
-            </td>
-            <td>
-              <div class="btn-group admin-user-menu">
-                <button type="button" class="btn btn-sm btn-default dropdown-toggle" data-toggle="dropdown">
-                  <i class="icon-settings"></i> <span class="caret"></span>
-                </button>
-                <ul class="dropdown-menu" role="menu">
-                  <li class="dropdown-header">{{ t('user_management.edit_menu') }}</li>
-                  <li>
-                    <a href="#"
-                        data-user-id="{{ sUserId }}"
-                        data-user-email="{{ sUser.email }}"
-                        data-target="#admin-password-reset-modal"
-                        data-toggle="modal">
-                      <i class="icon-fw icon-key"></i>
-                      {{ t('user_management.reset_password') }}
-                    </a>
-                  </li>
-                  <li class="divider"></li>
-                  <li class="dropdown-header">{{ t('status') }}</li>
-
-                  {% if sUser.status == 1 %}
-                  <form id="form_activate_{{ sUserId }}" action="/admin/user/{{ sUserId }}/activate" method="post">
-                    <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                  </form>
-                  <li>
-                    <a href="javascript:form_activate_{{ sUserId }}.submit()">
-                      <i class="icon-fw icon-user-following"></i> {{ t('user_management.accept') }}
-                    </a>
-                  </li>
-                  {% endif  %}
-
-                  {% if sUser.status == 2 %}
-                  <form id="form_suspend_{{ sUserId }}" action="/admin/user/{{ sUserId }}/suspend" method="post">
-                    <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                  </form>
-                  <li>
-                    {% if sUser.username != user.username %}
-                    <a href="javascript:form_suspend_{{ sUserId }}.submit()">
-                      <i class="icon-fw icon-ban"></i>
-                      {{ t('user_management.deactivate_account') }}
-                    </a>
-                    {% else %}
-                    <a disabled>
-                      <i class="icon-fw icon-ban"></i>
-                      {{ t('user_management.deactivate_account') }}
-                    </a>
-                    <p class="alert alert-danger m-l-10 m-r-10 p-10">{{ t("user_management.your_own") }}</p>
-                    {% endif %}
-                  </li>
-                  {% endif %}
-
-                  {% if sUser.status == 3 %}
-                  <form id="form_activate_{{ sUserId }}" action="/admin/user/{{ sUserId }}/activate" method="post">
-                    <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                  </form>
-                  <form id="form_remove_{{ sUserId }}" action="/admin/user/{{ sUserId }}/remove" method="post">
-                    <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                  </form>
-                  <li>
-                    <a href="javascript:form_activate_{{ sUserId }}.submit()">
-                      <i class="icon-fw icon-action-redo"></i> {{ t('Undo') }}
-                    </a>
-                  </li>
-                  <li>
-                    {# label は同じだけど、こっちは論理削除 #}
-                    <a href="javascript:form_remove_{{ sUserId }}.submit()">
-                      <i class="icon-fw icon-fire text-danger"></i> {{ t('Delete') }}
-                    </a>
-                  </li>
-                  {% endif %}
-
-                  {% if sUser.status == 1 || sUser.status == 5 %}
-                  <form id="form_removeCompletely_{{ sUserId }}" action="/admin/user/{{ sUser._id.toString() }}/removeCompletely" method="post">
-                    <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                  </form>
-                  <li class="dropdown-button">
-                    {# label は同じだけど、こっちは物理削除 #}
-                    <a href="javascript:form_removeCompletely_{{ sUserId }}.submit()">
-                      <i class="icon-fw icon-fire text-danger"></i> {{ t('Delete') }}
-                    </a>
-                  </li>
-                  {% endif %}
-
-                  {% if sUser.status == 2 %} {# activated な人だけこのメニューを表示 #}
-                  <li class="divider"></li>
-                  <li class="dropdown-header">{{ t('user_management.administrator_menu') }}</li>
-
-                  {% if sUser.admin %}
-                  <form id="form_removeFromAdmin_{{ sUserId }}" action="/admin/user/{{ sUser._id.toString() }}/removeFromAdmin" method="post">
-                    <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                  </form>
-                  <li>
-                    {% if sUser.username != user.username %}
-                      <a href="javascript:form_removeFromAdmin_{{ sUserId }}.submit()">
-                        <i class="icon-fw icon-user-unfollow"></i> {{ t("user_management.remove_admin_access") }}
-                      </a>
-                    {% else %}
-                      <a disabled>
-                        <i class="icon-fw icon-user-unfollow"></i> {{ t("user_management.remove_admin_access") }}
-                      </a>
-                      <p class="alert alert-danger m-l-10 m-r-10 p-10">{{ t("user_management.cannot_remove") }}</p>
-                    {% endif %}
-                  </li>
-                  {% else %}
-                  <form id="form_makeAdmin_{{ sUserId }}" action="/admin/user/{{ sUser._id.toString() }}/makeAdmin" method="post">
-                    <input type="hidden" name="_csrf" value="{{ csrf() }}">
-                  </form>
-                  <li>
-                    <a href="javascript:form_makeAdmin_{{ sUserId }}.submit()">
-                      <i class="icon-fw icon-magic-wand"></i> {{ t("user_management.give_admin_access") }}
-                    </a>
-                  </li>
-                  {% endif %}
-
-                  {% endif %}
-                </ul>
-              </div>
-            </td>
-          </tr>
-          {% endfor %}
-        </tbody>
-      </table>
-
-      {% include '../widget/pager.html' with {path: "/admin/users", pager: pager} %}
-
-    </div>
-  </div>
-</div>
-{% endblock content_main %}
-
-{% block content_footer %}
-{% endblock content_footer %} -->

src/server/views/admin/export.html

@@ -1,11 +1,11 @@
 {% extends '../layout/admin.html' %}
 
-{% block html_title %}{{ customizeService.generateCustomTitle(t('Export Data')) }}{% endblock %}
+{% block html_title %}{{ customizeService.generateCustomTitle(t('Export Archive Data')) }}{% endblock %}
 
 {% block content_header %}
 <div class="header-wrap">
   <header id="page-header">
-    <h1 id="admin-title" class="title">{{ t('Export Data') }}</h1>
+    <h1 id="admin-title" class="title">{{ t('Export Archive Data') }}</h1>
   </header>
 </div>
 {% endblock %}

src/server/views/admin/markdown.html

@@ -22,33 +22,6 @@
   </div>
 
 </div>
-
-<script>
-  // give a space between items in textarea(',' => ', ')
-  for (var i = 0; i < $('textarea.xss-list').length; i++) {
-    $($('textarea.xss-list')[i]).val($($('textarea.xss-list')[i]).val().replace(/,/g, ', '));
-  };
-
-  $('input[name="markdownSetting[markdown:xss:isEnabledPrevention]"]').change(function() {
-    if ($(this).val() === 'true') {
-      $('#xss-hide-when-disabled').slideDown();
-    }
-    else {
-      $('#xss-hide-when-disabled').slideUp();
-    }
-  });
-
-  $('#btn-import-tags').on('click', () => {
-    var $tagWhiteList = $('textarea[name="markdownSetting[markdown:xss:tagWhiteList]"]');
-    var $recommendedTagList = $('textarea[name="recommendedTags"]');
-    $tagWhiteList.val($recommendedTagList.val());
-  });
-  $('#btn-import-attrs').on('click', () => {
-    var $attrWhiteList = $('textarea[name="markdownSetting[markdown:xss:attrWhiteList]"]');
-    var $recommendedAttrList = $('textarea[name="recommendedAttrs"]');
-    $attrWhiteList.val($recommendedAttrList.val());
-  });
-</script>
 {% endblock content_main %}
 
 {% block content_footer %}

src/server/views/admin/users.html

@@ -30,9 +30,8 @@
     {% include './widget/menu.html' with {current: 'user'} %}
   </div>
   <div
-  class="col-md-9"
-  id ="admin-user-page"
-  users= "{{ users | json }}"
+    class="col-md-9"
+    id ="admin-user-page"
   >
   </div>
 </div>

src/server/views/admin/widget/menu.html

@@ -8,7 +8,7 @@
   <li class="{% if current == 'markdown'%}active{% endif %}"><a href="/admin/markdown"><i class="icon-fw icon-note"></i> {{ t('Markdown Settings') }}</a></li>
   <li class="{% if current == 'customize'%}active{% endif %}"><a href="/admin/customize"><i class="icon-fw icon-wrench"></i> {{ t('Customize') }}</a></li>
   <li class="{% if current == 'importer'%}active{% endif %}"><a href="/admin/importer"><i class="icon-fw icon-cloud-upload"></i> {{ t('Import Data') }}</a></li>
-  <li class="{% if current == 'export'%}active{% endif %}"><a href="/admin/export"><i class="icon-fw icon-cloud-download"></i> {{ t('Export Data') }}</a></li>
+  <li class="{% if current == 'export'%}active{% endif %}"><a href="/admin/export"><i class="icon-fw icon-cloud-download"></i> {{ t('Export Archive Data') }}</a></li>
   <li class="{% if current == 'notification'%}active{% endif %}"><a href="/admin/notification"><i class="icon-fw icon-bell"></i> {{ t('Notification Settings') }}</a></li>
   <li class="{% if current == 'user' || current == 'external-account' %}active{% endif %}"><a href="/admin/users"><i class="icon-fw icon-user"></i> {{ t('User_Management') }}</a></li>
   <li class="{% if current == 'user-group'%}active{% endif %}"><a href="/admin/user-groups"><i class="icon-fw icon-people"></i> {{ t('UserGroup Management') }}</a></li>

src/server/views/layout/layout.html

@@ -136,6 +136,11 @@
             <i class="icon-pencil"></i><span>{{ t('New') }}</span>
           </a>
         </li>
+        <li class="nav-item-help">
+          <a href="https://docs.growi.org/" target="_blank">
+            <i class="icon-question"></i><span>{{ t('Help') }}</span><span class="text-muted small"><i class="icon-share-alt"></i></span>
+          </a>
+        </li>
         <li class="dropdown">
           <a class="dropdown-toggle waves-effect waves-light" data-toggle="dropdown">
             <img src="{{ user|picture }}" class="picture img-circle" width="25" /> <span class="user-name">{{ user.name }}</span>

src/test/global-setup.js

@@ -1,9 +1,14 @@
-const mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || process.env.MONGO_URI || 'mongodb://localhost/growi_test';
+// check env
+if (process.env.NODE_ENV !== 'test') {
+  throw new Error('\'process.env.NODE_ENV\' must be \'test\'');
+}
 
 const mongoose = require('mongoose');
 
+const { getMongoUri } = require('../lib/util/mongoose-utils');
+
 module.exports = async() => {
-  await mongoose.connect(mongoUri, { useNewUrlParser: true });
+  await mongoose.connect(getMongoUri(), { useNewUrlParser: true });
   await mongoose.connection.dropDatabase();
   await mongoose.disconnect();
 };

src/test/setup.js

@@ -1,13 +1,13 @@
-const mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || process.env.MONGO_URI || 'mongodb://localhost/growi_test';
-
 const mongoose = require('mongoose');
 
+const { getMongoUri } = require('@commons/util/mongoose-utils');
+
 mongoose.Promise = global.Promise;
 
 jest.setTimeout(30000); // default 5000
 
 beforeAll(async(done) => {
-  await mongoose.connect(mongoUri, { useNewUrlParser: true });
+  await mongoose.connect(getMongoUri(), { useNewUrlParser: true });
   done();
 });
 

yarn.lock

@@ -1682,6 +1682,15 @@ array-unique@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
 
+array.prototype.flatmap@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.2.2.tgz#28d621d351c19a62b84331b01669395ef6cef4c4"
+  integrity sha512-ZZtPLE74KNE+0XcPv/vQmcivxN+8FhwOLvt2udHauO0aDEpsXDQrmd5HuJGpgPVyaV8HvkDPWnJ2iaem0oCKtA==
+  dependencies:
+    define-properties "^1.1.3"
+    es-abstract "^1.15.0"
+    function-bind "^1.1.1"
+
 arraybuffer.slice@~0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675"
@@ -2111,10 +2120,6 @@ block-stream@*:
   dependencies:
     inherits "~2.0.0"
 
-bluebird@3.0.5:
-  version "3.0.5"
-  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.0.5.tgz#2ff9d07c9b3edb29d6d280fe07528365e7ecd392"
-
 bluebird@3.5.1, bluebird@^3.5.1:
   version "3.5.1"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9"
@@ -4304,6 +4309,22 @@ es-abstract@^1.11.0, es-abstract@^1.12.0:
     is-regex "^1.0.4"
     object-keys "^1.0.12"
 
+es-abstract@^1.15.0:
+  version "1.15.0"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.15.0.tgz#8884928ec7e40a79e3c9bc812d37d10c8b24cc57"
+  integrity sha512-bhkEqWJ2t2lMeaJDuk7okMkJWI/yqgH/EoGwpcvv0XW9RWQsRspI4wt6xuyuvMvvQE3gg/D9HXppgk21w78GyQ==
+  dependencies:
+    es-to-primitive "^1.2.0"
+    function-bind "^1.1.1"
+    has "^1.0.3"
+    has-symbols "^1.0.0"
+    is-callable "^1.1.4"
+    is-regex "^1.0.4"
+    object-inspect "^1.6.0"
+    object-keys "^1.1.1"
+    string.prototype.trimleft "^2.1.0"
+    string.prototype.trimright "^2.1.0"
+
 es-abstract@^1.4.3:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.11.0.tgz#cce87d518f0496893b1a30cd8461835535480681"
@@ -8399,11 +8420,10 @@ mongoose-legacy-pluralize@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/mongoose-legacy-pluralize/-/mongoose-legacy-pluralize-1.0.2.tgz#3ba9f91fa507b5186d399fb40854bff18fb563e4"
 
-mongoose-paginate@^5.0.3:
-  version "5.0.3"
-  resolved "https://registry.yarnpkg.com/mongoose-paginate/-/mongoose-paginate-5.0.3.tgz#d7ae49ed5bf64f1f7af7620ea865b67058c55371"
-  dependencies:
-    bluebird "3.0.5"
+mongoose-paginate-v2@^1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/mongoose-paginate-v2/-/mongoose-paginate-v2-1.3.2.tgz#4a6077255156c555879c857eb0350b16272ed113"
+  integrity sha512-z8fmLaUjJ8u6Q/zxd/6JEbwKB+MY7lp2NahWlFdPYqiVHGVuL2cOpW99t4JA+EgW59V2zxwv8ZSoN0mFDaVrqw==
 
 mongoose-schema-jsonschema@>=1.2.1:
   version "1.2.1"
@@ -9020,6 +9040,11 @@ object-hash@>=1.3.1, object-hash@^1.3.1:
   resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df"
   integrity sha512-OSuu/pU4ENM9kmREg0BdNrUDIl1heYa4mBZacJc+vVWz4GtAwu7jO8s4AIt2aGRUTqxykpWzI3Oqnsm13tTMDA==
 
+object-inspect@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.6.0.tgz#c70b6cbf72f274aab4c34c0c82f5167bf82cf15b"
+  integrity sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==
+
 object-keys@^1.0.11, object-keys@^1.0.8:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
@@ -9028,6 +9053,11 @@ object-keys@^1.0.12:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.0.tgz#11bd22348dd2e096a045ab06f6c85bcc340fa032"
 
+object-keys@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e"
+  integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==
+
 object-keys@~0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336"
@@ -12193,6 +12223,22 @@ string.prototype.padend@^3.0.0:
     es-abstract "^1.4.3"
     function-bind "^1.0.2"
 
+string.prototype.trimleft@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/string.prototype.trimleft/-/string.prototype.trimleft-2.1.0.tgz#6cc47f0d7eb8d62b0f3701611715a3954591d634"
+  integrity sha512-FJ6b7EgdKxxbDxc79cOlok6Afd++TTs5szo+zJTUyow3ycrRfJVE2pq3vcN53XexvKZu/DJMDfeI/qMiZTrjTw==
+  dependencies:
+    define-properties "^1.1.3"
+    function-bind "^1.1.1"
+
+string.prototype.trimright@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/string.prototype.trimright/-/string.prototype.trimright-2.1.0.tgz#669d164be9df9b6f7559fa8e89945b168a5a6c58"
+  integrity sha512-fXZTSV55dNBwv16uw+hh5jkghxSnc5oHq+5K/gXgizHwAvMetdAJlHqqoFC1FSDVPYWLkAKl2cxpUT41sV7nSg==
+  dependencies:
+    define-properties "^1.1.3"
+    function-bind "^1.1.1"
+
 string@^3.0.1:
   version "3.3.3"
   resolved "https://registry.yarnpkg.com/string/-/string-3.3.3.tgz#5ea211cd92d228e184294990a6cc97b366a77cb0"