Merge pull request #1278 from weseek/imprv/rebuild-index

Imprv/rebuild index

config/migrate.js

@@ -7,15 +7,9 @@
 
 require('module-alias/register');
 
-function getMongoUri(env) {
-  return env.MONGOLAB_URI // for B.C.
-    || env.MONGODB_URI // MONGOLAB changes their env name
-    || env.MONGOHQ_URL
-    || env.MONGO_URI
-    || ((env.NODE_ENV === 'test') ? 'mongodb://localhost/growi_test' : 'mongodb://localhost/growi');
-}
+const { getMongoUri } = require('@commons/util/mongoose-utils');
 
-const mongoUri = getMongoUri(process.env);
+const mongoUri = getMongoUri();
 const match = mongoUri.match(/^(.+)\/([^/]+)$/);
 
 module.exports = {

package.json

@@ -72,6 +72,7 @@
     "@google-cloud/storage": "^3.3.0",
     "JSONStream": "^1.3.5",
     "archiver": "^3.1.1",
+    "array.prototype.flatmap": "^1.2.2",
     "async": "^3.0.1",
     "aws-sdk": "^2.88.0",
     "axios": "^0.19.0",

src/client/js/app.jsx

@@ -38,14 +38,13 @@ import UserGroupDetailPage from './components/Admin/UserGroupDetail/UserGroupDet
 import CustomCssEditor from './components/Admin/CustomCssEditor';
 import CustomScriptEditor from './components/Admin/CustomScriptEditor';
 import CustomHeaderEditor from './components/Admin/CustomHeaderEditor';
-import AdminRebuildSearch from './components/Admin/AdminRebuildSearch';
 import MarkdownSetting from './components/Admin/MarkdownSetting/MarkDownSetting';
 import Users from './components/Admin/Users/Users';
 import ManageExternalAccount from './components/Admin/Users/ManageExternalAccount';
 import UserGroupPage from './components/Admin/UserGroup/UserGroupPage';
 import Customize from './components/Admin/Customize/Customize';
 import Importer from './components/Admin/Importer';
-import FullTextSearchManagement from './components/Admin/FullTextSearchManagement/FullTextSearchPage';
+import FullTextSearchManagement from './components/Admin/FullTextSearchManagement';
 import ExportPage from './components/Admin/Export/ExportPage';
 
 import AppContainer from './services/AppContainer';
@@ -133,8 +132,6 @@ if (pageContainer.state.pageId != null) {
     'bookmark-button-lg':  <BookmarkButton pageId={pageContainer.state.pageId} crowi={appContainer} size="lg" />,
     'rename-page-name-input':  <PagePathAutoComplete crowi={appContainer} initializedPath={pageContainer.state.path} />,
     'duplicate-page-name-input':  <PagePathAutoComplete crowi={appContainer} initializedPath={pageContainer.state.path} />,
-
-    'admin-rebuild-search': <AdminRebuildSearch crowi={appContainer} />,
   }, componentMappings);
 }
 if (pageContainer.state.path != null) {

src/client/js/components/Admin/AdminRebuildSearch.jsx

@@ -1,82 +0,0 @@
-import React from 'react';
-import PropTypes from 'prop-types';
-
-import { createSubscribedElement } from '../UnstatedUtils';
-import WebsocketContainer from '../../services/WebsocketContainer';
-
-class AdminRebuildSearch extends React.Component {
-
-  constructor(props) {
-    super(props);
-
-    this.state = {
-      isCompleted: false,
-      total: 0,
-      current: 0,
-      skip: 0,
-    };
-  }
-
-  componentDidMount() {
-    const socket = this.props.websocketContainer.getWebSocket();
-
-    socket.on('admin:addPageProgress', (data) => {
-      const newStates = Object.assign(data, { isCompleted: false });
-      this.setState(newStates);
-    });
-
-    socket.on('admin:finishAddPage', (data) => {
-      const newStates = Object.assign(data, { isCompleted: true });
-      this.setState(newStates);
-    });
-  }
-
-  render() {
-    const {
-      total, current, skip, isCompleted,
-    } = this.state;
-    if (total === 0) {
-      return null;
-    }
-
-    const progressBarLabel = isCompleted ? 'Completed' : `Processing.. ${current}/${total} (${skip} skips)`;
-    const progressBarWidth = isCompleted ? '100%' : `${(current / total) * 100}%`;
-    const progressBarClassNames = isCompleted
-      ? 'progress-bar progress-bar-success'
-      : 'progress-bar progress-bar-striped progress-bar-animated active';
-
-    return (
-      <div>
-        <h5>
-          {progressBarLabel}
-          <span className="pull-right">{progressBarWidth}</span>
-        </h5>
-        <div className="progress progress-sm">
-          <div
-            className={progressBarClassNames}
-            role="progressbar"
-            aria-valuemin="0"
-            aria-valuenow={current}
-            aria-valuemax={total}
-            style={{ width: progressBarWidth }}
-          >
-          </div>
-        </div>
-      </div>
-    );
-  }
-
-}
-
-/**
- * Wrapper component for using unstated
- */
-const AdminRebuildSearchWrapper = (props) => {
-  return createSubscribedElement(AdminRebuildSearch, props, [WebsocketContainer]);
-};
-
-AdminRebuildSearch.propTypes = {
-  websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
-};
-
-export default AdminRebuildSearchWrapper;

src/client/js/components/Admin/Export/ExportingProgressBar.jsx → src/client/js/components/Admin/Common/ProgressBar.jsx

@@ -2,7 +2,7 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 
-class ExportingProgressBar extends React.Component {
+class ProgressBar extends React.Component {
 
 
   render() {
@@ -35,11 +35,11 @@ class ExportingProgressBar extends React.Component {
 
 }
 
-ExportingProgressBar.propTypes = {
+ProgressBar.propTypes = {
   header: PropTypes.string.isRequired,
   currentCount: PropTypes.number.isRequired,
   totalCount: PropTypes.number.isRequired,
   isInProgress: PropTypes.bool,
 };
 
-export default withTranslation()(ExportingProgressBar);
+export default withTranslation()(ProgressBar);

src/client/js/components/Admin/Export/ExportPage.jsx

@@ -9,9 +9,10 @@ import AppContainer from '../../../services/AppContainer';
 import WebsocketContainer from '../../../services/WebsocketContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
+import ProgressBar from '../Common/ProgressBar';
+
 import ExportZipFormModal from './ExportZipFormModal';
 import ZipFileTable from './ZipFileTable';
-import ExportingProgressBar from './ExportingProgressBar';
 
 class ExportPage extends React.Component {
 
@@ -158,7 +159,7 @@ class ExportPage extends React.Component {
       const { collectionName, currentCount, totalCount } = progressData;
       return (
         <div className="col-md-6" key={collectionName}>
-          <ExportingProgressBar
+          <ProgressBar
             header={collectionName}
             currentCount={currentCount}
             totalCount={totalCount}
@@ -181,7 +182,7 @@ class ExportPage extends React.Component {
     return (
       <div className="row px-3">
         <div className="col-md-12" key="progressBarForZipping">
-          <ExportingProgressBar
+          <ProgressBar
             header="Zip Files"
             currentCount={1}
             totalCount={1}

src/client/js/components/Admin/FullTextSearchManagement.jsx

@@ -0,0 +1,35 @@
+import React, { Fragment } from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+import { createSubscribedElement } from '../UnstatedUtils';
+import AppContainer from '../../services/AppContainer';
+
+import RebuildIndex from './FullTextSearchManagement/RebuildIndex';
+
+
+class FullTextSearchManagement extends React.Component {
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <Fragment>
+        <h2> { t('full_text_search_management.elasticsearch_management') } </h2>
+        <RebuildIndex />
+      </Fragment>
+    );
+  }
+
+}
+
+const FullTextSearchManagementWrapper = (props) => {
+  return createSubscribedElement(FullTextSearchManagement, props, [AppContainer]);
+};
+
+FullTextSearchManagement.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+};
+
+export default withTranslation()(FullTextSearchManagementWrapper);

src/client/js/components/Admin/FullTextSearchManagement/FullTextSearchPage.jsx

@@ -1,75 +0,0 @@
-import React, { Fragment } from 'react';
-import PropTypes from 'prop-types';
-import { withTranslation } from 'react-i18next';
-
-import AdminRebuildSearch from '../AdminRebuildSearch';
-import AppContainer from '../../../services/AppContainer';
-
-import { createSubscribedElement } from '../../UnstatedUtils';
-import { toastSuccess, toastError } from '../../../util/apiNotification';
-
-
-class FullTextSearchManagement extends React.Component {
-
-  constructor(props) {
-    super(props);
-
-    this.buildIndex = this.buildIndex.bind(this);
-  }
-
-  async buildIndex() {
-
-    const { appContainer } = this.props;
-    const pageId = this.pageId;
-
-    try {
-      const res = await appContainer.apiPost('/admin/search/build', { page_id: pageId });
-      if (!res.ok) {
-        throw new Error(res.message);
-      }
-      else {
-        toastSuccess('Building request is successfully posted.');
-      }
-    }
-    catch (e) {
-      toastError(e, (new Error('エラーが発生しました')));
-    }
-  }
-
-  render() {
-    const { t } = this.props;
-
-    return (
-      <Fragment>
-        <fieldset>
-          <legend> { t('full_text_search_management.elasticsearch_management') } </legend>
-          <div className="form-group form-horizontal">
-            <div className="col-xs-3 control-label"></div>
-            <div className="col-xs-7">
-              <button type="submit" className="btn btn-inverse" onClick={this.buildIndex}>{ t('full_text_search_management.build_button') }</button>
-              <p className="help-block">
-                { t('full_text_search_management.rebuild_description_1') }<br />
-                { t('full_text_search_management.rebuild_description_2') }<br />
-                { t('full_text_search_management.rebuild_description_3') }<br />
-              </p>
-            </div>
-          </div>
-        </fieldset>
-
-        <AdminRebuildSearch />
-      </Fragment>
-    );
-  }
-
-}
-
-const FullTextSearchManagementWrapper = (props) => {
-  return createSubscribedElement(FullTextSearchManagement, props, [AppContainer]);
-};
-
-FullTextSearchManagement.propTypes = {
-  t: PropTypes.func.isRequired, // i18next
-  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
-};
-
-export default withTranslation()(FullTextSearchManagementWrapper);

src/client/js/components/Admin/FullTextSearchManagement/RebuildIndex.jsx

@@ -0,0 +1,133 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { withTranslation } from 'react-i18next';
+
+import { createSubscribedElement } from '../../UnstatedUtils';
+import AppContainer from '../../../services/AppContainer';
+import WebsocketContainer from '../../../services/WebsocketContainer';
+import { toastSuccess, toastError } from '../../../util/apiNotification';
+
+import ProgressBar from '../Common/ProgressBar';
+
+class RebuildIndex extends React.Component {
+
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      isProcessing: false,
+      isCompleted: false,
+
+      total: 0,
+      current: 0,
+      skip: 0,
+    };
+
+    this.buildIndex = this.buildIndex.bind(this);
+  }
+
+  componentDidMount() {
+    const socket = this.props.websocketContainer.getWebSocket();
+
+    socket.on('admin:addPageProgress', (data) => {
+      this.setState({
+        isProcessing: true,
+        ...data,
+      });
+    });
+
+    socket.on('admin:finishAddPage', (data) => {
+      this.setState({
+        isProcessing: false,
+        isCompleted: true,
+        ...data,
+      });
+    });
+  }
+
+  async buildIndex() {
+
+    const { appContainer } = this.props;
+    const pageId = this.pageId;
+
+    try {
+      const res = await appContainer.apiPost('/admin/search/build', { page_id: pageId });
+      if (!res.ok) {
+        throw new Error(res.message);
+      }
+
+      this.setState({ isProcessing: true });
+      toastSuccess('Rebuilding is requested');
+    }
+    catch (e) {
+      toastError(e);
+    }
+  }
+
+  renderProgressBar() {
+    const {
+      total, current, skip, isProcessing, isCompleted,
+    } = this.state;
+    const showProgressBar = isProcessing || isCompleted;
+
+    if (!showProgressBar) {
+      return null;
+    }
+
+    const header = isCompleted ? 'Completed' : `Processing.. (${skip} skips)`;
+
+    return (
+      <ProgressBar
+        header={header}
+        currentCount={current}
+        totalCount={total}
+      />
+    );
+  }
+
+  render() {
+    const { t } = this.props;
+
+    return (
+      <>
+        <div className="row">
+          <div className="col-xs-3 control-label"></div>
+          <div className="col-xs-9">
+            { this.renderProgressBar() }
+
+            <button
+              type="submit"
+              className="btn btn-inverse"
+              onClick={this.buildIndex}
+              disabled={this.state.isProcessing}
+            >
+              { t('full_text_search_management.build_button') }
+            </button>
+
+            <p className="help-block">
+              { t('full_text_search_management.rebuild_description_1') }<br />
+              { t('full_text_search_management.rebuild_description_2') }<br />
+              { t('full_text_search_management.rebuild_description_3') }<br />
+            </p>
+          </div>
+        </div>
+      </>
+    );
+  }
+
+}
+
+/**
+ * Wrapper component for using unstated
+ */
+const RebuildIndexWrapper = (props) => {
+  return createSubscribedElement(RebuildIndex, props, [AppContainer, WebsocketContainer]);
+};
+
+RebuildIndex.propTypes = {
+  t: PropTypes.func.isRequired, // i18next
+  appContainer: PropTypes.instanceOf(AppContainer).isRequired,
+  websocketContainer: PropTypes.instanceOf(WebsocketContainer).isRequired,
+};
+
+export default withTranslation()(RebuildIndexWrapper);

src/client/js/components/SearchPage/SearchResultList.jsx

@@ -15,12 +15,16 @@ class SearchResultList extends React.Component {
 
   render() {
     const resultList = this.props.pages.map((page) => {
+      const showTags = (page.tags != null) && (page.tags.length > 0);
+
       return (
         <div id={page._id} key={page._id} className="search-result-page mb-5">
-          <h2><a href={page.path}>{page.path}</a></h2>
-          { page.tags.length > 0 && (
-            <span><i className="tag-icon icon-tag"></i> {page.tags.join(', ')}</span>
-          )}
+          <h2>
+            <a href={page.path}>{page.path}</a>
+            { showTags && (
+              <div className="mt-1 small"><i className="tag-icon icon-tag"></i> {page.tags.join(', ')}</div>
+            )}
+          </h2>
           <RevisionLoader
             growiRenderer={this.growiRenderer}
             pageId={page._id}

src/lib/util/mongoose-utils.js

@@ -1,5 +1,15 @@
 const mongoose = require('mongoose');
 
+const getMongoUri = () => {
+  const { env } = process;
+
+  return env.MONGOLAB_URI // for B.C.
+    || env.MONGODB_URI // MONGOLAB changes their env name
+    || env.MONGOHQ_URL
+    || env.MONGO_URI
+    || ((env.NODE_ENV === 'test') ? 'mongodb://localhost/growi_test' : 'mongodb://localhost/growi');
+};
+
 const getModelSafely = (modelName) => {
   if (mongoose.modelNames().includes(modelName)) {
     return mongoose.model(modelName);
@@ -8,5 +18,6 @@ const getModelSafely = (modelName) => {
 };
 
 module.exports = {
+  getMongoUri,
   getModelSafely,
 };

src/server/crowi/index.js

@@ -6,6 +6,7 @@ const pkg = require('@root/package.json');
 const InterceptorManager = require('@commons/service/interceptor-manager');
 const CdnResourcesService = require('@commons/service/cdn-resources-service');
 const Xss = require('@commons/service/xss');
+const { getMongoUri } = require('@commons/util/mongoose-utils');
 
 const path = require('path');
 
@@ -74,14 +75,6 @@ function Crowi(rootdir) {
   };
 }
 
-function getMongoUrl(env) {
-  return env.MONGOLAB_URI // for B.C.
-    || env.MONGODB_URI // MONGOLAB changes their env name
-    || env.MONGOHQ_URL
-    || env.MONGO_URI
-    || ((process.env.NODE_ENV === 'test') ? 'mongodb://localhost/growi_test' : 'mongodb://localhost/growi');
-}
-
 Crowi.prototype.init = async function() {
   await this.setupDatabase();
   await this.setupModels();
@@ -202,10 +195,10 @@ Crowi.prototype.event = function(name, event) {
 };
 
 Crowi.prototype.setupDatabase = function() {
-  // mongoUri = mongodb://user:password@host/dbname
   mongoose.Promise = global.Promise;
 
-  const mongoUri = getMongoUrl(this.env);
+  // mongoUri = mongodb://user:password@host/dbname
+  const mongoUri = getMongoUri();
 
   return mongoose.connect(mongoUri, { useNewUrlParser: true });
 };
@@ -216,7 +209,7 @@ Crowi.prototype.setupSessionConfig = function() {
   const sessionAge = (1000 * 3600 * 24 * 30);
   const redisUrl = this.env.REDISTOGO_URL || this.env.REDIS_URI || this.env.REDIS_URL || null;
 
-  const mongoUrl = getMongoUrl(this.env);
+  const mongoUrl = getMongoUri();
   let sessionConfig;
 
   return new Promise(((resolve, reject) => {

src/server/models/bookmark.js

@@ -21,6 +21,23 @@ module.exports = function(crowi) {
     return await this.count({ page: pageId });
   };
 
+  /**
+   * @return {object} key: page._id, value: bookmark count
+   */
+  bookmarkSchema.statics.getPageIdToCountMap = async function(pageIds) {
+    const results = await this.aggregate()
+      .match({ page: { $in: pageIds } })
+      .group({ _id: '$page', count: { $sum: 1 } });
+
+    // convert to map
+    const idToCountMap = {};
+    results.forEach((result) => {
+      idToCountMap[result._id] = result.count;
+    });
+
+    return idToCountMap;
+  };
+
   bookmarkSchema.statics.populatePage = async function(bookmarks) {
     const Bookmark = this;
     const User = crowi.model('User');
@@ -122,12 +139,12 @@ module.exports = function(crowi) {
     }
   };
 
-  bookmarkSchema.statics.removeBookmark = async function(page, user) {
+  bookmarkSchema.statics.removeBookmark = async function(pageId, user) {
     const Bookmark = this;
 
     try {
-      const data = await Bookmark.findOneAndRemove({ page, user });
-      bookmarkEvent.emit('delete', page);
+      const data = await Bookmark.findOneAndRemove({ page: pageId, user });
+      bookmarkEvent.emit('delete', pageId);
       return data;
     }
     catch (err) {

src/server/models/page-tag-relation.js

@@ -1,6 +1,8 @@
 // disable no-return-await for model functions
 /* eslint-disable no-return-await */
 
+const flatMap = require('array.prototype.flatmap');
+
 const mongoose = require('mongoose');
 const mongoosePaginate = require('mongoose-paginate');
 
@@ -60,6 +62,52 @@ class PageTagRelation {
     return relations.map((relation) => { return relation.relatedTag.name });
   }
 
+  /**
+   * @return {object} key: Page._id, value: array of tag names
+   */
+  static async getIdToTagNamesMap(pageIds) {
+    /**
+     * @see https://docs.mongodb.com/manual/reference/operator/aggregation/group/#pivot-data
+     *
+     * results will be:
+     * [
+     *   { _id: 58dca7b2c435b3480098dbbc, tagIds: [ 5da630f71a677515601e36d7, 5da77163ec786e4fe43e0e3e ]},
+     *   { _id: 58dca7b2c435b3480098dbbd, tagIds: [ ... ]},
+     *   ...
+     * ]
+     */
+    const results = await this.aggregate()
+      .match({ relatedPage: { $in: pageIds } })
+      .group({ _id: '$relatedPage', tagIds: { $push: '$relatedTag' } });
+
+    if (results.length === 0) {
+      return {};
+    }
+
+    results.flatMap = flatMap.shim(); // TODO: remove after upgrading to node v12
+
+    // extract distinct tag ids
+    const allTagIds = results
+      .flatMap(result => result.tagIds); // map + flatten
+    const distinctTagIds = Array.from(new Set(allTagIds));
+
+    // retrieve tag documents
+    const Tag = mongoose.model('Tag');
+    const tagIdToNameMap = await Tag.getIdToNameMap(distinctTagIds);
+
+    // convert to map
+    const idToTagNamesMap = {};
+    results.forEach((result) => {
+      const tagNames = result.tagIds
+        .map(tagId => tagIdToNameMap[tagId])
+        .filter(tagName => tagName != null); // filter null object
+
+      idToTagNamesMap[result._id] = tagNames;
+    });
+
+    return idToTagNamesMap;
+  }
+
   static async updatePageTags(pageId, tags) {
     if (pageId == null || tags == null) {
       throw new Error('args \'pageId\' and \'tags\' are required.');

src/server/models/page.js

@@ -923,21 +923,6 @@ module.exports = function(crowi) {
     return { templateBody, templateTags };
   };
 
-  /**
-   * Bulk get (for internal only)
-   */
-  pageSchema.statics.getStreamOfFindAll = function(options) {
-    const criteria = { redirectTo: null };
-
-    return this.find(criteria)
-      .populate([
-        { path: 'creator', model: 'User' },
-        { path: 'revision', model: 'Revision' },
-      ])
-      .lean()
-      .cursor();
-  };
-
   async function pushRevision(pageData, newRevision, user) {
     await newRevision.save();
     debug('Successfully saved new revision', newRevision);
@@ -1384,10 +1369,6 @@ module.exports = function(crowi) {
     return addSlashOfEnd(path);
   };
 
-  pageSchema.statics.allPageCount = function() {
-    return this.count({ redirectTo: null });
-  };
-
   pageSchema.statics.GRANT_PUBLIC = GRANT_PUBLIC;
   pageSchema.statics.GRANT_RESTRICTED = GRANT_RESTRICTED;
   pageSchema.statics.GRANT_SPECIFIED = GRANT_SPECIFIED;

src/server/models/tag.js

@@ -23,6 +23,17 @@ schema.plugin(mongoosePaginate);
  */
 class Tag {
 
+  static async getIdToNameMap(tagIds) {
+    const tags = await this.find({ _id: { $in: tagIds } });
+
+    const idToNameMap = {};
+    tags.forEach((tag) => {
+      idToNameMap[tag._id.toString()] = tag.name;
+    });
+
+    return idToNameMap;
+  }
+
   static async findOrCreate(tagName) {
     const tag = await this.findOne({ name: tagName });
     if (!tag) {

src/server/routes/admin.js

@@ -91,6 +91,14 @@ module.exports = function(crowi, app) {
     return pager;
   }
 
+  // setup websocket event for rebuild index
+  searchEvent.on('addPageProgress', (total, current, skip) => {
+    crowi.getIo().sockets.emit('admin:addPageProgress', { total, current, skip });
+  });
+  searchEvent.on('finishAddPage', (total, current, skip) => {
+    crowi.getIo().sockets.emit('admin:finishAddPage', { total, current, skip });
+  });
+
   actions.index = function(req, res) {
     return res.render('admin/index', {
       plugins: pluginUtils.listPlugins(crowi.rootDir),
@@ -1144,14 +1152,12 @@ module.exports = function(crowi, app) {
       return res.json(ApiResponse.error('ElasticSearch Integration is not set up.'));
     }
 
-    searchEvent.on('addPageProgress', (total, current, skip) => {
-      crowi.getIo().sockets.emit('admin:addPageProgress', { total, current, skip });
-    });
-    searchEvent.on('finishAddPage', (total, current, skip) => {
-      crowi.getIo().sockets.emit('admin:finishAddPage', { total, current, skip });
-    });
-
-    await search.buildIndex();
+    try {
+      search.buildIndex();
+    }
+    catch (err) {
+      return res.json(ApiResponse.error(err));
+    }
 
     return res.json(ApiResponse.success());
   };

src/server/util/search.js

@@ -6,6 +6,13 @@ const elasticsearch = require('elasticsearch');
 const debug = require('debug')('growi:lib:search');
 const logger = require('@alias/logger')('growi:lib:search');
 
+const {
+  Writable, Transform,
+} = require('stream');
+const streamToPromise = require('stream-to-promise');
+
+const BULK_REINDEX_SIZE = 100;
+
 function SearchClient(crowi, esUri) {
   this.DEFAULT_OFFSET = 0;
   this.DEFAULT_LIMIT = 50;
@@ -84,9 +91,8 @@ SearchClient.prototype.checkESVersion = async function() {
 
 SearchClient.prototype.registerUpdateEvent = function() {
   const pageEvent = this.crowi.event('page');
-  pageEvent.on('create', this.syncPageCreated.bind(this));
+  pageEvent.on('create', this.syncPageUpdated.bind(this));
   pageEvent.on('update', this.syncPageUpdated.bind(this));
-  pageEvent.on('updateTag', this.syncPageUpdated.bind(this));
   pageEvent.on('delete', this.syncPageDeleted.bind(this));
 
   const bookmarkEvent = this.crowi.event('bookmark');
@@ -98,7 +104,7 @@ SearchClient.prototype.registerUpdateEvent = function() {
 };
 
 SearchClient.prototype.shouldIndexed = function(page) {
-  return (page.redirectTo == null);
+  return page.creator != null && page.revision != null && page.redirectTo == null;
 };
 
 // BONSAI_URL is following format:
@@ -163,6 +169,7 @@ SearchClient.prototype.buildIndex = async function(uri) {
   // reindex to tmp index
   await this.createIndex(tmpIndexName);
   await client.reindex({
+    waitForCompletion: false,
     body: {
       source: { index: indexName },
       dest: { index: tmpIndexName },
@@ -225,38 +232,6 @@ function generateDocContentsRelatedToRestriction(page) {
   };
 }
 
-SearchClient.prototype.prepareBodyForUpdate = function(body, page) {
-  if (!Array.isArray(body)) {
-    throw new Error('Body must be an array.');
-  }
-
-  const command = {
-    update: {
-      _index: this.aliasName,
-      _type: 'pages',
-      _id: page._id.toString(),
-    },
-  };
-
-  let document = {
-    path: page.path,
-    body: page.revision.body,
-    comment_count: page.commentCount,
-    bookmark_count: page.bookmarkCount || 0,
-    like_count: page.liker.length || 0,
-    updated_at: page.updatedAt,
-    tag_names: page.tagNames,
-  };
-
-  document = Object.assign(document, generateDocContentsRelatedToRestriction(page));
-
-  body.push(command);
-  body.push({
-    doc: document,
-    doc_as_upsert: true,
-  });
-};
-
 SearchClient.prototype.prepareBodyForCreate = function(body, page) {
   if (!Array.isArray(body)) {
     throw new Error('Body must be an array.');
@@ -296,7 +271,7 @@ SearchClient.prototype.prepareBodyForDelete = function(body, page) {
 
   const command = {
     delete: {
-      _index: this.aliasName,
+      _index: this.indexName,
       _type: 'pages',
       _id: page._id.toString(),
     },
@@ -305,122 +280,183 @@ SearchClient.prototype.prepareBodyForDelete = function(body, page) {
   body.push(command);
 };
 
-SearchClient.prototype.addPages = async function(pages) {
+SearchClient.prototype.addAllPages = async function() {
+  const Page = this.crowi.model('Page');
+  return this.updateOrInsertPages(() => Page.find(), true);
+};
+
+SearchClient.prototype.updateOrInsertPageById = async function(pageId) {
+  const Page = this.crowi.model('Page');
+  return this.updateOrInsertPages(() => Page.findById(pageId));
+};
+
+/**
+ * @param {function} queryFactory factory method to generate a Mongoose Query instance
+ */
+SearchClient.prototype.updateOrInsertPages = async function(queryFactory, isEmittingProgressEvent = false) {
+  const Page = this.crowi.model('Page');
+  const { PageQueryBuilder } = Page;
   const Bookmark = this.crowi.model('Bookmark');
   const PageTagRelation = this.crowi.model('PageTagRelation');
-  const body = [];
 
-  /* eslint-disable no-await-in-loop */
-  for (const page of pages) {
-    page.bookmarkCount = await Bookmark.countByPageId(page._id);
-    const tagRelations = await PageTagRelation.find({ relatedPage: page._id }).populate('relatedTag');
-    page.tagNames = tagRelations.map((relation) => { return relation.relatedTag.name });
-    this.prepareBodyForCreate(body, page);
-  }
-  /* eslint-enable no-await-in-loop */
+  const searchEvent = this.searchEvent;
 
-  logger.debug('addPages(): Sending Request to ES', body);
-  return this.client.bulk({
-    body,
+  const prepareBodyForCreate = this.prepareBodyForCreate.bind(this);
+  const shouldIndexed = this.shouldIndexed.bind(this);
+  const bulkWrite = this.client.bulk.bind(this.client);
+
+  const findQuery = new PageQueryBuilder(queryFactory()).addConditionToExcludeRedirect().query;
+  const countQuery = new PageQueryBuilder(queryFactory()).addConditionToExcludeRedirect().query;
+
+  const totalCount = await countQuery.count();
+
+  const readStream = findQuery
+    // populate data which will be referenced by prepareBodyForCreate()
+    .populate([
+      { path: 'creator', model: 'User', select: 'username' },
+      { path: 'revision', model: 'Revision', select: 'body' },
+    ])
+    .snapshot()
+    .lean()
+    .cursor();
+
+  let skipped = 0;
+  const thinOutStream = new Transform({
+    objectMode: true,
+    async transform(doc, encoding, callback) {
+      if (shouldIndexed(doc)) {
+        this.push(doc);
+      }
+      else {
+        skipped++;
+      }
+      callback();
+    },
   });
-};
 
-SearchClient.prototype.updatePages = async function(pages) {
-  const self = this;
-  const PageTagRelation = this.crowi.model('PageTagRelation');
-  const body = [];
+  let batchBuffer = [];
+  const batchingStream = new Transform({
+    objectMode: true,
+    transform(doc, encoding, callback) {
+      batchBuffer.push(doc);
 
-  /* eslint-disable no-await-in-loop */
-  for (const page of pages) {
-    const tagRelations = await PageTagRelation.find({ relatedPage: page._id }).populate('relatedTag');
-    page.tagNames = tagRelations.map((relation) => { return relation.relatedTag.name });
-    self.prepareBodyForUpdate(body, page);
-  }
+      if (batchBuffer.length >= BULK_REINDEX_SIZE) {
+        this.push(batchBuffer);
+        batchBuffer = [];
+      }
 
-  logger.debug('updatePages(): Sending Request to ES', body);
-  return this.client.bulk({
-    body,
+      callback();
+    },
+    final(callback) {
+      if (batchBuffer.length > 0) {
+        this.push(batchBuffer);
+      }
+      callback();
+    },
   });
-};
 
-SearchClient.prototype.deletePages = function(pages) {
-  const self = this;
-  const body = [];
+  const appendBookmarkCountStream = new Transform({
+    objectMode: true,
+    async transform(chunk, encoding, callback) {
+      const pageIds = chunk.map(doc => doc._id);
 
-  pages.map((page) => {
-    self.prepareBodyForDelete(body, page);
-    return;
+      const idToCountMap = await Bookmark.getPageIdToCountMap(pageIds);
+      const idsHavingCount = Object.keys(idToCountMap);
+
+      // append count
+      chunk
+        .filter(doc => idsHavingCount.includes(doc._id.toString()))
+        .forEach((doc) => {
+          // append count from idToCountMap
+          doc.bookmarkCount = idToCountMap[doc._id.toString()];
+        });
+
+      this.push(chunk);
+      callback();
+    },
   });
 
-  logger.debug('deletePages(): Sending Request to ES', body);
-  return this.client.bulk({
-    body,
+  const appendTagNamesStream = new Transform({
+    objectMode: true,
+    async transform(chunk, encoding, callback) {
+      const pageIds = chunk.map(doc => doc._id);
+
+      const idToTagNamesMap = await PageTagRelation.getIdToTagNamesMap(pageIds);
+      const idsHavingTagNames = Object.keys(idToTagNamesMap);
+
+      // append tagNames
+      chunk
+        .filter(doc => idsHavingTagNames.includes(doc._id.toString()))
+        .forEach((doc) => {
+          // append tagName from idToTagNamesMap
+          doc.tagNames = idToTagNamesMap[doc._id.toString()];
+        });
+
+      this.push(chunk);
+      callback();
+    },
   });
-};
 
-SearchClient.prototype.addAllPages = async function() {
-  const self = this;
-  const Page = this.crowi.model('Page');
-  const allPageCount = await Page.allPageCount();
-  const Bookmark = this.crowi.model('Bookmark');
-  const PageTagRelation = this.crowi.model('PageTagRelation');
-  const cursor = Page.getStreamOfFindAll();
-  let body = [];
-  let sent = 0;
-  let skipped = 0;
-  let total = 0;
+  let count = 0;
+  const writeStream = new Writable({
+    objectMode: true,
+    async write(batch, encoding, callback) {
+      const body = [];
+      batch.forEach(doc => prepareBodyForCreate(body, doc));
 
-  return new Promise((resolve, reject) => {
-    const bulkSend = (body) => {
-      self.client
-        .bulk({
+      try {
+        const res = await bulkWrite({
           body,
           requestTimeout: Infinity,
-        })
-        .then((res) => {
-          logger.info('addAllPages add anyway (items, errors, took): ', (res.items || []).length, res.errors, res.took, 'ms');
-        })
-        .catch((err) => {
-          logger.error('addAllPages error on add anyway: ', err);
         });
-    };
 
-    cursor
-      .eachAsync(async(doc) => {
-        if (!doc.creator || !doc.revision || !self.shouldIndexed(doc)) {
-          // debug('Skipped', doc.path);
-          skipped++;
-          return;
+        count += (res.items || []).length;
+
+        logger.info(`Adding pages progressing: (count=${count}, errors=${res.errors}, took=${res.took}ms)`);
+
+        if (isEmittingProgressEvent) {
+          searchEvent.emit('addPageProgress', totalCount, count, skipped);
         }
-        total++;
+      }
+      catch (err) {
+        logger.error('addAllPages error on add anyway: ', err);
+      }
 
-        const bookmarkCount = await Bookmark.countByPageId(doc._id);
-        const tagRelations = await PageTagRelation.find({ relatedPage: doc._id }).populate('relatedTag');
-        const page = { ...doc, bookmarkCount, tagNames: tagRelations.map((relation) => { return relation.relatedTag.name }) };
-        self.prepareBodyForCreate(body, page);
+      callback();
+    },
+    final(callback) {
+      logger.info(`Adding pages has terminated: (totalCount=${totalCount}, skipped=${skipped})`);
 
-        if (body.length >= 4000) {
-          // send each 2000 docs. (body has 2 elements for each data)
-          sent++;
-          logger.debug('Sending request (seq, total, skipped)', sent, total, skipped);
-          bulkSend(body);
-          this.searchEvent.emit('addPageProgress', allPageCount, total, skipped);
+      if (isEmittingProgressEvent) {
+        searchEvent.emit('finishAddPage', totalCount, count, skipped);
+      }
+      callback();
+    },
+  });
 
-          body = [];
-        }
-      })
-      .then(() => {
-        // send all remaining data on body[]
-        logger.debug('Sending last body of bulk operation:', body.length);
-        bulkSend(body);
-        this.searchEvent.emit('finishAddPage', allPageCount, total, skipped);
-
-        resolve();
-      })
-      .catch((e) => {
-        logger.error('Error wile iterating cursor.eachAsync()', e);
-        reject(e);
-      });
+  readStream
+    .pipe(thinOutStream)
+    .pipe(batchingStream)
+    .pipe(appendBookmarkCountStream)
+    .pipe(appendTagNamesStream)
+    .pipe(writeStream);
+
+  return streamToPromise(readStream);
+
+};
+
+SearchClient.prototype.deletePages = function(pages) {
+  const self = this;
+  const body = [];
+
+  pages.map((page) => {
+    self.prepareBodyForDelete(body, page);
+    return;
+  });
+
+  logger.debug('deletePages(): Sending Request to ES', body);
+  return this.client.bulk({
+    body,
   });
 };
 
@@ -855,76 +891,44 @@ SearchClient.prototype.parseQueryString = function(queryString) {
   };
 };
 
-SearchClient.prototype.syncPageCreated = function(page, user, bookmarkCount = 0) {
-  debug('SearchClient.syncPageCreated', page.path);
-
-  if (!this.shouldIndexed(page)) {
-    return;
-  }
-
-  page.bookmarkCount = bookmarkCount;
-  this.addPages([page])
-    .then((res) => {
-      debug('ES Response', res);
-    })
-    .catch((err) => {
-      logger.error('ES Error', err);
-    });
-};
+SearchClient.prototype.syncPageUpdated = async function(page, user) {
+  logger.debug('SearchClient.syncPageUpdated', page.path);
 
-SearchClient.prototype.syncPageUpdated = function(page, user, bookmarkCount = 0) {
-  debug('SearchClient.syncPageUpdated', page.path);
-  // TODO delete
+  // delete if page should not indexed
   if (!this.shouldIndexed(page)) {
-    this.deletePages([page])
-      .then((res) => {
-        debug('deletePages: ES Response', res);
-      })
-      .catch((err) => {
-        logger.error('deletePages:ES Error', err);
-      });
-
+    try {
+      await this.deletePages([page]);
+    }
+    catch (err) {
+      logger.error('deletePages:ES Error', err);
+    }
     return;
   }
 
-  page.bookmarkCount = bookmarkCount;
-  this.updatePages([page])
-    .then((res) => {
-      debug('ES Response', res);
-    })
-    .catch((err) => {
-      logger.error('ES Error', err);
-    });
+  return this.updateOrInsertPageById(page._id);
 };
 
-SearchClient.prototype.syncPageDeleted = function(page, user) {
+SearchClient.prototype.syncPageDeleted = async function(page, user) {
   debug('SearchClient.syncPageDeleted', page.path);
 
-  this.deletePages([page])
-    .then((res) => {
-      debug('deletePages: ES Response', res);
-    })
-    .catch((err) => {
-      logger.error('deletePages:ES Error', err);
-    });
+  try {
+    return await this.deletePages([page]);
+  }
+  catch (err) {
+    logger.error('deletePages:ES Error', err);
+  }
 };
 
 SearchClient.prototype.syncBookmarkChanged = async function(pageId) {
-  const Page = this.crowi.model('Page');
-  const Bookmark = this.crowi.model('Bookmark');
-  const page = await Page.findById(pageId);
-  const bookmarkCount = await Bookmark.countByPageId(pageId);
+  logger.debug('SearchClient.syncBookmarkChanged', pageId);
 
-  page.bookmarkCount = bookmarkCount;
-  this.updatePages([page])
-    .then((res) => { return debug('ES Response', res) })
-    .catch((err) => { return logger.error('ES Error', err) });
+  return this.updateOrInsertPageById(pageId);
 };
 
 SearchClient.prototype.syncTagChanged = async function(page) {
-  this.updatePages([page])
-    .then((res) => { return debug('ES Response', res) })
-    .catch((err) => { return logger.error('ES Error', err) });
+  logger.debug('SearchClient.syncTagChanged', page.path);
+
+  return this.updateOrInsertPageById(page._id);
 };
 
 

src/test/global-setup.js

@@ -1,9 +1,14 @@
-const mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || process.env.MONGO_URI || 'mongodb://localhost/growi_test';
+// check env
+if (process.env.NODE_ENV !== 'test') {
+  throw new Error('\'process.env.NODE_ENV\' must be \'test\'');
+}
 
 const mongoose = require('mongoose');
 
+const { getMongoUri } = require('../lib/util/mongoose-utils');
+
 module.exports = async() => {
-  await mongoose.connect(mongoUri, { useNewUrlParser: true });
+  await mongoose.connect(getMongoUri(), { useNewUrlParser: true });
   await mongoose.connection.dropDatabase();
   await mongoose.disconnect();
 };

src/test/setup.js

@@ -1,13 +1,13 @@
-const mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || process.env.MONGO_URI || 'mongodb://localhost/growi_test';
-
 const mongoose = require('mongoose');
 
+const { getMongoUri } = require('@commons/util/mongoose-utils');
+
 mongoose.Promise = global.Promise;
 
 jest.setTimeout(30000); // default 5000
 
 beforeAll(async(done) => {
-  await mongoose.connect(mongoUri, { useNewUrlParser: true });
+  await mongoose.connect(getMongoUri(), { useNewUrlParser: true });
   done();
 });
 

yarn.lock

@@ -1682,6 +1682,15 @@ array-unique@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
 
+array.prototype.flatmap@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.2.2.tgz#28d621d351c19a62b84331b01669395ef6cef4c4"
+  integrity sha512-ZZtPLE74KNE+0XcPv/vQmcivxN+8FhwOLvt2udHauO0aDEpsXDQrmd5HuJGpgPVyaV8HvkDPWnJ2iaem0oCKtA==
+  dependencies:
+    define-properties "^1.1.3"
+    es-abstract "^1.15.0"
+    function-bind "^1.1.1"
+
 arraybuffer.slice@~0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675"
@@ -4304,6 +4313,22 @@ es-abstract@^1.11.0, es-abstract@^1.12.0:
     is-regex "^1.0.4"
     object-keys "^1.0.12"
 
+es-abstract@^1.15.0:
+  version "1.15.0"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.15.0.tgz#8884928ec7e40a79e3c9bc812d37d10c8b24cc57"
+  integrity sha512-bhkEqWJ2t2lMeaJDuk7okMkJWI/yqgH/EoGwpcvv0XW9RWQsRspI4wt6xuyuvMvvQE3gg/D9HXppgk21w78GyQ==
+  dependencies:
+    es-to-primitive "^1.2.0"
+    function-bind "^1.1.1"
+    has "^1.0.3"
+    has-symbols "^1.0.0"
+    is-callable "^1.1.4"
+    is-regex "^1.0.4"
+    object-inspect "^1.6.0"
+    object-keys "^1.1.1"
+    string.prototype.trimleft "^2.1.0"
+    string.prototype.trimright "^2.1.0"
+
 es-abstract@^1.4.3:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.11.0.tgz#cce87d518f0496893b1a30cd8461835535480681"
@@ -9020,6 +9045,11 @@ object-hash@>=1.3.1, object-hash@^1.3.1:
   resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df"
   integrity sha512-OSuu/pU4ENM9kmREg0BdNrUDIl1heYa4mBZacJc+vVWz4GtAwu7jO8s4AIt2aGRUTqxykpWzI3Oqnsm13tTMDA==
 
+object-inspect@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.6.0.tgz#c70b6cbf72f274aab4c34c0c82f5167bf82cf15b"
+  integrity sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==
+
 object-keys@^1.0.11, object-keys@^1.0.8:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
@@ -9028,6 +9058,11 @@ object-keys@^1.0.12:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.0.tgz#11bd22348dd2e096a045ab06f6c85bcc340fa032"
 
+object-keys@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e"
+  integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==
+
 object-keys@~0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336"
@@ -12193,6 +12228,22 @@ string.prototype.padend@^3.0.0:
     es-abstract "^1.4.3"
     function-bind "^1.0.2"
 
+string.prototype.trimleft@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/string.prototype.trimleft/-/string.prototype.trimleft-2.1.0.tgz#6cc47f0d7eb8d62b0f3701611715a3954591d634"
+  integrity sha512-FJ6b7EgdKxxbDxc79cOlok6Afd++TTs5szo+zJTUyow3ycrRfJVE2pq3vcN53XexvKZu/DJMDfeI/qMiZTrjTw==
+  dependencies:
+    define-properties "^1.1.3"
+    function-bind "^1.1.1"
+
+string.prototype.trimright@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/string.prototype.trimright/-/string.prototype.trimright-2.1.0.tgz#669d164be9df9b6f7559fa8e89945b168a5a6c58"
+  integrity sha512-fXZTSV55dNBwv16uw+hh5jkghxSnc5oHq+5K/gXgizHwAvMetdAJlHqqoFC1FSDVPYWLkAKl2cxpUT41sV7nSg==
+  dependencies:
+    define-properties "^1.1.3"
+    function-bind "^1.1.1"
+
 string@^3.0.1:
   version "3.3.3"
   resolved "https://registry.yarnpkg.com/string/-/string-3.3.3.tgz#5ea211cd92d228e184294990a6cc97b366a77cb0"