feat: including comments in full text search (#4703)

* added comment object to search mappings

* listen commentEvent when comment created

* filter comment

* success to log comment in a chunk object

* clean code

* rename key

* added documents comments propaty

* emit comment event when a comment is updated and deleted

* fix grammar

* clean code

* add comments to fields

* add comments to searchkeywords.phrase

* fix "rebuild index  error"

Co-authored-by: Yuki Takei <yuki@weseek.co.jp>

packages/app/resource/search/mappings.json

@@ -65,6 +65,20 @@
             }
           }
         },
+        "comments": {
+          "type": "text",
+          "fields": {
+            "ja": {
+              "type": "text",
+              "analyzer": "japanese"
+            },
+            "en": {
+              "type": "text",
+              "analyzer": "english_edge_ngram",
+              "search_analyzer": "standard"
+            }
+          }
+        },
         "username": {
           "type": "keyword"
         },

packages/app/src/components/Admin/ElasticsearchManagement/ElasticsearchManagement.jsx

@@ -53,7 +53,8 @@ class ElasticsearchManagement extends React.Component {
       });
     });
 
-    socket.on('finishAddPage', (data) => {
+    socket.on('finishAddPage', async(data) => {
+      await this.retrieveIndicesStatus();
       this.setState({
         isRebuildingProcessing: false,
         isRebuildingCompleted: true,

packages/app/src/server/crowi/index.js

@@ -81,6 +81,7 @@ function Crowi() {
     user: new (require('../events/user'))(this),
     page: new (require('../events/page'))(this),
     bookmark: new (require('../events/bookmark'))(this),
+    comment: new (require('../events/comment'))(this),
     tag: new (require('../events/tag'))(this),
     admin: new (require('../events/admin'))(this),
   };

packages/app/src/server/events/comment.ts

@@ -0,0 +1,17 @@
+
+import util from 'util';
+
+const events = require('events');
+
+function CommentEvent(crowi) {
+  this.crowi = crowi;
+
+  events.EventEmitter.call(this);
+}
+util.inherits(CommentEvent, events.EventEmitter);
+
+CommentEvent.prototype.onCreate = function(comment) {};
+CommentEvent.prototype.onUpdate = function(comment) {};
+CommentEvent.prototype.onDelete = function(comment) {};
+
+module.exports = CommentEvent;

packages/app/src/server/models/comment.js

@@ -51,6 +51,24 @@ module.exports = function(crowi) {
     return this.find({ revision: id }).sort({ createdAt: -1 });
   };
 
+
+  /**
+   * @return {object} key: page._id, value: comments
+   */
+  commentSchema.statics.getPageIdToCommentMap = async function(pageIds) {
+    const results = await this.aggregate()
+      .match({ page: { $in: pageIds } })
+      .group({ _id: '$page', comments: { $push: '$comment' } });
+
+    // convert to map
+    const idToCommentMap = {};
+    results.forEach((result, i) => {
+      idToCommentMap[result._id] = result.comments;
+    });
+
+    return idToCommentMap;
+  };
+
   commentSchema.statics.countCommentByPageId = function(page) {
     const self = this;
 

packages/app/src/server/routes/comment.js

@@ -231,6 +231,7 @@ module.exports = function(crowi, app) {
     const position = commentForm.comment_position || -1;
     const isMarkdown = commentForm.is_markdown;
     const replyTo = commentForm.replyTo;
+    const commentEvent = crowi.event('comment');
 
     // check whether accessible
     const isAccessible = await Page.isAccessiblePageByViewer(pageId, req.user);
@@ -241,6 +242,7 @@ module.exports = function(crowi, app) {
     let createdComment;
     try {
       createdComment = await Comment.create(pageId, req.user._id, revisionId, comment, position, isMarkdown, replyTo);
+      commentEvent.emit('create', createdComment);
     }
     catch (err) {
       logger.error(err);
@@ -345,6 +347,8 @@ module.exports = function(crowi, app) {
     const commentId = commentForm.comment_id;
     const revision = commentForm.revision_id;
 
+    const commentEvent = crowi.event('comment');
+
     if (commentStr === '') {
       return res.json(ApiResponse.error('Comment text is required'));
     }
@@ -375,6 +379,7 @@ module.exports = function(crowi, app) {
         { _id: commentId },
         { $set: { comment: commentStr, isMarkdown, revision } },
       );
+      commentEvent.emit('create', updatedComment);
     }
     catch (err) {
       logger.error(err);
@@ -428,6 +433,8 @@ module.exports = function(crowi, app) {
    * @apiParam {String} comment_id Comment Id.
    */
   api.remove = async function(req, res) {
+    const commentEvent = crowi.event('comment');
+
     const commentId = req.body.comment_id;
     if (!commentId) {
       return Promise.resolve(res.json(ApiResponse.error('\'comment_id\' is undefined')));
@@ -452,6 +459,7 @@ module.exports = function(crowi, app) {
 
       await comment.removeWithReplies();
       await Page.updateCommentCount(comment.page);
+      commentEvent.emit('delete', comment);
     }
     catch (err) {
       return res.json(ApiResponse.error(err));

packages/app/src/server/service/search-delegator/elasticsearch.js

@@ -314,6 +314,7 @@ class ElasticsearchDelegator {
       body: page.revision.body,
       // username: page.creator?.username, // available Node.js v14 and above
       username: page.creator != null ? page.creator.username : null,
+      comments: page.comments,
       comment_count: page.commentCount,
       bookmark_count: bookmarkCount,
       like_count: page.liker.length || 0,
@@ -371,6 +372,7 @@ class ElasticsearchDelegator {
     const Page = mongoose.model('Page');
     const { PageQueryBuilder } = Page;
     const Bookmark = mongoose.model('Bookmark');
+    const Comment = mongoose.model('Comment');
     const PageTagRelation = mongoose.model('PageTagRelation');
 
     const socket = this.socketIoService.getAdminSocket();
@@ -431,6 +433,28 @@ class ElasticsearchDelegator {
       },
     });
 
+
+    const appendCommentStream = new Transform({
+      objectMode: true,
+      async transform(chunk, encoding, callback) {
+        const pageIds = chunk.map(doc => doc._id);
+
+        const idToCommentMap = await Comment.getPageIdToCommentMap(pageIds);
+        const idsHavingComment = Object.keys(idToCommentMap);
+
+        // append comments
+        chunk
+          .filter(doc => idsHavingComment.includes(doc._id.toString()))
+          .forEach((doc) => {
+            // append comments from idToCommentMap
+            doc.comments = idToCommentMap[doc._id.toString()];
+          });
+
+        this.push(chunk);
+        callback();
+      },
+    });
+
     const appendTagNamesStream = new Transform({
       objectMode: true,
       async transform(chunk, encoding, callback) {
@@ -503,6 +527,7 @@ class ElasticsearchDelegator {
       .pipe(thinOutStream)
       .pipe(batchStream)
       .pipe(appendBookmarkCountStream)
+      .pipe(appendCommentStream)
       .pipe(appendTagNamesStream)
       .pipe(writeStream);
 
@@ -579,7 +604,7 @@ class ElasticsearchDelegator {
   }
 
   createSearchQuerySortedByScore(option) {
-    let fields = ['path', 'bookmark_count', 'comment_count', 'updated_at', 'tag_names'];
+    let fields = ['path', 'bookmark_count', 'comment_count', 'updated_at', 'tag_names', 'comments'];
     if (option) {
       fields = option.fields || fields;
     }
@@ -635,7 +660,7 @@ class ElasticsearchDelegator {
         multi_match: {
           query: parsedKeywords.match.join(' '),
           type: 'most_fields',
-          fields: ['path.ja^2', 'path.en^2', 'body.ja', 'body.en'],
+          fields: ['path.ja^2', 'path.en^2', 'body.ja', 'body.en', 'comments.ja', 'comments.en'],
         },
       };
       query.body.query.bool.must.push(q);
@@ -645,7 +670,7 @@ class ElasticsearchDelegator {
       const q = {
         multi_match: {
           query: parsedKeywords.not_match.join(' '),
-          fields: ['path.ja', 'path.en', 'body.ja', 'body.en'],
+          fields: ['path.ja', 'path.en', 'body.ja', 'body.en', 'comments.ja', 'comments.en'],
           operator: 'or',
         },
       };
@@ -657,12 +682,13 @@ class ElasticsearchDelegator {
       parsedKeywords.phrase.forEach((phrase) => {
         phraseQueries.push({
           multi_match: {
-            query: phrase, // each phrase is quoteted words
+            query: phrase, // each phrase is quoteted words like "This is GROWI"
             type: 'phrase',
             fields: [
               // Not use "*.ja" fields here, because we want to analyze (parse) search words
               'path.raw^2',
               'body',
+              'comments',
             ],
           },
         });
@@ -1023,6 +1049,12 @@ class ElasticsearchDelegator {
     return this.updateOrInsertPageById(pageId);
   }
 
+  async syncCommentChanged(comment) {
+    logger.debug('SearchClient.syncCommentChanged', comment);
+
+    return this.updateOrInsertPageById(comment.page);
+  }
+
   async syncTagChanged(page) {
     logger.debug('SearchClient.syncTagChanged', page.path);
 

packages/app/src/server/service/search.js

@@ -73,6 +73,11 @@ class SearchService {
     bookmarkEvent.on('create', this.delegator.syncBookmarkChanged.bind(this.delegator));
     bookmarkEvent.on('delete', this.delegator.syncBookmarkChanged.bind(this.delegator));
 
+    const commentEvent = this.crowi.event('comment');
+    commentEvent.on('create', this.delegator.syncCommentChanged.bind(this.delegator));
+    commentEvent.on('update', this.delegator.syncCommentChanged.bind(this.delegator));
+    commentEvent.on('delete', this.delegator.syncCommentChanged.bind(this.delegator));
+
     const tagEvent = this.crowi.event('tag');
     tagEvent.on('update', this.delegator.syncTagChanged.bind(this.delegator));
   }