Imprv/gw7606 elastic search 7 (#5041)

* https://youtrack.weseek.co.jp/issue/GW-7605
- change key for `host` to `node` for @elastic/elasticsearch Client initiation
- add ssl configuration on Client init
- change `httpAuth` to `auth` and it's parameter format
- make client.indices.exists() to destructuring variable assignment
- make stats() to destructuring variable assignment
- make getAlias() to destructuring variable assignment
- make .search() to destructuring variable assignment

* - Investigated the search feature can only working while using old index
- Investigated indexing not working
- Compare @elastic/elasticsearch `index status` and `index meta` with elasticsearch.js
- Check and debug the client API result on elasticsearch.ts
- Research writing index data flow
- Bring back requestTimeout on client initiation
- Remove requestTimeout on bulkWrite index

* - Adjust getConnectionInfo() method
- Remove string split and set the format in getConnectionInfo()
- Investigate rejectUnauthorized parameter on Client
- Clean code / remove TODO

* https://youtrack.weseek.co.jp/issue/GW-7632
- Adjust app/.env.developement, service/config-loader.ts and elasticsearch.ts
- Set rejectUnauthorized from global env config

* https://youtrack.weseek.co.jp/issue/GW-7606
- add both @elastic/elasticsearch v6 and v7
- add env variable of USE_ELASTICSEARCH_V6
- adjust elasticsearch.ts conditions to use elsticsearch client version based on env var
- add mappings-v7.json with adjusted format based on breaking changes docs
- change _type paramter when indexing
- remove type mappings parameter in searching query

* - implement isElasticsearchV6 conditions
- add this.elasticsearch condition based on useElasticsearchV6 config
- add conditions to read mappings.json based on useElasticsearchV6 config
- add conditions for all _type query parameter based on useElasticsearchV6 config
- modify phrase query object structure based on useElasticsearchV6 config
- modify not phrase query object structure based on useElasticsearchV6 config
- modify totalValue on SearchResult.jsx component based on elasticsearch meta return data version

packages/app/.env.development

@@ -14,6 +14,7 @@ MONGO_URI="mongodb://mongo:27017/growi"
 ELASTICSEARCH_URI="http://elasticsearch:9200/growi"
 ELASTICSEARCH_REQUEST_TIMEOUT=15000
 ELASTICSEARCH_REJECT_UNAUTHORIZED=false
+USE_ELASTICSEARCH_V6=false
 HACKMD_URI="http://localhost:3010"
 HACKMD_URI_FOR_SERVER="http://hackmd:3000"
 # DRAWIO_URI="http://localhost:8080/?offline=1&https=0"

packages/app/package.json

@@ -86,7 +86,8 @@
     "date-fns": "^2.23.0",
     "detect-indent": "^7.0.0",
     "diff": "^5.0.0",
-    "@elastic/elasticsearch": "^6.8.7",
+    "@elastic/elasticsearch6": "npm:@elastic/elasticsearch@^6.8.7",
+    "@elastic/elasticsearch7": "npm:@elastic/elasticsearch@^7.16.0",
     "entities": "^2.0.0",
     "esa-nodejs": "^0.0.7",
     "escape-string-regexp": "=4.0.0",

packages/app/resource/search/mappings-es7.json

@@ -0,0 +1,115 @@
+{
+  "settings": {
+    "analysis": {
+      "filter": {
+        "english_stop": {
+          "type":       "stop",
+          "stopwords":  "_english_"
+        }
+      },
+      "tokenizer": {
+        "edge_ngram_tokenizer": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20,
+          "token_chars": ["letter", "digit"]
+        }
+      },
+      "analyzer": {
+        "japanese": {
+          "tokenizer": "kuromoji_tokenizer",
+          "char_filter" : ["icu_normalizer"]
+        },
+        "english_edge_ngram": {
+          "tokenizer": "edge_ngram_tokenizer",
+          "filter": [
+            "lowercase",
+            "english_stop"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "path": {
+        "type": "text",
+        "fields": {
+          "raw": {
+            "type": "text",
+            "analyzer": "keyword"
+          },
+          "ja": {
+            "type": "text",
+            "analyzer": "japanese"
+          },
+          "en": {
+            "type": "text",
+            "analyzer": "english_edge_ngram",
+            "search_analyzer": "standard"
+          }
+        }
+      },
+      "body": {
+        "type": "text",
+        "fields": {
+          "ja": {
+            "type": "text",
+            "analyzer": "japanese"
+          },
+          "en": {
+            "type": "text",
+            "analyzer": "english_edge_ngram",
+            "search_analyzer": "standard"
+          }
+        }
+      },
+      "comments": {
+        "type": "text",
+        "fields": {
+          "ja": {
+            "type": "text",
+            "analyzer": "japanese"
+          },
+          "en": {
+            "type": "text",
+            "analyzer": "english_edge_ngram",
+            "search_analyzer": "standard"
+          }
+        }
+      },
+      "username": {
+        "type": "keyword"
+      },
+      "comment_count": {
+        "type": "integer"
+      },
+      "bookmark_count": {
+        "type": "integer"
+      },
+      "like_count": {
+        "type": "integer"
+      },
+      "grant": {
+        "type": "integer"
+      },
+      "granted_users": {
+        "type": "keyword"
+      },
+      "granted_group": {
+        "type": "keyword"
+      },
+      "created_at": {
+        "type": "date",
+        "format": "dateOptionalTime"
+      },
+      "updated_at": {
+        "type": "date",
+        "format": "dateOptionalTime"
+      },
+      "tag_names": {
+        "type": "keyword"
+      }
+    }
+  }
+}

packages/app/src/components/SearchPage/SearchResult.jsx

@@ -285,6 +285,15 @@ class SearchResult extends React.Component {
 
     const listView = this.renderListView(this.props.pages);
 
+    let totalValue;
+
+    if (typeof this.props.searchResultMeta.total === 'string') {
+      totalValue = this.props.searchResultMeta.total; // elasticsearch V6
+    }
+    else {
+      totalValue = this.props.searchResultMeta.total.value; // elasticsearch v7
+    }
+
     /*
     UI あとで考える
     <span className="search-result-meta">Found: {this.props.searchResultMeta.total} pages with "{this.props.searchingKeyword}"</span>
@@ -296,7 +305,7 @@ class SearchResult extends React.Component {
             <nav>
               <div className="d-flex align-items-start justify-content-between mt-1">
                 <div className="search-result-meta">
-                  <i className="icon-magnifier" /> Found {this.props.searchResultMeta.total} pages with &quot;{this.props.searchingKeyword}&quot;
+                  <i className="icon-magnifier" /> Found {totalValue} pages with &quot;{this.props.searchingKeyword}&quot;
                 </div>
                 <div className="text-nowrap">
                   {deletionModeButtons}

packages/app/src/server/service/config-loader.ts

@@ -268,6 +268,12 @@ const ENV_VAR_NAME_TO_CONFIG_INFO = {
     type:    ValueType.BOOLEAN,
     default: false,
   },
+  USE_ELASTICSEARCH_V6: {
+    ns:      'crowi',
+    key:     'app:useElasticsearchV6',
+    type:    ValueType.BOOLEAN,
+    default: false,
+  },
   MONGO_GRIDFS_TOTAL_LIMIT: {
     ns:      'crowi',
     key:     'gridfs:totalLimit',

packages/app/src/server/service/search-delegator/elasticsearch.ts

@@ -1,4 +1,5 @@
-import elasticsearch from '@elastic/elasticsearch';
+import elasticsearch6 from '@elastic/elasticsearch6';
+import elasticsearch7 from '@elastic/elasticsearch7';
 import mongoose from 'mongoose';
 
 import { URL } from 'url';
@@ -43,6 +44,10 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
 
   socketIoService!: any
 
+  isElasticsearchV6: boolean
+
+  elasticsearch: any
+
   client: any
 
   queries: any
@@ -56,6 +61,9 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     this.configManager = configManager;
     this.socketIoService = socketIoService;
 
+    this.isElasticsearchV6 = this.configManager.getConfig('crowi', 'app:useElasticsearchV6');
+
+    this.elasticsearch = this.isElasticsearchV6 ? elasticsearch6 : elasticsearch7;
     this.client = null;
 
     // In Elasticsearch RegExp, we don't need to used ^ and $.
@@ -92,7 +100,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
   initClient() {
     const { host, auth, indexName } = this.getConnectionInfo();
 
-    this.client = new elasticsearch.Client({
+    this.client = new this.elasticsearch.Client({
       node: host,
       ssl: { rejectUnauthorized: this.configManager.getConfig('crowi', 'app:elasticsearchRejectUnauthorized') },
       auth,
@@ -303,7 +311,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
   }
 
   async createIndex(index) {
-    const body = require('^/resource/search/mappings.json');
+    const body = this.isElasticsearchV6 ? require('^/resource/search/mappings.json') : require('^/resource/search/mappings-es7.json');
     return this.client.indices.create({ index, body });
   }
 
@@ -340,7 +348,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     const command = {
       index: {
         _index: this.indexName,
-        _type: 'pages',
+        _type: this.isElasticsearchV6 ? 'pages' : '_doc',
         _id: page._id.toString(),
       },
     };
@@ -376,7 +384,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     const command = {
       delete: {
         _index: this.indexName,
-        _type: 'pages',
+        _type: this.isElasticsearchV6 ? 'pages' : '_doc',
         _id: page._id.toString(),
       },
     };
@@ -639,15 +647,19 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     }
 
     // sort by score
-    const query = {
+    // eslint-disable-next-line prefer-const
+    let query = {
       index: this.aliasName,
-      type: 'pages',
       body: {
         query: {}, // query
         _source: fields,
       },
     };
 
+    if (this.isElasticsearchV6) {
+      Object.assign(query, { type: 'pages' });
+    }
+
     return query;
   }
 
@@ -718,7 +730,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     if (parsedKeywords.phrase.length > 0) {
       const phraseQueries: any[] = [];
       parsedKeywords.phrase.forEach((phrase) => {
-        phraseQueries.push({
+        const phraseQuery = {
           multi_match: {
             query: phrase, // each phrase is quoteted words like "This is GROWI"
             type: 'phrase',
@@ -729,16 +741,24 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
               'comments',
             ],
           },
-        });
+        };
+        if (this.isElasticsearchV6) {
+          phraseQueries.push(phraseQuery);
+        }
+        else {
+          query.body.query.bool.must.push(phraseQuery);
+        }
       });
 
-      query.body.query.bool.must.push(phraseQueries);
+      if (this.isElasticsearchV6) {
+        query.body.query.bool.must.push(phraseQueries);
+      }
     }
 
     if (parsedKeywords.not_phrase.length > 0) {
       const notPhraseQueries: any[] = [];
       parsedKeywords.not_phrase.forEach((phrase) => {
-        notPhraseQueries.push({
+        const notPhraseQuery = {
           multi_match: {
             query: phrase, // each phrase is quoteted words
             type: 'phrase',
@@ -748,10 +768,19 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
               'body',
             ],
           },
-        });
+        };
+
+        if (this.isElasticsearchV6) {
+          notPhraseQueries.push(notPhraseQuery);
+        }
+        else {
+          query.body.query.bool.must_not.push(notPhraseQuery);
+        }
       });
 
-      query.body.query.bool.must_not.push(notPhraseQueries);
+      if (this.isElasticsearchV6) {
+        query.body.query.bool.must_not.push(notPhraseQueries);
+      }
     }
 
     if (parsedKeywords.prefix.length > 0) {

yarn.lock

@@ -769,7 +769,7 @@
   resolved "https://registry.yarnpkg.com/@browser-bunyan/levels/-/levels-1.6.0.tgz#3a50b8118254aa2ac26caf9d2aafa72d157e374b"
   integrity sha512-wte6nXXZH62Y/RGysYRlOkKxuJn+4S8xEamMF0fDncxxy0SriCHYwGPyWGF0FWYWmRzbZuEkp7dNebBf9Xfeeg==
 
-"@elastic/elasticsearch@^6.8.7":
+"@elastic/elasticsearch6@npm:@elastic/elasticsearch@^6.8.7":
   version "6.8.8"
   resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-6.8.8.tgz#363d332d4de3a3ee5420ac0ced2eb4bfadf04548"
   integrity sha512-51Jp3ZZ0oPqYPNlPG58XJ773MqJBx91rGNWCgVvy2UtxjxHsExAJooesOyLcoADnW0Dhyxu6yB8tziHnmyl8Vw==
@@ -782,6 +782,16 @@
     pump "^3.0.0"
     secure-json-parse "^2.1.0"
 
+"@elastic/elasticsearch7@npm:@elastic/elasticsearch@^7.16.0":
+  version "7.16.0"
+  resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.16.0.tgz#c1c64b6f0343c0f5ca6893fb77ceecd763455024"
+  integrity sha512-lMY2MFZZFG3om7QNHninxZZOXYx3NdIUwEISZxqaI9dXPoL3DNhU31keqjvx1gN6T74lGXAzrRNP4ag8CJ/VXw==
+  dependencies:
+    debug "^4.3.1"
+    hpagent "^0.1.1"
+    ms "^2.1.3"
+    secure-json-parse "^2.4.0"
+
 "@emotion/is-prop-valid@^0.8.3":
   version "0.8.8"
   resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-0.8.8.tgz#db28b1c4368a259b60a97311d6a952d4fd01ac1a"
@@ -9827,6 +9837,11 @@ hosted-git-info@^4.0.0, hosted-git-info@^4.0.1:
   dependencies:
     lru-cache "^6.0.0"
 
+hpagent@^0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9"
+  integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==
+
 hsl-regex@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/hsl-regex/-/hsl-regex-1.0.0.tgz#d49330c789ed819e276a4c0d272dffa30b18fe6e"
@@ -13627,6 +13642,11 @@ ms@2.1.2:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
+ms@^2.1.3:
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
+  integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
+
 multer-autoreap@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/multer-autoreap/-/multer-autoreap-1.0.3.tgz#a50aaeb713fa9407ac940807f6c112c6ce9df280"
@@ -17611,7 +17631,7 @@ schema-utils@^3.0.0:
     ajv "^6.12.5"
     ajv-keywords "^3.5.2"
 
-secure-json-parse@^2.1.0:
+secure-json-parse@^2.1.0, secure-json-parse@^2.4.0:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85"
   integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==