Merge branch 'feat/implement-elasticsearch-6-and-7' of https://github.com/weseek/growi into imprv/7639-elasticsearch-reindex-on-boot

* 'feat/implement-elasticsearch-6-and-7' of https://github.com/weseek/growi:
  - add TODO for "remove cat method or keep it exist since not Implemented on elasticsearch service"
  - Implement typescript type from researched API response - Add type IndicesStatsResponse - Add type ReindexResponse - Add type IndicesGetAliasResponse - Add and adjust indices.stats on elasticsearch-client.ts - Add and adjust reindex on elasticsearch-client.ts - Add and adjust indices.getAlias on elasticsearch-client.ts ** for IndicesStatsResponse._all.* and IndicesStatsResponse.indices I use any ** for client.indices.getAlias I didn't add promise because the result is relative
  - Use eslint-disable camelcase for file on elasticsearch-client-types.ts - Add type IndicesExistsResponse - Add type ValidateQueryResponse - Add type ClusterHealthResponse - Add and adjust cluster.health on elasticsearch-client.ts - Add and adjust indices.exists on elasticsearch-client.ts - Add and adjust indices.validateQuery on elasticsearch-client.ts
  - add and adjust elasricsearch-client.ts based on latest crowi function - add elasticsearch-client-types.ts and adjust the type - adjust initClient() methods to use ElasticsearchClient object - add getType() instead of add conditions each query
  - Rename mappings.json to mappings-es6.json and leave mappings-es7.json as it is - Make conditions in search delegator for meta data of total value - Adjust SearchResult component for totalValue render - Check and debug the result on ES6 and ES7 for conditions https://youtrack.weseek.co.jp/issue/GW-7608

packages/app/src/components/SearchPage/SearchResult.jsx

@@ -285,15 +285,6 @@ class SearchResult extends React.Component {
 
     const listView = this.renderListView(this.props.pages);
 
-    let totalValue;
-
-    if (typeof this.props.searchResultMeta.total === 'string') {
-      totalValue = this.props.searchResultMeta.total; // elasticsearch V6
-    }
-    else {
-      totalValue = this.props.searchResultMeta.total.value; // elasticsearch v7
-    }
-
     /*
     UI あとで考える
     <span className="search-result-meta">Found: {this.props.searchResultMeta.total} pages with "{this.props.searchingKeyword}"</span>
@@ -305,7 +296,7 @@ class SearchResult extends React.Component {
             <nav>
               <div className="d-flex align-items-start justify-content-between mt-1">
                 <div className="search-result-meta">
-                  <i className="icon-magnifier" /> Found {totalValue} pages with &quot;{this.props.searchingKeyword}&quot;
+                  <i className="icon-magnifier" /> Found {this.props.searchResultMeta.total} pages with &quot;{this.props.searchingKeyword}&quot;
                 </div>
                 <div className="text-nowrap">
                   {deletionModeButtons}

packages/app/src/server/service/search-delegator/elasticsearch-client-types.ts

@@ -0,0 +1,113 @@
+/* eslint-disable camelcase */
+export type NodesInfoResponse = {
+  nodes: Record<
+    string,
+    {
+      version: string
+      plugins: Plugin[]
+    }
+  >
+}
+
+export type CatIndicesResponse = {
+  index: string
+}[]
+
+export type IndicesExistsResponse = boolean
+
+export type IndicesExistsAliasResponse = boolean
+
+export type CatAliasesResponse = {
+  alias: string
+  index: string
+  filter: string
+}[]
+
+export type BulkResponse = {
+  took: number
+  errors: boolean
+  items: Record<string, any>[]
+}
+
+export type SearchResponse = {
+  took: number
+  timed_out: boolean
+  _shards: {
+    total: number
+    successful: number
+    skipped: number
+    failed: number
+  }
+  hits: {
+    total: number | {
+      value: number
+      relation: string
+    } // 6.x.x | 7.x.x
+    max_score: number | null
+    hits: Record<string, {
+      _index: string
+      _type: string
+      _id: string
+      _score: number
+      _source: any
+    }>[]
+  }
+}
+
+export type ValidateQueryResponse = {
+  valid: boolean,
+  _shards: {
+    total: number,
+    successful: number,
+    failed: number
+  },
+  explanations: Record<string, any>[]
+}
+
+export type ClusterHealthResponse = {
+  cluster_name: string,
+  status: string,
+  timed_out: boolean,
+  number_of_nodes: number,
+  number_of_data_nodes: number,
+  active_primary_shards: number,
+  active_shards: number,
+  relocating_shards: number,
+  initializing_shards: number,
+  unassigned_shards: number,
+  delayed_unassigned_shards: number,
+  number_of_pending_tasks: number,
+  number_of_in_flight_fetch: number,
+  task_max_waiting_in_queue_millis: number,
+  active_shards_percent_as_number: number
+}
+
+export type IndicesStatsResponse = {
+  _shards: {
+    total: number,
+    successful: number,
+    failed: number
+  },
+  _all: {
+    primaries: any,
+    total: any
+  },
+  indices: any
+}
+
+export type ReindexResponse = {
+  took: number,
+  timed_out: boolean,
+  total: number,
+  updated: number,
+  created: number,
+  deleted: number,
+  batches: number,
+  noops: number,
+  version_conflicts: number,
+  retries: number,
+  throttled_millis: number,
+  requests_per_second: number,
+  throttled_until_millis: number,
+  failures: any | null
+}

packages/app/src/server/service/search-delegator/elasticsearch-client.ts

@@ -0,0 +1,83 @@
+/* eslint-disable implicit-arrow-linebreak */
+/* eslint-disable no-confusing-arrow */
+import { Client as ES6Client, ApiResponse as ES6ApiResponse, RequestParams as ES6RequestParams } from '@elastic/elasticsearch6';
+import { Client as ES7Client, ApiResponse as ES7ApiResponse, RequestParams as ES7RequestParams } from '@elastic/elasticsearch7';
+import {
+  BulkResponse,
+  CatAliasesResponse,
+  CatIndicesResponse,
+  IndicesExistsResponse,
+  IndicesExistsAliasResponse,
+  NodesInfoResponse,
+  SearchResponse,
+  ValidateQueryResponse,
+  ClusterHealthResponse,
+  IndicesStatsResponse,
+  ReindexResponse,
+} from './elasticsearch-client-types';
+
+type ApiResponse<T = any, C = any> = ES6ApiResponse<T, C> | ES7ApiResponse<T, C>
+
+export default class ElasticsearchClient {
+
+  client: ES6Client | ES7Client
+
+  constructor(client: ES6Client | ES7Client) {
+    this.client = client;
+  }
+
+  bulk(params: ES6RequestParams.Bulk & ES7RequestParams.Bulk): Promise<ApiResponse<BulkResponse>> {
+    return this.client instanceof ES6Client ? this.client.bulk(params) : this.client.bulk(params);
+  }
+
+  // TODO: cat is not used in current Implementation, remove cat?
+  cat = {
+    aliases: (params: ES6RequestParams.CatAliases & ES7RequestParams.CatAliases): Promise<ApiResponse<CatAliasesResponse>> =>
+      this.client instanceof ES6Client ? this.client.cat.aliases(params) : this.client.cat.aliases(params),
+    indices: (params: ES6RequestParams.CatIndices & ES7RequestParams.CatIndices): Promise<ApiResponse<CatIndicesResponse>> =>
+      this.client instanceof ES6Client ? this.client.cat.indices(params) : this.client.cat.indices(params),
+  }
+
+  cluster = {
+    health: (params: ES6RequestParams.ClusterHealth & ES7RequestParams.ClusterHealth): Promise<ApiResponse<ClusterHealthResponse>> =>
+      this.client instanceof ES6Client ? this.client.cluster.health(params) : this.client.cluster.health(params),
+  }
+
+  indices = {
+    create: (params: ES6RequestParams.IndicesCreate & ES7RequestParams.IndicesCreate) =>
+      this.client instanceof ES6Client ? this.client.indices.create(params) : this.client.indices.create(params),
+    delete: (params: ES6RequestParams.IndicesDelete & ES7RequestParams.IndicesDelete) =>
+      this.client instanceof ES6Client ? this.client.indices.delete(params) : this.client.indices.delete(params),
+    exists: (params: ES6RequestParams.IndicesExists & ES7RequestParams.IndicesExists): Promise<ApiResponse<IndicesExistsResponse>> =>
+      this.client instanceof ES6Client ? this.client.indices.exists(params) : this.client.indices.exists(params),
+    existsAlias: (params: ES6RequestParams.IndicesExistsAlias & ES7RequestParams.IndicesExistsAlias): Promise<ApiResponse<IndicesExistsAliasResponse>> =>
+      this.client instanceof ES6Client ? this.client.indices.existsAlias(params) : this.client.indices.existsAlias(params),
+    putAlias: (params: ES6RequestParams.IndicesPutAlias & ES7RequestParams.IndicesPutAlias) =>
+      this.client instanceof ES6Client ? this.client.indices.putAlias(params) : this.client.indices.putAlias(params),
+    getAlias: (params: ES6RequestParams.IndicesGetAlias & ES7RequestParams.IndicesGetAlias) =>
+      this.client instanceof ES6Client ? this.client.indices.getAlias(params) : this.client.indices.getAlias(params),
+    updateAliases: (params: ES6RequestParams.IndicesUpdateAliases & ES7RequestParams.IndicesUpdateAliases) =>
+      this.client instanceof ES6Client ? this.client.indices.updateAliases(params) : this.client.indices.updateAliases(params),
+    validateQuery: (params: ES6RequestParams.IndicesValidateQuery & ES7RequestParams.IndicesValidateQuery): Promise<ApiResponse<ValidateQueryResponse>> =>
+      this.client instanceof ES6Client ? this.client.indices.validateQuery(params) : this.client.indices.validateQuery(params),
+    stats: (params: ES6RequestParams.IndicesStats & ES7RequestParams.IndicesStats): Promise<ApiResponse<IndicesStatsResponse>> =>
+      this.client instanceof ES6Client ? this.client.indices.stats(params) : this.client.indices.stats(params),
+  }
+
+  nodes = {
+    info: (): Promise<ApiResponse<NodesInfoResponse>> => (this.client instanceof ES6Client ? this.client.nodes.info() : this.client.nodes.info()),
+  }
+
+  ping() {
+    return this.client instanceof ES6Client ? this.client.ping() : this.client.ping();
+  }
+
+  reindex(params: ES6RequestParams.Reindex & ES7RequestParams.Reindex): Promise<ApiResponse<ReindexResponse>> {
+    return this.client instanceof ES6Client ? this.client.reindex(params) : this.client.reindex(params);
+  }
+
+  search(params: ES6RequestParams.Search & ES7RequestParams.Search): Promise<ApiResponse<SearchResponse>> {
+    return this.client instanceof ES6Client ? this.client.search(params) : this.client.search(params);
+  }
+
+}

packages/app/src/server/service/search-delegator/elasticsearch.ts

@@ -14,6 +14,7 @@ import { SearchDelegatorName } from '~/interfaces/named-query';
 import {
   MetaData, SearchDelegator, Result, SearchableData, QueryTerms,
 } from '../../interfaces/search';
+import ElasticsearchClient from './elasticsearch-client';
 
 const logger = loggerFactory('growi:service:search-delegator:elasticsearch');
 
@@ -100,16 +101,19 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
   initClient() {
     const { host, auth, indexName } = this.getConnectionInfo();
 
-    this.client = new this.elasticsearch.Client({
+    this.client = new ElasticsearchClient(new this.elasticsearch.Client({
       node: host,
       ssl: { rejectUnauthorized: this.configManager.getConfig('crowi', 'app:elasticsearchRejectUnauthorized') },
       auth,
       requestTimeout: this.configManager.getConfig('crowi', 'app:elasticsearchRequestTimeout'),
-      // log: 'debug',
-    });
+    }));
     this.indexName = indexName;
   }
 
+  getType() {
+    return this.isElasticsearchV6 ? 'pages' : '_doc';
+  }
+
   /**
    * return information object to connect to ES
    * @return {object} { host, auth, indexName}
@@ -307,7 +311,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
   }
 
   async createIndex(index) {
-    const body = this.isElasticsearchV6 ? require('^/resource/search/mappings.json') : require('^/resource/search/mappings-es7.json');
+    const body = this.isElasticsearchV6 ? require('^/resource/search/mappings-es6.json') : require('^/resource/search/mappings-es7.json');
     return this.client.indices.create({ index, body });
   }
 
@@ -344,7 +348,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     const command = {
       index: {
         _index: this.indexName,
-        _type: this.isElasticsearchV6 ? 'pages' : '_doc',
+        _type: this.getType(),
         _id: page._id.toString(),
       },
     };
@@ -380,7 +384,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     const command = {
       delete: {
         _index: this.indexName,
-        _type: this.isElasticsearchV6 ? 'pages' : '_doc',
+        _type: this.getType(),
         _id: page._id.toString(),
       },
     };
@@ -607,16 +611,17 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
       logger.debug('ES returns explanations: ', result.explanations);
     }
 
-    const { body } = await this.client.search(query);
     const { body: result } = await this.client.search(query);
 
     // for debug
     logger.debug('ES result: ', result);
 
+    const totalValue = this.isElasticsearchV6 ? result.hits.total : result.hits.total.value;
+
     return {
       meta: {
         took: result.took,
-        total: result.hits.total,
+        total: totalValue,
         results: result.hits.hits.length,
       },
       data: result.hits.hits.map((elm) => {