Merge pull request #3210 from weseek/imprv/auto-reconnect-search-service

Imprv/auto reconnect search service

src/server/middlewares/auto-reconnect-to-search.js

@@ -0,0 +1,32 @@
+const loggerFactory = require('@alias/logger');
+
+const { ReconnectContext, nextTick } = require('../service/search-reconnect-context/reconnect-context');
+
+const logger = loggerFactory('growi:middlewares:auto-reconnect-to-search');
+
+module.exports = (crowi) => {
+  const { searchService } = crowi;
+  const reconnectContext = new ReconnectContext();
+
+  const reconnectHandler = async() => {
+    try {
+      logger.info('Auto reconnection is started.');
+      await searchService.reconnectClient();
+    }
+    catch (err) {
+      logger.error('Auto reconnection failed.');
+    }
+
+    return searchService.isReachable;
+  };
+
+  return (req, res, next) => {
+    if (searchService != null && !searchService.isReachable) {
+      // NON-BLOCKING CALL
+      // for the latency of the response
+      nextTick(reconnectContext, reconnectHandler);
+    }
+
+    return next();
+  };
+};

src/server/routes/apiv3/healthcheck.js

@@ -66,6 +66,7 @@ module.exports = (crowi) => {
     if (searchService.isConfigured) {
       try {
         info.searchInfo = await searchService.getInfoForHealth();
+        searchService.resetErrorStatus();
       }
       catch (err) {
         errors.push(new ErrorV3(`The Search Service is not connectable - ${err.message}`, 'healthcheck-search-unhealthy', err.stack));

src/server/routes/apiv3/search.js

@@ -77,7 +77,7 @@ module.exports = (crowi) => {
     }
 
     try {
-      await searchService.initClient();
+      await searchService.reconnectClient();
       return res.status(200).send();
     }
     catch (err) {

src/server/routes/index.js

@@ -4,6 +4,7 @@ const autoReap = require('multer-autoreap');
 autoReap.options.reapOnError = true; // continue reaping the file even if an error occurs
 
 module.exports = function(crowi, app) {
+  const autoReconnectToSearch = require('../middlewares/auto-reconnect-to-search')(crowi);
   const applicationNotInstalled = require('../middlewares/application-not-installed')(crowi);
   const applicationInstalled = require('../middlewares/application-installed')(crowi);
   const accessTokenParser = require('../middlewares/access-token-parser')(crowi);
@@ -32,7 +33,7 @@ module.exports = function(crowi, app) {
 
   /* eslint-disable max-len, comma-spacing, no-multi-spaces */
 
-  app.get('/'                        , applicationInstalled, loginRequired , page.showTopPage);
+  app.get('/'                        , applicationInstalled, loginRequired , autoReconnectToSearch, page.showTopPage);
 
   // API v3
   app.use('/api-docs', require('./apiv3/docs')(crowi));
@@ -175,6 +176,6 @@ module.exports = function(crowi, app) {
   app.get('/share/:linkId', page.showSharedPage);
 
   app.get('/*/$'                   , loginRequired , page.showPageWithEndOfSlash, page.notFound);
-  app.get('/*'                     , loginRequired , page.showPage, page.notFound);
+  app.get('/*'                     , loginRequired , autoReconnectToSearch, page.showPage, page.notFound);
 
 };

src/server/service/search-reconnect-context/reconnect-context.js

@@ -0,0 +1,80 @@
+const loggerFactory = require('@alias/logger');
+
+const logger = loggerFactory('growi:service:search-reconnect-context:reconnect-context');
+
+
+const RECONNECT_INTERVAL_SEC = 120;
+
+class ReconnectContext {
+
+  constructor() {
+    this.lastEvalDate = null;
+
+    this.reset(true);
+  }
+
+  reset() {
+    this.counter = 0;
+    this.stage = 0;
+  }
+
+  incrementCount() {
+    this.counter++;
+  }
+
+  incrementStage() {
+    this.counter = 0; // reset counter
+    this.stage++;
+  }
+
+  get shouldReconnectByCount() {
+    // https://www.google.com/search?q=10log10(x)-1+graph
+    const thresholdOfThisStage = 10 * Math.log10(this.stage) - 1;
+    return this.counter > thresholdOfThisStage;
+  }
+
+  get shouldReconnectByTime() {
+    if (this.lastEvalDate == null) {
+      this.lastEvalDate = new Date();
+      return true;
+    }
+
+    const thres = this.lastEvalDate.setSeconds(this.lastEvalDate.getSeconds() + RECONNECT_INTERVAL_SEC);
+    return thres < new Date();
+  }
+
+  get shouldReconnect() {
+    if (this.shouldReconnectByTime) {
+      logger.info('Server should reconnect by time');
+      return true;
+    }
+    if (this.shouldReconnectByCount) {
+      logger.info('Server should reconnect by count');
+      return true;
+    }
+    return false;
+  }
+
+}
+
+async function nextTick(context, reconnectHandler) {
+  context.incrementCount();
+
+  if (context.shouldReconnect) {
+    const isSuccessToReconnect = await reconnectHandler();
+
+    // success to reconnect
+    if (isSuccessToReconnect) {
+      context.reset();
+    }
+    // fail to reconnect
+    else {
+      context.incrementStage();
+    }
+  }
+}
+
+module.exports = {
+  ReconnectContext,
+  nextTick,
+};

src/server/service/search.js

@@ -11,7 +11,7 @@ class SearchService {
     this.isErrorOccuredOnSearching = null;
 
     try {
-      this.delegator = this.initDelegator();
+      this.delegator = this.generateDelegator();
     }
     catch (err) {
       logger.error(err);
@@ -39,7 +39,7 @@ class SearchService {
     return this.configManager.getConfig('crowi', 'app:elasticsearchUri') != null;
   }
 
-  initDelegator() {
+  generateDelegator() {
     logger.info('Initializing search delegator');
 
     if (this.isSearchboxEnabled) {
@@ -52,7 +52,6 @@ class SearchService {
       const ElasticsearchDelegator = require('./search-delegator/elasticsearch.js');
       return new ElasticsearchDelegator(this.configManager, this.crowi.socketIoService);
     }
-
   }
 
   registerUpdateEvent() {
@@ -69,12 +68,24 @@ class SearchService {
     tagEvent.on('update', this.delegator.syncTagChanged.bind(this.delegator));
   }
 
-  async initClient() {
-    // reset error flag
+  resetErrorStatus() {
     this.isErrorOccuredOnHealthcheck = false;
     this.isErrorOccuredOnSearching = false;
+  }
+
+  async reconnectClient() {
+    logger.info('Try to reconnect...');
+    this.delegator.initClient();
+
+    try {
+      await this.getInfoForHealth();
 
-    return this.delegator.initClient();
+      logger.info('Reconnecting succeeded.');
+      this.resetErrorStatus();
+    }
+    catch (err) {
+      throw err;
+    }
   }
 
   async getInfo() {