Merge pull request #4730 from weseek/imprv/search-query-interfaces

imprv: Create interfaces for search query feature

packages/app/src/server/interfaces/search.ts

@@ -0,0 +1,45 @@
+/* eslint-disable camelcase */
+import { SearchDelegatorName } from '~/interfaces/named-query';
+
+
+export type QueryTerms = {
+  match: string[],
+  not_match: string[],
+  phrase: string[],
+  not_phrase: string[],
+  prefix: string[],
+  not_prefix: string[],
+  tag: string[],
+  not_tag: string[],
+}
+
+export type ParsedQuery = {
+  queryString: string // original query string in request
+  nqNames: string[] // possible NamedQuery names found in query string
+}
+
+export interface SearchQueryParser {
+  parseSearchQuery(queryString: string): Promise<ParsedQuery>
+}
+
+export interface SearchResolver{
+  resolve(parsedQuery: ParsedQuery): Promise<[SearchDelegator, SearchableData]>
+}
+
+export interface SearchDelegator<T = unknown> {
+  name?: SearchDelegatorName
+  search(data: SearchableData | null, user, userGroups, option): Promise<Result<T> & MetaData>
+}
+
+export type Result<T> = {
+  data: T
+}
+
+export type MetaData = {
+  meta: { [key:string]: any }
+}
+
+export type SearchableData = {
+  queryString: string
+  terms: QueryTerms
+}

packages/app/src/server/models/page.ts

@@ -44,6 +44,17 @@ export interface PageModel extends Model<PageDocument> {
   findTargetAndAncestorsByPathOrId(pathOrId: string): Promise<TargetAndAncestorsResult>
   findChildrenByParentPathOrIdAndViewer(parentPathOrId: string, user, userGroups?): Promise<PageDocument[]>
   findAncestorsChildrenByPathAndViewer(path: string, user, userGroups?): Promise<Record<string, PageDocument[]>>
+
+  PageQueryBuilder: typeof PageQueryBuilder
+
+  GRANT_PUBLIC
+  GRANT_RESTRICTED
+  GRANT_SPECIFIED
+  GRANT_OWNER
+  GRANT_USER_GROUP
+  PAGE_GRANT_ERROR
+  STATUS_PUBLISHED
+  STATUS_DELETED
 }
 
 const ObjectId = mongoose.Schema.Types.ObjectId;

packages/app/src/server/service/config-loader.ts

@@ -262,12 +262,6 @@ const ENV_VAR_NAME_TO_CONFIG_INFO = {
     type:    ValueType.NUMBER,
     default: 8000, // msec
   },
-  SEARCHBOX_SSL_URL: {
-    ns:      'crowi',
-    key:     'app:searchboxSslUrl',
-    type:    ValueType.STRING,
-    default: null,
-  },
   MONGO_GRIDFS_TOTAL_LIMIT: {
     ns:      'crowi',
     key:     'gridfs:totalLimit',

packages/app/src/server/service/search-delegator/elasticsearch.js → packages/app/src/server/service/search-delegator/elasticsearch.ts

@@ -1,23 +1,39 @@
-import loggerFactory from '~/utils/logger';
+import elasticsearch from 'elasticsearch';
+import mongoose from 'mongoose';
 
-const logger = loggerFactory('growi:service:search-delegator:elasticsearch');
-const elasticsearch = require('elasticsearch');
-const mongoose = require('mongoose');
+import { URL } from 'url';
 
-const { URL } = require('url');
+import { Writable, Transform } from 'stream';
+import streamToPromise from 'stream-to-promise';
 
-const {
-  Writable, Transform,
-} = require('stream');
-const streamToPromise = require('stream-to-promise');
+import { createBatchStream } from '../../util/batch-stream';
+import loggerFactory from '~/utils/logger';
+import { PageDocument, PageModel } from '../../models/page';
+import {
+  MetaData, SearchDelegator, Result, SearchableData, QueryTerms,
+} from '../../interfaces/search';
 
-const { createBatchStream } = require('../../util/batch-stream');
+const logger = loggerFactory('growi:service:search-delegator:elasticsearch');
 
 const DEFAULT_OFFSET = 0;
 const DEFAULT_LIMIT = 50;
 const BULK_REINDEX_SIZE = 100;
 
-class ElasticsearchDelegator {
+type Data = any;
+
+class ElasticsearchDelegator implements SearchDelegator<Data> {
+
+  configManager!: any
+
+  socketIoService!: any
+
+  client: any
+
+  queries: any
+
+  indexName: string
+
+  esUri: string
 
   constructor(configManager, socketIoService) {
     this.configManager = configManager;
@@ -115,7 +131,7 @@ class ElasticsearchDelegator {
     let esVersion = 'unknown';
     const esNodeInfos = {};
 
-    for (const [nodeName, nodeInfo] of Object.entries(info.nodes)) {
+    for (const [nodeName, nodeInfo] of Object.entries<any>(info.nodes)) {
       esVersion = nodeInfo.version;
 
       const filteredInfo = {
@@ -160,7 +176,7 @@ class ElasticsearchDelegator {
     const isExistsTmpIndex = await client.indices.exists({ index: tmpIndexName });
 
     // create indices name list
-    const existingIndices = [];
+    const existingIndices: string[] = [];
     if (isExistsMainIndex) { existingIndices.push(indexName) }
     if (isExistsTmpIndex) { existingIndices.push(tmpIndexName) }
 
@@ -355,7 +371,7 @@ class ElasticsearchDelegator {
   }
 
   updateOrInsertDescendantsPagesById(page, user) {
-    const Page = mongoose.model('Page');
+    const Page = mongoose.model('Page') as PageModel;
     const { PageQueryBuilder } = Page;
     const builder = new PageQueryBuilder(Page.find());
     builder.addConditionToListWithDescendants(page.path);
@@ -365,13 +381,13 @@ class ElasticsearchDelegator {
   /**
    * @param {function} queryFactory factory method to generate a Mongoose Query instance
    */
-  async updateOrInsertPages(queryFactory, option = {}) {
+  async updateOrInsertPages(queryFactory, option: any = {}) {
     const { isEmittingProgressEvent = false, invokeGarbageCollection = false } = option;
 
-    const Page = mongoose.model('Page');
+    const Page = mongoose.model('Page') as PageModel;
     const { PageQueryBuilder } = Page;
-    const Bookmark = mongoose.model('Bookmark');
-    const PageTagRelation = mongoose.model('PageTagRelation');
+    const Bookmark = mongoose.model('Bookmark') as any; // TODO: typescriptize model
+    const PageTagRelation = mongoose.model('PageTagRelation') as any; // TODO: typescriptize model
 
     const socket = this.socketIoService.getAdminSocket();
 
@@ -527,7 +543,7 @@ class ElasticsearchDelegator {
    *   data: [ pages ...],
    * }
    */
-  async search(query) {
+  async searchKeyword(query) {
     // for debug
     if (process.env.NODE_ENV === 'development') {
       const result = await this.client.indices.validateQuery({
@@ -578,7 +594,7 @@ class ElasticsearchDelegator {
     return query;
   }
 
-  createSearchQuerySortedByScore(option) {
+  createSearchQuerySortedByScore(option?) {
     let fields = ['path', 'bookmark_count', 'comment_count', 'updated_at', 'tag_names'];
     if (option) {
       fields = option.fields || fields;
@@ -599,7 +615,7 @@ class ElasticsearchDelegator {
     return query;
   }
 
-  appendResultSize(query, from, size) {
+  appendResultSize(query, from?, size?) {
     query.from = from || DEFAULT_OFFSET;
     query.size = size || DEFAULT_LIMIT;
   }
@@ -624,12 +640,9 @@ class ElasticsearchDelegator {
     return query;
   }
 
-  appendCriteriaForQueryString(query, queryString) {
+  appendCriteriaForQueryString(query, parsedKeywords: QueryTerms) {
     query = this.initializeBoolQuery(query); // eslint-disable-line no-param-reassign
 
-    // parse
-    const parsedKeywords = this.parseQueryString(queryString);
-
     if (parsedKeywords.match.length > 0) {
       const q = {
         multi_match: {
@@ -653,7 +666,7 @@ class ElasticsearchDelegator {
     }
 
     if (parsedKeywords.phrase.length > 0) {
-      const phraseQueries = [];
+      const phraseQueries: any[] = [];
       parsedKeywords.phrase.forEach((phrase) => {
         phraseQueries.push({
           multi_match: {
@@ -672,7 +685,7 @@ class ElasticsearchDelegator {
     }
 
     if (parsedKeywords.not_phrase.length > 0) {
-      const notPhraseQueries = [];
+      const notPhraseQueries: any[] = [];
       parsedKeywords.not_phrase.forEach((phrase) => {
         notPhraseQueries.push({
           multi_match: {
@@ -725,12 +738,12 @@ class ElasticsearchDelegator {
 
     query = this.initializeBoolQuery(query); // eslint-disable-line no-param-reassign
 
-    const Page = mongoose.model('Page');
+    const Page = mongoose.model('Page') as PageModel;
     const {
       GRANT_PUBLIC, GRANT_RESTRICTED, GRANT_SPECIFIED, GRANT_OWNER, GRANT_USER_GROUP,
     } = Page;
 
-    const grantConditions = [
+    const grantConditions: any[] = [
       { term: { grant: GRANT_PUBLIC } },
     ];
 
@@ -797,44 +810,9 @@ class ElasticsearchDelegator {
     query.body.query.bool.filter.push({ bool: { should: grantConditions } });
   }
 
-  filterPortalPages(query) {
-    query = this.initializeBoolQuery(query); // eslint-disable-line no-param-reassign
-
-    query.body.query.bool.must_not.push(this.queries.USER);
-    query.body.query.bool.filter.push(this.queries.PORTAL);
-  }
-
-  filterPublicPages(query) {
-    query = this.initializeBoolQuery(query); // eslint-disable-line no-param-reassign
-
-    query.body.query.bool.must_not.push(this.queries.USER);
-    query.body.query.bool.filter.push(this.queries.PUBLIC);
-  }
-
-  filterUserPages(query) {
-    query = this.initializeBoolQuery(query); // eslint-disable-line no-param-reassign
-
-    query.body.query.bool.filter.push(this.queries.USER);
-  }
-
-  filterPagesByType(query, type) {
-    const Page = mongoose.model('Page');
-
-    switch (type) {
-      case Page.TYPE_PORTAL:
-        return this.filterPortalPages(query);
-      case Page.TYPE_PUBLIC:
-        return this.filterPublicPages(query);
-      case Page.TYPE_USER:
-        return this.filterUserPages(query);
-      default:
-        return query;
-    }
-  }
-
-  appendFunctionScore(query, queryString) {
+  async appendFunctionScore(query, queryString) {
     const User = mongoose.model('User');
-    const count = User.count({}) || 1;
+    const count = await User.count({}) || 1;
 
     const minScore = queryString.length * 0.1 - 1; // increase with length
     logger.debug('min_score: ', minScore);
@@ -856,32 +834,32 @@ class ElasticsearchDelegator {
     };
   }
 
-  async searchKeyword(queryString, user, userGroups, option) {
+  async search(data: SearchableData, user, userGroups, option): Promise<Result<Data> & MetaData> {
+    const { queryString, terms } = data;
+
     const from = option.offset || null;
     const size = option.limit || null;
-    const type = option.type || null;
     const query = this.createSearchQuerySortedByScore();
-    this.appendCriteriaForQueryString(query, queryString);
+    this.appendCriteriaForQueryString(query, terms);
 
-    this.filterPagesByType(query, type);
     await this.filterPagesByViewer(query, user, userGroups);
 
     this.appendResultSize(query, from, size);
 
-    this.appendFunctionScore(query, queryString);
+    await this.appendFunctionScore(query, queryString);
 
-    return this.search(query);
+    return this.searchKeyword(query);
   }
 
-  parseQueryString(queryString) {
-    const matchWords = [];
-    const notMatchWords = [];
-    const phraseWords = [];
-    const notPhraseWords = [];
-    const prefixPaths = [];
-    const notPrefixPaths = [];
-    const tags = [];
-    const notTags = [];
+  parseQueryString(queryString: string) {
+    const matchWords: string[] = [];
+    const notMatchWords: string[] = [];
+    const phraseWords: string[] = [];
+    const notPhraseWords: string[] = [];
+    const prefixPaths: string[] = [];
+    const notPrefixPaths: string[] = [];
+    const tags: string[] = [];
+    const notTags: string[] = [];
 
     queryString.trim();
     queryString = queryString.replace(/\s+/g, ' '); // eslint-disable-line no-param-reassign
@@ -970,11 +948,11 @@ class ElasticsearchDelegator {
 
   // remove pages whitch should nod Indexed
   async syncPagesUpdated(pages, user) {
-    const shoudDeletePages = [];
+    const shoudDeletePages: any[] = [];
     pages.forEach((page) => {
       logger.debug('SearchClient.syncPageUpdated', page.path);
       if (!this.shouldIndexed(page)) {
-        shoudDeletePages.append(page);
+        shoudDeletePages.push(page);
       }
     });
 
@@ -1031,4 +1009,4 @@ class ElasticsearchDelegator {
 
 }
 
-module.exports = ElasticsearchDelegator;
+export default ElasticsearchDelegator;

packages/app/src/server/service/search-delegator/searchbox.js

@@ -1,49 +0,0 @@
-import loggerFactory from '~/utils/logger';
-
-// eslint-disable-next-line no-unused-vars
-const logger = loggerFactory('growi:service:search-delegator:searchbox');
-
-const ElasticsearchDelegator = require('./elasticsearch');
-
-class SearchboxDelegator extends ElasticsearchDelegator {
-
-  /**
-   * @inheritdoc
-   */
-  getConnectionInfo() {
-    const searchboxSslUrl = this.configManager.getConfig('crowi', 'app:searchboxSslUrl');
-    const url = new URL(searchboxSslUrl);
-
-    const indexName = 'crowi';
-    const host = `${url.protocol}//${url.username}:${url.password}@${url.host}:443`;
-
-    return {
-      host,
-      httpAuth: '',
-      indexName,
-    };
-  }
-
-  /**
-   * @inheritdoc
-   */
-  async rebuildIndex() {
-    const { client, indexName, aliasName } = this;
-
-    // flush index
-    await client.indices.delete({
-      index: indexName,
-    });
-    await this.createIndex(indexName);
-    await this.addAllPages();
-
-    // put alias
-    await client.indices.putAlias({
-      name: aliasName,
-      index: indexName,
-    });
-  }
-
-}
-
-module.exports = SearchboxDelegator;

packages/app/src/server/service/search.js → packages/app/src/server/service/search.ts

@@ -1,9 +1,27 @@
+import mongoose from 'mongoose';
+import RE2 from 're2';
+
+import { NamedQueryModel, NamedQueryDocument } from '../models/named-query';
+import {
+  SearchDelegator, SearchQueryParser, SearchResolver, ParsedQuery, Result, MetaData, SearchableData,
+} from '../interfaces/search';
+
 import loggerFactory from '~/utils/logger';
 
 // eslint-disable-next-line no-unused-vars
 const logger = loggerFactory('growi:service:search');
 
-class SearchService {
+class SearchService implements SearchQueryParser, SearchResolver {
+
+  crowi!: any
+
+  configManager!: any
+
+  isErrorOccuredOnHealthcheck: boolean | null
+
+  isErrorOccuredOnSearching: boolean | null
+
+  delegator: any & SearchDelegator
 
   constructor(crowi) {
     this.crowi = crowi;
@@ -33,11 +51,6 @@ class SearchService {
     return this.isConfigured && !this.isErrorOccuredOnHealthcheck && !this.isErrorOccuredOnSearching;
   }
 
-  get isSearchboxEnabled() {
-    const uri = this.configManager.getConfig('crowi', 'app:searchboxSslUrl');
-    return uri != null && uri.length > 0;
-  }
-
   get isElasticsearchEnabled() {
     const uri = this.configManager.getConfig('crowi', 'app:elasticsearchUri');
     return uri != null && uri.length > 0;
@@ -46,14 +59,9 @@ class SearchService {
   generateDelegator() {
     logger.info('Initializing search delegator');
 
-    if (this.isSearchboxEnabled) {
-      const SearchboxDelegator = require('./search-delegator/searchbox');
-      logger.info('Searchbox is enabled');
-      return new SearchboxDelegator(this.configManager, this.crowi.socketIoService);
-    }
     if (this.isElasticsearchEnabled) {
       const ElasticsearchDelegator = require('./search-delegator/elasticsearch');
-      logger.info('Elasticsearch (not Searchbox) is enabled');
+      logger.info('Elasticsearch is enabled');
       return new ElasticsearchDelegator(this.configManager, this.crowi.socketIoService);
     }
 
@@ -135,19 +143,23 @@ class SearchService {
     return this.delegator.rebuildIndex();
   }
 
-  async searchKeyword(keyword, user, userGroups, searchOpts) {
-    try {
-      return await this.delegator.searchKeyword(keyword, user, userGroups, searchOpts);
-    }
-    catch (err) {
-      logger.error(err);
+  async parseSearchQuery(_queryString: string): Promise<ParsedQuery> {
+    // TODO: impl parser
+    return {} as ParsedQuery;
+  }
 
-      // switch error flag, `isReachable` to be `false`
-      this.isErrorOccuredOnSearching = true;
-      throw err;
-    }
+  async resolve(parsedQuery: ParsedQuery): Promise<[SearchDelegator, SearchableData]> {
+    // TODO: impl resolve
+    return [{}, {}] as [SearchDelegator, SearchableData];
+  }
+
+  async searchKeyword(keyword: string, user, userGroups, searchOpts): Promise<Result<any> & MetaData> {
+    // TODO: parse
+    // TODO: resolve
+    // TODO: search
+    return {} as Result<any> & MetaData;
   }
 
 }
 
-module.exports = SearchService;
+export default SearchService;

packages/app/src/test/integration/service/search-delegator/searchbox.test.js

@@ -1,36 +0,0 @@
-const SearchboxDelegator = require('~/server/service/search-delegator/searchbox');
-
-describe('SearchboxDelegator test', () => {
-
-  let delegator;
-
-  describe('getConnectionInfo()', () => {
-
-    let configManagerMock;
-    let searchEventMock;
-
-    beforeEach(() => {
-      configManagerMock = {};
-      searchEventMock = {};
-
-      // setup mock
-      configManagerMock.getConfig = jest.fn()
-        .mockReturnValue('https://paas:7e530aafad58c892a8778827ae80c879@thorin-us-east-1.searchly.com');
-
-      delegator = new SearchboxDelegator(configManagerMock, searchEventMock);
-    });
-
-    test('returns expected object', async() => {
-
-      const { host, httpAuth, indexName } = delegator.getConnectionInfo();
-
-      expect(configManagerMock.getConfig).toHaveBeenCalledWith('crowi', 'app:searchboxSslUrl');
-      expect(host).toBe('https://paas:7e530aafad58c892a8778827ae80c879@thorin-us-east-1.searchly.com:443');
-      expect(httpAuth).toBe('');
-      expect(indexName).toBe('crowi');
-    });
-
-  });
-
-
-});