Merge pull request #4744 from weseek/feat/impl-query-parser

feat: Implement search query parser and resolver

packages/app/src/server/interfaces/search.ts

@@ -13,17 +13,14 @@ export type QueryTerms = {
   not_tag: string[],
 }
 
-export type ParsedQuery = {
-  queryString: string // original query string in request
-  nqNames: string[] // possible NamedQuery names found in query string
-}
+export type ParsedQuery = { queryString: string, terms?: QueryTerms, delegatorName?: string }
 
 export interface SearchQueryParser {
   parseSearchQuery(queryString: string): Promise<ParsedQuery>
 }
 
 export interface SearchResolver{
-  resolve(parsedQuery: ParsedQuery): Promise<[SearchDelegator, SearchableData]>
+  resolve(parsedQuery: ParsedQuery): Promise<[SearchDelegator, SearchableData | null]>
 }
 
 export interface SearchDelegator<T = unknown> {

packages/app/src/server/service/search-delegator/elasticsearch.ts

@@ -851,84 +851,6 @@ class ElasticsearchDelegator implements SearchDelegator<Data> {
     return this.searchKeyword(query);
   }
 
-  parseQueryString(queryString: string) {
-    const matchWords: string[] = [];
-    const notMatchWords: string[] = [];
-    const phraseWords: string[] = [];
-    const notPhraseWords: string[] = [];
-    const prefixPaths: string[] = [];
-    const notPrefixPaths: string[] = [];
-    const tags: string[] = [];
-    const notTags: string[] = [];
-
-    queryString.trim();
-    queryString = queryString.replace(/\s+/g, ' '); // eslint-disable-line no-param-reassign
-
-    // First: Parse phrase keywords
-    const phraseRegExp = new RegExp(/(-?"[^"]+")/g);
-    const phrases = queryString.match(phraseRegExp);
-
-    if (phrases !== null) {
-      queryString = queryString.replace(phraseRegExp, ''); // eslint-disable-line no-param-reassign
-
-      phrases.forEach((phrase) => {
-        phrase.trim();
-        if (phrase.match(/^-/)) {
-          notPhraseWords.push(phrase.replace(/^-/, ''));
-        }
-        else {
-          phraseWords.push(phrase);
-        }
-      });
-    }
-
-    // Second: Parse other keywords (include minus keywords)
-    queryString.split(' ').forEach((word) => {
-      if (word === '') {
-        return;
-      }
-
-      // https://regex101.com/r/pN9XfK/1
-      const matchNegative = word.match(/^-(prefix:|tag:)?(.+)$/);
-      // https://regex101.com/r/3qw9FQ/1
-      const matchPositive = word.match(/^(prefix:|tag:)?(.+)$/);
-
-      if (matchNegative != null) {
-        if (matchNegative[1] === 'prefix:') {
-          notPrefixPaths.push(matchNegative[2]);
-        }
-        else if (matchNegative[1] === 'tag:') {
-          notTags.push(matchNegative[2]);
-        }
-        else {
-          notMatchWords.push(matchNegative[2]);
-        }
-      }
-      else if (matchPositive != null) {
-        if (matchPositive[1] === 'prefix:') {
-          prefixPaths.push(matchPositive[2]);
-        }
-        else if (matchPositive[1] === 'tag:') {
-          tags.push(matchPositive[2]);
-        }
-        else {
-          matchWords.push(matchPositive[2]);
-        }
-      }
-    });
-
-    return {
-      match: matchWords,
-      not_match: notMatchWords,
-      phrase: phraseWords,
-      not_phrase: notPhraseWords,
-      prefix: prefixPaths,
-      not_prefix: notPrefixPaths,
-      tag: tags,
-      not_tag: notTags,
-    };
-  }
-
   async syncPageUpdated(page, user) {
     logger.debug('SearchClient.syncPageUpdated', page.path);
 

packages/app/src/server/service/search.ts

@@ -1,9 +1,10 @@
 import mongoose from 'mongoose';
 import RE2 from 're2';
 
-import { NamedQueryModel, NamedQueryDocument } from '../models/named-query';
+import { NamedQueryModel } from '../models/named-query';
+import { SearchDelegatorName } from '~/interfaces/named-query';
 import {
-  SearchDelegator, SearchQueryParser, SearchResolver, ParsedQuery, Result, MetaData, SearchableData,
+  SearchDelegator, SearchQueryParser, SearchResolver, ParsedQuery, Result, MetaData, SearchableData, QueryTerms,
 } from '../interfaces/search';
 
 import loggerFactory from '~/utils/logger';
@@ -11,6 +12,13 @@ import loggerFactory from '~/utils/logger';
 // eslint-disable-next-line no-unused-vars
 const logger = loggerFactory('growi:service:search');
 
+const normalizeQueryString = (_queryString: string): string => {
+  let queryString = _queryString.trim();
+  queryString = queryString.replace(/\s+/g, ' ');
+
+  return queryString;
+};
+
 class SearchService implements SearchQueryParser, SearchResolver {
 
   crowi!: any
@@ -21,7 +29,9 @@ class SearchService implements SearchQueryParser, SearchResolver {
 
   isErrorOccuredOnSearching: boolean | null
 
-  delegator: any & SearchDelegator
+  fullTextSearchDelegator: any & SearchDelegator
+
+  nqDelegators: {[key in SearchDelegatorName]: SearchDelegator} // TODO: initialize
 
   constructor(crowi) {
     this.crowi = crowi;
@@ -31,20 +41,20 @@ class SearchService implements SearchQueryParser, SearchResolver {
     this.isErrorOccuredOnSearching = null;
 
     try {
-      this.delegator = this.generateDelegator();
+      this.fullTextSearchDelegator = this.generateDelegator();
     }
     catch (err) {
       logger.error(err);
     }
 
     if (this.isConfigured) {
-      this.delegator.init();
+      this.fullTextSearchDelegator.init();
       this.registerUpdateEvent();
     }
   }
 
   get isConfigured() {
-    return this.delegator != null;
+    return this.fullTextSearchDelegator != null;
   }
 
   get isReachable() {
@@ -70,19 +80,19 @@ class SearchService implements SearchQueryParser, SearchResolver {
 
   registerUpdateEvent() {
     const pageEvent = this.crowi.event('page');
-    pageEvent.on('create', this.delegator.syncPageUpdated.bind(this.delegator));
-    pageEvent.on('update', this.delegator.syncPageUpdated.bind(this.delegator));
-    pageEvent.on('deleteCompletely', this.delegator.syncPagesDeletedCompletely.bind(this.delegator));
-    pageEvent.on('delete', this.delegator.syncPageDeleted.bind(this.delegator));
-    pageEvent.on('updateMany', this.delegator.syncPagesUpdated.bind(this.delegator));
-    pageEvent.on('syncDescendants', this.delegator.syncDescendantsPagesUpdated.bind(this.delegator));
+    pageEvent.on('create', this.fullTextSearchDelegator.syncPageUpdated.bind(this.fullTextSearchDelegator));
+    pageEvent.on('update', this.fullTextSearchDelegator.syncPageUpdated.bind(this.fullTextSearchDelegator));
+    pageEvent.on('deleteCompletely', this.fullTextSearchDelegator.syncPagesDeletedCompletely.bind(this.fullTextSearchDelegator));
+    pageEvent.on('delete', this.fullTextSearchDelegator.syncPageDeleted.bind(this.fullTextSearchDelegator));
+    pageEvent.on('updateMany', this.fullTextSearchDelegator.syncPagesUpdated.bind(this.fullTextSearchDelegator));
+    pageEvent.on('syncDescendants', this.fullTextSearchDelegator.syncDescendantsPagesUpdated.bind(this.fullTextSearchDelegator));
 
     const bookmarkEvent = this.crowi.event('bookmark');
-    bookmarkEvent.on('create', this.delegator.syncBookmarkChanged.bind(this.delegator));
-    bookmarkEvent.on('delete', this.delegator.syncBookmarkChanged.bind(this.delegator));
+    bookmarkEvent.on('create', this.fullTextSearchDelegator.syncBookmarkChanged.bind(this.fullTextSearchDelegator));
+    bookmarkEvent.on('delete', this.fullTextSearchDelegator.syncBookmarkChanged.bind(this.fullTextSearchDelegator));
 
     const tagEvent = this.crowi.event('tag');
-    tagEvent.on('update', this.delegator.syncTagChanged.bind(this.delegator));
+    tagEvent.on('update', this.fullTextSearchDelegator.syncTagChanged.bind(this.fullTextSearchDelegator));
   }
 
   resetErrorStatus() {
@@ -92,7 +102,7 @@ class SearchService implements SearchQueryParser, SearchResolver {
 
   async reconnectClient() {
     logger.info('Try to reconnect...');
-    this.delegator.initClient();
+    this.fullTextSearchDelegator.initClient();
 
     try {
       await this.getInfoForHealth();
@@ -107,7 +117,7 @@ class SearchService implements SearchQueryParser, SearchResolver {
 
   async getInfo() {
     try {
-      return await this.delegator.getInfo();
+      return await this.fullTextSearchDelegator.getInfo();
     }
     catch (err) {
       logger.error(err);
@@ -117,7 +127,7 @@ class SearchService implements SearchQueryParser, SearchResolver {
 
   async getInfoForHealth() {
     try {
-      const result = await this.delegator.getInfoForHealth();
+      const result = await this.fullTextSearchDelegator.getInfoForHealth();
 
       this.isErrorOccuredOnHealthcheck = false;
       return result;
@@ -132,32 +142,170 @@ class SearchService implements SearchQueryParser, SearchResolver {
   }
 
   async getInfoForAdmin() {
-    return this.delegator.getInfoForAdmin();
+    return this.fullTextSearchDelegator.getInfoForAdmin();
   }
 
   async normalizeIndices() {
-    return this.delegator.normalizeIndices();
+    return this.fullTextSearchDelegator.normalizeIndices();
   }
 
   async rebuildIndex() {
-    return this.delegator.rebuildIndex();
+    return this.fullTextSearchDelegator.rebuildIndex();
   }
 
   async parseSearchQuery(_queryString: string): Promise<ParsedQuery> {
-    // TODO: impl parser
-    return {} as ParsedQuery;
+
+    const regexp = new RE2(/^\[nq:.+\]$/g); // https://regex101.com/r/FzDUvT/1
+    const replaceRegexp = new RE2(/\[nq:|\]/g);
+
+    const queryString = normalizeQueryString(_queryString);
+
+    // when Normal Query
+    if (!regexp.test(queryString)) {
+      return { queryString, terms: this.parseQueryString(queryString) };
+    }
+
+    // when Named Query
+    const NamedQuery = mongoose.model('NamedQuery') as NamedQueryModel;
+
+    const name = queryString.replace(replaceRegexp, '');
+    const nq = await NamedQuery.findOne({ name });
+
+    // will delegate to full-text search
+    if (nq == null) {
+      return { queryString, terms: this.parseQueryString(queryString) };
+    }
+
+    const { aliasOf, delegatorName } = nq;
+
+    let parsedQuery;
+    if (aliasOf != null) {
+      parsedQuery = { queryString: normalizeQueryString(aliasOf), terms: this.parseQueryString(aliasOf) };
+    }
+    if (delegatorName != null) {
+      parsedQuery = { queryString, delegatorName };
+    }
+
+    return parsedQuery;
   }
 
-  async resolve(parsedQuery: ParsedQuery): Promise<[SearchDelegator, SearchableData]> {
-    // TODO: impl resolve
-    return [{}, {}] as [SearchDelegator, SearchableData];
+  async resolve(parsedQuery: ParsedQuery): Promise<[SearchDelegator, SearchableData | null]> {
+    const { queryString, terms, delegatorName } = parsedQuery;
+    if (delegatorName != null) {
+      const nqDelegator = this.nqDelegators[delegatorName];
+      if (nqDelegator != null) {
+        return [nqDelegator, null];
+      }
+    }
+
+    const data = {
+      queryString,
+      terms: terms as QueryTerms,
+    };
+    return [this.fullTextSearchDelegator, data];
   }
 
   async searchKeyword(keyword: string, user, userGroups, searchOpts): Promise<Result<any> & MetaData> {
-    // TODO: parse
-    // TODO: resolve
-    // TODO: search
-    return {} as Result<any> & MetaData;
+    let parsedQuery;
+    // parse
+    try {
+      parsedQuery = await this.parseSearchQuery(keyword);
+    }
+    catch (err) {
+      logger.error('Error occurred while parseSearchQuery', err);
+      throw err;
+    }
+
+    let delegator;
+    let data;
+    // resolve
+    try {
+      [delegator, data] = await this.resolve(parsedQuery);
+    }
+    catch (err) {
+      logger.error('Error occurred while resolving search delegator', err);
+      throw err;
+    }
+
+    return delegator.search(data, user, userGroups, searchOpts);
+  }
+
+  parseQueryString(queryString: string): QueryTerms {
+    // terms
+    const matchWords: string[] = [];
+    const notMatchWords: string[] = [];
+    const phraseWords: string[] = [];
+    const notPhraseWords: string[] = [];
+    const prefixPaths: string[] = [];
+    const notPrefixPaths: string[] = [];
+    const tags: string[] = [];
+    const notTags: string[] = [];
+
+    // First: Parse phrase keywords
+    const phraseRegExp = new RegExp(/(-?"[^"]+")/g);
+    const phrases = queryString.match(phraseRegExp);
+
+    if (phrases !== null) {
+      queryString = queryString.replace(phraseRegExp, ''); // eslint-disable-line no-param-reassign
+
+      phrases.forEach((phrase) => {
+        phrase.trim();
+        if (phrase.match(/^-/)) {
+          notPhraseWords.push(phrase.replace(/^-/, ''));
+        }
+        else {
+          phraseWords.push(phrase);
+        }
+      });
+    }
+
+    // Second: Parse other keywords (include minus keywords)
+    queryString.split(' ').forEach((word) => {
+      if (word === '') {
+        return;
+      }
+
+      // https://regex101.com/r/pN9XfK/1
+      const matchNegative = word.match(/^-(prefix:|tag:)?(.+)$/);
+      // https://regex101.com/r/3qw9FQ/1
+      const matchPositive = word.match(/^(prefix:|tag:)?(.+)$/);
+
+      if (matchNegative != null) {
+        if (matchNegative[1] === 'prefix:') {
+          notPrefixPaths.push(matchNegative[2]);
+        }
+        else if (matchNegative[1] === 'tag:') {
+          notTags.push(matchNegative[2]);
+        }
+        else {
+          notMatchWords.push(matchNegative[2]);
+        }
+      }
+      else if (matchPositive != null) {
+        if (matchPositive[1] === 'prefix:') {
+          prefixPaths.push(matchPositive[2]);
+        }
+        else if (matchPositive[1] === 'tag:') {
+          tags.push(matchPositive[2]);
+        }
+        else {
+          matchWords.push(matchPositive[2]);
+        }
+      }
+    });
+
+    const terms = {
+      match: matchWords,
+      not_match: notMatchWords,
+      phrase: phraseWords,
+      not_phrase: notPhraseWords,
+      prefix: prefixPaths,
+      not_prefix: notPrefixPaths,
+      tag: tags,
+      not_tag: notTags,
+    };
+
+    return terms;
   }
 
 }

packages/app/src/test/integration/service/search/search-service.test.js

@@ -0,0 +1,141 @@
+import SearchService from '~/server/service/search';
+import NamedQuery from '~/server/models/named-query';
+
+const { getInstance } = require('../../setup-crowi');
+
+describe('SearchService test', () => {
+  let crowi;
+  let searchService;
+
+  const DEFAULT = 'FullTextSearch';
+
+  // let NamedQuery;
+
+  let dummyAliasOf;
+  let dummyDelegatorName;
+
+  let namedQuery1;
+  let namedQuery2;
+
+  const dummyDelegator = {
+    search() {
+      return;
+    },
+  };
+
+  beforeAll(async() => {
+    crowi = await getInstance();
+    searchService = new SearchService(crowi);
+    searchService.nqDelegators = {
+      FullTextSearch: dummyDelegator,
+    };
+  });
+
+
+  describe('parseQueryString()', () => {
+    test('should parse queryString', async() => {
+      const queryString = 'match -notmatch "phrase" -"notphrase" [nq:named_query] prefix:/pre1 -prefix:/pre2 tag:Tag1 -tag:Tag2';
+      const terms = await searchService.parseQueryString(queryString);
+
+      const expected = { // QueryTerms
+        match: ['match', '[nq:named_query]'],
+        not_match: ['notmatch'],
+        phrase: ['"phrase"'],
+        not_phrase: ['"notphrase"'],
+        prefix: ['/pre1'],
+        not_prefix: ['/pre2'],
+        tag: ['Tag1'],
+        not_tag: ['Tag2'],
+      };
+
+      expect(terms).toStrictEqual(expected);
+    });
+  });
+
+  describe('parseSearchQuery()', () => {
+    beforeAll(async() => {
+      dummyDelegatorName = DEFAULT;
+      dummyAliasOf = 'match -notmatch "phrase" -"notphrase" prefix:/pre1 -prefix:/pre2 tag:Tag1 -tag:Tag2';
+
+      await NamedQuery.insertMany([
+        { name: 'named_query1', delegatorName: dummyDelegatorName },
+        { name: 'named_query2', aliasOf: dummyAliasOf },
+      ]);
+
+      namedQuery1 = await NamedQuery.findOne({ name: 'named_query1' });
+      namedQuery2 = await NamedQuery.findOne({ name: 'named_query2' });
+    });
+
+    test('should return result with delegatorName', async() => {
+      const queryString = '[nq:named_query1]';
+      const parsedQuery = await searchService.parseSearchQuery(queryString);
+
+      const expected = {
+        queryString,
+        delegatorName: dummyDelegatorName,
+      };
+
+      expect(parsedQuery).toStrictEqual(expected);
+    });
+
+    test('should return result with expanded aliasOf value', async() => {
+      const queryString = '[nq:named_query2]';
+      const parsedQuery = await searchService.parseSearchQuery(queryString);
+      const expected = {
+        queryString: dummyAliasOf,
+        terms: {
+          match: ['match'],
+          not_match: ['notmatch'],
+          phrase: ['"phrase"'],
+          not_phrase: ['"notphrase"'],
+          prefix: ['/pre1'],
+          not_prefix: ['/pre2'],
+          tag: ['Tag1'],
+          not_tag: ['Tag2'],
+        },
+      };
+
+      expect(parsedQuery).toStrictEqual(expected);
+    });
+
+    test('should resolve as full-text search delegator', async() => {
+      const parsedQuery = {
+        queryString: dummyAliasOf,
+        terms: {
+          match: ['match'],
+          not_match: ['notmatch'],
+          phrase: ['"phrase"'],
+          not_phrase: ['"notphrase"'],
+          prefix: ['/pre1'],
+          not_prefix: ['/pre2'],
+          tag: ['Tag1'],
+          not_tag: ['Tag2'],
+        },
+      };
+
+      const [delegator, data] = await searchService.resolve(parsedQuery);
+
+      const expectedData = parsedQuery;
+
+      expect(data).toStrictEqual(expectedData);
+      // expect(typeof delegator.search).toBe('function'); TODO: enable test after implementing delegator initialization
+    });
+
+    test('should resolve as custom search delegator', async() => {
+      const queryString = '[nq:named_query1]';
+      const parsedQuery = {
+        queryString,
+        delegatorName: dummyDelegatorName,
+      };
+
+      const [delegator, data] = await searchService.resolve(parsedQuery);
+
+      const expectedData = null;
+
+      expect(data).toBe(expectedData);
+      expect(typeof delegator.search).toBe('function');
+    });
+
+  });
+
+});