|
|
@@ -11,6 +11,7 @@ import type { ISearchResult, ISearchResultData } from '~/interfaces/search';
|
|
|
import { SORT_AXIS, SORT_ORDER } from '~/interfaces/search';
|
|
|
import { SocketEventName } from '~/interfaces/websocket';
|
|
|
import PageTagRelation from '~/server/models/page-tag-relation';
|
|
|
+import type { SocketIoService } from '~/server/service/socket-io';
|
|
|
import loggerFactory from '~/utils/logger';
|
|
|
|
|
|
import type {
|
|
|
@@ -20,12 +21,20 @@ import type { PageModel } from '../../models/page';
|
|
|
import { createBatchStream } from '../../util/batch-stream';
|
|
|
import { configManager } from '../config-manager';
|
|
|
import type { UpdateOrInsertPagesOpts } from '../interfaces/search';
|
|
|
-// // import { embed, openaiClient, fileUpload } from '../openai';
|
|
|
-// import { getOrCreateSearchAssistant } from '../openai/assistant';
|
|
|
|
|
|
import { aggregatePipelineToIndex } from './aggregate-to-index';
|
|
|
import type { AggregatedPage, BulkWriteBody, BulkWriteCommand } from './bulk-write';
|
|
|
-import ElasticsearchClient from './elasticsearch-client';
|
|
|
+import {
|
|
|
+ getClient,
|
|
|
+ isES7ClientDelegator,
|
|
|
+ isES8ClientDelegator,
|
|
|
+ isES9ClientDelegator,
|
|
|
+ type SearchQuery,
|
|
|
+ type ES7SearchQuery,
|
|
|
+ type ES8SearchQuery,
|
|
|
+ type ES9SearchQuery,
|
|
|
+ type ElasticsearchClientDelegator,
|
|
|
+} from './elasticsearch-client-delegator';
|
|
|
|
|
|
const logger = loggerFactory('growi:service:search-delegator:elasticsearch');
|
|
|
|
|
|
@@ -43,7 +52,7 @@ const ES_SORT_AXIS = {
|
|
|
const ES_SORT_ORDER = {
|
|
|
[DESC]: 'desc',
|
|
|
[ASC]: 'asc',
|
|
|
-};
|
|
|
+} as const;
|
|
|
|
|
|
const AVAILABLE_KEYS = ['match', 'not_match', 'phrase', 'not_phrase', 'prefix', 'not_prefix', 'tag', 'not_tag'];
|
|
|
|
|
|
@@ -53,64 +62,41 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
|
|
|
name!: SearchDelegatorName.DEFAULT;
|
|
|
|
|
|
- socketIoService!: any;
|
|
|
-
|
|
|
- isElasticsearchV7: boolean;
|
|
|
-
|
|
|
- isElasticsearchReindexOnBoot: boolean;
|
|
|
+ private socketIoService!: SocketIoService;
|
|
|
|
|
|
- elasticsearch: any;
|
|
|
+ // TODO: https://redmine.weseek.co.jp/issues/168446
|
|
|
+ private isElasticsearchV7: boolean;
|
|
|
|
|
|
- client: ElasticsearchClient;
|
|
|
+ private isElasticsearchReindexOnBoot: boolean;
|
|
|
|
|
|
- queries: any;
|
|
|
+ private elasticsearchVersion: 7 | 8 | 9;
|
|
|
|
|
|
- indexName: string;
|
|
|
+ private client: ElasticsearchClientDelegator;
|
|
|
|
|
|
- esUri: string | undefined;
|
|
|
+ private indexName: string;
|
|
|
|
|
|
- constructor(socketIoService) {
|
|
|
+ constructor(socketIoService: SocketIoService) {
|
|
|
this.name = SearchDelegatorName.DEFAULT;
|
|
|
this.socketIoService = socketIoService;
|
|
|
|
|
|
- const elasticsearchVersion: number = configManager.getConfig('app:elasticsearchVersion');
|
|
|
+ const elasticsearchVersion = configManager.getConfig('app:elasticsearchVersion');
|
|
|
|
|
|
- if (elasticsearchVersion !== 7 && elasticsearchVersion !== 8) {
|
|
|
+ if (elasticsearchVersion !== 7 && elasticsearchVersion !== 8 && elasticsearchVersion !== 9) {
|
|
|
throw new Error('Unsupported Elasticsearch version. Please specify a valid number to \'ELASTICSEARCH_VERSION\'');
|
|
|
}
|
|
|
|
|
|
this.isElasticsearchV7 = elasticsearchVersion === 7;
|
|
|
|
|
|
- this.isElasticsearchReindexOnBoot = configManager.getConfig('app:elasticsearchReindexOnBoot');
|
|
|
+ this.elasticsearchVersion = elasticsearchVersion;
|
|
|
|
|
|
- // In Elasticsearch RegExp, we don't need to used ^ and $.
|
|
|
- // Ref: https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-regexp-query.html#_standard_operators
|
|
|
- this.queries = {
|
|
|
- PORTAL: {
|
|
|
- regexp: {
|
|
|
- 'path.raw': '.*/',
|
|
|
- },
|
|
|
- },
|
|
|
- PUBLIC: {
|
|
|
- regexp: {
|
|
|
- 'path.raw': '.*[^/]',
|
|
|
- },
|
|
|
- },
|
|
|
- USER: {
|
|
|
- prefix: {
|
|
|
- 'path.raw': '/user/',
|
|
|
- },
|
|
|
- },
|
|
|
- };
|
|
|
-
|
|
|
- this.initClient();
|
|
|
+ this.isElasticsearchReindexOnBoot = configManager.getConfig('app:elasticsearchReindexOnBoot');
|
|
|
}
|
|
|
|
|
|
- get aliasName() {
|
|
|
+ get aliasName(): string {
|
|
|
return `${this.indexName}-alias`;
|
|
|
}
|
|
|
|
|
|
- initClient() {
|
|
|
+ async initClient(): Promise<void> {
|
|
|
const { host, auth, indexName } = this.getConnectionInfo();
|
|
|
|
|
|
const rejectUnauthorized = configManager.getConfig('app:elasticsearchRejectUnauthorized');
|
|
|
@@ -121,7 +107,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
requestTimeout: configManager.getConfig('app:elasticsearchRequestTimeout'),
|
|
|
};
|
|
|
|
|
|
- this.client = new ElasticsearchClient(this.isElasticsearchV7, options, rejectUnauthorized);
|
|
|
+ this.client = await getClient({ version: this.elasticsearchVersion, options, rejectUnauthorized });
|
|
|
this.indexName = indexName;
|
|
|
}
|
|
|
|
|
|
@@ -135,7 +121,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
*/
|
|
|
getConnectionInfo() {
|
|
|
let indexName = 'crowi';
|
|
|
- let host = this.esUri;
|
|
|
+ let host: string | undefined;
|
|
|
let auth;
|
|
|
|
|
|
const elasticsearchUri = configManager.getConfig('app:elasticsearchUri');
|
|
|
@@ -161,6 +147,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
}
|
|
|
|
|
|
async init(): Promise<void> {
|
|
|
+ await this.initClient();
|
|
|
const normalizeIndices = await this.normalizeIndices();
|
|
|
if (this.isElasticsearchReindexOnBoot) {
|
|
|
try {
|
|
|
@@ -190,7 +177,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
let esVersion = 'unknown';
|
|
|
const esNodeInfos = {};
|
|
|
|
|
|
- for (const [nodeName, nodeInfo] of Object.entries<any>(info)) {
|
|
|
+ for (const [nodeName, nodeInfo] of Object.entries(info)) {
|
|
|
esVersion = nodeInfo.version;
|
|
|
|
|
|
const filteredInfo = {
|
|
|
@@ -269,7 +256,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
/**
|
|
|
* rebuild index
|
|
|
*/
|
|
|
- async rebuildIndex() {
|
|
|
+ async rebuildIndex(): Promise<void> {
|
|
|
const { client, indexName, aliasName } = this;
|
|
|
|
|
|
const tmpIndexName = `${indexName}-tmp`;
|
|
|
@@ -281,12 +268,10 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
|
|
|
// update alias
|
|
|
await client.indices.updateAliases({
|
|
|
- body: {
|
|
|
- actions: [
|
|
|
- { add: { alias: aliasName, index: tmpIndexName } },
|
|
|
- { remove: { alias: aliasName, index: indexName } },
|
|
|
- ],
|
|
|
- },
|
|
|
+ actions: [
|
|
|
+ { add: { alias: aliasName, index: tmpIndexName } },
|
|
|
+ { remove: { alias: aliasName, index: indexName } },
|
|
|
+ ],
|
|
|
});
|
|
|
|
|
|
// flush index
|
|
|
@@ -312,7 +297,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
|
|
|
}
|
|
|
|
|
|
- async normalizeIndices() {
|
|
|
+ async normalizeIndices(): Promise<void> {
|
|
|
const { client, indexName, aliasName } = this;
|
|
|
|
|
|
const tmpIndexName = `${indexName}-tmp`;
|
|
|
@@ -339,19 +324,36 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- async createIndex(index) {
|
|
|
- let mappings = this.isElasticsearchV7
|
|
|
- ? require('^/resource/search/mappings-es7.json')
|
|
|
- : require('^/resource/search/mappings-es8.json');
|
|
|
+ async createIndex(index: string) {
|
|
|
+ // TODO: https://redmine.weseek.co.jp/issues/168446
|
|
|
+ if (isES7ClientDelegator(this.client)) {
|
|
|
+ const { mappings } = await import('^/resource/search/mappings-es7');
|
|
|
+ return this.client.indices.create({
|
|
|
+ index,
|
|
|
+ body: {
|
|
|
+ ...mappings,
|
|
|
+ },
|
|
|
+ });
|
|
|
+ }
|
|
|
|
|
|
- if (process.env.CI) {
|
|
|
- mappings = require('^/resource/search/mappings-es8-for-ci.json');
|
|
|
+ if (isES8ClientDelegator(this.client)) {
|
|
|
+ const { mappings } = await import('^/resource/search/mappings-es8');
|
|
|
+ return this.client.indices.create({
|
|
|
+ index,
|
|
|
+ ...mappings,
|
|
|
+ });
|
|
|
}
|
|
|
|
|
|
- return this.client.indices.create({
|
|
|
- index,
|
|
|
- body: mappings,
|
|
|
- });
|
|
|
+ if (isES9ClientDelegator(this.client)) {
|
|
|
+ const { mappings } = process.env.CI == null
|
|
|
+ ? await import('^/resource/search/mappings-es9')
|
|
|
+ : await import('^/resource/search/mappings-es9-for-ci');
|
|
|
+
|
|
|
+ return this.client.indices.create({
|
|
|
+ index,
|
|
|
+ ...mappings,
|
|
|
+ });
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -397,7 +399,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
return [command, document];
|
|
|
}
|
|
|
|
|
|
- prepareBodyForDelete(body, page) {
|
|
|
+ prepareBodyForDelete(body, page): void {
|
|
|
if (!Array.isArray(body)) {
|
|
|
throw new Error('Body must be an array.');
|
|
|
}
|
|
|
@@ -434,7 +436,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
/**
|
|
|
* @param {function} queryFactory factory method to generate a Mongoose Query instance
|
|
|
*/
|
|
|
- async updateOrInsertPages(queryFactory, option: UpdateOrInsertPagesOpts = {}) {
|
|
|
+ async updateOrInsertPages(queryFactory, option: UpdateOrInsertPagesOpts = {}): Promise<void> {
|
|
|
const { shouldEmitProgress = false, invokeGarbageCollection = false } = option;
|
|
|
|
|
|
const Page = mongoose.model<IPage, PageModel>('Page');
|
|
|
@@ -481,28 +483,6 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
},
|
|
|
});
|
|
|
|
|
|
- // const appendEmbeddingStream = new Transform({
|
|
|
- // objectMode: true,
|
|
|
- // async transform(chunk: AggregatedPage[], encoding, callback) {
|
|
|
- // // append embedding
|
|
|
- // for await (const doc of chunk) {
|
|
|
- // doc.revisionBodyEmbedded = (await embed(doc.revision.body, doc.creator?.username))[0].embedding;
|
|
|
- // }
|
|
|
-
|
|
|
- // this.push(chunk);
|
|
|
- // callback();
|
|
|
- // },
|
|
|
- // });
|
|
|
-
|
|
|
- // const appendFileUploadedStream = new Transform({
|
|
|
- // objectMode: true,
|
|
|
- // async transform(chunk, encoding, callback) {
|
|
|
- // await fileUpload(chunk);
|
|
|
- // this.push(chunk);
|
|
|
- // callback();
|
|
|
- // },
|
|
|
- // });
|
|
|
-
|
|
|
let count = 0;
|
|
|
const writeStream = new Writable({
|
|
|
objectMode: true,
|
|
|
@@ -558,8 +538,6 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
readStream,
|
|
|
batchStream,
|
|
|
appendTagNamesStream,
|
|
|
- // appendEmbeddingStream,
|
|
|
- // appendFileUploadedStream,
|
|
|
writeStream,
|
|
|
);
|
|
|
}
|
|
|
@@ -581,26 +559,74 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
* data: [ pages ...],
|
|
|
* }
|
|
|
*/
|
|
|
- async searchKeyword(query): Promise<ISearchResult<ISearchResultData>> {
|
|
|
+ async searchKeyword(query: SearchQuery): Promise<ISearchResult<ISearchResultData>> {
|
|
|
|
|
|
// for debug
|
|
|
if (process.env.NODE_ENV === 'development') {
|
|
|
logger.debug('query: ', JSON.stringify(query, null, 2));
|
|
|
|
|
|
- const validateQueryResponse = await this.client.indices.validateQuery({
|
|
|
- index: query.index,
|
|
|
- type: query.type,
|
|
|
- explain: true,
|
|
|
- body: {
|
|
|
- query: query.body.query,
|
|
|
- },
|
|
|
- });
|
|
|
+
|
|
|
+ const validateQueryResponse = await (async() => {
|
|
|
+ if (isES7ClientDelegator(this.client)) {
|
|
|
+ const es7SearchQuery = query as ES7SearchQuery;
|
|
|
+ return this.client.indices.validateQuery({
|
|
|
+ explain: true,
|
|
|
+ index: es7SearchQuery.index,
|
|
|
+ body: {
|
|
|
+ query: es7SearchQuery.body?.query,
|
|
|
+ },
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ if (isES8ClientDelegator(this.client)) {
|
|
|
+ const es8SearchQuery = query as ES8SearchQuery;
|
|
|
+ return this.client.indices.validateQuery({
|
|
|
+ explain: true,
|
|
|
+ index: es8SearchQuery.index,
|
|
|
+ query: es8SearchQuery.body.query,
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ if (isES9ClientDelegator(this.client)) {
|
|
|
+ const es9SearchQuery = query as ES9SearchQuery;
|
|
|
+ return this.client.indices.validateQuery({
|
|
|
+ explain: true,
|
|
|
+ index: es9SearchQuery.index,
|
|
|
+ query: es9SearchQuery.body.query,
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ throw new Error('Unsupported Elasticsearch version');
|
|
|
+ })();
|
|
|
+
|
|
|
|
|
|
// for debug
|
|
|
logger.debug('ES result: ', validateQueryResponse);
|
|
|
}
|
|
|
|
|
|
- const searchResponse = await this.client.search(query);
|
|
|
+ const searchResponse = await (async() => {
|
|
|
+ if (isES7ClientDelegator(this.client)) {
|
|
|
+ return this.client.search(query as ES7SearchQuery);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (isES8ClientDelegator(this.client)) {
|
|
|
+ return this.client.search(query as ES8SearchQuery);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (isES9ClientDelegator(this.client)) {
|
|
|
+ const { body, ...rest } = query as ES9SearchQuery;
|
|
|
+ return this.client.search({
|
|
|
+ ...rest,
|
|
|
+ // Elimination of the body property since ES9
|
|
|
+ // https://raw.githubusercontent.com/elastic/elasticsearch-js/2f6200eb397df0e54d23848d769a93614ee1fb45/docs/release-notes/breaking-changes.md
|
|
|
+ query: body.query,
|
|
|
+ sort: body.sort,
|
|
|
+ highlight: body.highlight,
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ throw new Error('Unsupported Elasticsearch version');
|
|
|
+ })();
|
|
|
|
|
|
const _total = searchResponse?.hits?.total;
|
|
|
let total = 0;
|
|
|
@@ -627,45 +653,49 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
|
|
|
/**
|
|
|
* create search query for Elasticsearch
|
|
|
- *
|
|
|
- * @param {object | undefined} option optional paramas
|
|
|
* @returns {object} query object
|
|
|
*/
|
|
|
- createSearchQuery(option?) {
|
|
|
- let fields = ['path', 'bookmark_count', 'comment_count', 'seenUsers_count', 'updated_at', 'tag_names', 'comments'];
|
|
|
- if (option) {
|
|
|
- fields = option.fields || fields;
|
|
|
- }
|
|
|
+ createSearchQuery(): SearchQuery {
|
|
|
+ const fields = ['path', 'bookmark_count', 'comment_count', 'seenUsers_count', 'updated_at', 'tag_names', 'comments'];
|
|
|
|
|
|
// sort by score
|
|
|
- // eslint-disable-next-line prefer-const
|
|
|
- let query = {
|
|
|
+ const query: SearchQuery = {
|
|
|
index: this.aliasName,
|
|
|
_source: fields,
|
|
|
body: {
|
|
|
- query: {}, // query
|
|
|
+ query: {
|
|
|
+ bool: {},
|
|
|
+ },
|
|
|
},
|
|
|
};
|
|
|
|
|
|
return query;
|
|
|
}
|
|
|
|
|
|
- appendResultSize(query, from?, size?) {
|
|
|
+ appendResultSize(query: SearchQuery, from?: number, size?: number): void {
|
|
|
query.from = from || DEFAULT_OFFSET;
|
|
|
query.size = size || DEFAULT_LIMIT;
|
|
|
}
|
|
|
|
|
|
- appendSortOrder(query, sortAxis: SORT_AXIS, sortOrder: SORT_ORDER) {
|
|
|
+ appendSortOrder(query: SearchQuery, sortAxis: SORT_AXIS, sortOrder: SORT_ORDER): void {
|
|
|
+ if (query.body == null) {
|
|
|
+ throw new Error('query.body is not initialized');
|
|
|
+ }
|
|
|
+
|
|
|
// default sort order is score descending
|
|
|
const sort = ES_SORT_AXIS[sortAxis] || ES_SORT_AXIS[RELATION_SCORE];
|
|
|
const order = ES_SORT_ORDER[sortOrder] || ES_SORT_ORDER[DESC];
|
|
|
- query.body.sort = { [sort]: { order } };
|
|
|
+
|
|
|
+ query.body.sort = {
|
|
|
+ [sort]: { order },
|
|
|
+ };
|
|
|
+
|
|
|
}
|
|
|
|
|
|
- initializeBoolQuery(query) {
|
|
|
+ initializeBoolQuery(query: SearchQuery): SearchQuery {
|
|
|
// query is created by createSearchQuery()
|
|
|
- if (!query.body.query.bool) {
|
|
|
- query.body.query.bool = {};
|
|
|
+ if (query?.body?.query?.bool == null) {
|
|
|
+ throw new Error('query.body.query.bool is not initialized');
|
|
|
}
|
|
|
|
|
|
const isInitialized = (query) => { return !!query && Array.isArray(query) };
|
|
|
@@ -682,14 +712,30 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
return query;
|
|
|
}
|
|
|
|
|
|
- appendCriteriaForQueryString(query, parsedKeywords: ESQueryTerms): void {
|
|
|
+ appendCriteriaForQueryString(query: SearchQuery, parsedKeywords: ESQueryTerms): void {
|
|
|
query = this.initializeBoolQuery(query); // eslint-disable-line no-param-reassign
|
|
|
|
|
|
+ if (query.body?.query?.bool == null) {
|
|
|
+ throw new Error('query.body.query.bool is not initialized');
|
|
|
+ }
|
|
|
+
|
|
|
+ if (query.body?.query?.bool.must == null || !Array.isArray(query.body?.query?.bool.must)) {
|
|
|
+ throw new Error('query.body.query.bool.must is not initialized');
|
|
|
+ }
|
|
|
+
|
|
|
+ if (query.body?.query?.bool.must_not == null || !Array.isArray(query.body?.query?.bool.must_not)) {
|
|
|
+ throw new Error('query.body.query.bool.must_not is not initialized');
|
|
|
+ }
|
|
|
+
|
|
|
+ if (query.body?.query?.bool.filter == null || !Array.isArray(query.body?.query?.bool.filter)) {
|
|
|
+ throw new Error('query.body.query.bool.filter is not initialized');
|
|
|
+ }
|
|
|
+
|
|
|
if (parsedKeywords.match.length > 0) {
|
|
|
const q = {
|
|
|
multi_match: {
|
|
|
query: parsedKeywords.match.join(' '),
|
|
|
- type: 'most_fields',
|
|
|
+ type: 'most_fields' as const,
|
|
|
fields: ['path.ja^2', 'path.en^2', 'body.ja', 'body.en', 'comments.ja', 'comments.en'],
|
|
|
},
|
|
|
};
|
|
|
@@ -701,18 +747,18 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
multi_match: {
|
|
|
query: parsedKeywords.not_match.join(' '),
|
|
|
fields: ['path.ja', 'path.en', 'body.ja', 'body.en', 'comments.ja', 'comments.en'],
|
|
|
- operator: 'or',
|
|
|
+ operator: 'or' as const,
|
|
|
},
|
|
|
};
|
|
|
query.body.query.bool.must_not.push(q);
|
|
|
}
|
|
|
|
|
|
if (parsedKeywords.phrase.length > 0) {
|
|
|
- parsedKeywords.phrase.forEach((phrase) => {
|
|
|
+ for (const phrase of parsedKeywords.phrase) {
|
|
|
const phraseQuery = {
|
|
|
multi_match: {
|
|
|
- query: phrase, // each phrase is quoteted words like "This is GROWI"
|
|
|
- type: 'phrase',
|
|
|
+ query: phrase, // query is created by createSearchQuery()
|
|
|
+ type: 'phrase' as const,
|
|
|
fields: [
|
|
|
// Not use "*.ja" fields here, because we want to analyze (parse) search words
|
|
|
'path.raw^2',
|
|
|
@@ -722,15 +768,15 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
},
|
|
|
};
|
|
|
query.body.query.bool.must.push(phraseQuery);
|
|
|
- });
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
if (parsedKeywords.not_phrase.length > 0) {
|
|
|
- parsedKeywords.not_phrase.forEach((phrase) => {
|
|
|
+ for (const phrase of parsedKeywords.not_phrase) {
|
|
|
const notPhraseQuery = {
|
|
|
multi_match: {
|
|
|
query: phrase, // each phrase is quoteted words
|
|
|
- type: 'phrase',
|
|
|
+ type: 'phrase' as const,
|
|
|
fields: [
|
|
|
// Not use "*.ja" fields here, because we want to analyze (parse) search words
|
|
|
'path.raw^2',
|
|
|
@@ -739,7 +785,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
},
|
|
|
};
|
|
|
query.body.query.bool.must_not.push(notPhraseQuery);
|
|
|
- });
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
if (parsedKeywords.prefix.length > 0) {
|
|
|
@@ -771,12 +817,16 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- async filterPagesByViewer(query, user, userGroups) {
|
|
|
+ async filterPagesByViewer(query: SearchQuery, user, userGroups): Promise<void> {
|
|
|
const showPagesRestrictedByOwner = !configManager.getConfig('security:list-policy:hideRestrictedByOwner');
|
|
|
const showPagesRestrictedByGroup = !configManager.getConfig('security:list-policy:hideRestrictedByGroup');
|
|
|
|
|
|
query = this.initializeBoolQuery(query); // eslint-disable-line no-param-reassign
|
|
|
|
|
|
+ if (query.body?.query?.bool?.filter == null || !Array.isArray(query.body?.query?.bool?.filter)) {
|
|
|
+ throw new Error('query.body.query.bool is not initialized');
|
|
|
+ }
|
|
|
+
|
|
|
const Page = mongoose.model('Page') as unknown as PageModel;
|
|
|
const {
|
|
|
GRANT_PUBLIC, GRANT_SPECIFIED, GRANT_OWNER, GRANT_USER_GROUP,
|
|
|
@@ -835,7 +885,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
query.body.query.bool.filter.push({ bool: { should: grantConditions } });
|
|
|
}
|
|
|
|
|
|
- async appendFunctionScore(query, queryString) {
|
|
|
+ async appendFunctionScore(query, queryString): Promise<void> {
|
|
|
const User = mongoose.model('User');
|
|
|
const count = await User.count({}) || 1;
|
|
|
|
|
|
@@ -859,43 +909,11 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
};
|
|
|
}
|
|
|
|
|
|
- // async appendVectorScore(query, queryString: string, username?: string): Promise<void> {
|
|
|
-
|
|
|
- // const searchAssistant = await getOrCreateSearchAssistant();
|
|
|
-
|
|
|
- // // generate keywords for vector
|
|
|
- // const run = await openaiClient.beta.threads.createAndRunPoll({
|
|
|
- // assistant_id: searchAssistant.id,
|
|
|
- // thread: {
|
|
|
- // messages: [
|
|
|
- // { role: 'user', content: 'globalLang: "en_US", userLang: "ja_JP", user_input: "武井さんがジョインしたのはいつですか？"' },
|
|
|
- // { role: 'assistant', content: '武井さん 武井 takei yuki ジョイン join 入社 加入 雇用開始 年月日 start date join employee' },
|
|
|
- // { role: 'user', content: `globalLang: "en_US", userLang: "ja_JP", user_input: "${queryString}"` },
|
|
|
- // ],
|
|
|
- // },
|
|
|
- // });
|
|
|
- // const messages = await openaiClient.beta.threads.messages.list(run.thread_id, {
|
|
|
- // limit: 1,
|
|
|
- // });
|
|
|
- // const content = messages.data[0].content[0];
|
|
|
- // const keywordsForVector = content.type === 'text' ? content.text.value : queryString;
|
|
|
-
|
|
|
- // logger.debug('keywordsFor: ', keywordsForVector);
|
|
|
-
|
|
|
- // const queryVector = (await embed(queryString, username))[0].embedding;
|
|
|
-
|
|
|
- // query.body.query = {
|
|
|
- // script_score: {
|
|
|
- // query: { ...query.body.query },
|
|
|
- // script: {
|
|
|
- // source: "cosineSimilarity(params.query_vector, 'body_embedded') + 1.0",
|
|
|
- // params: { query_vector: queryVector },
|
|
|
- // },
|
|
|
- // },
|
|
|
- // };
|
|
|
- // }
|
|
|
-
|
|
|
- appendHighlight(query) {
|
|
|
+ appendHighlight(query: SearchQuery): void {
|
|
|
+ if (query.body == null) {
|
|
|
+ throw new Error('query.body is not initialized');
|
|
|
+ }
|
|
|
+
|
|
|
query.body.highlight = {
|
|
|
fragmenter: 'simple',
|
|
|
pre_tags: ["<em class='highlighted-keyword'>"],
|
|
|
@@ -928,15 +946,10 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
|
|
|
const query = this.createSearchQuery();
|
|
|
|
|
|
- if (option?.vector) {
|
|
|
- // await this.filterPagesByViewer(query, user, userGroups);
|
|
|
- // await this.appendVectorScore(query, queryString, user?.username);
|
|
|
- }
|
|
|
- else {
|
|
|
- this.appendCriteriaForQueryString(query, terms);
|
|
|
- await this.filterPagesByViewer(query, user, userGroups);
|
|
|
- await this.appendFunctionScore(query, queryString);
|
|
|
- }
|
|
|
+ this.appendCriteriaForQueryString(query, terms);
|
|
|
+ await this.filterPagesByViewer(query, user, userGroups);
|
|
|
+ await this.appendFunctionScore(query, queryString);
|
|
|
+
|
|
|
|
|
|
this.appendResultSize(query, from, size);
|
|
|
|
|
|
@@ -967,7 +980,7 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
|
|
|
}
|
|
|
|
|
|
// remove pages whitch should nod Indexed
|
|
|
- async syncPagesUpdated(pages, user) {
|
|
|
+ async syncPagesUpdated(pages, user): Promise<void> {
|
|
|
const shoudDeletePages: any[] = [];
|
|
|
|
|
|
// delete if page should not indexed
|
mergify[bot]