Merge remote-tracking branch 'origin/feat/openai-vector-searching' into imprv/openai-chat-sse

apps/app/public/static/locales/en_US/admin.json

@@ -1132,5 +1132,16 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "Users without administrative rights cannot access the administration screen"
+  },
+  "ai_integration": {
+    "ai_integration": "AI Integration",
+    "disable_mode_explanation": "Currently, AI integration is disabled. To enable it, please set the environment variable <code>AI_ENABLED</code> to true.",
+    "ai_search_management": "AI search management",
+    "rebuild_vector_store": "Rebuild Vector Store",
+    "rebuild_vector_store_label": "Rebuild",
+    "rebuild_vector_store_explanation1": "Delete the existing Vector Store and recreate the Vector Store on the public page.",
+    "rebuild_vector_store_explanation2": "This process may take several minutes.",
+    "rebuild_vector_store_requested": "Vector Store rebuild has been requested",
+    "rebuild_vector_store_failed": "Vector Store rebuild failed"
   }
 }

apps/app/public/static/locales/fr_FR/admin.json

@@ -1131,5 +1131,16 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "Seul les administrateurs peuvent accéder à cette page."
+  },
+  "ai_integration": {
+    "ai_integration": "Intégration de l'IA",
+    "disable_mode_explanation": "Actuellement, l'intégration de l'IA est désactivée. Pour l'activer, veuillez définir la variable d'environnement <code>AI_ENABLED</code> sur true",
+    "ai_search_management": "Gestion de la recherche par l'IA",
+    "rebuild_vector_store": "Reconstruire le magasin Vector",
+    "rebuild_vector_store_label": "Reconstruire",
+    "rebuild_vector_store_explanation1": "Supprimez le Vector Store existant et recréez le Vector Store sur la page publique.",
+    "rebuild_vector_store_explanation2": "Ce processus peut prendre plusieurs minutes.",
+    "rebuild_vector_store_requested": "La reconstruction du magasin Vector a été demandée",
+    "rebuild_vector_store_failed": "Échec de la reconstruction du magasin de vecteurs"
   }
 }

apps/app/public/static/locales/ja_JP/admin.json

@@ -1142,5 +1142,16 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "管理者権限のないユーザーでは管理画面にはアクセスできません"
+  },
+  "ai_integration": {
+    "ai_integration": "AI 連携",
+    "disable_mode_explanation": "現在、AI 連携は無効になっています。有効にする場合は環境変数 <code>AI_ENABLED</code> を true に設定してください。",
+    "ai_search_management": "AI 検索管理",
+    "rebuild_vector_store": "Vector Store のリビルド",
+    "rebuild_vector_store_label": "リビルド",
+    "rebuild_vector_store_explanation1": "既存の Vector Store を削除し、公開ページの Vector Store を再作成します。",
+    "rebuild_vector_store_explanation2": "この作業には数分かかる可能性があります。",
+    "rebuild_vector_store_requested": "Vector Store のリビルドを受け付けました",
+    "rebuild_vector_store_failed": "Vector Store のリビルドに失敗しました"
   }
 }

apps/app/public/static/locales/zh_CN/admin.json

@@ -1141,5 +1141,16 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "没有管理权限的用户无法访问管理屏幕"
+  },
+  "ai_integration": {
+    "ai_integration": "AI 集成",
+    "disable_mode_explanation": "目前，AI 集成已禁用。要启用它，请将环境变量 <code>AI_ENABLED</code> 设置为 true",
+    "ai_search_management": "AI 搜索管理",
+    "rebuild_vector_store": "重建矢量商店",
+    "rebuild_vector_store_label": "重建",
+    "rebuild_vector_store_explanation1": "删除现有的矢量存储，在公共页面上重新创建矢量存储。",
+    "rebuild_vector_store_explanation2": "这个过程可能需要几分钟。",
+    "rebuild_vector_store_requested": "已要求重建矢量存储库",
+    "rebuild_vector_store_failed": "向量存储区重建失败"
   }
 }

apps/app/src/client/components/Admin/AiIntegration/AiIntegration.tsx

@@ -0,0 +1,46 @@
+import { useCallback } from 'react';
+
+import { useTranslation } from 'react-i18next';
+
+import { apiv3Post } from '~/client/util/apiv3-client';
+import { toastSuccess, toastError } from '~/client/util/toastr';
+
+
+export const AiIntegration = (): JSX.Element => {
+  const { t } = useTranslation('admin');
+
+  const clickRebuildVectorStoreButtonHandler = useCallback(async() => {
+    try {
+      toastSuccess(t('ai_integration.rebuild_vector_store_requested'));
+      await apiv3Post('/openai/rebuild-vector-store');
+    }
+    catch {
+      toastError(t('ai_integration.rebuild_vector_store_failed'));
+    }
+  }, [t]);
+
+  return (
+    <div data-testid="admin-ai-integration">
+      <h2 className="admin-setting-header">{ t('ai_integration.ai_search_management') }</h2>
+
+      <div className="row">
+        <label className="col-md-3 col-form-label text-start text-md-end">{ t('ai_integration.rebuild_vector_store_label') }</label>
+        <div className="col-md-8">
+          {/* TODO: https://redmine.weseek.co.jp/issues/153978 */}
+          <button
+            type="submit"
+            className="btn btn-primary"
+            onClick={clickRebuildVectorStoreButtonHandler}
+          >
+            {t('ai_integration.rebuild_vector_store')}
+          </button>
+
+          <p className="form-text text-muted">
+            {t('ai_integration.rebuild_vector_store_explanation1')}<br />
+            {t('ai_integration.rebuild_vector_store_explanation2')}<br />
+          </p>
+        </div>
+      </div>
+    </div>
+  );
+};

apps/app/src/client/components/Admin/AiIntegration/AiIntegrationDisableMode.tsx

@@ -0,0 +1,28 @@
+import type { FC } from 'react';
+import React from 'react';
+
+import { useTranslation } from 'react-i18next';
+
+export const AiIntegrationDisableMode: FC = () => {
+  const { t } = useTranslation('admin');
+
+  return (
+    <div className="ccontainer-lg">
+      <div className="container">
+        <div className="row justify-content-md-center">
+          <div className="col-md-6 mt-5">
+            <div className="text-center">
+              {/* error icon large */}
+              <h1><span className="material-symbols-outlined">error</span></h1>
+              <h1 className="text-center">{t('ai_integration.ai_integration')}</h1>
+              <h3
+                // eslint-disable-next-line react/no-danger
+                dangerouslySetInnerHTML={{ __html: t('ai_integration.disable_mode_explanation') }}
+              />
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};

apps/app/src/components/Admin/Common/AdminNavigation.tsx

@@ -32,6 +32,7 @@ const MenuLabel = ({ menu }: { menu: string }) => {
     case 'user-groups':              return <><span className="material-symbols-outlined me-1">group</span>{          t('user_group_management.user_group_management') }</>;
     case 'audit-log':                return <><span className="material-symbols-outlined me-1">feed</span>{            t('audit_log_management.audit_log')}</>;
     case 'plugins':                  return <><span className="material-symbols-outlined me-1">extension</span>{          t('plugins.plugins')}</>;
+    case 'ai-integration':           return <><span className="material-symbols-outlined me-1">psychology</span>{          t('ai_integration.ai_integration')}</>;
     case 'search':                   return <><span className="material-symbols-outlined me-1">search</span>{       t('full_text_search_management.full_text_search_management') }</>;
     case 'cloud':                    return <><span className="material-symbols-outlined me-1">share</span>{       t('cloud_setting_management.to_cloud_settings')} </>;
     default:                         return <><span className="material-symbols-outlined me-1">home</span>{            t('wiki_management_homepage') }</>;
@@ -106,6 +107,7 @@ export const AdminNavigation = (): JSX.Element => {
         <MenuLink menu="user-groups" isListGroupItems={isListGroupItems} isActive={isActiveMenu(['/user-groups', 'user-group-detail'])} />
         <MenuLink menu="audit-log" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/audit-log')} />
         <MenuLink menu="plugins" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/plugins')} />
+        <MenuLink menu="ai-integration" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/ai-integration')} />
         <MenuLink menu="search" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/search')} />
         {growiCloudUri != null && growiAppIdForGrowiCloud != null
           && (
@@ -159,6 +161,7 @@ export const AdminNavigation = (): JSX.Element => {
             {isActiveMenu('/audit-log')             && <MenuLabel menu="audit-log" />}
             {isActiveMenu('/plugins')               && <MenuLabel menu="plugins" />}
             {isActiveMenu('/data-transfer')         && <MenuLabel menu="data-transfer" />}
+            {isActiveMenu('/ai-integration')                && <MenuLabel menu="ai-integration" />}
             {/* eslint-enable no-multi-spaces */}
           </span>
         </button>

apps/app/src/interfaces/ai.ts

@@ -0,0 +1,5 @@
+export const aiServiceType = {
+  OPEN_AI: 'openai',
+} as const;
+
+export const aiServiceTypes = Object.values(aiServiceType);

apps/app/src/pages/admin/ai-integration.page.tsx

@@ -0,0 +1,61 @@
+import type {
+  NextPage, GetServerSideProps, GetServerSidePropsContext,
+} from 'next';
+import { useTranslation } from 'next-i18next';
+import dynamic from 'next/dynamic';
+import Head from 'next/head';
+
+import type { CrowiRequest } from '~/interfaces/crowi-request';
+import type { CommonProps } from '~/pages/utils/commons';
+import { generateCustomTitle } from '~/pages/utils/commons';
+
+import { retrieveServerSideProps } from '../../utils/admin-page-util';
+
+const AdminLayout = dynamic(() => import('~/components/Layout/AdminLayout'), { ssr: false });
+const ForbiddenPage = dynamic(() => import('~/client/components/Admin/ForbiddenPage').then(mod => mod.ForbiddenPage), { ssr: false });
+const AiIntegration = dynamic(() => import('~/client/components/Admin/AiIntegration/AiIntegration').then(mod => mod.AiIntegration), { ssr: false });
+const AiIntegrationDisableMode = dynamic(
+  () => import('~/client/components/Admin/AiIntegration/AiIntegrationDisableMode').then(mod => mod.AiIntegrationDisableMode), { ssr: false },
+);
+
+type Props = CommonProps & {
+  aiEnabled: boolean,
+};
+
+const AdminAiIntegrationPage: NextPage<Props> = (props: Props) => {
+  const { t } = useTranslation('admin');
+
+  const title = t('ai_integration.ai_integration');
+  const headTitle = generateCustomTitle(props, title);
+
+  if (props.isAccessDeniedForNonAdminUser) {
+    return <ForbiddenPage />;
+  }
+
+  return (
+    <AdminLayout componentTitle={title}>
+      <Head>
+        <title>{headTitle}</title>
+      </Head>
+      {props.aiEnabled
+        ? <AiIntegration />
+        : <AiIntegrationDisableMode />
+      }
+    </AdminLayout>
+  );
+};
+
+const injectServerConfigurations = async(context: GetServerSidePropsContext, props: Props): Promise<void> => {
+  const req: CrowiRequest = context.req as CrowiRequest;
+  const { crowi } = req;
+  const { configManager } = crowi;
+
+  props.aiEnabled = configManager.getConfig('crowi', 'app:aiEnabled');
+};
+
+export const getServerSideProps: GetServerSideProps = async(context: GetServerSidePropsContext) => {
+  const props = await retrieveServerSideProps(context, injectServerConfigurations);
+  return props;
+};
+
+export default AdminAiIntegrationPage;

apps/app/src/server/middlewares/certify-ai-service.ts

@@ -0,0 +1,26 @@
+import type { NextFunction, Request, Response } from 'express';
+
+import { aiServiceTypes } from '~/interfaces/ai';
+import { configManager } from '~/server/service/config-manager';
+import loggerFactory from '~/utils/logger';
+
+const logger = loggerFactory('growi:middlewares:certify-ai-service');
+
+export const certifyAiService = (req: Request, res: Response & { apiv3Err }, next: NextFunction): void => {
+  const aiEnabled = configManager.getConfig('crowi', 'app:aiEnabled');
+  const aiServiceType = configManager.getConfig('crowi', 'app:aiServiceType');
+
+  if (!aiEnabled) {
+    const message = 'AI_ENABLED is not true';
+    logger.error(message);
+    return res.apiv3Err(message, 403);
+  }
+
+  if (aiServiceType == null || !aiServiceTypes.includes(aiServiceType)) {
+    const message = 'AI_SERVICE_TYPE is missing or contains an invalid value';
+    logger.error(message);
+    return res.apiv3Err(message, 403);
+  }
+
+  next();
+};

apps/app/src/server/routes/apiv3/openai/chat.ts

@@ -3,11 +3,14 @@ import type { ValidationChain } from 'express-validator';
 import { body } from 'express-validator';
 
 import type Crowi from '~/server/crowi';
+import { apiV3FormValidator } from '~/server/middlewares/apiv3-form-validator';
+import { certifyAiService } from '~/server/middlewares/certify-ai-service';
+import { configManager } from '~/server/service/config-manager';
 import { openaiClient } from '~/server/service/openai';
 import { getOrCreateChatAssistant } from '~/server/service/openai/assistant';
 import loggerFactory from '~/utils/logger';
 
-import { apiV3FormValidator } from '../../../middlewares/apiv3-form-validator';
+
 import type { ApiV3Response } from '../interfaces/apiv3-response';
 
 const logger = loggerFactory('growi:routes:apiv3:openai:chat');
@@ -31,9 +34,9 @@ export const chatHandlersFactory: ChatHandlersFactory = (crowi) => {
   ];
 
   return [
-    accessTokenParser, loginRequiredStrictly, validator, apiV3FormValidator,
+    accessTokenParser, loginRequiredStrictly, certifyAiService, validator, apiV3FormValidator,
     async(req: Req, res: ApiV3Response) => {
-      const vectorStoreId = process.env.OPENAI_VECTOR_STORE_ID;
+      const vectorStoreId = configManager.getConfig('crowi', 'app:openaiVectorStoreId');
       if (vectorStoreId == null) {
         return res.apiv3Err('OPENAI_VECTOR_STORE_ID is not setup', 503);
       }

apps/app/src/server/routes/apiv3/openai/index.ts

@@ -3,16 +3,19 @@ import express from 'express';
 import { chatHandlersFactory } from './chat';
 import { postMessageHandlersFactory } from './message';
 import { createThreadHandlersFactory } from './thread';
+import { rebuildVectorStoreHandlersFactory } from './rebuild-vector-store';
 
 const router = express.Router();
 
 module.exports = (crowi) => {
+  router.post('/rebuild-vector-store', rebuildVectorStoreHandlersFactory(crowi));
+
   // deprecated
   router.post('/chat', chatHandlersFactory(crowi));
-
   // create thread
   router.post('/thread', createThreadHandlersFactory(crowi));
   // post message and return streaming with SSE
   router.post('/message', postMessageHandlersFactory(crowi));
+
   return router;
 };

apps/app/src/server/routes/apiv3/openai/rebuild-vector-store.ts

@@ -0,0 +1,30 @@
+import type { Request, RequestHandler } from 'express';
+import type { ValidationChain } from 'express-validator';
+
+import type Crowi from '~/server/crowi';
+import { certifyAiService } from '~/server/middlewares/certify-ai-service';
+import loggerFactory from '~/utils/logger';
+
+import { apiV3FormValidator } from '../../../middlewares/apiv3-form-validator';
+import type { ApiV3Response } from '../interfaces/apiv3-response';
+
+const logger = loggerFactory('growi:routes:apiv3:openai:rebuild-vector-store');
+
+type RebuildVectorStoreFactory = (crowi: Crowi) => RequestHandler[];
+
+export const rebuildVectorStoreHandlersFactory: RebuildVectorStoreFactory = (crowi) => {
+  const accessTokenParser = require('~/server/middlewares/access-token-parser')(crowi);
+  const loginRequiredStrictly = require('~/server/middlewares/login-required')(crowi);
+  const adminRequired = require('~/server/middlewares/admin-required')(crowi);
+
+  const validator: ValidationChain[] = [
+    //
+  ];
+
+  return [
+    accessTokenParser, loginRequiredStrictly, adminRequired, certifyAiService, validator, apiV3FormValidator,
+    async(req: Request, res: ApiV3Response) => {
+      return res.apiv3({});
+    },
+  ];
+};

apps/app/src/server/service/config-loader.ts

@@ -736,6 +736,54 @@ const ENV_VAR_NAME_TO_CONFIG_INFO = {
     type: ValueType.NUMBER,
     default: 172800, // 2 days
   },
+  AI_ENABLED: {
+    ns: 'crowi',
+    key: 'app:aiEnabled',
+    type: ValueType.BOOLEAN,
+    default: false,
+  },
+  AI_SERVICE_TYPE: {
+    ns: 'crowi',
+    key: 'app:aiServiceType',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_API_KEY: {
+    ns: 'crowi',
+    key: 'app:openaiApiKey',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_DIMENSIONS: {
+    ns: 'crowi',
+    key: 'app:openaiDimensions',
+    type: ValueType.NUMBER,
+    default: null,
+  },
+  OPENAI_SEARCH_ASSISTANT_INSTRUCTIONS: {
+    ns: 'crowi',
+    key: 'app:openaiSearchAssistantInstructions',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_CHAT_ASSISTANT_INSTRUCTIONS: {
+    ns: 'crowi',
+    key: 'app:openaiChatAssistantInstructions',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_ASSISTANT_NAME_SUFFIX: {
+    ns: 'crowi',
+    key: 'app:openaiAssistantNameSuffix',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_VECTOR_STORE_ID: {
+    ns: 'crowi',
+    key: 'app:openaiVectorStoreId',
+    type: ValueType.STRING,
+    default: null,
+  },
 };
 
 

apps/app/src/server/service/openai/assistant/assistant.ts

@@ -35,7 +35,7 @@ const findAssistantByName = async(assistantName: string): Promise<OpenAI.Beta.As
 
 const getOrCreateAssistant = async(type: AssistantType): Promise<OpenAI.Beta.Assistant> => {
   const appSiteUrl = configManager.getConfig('crowi', 'app:siteUrl');
-  const assistantName = `GROWI ${type} Assistant for ${appSiteUrl} ${process.env.OPENAI_ASSISTANT_NAME_SUFFIX}`;
+  const assistantName = `GROWI ${type} Assistant for ${appSiteUrl} ${configManager.getConfig('crowi', 'app:openaiAssistantNameSuffix')}}`;
 
   const assistantOnRemote = await findAssistantByName(assistantName);
   if (assistantOnRemote != null) {
@@ -58,7 +58,7 @@ export const getOrCreateSearchAssistant = async(): Promise<OpenAI.Beta.Assistant
 
   searchAssistant = await getOrCreateAssistant(AssistantType.SEARCH);
   openaiClient.beta.assistants.update(searchAssistant.id, {
-    instructions: process.env.OPENAI_SEARCH_ASSISTANT_INSTRUCTIONS,
+    instructions: configManager.getConfig('crowi', 'app:openaiSearchAssistantInstructions'),
     tools: [{ type: 'file_search' }],
   });
 
@@ -74,7 +74,7 @@ export const getOrCreateChatAssistant = async(): Promise<OpenAI.Beta.Assistant>
 
   chatAssistant = await getOrCreateAssistant(AssistantType.CHAT);
   openaiClient.beta.assistants.update(chatAssistant.id, {
-    instructions: process.env.OPENAI_CHAT_ASSISTANT_INSTRUCTIONS,
+    instructions: configManager.getConfig('crowi', 'app:openaiChatAssistantInstructions'),
     tools: [{ type: 'file_search' }],
   });
 

apps/app/src/server/service/openai/client.ts

@@ -1,5 +1,7 @@
 import OpenAI from 'openai';
 
+import { configManager } from '~/server/service/config-manager';
+
 export const openaiClient = new OpenAI({
-  apiKey: process.env.OPENAI_API_KEY, // This is the default and can be omitted
+  apiKey: configManager?.getConfig('crowi', 'app:openaiApiKey'), // This is the default and can be omitted
 });

apps/app/src/server/service/openai/embeddings.ts

@@ -2,6 +2,8 @@ import crypto from 'crypto';
 
 import type { OpenAI } from 'openai';
 
+import { configManager } from '~/server/service/config-manager';
+
 import { openaiClient } from './client';
 
 
@@ -17,7 +19,7 @@ export const embed = async(input: string, username?: string): Promise<OpenAI.Emb
   const result = await openaiClient.embeddings.create({
     input,
     model: 'text-embedding-3-large',
-    dimensions: Number(process.env.OPENAI_DIMENSIONS),
+    dimensions: configManager.getConfig('crowi', 'app:openaiDimensions'),
     user,
   });
 

apps/app/src/server/service/openai/file-upload.ts

@@ -3,12 +3,14 @@ import { Readable } from 'stream';
 import type { IPageHasId } from '@growi/core';
 import { toFile } from 'openai';
 
+import { configManager } from '~/server/service/config-manager';
+
 import { openaiClient } from './client';
 
 type PageToUpload = Omit<IPageHasId, 'revision'> & { revision: { body: string } };
 
 export const fileUpload = async(pages: PageToUpload[]): Promise<void> => {
-  const vectorStoreId = process.env.OPENAI_VECTOR_STORE_ID;
+  const vectorStoreId = configManager.getConfig('crowi', 'app:openaiVectorStoreId');
   if (vectorStoreId == null) {
     return;
   }

apps/app/src/server/service/search-delegator/elasticsearch.ts

@@ -20,8 +20,8 @@ import type { PageModel } from '../../models/page';
 import { createBatchStream } from '../../util/batch-stream';
 import { configManager } from '../config-manager';
 import type { UpdateOrInsertPagesOpts } from '../interfaces/search';
-import { embed, openaiClient, fileUpload } from '../openai';
-import { getOrCreateSearchAssistant } from '../openai/assistant';
+// // import { embed, openaiClient, fileUpload } from '../openai';
+// import { getOrCreateSearchAssistant } from '../openai/assistant';
 
 import { aggregatePipelineToIndex } from './aggregate-to-index';
 import type { AggregatedPage, BulkWriteBody, BulkWriteCommand } from './bulk-write';
@@ -481,27 +481,27 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
       },
     });
 
-    const appendEmbeddingStream = new Transform({
-      objectMode: true,
-      async transform(chunk: AggregatedPage[], encoding, callback) {
-        // append embedding
-        for await (const doc of chunk) {
-          doc.revisionBodyEmbedded = (await embed(doc.revision.body, doc.creator?.username))[0].embedding;
-        }
-
-        this.push(chunk);
-        callback();
-      },
-    });
-
-    const appendFileUploadedStream = new Transform({
-      objectMode: true,
-      async transform(chunk, encoding, callback) {
-        await fileUpload(chunk);
-        this.push(chunk);
-        callback();
-      },
-    });
+    // const appendEmbeddingStream = new Transform({
+    //   objectMode: true,
+    //   async transform(chunk: AggregatedPage[], encoding, callback) {
+    //     // append embedding
+    //     for await (const doc of chunk) {
+    //       doc.revisionBodyEmbedded = (await embed(doc.revision.body, doc.creator?.username))[0].embedding;
+    //     }
+
+    //     this.push(chunk);
+    //     callback();
+    //   },
+    // });
+
+    // const appendFileUploadedStream = new Transform({
+    //   objectMode: true,
+    //   async transform(chunk, encoding, callback) {
+    //     await fileUpload(chunk);
+    //     this.push(chunk);
+    //     callback();
+    //   },
+    // });
 
     let count = 0;
     const writeStream = new Writable({
@@ -556,8 +556,8 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
     readStream
       .pipe(batchStream)
       .pipe(appendTagNamesStream)
-      .pipe(appendEmbeddingStream)
-      .pipe(appendFileUploadedStream)
+      // .pipe(appendEmbeddingStream)
+      // .pipe(appendFileUploadedStream)
       .pipe(writeStream);
 
     return streamToPromise(writeStream);
@@ -858,41 +858,41 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
     };
   }
 
-  async appendVectorScore(query, queryString: string, username?: string): Promise<void> {
-
-    const searchAssistant = await getOrCreateSearchAssistant();
-
-    // generate keywords for vector
-    const run = await openaiClient.beta.threads.createAndRunPoll({
-      assistant_id: searchAssistant.id,
-      thread: {
-        messages: [
-          { role: 'user', content: 'globalLang: "en_US", userLang: "ja_JP", user_input: "武井さんがジョインしたのはいつですか？"' },
-          { role: 'assistant', content: '武井さん 武井 takei yuki ジョイン join 入社 加入 雇用開始 年月日 start date join employee' },
-          { role: 'user', content: `globalLang: "en_US", userLang: "ja_JP", user_input: "${queryString}"` },
-        ],
-      },
-    });
-    const messages = await openaiClient.beta.threads.messages.list(run.thread_id, {
-      limit: 1,
-    });
-    const content = messages.data[0].content[0];
-    const keywordsForVector = content.type === 'text' ? content.text.value : queryString;
-
-    logger.debug('keywordsFor: ', keywordsForVector);
-
-    const queryVector = (await embed(queryString, username))[0].embedding;
-
-    query.body.query = {
-      script_score: {
-        query: { ...query.body.query },
-        script: {
-          source: "cosineSimilarity(params.query_vector, 'body_embedded') + 1.0",
-          params: { query_vector: queryVector },
-        },
-      },
-    };
-  }
+  // async appendVectorScore(query, queryString: string, username?: string): Promise<void> {
+
+  //   const searchAssistant = await getOrCreateSearchAssistant();
+
+  //   // generate keywords for vector
+  //   const run = await openaiClient.beta.threads.createAndRunPoll({
+  //     assistant_id: searchAssistant.id,
+  //     thread: {
+  //       messages: [
+  //         { role: 'user', content: 'globalLang: "en_US", userLang: "ja_JP", user_input: "武井さんがジョインしたのはいつですか？"' },
+  //         { role: 'assistant', content: '武井さん 武井 takei yuki ジョイン join 入社 加入 雇用開始 年月日 start date join employee' },
+  //         { role: 'user', content: `globalLang: "en_US", userLang: "ja_JP", user_input: "${queryString}"` },
+  //       ],
+  //     },
+  //   });
+  //   const messages = await openaiClient.beta.threads.messages.list(run.thread_id, {
+  //     limit: 1,
+  //   });
+  //   const content = messages.data[0].content[0];
+  //   const keywordsForVector = content.type === 'text' ? content.text.value : queryString;
+
+  //   logger.debug('keywordsFor: ', keywordsForVector);
+
+  //   const queryVector = (await embed(queryString, username))[0].embedding;
+
+  //   query.body.query = {
+  //     script_score: {
+  //       query: { ...query.body.query },
+  //       script: {
+  //         source: "cosineSimilarity(params.query_vector, 'body_embedded') + 1.0",
+  //         params: { query_vector: queryVector },
+  //       },
+  //     },
+  //   };
+  // }
 
   appendHighlight(query) {
     query.body.highlight = {
@@ -928,8 +928,8 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
     const query = this.createSearchQuery();
 
     if (option?.vector) {
-      await this.filterPagesByViewer(query, user, userGroups);
-      await this.appendVectorScore(query, queryString, user?.username);
+      // await this.filterPagesByViewer(query, user, userGroups);
+      // await this.appendVectorScore(query, queryString, user?.username);
     }
     else {
       this.appendCriteriaForQueryString(query, terms);