Merge pull request #9122 from weseek/153985-add-openai-menu-to-the-admin-page

feat: Add AI Search menu to the admin page

apps/app/public/static/locales/en_US/admin.json

@@ -1132,5 +1132,9 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "Users without administrative rights cannot access the administration screen"
+  },
+  "ai_integration": {
+    "ai_integration": "AI Integration",
+    "disable_mode_explanation": "Currently, AI integration is disabled. To enable it, please set the environment variable <code>AI_ENABLED</code> to true."
   }
 }

apps/app/public/static/locales/fr_FR/admin.json

@@ -1131,5 +1131,9 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "Seul les administrateurs peuvent accéder à cette page."
+  },
+  "ai_integration": {
+    "ai_integration": "Intégration de l'IA",
+    "disable_mode_explanation": "Actuellement, l'intégration de l'IA est désactivée. Pour l'activer, veuillez définir la variable d'environnement <code>AI_ENABLED</code> sur true"
   }
 }

apps/app/public/static/locales/ja_JP/admin.json

@@ -1142,5 +1142,9 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "管理者権限のないユーザーでは管理画面にはアクセスできません"
+  },
+  "ai_integration": {
+    "ai_integration": "AI 連携",
+    "disable_mode_explanation": "現在、AI 連携は無効になっています。有効にする場合は環境変数 <code>AI_ENABLED</code> を true に設定してください。"
   }
 }

apps/app/public/static/locales/zh_CN/admin.json

@@ -1141,5 +1141,9 @@
   },
   "forbidden_page": {
     "do_not_have_admin_permission": "没有管理权限的用户无法访问管理屏幕"
+  },
+  "ai_integration": {
+    "ai_integration": "AI 集成",
+    "disable_mode_explanation": "目前，AI 集成已禁用。要启用它，请将环境变量 <code>AI_ENABLED</code> 设置为 true”"
   }
 }

apps/app/src/client/components/Admin/AiIntegration/AiIntegrationDisableMode.tsx

@@ -0,0 +1,28 @@
+import type { FC } from 'react';
+import React from 'react';
+
+import { useTranslation } from 'react-i18next';
+
+export const AiIntegrationDisableMode: FC = () => {
+  const { t } = useTranslation('admin');
+
+  return (
+    <div className="ccontainer-lg">
+      <div className="container">
+        <div className="row justify-content-md-center">
+          <div className="col-md-6 mt-5">
+            <div className="text-center">
+              {/* error icon large */}
+              <h1><span className="material-symbols-outlined">error</span></h1>
+              <h1 className="text-center">{t('ai_integration.ai_integration')}</h1>
+              <h3
+                // eslint-disable-next-line react/no-danger
+                dangerouslySetInnerHTML={{ __html: t('ai_integration.disable_mode_explanation') }}
+              />
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};

apps/app/src/components/Admin/Common/AdminNavigation.tsx

@@ -32,6 +32,7 @@ const MenuLabel = ({ menu }: { menu: string }) => {
     case 'user-groups':              return <><span className="material-symbols-outlined me-1">group</span>{          t('user_group_management.user_group_management') }</>;
     case 'audit-log':                return <><span className="material-symbols-outlined me-1">feed</span>{            t('audit_log_management.audit_log')}</>;
     case 'plugins':                  return <><span className="material-symbols-outlined me-1">extension</span>{          t('plugins.plugins')}</>;
+    case 'ai-integration':           return <><span className="material-symbols-outlined me-1">psychology</span>{          t('ai_integration.ai_integration')}</>;
     case 'search':                   return <><span className="material-symbols-outlined me-1">search</span>{       t('full_text_search_management.full_text_search_management') }</>;
     case 'cloud':                    return <><span className="material-symbols-outlined me-1">share</span>{       t('cloud_setting_management.to_cloud_settings')} </>;
     default:                         return <><span className="material-symbols-outlined me-1">home</span>{            t('wiki_management_homepage') }</>;
@@ -106,6 +107,7 @@ export const AdminNavigation = (): JSX.Element => {
         <MenuLink menu="user-groups" isListGroupItems={isListGroupItems} isActive={isActiveMenu(['/user-groups', 'user-group-detail'])} />
         <MenuLink menu="audit-log" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/audit-log')} />
         <MenuLink menu="plugins" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/plugins')} />
+        <MenuLink menu="ai-integration" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/ai-integration')} />
         <MenuLink menu="search" isListGroupItems={isListGroupItems} isActive={isActiveMenu('/search')} />
         {growiCloudUri != null && growiAppIdForGrowiCloud != null
           && (
@@ -159,6 +161,7 @@ export const AdminNavigation = (): JSX.Element => {
             {isActiveMenu('/audit-log')             && <MenuLabel menu="audit-log" />}
             {isActiveMenu('/plugins')               && <MenuLabel menu="plugins" />}
             {isActiveMenu('/data-transfer')         && <MenuLabel menu="data-transfer" />}
+            {isActiveMenu('/ai-integration')                && <MenuLabel menu="ai-integration" />}
             {/* eslint-enable no-multi-spaces */}
           </span>
         </button>

apps/app/src/pages/admin/ai-integration.page.tsx

@@ -0,0 +1,60 @@
+import type {
+  NextPage, GetServerSideProps, GetServerSidePropsContext,
+} from 'next';
+import { useTranslation } from 'next-i18next';
+import dynamic from 'next/dynamic';
+import Head from 'next/head';
+
+import type { CrowiRequest } from '~/interfaces/crowi-request';
+import type { CommonProps } from '~/pages/utils/commons';
+import { generateCustomTitle } from '~/pages/utils/commons';
+
+import { retrieveServerSideProps } from '../../utils/admin-page-util';
+
+const AdminLayout = dynamic(() => import('~/components/Layout/AdminLayout'), { ssr: false });
+const ForbiddenPage = dynamic(() => import('~/client/components/Admin/ForbiddenPage').then(mod => mod.ForbiddenPage), { ssr: false });
+const AiIntegrationDisableMode = dynamic(
+  () => import('~/client/components/Admin/AiIntegration/AiIntegrationDisableMode').then(mod => mod.AiIntegrationDisableMode), { ssr: false },
+);
+
+type Props = CommonProps & {
+  aiEnabled: boolean,
+};
+
+const AdminAiIntegrationPage: NextPage<Props> = (props) => {
+  const { t } = useTranslation('admin');
+
+  const title = t('ai_integration.ai_integration');
+  const headTitle = generateCustomTitle(props, title);
+
+  if (props.isAccessDeniedForNonAdminUser) {
+    return <ForbiddenPage />;
+  }
+
+  return (
+    <AdminLayout componentTitle={title}>
+      <Head>
+        <title>{headTitle}</title>
+      </Head>
+      {props.aiEnabled
+        ? <></> // TODO: implement admin page
+        : <AiIntegrationDisableMode />
+      }
+    </AdminLayout>
+  );
+};
+
+const injectServerConfigurations = async(context: GetServerSidePropsContext, props: Props): Promise<void> => {
+  const req: CrowiRequest = context.req as CrowiRequest;
+  const { crowi } = req;
+  const { configManager } = crowi;
+
+  props.aiEnabled = configManager.getConfig('crowi', 'app:aiEnabled');
+};
+
+export const getServerSideProps: GetServerSideProps = async(context: GetServerSidePropsContext) => {
+  const props = await retrieveServerSideProps(context, injectServerConfigurations);
+  return props;
+};
+
+export default AdminAiIntegrationPage;

apps/app/src/server/routes/apiv3/openai/chat.ts

@@ -3,6 +3,7 @@ import type { ValidationChain } from 'express-validator';
 import { body } from 'express-validator';
 
 import type Crowi from '~/server/crowi';
+import { configManager } from '~/server/service/config-manager';
 import { openaiClient } from '~/server/service/openai';
 import { getOrCreateChatAssistant } from '~/server/service/openai/assistant';
 import loggerFactory from '~/utils/logger';
@@ -33,7 +34,7 @@ export const chatHandlersFactory: ChatHandlersFactory = (crowi) => {
   return [
     accessTokenParser, loginRequiredStrictly, validator, apiV3FormValidator,
     async(req: Req, res: ApiV3Response) => {
-      const vectorStoreId = process.env.OPENAI_VECTOR_STORE_ID;
+      const vectorStoreId = configManager.getConfig('crowi', 'app:openaiVectorStoreId');
       if (vectorStoreId == null) {
         return res.apiv3Err('OPENAI_VECTOR_STORE_ID is not setup', 503);
       }

apps/app/src/server/service/config-loader.ts

@@ -736,6 +736,54 @@ const ENV_VAR_NAME_TO_CONFIG_INFO = {
     type: ValueType.NUMBER,
     default: 172800, // 2 days
   },
+  AI_ENABLED: {
+    ns: 'crowi',
+    key: 'app:aiEnabled',
+    type: ValueType.BOOLEAN,
+    default: false,
+  },
+  AI_SERVICE_TYPE: {
+    ns: 'crowi',
+    key: 'app:aiServiceType',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_API_KEY: {
+    ns: 'crowi',
+    key: 'app:openaiApiKey',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_DIMENSIONS: {
+    ns: 'crowi',
+    key: 'app:openaiDimensions',
+    type: ValueType.NUMBER,
+    default: null,
+  },
+  OPENAI_SEARCH_ASSISTANT_INSTRUCTIONS: {
+    ns: 'crowi',
+    key: 'app:openaiSearchAssistantInstructions',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_CHAT_ASSISTANT_INSTRUCTIONS: {
+    ns: 'crowi',
+    key: 'app:openaiChatAssistantInstructions',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_ASSISTANT_NAME_SUFFIX: {
+    ns: 'crowi',
+    key: 'app:openaiAssistantNameSuffix',
+    type: ValueType.STRING,
+    default: null,
+  },
+  OPENAI_VECTOR_STORE_ID: {
+    ns: 'crowi',
+    key: 'app:openaiVectorStoreId',
+    type: ValueType.STRING,
+    default: null,
+  },
 };
 
 

apps/app/src/server/service/openai/assistant/assistant.ts

@@ -35,7 +35,7 @@ const findAssistantByName = async(assistantName: string): Promise<OpenAI.Beta.As
 
 const getOrCreateAssistant = async(type: AssistantType): Promise<OpenAI.Beta.Assistant> => {
   const appSiteUrl = configManager.getConfig('crowi', 'app:siteUrl');
-  const assistantName = `GROWI ${type} Assistant for ${appSiteUrl} ${process.env.OPENAI_ASSISTANT_NAME_SUFFIX}`;
+  const assistantName = `GROWI ${type} Assistant for ${appSiteUrl} ${configManager.getConfig('crowi', 'app:openaiAssistantNameSuffix')}}`;
 
   const assistantOnRemote = await findAssistantByName(assistantName);
   if (assistantOnRemote != null) {
@@ -58,7 +58,7 @@ export const getOrCreateSearchAssistant = async(): Promise<OpenAI.Beta.Assistant
 
   searchAssistant = await getOrCreateAssistant(AssistantType.SEARCH);
   openaiClient.beta.assistants.update(searchAssistant.id, {
-    instructions: process.env.OPENAI_SEARCH_ASSISTANT_INSTRUCTIONS,
+    instructions: configManager.getConfig('crowi', 'app:openaiSearchAssistantInstructions'),
     tools: [{ type: 'file_search' }],
   });
 
@@ -74,7 +74,7 @@ export const getOrCreateChatAssistant = async(): Promise<OpenAI.Beta.Assistant>
 
   chatAssistant = await getOrCreateAssistant(AssistantType.CHAT);
   openaiClient.beta.assistants.update(chatAssistant.id, {
-    instructions: process.env.OPENAI_CHAT_ASSISTANT_INSTRUCTIONS,
+    instructions: configManager.getConfig('crowi', 'app:openaiChatAssistantInstructions'),
     tools: [{ type: 'file_search' }],
   });
 

apps/app/src/server/service/openai/client.ts

@@ -1,5 +1,7 @@
 import OpenAI from 'openai';
 
+import { configManager } from '~/server/service/config-manager';
+
 export const openaiClient = new OpenAI({
-  apiKey: process.env.OPENAI_API_KEY, // This is the default and can be omitted
+  apiKey: configManager?.getConfig('crowi', 'app:openaiApiKey'), // This is the default and can be omitted
 });

apps/app/src/server/service/openai/embeddings.ts

@@ -2,6 +2,8 @@ import crypto from 'crypto';
 
 import type { OpenAI } from 'openai';
 
+import { configManager } from '~/server/service/config-manager';
+
 import { openaiClient } from './client';
 
 
@@ -17,7 +19,7 @@ export const embed = async(input: string, username?: string): Promise<OpenAI.Emb
   const result = await openaiClient.embeddings.create({
     input,
     model: 'text-embedding-3-large',
-    dimensions: Number(process.env.OPENAI_DIMENSIONS),
+    dimensions: configManager.getConfig('crowi', 'app:openaiDimensions'),
     user,
   });
 

apps/app/src/server/service/openai/file-upload.ts

@@ -3,12 +3,14 @@ import { Readable } from 'stream';
 import type { IPageHasId } from '@growi/core';
 import { toFile } from 'openai';
 
+import { configManager } from '~/server/service/config-manager';
+
 import { openaiClient } from './client';
 
 type PageToUpload = Omit<IPageHasId, 'revision'> & { revision: { body: string } };
 
 export const fileUpload = async(pages: PageToUpload[]): Promise<void> => {
-  const vectorStoreId = process.env.OPENAI_VECTOR_STORE_ID;
+  const vectorStoreId = configManager.getConfig('crowi', 'app:openaiVectorStoreId');
   if (vectorStoreId == null) {
     return;
   }

apps/app/src/server/service/search-delegator/elasticsearch.ts

@@ -20,8 +20,8 @@ import type { PageModel } from '../../models/page';
 import { createBatchStream } from '../../util/batch-stream';
 import { configManager } from '../config-manager';
 import type { UpdateOrInsertPagesOpts } from '../interfaces/search';
-import { embed, openaiClient, fileUpload } from '../openai';
-import { getOrCreateSearchAssistant } from '../openai/assistant';
+// // import { embed, openaiClient, fileUpload } from '../openai';
+// import { getOrCreateSearchAssistant } from '../openai/assistant';
 
 import { aggregatePipelineToIndex } from './aggregate-to-index';
 import type { AggregatedPage, BulkWriteBody, BulkWriteCommand } from './bulk-write';
@@ -481,27 +481,27 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
       },
     });
 
-    const appendEmbeddingStream = new Transform({
-      objectMode: true,
-      async transform(chunk: AggregatedPage[], encoding, callback) {
-        // append embedding
-        for await (const doc of chunk) {
-          doc.revisionBodyEmbedded = (await embed(doc.revision.body, doc.creator?.username))[0].embedding;
-        }
-
-        this.push(chunk);
-        callback();
-      },
-    });
-
-    const appendFileUploadedStream = new Transform({
-      objectMode: true,
-      async transform(chunk, encoding, callback) {
-        await fileUpload(chunk);
-        this.push(chunk);
-        callback();
-      },
-    });
+    // const appendEmbeddingStream = new Transform({
+    //   objectMode: true,
+    //   async transform(chunk: AggregatedPage[], encoding, callback) {
+    //     // append embedding
+    //     for await (const doc of chunk) {
+    //       doc.revisionBodyEmbedded = (await embed(doc.revision.body, doc.creator?.username))[0].embedding;
+    //     }
+
+    //     this.push(chunk);
+    //     callback();
+    //   },
+    // });
+
+    // const appendFileUploadedStream = new Transform({
+    //   objectMode: true,
+    //   async transform(chunk, encoding, callback) {
+    //     await fileUpload(chunk);
+    //     this.push(chunk);
+    //     callback();
+    //   },
+    // });
 
     let count = 0;
     const writeStream = new Writable({
@@ -556,8 +556,8 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
     readStream
       .pipe(batchStream)
       .pipe(appendTagNamesStream)
-      .pipe(appendEmbeddingStream)
-      .pipe(appendFileUploadedStream)
+      // .pipe(appendEmbeddingStream)
+      // .pipe(appendFileUploadedStream)
       .pipe(writeStream);
 
     return streamToPromise(writeStream);
@@ -858,41 +858,41 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
     };
   }
 
-  async appendVectorScore(query, queryString: string, username?: string): Promise<void> {
-
-    const searchAssistant = await getOrCreateSearchAssistant();
-
-    // generate keywords for vector
-    const run = await openaiClient.beta.threads.createAndRunPoll({
-      assistant_id: searchAssistant.id,
-      thread: {
-        messages: [
-          { role: 'user', content: 'globalLang: "en_US", userLang: "ja_JP", user_input: "武井さんがジョインしたのはいつですか？"' },
-          { role: 'assistant', content: '武井さん 武井 takei yuki ジョイン join 入社 加入 雇用開始 年月日 start date join employee' },
-          { role: 'user', content: `globalLang: "en_US", userLang: "ja_JP", user_input: "${queryString}"` },
-        ],
-      },
-    });
-    const messages = await openaiClient.beta.threads.messages.list(run.thread_id, {
-      limit: 1,
-    });
-    const content = messages.data[0].content[0];
-    const keywordsForVector = content.type === 'text' ? content.text.value : queryString;
-
-    logger.debug('keywordsFor: ', keywordsForVector);
-
-    const queryVector = (await embed(queryString, username))[0].embedding;
-
-    query.body.query = {
-      script_score: {
-        query: { ...query.body.query },
-        script: {
-          source: "cosineSimilarity(params.query_vector, 'body_embedded') + 1.0",
-          params: { query_vector: queryVector },
-        },
-      },
-    };
-  }
+  // async appendVectorScore(query, queryString: string, username?: string): Promise<void> {
+
+  //   const searchAssistant = await getOrCreateSearchAssistant();
+
+  //   // generate keywords for vector
+  //   const run = await openaiClient.beta.threads.createAndRunPoll({
+  //     assistant_id: searchAssistant.id,
+  //     thread: {
+  //       messages: [
+  //         { role: 'user', content: 'globalLang: "en_US", userLang: "ja_JP", user_input: "武井さんがジョインしたのはいつですか？"' },
+  //         { role: 'assistant', content: '武井さん 武井 takei yuki ジョイン join 入社 加入 雇用開始 年月日 start date join employee' },
+  //         { role: 'user', content: `globalLang: "en_US", userLang: "ja_JP", user_input: "${queryString}"` },
+  //       ],
+  //     },
+  //   });
+  //   const messages = await openaiClient.beta.threads.messages.list(run.thread_id, {
+  //     limit: 1,
+  //   });
+  //   const content = messages.data[0].content[0];
+  //   const keywordsForVector = content.type === 'text' ? content.text.value : queryString;
+
+  //   logger.debug('keywordsFor: ', keywordsForVector);
+
+  //   const queryVector = (await embed(queryString, username))[0].embedding;
+
+  //   query.body.query = {
+  //     script_score: {
+  //       query: { ...query.body.query },
+  //       script: {
+  //         source: "cosineSimilarity(params.query_vector, 'body_embedded') + 1.0",
+  //         params: { query_vector: queryVector },
+  //       },
+  //     },
+  //   };
+  // }
 
   appendHighlight(query) {
     query.body.highlight = {
@@ -928,8 +928,8 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
     const query = this.createSearchQuery();
 
     if (option?.vector) {
-      await this.filterPagesByViewer(query, user, userGroups);
-      await this.appendVectorScore(query, queryString, user?.username);
+      // await this.filterPagesByViewer(query, user, userGroups);
+      // await this.appendVectorScore(query, queryString, user?.username);
     }
     else {
       this.appendCriteriaForQueryString(query, terms);