Merge branch 'feat/growi-ai-next' into support/162312-i18n-for-growi-ai-next

apps/app/src/features/openai/interfaces/thread-relation.ts

@@ -1,10 +1,10 @@
 import type { IUser, Ref, HasObjectId } from '@growi/core';
 
-import type { IVectorStore } from './vector-store';
+import type { AiAssistant } from './ai-assistant';
 
 export interface IThreadRelation {
   userId: Ref<IUser>
-  vectorStore: Ref<IVectorStore>
+  aiAssistant: Ref<AiAssistant>
   threadId: string;
   title?: string;
   expiredAt: Date;

apps/app/src/features/openai/server/models/thread-relation.ts

@@ -25,9 +25,9 @@ const schema = new Schema<ThreadRelationDocument, ThreadRelationModel>({
     ref: 'User',
     required: true,
   },
-  vectorStore: {
+  aiAssistant: {
     type: Schema.Types.ObjectId,
-    ref: 'VectorStore',
+    ref: 'AiAssistant',
     required: true,
   },
   threadId: {

apps/app/src/features/openai/server/routes/delete-ai-assistant.ts

@@ -11,7 +11,7 @@ import { apiV3FormValidator } from '~/server/middlewares/apiv3-form-validator';
 import type { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response';
 import loggerFactory from '~/utils/logger';
 
-import { getOpenaiService } from '../services/openai';
+import { deleteAiAssistant } from '../services/delete-ai-assistant';
 
 import { certifyAiService } from './middlewares/certify-ai-service';
 
@@ -41,13 +41,8 @@ export const deleteAiAssistantsFactory: DeleteAiAssistantsFactory = (crowi) => {
       const { id } = req.params;
       const { user } = req;
 
-      const openaiService = getOpenaiService();
-      if (openaiService == null) {
-        return res.apiv3Err(new ErrorV3('GROWI AI is not enabled'), 501);
-      }
-
       try {
-        const deletedAiAssistant = await openaiService.deleteAiAssistant(user._id, id);
+        const deletedAiAssistant = await deleteAiAssistant(user._id, id);
         return res.apiv3({ deletedAiAssistant });
       }
       catch (err) {

apps/app/src/features/openai/server/routes/get-threads.ts

@@ -48,8 +48,7 @@ export const getThreadsFactory: GetThreadsFactory = (crowi) => {
           return res.apiv3Err(new ErrorV3('The specified AI assistant is not usable'), 400);
         }
 
-        const vectorStoreRelation = await openaiService.getVectorStoreRelation(aiAssistantId);
-        const threads = await openaiService.getThreads(vectorStoreRelation._id);
+        const threads = await openaiService.getThreadsByAiAssistantId(aiAssistantId);
 
         return res.apiv3({ threads });
       }

apps/app/src/features/openai/server/routes/middlewares/upsert-ai-assistant-validator.ts

@@ -10,20 +10,17 @@ export const upsertAiAssistantValidator: ValidationChain[] = [
     .withMessage('name must be a string')
     .not()
     .isEmpty()
-    .withMessage('name is required')
-    .escape(),
+    .withMessage('name is required'),
 
   body('description')
     .optional()
     .isString()
-    .withMessage('description must be a string')
-    .escape(),
+    .withMessage('description must be a string'),
 
   body('additionalInstruction')
     .optional()
     .isString()
-    .withMessage('additionalInstruction must be a string')
-    .escape(),
+    .withMessage('additionalInstruction must be a string'),
 
   body('pagePathPatterns')
     .isArray()

apps/app/src/features/openai/server/routes/thread.ts

@@ -50,8 +50,7 @@ export const createThreadHandlersFactory: CreateThreadFactory = (crowi) => {
           return res.apiv3Err(new ErrorV3('The specified AI assistant is not usable'), 400);
         }
 
-        const vectorStoreRelation = await openaiService.getVectorStoreRelation(aiAssistantId);
-        const thread = await openaiService.createThread(req.user._id, vectorStoreRelation, initialUserMessage);
+        const thread = await openaiService.createThread(req.user._id, aiAssistantId, initialUserMessage);
 
         return res.apiv3(thread);
       }

apps/app/src/features/openai/server/services/client-delegator/azure-openai-client-delegator.ts

@@ -33,6 +33,16 @@ export class AzureOpenaiClientDelegator implements IOpenaiClientDelegator {
     });
   }
 
+  async updateThread(threadId: string, vectorStoreId: string): Promise<OpenAI.Beta.Threads.Thread> {
+    return this.client.beta.threads.update(threadId, {
+      tool_resources: {
+        file_search: {
+          vector_store_ids: [vectorStoreId],
+        },
+      },
+    });
+  }
+
   async retrieveThread(threadId: string): Promise<OpenAI.Beta.Threads.Thread> {
     return this.client.beta.threads.retrieve(threadId);
   }

apps/app/src/features/openai/server/services/client-delegator/interfaces.ts

@@ -5,6 +5,7 @@ import type { MessageListParams } from '../../../interfaces/message';
 
 export interface IOpenaiClientDelegator {
   createThread(vectorStoreId: string): Promise<OpenAI.Beta.Threads.Thread>
+  updateThread(threadId: string, vectorStoreId: string): Promise<OpenAI.Beta.Threads.Thread>
   retrieveThread(threadId: string): Promise<OpenAI.Beta.Threads.Thread>
   deleteThread(threadId: string): Promise<OpenAI.Beta.Threads.ThreadDeleted>
   getMessages(threadId: string, options?: MessageListParams): Promise<OpenAI.Beta.Threads.Messages.MessagesPage>

apps/app/src/features/openai/server/services/client-delegator/openai-client-delegator.ts

@@ -38,6 +38,16 @@ export class OpenaiClientDelegator implements IOpenaiClientDelegator {
     return this.client.beta.threads.retrieve(threadId);
   }
 
+  async updateThread(threadId: string, vectorStoreId: string): Promise<OpenAI.Beta.Threads.Thread> {
+    return this.client.beta.threads.update(threadId, {
+      tool_resources: {
+        file_search: {
+          vector_store_ids: [vectorStoreId],
+        },
+      },
+    });
+  }
+
   async deleteThread(threadId: string): Promise<OpenAI.Beta.Threads.ThreadDeleted> {
     return this.client.beta.threads.del(threadId);
   }

apps/app/src/features/openai/server/services/cron/vector-store-file-deletion-cron.ts

@@ -47,7 +47,7 @@ export class VectorStoreFileDeletionCronService {
   }
 
   private async executeJob(): Promise<void> {
-    await this.openaiService.deleteObsolatedVectorStoreRelations();
+    await this.openaiService.deleteObsoletedVectorStoreRelations();
     await this.openaiService.deleteObsoleteVectorStoreFile(this.vectorStoreFileDeletionBarchSize, this.vectorStoreFileDeletionApiCallInterval);
   }
 

apps/app/src/features/openai/server/services/delete-ai-assistant.ts

@@ -0,0 +1,51 @@
+import {
+  getIdStringForRef, type IUserHasId,
+} from '@growi/core';
+import createError from 'http-errors';
+
+import loggerFactory from '~/utils/logger';
+
+import type { AiAssistantDocument } from '../models/ai-assistant';
+import AiAssistantModel from '../models/ai-assistant';
+
+import { isAiEnabled } from './is-ai-enabled';
+import { getOpenaiService } from './openai';
+
+const logger = loggerFactory('growi:service:openai:delete-ai-assistant');
+
+
+export const deleteAiAssistant = async(ownerId: string, aiAssistantId: string): Promise<AiAssistantDocument> => {
+  const openaiService = getOpenaiService();
+  if (openaiService == null) {
+    throw createError('openaiService is not initialized', 500);
+  }
+
+  const aiAssistant = await AiAssistantModel.findOne({ owner: ownerId, _id: aiAssistantId });
+  if (aiAssistant == null) {
+    throw createError(404, 'AiAssistant document does not exist');
+  }
+
+  const vectorStoreRelationId = getIdStringForRef(aiAssistant.vectorStore);
+  await openaiService.deleteVectorStore(vectorStoreRelationId);
+
+  const deletedAiAssistant = await aiAssistant.remove();
+  return deletedAiAssistant;
+};
+
+export const deleteUserAiAssistant = async(user: IUserHasId): Promise<void> => {
+  if (isAiEnabled()) {
+    const aiAssistants = await AiAssistantModel.find({ owner: user });
+    for await (const aiAssistant of aiAssistants) {
+      try {
+        await deleteAiAssistant(user._id, aiAssistant._id);
+      }
+      catch (err) {
+        logger.error(`Failed to delete AiAssistant ${aiAssistant._id}`);
+      }
+    }
+  }
+
+  // Cannot delete OpenAI VectorStore entities without enabling openaiService.
+  // Delete OpenAI VectorStore entities through "deleteVectorStoresOrphanedFromAiAssistant" when app starts with openaiService enabled
+  await AiAssistantModel.deleteMany({ owner: user });
+};

apps/app/src/features/openai/server/services/normalize-data/normalize-thread-relation-expired-at/normalize-thread-relation-expired-at.integ.ts

@@ -15,7 +15,7 @@ describe('normalizeExpiredAtForThreadRelations', () => {
     const threadRelation = new ThreadRelation({
       userId: new Types.ObjectId(),
       threadId: 'test-thread',
-      vectorStore: new Types.ObjectId(),
+      aiAssistant: new Types.ObjectId(),
       expiredAt: expiredDate,
     });
     await threadRelation.save();
@@ -37,7 +37,7 @@ describe('normalizeExpiredAtForThreadRelations', () => {
     const threadRelation = new ThreadRelation({
       userId: new Types.ObjectId(),
       threadId: 'test-thread-2',
-      vectorStore: new Types.ObjectId(),
+      aiAssistant: new Types.ObjectId(),
       expiredAt: nonExpiredDate,
     });
     await threadRelation.save();
@@ -57,7 +57,7 @@ describe('normalizeExpiredAtForThreadRelations', () => {
     const threadRelation = new ThreadRelation({
       userId: new Types.ObjectId(),
       threadId: 'test-thread-3',
-      vectorStore: new Types.ObjectId(),
+      aiAssistant: new Types.ObjectId(),
       expiredAt: nonExpiredDate,
     });
     await threadRelation.save();

apps/app/src/features/openai/server/services/openai.ts

@@ -65,17 +65,13 @@ const convertPathPatternsToRegExp = (pagePathPatterns: string[]): Array<string |
 };
 
 export interface IOpenaiService {
-  createThread(
-    userId: string, vectorStoreRelation: VectorStoreDocument, initialUserMessage: string
-  ): Promise<ThreadRelationDocument>;
-  getThreads(vectorStoreRelationId: string): Promise<ThreadRelationDocument[]>
+  createThread(userId: string, aiAssistantId: string, initialUserMessage: string): Promise<ThreadRelationDocument>;
+  getThreadsByAiAssistantId(aiAssistantId: string): Promise<ThreadRelationDocument[]>
   deleteThread(threadRelationId: string): Promise<ThreadRelationDocument>;
   deleteExpiredThreads(limit: number, apiCallInterval: number): Promise<void>; // for CronJob
-  deleteObsolatedVectorStoreRelations(): Promise<void> // for CronJob
+  deleteObsoletedVectorStoreRelations(): Promise<void> // for CronJob
   deleteVectorStore(vectorStoreRelationId: string): Promise<void>;
   getMessageData(threadId: string, lang?: Lang, options?: MessageListParams): Promise<OpenAI.Beta.Threads.Messages.MessagesPage>;
-  getVectorStoreRelation(aiAssistantId: string): Promise<VectorStoreDocument>
-  getVectorStoreRelationsByPageIds(pageId: Types.ObjectId[]): Promise<VectorStoreDocument[]>;
   createVectorStoreFile(vectorStoreRelation: VectorStoreDocument, pages: PageDocument[]): Promise<void>;
   createVectorStoreFileOnPageCreate(pages: PageDocument[]): Promise<void>;
   updateVectorStoreFileOnPageUpdate(page: HydratedDocument<PageDocument>): Promise<void>;
@@ -86,7 +82,6 @@ export interface IOpenaiService {
   createAiAssistant(data: Omit<AiAssistant, 'vectorStore'>): Promise<AiAssistantDocument>;
   updateAiAssistant(aiAssistantId: string, data: Omit<AiAssistant, 'vectorStore'>): Promise<AiAssistantDocument>;
   getAccessibleAiAssistants(user: IUserHasId): Promise<AccessibleAiAssistants>
-  deleteAiAssistant(ownerId: string, aiAssistantId: string): Promise<AiAssistantDocument>
   isLearnablePageLimitExceeded(user: IUserHasId, pagePathPatterns: string[]): Promise<boolean>;
 }
 class OpenaiService implements IOpenaiService {
@@ -122,7 +117,9 @@ class OpenaiService implements IOpenaiService {
     return threadTitle;
   }
 
-  async createThread(userId: string, vectorStoreRelation: VectorStoreDocument, initialUserMessage: string): Promise<ThreadRelationDocument> {
+  async createThread(userId: string, aiAssistantId: string, initialUserMessage: string): Promise<ThreadRelationDocument> {
+    const vectorStoreRelation = await this.getVectorStoreRelationByAiAssistantId(aiAssistantId);
+
     let threadTitle: string | null = null;
     if (initialUserMessage != null) {
       try {
@@ -137,8 +134,8 @@ class OpenaiService implements IOpenaiService {
       const thread = await this.client.createThread(vectorStoreRelation.vectorStoreId);
       const threadRelation = await ThreadRelationModel.create({
         userId,
+        aiAssistant: aiAssistantId,
         threadId: thread.id,
-        vectorStore: vectorStoreRelation._id,
         title: threadTitle,
       });
       return threadRelation;
@@ -148,8 +145,21 @@ class OpenaiService implements IOpenaiService {
     }
   }
 
-  async getThreads(vectorStoreRelationId: string): Promise<ThreadRelationDocument[]> {
-    const threadRelations = await ThreadRelationModel.find({ vectorStore: vectorStoreRelationId });
+  async updateThreads(aiAssistantId: string, vectorStoreId: string): Promise<void> {
+    const threadRelations = await this.getThreadsByAiAssistantId(aiAssistantId);
+    for await (const threadRelation of threadRelations) {
+      try {
+        const updatedThreadResponse = await this.client.updateThread(threadRelation.threadId, vectorStoreId);
+        logger.debug('Update thread', updatedThreadResponse);
+      }
+      catch (err) {
+        logger.error(err);
+      }
+    }
+  }
+
+  async getThreadsByAiAssistantId(aiAssistantId: string): Promise<ThreadRelationDocument[]> {
+    const threadRelations = await ThreadRelationModel.find({ aiAssistant: aiAssistantId });
     return threadRelations;
   }
 
@@ -211,7 +221,7 @@ class OpenaiService implements IOpenaiService {
   }
 
 
-  async getVectorStoreRelation(aiAssistantId: string): Promise<VectorStoreDocument> {
+  async getVectorStoreRelationByAiAssistantId(aiAssistantId: string): Promise<VectorStoreDocument> {
     const aiAssistant = await AiAssistantModel.findById({ _id: aiAssistantId }).populate('vectorStore');
     if (aiAssistant == null) {
       throw createError(404, 'AiAssistant document does not exist');
@@ -376,7 +386,7 @@ class OpenaiService implements IOpenaiService {
   }
 
   // Deletes all VectorStore documents that are marked as deleted (isDeleted: true) and have no associated VectorStoreFileRelation documents
-  async deleteObsolatedVectorStoreRelations(): Promise<void> {
+  async deleteObsoletedVectorStoreRelations(): Promise<void> {
     const deletedVectorStoreRelations = await VectorStoreModel.find({ isDeleted: true });
     if (deletedVectorStoreRelations.length === 0) {
       return;
@@ -812,6 +822,8 @@ class OpenaiService implements IOpenaiService {
 
       newVectorStoreRelation = await this.createVectorStore(data.name);
 
+      this.updateThreads(aiAssistantId, newVectorStoreRelation.vectorStoreId);
+
       // VectorStore creation process does not await
       this.createVectorStoreFileWithStream(newVectorStoreRelation, conditions);
     }
@@ -865,19 +877,6 @@ class OpenaiService implements IOpenaiService {
     };
   }
 
-  async deleteAiAssistant(ownerId: string, aiAssistantId: string): Promise<AiAssistantDocument> {
-    const aiAssistant = await AiAssistantModel.findOne({ owner: ownerId, _id: aiAssistantId });
-    if (aiAssistant == null) {
-      throw createError(404, 'AiAssistant document does not exist');
-    }
-
-    const vectorStoreRelationId = getIdStringForRef(aiAssistant.vectorStore);
-    await this.deleteVectorStore(vectorStoreRelationId);
-
-    const deletedAiAssistant = await aiAssistant.remove();
-    return deletedAiAssistant;
-  }
-
   async isLearnablePageLimitExceeded(user: IUserHasId, pagePathPatterns: string[]): Promise<boolean> {
     const normalizedPagePathPatterns = removeGlobPath(pagePathPatterns);
 

apps/app/src/server/routes/apiv3/users.js

@@ -9,6 +9,7 @@ import { body, query } from 'express-validator';
 import { isEmail } from 'validator';
 
 import ExternalUserGroupRelation from '~/features/external-user-group/server/models/external-user-group-relation';
+import { deleteUserAiAssistant } from '~/features/openai/server/services/delete-ai-assistant';
 import { SupportedAction } from '~/interfaces/activity';
 import { accessTokenParser } from '~/server/middlewares/access-token-parser';
 import Activity from '~/server/models/activity';
@@ -809,6 +810,8 @@ module.exports = (crowi) => {
       await user.statusDelete();
       await ExternalAccount.remove({ user });
 
+      deleteUserAiAssistant(user);
+
       const serializedUser = serializeUserSecurely(user);
 
       activityEvent.emit('update', res.locals.activity._id, { action: SupportedAction.ACTION_ADMIN_USERS_REMOVE });

apps/app/src/server/service/normalize-data/delete-legacy-knowledge-assistant-vector-store.ts → apps/app/src/server/service/normalize-data/delete-vector-stores-orphaned-from-ai-assistant.ts

@@ -2,8 +2,11 @@ import AiAssistantModel from '~/features/openai/server/models/ai-assistant';
 import VectorStoreRelationModel from '~/features/openai/server/models/vector-store';
 import { isAiEnabled } from '~/features/openai/server/services/is-ai-enabled';
 import { getOpenaiService } from '~/features/openai/server/services/openai';
+import loggerFactory from '~/utils/logger';
 
-export const deleteLegacyKnowledgeAssistantVectorStore = async(): Promise<void> => {
+const logger = loggerFactory('growi:service:normalize-data:delete-vector-stores-orphaned-from-ai-assistant');
+
+export const deleteVectorStoresOrphanedFromAiAssistant = async(): Promise<void> => {
   if (!isAiEnabled()) {
     return;
   }
@@ -20,7 +23,12 @@ export const deleteLegacyKnowledgeAssistantVectorStore = async(): Promise<void>
   // Logically delete only the VectorStore entities, leaving related documents to be automatically deleted by cron job
   const openaiService = getOpenaiService();
   for await (const vectorStoreRelation of nonDeletedLegacyKnowledgeAssistantVectorStoreRelations) {
-    const vectorStoreFileRelationId = vectorStoreRelation._id;
-    await openaiService?.deleteVectorStore(vectorStoreFileRelationId);
+    try {
+      const vectorStoreFileRelationId = vectorStoreRelation._id;
+      await openaiService?.deleteVectorStore(vectorStoreFileRelationId);
+    }
+    catch (err) {
+      logger.error(err);
+    }
   }
 };

apps/app/src/server/service/normalize-data/index.ts

@@ -3,7 +3,7 @@ import loggerFactory from '~/utils/logger';
 
 import { convertNullToEmptyGrantedArrays } from './convert-null-to-empty-granted-arrays';
 import { convertRevisionPageIdToObjectId } from './convert-revision-page-id-to-objectid';
-import { deleteLegacyKnowledgeAssistantVectorStore } from './delete-legacy-knowledge-assistant-vector-store';
+import { deleteVectorStoresOrphanedFromAiAssistant } from './delete-vector-stores-orphaned-from-ai-assistant';
 import { renameDuplicateRootPages } from './rename-duplicate-root-pages';
 
 const logger = loggerFactory('growi:service:NormalizeData');
@@ -13,7 +13,7 @@ export const normalizeData = async(): Promise<void> => {
   await convertRevisionPageIdToObjectId();
   await normalizeExpiredAtForThreadRelations();
   await convertNullToEmptyGrantedArrays();
-  await deleteLegacyKnowledgeAssistantVectorStore();
+  await deleteVectorStoresOrphanedFromAiAssistant();
 
   logger.info('normalizeData has been executed');
   return;