Merge pull request #9698 from weseek/feat/162489-delete-vectorstore-of-old-knowledge-assistant-through-normalize-data

feat: Delete VectorStore of old knowledge assistant through normalizeData

apps/app/src/features/openai/server/services/openai.ts

@@ -71,6 +71,7 @@ export interface IOpenaiService {
   deleteThread(threadRelationId: string): Promise<ThreadRelationDocument>;
   deleteExpiredThreads(limit: number, apiCallInterval: number): Promise<void>; // for CronJob
   deleteObsolatedVectorStoreRelations(): Promise<void> // for CronJob
+  deleteVectorStore(vectorStoreRelationId: string): Promise<void>;
   getMessageData(threadId: string, lang?: Lang, options?: MessageListParams): Promise<OpenAI.Beta.Threads.Messages.MessagesPage>;
   getVectorStoreRelation(aiAssistantId: string): Promise<VectorStoreDocument>
   getVectorStoreRelationsByPageIds(pageId: Types.ObjectId[]): Promise<VectorStoreDocument[]>;
@@ -366,14 +367,15 @@ class OpenaiService implements IOpenaiService {
     return uploadedFile;
   }
 
-  private async deleteVectorStore(vectorStoreRelationId: string): Promise<void> {
+  async deleteVectorStore(vectorStoreRelationId: string): Promise<void> {
     const vectorStoreDocument: VectorStoreDocument | null = await VectorStoreModel.findOne({ _id: vectorStoreRelationId, isDeleted: false });
     if (vectorStoreDocument == null) {
       return;
     }
 
     try {
-      await this.client.deleteVectorStore(vectorStoreDocument.vectorStoreId);
+      const deleteVectorStoreResponse = await this.client.deleteVectorStore(vectorStoreDocument.vectorStoreId);
+      logger.debug('Delete vector store', deleteVectorStoreResponse);
       await vectorStoreDocument.markAsDeleted();
     }
     catch (err) {

apps/app/src/server/service/normalize-data/delete-legacy-knowledge-assistant-vector-store.ts

@@ -0,0 +1,26 @@
+import AiAssistantModel from '~/features/openai/server/models/ai-assistant';
+import VectorStoreRelationModel from '~/features/openai/server/models/vector-store';
+import { isAiEnabled } from '~/features/openai/server/services/is-ai-enabled';
+import { getOpenaiService } from '~/features/openai/server/services/openai';
+
+export const deleteLegacyKnowledgeAssistantVectorStore = async(): Promise<void> => {
+  if (!isAiEnabled()) {
+    return;
+  }
+
+  // Identify VectorStoreRelation documents not related to existing aiAssistant documents as those used by old knowledge assistant
+  // Retrieve these VectorStoreRelation documents used by old knowledge assistant
+  // Only one active ({isDeleted: false}) VectorStoreRelation document should exist for old knowledge assistant, so only one should be returned
+  const aiAssistantVectorStoreIds = await AiAssistantModel.distinct('vectorStore');
+  const nonDeletedLegacyKnowledgeAssistantVectorStoreRelations = await VectorStoreRelationModel.find({
+    _id: { $nin: aiAssistantVectorStoreIds },
+    isDeleted: false,
+  });
+
+  // Logically delete only the VectorStore entities, leaving related documents to be automatically deleted by cron job
+  const openaiService = getOpenaiService();
+  for await (const vectorStoreRelation of nonDeletedLegacyKnowledgeAssistantVectorStoreRelations) {
+    const vectorStoreFileRelationId = vectorStoreRelation._id;
+    await openaiService?.deleteVectorStore(vectorStoreFileRelationId);
+  }
+};

apps/app/src/server/service/normalize-data/index.ts

@@ -3,6 +3,7 @@ import loggerFactory from '~/utils/logger';
 
 import { convertNullToEmptyGrantedArrays } from './convert-null-to-empty-granted-arrays';
 import { convertRevisionPageIdToObjectId } from './convert-revision-page-id-to-objectid';
+import { deleteLegacyKnowledgeAssistantVectorStore } from './delete-legacy-knowledge-assistant-vector-store';
 import { renameDuplicateRootPages } from './rename-duplicate-root-pages';
 
 const logger = loggerFactory('growi:service:NormalizeData');
@@ -12,6 +13,7 @@ export const normalizeData = async(): Promise<void> => {
   await convertRevisionPageIdToObjectId();
   await normalizeExpiredAtForThreadRelations();
   await convertNullToEmptyGrantedArrays();
+  await deleteLegacyKnowledgeAssistantVectorStore();
 
   logger.info('normalizeData has been executed');
   return;