Impl 153988

apps/app/src/server/service/openai/openai-client-delegator.ts

@@ -1,4 +1,5 @@
 import OpenAI from 'openai';
+import { type Uploadable } from 'openai/uploads';
 
 import { aiServiceType as serviceType, aiServiceTypes } from '~/interfaces/ai';
 import { configManager } from '~/server/service/config-manager';
@@ -16,7 +17,7 @@ export default class OpenaiClient {
     const aiServiceType = configManager.getConfig('crowi', 'app:aiServiceType');
 
     if (!aiEnabled) {
-      throw new Error('I_ENABLED is not true');
+      throw new Error('AI_ENABLED is not true');
     }
 
     if (aiServiceType == null || !aiServiceTypes.includes(aiServiceType)) {
@@ -59,4 +60,10 @@ export default class OpenaiClient {
       : null;
   }
 
+  async uploadAndPoll(files: Uploadable[]): Promise<OpenAI.Beta.VectorStores.FileBatches.VectorStoreFileBatch | null> {
+    return this.isOpenai
+      ? this.client.beta.vectorStores.fileBatches.uploadAndPoll(this.openaiVectorStoreId, { files })
+      : null;
+  }
+
 }

apps/app/src/server/service/openai/openai.ts

@@ -1,3 +1,11 @@
+import { Readable } from 'stream';
+
+import { PageGrant } from '@growi/core';
+import type { HydratedDocument } from 'mongoose';
+import mongoose from 'mongoose';
+import { toFile } from 'openai';
+
+import type { PageDocument, PageModel } from '~/server/models/page';
 import { configManager } from '~/server/service/config-manager';
 
 import OpenaiClient from './openai-client-delegator';
@@ -19,17 +27,31 @@ class OpenaiService implements IOpenaiService {
 
   async rebuildVectorStore() {
     // Delete an existing VectorStoreFile
-    const vectorStoreFileData = await this.client.getVectorStoreFiles();
-    const vectorStoreFiles = vectorStoreFileData?.data;
-    if (vectorStoreFiles != null && vectorStoreFiles.length > 0) {
-      vectorStoreFiles.forEach(async(vectorStoreFile) => {
-        await this.client.deleteVectorStoreFiles(vectorStoreFile.id);
-      });
-    }
+    // const vectorStoreFileData = await this.client.getVectorStoreFiles();
+    // const vectorStoreFiles = vectorStoreFileData?.data;
+    // if (vectorStoreFiles != null && vectorStoreFiles.length > 0) {
+    //   vectorStoreFiles.forEach(async(vectorStoreFile) => {
+    //     await this.client.deleteVectorStoreFiles(vectorStoreFile.id);
+    //   });
+    // }
 
     // Create all public pages VectorStoreFile
-    // TODO: https://redmine.weseek.co.jp/issues/153988
+    const page = mongoose.model<HydratedDocument<PageDocument>, PageModel>('Page');
+    const allPublicPages = await page.find({ grant: PageGrant.GRANT_PUBLIC }).populate('revision');
+
+    const filesPromise = allPublicPages
+      .filter(page => page.revision?.body != null && page.revision.body.length > 0)
+      .map(async(page) => {
+        const file = await toFile(Readable.from(page.revision.body), `${page._id}.md`);
+        return file;
+      });
+
+    if (filesPromise.length === 0) {
+      return;
+    }
 
+    const files = await Promise.all(filesPromise);
+    await this.client.uploadAndPoll(files);
   }
 
 }