Keep uploaded files in memory

apps/app/src/features/openai/server/services/openai.ts

@@ -79,7 +79,7 @@ export interface IOpenaiService {
   createVectorStoreFile(vectorStoreRelation: VectorStoreDocument, pages: PageDocument[]): Promise<void>;
   createVectorStoreFileOnPageCreate(pages: PageDocument[]): Promise<void>;
   updateVectorStoreFileOnPageUpdate(page: HydratedDocument<PageDocument>): Promise<void>;
-  createVectorStoreFileOnUploadAttachment(page: HydratedDocument<PageDocument>, file: Express.Multer.File): Promise<void>;
+  createVectorStoreFileOnUploadAttachment(page: HydratedDocument<PageDocument>, buffer: Buffer, fileName: string): Promise<void>;
   deleteVectorStoreFile(vectorStoreRelationId: Types.ObjectId, pageId: Types.ObjectId): Promise<void>;
   deleteVectorStoreFilesByPageIds(pageIds: Types.ObjectId[]): Promise<void>;
   deleteObsoleteVectorStoreFile(limit: number, apiCallInterval: number): Promise<void>; // for CronJob
@@ -577,15 +577,15 @@ class OpenaiService implements IOpenaiService {
     }
   }
 
-  async createVectorStoreFileOnUploadAttachment(page: HydratedDocument<PageDocument>, file: Express.Multer.File): Promise<void> {
+  async createVectorStoreFileOnUploadAttachment(page: HydratedDocument<PageDocument>, buffer: Buffer, fileName: string): Promise<void> {
     const aiAssistants = await this.findAiAssistantByPagePath([page.path], { shouldPopulateVectorStore: true });
     if (aiAssistants.length === 0) {
       return;
     }
 
-    const fileStream = fs.createReadStream(file.path, { flags: 'r', autoClose: true });
-    const file_ = await toFile(fileStream, file.originalname);
+    const file_ = await toFile(buffer, fileName);
     const uploadedFile = await this.client.uploadFile(file_);
+    logger.debug('uploadedFile', uploadedFile);
 
     for await (const aiAssistant of aiAssistants) {
       const pagesToVectorize = await this.filterPagesByAccessScope(aiAssistant, [page]);

apps/app/src/server/routes/apiv3/attachment.js

@@ -1,3 +1,5 @@
+import fs from 'fs';
+
 import { ErrorV3 } from '@growi/core/dist/models';
 import { serializeUserSecurely } from '@growi/core/dist/models/serializers';
 import express from 'express';
@@ -371,9 +373,12 @@ module.exports = (crowi) => {
         };
 
         if (isVectorStoreCompatible(file)) {
+          const fileName = file.originalname;
+          // Keep uploaded files in memory as they are deleted by autoReap middleware
+          const fileContent = fs.readFileSync(file.path);
           const openaiService = getOpenaiService();
           // no await
-          openaiService?.createVectorStoreFileOnUploadAttachment(page, file);
+          openaiService?.createVectorStoreFileOnUploadAttachment(page, fileContent, fileName);
         }
 
         activityEvent.emit('update', res.locals.activity._id, { action: SupportedAction.ACTION_ATTACHMENT_ADD });