imprv createVectorStoreFile

apps/app/src/server/models/page.ts

@@ -7,6 +7,7 @@ import {
   type IPage,
   GroupType, type HasObjectId,
 } from '@growi/core';
+import type { IPagePopulatedToShowRevision } from '@growi/core/dist/interfaces';
 import { getIdForRef, isPopulated } from '@growi/core/dist/interfaces';
 import { isTopPage, hasSlash } from '@growi/core/dist/utils/page-path-utils';
 import { addTrailingSlash, normalizePath } from '@growi/core/dist/utils/path-utils';
@@ -50,10 +51,9 @@ export interface PageDocument extends IPage, Document<Types.ObjectId> {
   [x:string]: any // for obsolete methods
   getLatestRevisionBodyLength(): Promise<number | null | undefined>
   calculateAndUpdateLatestRevisionBodyLength(this: PageDocument): Promise<void>
-  populateDataToShowRevision(shouldExcludeBody?: boolean): Promise<PageDocument>
+  populateDataToShowRevision(shouldExcludeBody?: boolean): Promise<IPagePopulatedToShowRevision>
 }
 
-
 type TargetAndAncestorsResult = {
   targetAndAncestors: PageDocument[]
   rootPage: PageDocument

apps/app/src/server/service/openai/openai.ts

@@ -1,10 +1,10 @@
 import { Readable } from 'stream';
 
-import type { IRevisionHasId } from '@growi/core';
 import { PageGrant, isPopulated } from '@growi/core';
-import type { HydratedDocument } from 'mongoose';
+import type { HydratedDocument, Types } from 'mongoose';
 import mongoose from 'mongoose';
 import { toFile } from 'openai';
+import type { FileLike } from 'openai/uploads.mjs';
 
 import { OpenaiServiceTypes } from '~/interfaces/ai';
 import type { PageDocument, PageModel } from '~/server/models/page';
@@ -15,6 +15,10 @@ import { getClient } from './client-delegator';
 
 const logger = loggerFactory('growi:service:openai');
 
+const createFileForVectorStore = async(pageId: Types.ObjectId, body: string): Promise<FileLike> => {
+  return toFile(Readable.from(body), `${pageId}.md`);
+};
+
 export interface IOpenaiService {
   createVectorStoreFile(pages: PageDocument[]): Promise<void>;
   rebuildVectorStoreAll(): Promise<void>;
@@ -28,14 +32,19 @@ class OpenaiService implements IOpenaiService {
   }
 
   async createVectorStoreFile(pages: PageDocument[]): Promise<void> {
-    const filesPromise = pages
-      .filter(page => page.grant === PageGrant.GRANT_PUBLIC && page.revision != null && isPopulated(page.revision) && page.revision.body.length > 0)
-      .map(async(page) => {
-        // The above filters ensure that revisions are populated
-        const revision = page?.revision as IRevisionHasId;
-        const file = await toFile(Readable.from(revision.body), `${page._id}.md`);
-        return file;
-      });
+    const filesPromise: Promise<FileLike>[] = [];
+    pages.forEach(async(page) => {
+      if (page._id != null && page.grant === PageGrant.GRANT_PUBLIC && page.revision != null) {
+        if (isPopulated(page.revision) && page.revision.body.length > 0) {
+          filesPromise.push(createFileForVectorStore(page._id, page.revision.body));
+        }
+
+        const pagePopulatedToShowRevision = await page.populateDataToShowRevision();
+        if (pagePopulatedToShowRevision.revision != null && pagePopulatedToShowRevision.revision.body.length > 0) {
+          filesPromise.push(createFileForVectorStore(page._id, pagePopulatedToShowRevision.revision.body));
+        }
+      }
+    });
 
     if (filesPromise.length === 0) {
       return;