Merge branch 'master' into feat/page-bulk-export

apps/app/package.json

@@ -237,7 +237,7 @@
     "@types/archiver": "^6.0.2",
     "@types/express": "^4.17.21",
     "@types/jest": "^29.5.2",
-    "@types/node-cron": "^3.0.2",
+    "@types/node-cron": "^3.0.11",
     "@types/react-input-autosize": "^2.2.4",
     "@types/react-scroll": "^1.8.4",
     "@types/react-stickynode": "^4.0.3",
@@ -278,8 +278,8 @@
     "react-hotkeys": "^2.0.0",
     "react-input-autosize": "^3.0.0",
     "react-toastify": "^9.1.3",
-    "remark-github-admonitions-to-directives": "^2.0.0",
     "rehype-rewrite": "^4.0.2",
+    "remark-github-admonitions-to-directives": "^2.0.0",
     "replacestream": "^4.0.3",
     "sass": "^1.53.0",
     "simple-load-script": "^1.0.2",

apps/app/public/static/locales/en_US/admin.json

@@ -15,7 +15,7 @@
     "scope_of_page_disclosure": "Scope of page disclosure",
     "set_point": "Set point",
     "Guest Users Access": "Guest users access",
-    "readonly_users_access": "ROM users' access",
+    "readonly_users_access": "Read only users' access",
     "always_hidden": "Always hidden",
     "always_displayed": "Always displayed",
     "displayed_or_hidden": "Hidden / Displayed",
@@ -87,9 +87,9 @@
       "deny": "Deny (Registered users only)",
       "readonly": "Accept (Guests can read only)"
     },
-    "rom_users_comment": {
-      "deny": "Deny (Prohibit ROM users from comment management)",
-      "accept": "Allow (ROM users can manage comments)"
+    "read_only_users_comment": {
+      "deny": "Deny (Prohibit reead only users from comment management)",
+      "accept": "Allow (Read only users can manage comments)"
     },
     "registration_mode": {
       "open": "Open (Anyone can register)",

apps/app/public/static/locales/en_US/translation.json

@@ -491,7 +491,10 @@
     "placeholder": "Ask me anything.",
     "caution_against_hallucination": "Please verify the information and check the sources.",
     "progress_label": "Generating answers",
-    "failed_to_create_or_retrieve_thread": "Failed to create or retrieve thread"
+    "failed_to_create_or_retrieve_thread": "Failed to create or retrieve thread",
+    "rate_limit_exceeded": "You have reached your usage limit for OpenAI's API. To use the Knowledge Assistant again, please add credits from the OpenAI billing page.",
+    "rate_limit_exceeded_for_growi_cloud": "You have reached your OpenAI API usage limit. To use the Knowledge Assistant again, please add credits from the GROWI.cloud admin page for Hosted users or from the OpenAI billing page for Owned users."
+
   },
   "link_edit": {
     "edit_link": "Edit Link",

apps/app/public/static/locales/fr_FR/admin.json

@@ -15,7 +15,7 @@
     "scope_of_page_disclosure": "Confidentialité de la page",
     "set_point": "Valeur",
     "Guest Users Access": "Accès invité",
-    "readonly_users_access": "Accès des utilisateurs ROM",
+    "readonly_users_access": "Accès des utilisateurs lecture seule",
     "always_hidden": "Toujours caché",
     "always_displayed": "Toujours affiché",
     "displayed_or_hidden": "Caché / Affiché",
@@ -87,9 +87,9 @@
       "deny": "Refuser (Utilisateurs inscrits seulement)",
       "readonly": "Autoriser (Lecture seule)"
     },
-    "rom_users_comment": {
-      "deny": "Refuser (Interdire la gestion des commentaires aux utilisateurs ROM)",
-      "accept": "Autoriser (Les utilisateurs ROM peuvent gérer les commentaires)"
+    "read_only_users_comment": {
+      "deny": "Refuser (Interdire la gestion des commentaires aux utilisateurs lecture seule)",
+      "accept": "Autoriser (Les utilisateurs lecture seule peuvent gérer les commentaires)"
     },
     "registration_mode": {
       "open": "Ouvert (Tout le monde peut s'inscrire)",

apps/app/public/static/locales/fr_FR/translation.json

@@ -485,7 +485,9 @@
     "placeholder": "Demandez-moi n'importe quoi.",
     "caution_against_hallucination": "Veuillez vérifier les informations et consulter les sources.",
     "progress_label": "Génération des réponses",
-    "failed_to_create_or_retrieve_thread": "Échec de la création ou de la récupération du fil de discussion"
+    "failed_to_create_or_retrieve_thread": "Échec de la création ou de la récupération du fil de discussion",
+    "rate_limit_exceeded": "Vous avez atteint votre limite d'utilisation de l'API de l'OpenAI. Pour utiliser à nouveau l'assistant de connaissance, veuillez ajouter des crédits à partir de la page de facturation d'OpenAI.",
+    "rate_limit_exceeded_for_growi_cloud": "Vous avez atteint votre limite d'utilisation de l'API de l'OpenAI. Pour utiliser à nouveau l'assistant de connaissance, veuillez ajouter des crédits à partir de la page d'administration de GROWI.cloud pour les utilisateurs hébergés ou à partir de la page de facturation de l'OpenAI pour les utilisateurs propriétaires."
   },
   "link_edit": {
     "edit_link": "Modifier lien",

apps/app/public/static/locales/ja_JP/admin.json

@@ -24,7 +24,7 @@
     "scope_of_page_disclosure": "ページの公開範囲",
     "set_point": "設定値",
     "Guest Users Access":"ゲストユーザーのアクセス",
-    "readonly_users_access": "ROMユーザーのアクセス",
+    "readonly_users_access": "閲覧のみユーザーのアクセス",
     "always_hidden": "非表示 (固定)",
     "always_displayed": "表示 (固定)",
     "displayed_or_hidden": "非表示 / 表示",
@@ -96,9 +96,9 @@
       "deny": "拒否 (アカウントを持つユーザーのみ利用可能)",
       "readonly": "許可 (ゲストユーザーも閲覧のみ可能)"
     },
-    "rom_users_comment": {
-      "deny": "拒否 (ROMユーザーのコメント操作を禁止)",
-      "accept": "許可 (ROMユーザーもコメント操作可能)"
+    "read_only_users_comment": {
+      "deny": "拒否 (閲覧のみユーザーのコメント操作を禁止)",
+      "accept": "許可 (閲覧のみユーザーもコメント操作可能)"
     },
     "registration_mode": {
       "open": "公開 (だれでも登録可能)",

apps/app/public/static/locales/ja_JP/translation.json

@@ -524,7 +524,9 @@
     "placeholder": "ききたいことを入力してください",
     "caution_against_hallucination": "情報が正しいか出典を確認しましょう",
     "progress_label": "回答を生成しています",
-    "failed_to_create_or_retrieve_thread": "スレッドの作成または取得に失敗しました"
+    "failed_to_create_or_retrieve_thread": "スレッドの作成または取得に失敗しました",
+    "rate_limit_exceeded": "OpenAI の API の利用上限に達しました。ナレッジアシスタントを再度利用するには OpenAI の請求ページからクレジットを追加してください。",
+    "rate_limit_exceeded_for_growi_cloud": "OpenAI の API の利用上限に達しました。ナレッジアシスタントを再度利用するには Hosted の場合は GROWI.cloud の管理画面から Owned の場合は OpenAI の請求ページからクレジットを追加してください。"
   },
   "link_edit": {
     "edit_link": "リンク編集",

apps/app/public/static/locales/zh_CN/admin.json

@@ -27,7 +27,7 @@
     "always_hidden": "总是隐藏",
     "displayed_or_hidden": "隐藏 / 显示",
     "Guest Users Access": "来宾用户访问",
-    "readonly_users_access": "ROM用户的访问",
+    "readonly_users_access": "只浏览用户的访问",
 		"Fixed by env var": "这是由env var<code>%s=%s</code>修复的。",
 		"register_limitation": "注册限制",
 		"register_limitation_desc": "限制新用户注册",
@@ -96,9 +96,9 @@
 			"deny": "拒绝（仅限注册用户）",
 			"readonly": "接受（来宾可以只读）"
 		},
-    "rom_users_comment": {
-      "deny": "拒绝 (禁止ROM用户操作评论)",
-      "accept": "允许 (ROM用户可以管理评论)"
+    "read_only_users_comment": {
+      "deny": "拒绝 (禁止只浏览用户操作评论)",
+      "accept": "允许 (只浏览用户可以管理评论)"
     },
 		"registration_mode": {
 			"open": "打开（任何人都可以注册）",

apps/app/public/static/locales/zh_CN/translation.json

@@ -480,7 +480,9 @@
     "placeholder": "问我任何问题。",
     "caution_against_hallucination": "请核实信息并检查来源。",
     "progress_label": "生成答案中",
-    "failed_to_create_or_retrieve_thread": "创建或获取线程失败"
+    "failed_to_create_or_retrieve_thread": "创建或获取线程失败",
+    "rate_limit_exceeded": "您已达到 OpenAI API 的使用上限。要再次使用知识助手，请从 OpenAI 账单页面添加点数。",
+    "rate_limit_exceeded_for_growi_cloud": "您已达到 OpenAI API 使用上限。如需再次使用知识助手，请从GROWI.cloud管理页面为托管用户添加点数，或从OpenAI计费页面为自有用户添加点数。"
   },
   "link_edit": {
     "edit_link": "Edit Link",

apps/app/src/client/components/Admin/Security/SecuritySetting.jsx

@@ -526,16 +526,16 @@ class SecuritySetting extends React.Component {
                 aria-expanded="true"
               >
                 <span className="float-start">
-                  {isRomUserAllowedToComment === true && t('security_settings.rom_users_comment.accept')}
-                  {isRomUserAllowedToComment === false && t('security_settings.rom_users_comment.deny')}
+                  {isRomUserAllowedToComment === true && t('security_settings.read_only_users_comment.accept')}
+                  {isRomUserAllowedToComment === false && t('security_settings.read_only_users_comment.deny')}
                 </span>
               </button>
               <div className="dropdown-menu" aria-labelledby="dropdownMenuButton">
                 <button className="dropdown-item" type="button" onClick={() => { adminGeneralSecurityContainer.switchIsRomUserAllowedToComment(false) }}>
-                  {t('security_settings.rom_users_comment.deny')}
+                  {t('security_settings.read_only_users_comment.deny')}
                 </button>
                 <button className="dropdown-item" type="button" onClick={() => { adminGeneralSecurityContainer.switchIsRomUserAllowedToComment(true) }}>
-                  {t('security_settings.rom_users_comment.accept')}
+                  {t('security_settings.read_only_users_comment.accept')}
                 </button>
               </div>
             </div>

apps/app/src/components/ReactMarkdownComponents/CodeBlock.tsx

@@ -13,6 +13,17 @@ Object.entries<object>(oneDark).forEach(([key, value]) => {
 });
 
 
+type InlineCodeBlockProps = {
+  children: ReactNode,
+  className?: string,
+}
+
+const InlineCodeBlockSubstance = (props: InlineCodeBlockProps): JSX.Element => {
+  const { children, className, ...rest } = props;
+  return <code className={`code-inline ${className ?? ''}`} {...rest}>{children}</code>;
+};
+
+
 function extractChildrenToIgnoreReactNode(children: ReactNode): ReactNode {
 
   if (children == null) {
@@ -70,15 +81,15 @@ function CodeBlockSubstance({ lang, children }: { lang: string, children: ReactN
 type CodeBlockProps = {
   children: ReactNode,
   className?: string,
-  inline?: string, // "" or undefined
+  inline?: true,
 }
 
 export const CodeBlock = (props: CodeBlockProps): JSX.Element => {
 
   // TODO: set border according to the value of 'customize:highlightJsStyleBorder'
   const { className, children, inline } = props;
-  if (inline != null) {
-    return <code className={`code-inline ${className ?? ''}`}>{children}</code>;
+  if (inline) {
+    return <InlineCodeBlockSubstance className={`code-inline ${className ?? ''}`}>{children}</InlineCodeBlockSubstance>;
   }
 
   const match = /language-(\w+)(:?.+)?/.exec(className || '');

apps/app/src/features/callout/components/CalloutViewer.module.scss

@@ -1,24 +1,12 @@
 @use '@growi/core-styles/scss/bootstrap/init' as bs;
 
 // == Colors
-@include bs.color-mode(light) {
-  .callout-viewer {
-    --callout-accent-note: hsl(212, 92%, 45%);
-    --callout-accent-tip: hsl(137, 66%, 30%);
-    --callout-accent-important: hsl(261, 69%, 59%);
-    --callout-accent-warning: hsl(40, 100%, 30%);
-    --callout-accent-caution: hsl(356, 71%, 48%);
-  }
-}
-
-@include bs.color-mode(dark) {
-  .callout-viewer {
-    --callout-accent-note: hsl(215, 93%, 58%);
-    --callout-accent-tip: hsl(128, 49%, 49%);
-    --callout-accent-important: hsl(262, 89%, 71%);
-    --callout-accent-warning: hsl(41, 72%, 48%);
-    --callout-accent-caution: hsl(3, 93%, 63%);
-  }
+.callout-viewer {
+  --callout-accent-note: var(--bs-info);
+  --callout-accent-tip: var(--bs-success);
+  --callout-accent-important: var(--bs-primary);
+  --callout-accent-warning: var(--bs-warning);
+  --callout-accent-caution: var(--bs-danger);
 }
 
 .callout-viewer :global{

apps/app/src/features/openai/chat/components/AiChatModal/AiChatModal.tsx

@@ -9,10 +9,11 @@ import {
 
 import { apiv3Post } from '~/client/util/apiv3-client';
 import { toastError } from '~/client/util/toastr';
+import { useGrowiCloudUri } from '~/stores-universal/context';
 import loggerFactory from '~/utils/logger';
 
 import { useRagSearchModal } from '../../../client/stores/rag-search';
-import { MessageErrorCode } from '../../../interfaces/message-error';
+import { MessageErrorCode, StreamErrorCode } from '../../../interfaces/message-error';
 
 import { MessageCard } from './MessageCard';
 import { ResizableTextarea } from './ResizableTextArea';
@@ -48,6 +49,8 @@ const AiChatModalSubstance = (): JSX.Element => {
   const [messageLogs, setMessageLogs] = useState<Message[]>([]);
   const [generatingAnswerMessage, setGeneratingAnswerMessage] = useState<Message>();
 
+  const { data: growiCloudUri } = useGrowiCloudUri();
+
   const isGenerating = generatingAnswerMessage != null;
 
   useEffect(() => {
@@ -141,14 +144,28 @@ const AiChatModalSubstance = (): JSX.Element => {
 
         const chunk = decoder.decode(value);
 
-        // Extract text values from the chunk
-        const textValues = chunk
-          .split('\n\n')
-          .filter(line => line.trim().startsWith('data:'))
-          .map((line) => {
+        const textValues: string[] = [];
+        const lines = chunk.split('\n\n');
+        lines.forEach((line) => {
+          const trimedLine = line.trim();
+          if (trimedLine.startsWith('data:')) {
             const data = JSON.parse(line.replace('data: ', ''));
-            return data.content[0].text.value;
-          });
+            textValues.push(data.content[0].text.value);
+          }
+          else if (trimedLine.startsWith('error:')) {
+            const error = JSON.parse(line.replace('error: ', ''));
+            logger.error(error.errorMessage);
+            form.setError('input', { type: 'manual', message: error.message });
+
+            if (error.code === StreamErrorCode.RATE_LIMIT_EXCEEDED) {
+              const toastErrorMessage = growiCloudUri != null
+                ? 'modal_aichat.rate_limit_exceeded_for_growi_cloud'
+                : 'modal_aichat.rate_limit_exceeded';
+              toastError(t(toastErrorMessage));
+            }
+          }
+        });
+
 
         // append text values to the assistant message
         setGeneratingAnswerMessage((prevMessage) => {
@@ -168,7 +185,7 @@ const AiChatModalSubstance = (): JSX.Element => {
       form.setError('input', { type: 'manual', message: err.toString() });
     }
 
-  }, [form, isGenerating, messageLogs, t, threadId]);
+  }, [form, growiCloudUri, isGenerating, messageLogs, t, threadId]);
 
   const keyDownHandler = (event: KeyboardEvent<HTMLTextAreaElement>) => {
     if (event.key === 'Enter' && (event.ctrlKey || event.metaKey)) {

apps/app/src/features/openai/interfaces/message-error.ts

@@ -1,3 +1,9 @@
 export const MessageErrorCode = {
   THREAD_ID_IS_NOT_SET: 'thread-id-is-not-set',
 } as const;
+
+export const StreamErrorCode = {
+  RATE_LIMIT_EXCEEDED: 'rate_limit_exceeded',
+} as const;
+
+export type StreamErrorCode = typeof StreamErrorCode[keyof typeof StreamErrorCode];

apps/app/src/features/openai/server/models/thread-relation.ts

@@ -0,0 +1,57 @@
+import type mongoose from 'mongoose';
+import { type Model, type Document, Schema } from 'mongoose';
+
+import { getOrCreateModel } from '~/server/util/mongoose-utils';
+
+const DAYS_UNTIL_EXPIRATION = 30;
+
+const generateExpirationDate = (): Date => {
+  const currentDate = new Date();
+  const expirationDate = new Date(currentDate.setDate(currentDate.getDate() + DAYS_UNTIL_EXPIRATION));
+  return expirationDate;
+};
+
+interface ThreadRelation {
+  userId: mongoose.Types.ObjectId;
+  threadId: string;
+  expiredAt: Date;
+}
+
+interface ThreadRelationDocument extends ThreadRelation, Document {
+  updateThreadExpiration(): Promise<void>;
+}
+
+interface ThreadRelationModel extends Model<ThreadRelationDocument> {
+  getExpiredThreadRelations(limit?: number): Promise<ThreadRelationDocument[] | undefined>;
+}
+
+const schema = new Schema<ThreadRelationDocument, ThreadRelationModel>({
+  userId: {
+    type: Schema.Types.ObjectId,
+    ref: 'User',
+    required: true,
+  },
+  threadId: {
+    type: String,
+    required: true,
+    unique: true,
+  },
+  expiredAt: {
+    type: Date,
+    default: generateExpirationDate,
+    required: true,
+  },
+});
+
+schema.statics.getExpiredThreadRelations = async function(limit?: number): Promise<ThreadRelationDocument[] | undefined> {
+  const currentDate = new Date();
+  const expiredThreadRelations = await this.find({ expiredAt: { $lte: currentDate } }).limit(limit ?? 100).exec();
+  return expiredThreadRelations;
+};
+
+schema.methods.updateThreadExpiration = async function(): Promise<void> {
+  this.expiredAt = generateExpirationDate();
+  await this.save();
+};
+
+export default getOrCreateModel<ThreadRelationDocument, ThreadRelationModel>('ThreadRelation', schema);

apps/app/src/features/openai/server/routes/message.ts

@@ -11,7 +11,7 @@ import { apiV3FormValidator } from '~/server/middlewares/apiv3-form-validator';
 import type { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response';
 import loggerFactory from '~/utils/logger';
 
-import { MessageErrorCode } from '../../interfaces/message-error';
+import { MessageErrorCode, StreamErrorCode } from '../../interfaces/message-error';
 import { openaiClient } from '../services';
 
 import { certifyAiService } from './middlewares/certify-ai-service';
@@ -80,6 +80,18 @@ export const postMessageHandlersFactory: PostMessageHandlersFactory = (crowi) =>
         res.write(`data: ${JSON.stringify(delta)}\n\n`);
       };
 
+      const sendError = (code: StreamErrorCode, message: string) => {
+        res.write(`error: ${JSON.stringify({ code, message })}\n\n`);
+      };
+
+      stream.on('event', (delta) => {
+        if (delta.event === 'thread.run.failed') {
+          if (delta.data.last_error?.code === StreamErrorCode.RATE_LIMIT_EXCEEDED) {
+            logger.error(delta.data.last_error.message);
+            sendError(StreamErrorCode.RATE_LIMIT_EXCEEDED, delta.data.last_error.message);
+          }
+        }
+      });
       stream.on('messageDelta', messageDeltaHandler);
       stream.once('messageDone', () => {
         stream.off('messageDelta', messageDeltaHandler);

apps/app/src/features/openai/server/routes/thread.ts

@@ -1,23 +1,21 @@
+import type { IUserHasId } from '@growi/core/dist/interfaces';
 import type { Request, RequestHandler } from 'express';
 import type { ValidationChain } from 'express-validator';
 import { body } from 'express-validator';
+import { filterXSS } from 'xss';
 
 import type Crowi from '~/server/crowi';
 import { apiV3FormValidator } from '~/server/middlewares/apiv3-form-validator';
 import type { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response';
 import loggerFactory from '~/utils/logger';
 
-import { openaiClient } from '../services';
 import { getOpenaiService } from '../services/openai';
 
 import { certifyAiService } from './middlewares/certify-ai-service';
 
 const logger = loggerFactory('growi:routes:apiv3:openai:thread');
 
-type CreateThreadReq = Request<undefined, ApiV3Response, {
-  userMessage: string,
-  threadId?: string,
-}>
+type CreateThreadReq = Request<undefined, ApiV3Response, { threadId?: string }> & { user: IUserHasId };
 
 type CreateThreadFactory = (crowi: Crowi) => RequestHandler[];
 
@@ -32,24 +30,11 @@ export const createThreadHandlersFactory: CreateThreadFactory = (crowi) => {
   return [
     accessTokenParser, loginRequiredStrictly, certifyAiService, validator, apiV3FormValidator,
     async(req: CreateThreadReq, res: ApiV3Response) => {
-      const openaiService = getOpenaiService();
-      if (openaiService == null) {
-        return res.apiv3Err('OpenaiService is not available', 503);
-      }
-
       try {
-        const vectorStore = await openaiService.getOrCreateVectorStoreForPublicScope();
-        const threadId = req.body.threadId;
-        const thread = threadId == null
-          ? await openaiClient.beta.threads.create({
-            tool_resources: {
-              file_search: {
-                vector_store_ids: [vectorStore.vectorStoreId],
-              },
-            },
-          })
-          : await openaiClient.beta.threads.retrieve(threadId);
-
+        const openaiService = getOpenaiService();
+        const filterdThreadId = req.body.threadId != null ? filterXSS(req.body.threadId) : undefined;
+        const vectorStore = await openaiService?.getOrCreateVectorStoreForPublicScope();
+        const thread = await openaiService?.getOrCreateThread(req.user._id, vectorStore?.vectorStoreId, filterdThreadId);
         return res.apiv3({ thread });
       }
       catch (err) {

apps/app/src/features/openai/server/services/client-delegator/azure-openai-client-delegator.ts

@@ -22,6 +22,24 @@ export class AzureOpenaiClientDelegator implements IOpenaiClientDelegator {
     // TODO: initialize openaiVectorStoreId property
   }
 
+  async createThread(vectorStoreId: string): Promise<OpenAI.Beta.Threads.Thread> {
+    return this.client.beta.threads.create({
+      tool_resources: {
+        file_search: {
+          vector_store_ids: [vectorStoreId],
+        },
+      },
+    });
+  }
+
+  async retrieveThread(threadId: string): Promise<OpenAI.Beta.Threads.Thread> {
+    return this.client.beta.threads.retrieve(threadId);
+  }
+
+  async deleteThread(threadId: string): Promise<OpenAI.Beta.Threads.ThreadDeleted> {
+    return this.client.beta.threads.del(threadId);
+  }
+
   async createVectorStore(scopeType:VectorStoreScopeType): Promise<OpenAI.Beta.VectorStores.VectorStore> {
     return this.client.beta.vectorStores.create({ name: `growi-vector-store-{${scopeType}` });
   }

apps/app/src/features/openai/server/services/client-delegator/interfaces.ts

@@ -4,6 +4,9 @@ import type { Uploadable } from 'openai/uploads';
 import type { VectorStoreScopeType } from '~/features/openai/server/models/vector-store';
 
 export interface IOpenaiClientDelegator {
+  createThread(vectorStoreId: string): Promise<OpenAI.Beta.Threads.Thread>
+  retrieveThread(threadId: string): Promise<OpenAI.Beta.Threads.Thread>
+  deleteThread(threadId: string): Promise<OpenAI.Beta.Threads.ThreadDeleted>
   retrieveVectorStore(vectorStoreId: string): Promise<OpenAI.Beta.VectorStores.VectorStore>
   createVectorStore(scopeType:VectorStoreScopeType): Promise<OpenAI.Beta.VectorStores.VectorStore>
   uploadFile(file: Uploadable): Promise<OpenAI.Files.FileObject>

apps/app/src/features/openai/server/services/client-delegator/openai-client-delegator.ts

@@ -24,6 +24,24 @@ export class OpenaiClientDelegator implements IOpenaiClientDelegator {
     this.client = new OpenAI({ apiKey });
   }
 
+  async createThread(vectorStoreId: string): Promise<OpenAI.Beta.Threads.Thread> {
+    return this.client.beta.threads.create({
+      tool_resources: {
+        file_search: {
+          vector_store_ids: [vectorStoreId],
+        },
+      },
+    });
+  }
+
+  async retrieveThread(threadId: string): Promise<OpenAI.Beta.Threads.Thread> {
+    return this.client.beta.threads.retrieve(threadId);
+  }
+
+  async deleteThread(threadId: string): Promise<OpenAI.Beta.Threads.ThreadDeleted> {
+    return this.client.beta.threads.del(threadId);
+  }
+
   async createVectorStore(scopeType:VectorStoreScopeType): Promise<OpenAI.Beta.VectorStores.VectorStore> {
     return this.client.beta.vectorStores.create({ name: `growi-vector-store-${scopeType}` });
   }

apps/app/src/features/openai/server/services/openai-api-error-handler.ts

@@ -0,0 +1,29 @@
+import OpenAI from 'openai';
+
+import loggerFactory from '~/utils/logger';
+
+const logger = loggerFactory('growi:service:openai');
+
+// Error Code Reference
+// https://platform.openai.com/docs/guides/error-codes/api-errors
+
+// Error Handling Reference
+// https://github.com/openai/openai-node/tree/d08bf1a8fa779e6a9349d92ddf65530dd84e686d?tab=readme-ov-file#handling-errors
+
+type ErrorHandler = {
+  notFoundError?: () => Promise<void>;
+}
+
+export const oepnaiApiErrorHandler = async(error: unknown, handler: ErrorHandler): Promise<void> => {
+  if (!(error instanceof OpenAI.APIError)) {
+    return;
+  }
+
+  logger.error(error);
+
+  if (error.status === 404 && handler.notFoundError != null) {
+    await handler.notFoundError();
+    return;
+  }
+
+};

apps/app/src/features/openai/server/services/openai.ts

@@ -7,6 +7,7 @@ import mongoose from 'mongoose';
 import type OpenAI from 'openai';
 import { toFile } from 'openai';
 
+import ThreadRelationModel from '~/features/openai/server/models/thread-relation';
 import VectorStoreModel, { VectorStoreScopeType, type VectorStoreDocument } from '~/features/openai/server/models/vector-store';
 import VectorStoreFileRelationModel, {
   type VectorStoreFileRelation,
@@ -19,8 +20,8 @@ import loggerFactory from '~/utils/logger';
 
 import { OpenaiServiceTypes } from '../../interfaces/ai';
 
-
 import { getClient } from './client-delegator';
+import { oepnaiApiErrorHandler } from './openai-api-error-handler';
 
 const BATCH_SIZE = 100;
 
@@ -29,7 +30,9 @@ const logger = loggerFactory('growi:service:openai');
 let isVectorStoreForPublicScopeExist = false;
 
 export interface IOpenaiService {
+  getOrCreateThread(userId: string, vectorStoreId?: string, threadId?: string): Promise<OpenAI.Beta.Threads.Thread | undefined>;
   getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument>;
+  deleteExpiredThreads(limit: number, apiCallInterval: number): Promise<void>;
   createVectorStoreFile(pages: PageDocument[]): Promise<void>;
   deleteVectorStoreFile(pageId: Types.ObjectId): Promise<void>;
   rebuildVectorStoreAll(): Promise<void>;
@@ -42,6 +45,63 @@ class OpenaiService implements IOpenaiService {
     return getClient({ openaiServiceType });
   }
 
+  public async getOrCreateThread(userId: string, vectorStoreId?: string, threadId?: string): Promise<OpenAI.Beta.Threads.Thread> {
+    if (vectorStoreId != null && threadId == null) {
+      try {
+        const thread = await this.client.createThread(vectorStoreId);
+        await ThreadRelationModel.create({ userId, threadId: thread.id });
+        return thread;
+      }
+      catch (err) {
+        throw new Error(err);
+      }
+    }
+
+    const threadRelation = await ThreadRelationModel.findOne({ threadId });
+    if (threadRelation == null) {
+      throw new Error('ThreadRelation document is not exists');
+    }
+
+    // Check if a thread entity exists
+    // If the thread entity does not exist, the thread-relation document is deleted
+    try {
+      const thread = await this.client.retrieveThread(threadRelation.threadId);
+
+      // Update expiration date if thread entity exists
+      await threadRelation.updateThreadExpiration();
+
+      return thread;
+    }
+    catch (err) {
+      await oepnaiApiErrorHandler(err, { notFoundError: async() => { await threadRelation.remove() } });
+      throw new Error(err);
+    }
+  }
+
+  public async deleteExpiredThreads(limit: number, apiCallInterval: number): Promise<void> {
+    const expiredThreadRelations = await ThreadRelationModel.getExpiredThreadRelations(limit);
+    if (expiredThreadRelations == null) {
+      return;
+    }
+
+    const deletedThreadIds: string[] = [];
+    for await (const expiredThreadRelation of expiredThreadRelations) {
+      try {
+        const deleteThreadResponse = await this.client.deleteThread(expiredThreadRelation.threadId);
+        logger.debug('Delete thread', deleteThreadResponse);
+        deletedThreadIds.push(expiredThreadRelation.threadId);
+
+        // sleep
+        await new Promise(resolve => setTimeout(resolve, apiCallInterval));
+      }
+      catch (err) {
+        logger.error(err);
+      }
+    }
+
+    await ThreadRelationModel.deleteMany({ threadId: { $in: deletedThreadIds } });
+  }
+
   public async getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument> {
     const vectorStoreDocument = await VectorStoreModel.findOne({ scorpeType: VectorStoreScopeType.PUBLIC });
 
@@ -50,11 +110,17 @@ class OpenaiService implements IOpenaiService {
     }
 
     if (vectorStoreDocument != null && !isVectorStoreForPublicScopeExist) {
-      const vectorStore = await this.client.retrieveVectorStore(vectorStoreDocument.vectorStoreId);
-      if (vectorStore != null) {
+      try {
+        // Check if vector store entity exists
+        // If the vector store entity does not exist, the vector store document is deleted
+        await this.client.retrieveVectorStore(vectorStoreDocument.vectorStoreId);
         isVectorStoreForPublicScopeExist = true;
         return vectorStoreDocument;
       }
+      catch (err) {
+        await oepnaiApiErrorHandler(err, { notFoundError: async() => { await vectorStoreDocument.remove() } });
+        throw new Error(err);
+      }
     }
 
     const newVectorStore = await this.client.createVectorStore(VectorStoreScopeType.PUBLIC);
@@ -74,7 +140,7 @@ class OpenaiService implements IOpenaiService {
     return uploadedFile;
   }
 
-  async createVectorStoreFile(pages: Array<PageDocument>): Promise<void> {
+  async createVectorStoreFile(pages: Array<HydratedDocument<PageDocument>>): Promise<void> {
     const vectorStoreFileRelationsMap: Map<string, VectorStoreFileRelation> = new Map();
     const processUploadFile = async(page: PageDocument) => {
       if (page._id != null && page.grant === PageGrant.GRANT_PUBLIC && page.revision != null) {
@@ -112,22 +178,22 @@ class OpenaiService implements IOpenaiService {
     }
 
     try {
+      // Save vector store file relation
+      await VectorStoreFileRelationModel.upsertVectorStoreFileRelations(vectorStoreFileRelations);
+
       // Create vector store file
       const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
       const createVectorStoreFileBatchResponse = await this.client.createVectorStoreFileBatch(vectorStore.vectorStoreId, uploadedFileIds);
       logger.debug('Create vector store file', createVectorStoreFileBatchResponse);
-
-      // Save vector store file relation
-      await VectorStoreFileRelationModel.upsertVectorStoreFileRelations(vectorStoreFileRelations);
     }
     catch (err) {
       logger.error(err);
 
       // Delete all uploaded files if createVectorStoreFileBatch fails
-      uploadedFileIds.forEach(async(fileId) => {
-        const deleteFileResponse = await this.client.deleteFile(fileId);
-        logger.debug('Delete vector store file (Due to createVectorStoreFileBatch failure)', deleteFileResponse);
-      });
+      const pageIds = pages.map(page => page._id);
+      for await (const pageId of pageIds) {
+        await this.deleteVectorStoreFile(pageId);
+      }
     }
 
   }
@@ -140,9 +206,8 @@ class OpenaiService implements IOpenaiService {
     }
 
     const deletedFileIds: string[] = [];
-    for (const fileId of vectorStoreFileRelation.fileIds) {
+    for await (const fileId of vectorStoreFileRelation.fileIds) {
       try {
-        // eslint-disable-next-line no-await-in-loop
         const deleteFileResponse = await this.client.deleteFile(fileId);
         logger.debug('Delete vector store file', deleteFileResponse);
         deletedFileIds.push(fileId);
@@ -174,7 +239,7 @@ class OpenaiService implements IOpenaiService {
     const createVectorStoreFile = this.createVectorStoreFile.bind(this);
     const createVectorStoreFileStream = new Transform({
       objectMode: true,
-      async transform(chunk: PageDocument[], encoding, callback) {
+      async transform(chunk: HydratedDocument<PageDocument>[], encoding, callback) {
         await createVectorStoreFile(chunk);
         this.push(chunk);
         callback();

apps/app/src/features/openai/server/services/thread-deletion-cron.ts

@@ -0,0 +1,61 @@
+import nodeCron from 'node-cron';
+
+import { configManager } from '~/server/service/config-manager';
+import loggerFactory from '~/utils/logger';
+
+import { getOpenaiService, type IOpenaiService } from './openai';
+
+const logger = loggerFactory('growi:service:thread-deletion-cron');
+
+class ThreadDeletionCronService {
+
+  cronJob: nodeCron.ScheduledTask;
+
+  openaiService: IOpenaiService;
+
+  threadDeletionCronExpression: string;
+
+  threadDeletionBarchSize: number;
+
+  threadDeletionApiCallInterval: number;
+
+  startCron(): void {
+    const isAiEnabled = configManager.getConfig('crowi', 'app:aiEnabled');
+    if (!isAiEnabled) {
+      return;
+    }
+
+    const openaiService = getOpenaiService();
+    if (openaiService == null) {
+      throw new Error('OpenAI service is not initialized');
+    }
+
+    this.openaiService = openaiService;
+    this.threadDeletionCronExpression = configManager.getConfig('crowi', 'openai:threadDeletionCronExpression');
+    this.threadDeletionBarchSize = configManager.getConfig('crowi', 'openai:threadDeletionBarchSize');
+    this.threadDeletionApiCallInterval = configManager.getConfig('crowi', 'openai:threadDeletionApiCallInterval');
+
+    this.cronJob?.stop();
+    this.cronJob = this.generateCronJob();
+    this.cronJob.start();
+  }
+
+  private async executeJob(): Promise<void> {
+    // Must be careful of OpenAI's rate limit
+    await this.openaiService.deleteExpiredThreads(this.threadDeletionBarchSize, this.threadDeletionApiCallInterval);
+  }
+
+  private generateCronJob() {
+    return nodeCron.schedule(this.threadDeletionCronExpression, async() => {
+      try {
+        await this.executeJob();
+      }
+      catch (e) {
+        logger.error(e);
+      }
+    });
+  }
+
+}
+
+export default ThreadDeletionCronService;

apps/app/src/server/crowi/index.js

@@ -12,6 +12,7 @@ import pkg from '^/package.json';
 
 import { KeycloakUserGroupSyncService } from '~/features/external-user-group/server/service/keycloak-user-group-sync';
 import { LdapUserGroupSyncService } from '~/features/external-user-group/server/service/ldap-user-group-sync';
+import OpenaiThreadDeletionCronService from '~/features/openai/server/services/thread-deletion-cron';
 import { PageBulkExportJobInProgressStatus } from '~/features/page-bulk-export/interfaces/page-bulk-export';
 import PageBulkExportJob from '~/features/page-bulk-export/server/models/page-bulk-export-job';
 import instanciatePageBulkExportService, { pageBulkExportService } from '~/features/page-bulk-export/server/service/page-bulk-export';
@@ -106,6 +107,7 @@ class Crowi {
     this.activityService = null;
     this.commentService = null;
     this.questionnaireService = null;
+    this.openaiThreadDeletionCronService = null;
 
     this.tokens = null;
 
@@ -321,6 +323,9 @@ Crowi.prototype.setupCron = function() {
 
   instanciatePageBulkExportJobCronService(this);
   pageBulkExportJobCronService.startCron();
+
+  this.openaiThreadDeletionCronService = new OpenaiThreadDeletionCronService();
+  this.openaiThreadDeletionCronService.startCron();
 };
 
 Crowi.prototype.setupQuestionnaireService = function() {

apps/app/src/server/service/config-loader.ts

@@ -825,6 +825,24 @@ const ENV_VAR_NAME_TO_CONFIG_INFO: Record<string, EnvConfig> = {
     type: ValueType.STRING,
     default: null,
   },
+  OPENAI_THREAD_DELETION_CRON_EXPRESSION: {
+    ns: 'crowi',
+    key: 'openai:threadDeletionCronExpression',
+    type: ValueType.STRING,
+    default: '0 * * * *', // every hour
+  },
+  OPENAI_THREAD_DELETION_BARCH_SIZE: {
+    ns: 'crowi',
+    key: 'openai:threadDeletionBarchSize',
+    type: ValueType.NUMBER,
+    default: 100,
+  },
+  OPENAI_THREAD_DELETION_API_CALL_INTERVAL: {
+    ns: 'crowi',
+    key: 'openai:threadDeletionApiCallInterval',
+    type: ValueType.NUMBER,
+    default: 36000, // msec
+  },
 };
 
 

apps/app/src/services/renderer/remark-plugins/codeblock.ts

@@ -10,7 +10,7 @@ export const remarkPlugin: Plugin = () => {
   return (tree) => {
     visit(tree, 'inlineCode', (node: InlineCode) => {
       const data = node.data || (node.data = {});
-      data.hProperties = { inline: true };
+      data.hProperties = { inline: 'true' }; // set 'true' explicitly because the empty string is evaluated as false for `if (inline) { ... }`
     });
   };
 };

apps/app/src/services/renderer/renderer.tsx

@@ -44,15 +44,18 @@ let commonSanitizeOption: SanitizeOption;
 export const getCommonSanitizeOption = (config:RendererConfig): SanitizeOption => {
   if (commonSanitizeOption == null || config.sanitizeType !== currentInitializedSanitizeType) {
     // initialize
-    commonSanitizeOption = {
-      tagNames: config.sanitizeType === RehypeSanitizeType.RECOMMENDED
-        ? recommendedTagNames
-        : config.customTagWhitelist ?? recommendedTagNames,
-      attributes: config.sanitizeType === RehypeSanitizeType.RECOMMENDED
-        ? recommendedAttributes
-        : config.customAttrWhitelist ?? recommendedAttributes,
-      clobberPrefix: '', // remove clobber prefix
-    };
+    commonSanitizeOption = deepmerge(
+      {
+        tagNames: config.sanitizeType === RehypeSanitizeType.RECOMMENDED
+          ? recommendedTagNames
+          : config.customTagWhitelist ?? recommendedTagNames,
+        attributes: config.sanitizeType === RehypeSanitizeType.RECOMMENDED
+          ? recommendedAttributes
+          : config.customAttrWhitelist ?? recommendedAttributes,
+        clobberPrefix: '', // remove clobber prefix
+      },
+      codeBlock.sanitizeOption,
+    );
 
     currentInitializedSanitizeType = config.sanitizeType;
   }
@@ -123,6 +126,7 @@ export const generateSSRViewOptions = (
     config: RendererConfig,
     pagePath: string,
 ): RendererOptions => {
+
   const options = generateCommonOptions(pagePath);
 
   const { remarkPlugins, rehypePlugins } = options;
@@ -140,9 +144,7 @@ export const generateSSRViewOptions = (
   }
 
   const rehypeSanitizePlugin: Pluggable | (() => void) = config.isEnabledXssPrevention
-    ? [sanitize, deepmerge(
-      getCommonSanitizeOption(config),
-    )]
+    ? [sanitize, getCommonSanitizeOption(config)]
     : () => {};
 
   // add rehype plugins

yarn.lock

@@ -4713,7 +4713,7 @@
   resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
   integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==
 
-"@types/node-cron@^3.0.2":
+"@types/node-cron@^3.0.11":
   version "3.0.11"
   resolved "https://registry.yarnpkg.com/@types/node-cron/-/node-cron-3.0.11.tgz#70b7131f65038ae63cfe841354c8aba363632344"
   integrity sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==