Merge branch 'master' into support/turn-off-markdown-splitter

apps/app/docker/Dockerfile

@@ -1,4 +1,4 @@
-# syntax = docker/dockerfile:1.4
+# syntax = docker/dockerfile:1
 
 
 ##
@@ -6,15 +6,22 @@
 ##
 FROM node:20-slim AS base
 
-ENV optDir /opt
+ENV optDir=/opt
 
 WORKDIR ${optDir}
 
+# install tools
+RUN apt-get update && apt-get install -y ca-certificates wget curl --no-install-recommends
+
+# install git and git-lfs
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash \
+  && apt-get update && apt-get install -y git git-lfs --no-install-recommends \
+  && git lfs install
+
 # install pnpm
-RUN apt-get update && apt-get install -y ca-certificates wget --no-install-recommends \
-  && wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.shrc" SHELL="$(which sh)" sh -
-ENV PNPM_HOME "/root/.local/share/pnpm"
-ENV PATH "$PNPM_HOME:$PATH"
+RUN wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.shrc" SHELL="$(which sh)" sh -
+ENV PNPM_HOME="/root/.local/share/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
 
 # install turbo
 RUN pnpm add turbo --global
@@ -26,7 +33,7 @@ RUN pnpm add turbo --global
 ##
 FROM base AS builder
 
-ENV optDir /opt
+ENV optDir=/opt
 
 WORKDIR ${optDir}
 
@@ -62,12 +69,12 @@ RUN tar -zcf packages.tar.gz \
 ## release
 ##
 FROM node:20-slim
-LABEL maintainer Yuki Takei <yuki@weseek.co.jp>
+LABEL maintainer="Yuki Takei <yuki@weseek.co.jp>"
 
-ENV NODE_ENV production
+ENV NODE_ENV="production"
 
-ENV optDir /opt
-ENV appDir ${optDir}/growi
+ENV optDir=/opt
+ENV appDir=${optDir}/growi
 
 # Add gosu
 # see: https://github.com/tianon/gosu/blob/1.13/INSTALL.md
@@ -83,7 +90,7 @@ RUN apt-get update && apt-get install -y sudo ca-certificates wget --no-install-
   && wget -qO- https://get.pnpm.io/install.sh | ENV="$HOME/.shrc" SHELL="$(which sh)" sudo -u node sh - \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
 ENV PNPM_HOME="/home/node/.local/share/pnpm"
-ENV PATH "$PNPM_HOME:$PATH"
+ENV PATH="$PNPM_HOME:$PATH"
 
 COPY --from=builder --chown=node:node \
   ${optDir}/packages.tar.gz ${appDir}/

apps/app/src/features/openai/server/models/vector-store-file-relation.ts

@@ -5,6 +5,7 @@ import { type Model, type Document, Schema } from 'mongoose';
 import { getOrCreateModel } from '~/server/util/mongoose-utils';
 
 export interface VectorStoreFileRelation {
+  vectorStoreRelationId: mongoose.Types.ObjectId;
   pageId: mongoose.Types.ObjectId;
   fileIds: string[];
   isAttachedToVectorStore: boolean;
@@ -18,7 +19,7 @@ interface VectorStoreFileRelationModel extends Model<VectorStoreFileRelation> {
 }
 
 export const prepareVectorStoreFileRelations = (
-    pageId: Types.ObjectId, fileId: string, relationsMap: Map<string, VectorStoreFileRelation>,
+    vectorStoreRelationId: Types.ObjectId, pageId: Types.ObjectId, fileId: string, relationsMap: Map<string, VectorStoreFileRelation>,
 ): Map<string, VectorStoreFileRelation> => {
   const pageIdStr = pageId.toHexString();
   const existingData = relationsMap.get(pageIdStr);
@@ -30,6 +31,7 @@ export const prepareVectorStoreFileRelations = (
   // If the data doesn't exist, create a new one and add it to the map
   else {
     relationsMap.set(pageIdStr, {
+      vectorStoreRelationId,
       pageId,
       fileIds: [fileId],
       isAttachedToVectorStore: false,
@@ -40,11 +42,15 @@ export const prepareVectorStoreFileRelations = (
 };
 
 const schema = new Schema<VectorStoreFileRelationDocument, VectorStoreFileRelationModel>({
+  vectorStoreRelationId: {
+    type: Schema.Types.ObjectId,
+    ref: 'VectorStore',
+    required: true,
+  },
   pageId: {
     type: Schema.Types.ObjectId,
     ref: 'Page',
     required: true,
-    unique: true,
   },
   fileIds: [{
     type: String,
@@ -57,13 +63,18 @@ const schema = new Schema<VectorStoreFileRelationDocument, VectorStoreFileRelati
   },
 });
 
+// define unique compound index
+schema.index({ vectorStoreRelationId: 1, pageId: 1 }, { unique: true });
+
 schema.statics.upsertVectorStoreFileRelations = async function(vectorStoreFileRelations: VectorStoreFileRelation[]): Promise<void> {
   await this.bulkWrite(
     vectorStoreFileRelations.map((data) => {
       return {
         updateOne: {
-          filter: { pageId: data.pageId },
-          update: { $addToSet: { fileIds: { $each: data.fileIds } } },
+          filter: { pageId: data.pageId, vectorStoreRelationId: data.vectorStoreRelationId },
+          update: {
+            $addToSet: { fileIds: { $each: data.fileIds } },
+          },
           upsert: true,
         },
       };

apps/app/src/features/openai/server/models/vector-store.ts

@@ -11,10 +11,13 @@ export type VectorStoreScopeType = typeof VectorStoreScopeType[keyof typeof Vect
 const VectorStoreScopeTypes = Object.values(VectorStoreScopeType);
 interface VectorStore {
   vectorStoreId: string
-  scorpeType: VectorStoreScopeType
+  scopeType: VectorStoreScopeType
+  isDeleted: boolean
 }
 
-export interface VectorStoreDocument extends VectorStore, Document {}
+export interface VectorStoreDocument extends VectorStore, Document {
+  markAsDeleted(): Promise<void>
+}
 
 type VectorStoreModel = Model<VectorStore>
 
@@ -24,11 +27,21 @@ const schema = new Schema<VectorStoreDocument, VectorStoreModel>({
     required: true,
     unique: true,
   },
-  scorpeType: {
+  scopeType: {
     enum: VectorStoreScopeTypes,
     type: String,
     required: true,
   },
+  isDeleted: {
+    type: Boolean,
+    default: false,
+    required: true,
+  },
 });
 
+schema.methods.markAsDeleted = async function(): Promise<void> {
+  this.isDeleted = true;
+  await this.save();
+};
+
 export default getOrCreateModel<VectorStoreDocument, VectorStoreModel>('VectorStore', schema);

apps/app/src/features/openai/server/services/client-delegator/azure-openai-client-delegator.ts

@@ -48,6 +48,10 @@ export class AzureOpenaiClientDelegator implements IOpenaiClientDelegator {
     return this.client.beta.vectorStores.retrieve(vectorStoreId);
   }
 
+  async deleteVectorStore(vectorStoreId: string): Promise<OpenAI.Beta.VectorStores.VectorStoreDeleted> {
+    return this.client.beta.vectorStores.del(vectorStoreId);
+  }
+
   async uploadFile(file: Uploadable): Promise<OpenAI.Files.FileObject> {
     return this.client.files.create({ file, purpose: 'assistants' });
   }

apps/app/src/features/openai/server/services/client-delegator/interfaces.ts

@@ -9,6 +9,7 @@ export interface IOpenaiClientDelegator {
   deleteThread(threadId: string): Promise<OpenAI.Beta.Threads.ThreadDeleted>
   retrieveVectorStore(vectorStoreId: string): Promise<OpenAI.Beta.VectorStores.VectorStore>
   createVectorStore(scopeType:VectorStoreScopeType): Promise<OpenAI.Beta.VectorStores.VectorStore>
+  deleteVectorStore(vectorStoreId: string): Promise<OpenAI.Beta.VectorStores.VectorStoreDeleted>
   uploadFile(file: Uploadable): Promise<OpenAI.Files.FileObject>
   createVectorStoreFileBatch(vectorStoreId: string, fileIds: string[]): Promise<OpenAI.Beta.VectorStores.FileBatches.VectorStoreFileBatch>
   deleteFile(fileId: string): Promise<OpenAI.Files.FileDeleted>;

apps/app/src/features/openai/server/services/client-delegator/openai-client-delegator.ts

@@ -50,6 +50,10 @@ export class OpenaiClientDelegator implements IOpenaiClientDelegator {
     return this.client.beta.vectorStores.retrieve(vectorStoreId);
   }
 
+  async deleteVectorStore(vectorStoreId: string): Promise<OpenAI.Beta.VectorStores.VectorStoreDeleted> {
+    return this.client.beta.vectorStores.del(vectorStoreId);
+  }
+
   async uploadFile(file: Uploadable): Promise<OpenAI.Files.FileObject> {
     return this.client.files.create({ file, purpose: 'assistants' });
   }

apps/app/src/features/openai/server/services/openai.ts

@@ -35,9 +35,11 @@ type VectorStoreFileRelationsMap = Map<string, VectorStoreFileRelation>
 export interface IOpenaiService {
   getOrCreateThread(userId: string, vectorStoreId?: string, threadId?: string): Promise<OpenAI.Beta.Threads.Thread | undefined>;
   getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument>;
-  deleteExpiredThreads(limit: number, apiCallInterval: number): Promise<void>;
+  deleteExpiredThreads(limit: number, apiCallInterval: number): Promise<void>; // for CronJob
+  deleteObsolatedVectorStoreRelations(): Promise<void> // for CronJob
   createVectorStoreFile(pages: PageDocument[]): Promise<void>;
-  deleteVectorStoreFile(pageId: Types.ObjectId): Promise<void>;
+  deleteVectorStoreFile(vectorStoreRelationId: Types.ObjectId, pageId: Types.ObjectId): Promise<void>;
+  deleteObsoleteVectorStoreFile(limit: number, apiCallInterval: number): Promise<void>; // for CronJob
   rebuildVectorStoreAll(): Promise<void>;
   rebuildVectorStore(page: HydratedDocument<PageDocument>): Promise<void>;
 }
@@ -106,7 +108,7 @@ class OpenaiService implements IOpenaiService {
   }
 
   public async getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument> {
-    const vectorStoreDocument = await VectorStoreModel.findOne({ scorpeType: VectorStoreScopeType.PUBLIC });
+    const vectorStoreDocument: VectorStoreDocument | null = await VectorStoreModel.findOne({ scopeType: VectorStoreScopeType.PUBLIC, isDeleted: false });
 
     if (vectorStoreDocument != null && isVectorStoreForPublicScopeExist) {
       return vectorStoreDocument;
@@ -121,7 +123,7 @@ class OpenaiService implements IOpenaiService {
         return vectorStoreDocument;
       }
       catch (err) {
-        await oepnaiApiErrorHandler(err, { notFoundError: async() => { await vectorStoreDocument.remove() } });
+        await oepnaiApiErrorHandler(err, { notFoundError: vectorStoreDocument.markAsDeleted });
         throw new Error(err);
       }
     }
@@ -129,8 +131,8 @@ class OpenaiService implements IOpenaiService {
     const newVectorStore = await this.client.createVectorStore(VectorStoreScopeType.PUBLIC);
     const newVectorStoreDocument = await VectorStoreModel.create({
       vectorStoreId: newVectorStore.id,
-      scorpeType: VectorStoreScopeType.PUBLIC,
-    });
+      scopeType: VectorStoreScopeType.PUBLIC,
+    }) as VectorStoreDocument;
 
     isVectorStoreForPublicScopeExist = true;
 
@@ -158,20 +160,37 @@ class OpenaiService implements IOpenaiService {
     return uploadedFile;
   }
 
+  private async deleteVectorStore(vectorStoreScopeType: VectorStoreScopeType): Promise<void> {
+    const vectorStoreDocument: VectorStoreDocument | null = await VectorStoreModel.findOne({ scopeType: vectorStoreScopeType, isDeleted: false });
+    if (vectorStoreDocument == null) {
+      return;
+    }
+
+    try {
+      await this.client.deleteVectorStore(vectorStoreDocument.vectorStoreId);
+      await vectorStoreDocument.markAsDeleted();
+    }
+    catch (err) {
+      await oepnaiApiErrorHandler(err, { notFoundError: vectorStoreDocument.markAsDeleted });
+      throw new Error(err);
+    }
+  }
+
   async createVectorStoreFile(pages: Array<HydratedDocument<PageDocument>>): Promise<void> {
+    const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
     const vectorStoreFileRelationsMap: VectorStoreFileRelationsMap = new Map();
     const processUploadFile = async(page: PageDocument) => {
       if (page._id != null && page.grant === PageGrant.GRANT_PUBLIC && page.revision != null) {
         if (isPopulated(page.revision) && page.revision.body.length > 0) {
           const uploadedFile = await this.uploadFile(page._id, page.revision.body);
-          prepareVectorStoreFileRelations(page._id, uploadedFile.id, vectorStoreFileRelationsMap);
+          prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
           return;
         }
 
         const pagePopulatedToShowRevision = await page.populateDataToShowRevision();
         if (pagePopulatedToShowRevision.revision != null && pagePopulatedToShowRevision.revision.body.length > 0) {
           const uploadedFile = await this.uploadFile(page._id, pagePopulatedToShowRevision.revision.body);
-          prepareVectorStoreFileRelations(page._id, uploadedFile.id, vectorStoreFileRelationsMap);
+          prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
         }
       }
     };
@@ -202,7 +221,6 @@ class OpenaiService implements IOpenaiService {
       await VectorStoreFileRelationModel.upsertVectorStoreFileRelations(vectorStoreFileRelations);
 
       // Create vector store file
-      const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
       const createVectorStoreFileBatchResponse = await this.client.createVectorStoreFileBatch(vectorStore.vectorStoreId, uploadedFileIds);
       logger.debug('Create vector store file', createVectorStoreFileBatchResponse);
 
@@ -214,15 +232,40 @@ class OpenaiService implements IOpenaiService {
 
       // Delete all uploaded files if createVectorStoreFileBatch fails
       for await (const pageId of pageIds) {
-        await this.deleteVectorStoreFile(pageId);
+        await this.deleteVectorStoreFile(vectorStore._id, pageId);
       }
     }
 
   }
 
-  async deleteVectorStoreFile(pageId: Types.ObjectId): Promise<void> {
+  // Deletes all VectorStore documents that are marked as deleted (isDeleted: true) and have no associated VectorStoreFileRelation documents
+  async deleteObsolatedVectorStoreRelations(): Promise<void> {
+    const deletedVectorStoreRelations = await VectorStoreModel.find({ isDeleted: true });
+    if (deletedVectorStoreRelations.length === 0) {
+      return;
+    }
+
+    const currentVectorStoreRelationIds: Types.ObjectId[] = await VectorStoreFileRelationModel.aggregate([
+      {
+        $group: {
+          _id: '$vectorStoreRelationId',
+          relationCount: { $sum: 1 },
+        },
+      },
+      { $match: { relationCount: { $gt: 0 } } },
+      { $project: { _id: 1 } },
+    ]);
+
+    if (currentVectorStoreRelationIds.length === 0) {
+      return;
+    }
+
+    await VectorStoreModel.deleteMany({ _id: { $nin: currentVectorStoreRelationIds }, isDeleted: true });
+  }
+
+  async deleteVectorStoreFile(vectorStoreRelationId: Types.ObjectId, pageId: Types.ObjectId, apiCallInterval?: number): Promise<void> {
     // Delete vector store file and delete vector store file relation
-    const vectorStoreFileRelation = await VectorStoreFileRelationModel.findOne({ pageId });
+    const vectorStoreFileRelation = await VectorStoreFileRelationModel.findOne({ vectorStoreRelationId, pageId });
     if (vectorStoreFileRelation == null) {
       return;
     }
@@ -233,8 +276,13 @@ class OpenaiService implements IOpenaiService {
         const deleteFileResponse = await this.client.deleteFile(fileId);
         logger.debug('Delete vector store file', deleteFileResponse);
         deletedFileIds.push(fileId);
+        if (apiCallInterval != null) {
+          // sleep
+          await new Promise(resolve => setTimeout(resolve, apiCallInterval));
+        }
       }
       catch (err) {
+        await oepnaiApiErrorHandler(err, { notFoundError: async() => { deletedFileIds.push(fileId) } });
         logger.error(err);
       }
     }
@@ -250,8 +298,34 @@ class OpenaiService implements IOpenaiService {
     await vectorStoreFileRelation.save();
   }
 
+  async deleteObsoleteVectorStoreFile(limit: number, apiCallInterval: number): Promise<void> {
+    // Retrieves all VectorStore documents that are marked as deleted
+    const deletedVectorStoreRelations = await VectorStoreModel.find({ isDeleted: true });
+    if (deletedVectorStoreRelations.length === 0) {
+      return;
+    }
+
+    // Retrieves VectorStoreFileRelation documents associated with deleted VectorStore documents
+    const obsoleteVectorStoreFileRelations = await VectorStoreFileRelationModel.find(
+      { vectorStoreRelationId: { $in: deletedVectorStoreRelations.map(deletedVectorStoreRelation => deletedVectorStoreRelation._id) } },
+    ).limit(limit);
+    if (obsoleteVectorStoreFileRelations.length === 0) {
+      return;
+    }
+
+    // Delete obsolete VectorStoreFile
+    for await (const vectorStoreFileRelation of obsoleteVectorStoreFileRelations) {
+      try {
+        await this.deleteVectorStoreFile(vectorStoreFileRelation.vectorStoreRelationId, vectorStoreFileRelation.pageId, apiCallInterval);
+      }
+      catch (err) {
+        logger.error(err);
+      }
+    }
+  }
+
   async rebuildVectorStoreAll() {
-    // TODO: https://redmine.weseek.co.jp/issues/154364
+    await this.deleteVectorStore(VectorStoreScopeType.PUBLIC);
 
     // Create all public pages VectorStoreFile
     const Page = mongoose.model<HydratedDocument<PageDocument>, PageModel>('Page');
@@ -274,7 +348,8 @@ class OpenaiService implements IOpenaiService {
   }
 
   async rebuildVectorStore(page: HydratedDocument<PageDocument>) {
-    await this.deleteVectorStoreFile(page._id);
+    const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
+    await this.deleteVectorStoreFile(vectorStore._id, page._id);
     await this.createVectorStoreFile([page]);
   }
 

apps/app/src/features/openai/server/services/vector-store-file-deletion-cron.ts

@@ -0,0 +1,61 @@
+import nodeCron from 'node-cron';
+
+import { configManager } from '~/server/service/config-manager';
+import loggerFactory from '~/utils/logger';
+
+import { getOpenaiService, type IOpenaiService } from './openai';
+
+const logger = loggerFactory('growi:service:vector-store-file-deletion-cron');
+
+class VectorStoreFileDeletionCronService {
+
+  cronJob: nodeCron.ScheduledTask;
+
+  openaiService: IOpenaiService;
+
+  vectorStoreFileDeletionCronExpression: string;
+
+  vectorStoreFileDeletionBarchSize: number;
+
+  vectorStoreFileDeletionApiCallInterval: number;
+
+  startCron(): void {
+    const isAiEnabled = configManager.getConfig('crowi', 'app:aiEnabled');
+    if (!isAiEnabled) {
+      return;
+    }
+
+    const openaiService = getOpenaiService();
+    if (openaiService == null) {
+      throw new Error('OpenAI service is not initialized');
+    }
+
+    this.openaiService = openaiService;
+    this.vectorStoreFileDeletionCronExpression = configManager.getConfig('crowi', 'openai:vectorStoreFileDeletionCronExpression');
+    this.vectorStoreFileDeletionBarchSize = configManager.getConfig('crowi', 'openai:vectorStoreFileDeletionBarchSize');
+    this.vectorStoreFileDeletionApiCallInterval = configManager.getConfig('crowi', 'openai:vectorStoreFileDeletionApiCallInterval');
+
+    this.cronJob?.stop();
+    this.cronJob = this.generateCronJob();
+    this.cronJob.start();
+  }
+
+  private async executeJob(): Promise<void> {
+    await this.openaiService.deleteObsolatedVectorStoreRelations();
+    await this.openaiService.deleteObsoleteVectorStoreFile(this.vectorStoreFileDeletionBarchSize, this.vectorStoreFileDeletionApiCallInterval);
+  }
+
+  private generateCronJob() {
+    return nodeCron.schedule(this.vectorStoreFileDeletionCronExpression, async() => {
+      try {
+        await this.executeJob();
+      }
+      catch (e) {
+        logger.error(e);
+      }
+    });
+  }
+
+}
+
+export default VectorStoreFileDeletionCronService;

apps/app/src/server/crowi/index.js

@@ -13,6 +13,7 @@ import pkg from '^/package.json';
 import { KeycloakUserGroupSyncService } from '~/features/external-user-group/server/service/keycloak-user-group-sync';
 import { LdapUserGroupSyncService } from '~/features/external-user-group/server/service/ldap-user-group-sync';
 import OpenaiThreadDeletionCronService from '~/features/openai/server/services/thread-deletion-cron';
+import OpenaiVectorStoreFileDeletionCronService from '~/features/openai/server/services/vector-store-file-deletion-cron';
 import QuestionnaireService from '~/features/questionnaire/server/service/questionnaire';
 import QuestionnaireCronService from '~/features/questionnaire/server/service/questionnaire-cron';
 import loggerFactory from '~/utils/logger';
@@ -113,6 +114,7 @@ class Crowi {
     this.questionnaireService = null;
     this.questionnaireCronService = null;
     this.openaiThreadDeletionCronService = null;
+    this.openaiVectorStoreFileDeletionCronService = null;
 
     this.tokens = null;
 
@@ -326,6 +328,9 @@ Crowi.prototype.setupCron = function() {
 
   this.openaiThreadDeletionCronService = new OpenaiThreadDeletionCronService();
   this.openaiThreadDeletionCronService.startCron();
+
+  this.openaiThreadDeletionCronService = new OpenaiVectorStoreFileDeletionCronService();
+  this.openaiThreadDeletionCronService.startCron();
 };
 
 Crowi.prototype.setupQuestionnaireService = function() {

apps/app/src/server/service/config-loader.ts

@@ -826,6 +826,24 @@ As this system is a Retrieval Augmented Generation (RAG), focus on answering que
     type: ValueType.NUMBER,
     default: 36000, // msec
   },
+  OPENAI_VECTOR_STORE_FILE_DELETION_CRON_EXPRESSION: {
+    ns: 'crowi',
+    key: 'openai:vectorStoreFileDeletionCronExpression',
+    type: ValueType.STRING,
+    default: '0 * * * *', // every hour
+  },
+  OPENAI_VECTOR_STORE_FILE_DELETION_BARCH_SIZE: {
+    ns: 'crowi',
+    key: 'openai:vectorStoreFileDeletionBarchSize',
+    type: ValueType.NUMBER,
+    default: 100,
+  },
+  OPENAI_VECTOR_STORE_FILE_DELETION_API_CALL_INTERVAL: {
+    ns: 'crowi',
+    key: 'openai:vectorStoreFileDeletionApiCallInterval',
+    type: ValueType.NUMBER,
+    default: 36000, // msec
+  },
 };
 
 

apps/app/src/server/service/page/index.ts

@@ -1902,8 +1902,11 @@ class PageService implements IPageService {
     ]);
 
     const openaiService = getOpenaiService();
-    const deleteVectorStoreFilePromises = pageIds.map(pageId => openaiService?.deleteVectorStoreFile(pageId));
-    await Promise.allSettled(deleteVectorStoreFilePromises);
+    if (openaiService != null) {
+      const vectorStore = await openaiService.getOrCreateVectorStoreForPublicScope();
+      const deleteVectorStoreFilePromises = pageIds.map(pageId => openaiService.deleteVectorStoreFile(vectorStore._id, pageId));
+      await Promise.allSettled(deleteVectorStoreFilePromises);
+    }
   }
 
   // delete multiple pages