Merge pull request #4560 from weseek/feat/page-migration-remove-unique-constraint

feat: page migration remove unique constraint

packages/app/src/client/services/AdminAppContainer.js

@@ -452,7 +452,7 @@ export default class AdminAppContainer extends Container {
   /**
    * Start v5 page migration
    * @memberOf AdminAppContainer
-   * @property action takes only 'upgrade' for now. 'upgrade' will start or resume migration
+   * @property action takes only 'initialMigration' for now. 'initialMigration' will start or resume migration
    */
   async v5PageMigrationHandler(action) {
     const response = await this.appContainer.apiv3.post('/pages/v5-schema-migration', { action });

packages/app/src/components/Admin/App/V5PageMigration.tsx

@@ -17,7 +17,7 @@ const V5PageMigration: FC<Props> = (props: Props) => {
   const onConfirm = async() => {
     setIsV5PageMigrationModalShown(false);
     try {
-      const { isV5Compatible } = await adminAppContainer.v5PageMigrationHandler('upgrade');
+      const { isV5Compatible } = await adminAppContainer.v5PageMigrationHandler('initialMigration');
       if (isV5Compatible) {
 
         return toastSuccess(t('admin:v5_page_migration.already_upgraded'));

packages/app/src/server/models/page.js

@@ -42,7 +42,7 @@ const pageSchema = new mongoose.Schema({
   },
   isEmpty: { type: Boolean, default: false },
   path: {
-    type: String, required: true, index: true, unique: true,
+    type: String, required: true,
   },
   revision: { type: ObjectId, ref: 'Revision' },
   redirectTo: { type: String, index: true },
@@ -71,6 +71,9 @@ const pageSchema = new mongoose.Schema({
 pageSchema.plugin(mongoosePaginate);
 pageSchema.plugin(uniqueValidator);
 
+// TODO: test this after modifying Page.create
+// ensure v4 compatibility using partial index
+pageSchema.index({ path: 1 }, { unique: true, partialFilterExpression: { parent: null } });
 
 /**
  * return an array of ancestors paths that is extracted from specified pagePath

packages/app/src/server/routes/apiv3/pages.js

@@ -684,30 +684,29 @@ module.exports = (crowi) => {
 
   });
 
-  // TODO: use socket conn to show progress
   router.post('/v5-schema-migration', accessTokenParser, loginRequired, adminRequired, csrf, validator.v5PageMigration, apiV3FormValidator, async(req, res) => {
     const { action } = req.body;
+    const isV5Compatible = crowi.configManager.getConfig('crowi', 'app:isV5Compatible');
 
-    switch (action) {
-      case 'upgrade':
-
-        try {
-          const Page = crowi.model('Page');
-          // not await
-          crowi.pageService.v5RecursiveMigration(Page.GRANT_PUBLIC);
-        }
-        catch (err) {
-          logger.error('Error\n', err);
-          return res.apiv3Err(new ErrorV3('Failed to migrate pages. Please try again.', 'v5_migration_failed'), 500);
-        }
-        break;
+    try {
+      switch (action) {
+        case 'initialMigration':
+          if (!isV5Compatible) {
+            const Page = crowi.model('Page');
+            // this method throws and emit socketIo event when error occurs
+            crowi.pageService.v5InitialMigration(Page.GRANT_PUBLIC); // not await
+          }
+          break;
 
-      default:
-        logger.error(`${action} action is not supported.`);
-        return res.apiv3Err(new ErrorV3('This action is not supported.', 'not_supported'), 400);
+        default:
+          logger.error(`${action} action is not supported.`);
+          return res.apiv3Err(new ErrorV3('This action is not supported.', 'not_supported'), 400);
+      }
+    }
+    catch (err) {
+      return res.apiv3Err(new ErrorV3(`Failed to migrate pages: ${err.message}`), 500);
     }
 
-    const isV5Compatible = crowi.configManager.getConfig('crowi', 'app:isV5Compatible');
     return res.apiv3({ isV5Compatible });
   });
 

packages/app/src/server/service/page.js

@@ -738,7 +738,54 @@ class PageService {
     }
   }
 
-  async v5RecursiveMigration(grant, rootPath = null) {
+  async v5InitialMigration(grant) {
+    const socket = this.crowicrowi.socketIoService.getAdminSocket();
+    try {
+      await this._v5RecursiveMigration(grant);
+    }
+    catch (err) {
+      logger.error('V5 initial miration failed.', err);
+      socket.emit('v5InitialMirationFailed', { error: err.message });
+
+      throw err;
+    }
+
+    const Page = this.crowi.model('Page');
+    const indexStatus = await Page.aggregate([{ $indexStats: {} }]);
+    const pathIndexStatus = indexStatus.filter(status => status.name === 'path_1')?.[0];
+    const isPathIndexExists = pathIndexStatus != null;
+    const isUnique = isPathIndexExists && pathIndexStatus.spec?.unique === true;
+
+    if (isUnique || !isPathIndexExists) {
+      try {
+        await this._v5NormalizeIndex(isPathIndexExists);
+      }
+      catch (err) {
+        logger.error('V5 index normalization failed.', err);
+        socket.emit('v5IndexNormalizationFailed', { error: err.message });
+
+        throw err;
+      }
+    }
+
+    await this._setIsV5CompatibleTrue();
+  }
+
+  async _setIsV5CompatibleTrue() {
+    try {
+      await this.crowi.configManager.updateConfigsInTheSameNamespace('crowi', {
+        'app:isV5Compatible': true,
+      });
+      logger.info('Successfully migrated all public pages.');
+    }
+    catch (err) {
+      logger.warn('Failed to update app:isV5Compatible to true.');
+      throw err;
+    }
+  }
+
+  // TODO: use websocket to show progress
+  async _v5RecursiveMigration(grant, rootPath) {
     const BATCH_SIZE = 100;
     const PAGES_LIMIT = 1000;
     const Page = this.crowi.model('Page');
@@ -767,7 +814,7 @@ class PageService {
       baseAggregation = baseAggregation.limit(Math.floor(total * 0.3));
     }
 
-    const randomPagesStream = await baseAggregation.cursor({ batchSize: BATCH_SIZE }).exec();
+    const pagesStream = await baseAggregation.cursor({ batchSize: BATCH_SIZE }).exec();
 
     // use batch stream
     const batchStream = createBatchStream(BATCH_SIZE);
@@ -800,6 +847,7 @@ class PageService {
         }
         catch (err) {
           logger.error('Failed to insert empty pages.', err);
+          throw err;
         }
 
         // find parents again
@@ -837,6 +885,7 @@ class PageService {
         }
         catch (err) {
           logger.error('Failed to update page.parent.', err);
+          throw err;
         }
 
         callback();
@@ -846,24 +895,41 @@ class PageService {
       },
     });
 
-    randomPagesStream
+    pagesStream
       .pipe(batchStream)
       .pipe(migratePagesStream);
 
     await streamToPromise(migratePagesStream);
+
     if (await Page.exists({ grant, parent: null, path: { $ne: '/' } })) {
       return this.v5RecursiveMigration(grant, rootPath);
     }
 
+  }
+
+  async _v5NormalizeIndex(isPathIndexExists) {
+    const collection = mongoose.connection.collection('pages');
+
+    if (isPathIndexExists) {
+      try {
+        // drop pages.path_1 indexes
+        await collection.dropIndex('path_1');
+        logger.info('Succeeded to drop unique indexes from pages.path.');
+      }
+      catch (err) {
+        logger.warn('Failed to drop unique indexes from pages.path.', err);
+        throw err;
+      }
+    }
+
     try {
-      await this.crowi.configManager.updateConfigsInTheSameNamespace('crowi', {
-        'app:isV5Compatible': true,
-      });
-      logger.info('Successfully migrated all public pages.');
+      // create indexes without
+      await collection.createIndex({ path: 1 }, { unique: false });
+      logger.info('Succeeded to create non-unique indexes on pages.path.');
     }
     catch (err) {
-      // just to know
-      logger.error('Failed to update app:isV5Compatible to true.');
+      logger.warn('Failed to create non-unique indexes on pages.path.', err);
+      throw err;
     }
   }