Merge pull request #5617 from weseek/imprv/avoid-rerunning-v5-migration

imprv: Show v5 migration status with a progress bar

packages/app/resource/locales/en_US/admin/admin.json

@@ -28,7 +28,10 @@
     "modal_migration_warning": "This process may take long. It is highly recommended that administrators tell users not to create, modify, or delete pages during the conversion.",
     "start_upgrading": "Start converting to v5 compatibility",
     "successfully_started": "Succeeded to start the conversion",
-    "already_upgraded": "You have already completed the conversion to v5 compatibility"
+    "already_upgraded": "You have already completed the conversion to v5 compatibility",
+    "header_upgrading_progress": "Upgrade Progress",
+    "migration_succeeded": "Your upgrade has been successfully completed! Exit maintenance mode and GROWI can be used.",
+    "migration_failed": "Upgrade failed. Please refer to the GROWI docs for information on what to do in the event of failure."
   },
   "maintenance_mode": {
     "maintenance_mode": "Maintenance Mode",

packages/app/resource/locales/ja_JP/admin/admin.json

@@ -28,7 +28,10 @@
     "modal_migration_warning": "管理者はユーザーに、v5 互換形式への変換中はページを作成・変更・削除しないように伝えることを強くお勧めします。",
     "start_upgrading": "v5 互換形式への変換を開始",
     "successfully_started": "正常に v5 互換形式への変換が開始されました",
-    "already_upgraded": "v5 互換形式への変換は既に完了しています"
+    "already_upgraded": "v5 互換形式への変換は既に完了しています",
+    "header_upgrading_progress": "アップグレード進行度",
+    "migration_succeeded": "アップグレードが正常に完了しました！メンテナンスモードを終了して、GROWI を使用することができます。",
+    "migration_failed": "アップグレードが失敗しました。失敗した場合の対処法は GROWI docs を参照してください。"
   },
   "maintenance_mode": {
     "maintenance_mode": "メンテナンスモード",

packages/app/resource/locales/zh_CN/admin/admin.json

@@ -28,7 +28,10 @@
     "modal_migration_warning": "这个过程可能需要很长时间。强烈建议管理员告诉用户在转换期间不要创建、修改或删除页面。",
     "start_upgrading": "开始转换为v5兼容性",
     "successfully_started": "成功开始转换",
-    "already_upgraded": "你已经完成了向v5兼容性的转换"
+    "already_upgraded": "你已经完成了向v5兼容性的转换",
+    "header_upgrading_progress": "升级进度",
+    "migration_succeeded": "您的升级已经成功完成! 退出维护模式，可以使用GROWI。",
+    "migration_failed": "升级失败。请参考GROWI的文档，了解在失败情况下该如何处理。"
   },
   "maintenance_mode": {
     "maintenance_mode": "维护模式",

packages/app/src/client/services/ContextExtractor.tsx

@@ -15,7 +15,7 @@ import {
   usePreferDrawerModeByUser, usePreferDrawerModeOnEditByUser, useSidebarCollapsed, useCurrentSidebarContents, useCurrentProductNavWidth,
   useSelectedGrant, useSelectedGrantGroupId, useSelectedGrantGroupName,
 } from '~/stores/ui';
-import { useSetupGlobalSocket } from '~/stores/websocket';
+import { useSetupGlobalSocket, useSetupGlobalAdminSocket } from '~/stores/websocket';
 import { IUserUISettings } from '~/interfaces/user-ui-settings';
 
 const { isTrashPage: _isTrashPage } = pagePathUtils;
@@ -161,6 +161,7 @@ const ContextExtractorOnce: FC = () => {
 
   // Global Socket
   useSetupGlobalSocket();
+  useSetupGlobalAdminSocket();
 
   return null;
 };

packages/app/src/components/Admin/App/V5PageMigration.tsx

@@ -1,19 +1,75 @@
-import React, { FC, useState } from 'react';
+import React, {
+  FC, useCallback, useEffect, useState,
+} from 'react';
 import { useTranslation } from 'react-i18next';
 import { ConfirmModal } from './ConfirmModal';
 import AdminAppContainer from '../../../client/services/AdminAppContainer';
 import { withUnstatedContainers } from '../../UnstatedUtils';
 import { toastSuccess, toastError } from '../../../client/util/apiNotification';
+import { useGlobalAdminSocket } from '~/stores/websocket';
+import LabeledProgressBar from '../Common/LabeledProgressBar';
+import {
+  SocketEventName, PMStartedData, PMMigratingData, PMErrorCountData, PMEndedData,
+} from '~/interfaces/websocket';
 
 type Props = {
   adminAppContainer: typeof AdminAppContainer & { v5PageMigrationHandler: () => Promise<{ isV5Compatible: boolean }> },
 }
 
 const V5PageMigration: FC<Props> = (props: Props) => {
+  // Modal
   const [isV5PageMigrationModalShown, setIsV5PageMigrationModalShown] = useState(false);
-  const { adminAppContainer } = props;
+  // Progress bar
+  const [isInProgress, setProgressing] = useState<boolean | undefined>(undefined); // use false as ended
+  const [total, setTotal] = useState<number>(0);
+  const [skip, setSkip] = useState<number>(0);
+  const [current, setCurrent] = useState<number>(0);
+  const [isSucceeded, setSucceeded] = useState<boolean | undefined>(undefined);
+
+  const { data: adminSocket } = useGlobalAdminSocket();
   const { t } = useTranslation();
 
+  const { adminAppContainer } = props;
+
+  /*
+   * Local components
+   */
+  const renderResultMessage = useCallback((isSucceeded: boolean) => {
+    return (
+      <>
+        {
+          isSucceeded
+            ? <p className="text-success p-1">{t('admin:v5_page_migration.migration_succeeded')}</p>
+            : <p className="text-danger p-1">{t('admin:v5_page_migration.migration_failed')}</p>
+        }
+      </>
+    );
+  }, [t]);
+
+  const renderProgressBar = () => {
+    if (isInProgress == null) {
+      return <></>;
+    }
+
+    return (
+      <>
+        {
+          isSucceeded != null && renderResultMessage(isSucceeded)
+        }
+        <LabeledProgressBar
+          header={t('admin:v5_page_migration.header_upgrading_progress')}
+          currentCount={current}
+          errorsCount={skip}
+          totalCount={total}
+          isInProgress={isInProgress}
+        />
+      </>
+    );
+  };
+
+  /*
+   * Functions
+   */
   const onConfirm = async() => {
     setIsV5PageMigrationModalShown(false);
     try {
@@ -29,6 +85,39 @@ const V5PageMigration: FC<Props> = (props: Props) => {
     }
   };
 
+  /*
+   * Use Effect
+   */
+  // Setup Admin Socket
+  useEffect(() => {
+    adminSocket?.once(SocketEventName.PMStarted, (data: PMStartedData) => {
+      setProgressing(true);
+      setTotal(data.total);
+    });
+
+    adminSocket?.on(SocketEventName.PMMigrating, (data: PMMigratingData) => {
+      setProgressing(true);
+      setCurrent(data.count);
+    });
+
+    adminSocket?.on(SocketEventName.PMErrorCount, (data: PMErrorCountData) => {
+      setProgressing(true);
+      setSkip(data.skip);
+    });
+
+    adminSocket?.once(SocketEventName.PMEnded, (data: PMEndedData) => {
+      setProgressing(false);
+      setSucceeded(data.isSucceeded);
+    });
+
+    return () => {
+      adminSocket?.off(SocketEventName.PMStarted);
+      adminSocket?.off(SocketEventName.PMMigrating);
+      adminSocket?.off(SocketEventName.PMErrorCount);
+      adminSocket?.off(SocketEventName.PMEnded);
+    };
+  }, [adminSocket]);
+
   return (
     <>
       <ConfirmModal
@@ -48,9 +137,10 @@ const V5PageMigration: FC<Props> = (props: Props) => {
           {t('admin:v5_page_migration.migration_note')}
         </span>
       </p>
+      {renderProgressBar()}
       <div className="row my-3">
         <div className="mx-auto">
-          <button type="button" className="btn btn-warning" onClick={() => setIsV5PageMigrationModalShown(true)}>
+          <button type="button" className="btn btn-warning" onClick={() => setIsV5PageMigrationModalShown(true)} disabled={isInProgress != null}>
             {t('admin:v5_page_migration.upgrade_to_v5')}
           </button>
         </div>

packages/app/src/interfaces/websocket.ts

@@ -1,5 +1,12 @@
 export const SocketEventName = {
-  UpdateDescCount: 'UpdateDsecCount',
+  // Update descendantCount
+  UpdateDescCount: 'UpdateDescCount',
+
+  // Public migration
+  PMStarted: 'PublicMigrationStarted',
+  PMMigrating: 'PublicMigrationMigrating',
+  PMErrorCount: 'PublicMigrationErrorCount',
+  PMEnded: 'PublicMigrationEnded',
 } as const;
 export type SocketEventName = typeof SocketEventName[keyof typeof SocketEventName];
 
@@ -10,3 +17,8 @@ type DescendantCount = number;
  */
 export type UpdateDescCountRawData = Record<PageId, DescendantCount>;
 export type UpdateDescCountData = Map<PageId, DescendantCount>;
+
+export type PMStartedData = { total: number };
+export type PMMigratingData = { count: number };
+export type PMErrorCountData = { skip: number };
+export type PMEndedData = { isSucceeded: boolean };

packages/app/src/server/service/page.ts

@@ -2557,13 +2557,14 @@ class PageService {
     return this._normalizeParentRecursively(pathAndRegExpsToNormalize, ancestorPaths, grantFiltersByUser, user);
   }
 
-  // TODO: use websocket to show progress
   private async _normalizeParentRecursively(
-      pathOrRegExps: (RegExp | string)[], publicPathsToNormalize: string[], grantFiltersByUser: { $or: any[] }, user,
+      pathOrRegExps: (RegExp | string)[], publicPathsToNormalize: string[], grantFiltersByUser: { $or: any[] }, user, count = 0, skiped = 0, isFirst = true,
   ): Promise<void> {
     const BATCH_SIZE = 100;
     const PAGES_LIMIT = 1000;
 
+    const socket = this.crowi.socketIoService.getAdminSocket();
+
     const Page = mongoose.model('Page') as unknown as PageModel;
     const { PageQueryBuilder } = Page;
 
@@ -2617,6 +2618,9 @@ class PageService {
 
     // Limit pages to get
     const total = await Page.countDocuments(mergedFilter);
+    if (isFirst) {
+      socket.emit(SocketEventName.PMStarted, { total });
+    }
     if (total > PAGES_LIMIT) {
       baseAggregation = baseAggregation.limit(Math.floor(total * 0.3));
     }
@@ -2624,8 +2628,9 @@ class PageService {
     const pagesStream = await baseAggregation.cursor({ batchSize: BATCH_SIZE });
     const batchStream = createBatchStream(BATCH_SIZE);
 
-    let countPages = 0;
     let shouldContinue = true;
+    let nextCount = count;
+    let nextSkiped = skiped;
 
     const migratePagesStream = new Writable({
       objectMode: true,
@@ -2710,12 +2715,17 @@ class PageService {
         try {
           const res = await Page.bulkWrite(updateManyOperations);
 
-          countPages += res.result.nModified;
-          logger.info(`Page migration processing: (count=${countPages})`);
+          nextCount += res.result.nModified;
+          nextSkiped += res.result.writeErrors.length;
+          logger.info(`Page migration processing: (migratedPages=${res.result.nModified})`);
+
+          socket.emit(SocketEventName.PMMigrating, { count: nextCount });
+          socket.emit(SocketEventName.PMErrorCount, { skip: nextSkiped });
 
           // Throw if any error is found
           if (res.result.writeErrors.length > 0) {
             logger.error('Failed to migrate some pages', res.result.writeErrors);
+            socket.emit(SocketEventName.PMEnded, { isSucceeded: false });
             throw Error('Failed to migrate some pages');
           }
 
@@ -2723,6 +2733,7 @@ class PageService {
           if (res.result.nModified === 0 && res.result.nMatched === 0) {
             shouldContinue = false;
             logger.error('Migration is unable to continue', 'parentPaths:', parentPaths, 'bulkWriteResult:', res);
+            socket.emit(SocketEventName.PMEnded, { isSucceeded: false });
           }
         }
         catch (err) {
@@ -2744,9 +2755,11 @@ class PageService {
     await streamToPromise(migratePagesStream);
 
     if (await Page.exists(mergedFilter) && shouldContinue) {
-      return this._normalizeParentRecursively(pathOrRegExps, publicPathsToNormalize, grantFiltersByUser, user);
+      return this._normalizeParentRecursively(pathOrRegExps, publicPathsToNormalize, grantFiltersByUser, user, nextCount, nextSkiped, false);
     }
 
+    // End
+    socket.emit(SocketEventName.PMEnded, { isSucceeded: true });
   }
 
   private async _v5NormalizeIndex() {

packages/app/src/stores/websocket.tsx

@@ -9,6 +9,12 @@ const logger = loggerFactory('growi:stores:ui');
 export const GLOBAL_SOCKET_NS = '/';
 export const GLOBAL_SOCKET_KEY = 'globalSocket';
 
+export const GLOBAL_ADMIN_SOCKET_NS = '/admin';
+export const GLOBAL_ADMIN_SOCKET_KEY = 'globalAdminSocket';
+
+/*
+ * Global Socket
+ */
 export const useSetupGlobalSocket = (): SWRResponse<Socket, Error> => {
   const socket = io(GLOBAL_SOCKET_NS, {
     transports: ['websocket'],
@@ -23,3 +29,21 @@ export const useSetupGlobalSocket = (): SWRResponse<Socket, Error> => {
 export const useGlobalSocket = (): SWRResponse<Socket, Error> => {
   return useStaticSWR(GLOBAL_SOCKET_KEY);
 };
+
+/*
+ * Global Admin Socket
+ */
+export const useSetupGlobalAdminSocket = (): SWRResponse<Socket, Error> => {
+  const socket = io(GLOBAL_ADMIN_SOCKET_NS, {
+    transports: ['websocket'],
+  });
+
+  socket.on('error', (err) => { logger.error(err) });
+  socket.on('connect_error', (err) => { logger.error('Failed to connect with websocket.', err) });
+
+  return useStaticSWR(GLOBAL_ADMIN_SOCKET_KEY, socket);
+};
+
+export const useGlobalAdminSocket = (): SWRResponse<Socket, Error> => {
+  return useStaticSWR(GLOBAL_ADMIN_SOCKET_KEY);
+};