Merge remote-tracking branch 'origin/dev/5.0.x' into imprv/descendants-page-list-modal

packages/app/resource/locales/en_US/admin/admin.json

@@ -189,6 +189,9 @@
     "beta_warning": "This function is Beta.",
     "import_from": "Import from {{from}}",
     "import_growi_archive": "Import GROWI archive",
+    "error": {
+      "only_upsert_available": "Only 'Upsert' option is available for pages collection."
+    },
     "growi_settings": {
       "description_of_import_mode": {
         "about": "When you import data with the same name as an existing one, choose from the following three modes below.",

packages/app/resource/locales/ja_JP/admin/admin.json

@@ -207,6 +207,9 @@
     "beta_warning": "この機能はベータ版です",
     "import_from": "{{from}} からインポート",
     "import_growi_archive": "GROWI アーカイブをインポート",
+    "error": {
+      "only_upsert_available": "pages コレクションには 'Upsert' オプションのみ対応しています"
+    },
     "growi_settings": {
       "description_of_import_mode": {
         "about": "既存のデータと同名であるデータをインポートする際の挙動は以下の3つのモードから選べます。",

packages/app/resource/locales/zh_CN/admin/admin.json

@@ -199,6 +199,9 @@
     "beta_warning": "这个函数是Beta。",
     "import_from": "Import from {{from}}",
     "import_growi_archive": "Import GROWI archive",
+    "error": {
+      "only_upsert_available": "Only 'Upsert' option is available for pages collection."
+    },
     "growi_settings": {
       "description_of_import_mode": {
         "about": "When you import data with the same name as an existing one, choose from the following three modes below.",

packages/app/src/client/services/PageContainer.js

@@ -138,71 +138,6 @@ export default class PageContainer extends Container {
     return 'PageContainer';
   }
 
-
-  /**
-   * whether to display reaction buttons
-   * ex.) like, bookmark
-   */
-  get isAbleToShowPageReactionButtons() {
-    const { isTrashPage, isPageExist } = this.state;
-    const { isSharedUser } = this.appContainer;
-
-    return (!isTrashPage && isPageExist && !isSharedUser);
-  }
-
-  /**
-   * whether to display tag labels
-   */
-  get isAbleToShowTagLabel() {
-    const { isUserPage } = this.state;
-    const { isSharedUser } = this.appContainer;
-
-    return (!isUserPage && !isSharedUser);
-  }
-
-  /**
-   * whether to display page management
-   * ex.) duplicate, rename
-   */
-  get isAbleToShowPageManagement() {
-    const { isPageExist, isTrashPage } = this.state;
-    const { isSharedUser } = this.appContainer;
-
-    return (isPageExist && !isTrashPage && !isSharedUser);
-  }
-
-  /**
-   * whether to display pageEditorModeManager
-   * ex.) view, edit, hackmd
-   */
-  get isAbleToShowPageEditorModeManager() {
-    const { isNotCreatable, isTrashPage } = this.state;
-    const { isSharedUser } = this.appContainer;
-
-    return (!isNotCreatable && !isTrashPage && !isSharedUser);
-  }
-
-  /**
-   * whether to display pageAuthors
-   * ex.) creator, lastUpdateUser
-   */
-  get isAbleToShowPageAuthors() {
-    const { isPageExist, isUserPage } = this.state;
-
-    return (isPageExist && !isUserPage);
-  }
-
-  /**
-   * whether to like button
-   * not displayed on user page
-   */
-  get isAbleToShowLikeButtons() {
-    const { isUserPage } = this.state;
-    const { isSharedUser } = this.appContainer;
-
-    return (!isUserPage && !isSharedUser);
-  }
-
   /**
    * whether to Empty Trash Page
    * not displayed when guest user and not on trash page

packages/app/src/components/Admin/ImportData/GrowiArchive/ImportForm.jsx

@@ -287,7 +287,9 @@ class ImportForm extends React.Component {
   }
 
   async import() {
-    const { appContainer, fileName, onPostImport } = this.props;
+    const {
+      appContainer, fileName, onPostImport, t,
+    } = this.props;
     const { selectedCollections, optionsMap } = this.state;
 
     // init progress data
@@ -312,6 +314,9 @@ class ImportForm extends React.Component {
       toastSuccess(undefined, 'Import process has requested.');
     }
     catch (err) {
+      if (err.code === 'only_upsert_available') {
+        toastError(t('admin:importer_management.error.only_upsert_available'));
+      }
       toastError(err, 'Import request failed.');
     }
   }

packages/app/src/components/Navbar/GrowiContextualSubNavigation.tsx

@@ -175,6 +175,9 @@ const GrowiContextualSubNavigation = (props) => {
       showDrawerToggler={isDrawerMode}
       showTagLabel={isAbleToShowTagLabel}
       showPageAuthors={isAbleToShowPageAuthors}
+      isGuestUser={isGuestUser}
+      isDrawerMode={isDrawerMode}
+      isCompactMode={isCompactMode}
       tags={tagsInfoData?.tags || []}
       tagsUpdatedHandler={tagsUpdatedHandler}
       controls={ControlComponents}

packages/app/src/server/models/page.ts

@@ -145,7 +145,7 @@ schema.statics.createEmptyPagesByPaths = async function(paths: string[], publicO
 };
 
 schema.statics.createEmptyPage = async function(
-    path: string, parent: any, descendantCount: number, // TODO: improve type including IPage at https://redmine.weseek.co.jp/issues/86506
+    path: string, parent: any, descendantCount = 0, // TODO: improve type including IPage at https://redmine.weseek.co.jp/issues/86506
 ): Promise<PageDocument & { _id: any }> {
   if (parent == null) {
     throw Error('parent must not be null');
@@ -459,8 +459,10 @@ schema.statics.incrementDescendantCountOfPageIds = async function(pageIds: Objec
   await this.updateMany({ _id: { $in: pageIds } }, { $inc: { descendantCount: increment } });
 };
 
-// update descendantCount of a page with provided id
-schema.statics.recountDescendantCountOfSelfAndDescendants = async function(id: ObjectIdLike):Promise<void> {
+/**
+ * recount descendantCount of a page with the provided id and return it
+ */
+schema.statics.recountDescendantCount = async function(id: ObjectIdLike):Promise<number> {
   const res = await this.aggregate(
     [
       {
@@ -498,8 +500,7 @@ schema.statics.recountDescendantCountOfSelfAndDescendants = async function(id: O
     ],
   );
 
-  const query = { descendantCount: res.length === 0 ? 0 : res[0].descendantCount };
-  await this.findByIdAndUpdate(id, query);
+  return res.length === 0 ? 0 : res[0].descendantCount;
 };
 
 schema.statics.findAncestorsUsingParentRecursively = async function(pageId: ObjectIdLike, shouldIncludeTarget: boolean) {
@@ -534,7 +535,7 @@ export default (crowi: Crowi): any => {
   }
 
   schema.statics.create = async function(path: string, body: string, user, options: PageCreateOptions = {}) {
-    if (crowi.pageGrantService == null || crowi.configManager == null) {
+    if (crowi.pageGrantService == null || crowi.configManager == null || crowi.pageService == null) {
       throw Error('Crowi is not setup');
     }
 
@@ -593,6 +594,9 @@ export default (crowi: Crowi): any => {
     let page;
     if (emptyPage != null) {
       page = emptyPage;
+      const descendantCount = await this.recountDescendantCount(page._id);
+
+      page.descendantCount = descendantCount;
       page.isEmpty = false;
     }
     else {
@@ -622,7 +626,7 @@ export default (crowi: Crowi): any => {
 
     let savedPage = await page.save();
 
-    await crowi.pageService?.updateDescendantCountOfAncestors(page._id, 1, false);
+    await crowi.pageService.updateDescendantCountOfAncestors(page._id, 1, false);
 
     /*
      * After save

packages/app/src/server/routes/apiv3/import.js

@@ -1,3 +1,5 @@
+import mongoose from 'mongoose';
+
 import loggerFactory from '~/utils/logger';
 
 const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-line no-unused-vars
@@ -204,8 +206,19 @@ module.exports = (crowi) => {
    */
   router.post('/', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
     // TODO: add express validator
-
     const { fileName, collections, optionsMap } = req.body;
+
+    // pages collection can only be imported by upsert if isV5Compatible is true
+    const isV5Compatible = crowi.configManager.getConfig('crowi', 'app:isV5Compatible');
+    const isImportPagesCollection = collections.includes('pages');
+    if (isV5Compatible && isImportPagesCollection) {
+      const option = new GrowiArchiveImportOption(null, optionsMap.pages);
+      if (option.mode !== 'upsert') {
+        return res.apiv3Err(new ErrorV3('Upsert is only available for importing pages collection.', 'only_upsert_available'));
+      }
+    }
+
+
     const zipFile = importService.getFile(fileName);
 
     // return response first

packages/app/src/server/routes/apiv3/overwrite-params/pages.js

@@ -1,4 +1,8 @@
 const mongoose = require('mongoose');
+const { format } = require('date-fns');
+const { pagePathUtils } = require('@growi/core');
+
+const { isTopPage } = pagePathUtils;
 
 // eslint-disable-next-line no-unused-vars
 const ImportOptionForPages = require('~/models/admin/import-option-for-pages');
@@ -45,6 +49,10 @@ class PageOverwriteParamsFactory {
       return null;
     };
 
+    params.descendantCount = (value, { document, schema, propertyName }) => {
+      return 0;
+    };
+
     if (option.initPageMetadatas) {
       params.liker = [];
       params.seenUsers = [];

packages/app/src/server/routes/apiv3/pages.js

@@ -638,9 +638,11 @@ module.exports = (crowi) => {
     const newParentPage = await crowi.pageService.duplicate(page, newPagePath, req.user, isRecursively);
     const result = { page: serializePageSecurely(newParentPage) };
 
-    page.path = newPagePath;
+    // copy the page since it's used and updated in crowi.pageService.duplicate
+    const copyPage = { ...page };
+    copyPage.path = newPagePath;
     try {
-      await globalNotificationService.fire(GlobalNotificationSetting.EVENT.PAGE_CREATE, page, req.user);
+      await globalNotificationService.fire(GlobalNotificationSetting.EVENT.PAGE_CREATE, copyPage, req.user);
     }
     catch (err) {
       logger.error('Create grobal notification failed', err);
@@ -708,12 +710,11 @@ module.exports = (crowi) => {
 
   router.post('/v5-schema-migration', accessTokenParser, loginRequired, adminRequired, csrf, async(req, res) => {
     const isV5Compatible = crowi.configManager.getConfig('crowi', 'app:isV5Compatible');
-    const Page = crowi.model('Page');
 
     try {
       if (!isV5Compatible) {
         // this method throws and emit socketIo event when error occurs
-        crowi.pageService.v5InitialMigration(Page.GRANT_PUBLIC); // not await
+        crowi.pageService.normalizeAllPublicPages(); // not await
       }
     }
     catch (err) {

packages/app/src/server/routes/page.js

@@ -338,6 +338,7 @@ module.exports = function(crowi, app) {
     const offset = parseInt(req.query.offset) || 0;
     await addRenderVarsForDescendants(renderVars, path, req.user, offset, limit, true);
     await addRenderVarsForPageTree(renderVars, pathOrId, req.user);
+
     addRenderVarsWhenNotFound(renderVars, pathOrId);
 
     return res.render(view, renderVars);

packages/app/src/server/service/import.js

@@ -182,6 +182,13 @@ class ImportService {
     // init status object
     this.currentProgressingStatus = new CollectionProgressingStatus(collections);
 
+    const isV5Compatible = this.crowi.configManager.getConfig('crowi', 'app:isV5Compatible');
+    const isImportPagesCollection = collections.includes('pages');
+    const shouldNormalizePages = isV5Compatible && isImportPagesCollection;
+
+    // set isV5Compatible to false
+    if (shouldNormalizePages) await this.crowi.configManager.updateConfigsInTheSameNamespace('crowi', { 'app:isV5Compatible': false });
+
     // process serially so as not to waste memory
     const promises = collections.map((collectionName) => {
       const importSettings = importSettingsMap[collectionName];
@@ -199,6 +206,9 @@ class ImportService {
       }
     }
 
+    // run normalizeAllPublicPages
+    if (shouldNormalizePages) await this.crowi.pageService.normalizeAllPublicPages();
+
     this.currentProgressingStatus = null;
     this.emitTerminateEvent();
   }
@@ -333,6 +343,8 @@ class ImportService {
 
     // upsert
     switch (collectionName) {
+      case 'pages':
+        return bulk.find({ path: document.path }).upsert().replaceOne(document);
       default:
         return bulk.find({ _id: document._id }).upsert().replaceOne(document);
     }

packages/app/src/server/service/installer.ts

@@ -115,6 +115,7 @@ export class InstallerService {
     const rootPage = await Page.findOne({ path: '/' });
     const rootRevision = await Revision.findOne({ path: '/' });
     rootPage.creator = adminUser._id;
+    rootPage.lastUpdateUser = adminUser._id;
     rootRevision.creator = adminUser._id;
     await Promise.all([rootPage.save(), rootRevision.save()]);
 

packages/app/src/server/service/page.ts

@@ -700,9 +700,17 @@ class PageService {
 
     newPagePath = this.crowi.xss.process(newPagePath); // eslint-disable-line no-param-reassign
 
-    const createdPage = await (Page.create as CreateMethod)(
-      newPagePath, page.revision.body, user, options,
-    );
+    let createdPage;
+
+    if (page.isEmpty) {
+      const parent = await Page.getParentAndFillAncestors(newPagePath);
+      createdPage = await Page.createEmptyPage(newPagePath, parent);
+    }
+    else {
+      createdPage = await (Page.create as CreateMethod)(
+        newPagePath, page.revision.body, user, options,
+      );
+    }
 
     // take over tags
     const originTags = await page.findRelatedTagsById();
@@ -718,12 +726,16 @@ class PageService {
 
     // TODO: resume
     if (isRecursively) {
-      this.duplicateDescendantsWithStream(page, newPagePath, user, shouldUseV4Process);
+      this.resumableDuplicateDescendants(page, newPagePath, user, shouldUseV4Process, createdPage._id);
     }
-
     return result;
   }
 
+  async resumableDuplicateDescendants(page, newPagePath, user, shouldUseV4Process, createdPageId) {
+    const descendantCountAppliedToAncestors = await this.duplicateDescendantsWithStream(page, newPagePath, user, shouldUseV4Process);
+    await this.updateDescendantCountOfAncestors(createdPageId, descendantCountAppliedToAncestors, false);
+  }
+
   async duplicateV4(page, newPagePath, user, isRecursively) {
     const Page = this.crowi.model('Page');
     const PageTagRelation = mongoose.model('PageTagRelation') as any; // TODO: Typescriptize model
@@ -827,14 +839,7 @@ class PageService {
       pageIdMapping[page._id] = newPageId;
 
       let newPage;
-      if (page.isEmpty) {
-        newPage = {
-          _id: newPageId,
-          path: newPagePath,
-          isEmpty: true,
-        };
-      }
-      else {
+      if (!page.isEmpty) {
         newPage = {
           _id: newPageId,
           path: newPagePath,
@@ -845,14 +850,11 @@ class PageService {
           lastUpdateUser: user._id,
           revision: revisionId,
         };
+        newRevisions.push({
+          _id: revisionId, pageId: newPageId, body: pageIdRevisionMapping[page._id].body, author: user._id, format: 'markdown',
+        });
       }
-
       newPages.push(newPage);
-
-      newRevisions.push({
-        _id: revisionId, pageId: newPageId, body: pageIdRevisionMapping[page._id].body, author: user._id, format: 'markdown',
-      });
-
     });
 
     await Page.insertMany(newPages, { ordered: false });
@@ -922,11 +924,13 @@ class PageService {
     const normalizeParentAndDescendantCountOfDescendants = this.normalizeParentAndDescendantCountOfDescendants.bind(this);
     const pageEvent = this.pageEvent;
     let count = 0;
+    let nNonEmptyDuplicatedPages = 0;
     const writeStream = new Writable({
       objectMode: true,
       async write(batch, encoding, callback) {
         try {
           count += batch.length;
+          nNonEmptyDuplicatedPages += batch.filter(page => !page.isEmpty).length;
           await duplicateDescendants(batch, user, pathRegExp, newPagePathPrefix, shouldUseV4Process);
           logger.debug(`Adding pages progressing: (count=${count})`);
         }
@@ -962,6 +966,9 @@ class PageService {
       .pipe(createBatchStream(BULK_REINDEX_SIZE))
       .pipe(writeStream);
 
+    await streamToPromise(writeStream);
+
+    return nNonEmptyDuplicatedPages;
   }
 
   private async duplicateDescendantsWithStreamV4(page, newPagePath, user) {
@@ -1000,6 +1007,9 @@ class PageService {
       .pipe(createBatchStream(BULK_REINDEX_SIZE))
       .pipe(writeStream);
 
+    await streamToPromise(writeStream);
+
+    return count;
   }
 
   /*
@@ -1868,7 +1878,7 @@ class PageService {
   }
 
   // TODO: use socket to send status to the client
-  async v5InitialMigration(grant) {
+  async normalizeAllPublicPages() {
     // const socket = this.crowi.socketIoService.getAdminSocket();
 
     let isUnique;
@@ -1894,7 +1904,8 @@ class PageService {
 
     // then migrate
     try {
-      await this.normalizeParentRecursively(grant, null, true);
+      const Page = mongoose.model('Page') as unknown as PageModel;
+      await this.normalizeParentRecursively(Page.GRANT_PUBLIC, null, true);
     }
     catch (err) {
       logger.error('V5 initial miration failed.', err);
@@ -2158,7 +2169,8 @@ class PageService {
       objectMode: true,
       async write(pageDocuments, encoding, callback) {
         for await (const document of pageDocuments) {
-          await Page.recountDescendantCountOfSelfAndDescendants(document._id);
+          const descendantCount = await Page.recountDescendantCount(document._id);
+          await Page.findByIdAndUpdate(document._id, { descendantCount });
         }
         callback();
       },

packages/app/test/integration/service/v5-migration.test.js

@@ -75,7 +75,7 @@ describe('V5 page migration', () => {
 
   });
 
-  describe('v5InitialMigration()', () => {
+  describe('normalizeAllPublicPages()', () => {
     jest.setTimeout(60000);
     let createPagePaths;
     let allPossiblePagePaths;
@@ -132,7 +132,7 @@ describe('V5 page migration', () => {
       ]);
 
       // migrate
-      await crowi.pageService.v5InitialMigration(Page.GRANT_PUBLIC);
+      await crowi.pageService.normalizeAllPublicPages(Page.GRANT_PUBLIC);
       jest.setTimeout(30000);
     });