Merge branch 'feat/pt-dev-3' into feat/pt-dev-4

packages/app/src/interfaces/page.ts

@@ -14,7 +14,7 @@ export type IPage = {
   createdAt: Date,
   updatedAt: Date,
   seenUsers: Ref<IUser>[],
-  parent: Ref<IPage>,
+  parent: Ref<IPage> | null,
   isEmpty: boolean,
   redirectTo: string,
   grant: number,

packages/app/src/migrations/20211129125654-initialize-private-legacy-pages-named-query.js

@@ -0,0 +1,47 @@
+import mongoose from 'mongoose';
+
+import { getModelSafely, getMongoUri, mongoOptions } from '@growi/core';
+import NamedQuery from '~/server/models/named-query';
+import { SearchDelegatorName } from '~/interfaces/named-query';
+import loggerFactory from '~/utils/logger';
+
+
+const logger = loggerFactory('growi:migrate:initialize-private-legacy-pages-named-query');
+
+module.exports = {
+  async up(db, next) {
+    mongoose.connect(getMongoUri(), mongoOptions);
+
+    try {
+      await NamedQuery.insertMany({
+        name: SearchDelegatorName.PRIVATE_LEGACY_PAGES,
+        delegatorName: SearchDelegatorName.PRIVATE_LEGACY_PAGES,
+      });
+    }
+    catch (err) {
+      logger.error('Failed to migrate named query for private legacy pages search delagator.', err);
+      throw err;
+    }
+
+    next();
+    logger.info('Successfully migrated named query for private legacy pages search delagator.');
+  },
+
+  async down(db, next) {
+    mongoose.connect(getMongoUri(), mongoOptions);
+
+    try {
+      await NamedQuery.findOneAndDelete({
+        name: SearchDelegatorName.PRIVATE_LEGACY_PAGES,
+        delegatorName: SearchDelegatorName.PRIVATE_LEGACY_PAGES,
+      });
+    }
+    catch (err) {
+      logger.error('Failed to delete named query for private legacy pages search delagator.', err);
+      throw err;
+    }
+
+    next();
+    logger.info('Successfully deleted named query for private legacy pages search delagator.');
+  },
+};

packages/app/src/server/crowi/index.js

@@ -18,6 +18,7 @@ import { projectRoot } from '~/utils/project-dir-utils';
 import ConfigManager from '../service/config-manager';
 import AppService from '../service/app';
 import AclService from '../service/acl';
+import SearchService from '../service/search';
 import AttachmentService from '../service/attachment';
 import { SlackIntegrationService } from '../service/slack-integration';
 import { UserNotificationService } from '../service/user-notification';
@@ -370,7 +371,6 @@ Crowi.prototype.setupPassport = async function() {
 };
 
 Crowi.prototype.setupSearcher = async function() {
-  const SearchService = require('~/server/service/search');
   this.searchService = new SearchService(this);
 };
 

packages/app/src/server/interfaces/search.ts

@@ -33,7 +33,7 @@ export type Result<T> = {
 }
 
 export type MetaData = {
-  meta: { [key:string]: any }
+  meta?: { [key:string]: any }
 }
 
 export type SearchableData = {

packages/app/src/server/middlewares/auto-reconnect-to-search.js

@@ -14,7 +14,7 @@ module.exports = (crowi) => {
       await searchService.reconnectClient();
     }
     catch (err) {
-      logger.error('Auto reconnection failed.');
+      logger.error('Auto reconnection failed.', err);
     }
 
     return searchService.isReachable;

packages/app/src/server/models/obsolete-page.js

@@ -219,6 +219,26 @@ export class PageQueryBuilder {
     return this;
   }
 
+  addConditionAsNonRootPage() {
+    this.query = this.query.and({ path: { $ne: '/' } });
+
+    return this;
+  }
+
+  addConditionAsNotMigrated() {
+    this.query = this.query
+      .and({ parent: null });
+
+    return this;
+  }
+
+  addConditionAsMigrated() {
+    this.query = this.query
+      .and({ parent: { $ne: null } });
+
+    return this;
+  }
+
   /*
    * Add this condition when get any ancestor pages including the target's parent
    */
@@ -672,13 +692,15 @@ export const getPageSchema = (crowi) => {
     return await findListFromBuilderAndViewer(builder, currentUser, showAnyoneKnowsLink, opt);
   };
 
-  pageSchema.statics.findListByPageIds = async function(ids, option) {
+  pageSchema.statics.findListByPageIds = async function(ids, option, excludeRedirect = true) {
     const User = crowi.model('User');
 
     const opt = Object.assign({}, option);
     const builder = new PageQueryBuilder(this.find({ _id: { $in: ids } }));
 
-    builder.addConditionToExcludeRedirect();
+    if (excludeRedirect) {
+      builder.addConditionToExcludeRedirect();
+    }
     builder.addConditionToPagenate(opt.offset, opt.limit);
 
     // count

packages/app/src/server/routes/apiv3/pages.js

@@ -186,6 +186,7 @@ module.exports = (crowi) => {
     ],
     v5PageMigration: [
       body('action').isString().withMessage('action is required'),
+      body('pageIds').isArray().withMessage('pageIds must be an array'),
     ],
   };
 
@@ -685,18 +686,21 @@ module.exports = (crowi) => {
   });
 
   router.post('/v5-schema-migration', accessTokenParser, loginRequired, adminRequired, csrf, validator.v5PageMigration, apiV3FormValidator, async(req, res) => {
-    const { action } = req.body;
+    const { action, pageIds } = req.body;
     const isV5Compatible = crowi.configManager.getConfig('crowi', 'app:isV5Compatible');
+    const Page = crowi.model('Page');
 
     try {
       switch (action) {
         case 'initialMigration':
           if (!isV5Compatible) {
-            const Page = crowi.model('Page');
             // this method throws and emit socketIo event when error occurs
             crowi.pageService.v5InitialMigration(Page.GRANT_PUBLIC); // not await
           }
           break;
+        case 'privateLegacyPages':
+          crowi.pageService.v5MigrationByPageIds(pageIds);
+          break;
 
         default:
           logger.error(`${action} action is not supported.`);

packages/app/src/server/service/page.js

@@ -1,5 +1,4 @@
 import { pagePathUtils } from '@growi/core';
-import Page from '~/components/Page';
 import loggerFactory from '~/utils/logger';
 
 const mongoose = require('mongoose');
@@ -776,6 +775,28 @@ class PageService {
     }
   }
 
+  async v5MigrationByPageIds(pageIds) {
+    const Page = mongoose.model('Page');
+
+    if (pageIds == null || pageIds.length === 0) {
+      return;
+    }
+
+    // generate regexps
+    const regexps = await this._generateRegExpsByPageIds(pageIds);
+
+    // migrate recursively
+    try {
+      await this._v5RecursiveMigration(null, regexps);
+    }
+    catch (err) {
+      logger.error('V5 initial miration failed.', err);
+      // socket.emit('v5InitialMirationFailed', { error: err.message }); TODO: use socket to tell user
+
+      throw err;
+    }
+  }
+
   async v5InitialMigration(grant) {
     const socket = this.crowi.socketIoService.getAdminSocket();
     try {
@@ -809,6 +830,27 @@ class PageService {
     await this._setIsV5CompatibleTrue();
   }
 
+  /*
+   * returns an array of js RegExp instance instead of RE2 instance for mongo filter
+   */
+  async _generateRegExpsByPageIds(pageIds) {
+    const Page = mongoose.model('Page');
+
+    let result;
+    try {
+      result = await Page.findListByPageIds(pageIds, null, false);
+    }
+    catch (err) {
+      logger.error('Failed to find pages by ids', err);
+      throw err;
+    }
+
+    const { pages } = result;
+    const regexps = pages.map(page => new RegExp(`^${page.path}`));
+
+    return regexps;
+  }
+
   async _setIsV5CompatibleTrue() {
     try {
       await this.crowi.configManager.updateConfigsInTheSameNamespace('crowi', {
@@ -823,21 +865,38 @@ class PageService {
   }
 
   // TODO: use websocket to show progress
-  async _v5RecursiveMigration(grant, rootPath) {
+  async _v5RecursiveMigration(grant, regexps) {
     const BATCH_SIZE = 100;
     const PAGES_LIMIT = 1000;
     const Page = this.crowi.model('Page');
     const { PageQueryBuilder } = Page;
 
-    const total = await Page.countDocuments({ grant, parent: null });
+    // generate filter
+    let filter = {
+      parent: null,
+      path: { $ne: '/' },
+    };
+    if (grant != null) {
+      filter = {
+        ...filter,
+        grant,
+      };
+    }
+    if (regexps != null && regexps.length !== 0) {
+      filter = {
+        ...filter,
+        path: {
+          $in: regexps,
+        },
+      };
+    }
+
+    const total = await Page.countDocuments(filter);
 
     let baseAggregation = Page
       .aggregate([
         {
-          $match: {
-            grant,
-            parent: null,
-          },
+          $match: filter,
         },
         {
           $project: { // minimize data to fetch
@@ -921,8 +980,8 @@ class PageService {
 
     await streamToPromise(migratePagesStream);
 
-    if (await Page.exists({ grant, parent: null, path: { $ne: '/' } })) {
-      return this._v5RecursiveMigration(grant, rootPath);
+    if (await Page.exists(filter)) {
+      return this._v5RecursiveMigration(grant, regexps);
     }
 
   }

packages/app/src/server/service/search-delegator/private-legacy-pages.ts

@@ -0,0 +1,53 @@
+import mongoose from 'mongoose';
+
+import { PageModel, PageDocument } from '~/server/models/page';
+import { SearchDelegatorName } from '~/interfaces/named-query';
+import { IPage } from '~/interfaces/page';
+import {
+  MetaData, Result, SearchableData, SearchDelegator,
+} from '../../interfaces/search';
+
+
+type Data = {
+  pages: IPage[]
+}
+
+class PrivateLegacyPagesDelegator implements SearchDelegator<Data> {
+
+  name!: SearchDelegatorName.PRIVATE_LEGACY_PAGES
+
+  async search(data: SearchableData | null, user, userGroups, option): Promise<Result<Data> & MetaData> {
+    const { offset, limit } = option;
+
+    if (offset == null || limit == null) {
+      throw Error('PrivateLegacyPagesDelegator requires pagination options (offset, limit).');
+    }
+    if (user == null && userGroups == null) {
+      throw Error('Either of user and userGroups must not be null.');
+    }
+
+    // find private legacy pages
+    const Page = mongoose.model('Page') as PageModel;
+    const { PageQueryBuilder } = Page;
+
+    const queryBuilder = new PageQueryBuilder(Page.find());
+
+    const pages: PageDocument[] = await queryBuilder
+      .addConditionAsNonRootPage()
+      .addConditionAsNotMigrated()
+      .addConditionToFilteringByViewer(user, userGroups)
+      .addConditionToPagenate(offset, limit)
+      .query
+      .lean()
+      .exec();
+
+    return {
+      data: {
+        pages,
+      },
+    };
+  }
+
+}
+
+export default PrivateLegacyPagesDelegator;

packages/app/src/server/service/search.ts

@@ -1,11 +1,14 @@
 import mongoose from 'mongoose';
 import RE2 from 're2';
 
-import { NamedQueryModel } from '../models/named-query';
 import { SearchDelegatorName } from '~/interfaces/named-query';
+
+import { NamedQueryModel } from '../models/named-query';
 import {
   SearchDelegator, SearchQueryParser, SearchResolver, ParsedQuery, Result, MetaData, SearchableData, QueryTerms,
 } from '../interfaces/search';
+import ElasticsearchDelegator from './search-delegator/elasticsearch';
+import PrivateLegacyPagesDelegator from './search-delegator/private-legacy-pages';
 
 import loggerFactory from '~/utils/logger';
 
@@ -31,7 +34,7 @@ class SearchService implements SearchQueryParser, SearchResolver {
 
   fullTextSearchDelegator: any & SearchDelegator
 
-  nqDelegators: {[key in SearchDelegatorName]: SearchDelegator} // TODO: initialize
+  nqDelegators: {[key in SearchDelegatorName]: SearchDelegator}
 
   constructor(crowi) {
     this.crowi = crowi;
@@ -41,7 +44,9 @@ class SearchService implements SearchQueryParser, SearchResolver {
     this.isErrorOccuredOnSearching = null;
 
     try {
-      this.fullTextSearchDelegator = this.generateDelegator();
+      this.fullTextSearchDelegator = this.generateFullTextSearchDelegator();
+      this.nqDelegators = this.generateNQDelegators(this.fullTextSearchDelegator);
+      logger.info('Succeeded to initialize search delegators');
     }
     catch (err) {
       logger.error(err);
@@ -66,11 +71,10 @@ class SearchService implements SearchQueryParser, SearchResolver {
     return uri != null && uri.length > 0;
   }
 
-  generateDelegator() {
+  generateFullTextSearchDelegator() {
     logger.info('Initializing search delegator');
 
     if (this.isElasticsearchEnabled) {
-      const ElasticsearchDelegator = require('./search-delegator/elasticsearch');
       logger.info('Elasticsearch is enabled');
       return new ElasticsearchDelegator(this.configManager, this.crowi.socketIoService);
     }
@@ -78,6 +82,13 @@ class SearchService implements SearchQueryParser, SearchResolver {
     logger.info('No elasticsearch URI is specified so that full text search is disabled.');
   }
 
+  generateNQDelegators(defaultDelegator: SearchDelegator): {[key in SearchDelegatorName]: SearchDelegator} {
+    return {
+      [SearchDelegatorName.DEFAULT]: defaultDelegator,
+      [SearchDelegatorName.PRIVATE_LEGACY_PAGES]: new PrivateLegacyPagesDelegator(),
+    };
+  }
+
   registerUpdateEvent() {
     const pageEvent = this.crowi.event('page');
     pageEvent.on('create', this.fullTextSearchDelegator.syncPageUpdated.bind(this.fullTextSearchDelegator));
@@ -154,7 +165,6 @@ class SearchService implements SearchQueryParser, SearchResolver {
   }
 
   async parseSearchQuery(_queryString: string): Promise<ParsedQuery> {
-
     const regexp = new RE2(/^\[nq:.+\]$/g); // https://regex101.com/r/FzDUvT/1
     const replaceRegexp = new RE2(/\[nq:|\]/g);
 
@@ -202,7 +212,7 @@ class SearchService implements SearchQueryParser, SearchResolver {
       queryString,
       terms: terms as QueryTerms,
     };
-    return [this.fullTextSearchDelegator, data];
+    return [this.nqDelegators[SearchDelegatorName.DEFAULT], data];
   }
 
   async searchKeyword(keyword: string, user, userGroups, searchOpts): Promise<Result<any> & MetaData> {

packages/app/src/test/integration/service/page.test.js

@@ -706,6 +706,7 @@ describe('PageService', () => {
       deleteManyPageSpy = jest.spyOn(Page, 'deleteMany').mockImplementation();
       removeAllAttachmentsSpy = jest.spyOn(crowi.attachmentService, 'removeAllAttachments').mockImplementation();
     });
+
     test('deleteCompletelyOperation', async() => {
       await crowi.pageService.deleteCompletelyOperation([parentForDeleteCompletely._id], [parentForDeleteCompletely.path], { });
 
@@ -820,5 +821,55 @@ describe('PageService', () => {
     });
   });
 
+  describe('v5MigrationByPageIds()', () => {
+    test('should migrate all pages specified by pageIds', async() => {
+      jest.restoreAllMocks();
+
+      // initialize pages for test
+      const pages = await Page.insertMany([
+        {
+          path: '/private1',
+          grant: Page.GRANT_OWNER,
+          creator: testUser1,
+          lastUpdateUser: testUser1,
+        },
+        {
+          path: '/dummyParent/private1',
+          grant: Page.GRANT_OWNER,
+          creator: testUser1,
+          lastUpdateUser: testUser1,
+        },
+        {
+          path: '/dummyParent/private1/private2',
+          grant: Page.GRANT_OWNER,
+          creator: testUser1,
+          lastUpdateUser: testUser1,
+        },
+        {
+          path: '/dummyParent/private1/private3',
+          grant: Page.GRANT_OWNER,
+          creator: testUser1,
+          lastUpdateUser: testUser1,
+        },
+      ]);
+
+      const pageIds = pages.map(page => page._id);
+      // migrate
+      await crowi.pageService.v5MigrationByPageIds(pageIds);
+
+      const migratedPages = await Page.find({
+        path: {
+          $in: ['/private1', '/dummyParent', '/dummyParent/private1', '/dummyParent/private1/private2', '/dummyParent/private1/private3'],
+        },
+      });
+      const migratedPagePaths = migratedPages.filter(doc => doc.parent != null).map(doc => doc.path);
+
+      const expected = ['/private1', '/dummyParent', '/dummyParent/private1', '/dummyParent/private1/private2', '/dummyParent/private1/private3'];
+
+      expect(migratedPagePaths.sort()).toStrictEqual(expected.sort());
+    });
+
+  });
+
 
 });

packages/app/src/test/integration/service/search/search-service.test.js

@@ -1,3 +1,5 @@
+import mongoose from 'mongoose';
+
 import SearchService from '~/server/service/search';
 import NamedQuery from '~/server/models/named-query';
 
@@ -8,16 +10,16 @@ describe('SearchService test', () => {
   let searchService;
 
   const DEFAULT = 'FullTextSearch';
+  const PRIVATE_LEGACY_PAGES = 'PrivateLegacyPages';
 
   // let NamedQuery;
 
   let dummyAliasOf;
-  let dummyDelegatorName;
 
   let namedQuery1;
   let namedQuery2;
 
-  const dummyDelegator = {
+  const dummyFullTextSearchDelegator = {
     search() {
       return;
     },
@@ -27,18 +29,29 @@ describe('SearchService test', () => {
     crowi = await getInstance();
     searchService = new SearchService(crowi);
     searchService.nqDelegators = {
-      FullTextSearch: dummyDelegator,
+      ...searchService.nqDelegators,
+      [DEFAULT]: dummyFullTextSearchDelegator, // override with dummy full-text search delegator
     };
+
+    dummyAliasOf = 'match -notmatch "phrase" -"notphrase" prefix:/pre1 -prefix:/pre2 tag:Tag1 -tag:Tag2';
+
+    await NamedQuery.insertMany([
+      { name: 'named_query1', delegatorName: PRIVATE_LEGACY_PAGES },
+      { name: 'named_query2', aliasOf: dummyAliasOf },
+    ]);
+
+    namedQuery1 = await NamedQuery.findOne({ name: 'named_query1' });
+    namedQuery2 = await NamedQuery.findOne({ name: 'named_query2' });
   });
 
 
   describe('parseQueryString()', () => {
     test('should parse queryString', async() => {
-      const queryString = 'match -notmatch "phrase" -"notphrase" [nq:named_query] prefix:/pre1 -prefix:/pre2 tag:Tag1 -tag:Tag2';
+      const queryString = 'match -notmatch "phrase" -"notphrase" prefix:/pre1 -prefix:/pre2 tag:Tag1 -tag:Tag2';
       const terms = await searchService.parseQueryString(queryString);
 
       const expected = { // QueryTerms
-        match: ['match', '[nq:named_query]'],
+        match: ['match'],
         not_match: ['notmatch'],
         phrase: ['"phrase"'],
         not_phrase: ['"notphrase"'],
@@ -53,18 +66,6 @@ describe('SearchService test', () => {
   });
 
   describe('parseSearchQuery()', () => {
-    beforeAll(async() => {
-      dummyDelegatorName = DEFAULT;
-      dummyAliasOf = 'match -notmatch "phrase" -"notphrase" prefix:/pre1 -prefix:/pre2 tag:Tag1 -tag:Tag2';
-
-      await NamedQuery.insertMany([
-        { name: 'named_query1', delegatorName: dummyDelegatorName },
-        { name: 'named_query2', aliasOf: dummyAliasOf },
-      ]);
-
-      namedQuery1 = await NamedQuery.findOne({ name: 'named_query1' });
-      namedQuery2 = await NamedQuery.findOne({ name: 'named_query2' });
-    });
 
     test('should return result with delegatorName', async() => {
       const queryString = '[nq:named_query1]';
@@ -72,7 +73,7 @@ describe('SearchService test', () => {
 
       const expected = {
         queryString,
-        delegatorName: dummyDelegatorName,
+        delegatorName: PRIVATE_LEGACY_PAGES,
       };
 
       expect(parsedQuery).toStrictEqual(expected);
@@ -97,7 +98,9 @@ describe('SearchService test', () => {
 
       expect(parsedQuery).toStrictEqual(expected);
     });
+  });
 
+  describe('resolve()', () => {
     test('should resolve as full-text search delegator', async() => {
       const parsedQuery = {
         queryString: dummyAliasOf,
@@ -118,14 +121,14 @@ describe('SearchService test', () => {
       const expectedData = parsedQuery;
 
       expect(data).toStrictEqual(expectedData);
-      // expect(typeof delegator.search).toBe('function'); TODO: enable test after implementing delegator initialization
+      expect(typeof delegator.search).toBe('function');
     });
 
     test('should resolve as custom search delegator', async() => {
       const queryString = '[nq:named_query1]';
       const parsedQuery = {
         queryString,
-        delegatorName: dummyDelegatorName,
+        delegatorName: PRIVATE_LEGACY_PAGES,
       };
 
       const [delegator, data] = await searchService.resolve(parsedQuery);
@@ -135,7 +138,69 @@ describe('SearchService test', () => {
       expect(data).toBe(expectedData);
       expect(typeof delegator.search).toBe('function');
     });
+  });
+
+  describe('searchKeyword()', () => {
+    test('should search with custom search delegator', async() => {
+      const Page = mongoose.model('Page');
+      const User = mongoose.model('User');
+      await User.insertMany([
+        { name: 'dummyuser1', username: 'dummyuser1', email: 'dummyuser1@example.com' },
+        { name: 'dummyuser2', username: 'dummyuser2', email: 'dummyuser2@example.com' },
+      ]);
 
+      const testUser1 = await User.findOne({ username: 'dummyuser1' });
+      const testUser2 = await User.findOne({ username: 'dummyuser2' });
+
+      await Page.insertMany([
+        {
+          path: '/user1',
+          grant: Page.GRANT_PUBLIC,
+          creator: testUser1,
+          lastUpdateUser: testUser1,
+        },
+        {
+          path: '/user1_owner',
+          grant: Page.GRANT_OWNER,
+          creator: testUser1,
+          lastUpdateUser: testUser1,
+          grantedUsers: [testUser1._id],
+        },
+        {
+          path: '/user2_public',
+          grant: Page.GRANT_PUBLIC,
+          creator: testUser2,
+          lastUpdateUser: testUser2,
+        },
+      ]);
+
+      const page1 = await Page.findOne({ path: '/user1' });
+
+      await Page.insertMany([
+        {
+          path: '/user1/hasParent',
+          grant: Page.GRANT_PUBLIC,
+          creator: testUser1,
+          lastUpdateUser: testUser1,
+          parent: page1,
+        },
+      ]);
+
+      const queryString = '[nq:named_query1]';
+      const parsedQuery = {
+        queryString,
+        delegatorName: PRIVATE_LEGACY_PAGES,
+      };
+
+      const [delegator, data] = await searchService.resolve(parsedQuery);
+
+      const result = await delegator.search(data, testUser1, null, { limit: 0, offset: 0 });
+
+      const resultPaths = result.data.pages.map(page => page.path);
+      const flag = resultPaths.includes('/user1') && resultPaths.includes('/user1_owner') && resultPaths.includes('/user2_public');
+
+      expect(flag).toBe(true);
+    });
   });
 
 });