Merge remote-tracking branch 'origin/dev/3.5.x'

# Conflicts:
#	CHANGES.md
#	package.json

CHANGES.md

@@ -20,6 +20,12 @@ Upgrading Guide: https://docs.growi.org/en/admin-guide/upgrading/36x.html
 * Support: Upgrade libs
     * growi-commons
 
+## 3.5.25
+
+* Improvement: Disable ESC key to close Handsontable Modal
+* Fix: Exported data of empty collection is broken
+* Fix: Some components crash after when the page with attachment has exported/imported
+
 ## 3.5.24
 
 * Fix: Plugins are not working on Heroku

package.json

@@ -107,6 +107,7 @@
     "i18next-express-middleware": "^1.4.1",
     "i18next-node-fs-backend": "^2.1.0",
     "i18next-sprintf-postprocessor": "^0.2.2",
+    "is-iso-date": "^0.0.1",
     "md5": "^2.2.1",
     "method-override": "^3.0.0",
     "migrate-mongo": "^7.0.1",

src/client/js/components/Admin/ExportArchiveDataPage.jsx

@@ -15,6 +15,10 @@ import ProgressBar from './Common/ProgressBar';
 import SelectCollectionsModal from './ExportArchiveData/SelectCollectionsModal';
 import ArchiveFilesTable from './ExportArchiveData/ArchiveFilesTable';
 
+const IGNORED_COLLECTION_NAMES = [
+  'sessions',
+];
+
 class ExportArchiveDataPage extends React.Component {
 
   constructor(props) {
@@ -46,9 +50,14 @@ class ExportArchiveDataPage extends React.Component {
     ]);
     // TODO: toastSuccess, toastError
 
+    // filter only not ignored collection names
+    const filteredCollections = collections.filter((collectionName) => {
+      return !IGNORED_COLLECTION_NAMES.includes(collectionName);
+    });
+
     const { zipFileStats, isExporting, progressList } = status;
     this.setState({
-      collections,
+      collections: filteredCollections,
       zipFileStats,
       isExporting,
       progressList,

src/client/js/components/Admin/ImportData/GrowiArchive/UploadForm.jsx

@@ -57,7 +57,7 @@ class UploadForm extends React.Component {
                 type="file"
                 name="file"
                 className="form-control-file"
-                accept=".growi.zip"
+                accept=".zip"
                 ref={this.inputRef}
                 onChange={this.changeFileName}
               />

src/client/js/components/PageEditor/HandsontableModal.jsx

@@ -412,7 +412,7 @@ export default class HandsontableModal extends React.PureComponent {
     const dialogClassName = dialogClassNames.join(' ');
 
     return (
-      <Modal show={this.state.show} onHide={this.cancel} bsSize="large" dialogClassName={dialogClassName}>
+      <Modal show={this.state.show} onHide={this.cancel} bsSize="large" dialogClassName={dialogClassName} keyboard={false}>
         <Modal.Header closeButton>
           { this.renderExpandOrContractButton() }
           <Modal.Title>Edit Table</Modal.Title>

src/client/js/components/User/UserPicture.jsx

@@ -2,24 +2,27 @@ import React from 'react';
 import md5 from 'md5';
 import PropTypes from 'prop-types';
 
+const DEFAULT_IMAGE = '/images/icons/user.svg';
+
 // TODO UserComponent?
 export default class UserPicture extends React.Component {
 
   getUserPicture(user) {
+    let pictPath;
+
     // gravatar
     if (user.isGravatarEnabled === true) {
-      return this.generateGravatarSrc(user);
+      pictPath = this.generateGravatarSrc(user);
     }
     // uploaded image
     if (user.image != null) {
-      return user.image;
+      pictPath = user.image;
     }
     if (user.imageAttachment != null) {
       return user.imageAttachment.filePathProxied;
     }
 
-    return '/images/icons/user.svg';
-
+    return pictPath || DEFAULT_IMAGE;
   }
 
   generateGravatarSrc(user) {
@@ -38,9 +41,22 @@ export default class UserPicture extends React.Component {
     return className.join(' ');
   }
 
+  renderForNull() {
+    return (
+      <img
+        src={DEFAULT_IMAGE}
+        alt="someone"
+        className={this.getClassName()}
+      />
+    );
+  }
+
   render() {
     const user = this.props.user;
-    const href = `/user/${user.username}`;
+
+    if (user == null) {
+      return this.renderForNull();
+    }
 
     const imgElem = (
       <img
@@ -53,14 +69,14 @@ export default class UserPicture extends React.Component {
     return (
       (this.props.withoutLink)
         ? <span>{imgElem}</span>
-        : <a href={href}>{imgElem}</a>
+        : <a href={`/user/${user.username}`}>{imgElem}</a>
     );
   }
 
 }
 
 UserPicture.propTypes = {
-  user: PropTypes.object.isRequired,
+  user: PropTypes.object,
   size: PropTypes.string,
   withoutLink: PropTypes.bool,
 };

src/client/js/components/User/Username.jsx

@@ -3,9 +3,17 @@ import PropTypes from 'prop-types';
 
 export default class Username extends React.Component {
 
+  renderForNull() {
+    return <span>anyone</span>;
+  }
+
   render() {
     const { user } = this.props;
 
+    if (user == null) {
+      return this.renderForNull();
+    }
+
     const name = user.name || '(no name)';
     const username = user.username;
     const href = `/user/${user.username}`;
@@ -18,5 +26,5 @@ export default class Username extends React.Component {
 }
 
 Username.propTypes = {
-  user: PropTypes.object.isRequired,
+  user: PropTypes.oneOfType([PropTypes.object, PropTypes.string]), // Possibility of receiving a string of 'null'
 };

src/server/routes/apiv3/import.js

@@ -5,9 +5,6 @@ const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-lin
 const path = require('path');
 const multer = require('multer');
 
-// eslint-disable-next-line no-unused-vars
-const { ObjectId } = require('mongoose').Types;
-
 const express = require('express');
 
 const GrowiArchiveImportOption = require('@commons/models/admin/growi-archive-import-option');
@@ -54,6 +51,8 @@ const generateOverwriteParams = (collectionName, req, options) => {
       return require('./overwrite-params/pages')(req, options);
     case 'revisions':
       return require('./overwrite-params/revisions')(req, options);
+    case 'attachmentFiles.chunks':
+      return require('./overwrite-params/attachmentFiles.chunks')(req, options);
     default:
       return {};
   }

src/server/routes/apiv3/mongo.js

@@ -34,7 +34,8 @@ module.exports = (crowi) => {
    *                      type: string
    */
   router.get('/collections', async(req, res) => {
-    const collections = Object.keys(mongoose.connection.collections);
+    const listCollectionsResult = await mongoose.connection.db.listCollections().toArray();
+    const collections = listCollectionsResult.map(collectionObj => collectionObj.name);
 
     // TODO: use res.apiv3
     return res.json({

src/server/routes/apiv3/overwrite-params/attachmentFiles.chunks.js

@@ -0,0 +1,32 @@
+const { Binary } = require('mongodb');
+const { ObjectId } = require('mongoose').Types;
+
+class AttachmentFilesChunksOverwriteParamsFactory {
+
+  /**
+   * generate overwrite params object
+   * @param {object} req
+   * @param {ImportOptionForPages} option
+   * @return object
+   *  key: property name
+   *  value: any value or a function `(value, { document, schema, propertyName }) => { return newValue }`
+   */
+  static generate(req, option) {
+    const params = {};
+
+    // Date
+    params.files_id = (value, { document, schema, propertyName }) => {
+      return ObjectId(value);
+    };
+
+    // Binary
+    params.data = (value, { document, schema, propertyName }) => {
+      return Binary(value);
+    };
+
+    return params;
+  }
+
+}
+
+module.exports = (req, option) => AttachmentFilesChunksOverwriteParamsFactory.generate(req, option);

src/server/routes/attachment.js

@@ -182,7 +182,7 @@ module.exports = function(crowi, app) {
 
     const attachment = await Attachment.findOne({ filePath });
 
-    return responseForAttachment(res, req.user, attachment);
+    return responseForAttachment(req, res, attachment);
   };
 
   /**

src/server/service/export.js

@@ -139,6 +139,10 @@ class ExportService {
         callback();
       },
       final(callback) {
+        // write beginning brace
+        if (isFirst) {
+          this.push('[');
+        }
         // write ending brace
         this.push(']');
         callback();

src/server/service/growi-bridge.js

@@ -11,26 +11,9 @@ const unzipper = require('unzipper');
 class GrowiBridgeService {
 
   constructor(crowi) {
+    this.crowi = crowi;
     this.encoding = 'utf-8';
     this.metaFileName = 'meta.json';
-
-    // { pages: Page, users: User, ... }
-    this.collectionMap = {};
-    this.initCollectionMap(crowi.models);
-  }
-
-  /**
-   * initialize collection map
-   *
-   * @memberOf GrowiBridgeService
-   * @param {object} models from models/index.js
-   */
-  initCollectionMap(models) {
-    for (const model of Object.values(models)) {
-      if (model.collection != null) {
-        this.collectionMap[model.collection.name] = model;
-      }
-    }
   }
 
   /**
@@ -61,11 +44,9 @@ class GrowiBridgeService {
    * @return {object} instance of mongoose model
    */
   getModelFromCollectionName(collectionName) {
-    const Model = this.collectionMap[collectionName];
-
-    if (Model == null) {
-      throw new Error(`cannot find a model for collection name "${collectionName}"`);
-    }
+    const Model = Object.values(this.crowi.models).find((m) => {
+      return m.collection != null && m.collection.name === collectionName;
+    });
 
     return Model;
   }

src/server/service/import.js

@@ -2,12 +2,17 @@ const logger = require('@alias/logger')('growi:services:ImportService'); // esli
 const fs = require('fs');
 const path = require('path');
 
+const isIsoDate = require('is-iso-date');
+const parseISO = require('date-fns/parseISO');
+
 const { Writable, Transform } = require('stream');
 const JSONStream = require('JSONStream');
 const streamToPromise = require('stream-to-promise');
 const unzipper = require('unzipper');
 
-const { ObjectId } = require('mongoose').Types;
+const mongoose = require('mongoose');
+
+const { ObjectId } = mongoose.Types;
 
 const { createBatchStream } = require('../util/batch-stream');
 const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
@@ -90,16 +95,27 @@ class ImportService {
    *
    * @memberOf ImportService
    * @param {any} value value from imported document
-   * @param {{ document: object, schema: object, key: string }}
+   * @param {{ document: object, schema: object, propertyName: string }}
    * @return {any} new value for the document
    */
   keepOriginal(value, { document, schema, propertyName }) {
-    let _value;
-    if (schema[propertyName].instance === 'ObjectID' && ObjectId.isValid(value)) {
+    let _value = value;
+
+    // _id
+    if (propertyName === '_id' && ObjectId.isValid(value)) {
       _value = ObjectId(value);
     }
-    else {
-      _value = value;
+    // Date
+    else if (isIsoDate(value)) {
+      _value = parseISO(value);
+    }
+
+    // Model
+    if (schema != null) {
+      // ObjectID
+      if (schema[propertyName] != null && schema[propertyName].instance === 'ObjectID' && ObjectId.isValid(value)) {
+        _value = ObjectId(value);
+      }
     }
 
     return _value;
@@ -177,18 +193,20 @@ class ImportService {
     const execUnorderedBulkOpSafely = this.execUnorderedBulkOpSafely.bind(this);
     const emitProgressEvent = this.emitProgressEvent.bind(this);
 
+    const collection = mongoose.connection.collection(collectionName);
+
     const { mode, jsonFileName, overwriteParams } = importSettings;
-    const Model = this.growiBridgeService.getModelFromCollectionName(collectionName);
-    const jsonFile = this.getFile(jsonFileName);
     const collectionProgress = this.currentProgressingStatus.progressMap[collectionName];
 
     try {
+      const jsonFile = this.getFile(jsonFileName);
+
       // validate options
       this.validateImportSettings(collectionName, importSettings);
 
       // flush
       if (mode === 'flushAndInsert') {
-        await Model.remove({});
+        await collection.deleteMany({});
       }
 
       // stream 1
@@ -214,7 +232,7 @@ class ImportService {
       const writeStream = new Writable({
         objectMode: true,
         async write(batch, encoding, callback) {
-          const unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+          const unorderedBulkOp = collection.initializeUnorderedBulkOp();
 
           // documents are not persisted until unorderedBulkOp.execute()
           batch.forEach((document) => {
@@ -363,7 +381,8 @@ class ImportService {
     }
     catch (err) {
       result = err.result;
-      errors = err.writeErrors.map((err) => {
+      errors = err.writeErrors || [err];
+      errors.map((err) => {
         const moreDetailErr = err.err;
         return { _id: moreDetailErr.op._id, message: err.errmsg };
       });
@@ -390,27 +409,33 @@ class ImportService {
    */
   convertDocuments(collectionName, document, overwriteParams) {
     const Model = this.growiBridgeService.getModelFromCollectionName(collectionName);
-    const schema = Model.schema.paths;
+    const schema = (Model != null) ? Model.schema.paths : null;
     const convertMap = this.convertMap[collectionName];
 
-    if (convertMap == null) {
-      throw new Error(`attribute map is not defined for ${collectionName}`);
-    }
-
     const _document = {};
 
-    // assign value from documents being imported
-    Object.entries(convertMap).forEach(([propertyName, convertedValue]) => {
-      const value = document[propertyName];
+    // not Mongoose Model
+    if (convertMap == null) {
+      // apply keepOriginal to all of properties
+      Object.entries(document).forEach(([propertyName, value]) => {
+        _document[propertyName] = this.keepOriginal(value, { document, propertyName });
+      });
+    }
+    // Mongoose Model
+    else {
+      // assign value from documents being imported
+      Object.entries(convertMap).forEach(([propertyName, convertedValue]) => {
+        const value = document[propertyName];
 
-      // distinguish between null and undefined
-      if (value === undefined) {
-        return; // next entry
-      }
+        // distinguish between null and undefined
+        if (value === undefined) {
+          return; // next entry
+        }
 
-      const convertFunc = (typeof convertedValue === 'function') ? convertedValue : null;
-      _document[propertyName] = (convertFunc != null) ? convertFunc(value, { document, propertyName, schema }) : convertedValue;
-    });
+        const convertFunc = (typeof convertedValue === 'function') ? convertedValue : null;
+        _document[propertyName] = (convertFunc != null) ? convertFunc(value, { document, propertyName, schema }) : convertedValue;
+      });
+    }
 
     // overwrite documents with custom values
     Object.entries(overwriteParams).forEach(([propertyName, overwriteValue]) => {

yarn.lock

@@ -6616,6 +6616,11 @@ is-installed-globally@^0.1.0:
     global-dirs "^0.1.0"
     is-path-inside "^1.0.0"
 
+is-iso-date@^0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/is-iso-date/-/is-iso-date-0.0.1.tgz#d1727b0a4f40cf4dd0dbf95a56a58cc991bb76e2"
+  integrity sha1-0XJ7Ck9Az03Q2/laVqWMyZG7duI=
+
 is-npm@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4"