Merge pull request #563 from weseek/feat/esa-importer

Feat/esa importer

lib/routes/admin.js

@@ -979,7 +979,7 @@ module.exports = function(crowi, app) {
     return res.render('admin/importer', {
       settingForm: settingForm,
     });
-  }
+  };
 
   actions.api = {};
   actions.api.appSetting = function(req, res) {
@@ -1222,10 +1222,10 @@ module.exports = function(crowi, app) {
    */
   actions.api.importDataFromEsa = async(req, res) => {
     const user = req.user;
-    const result = await importer.importPagesFromEsa(user);
+    const errors = await importer.importDataFromEsa(user);
 
-    if (result.errors) {
-      return res.json({ status: false, message: `<br> - ${result.errors.join('<br> - ')}` });
+    if (errors) {
+      return res.json({ status: false, message: `<br> - ${errors.join('<br> - ')}` });
     }
     return res.json({ status: true });
   };

lib/util/createGrowiPagesFromImports.js

@@ -12,8 +12,6 @@ module.exports = crowi => {
    * }]
    */
   const createGrowiPages = (pages) => {
-    let createdPages = [];
-    let rejectedPages = [];
     let errors = [];
 
     return new Promise((resolve, reject) => {
@@ -28,14 +26,12 @@ module.exports = crowi => {
           if (isCreatableName && !isPageNameTaken) {
             try {
               await Page.create(path, body, user, { grant: Page.GRANT_PUBLIC, grantUserGroupId: null });
-              createdPages.push(page);
             }
             catch (err) {
               errors.push(err);
             }
           }
           else {
-            rejectedPages.push(page);
             if (!isCreatableName) {
               errors.push(new Error(`${path} is not a creatable name in Growi`));
             }
@@ -50,11 +46,7 @@ module.exports = crowi => {
 
       Promise.all(promises)
         .then(() => {
-          resolve({
-            createdPages,
-            rejectedPages,
-            errors,
-          });
+          resolve(errors);
         });
     });
   };

lib/util/importer.js

@@ -4,9 +4,8 @@
 module.exports = crowi => {
   'use strict';
 
-  const debug = require('debug')('growi:lib:importer');
+  const logger = require('@alias/logger')('growi:util:importer');
   const esa = require('esa-nodejs');
-  const User = crowi.model('User');
   const config = crowi.getConfig();
   const createGrowiPages = require('./createGrowiPagesFromImports')(crowi);
   let importer = {};
@@ -20,33 +19,39 @@ module.exports = crowi => {
       team:        config.crowi['importer:esa:team_name'],
       accessToken: config.crowi['importer:esa:access_token'],
     });
-    debug('esa client is initialized');
+    logger.info('initialize esa importer');
   };
 
   /**
    * Import page data from esa to GROWI
    */
-  importer.importPagesFromEsa = user => {
-    return new Promise(async(resolve, reject) => {
-      const rawData = await getAllDataFromEsa();
-      const data = convertEsaDataForGrowi(rawData, user);
-      const result = await createGrowiPages(data, user);
+  importer.importDataFromEsa = user => {
+    return new Promise((resolve, reject) => {
+      const firstPage = 1;
+      const errors = importPostsFromEsa(firstPage, user, []);
 
-      resolve(result);
+      resolve(errors);
     });
   };
 
-  /**
-   * Get posts from esa (Promise wrapper)
-   */
-  const getAllDataFromEsa = () => {
+  const importPostsFromEsa = (pageNum, user, errors) => {
     return new Promise((resolve, reject) => {
-      esaClient.api.posts((err, res) => {
+      esaClient.api.posts({page: pageNum, per_page: 100}, async(err, res) => {
+        const nextPage = res.body.next_page;
+        const postsReceived = res.body.posts;
+
         if (err) {
-          debug(err);
-          return reject(err);
+          reject(`error in page ${pageNum}: ${err}`);
         }
-        resolve(res);
+
+        const data = convertEsaDataForGrowi(postsReceived, user);
+        const newErrors = await createGrowiPages(data);
+
+        if (nextPage) {
+          return resolve(importPostsFromEsa(nextPage, user, errors.concat(newErrors)));
+        }
+
+        resolve(errors.concat(newErrors));
       });
     });
   };
@@ -54,9 +59,9 @@ module.exports = crowi => {
   /**
    * Convert data into usable format for createGrowiPagesFromImports
    */
-  const convertEsaDataForGrowi = (rawData, user) => {
+  const convertEsaDataForGrowi = (pages, user) => {
     const basePage = '';
-    const data = rawData.body.posts.map(post => {
+    const data = pages.map(post => {
       const category = post.category;
       const name = post.name;
       let path = '';
@@ -91,7 +96,6 @@ module.exports = crowi => {
         resolve();
       }
       catch (err) {
-        debug(err);
         reject(err);
       }
     });
@@ -104,7 +108,6 @@ module.exports = crowi => {
     return new Promise((resolve, reject) => {
       esaClient.api.team((err, res) => {
         if (err) {
-          debug(err);
           return reject(err);
         }
         resolve(res);

lib/views/admin/importer.html

@@ -64,7 +64,7 @@
             <input type="hidden" name="_csrf" value="{{ csrf() }}" />
             <div class="col-xs-offset-3 col-xs-6">
               <button id="testConnectionToEsa" type="button" class="btn btn-primary" data-action="/_api/admin/import/esa"
-                  data-success-message="Import posts from esa success." data-error-message="Import posts from esa failed.">
+                  data-success-message="Import posts from esa success." data-error-message="Error occurred in importing pages from esa.io">
                 {{ t("importer_management.import") }}
               </button>
               <button type="submit" class="btn btn-secondary">{# the first element is the default button to submit #}