(WIP)add qiita importer form

lib/locales/en-US/translation.json

@@ -484,9 +484,15 @@
     "esa_settings": {
       "title": "Settings of esa importer",
       "team_name": "Team name",
-      "access_token": "Access token"
+      "access_token": "Access token",
+      "test_connection": "Test connection to esa"
     },
-    "import": "Import",
-    "test_connection": "Test connection to esa"
+    "qiita_settings": {
+      "title": "Settings of qiita:team importer",
+      "team_name": "Team name",
+      "access_token": "Access token",
+      "test_connection": "Test connection to qiita:team"
+    },
+    "import": "Import"
   }
 }

lib/locales/ja/translation.json

@@ -501,9 +501,15 @@
     "esa_settings": {
       "title": "esaインポータ設定",
       "team_name": "チーム名",
-      "access_token": "アクセストークン"
+      "access_token": "アクセストークン",
+      "test_connection": "接続テスト"
     },
-    "import": "インポート",
-    "test_connection": "接続テスト"
+    "qiita_settings": {
+      "title": "qiita:teamインポータ設定",
+      "team_name": "チーム名",
+      "access_token": "アクセストークン",
+      "test_connection": "接続テスト"
+    },
+    "import": "インポート"
   }
 }

lib/models/config.js

@@ -100,6 +100,8 @@ module.exports = function(crowi) {
 
       'importer:esa:team_name': '',
       'importer:esa:access_token': '',
+      'importer:qiita:team_name': '',
+      'importer:qiita:access_token': '',
     };
     /* eslint-enable */
   }

lib/routes/admin.js

@@ -1230,6 +1230,22 @@ module.exports = function(crowi, app) {
     return res.json({ status: true });
   };
 
+  /**
+   * Import all posts from qiita
+   *
+   * @param {*} req
+   * @param {*} res
+   */
+  actions.api.importDataFromQiita = async(req, res) => {
+    const user = req.user;
+    const errors = await importer.importDataFromQiita(user);
+
+    if (errors) {
+      return res.json({ status: false, message: `<br> - ${errors.join('<br> - ')}` });
+    }
+    return res.json({ status: true });
+  };
+
   /**
    * Test connection to esa and response result with json
    *
@@ -1246,6 +1262,22 @@ module.exports = function(crowi, app) {
     }
   };
 
+    /**
+   * Test connection to qiita and response result with json
+   *
+   * @param {*} req
+   * @param {*} res
+   */
+  actions.api.testQiitaAPI = async(req, res) => {
+    try {
+      await importer.testConnectionToQiita();
+      return res.json({ status: true });
+    }
+    catch (err) {
+      return res.json({ status: false, message: `${err}` });
+    }
+  };
+
   /**
    * save settings, update config cache, and response json
    *

lib/routes/index.js

@@ -142,9 +142,12 @@ module.exports = function(crowi, app) {
 
   // importer management for admin
   app.get('/admin/importer'                , loginRequired(crowi, app) , middleware.adminRequired() , admin.importer.index);
-  app.post('/_api/admin/settings/importer' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importer , admin.api.importerSetting);
+  app.post('/_api/admin/settings/importerEsa' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importer , admin.api.importerSetting);
+  app.post('/_api/admin/settings/importerQiita' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importer , admin.api.importerSetting);
   app.post('/_api/admin/import/esa'        , loginRequired(crowi, app) , middleware.adminRequired() , admin.api.importDataFromEsa);
   app.post('/_api/admin/import/testEsaAPI' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importer , admin.api.testEsaAPI);
+  app.post('/_api/admin/import/qiita'        , loginRequired(crowi, app) , middleware.adminRequired() , admin.api.importDataFromQiita);
+  app.post('/_api/admin/import/testQiitaAPI' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importer , admin.api.testQiitaAPI);
 
   app.get('/me'                       , loginRequired(crowi, app) , me.index);
   app.get('/me/password'              , loginRequired(crowi, app) , me.password);

lib/util/importer.js

@@ -6,6 +6,7 @@ module.exports = crowi => {
 
   const logger = require('@alias/logger')('growi:util:importer');
   const esa = require('esa-nodejs');
+  const qiita = require('./restQiitaAPI');
   const config = crowi.getConfig();
   const createGrowiPages = require('./createGrowiPagesFromImports')(crowi);
   let importer = {};
@@ -56,6 +57,40 @@ module.exports = crowi => {
     });
   };
 
+    /**
+   * Import page data from esa to GROWI
+   */
+  importer.importDataFromQiita = user => {
+    return new Promise((resolve, reject) => {
+      const firstPage = 1;
+      const errors = importPostsFromQiita(firstPage, user, []);
+
+      resolve(errors);
+    });
+  };
+
+  const importPostsFromQiita = (pageNum, user, errors) => {
+    return new Promise((resolve, reject) => {
+      esaClient.api.posts({page: pageNum, per_page: 100}, async(err, res) => { //QIITA
+        const nextPage = res.body.next_page;
+        const postsReceived = res.body.posts;
+
+        if (err) {
+          reject(`error in page ${pageNum}: ${err}`);
+        }
+
+        const data = convertQiitaDataForGrowi(postsReceived, user);
+        const newErrors = await createGrowiPages(data);
+
+        if (nextPage) {
+          return resolve(importPostsFromQiita(nextPage, user, errors.concat(newErrors)));
+        }
+
+        resolve(errors.concat(newErrors));
+      });
+    });
+  };
+
   /**
    * Convert data into usable format for createGrowiPagesFromImports
    */
@@ -86,6 +121,36 @@ module.exports = crowi => {
     return data;
   };
 
+    /**
+   * Convert data into usable format for createGrowiPagesFromImports
+   */
+  const convertQiitaDataForGrowi = (pages, user) => {
+    const basePage = '';
+    const data = pages.map(post => {
+      const category = post.category;
+      const name = post.name;
+      let path = '';
+
+      if (category && name) {
+        path = `${category}/${name}`;
+      }
+      else if (category) {
+        path = category;
+      }
+      else if (name) {
+        path = name;
+      }
+
+      return {
+        path: `${basePage}/${path}`,
+        body: post.body_md,
+        user: user,
+      };
+    });
+
+    return data;
+  };
+
   /**
    * Import page data from esa to GROWI
    */
@@ -101,6 +166,21 @@ module.exports = crowi => {
     });
   };
 
+    /**
+   * Import page data from qiita to GROWI
+   */
+  importer.testConnectionToQiita = () => {
+    return new Promise(async(resolve, reject) => {
+      try {
+        await getTeamNameFromQiita();
+        resolve();
+      }
+      catch (err) {
+        reject(err);
+      }
+    });
+  };
+
   /**
    * Get teams from esa (Promise wrapper)
    */
@@ -115,6 +195,20 @@ module.exports = crowi => {
     });
   };
 
+    /**
+   * Get teams from esa (Promise wrapper)
+   */
+  const getTeamNameFromQiita = () => {
+    return new Promise((resolve, reject) => {
+      esaClient.api.team((err, res) => { //QIITA
+        if (err) {
+          return reject(err);
+        }
+        resolve(res);
+      });
+    });
+  };
+
   // initialize when server starts
   importer.initialize();
 

lib/views/admin/importer.html

@@ -39,11 +39,10 @@
       </div>
       {% endif %}
 
-      <!-- Importer management forms -->
-      <form action="/_api/admin/settings/importer" method="post" class="form-horizontal" id="importerSettingForm" role="form"
+      <!-- esa Importer management forms -->
+      <form action="/_api/admin/settings/importerEsa" method="post" class="form-horizontal" id="importerSettingFormEsa" role="form"
           data-success-messaage="更新しました">
         <fieldset>
-          <!-- esa importer -->
           <div class="form-group">
             <legend>{{ t('importer_management.esa_settings.title') }}</legend>
             <div class="form-group">
@@ -73,7 +72,47 @@
               <span class="col-xs-offset-1">
                 <button id="importFromEsa" type="button" class="btn btn-default" data-action="/_api/admin/import/testEsaAPI"
                     data-success-message="Test connection to esa success." data-error-message="Test connection to esa failed.">
-                  {{ t("importer_management.test_connection") }}
+                  {{ t("importer_management.esa_settings.test_connection") }}
+                </button>
+              </span>
+            </div>
+          </div>
+        </fieldset>
+      </form>
+      <!-- qiita:team Importer management forms -->
+      <form action="/_api/admin/settings/importer" method="post" class="form-horizontal" id="importerSettingFormQiita" role="form"
+      data-success-messaage="更新しました">
+        <fieldset>
+          <div class="form-group">
+            <legend>{{ t('importer_management.qiita_settings.title') }}</legend>
+            <div class="form-group">
+              <label for="settingForm[importer:qiita:team_name]" class="col-xs-3 control-label">{{ t('importer_management.qiita_settings.team_name') }}</label>
+              <div class="col-xs-6">
+                <input class="form-control" type="text" name="settingForm[importer:qiita:team_name]" value="{{ settingForm['importer:qiita:team_name'] | default('') }}">
+              </div>
+            </div>
+            <div class="form-group">
+              <label for="settingForm[importer:qiita:access_token]" class="col-xs-3 control-label">{{ t('importer_management.qiita_settings.access_token') }}</label>
+              <div class="col-xs-6">
+                <input class="form-control" type="text" name="settingForm[importer:qiita:access_token]" value="{{ settingForm['importer:qiita:access_token'] | default('') }}">
+              </div>
+            </div>
+          </div>
+
+          <div class="form-group">
+            <input type="hidden" name="_csrf" value="{{ csrf() }}" />
+            <div class="col-xs-offset-3 col-xs-6">
+              <button id="testConnectionToQiita" type="button" class="btn btn-primary" data-action="/_api/admin/import/qiita"
+                  data-success-message="Import posts from qiita:team success." data-error-message="Error occurred in importing pages from qiita:team">
+                {{ t("importer_management.import") }}
+              </button>
+              <button type="submit" class="btn btn-secondary">{# the first element is the default button to submit #}
+                {{ t('Update') }}
+              </button>
+              <span class="col-xs-offset-1">
+                <button id="importFromQiita" type="button" class="btn btn-default" data-action="/_api/admin/import/testQiitaAPI"
+                    data-success-message="Test connection to qiita:team success." data-error-message="Test connection to qiita:team failed.">
+                  {{ t("importer_management.qiita_settings.test_connection") }}
                 </button>
               </span>
             </div>
@@ -140,7 +179,7 @@
    * Handle button
    */
   $('#testConnectionToEsa, #importFromEsa').each(function() {
-    var $form = $('#importerSettingForm');
+    var $form = $('#importerSettingFormEsa');
     var $button = $(this);
     var $action = $button.attr('data-action');
     var $success_msg = $button.attr('data-success-message');
@@ -151,7 +190,7 @@
   /**
    * Handle submit button
    */
-  $('#importerSettingForm').each(function() {
+  $('#importerSettingFormEsa').each(function() {
     var $form = $(this);
     var $button = $("#importerSettingForm input[type='submit']");
     var $action = $form.attr('action');

yarn.lock

@@ -7189,6 +7189,10 @@ redis@^2.1.0:
     redis-commands "^1.2.0"
     redis-parser "^2.6.0"
 
+reduce-component@1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/reduce-component/-/reduce-component-1.0.1.tgz#e0c93542c574521bea13df0f9488ed82ab77c5da"
+
 reduce-css-calc@^2.0.0:
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/reduce-css-calc/-/reduce-css-calc-2.1.4.tgz#c20e9cda8445ad73d4ff4bea960c6f8353791708"
@@ -8239,6 +8243,28 @@ stylehacks@^4.0.0:
     postcss "^6.0.0"
     postcss-selector-parser "^3.0.0"
 
+superagent-no-cache@^0.1.0:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/superagent-no-cache/-/superagent-no-cache-0.1.1.tgz#58ed8de9aeff053a9c98ae01dec4fde4b9f85fda"
+  dependencies:
+    component-ie "^1.0.0"
+
+superagent@^1.2.0:
+  version "1.8.5"
+  resolved "https://registry.yarnpkg.com/superagent/-/superagent-1.8.5.tgz#1c0ddc3af30e80eb84ebc05cb2122da8fe940b55"
+  dependencies:
+    component-emitter "~1.2.0"
+    cookiejar "2.0.6"
+    debug "2"
+    extend "3.0.0"
+    form-data "1.0.0-rc3"
+    formidable "~1.0.14"
+    methods "~1.1.1"
+    mime "1.3.4"
+    qs "2.3.3"
+    readable-stream "1.0.27-1"
+    reduce-component "1.0.1"
+
 supports-color@4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.4.0.tgz#883f7ddabc165142b2a61427f3352ded195d1a3e"