Merge pull request #567 from weseek/feat/importer-qiita

Feat/importer qiita

lib/crowi/index.js

@@ -38,6 +38,7 @@ function Crowi(rootdir, env) {
   this.interceptorManager = {};
   this.passportService = null;
   this.globalNotificationService = null;
+  this.restQiitaAPIService = null;
   this.xss = new Xss();
 
   this.tokens = null;
@@ -93,6 +94,8 @@ Crowi.prototype.init = function() {
       return self.setupCsrf();
     }).then(function() {
       return self.setUpGlobalNotification();
+    }).then(function() {
+      return self.setUpRestQiitaAPI();
     });
 };
 
@@ -253,6 +256,10 @@ Crowi.prototype.getGlobalNotificationService = function() {
   return this.globalNotificationService;
 };
 
+Crowi.prototype.getRestQiitaAPIService = function() {
+  return this.restQiitaAPIService;
+};
+
 Crowi.prototype.setupPassport = function() {
   const config = this.getConfig();
   const Config = this.model('Config');
@@ -276,7 +283,7 @@ Crowi.prototype.setupPassport = function() {
     this.passportService.setupLdapStrategy();
     this.passportService.setupGoogleStrategy();
     this.passportService.setupGitHubStrategy();
-    this.passportService.setupTwitterStrategy(); 
+    this.passportService.setupTwitterStrategy();
   }
   catch (err) {
     logger.error(err);
@@ -470,4 +477,14 @@ Crowi.prototype.setUpGlobalNotification = function() {
   }
 };
 
+/**
+ * setup RestQiitaAPIService
+ */
+Crowi.prototype.setUpRestQiitaAPI = function() {
+  const RestQiitaAPIService = require('../service/rest-qiita-API');
+  if (this.restQiitaAPIService == null) {
+    this.restQiitaAPIService = new RestQiitaAPIService(this);
+  }
+};
+
 module.exports = Crowi;

lib/form/admin/importerQiita.js

@@ -0,0 +1,10 @@
+'use strict';
+
+var form = require('express-form')
+  , field = form.field;
+
+module.exports = form(
+  field('settingForm[importer:qiita:access_token]').required(),
+  field('settingForm[importer:qiita:team_name]').required(),
+);
+

lib/form/index.js

@@ -14,7 +14,8 @@ module.exports = {
     app: require('./admin/app'),
     mail: require('./admin/mail'),
     aws: require('./admin/aws'),
-    importer: require('./admin/importer'),
+    importerEsa: require('./admin/importerEsa'),
+    importerQiita: require('./admin/importerQiita'),
     plugin: require('./admin/plugin'),
     securityGeneral: require('./admin/securityGeneral'),
     securityGoogle: require('./admin/securityGoogle'),

lib/locales/en-US/translation.json

@@ -494,9 +494,15 @@
     "esa_settings": {
       "title": "Settings of esa importer",
       "team_name": "Team name",
-      "access_token": "Access token"
+      "access_token": "Access token",
+      "test_connection": "Test connection to esa"
     },
-    "import": "Import",
-    "test_connection": "Test connection to esa"
+    "qiita_settings": {
+      "title": "Settings of qiita:team importer",
+      "team_name": "Team name",
+      "access_token": "Access token",
+      "test_connection": "Test connection to qiita:team"
+    },
+    "import": "Import"
   }
 }

lib/locales/ja/translation.json

@@ -511,9 +511,15 @@
     "esa_settings": {
       "title": "esaインポータ設定",
       "team_name": "チーム名",
-      "access_token": "アクセストークン"
+      "access_token": "アクセストークン",
+      "test_connection": "接続テスト"
     },
-    "import": "インポート",
-    "test_connection": "接続テスト"
+    "qiita_settings": {
+      "title": "qiita:teamインポータ設定",
+      "team_name": "チーム名",
+      "access_token": "アクセストークン",
+      "test_connection": "接続テスト"
+    },
+    "import": "インポート"
   }
 }

lib/models/config.js

@@ -101,6 +101,8 @@ module.exports = function(crowi) {
 
       'importer:esa:team_name': '',
       'importer:esa:access_token': '',
+      'importer:qiita:team_name': '',
+      'importer:qiita:access_token': '',
     };
     /* eslint-enable */
   }

lib/routes/admin.js

@@ -1112,7 +1112,7 @@ module.exports = function(crowi, app) {
     debug('form content', form);
     await saveSettingAsync(form);
     const config = await crowi.getConfig();
-    
+
 
     // reset strategy
     await crowi.passportService.resetTwitterStrategy();
@@ -1226,12 +1226,29 @@ module.exports = function(crowi, app) {
   };
 
   /**
-   * save settings, update config cache, and response json
+   * save esa settings, update config cache, and response json
+   *
+   * @param {*} req
+   * @param {*} res
+   */
+  actions.api.importerSettingEsa = async(req, res) => {
+    const form = req.form.settingForm;
+
+    if (!req.form.isValid) {
+      return res.json({status: false, message: req.form.errors.join('\n')});
+    }
+
+    await saveSetting(req, res, form);
+    await importer.initializeEsaClient();
+  };
+
+  /**
+   * save qiita settings, update config cache, and response json
    *
    * @param {*} req
    * @param {*} res
    */
-  actions.api.importerSetting = async(req, res) => {
+  actions.api.importerSettingQiita = async(req, res) => {
     const form = req.form.settingForm;
 
     if (!req.form.isValid) {
@@ -1239,7 +1256,7 @@ module.exports = function(crowi, app) {
     }
 
     await saveSetting(req, res, form);
-    await importer.initialize();
+    await importer.initializeQiitaClient();
   };
 
   /**
@@ -1250,9 +1267,39 @@ module.exports = function(crowi, app) {
    */
   actions.api.importDataFromEsa = async(req, res) => {
     const user = req.user;
-    const errors = await importer.importDataFromEsa(user);
+    let errors;
 
-    if (errors) {
+    try {
+      errors = await importer.importDataFromEsa(user);
+    }
+    catch (err) {
+      errors = [err];
+    }
+
+    if (errors.length > 0) {
+      return res.json({ status: false, message: `<br> - ${errors.join('<br> - ')}` });
+    }
+    return res.json({ status: true });
+  };
+
+  /**
+   * Import all posts from qiita
+   *
+   * @param {*} req
+   * @param {*} res
+   */
+  actions.api.importDataFromQiita = async(req, res) => {
+    const user = req.user;
+    let errors;
+
+    try {
+      errors = await importer.importDataFromQiita(user);
+    }
+    catch (err) {
+      errors = [err];
+    }
+
+    if (errors.length > 0) {
       return res.json({ status: false, message: `<br> - ${errors.join('<br> - ')}` });
     }
     return res.json({ status: true });
@@ -1274,6 +1321,22 @@ module.exports = function(crowi, app) {
     }
   };
 
+  /**
+   * Test connection to qiita and response result with json
+   *
+   * @param {*} req
+   * @param {*} res
+   */
+  actions.api.testQiitaAPI = async(req, res) => {
+    try {
+      await importer.testConnectionToQiita();
+      return res.json({ status: true });
+    }
+    catch (err) {
+      return res.json({ status: false, message: `${err}` });
+    }
+  };
+
   /**
    * save settings, update config cache, and response json
    *

lib/routes/index.js

@@ -77,7 +77,7 @@ module.exports = function(crowi, app) {
   app.get('/passport/twitter'                     , loginPassport.loginWithTwitter);
   app.get('/passport/google/callback'             , loginPassport.loginPassportGoogleCallback);
   app.get('/passport/github/callback'             , loginPassport.loginPassportGitHubCallback);
-  app.get('/passport/twitter/callback'             , loginPassport.loginPassportTwitterCallback); 
+  app.get('/passport/twitter/callback'             , loginPassport.loginPassportTwitterCallback);
 
   // markdown admin
   app.get('/admin/markdown'                   , loginRequired(crowi, app) , middleware.adminRequired() , admin.markdown.index);
@@ -145,9 +145,12 @@ module.exports = function(crowi, app) {
 
   // importer management for admin
   app.get('/admin/importer'                , loginRequired(crowi, app) , middleware.adminRequired() , admin.importer.index);
-  app.post('/_api/admin/settings/importer' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importer , admin.api.importerSetting);
+  app.post('/_api/admin/settings/importerEsa' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importerEsa , admin.api.importerSettingEsa);
+  app.post('/_api/admin/settings/importerQiita' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importerQiita , admin.api.importerSettingQiita);
   app.post('/_api/admin/import/esa'        , loginRequired(crowi, app) , middleware.adminRequired() , admin.api.importDataFromEsa);
-  app.post('/_api/admin/import/testEsaAPI' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importer , admin.api.testEsaAPI);
+  app.post('/_api/admin/import/testEsaAPI' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importerEsa , admin.api.testEsaAPI);
+  app.post('/_api/admin/import/qiita'        , loginRequired(crowi, app) , middleware.adminRequired() , admin.api.importDataFromQiita);
+  app.post('/_api/admin/import/testQiitaAPI' , loginRequired(crowi, app) , middleware.adminRequired() , csrf , form.admin.importerQiita , admin.api.testQiitaAPI);
 
   app.get('/me'                       , loginRequired(crowi, app) , me.index);
   app.get('/me/password'              , loginRequired(crowi, app) , me.password);

lib/service/rest-qiita-API.js

@@ -0,0 +1,87 @@
+const debug = require('debug')('growi:service:GlobalNotification');
+
+function getAxios(team, token) {
+  return require('axios').create({
+    baseURL: `https://${team}.qiita.com/api/v2`,
+    headers: {
+      'Content-Type': 'application/json',
+      'X-Requested-With': 'XMLHttpRequest',
+      'authorization': `Bearer ${token}`
+    },
+    responseType: 'json'
+  });
+}
+
+/**
+ * the service class of restQiitaAPI
+ * Qiita API v2 documant https://qiita.com/api/v2/docs
+ */
+
+class RestQiitaAPIService {
+
+  constructor(crowi) {
+    this.crowi = crowi;
+    this.config = crowi.getConfig();
+    this.team = this.config.crowi['importer:qiita:team_name'];
+    this.token = this.config.crowi['importer:qiita:access_token'];
+    this.axios = getAxios(this.team, this.token);
+  }
+
+  /**
+   * @memberof RestQiitaAPI
+   * @param {string} team
+   * @param {string} token
+   */
+  async reset() {
+    this.team = this.config.crowi['importer:qiita:team_name'];
+    this.token = this.config.crowi['importer:qiita:access_token'];
+    this.axios = getAxios(this.team, this.token);
+  }
+
+  /**
+   * get Qiita API
+   * @memberof RestQiitaAPI
+   * @param {string} path
+   */
+  async restAPI(path) {
+    return this.axios.get(path)
+      .then(function(res) {
+        const data = res.data;
+        const total = res.headers['total-count'];
+
+        return {data, total};
+      });
+  }
+
+  /**
+   * get Qiita user
+   * @memberof RestQiitaAPI
+   */
+  async getQiitaUser() {
+    const res = await this.restAPI('/users');
+    const user = res.data;
+
+    if (user.length > 0) {
+      return user;
+    }
+  }
+
+
+  /**
+   * get Qiita pages
+   * @memberof RestQiitaAPI
+   * @param {string} pageNum
+   * @param {string} per_page
+   */
+  async getQiitaPages(pageNum, per_page) {
+    const res = await this.restAPI(`/items?page=${pageNum}&per_page=${per_page}`);
+    const pages = res.data;
+    const total = res.total;
+
+    if (pages.length > 0) {
+      return {pages, total};
+    }
+  }
+}
+
+module.exports = RestQiitaAPIService;

lib/util/importer.js

@@ -8,13 +8,14 @@ module.exports = crowi => {
   const esa = require('esa-nodejs');
   const config = crowi.getConfig();
   const createGrowiPages = require('./createGrowiPagesFromImports')(crowi);
+  const restQiitaAPIService = crowi.getRestQiitaAPIService();
   let importer = {};
   let esaClient = {};
 
   /**
    * Initialize importer
    */
-  importer.initialize = () => {
+  importer.initializeEsaClient = () => {
     esaClient = esa({
       team:        config.crowi['importer:esa:team_name'],
       accessToken: config.crowi['importer:esa:access_token'],
@@ -22,6 +23,14 @@ module.exports = crowi => {
     logger.info('initialize esa importer');
   };
 
+  /**
+   * Initialize importer
+   */
+  importer.initializeQiitaClient = () => {
+    restQiitaAPIService.reset();
+    logger.info('initialize qiita importer');
+  };
+
   /**
    * Import page data from esa to GROWI
    */
@@ -34,6 +43,10 @@ module.exports = crowi => {
     });
   };
 
+  /**
+   * post page data from esa and create GROWI page
+   * @param {string} pageNum default value is '1'
+   */
   const importPostsFromEsa = (pageNum, user, errors) => {
     return new Promise((resolve, reject) => {
       esaClient.api.posts({page: pageNum, per_page: 100}, async(err, res) => {
@@ -56,6 +69,35 @@ module.exports = crowi => {
     });
   };
 
+  /**
+   * Import page data from qiita to GROWI
+   */
+  importer.importDataFromQiita = async(user) => {
+    const firstPage = 1;
+    const errors = await importPostsFromQiita(firstPage, user, []);
+    return errors;
+  };
+
+  /**
+   * post page data from qiita and create GROWI page
+   * @param {string} pageNum default value is '1'
+   */
+  const importPostsFromQiita = async(pageNum, user, errors) => {
+    const per_page = '100';
+    const res = await restQiitaAPIService.getQiitaPages(pageNum, per_page);
+    const next = pageNum * per_page;
+    const postsReceived = res.pages;
+    const pageTotal = res.total;
+    const data = convertQiitaDataForGrowi(postsReceived, user);
+
+    const newErrors = await createGrowiPages(data);
+    if (next < pageTotal) {
+      return importPostsFromQiita(next, user, errors.concat(newErrors));
+    }
+
+    return errors.concat(newErrors);
+  };
+
   /**
    * Convert data into usable format for createGrowiPagesFromImports
    */
@@ -64,6 +106,7 @@ module.exports = crowi => {
     const data = pages.map(post => {
       const category = post.category;
       const name = post.name;
+
       let path = '';
 
       if (category && name) {
@@ -87,18 +130,36 @@ module.exports = crowi => {
   };
 
   /**
-   * Import page data from esa to GROWI
+   * Convert data into usable format for createGrowiPagesFromImports
    */
-  importer.testConnectionToEsa = () => {
-    return new Promise(async(resolve, reject) => {
-      try {
-        await getTeamNameFromEsa();
-        resolve();
-      }
-      catch (err) {
-        reject(err);
-      }
+  const convertQiitaDataForGrowi = (pages, user) => {
+    const basePage = '';
+    const data = pages.map(post => {
+      const title = post.title;
+      let path = title;
+
+      return {
+        path: `${basePage}/${path}`,
+        body: post.body,
+        user: user,
+      };
     });
+
+    return data;
+  };
+
+  /**
+   * Import page data from esa to GROWI
+   */
+  importer.testConnectionToEsa = async() => {
+    await getTeamNameFromEsa();
+  };
+
+  /**
+   * Import page data from qiita to GROWI
+   */
+  importer.testConnectionToQiita = async() => {
+    await restQiitaAPIService.getQiitaUser();
   };
 
   /**
@@ -116,7 +177,8 @@ module.exports = crowi => {
   };
 
   // initialize when server starts
-  importer.initialize();
+  importer.initializeEsaClient();
+  importer.initializeQiitaClient();
 
   return importer;
 };

lib/views/admin/importer.html

@@ -39,13 +39,13 @@
       </div>
       {% endif %}
 
-      <!-- Importer management forms -->
-      <form action="/_api/admin/settings/importer" method="post" class="form-horizontal" id="importerSettingForm" role="form"
+      <!-- esa Importer management forms -->
+      <form action="/_api/admin/settings/importerEsa" method="post" class="form-horizontal" id="importerSettingFormEsa" role="form"
           data-success-messaage="更新しました">
         <fieldset>
-          <!-- esa importer -->
           <div class="form-group">
             <legend>{{ t('importer_management.esa_settings.title') }}</legend>
+            <input type="password" name="dummypass" style="visibility: hidden; top: -100px; left: -100px;" />
             <div class="form-group">
               <label for="settingForm[importer:esa:team_name]" class="col-xs-3 control-label">{{ t('importer_management.esa_settings.team_name') }}</label>
               <div class="col-xs-6">
@@ -55,7 +55,7 @@
             <div class="form-group">
               <label for="settingForm[importer:esa:access_token]" class="col-xs-3 control-label">{{ t('importer_management.esa_settings.access_token') }}</label>
               <div class="col-xs-6">
-                <input class="form-control" type="text" name="settingForm[importer:esa:access_token]" value="{{ settingForm['importer:esa:access_token'] | default('') }}">
+                <input class="form-control" type="password" name="settingForm[importer:esa:access_token]" value="{{ settingForm['importer:esa:access_token'] | default('') }}">
               </div>
             </div>
           </div>
@@ -63,7 +63,7 @@
           <div class="form-group">
             <input type="hidden" name="_csrf" value="{{ csrf() }}" />
             <div class="col-xs-offset-3 col-xs-6">
-              <button id="testConnectionToEsa" type="button" class="btn btn-primary" data-action="/_api/admin/import/esa"
+              <button id="testConnectionToEsa" type="button" class="btn btn-primary btn-esa" data-action="/_api/admin/import/esa" name="Esa"
                   data-success-message="Import posts from esa success." data-error-message="Error occurred in importing pages from esa.io">
                 {{ t("importer_management.import") }}
               </button>
@@ -71,9 +71,50 @@
                 {{ t('Update') }}
               </button>
               <span class="col-xs-offset-1">
-                <button id="importFromEsa" type="button" class="btn btn-default" data-action="/_api/admin/import/testEsaAPI"
+                <button id="importFromEsa" type="button" class="btn btn-default btn-esa" data-action="/_api/admin/import/testEsaAPI" name="Esa"
                     data-success-message="Test connection to esa success." data-error-message="Test connection to esa failed.">
-                  {{ t("importer_management.test_connection") }}
+                  {{ t("importer_management.esa_settings.test_connection") }}
+                </button>
+              </span>
+            </div>
+          </div>
+        </fieldset>
+      </form>
+      <!-- qiita:team Importer management forms -->
+      <form action="/_api/admin/settings/importerQiita" method="post" class="form-horizontal" id="importerSettingFormQiita" role="form"
+      data-success-messaage="更新しました">
+        <fieldset>
+          <div class="form-group">
+            <legend>{{ t('importer_management.qiita_settings.title') }}</legend>
+            <input type="password" name="dummypass" style="visibility: hidden; top: -100px; left: -100px;" />
+            <div class="form-group">
+              <label for="settingForm[importer:qiita:team_name]" class="col-xs-3 control-label">{{ t('importer_management.qiita_settings.team_name') }}</label>
+              <div class="col-xs-6">
+                <input class="form-control" type="text" name="settingForm[importer:qiita:team_name]" value="{{ settingForm['importer:qiita:team_name'] | default('') }}">
+              </div>
+            </div>
+            <div class="form-group">
+              <label for="settingForm[importer:qiita:access_token]" class="col-xs-3 control-label">{{ t('importer_management.qiita_settings.access_token') }}</label>
+              <div class="col-xs-6">
+                <input class="form-control" type="password" name="settingForm[importer:qiita:access_token]" value="{{ settingForm['importer:qiita:access_token'] | default('') }}">
+              </div>
+            </div>
+          </div>
+
+          <div class="form-group">
+            <input type="hidden" name="_csrf" value="{{ csrf() }}" />
+            <div class="col-xs-offset-3 col-xs-6">
+              <button id="testConnectionToQiita" type="button" class="btn btn-primary btn-qiita" data-action="/_api/admin/import/qiita" name="Qiita"
+                  data-success-message="Import posts from qiita:team success." data-error-message="Error occurred in importing pages from qiita:team">
+                {{ t("importer_management.import") }}
+              </button>
+              <button type="submit" class="btn btn-secondary">{# the first element is the default button to submit #}
+                {{ t('Update') }}
+              </button>
+              <span class="col-xs-offset-1">
+                <button id="importFromQiita" type="button" class="btn btn-default btn-qiita" data-action="/_api/admin/import/testQiitaAPI" name="Qiita"
+                    data-success-message="Test connection to qiita:team success." data-error-message="Test connection to qiita:team failed.">
+                  {{ t("importer_management.qiita_settings.test_connection") }}
                 </button>
               </span>
             </div>
@@ -137,10 +178,10 @@
   }
 
   /**
-   * Handle button
+   * Handle button esa
    */
-  $('#testConnectionToEsa, #importFromEsa').each(function() {
-    var $form = $('#importerSettingForm');
+  $('.btn-esa, .btn-qiita').each(function() {
+    var $form = $('#importerSettingForm' + $(this).attr('name'));
     var $button = $(this);
     var $action = $button.attr('data-action');
     var $success_msg = $button.attr('data-success-message');
@@ -149,11 +190,11 @@
   });
 
   /**
-   * Handle submit button
+   * Handle submit button esa
    */
-  $('#importerSettingForm').each(function() {
+  $('#importerSettingFormEsa, #importerSettingFormQiita').each(function() {
     var $form = $(this);
-    var $button = $("#importerSettingForm input[type='submit']");
+    var $button = $("#importerSettingForm" + $(this).attr('name') + " button[type='submit']");
     var $action = $form.attr('action');
     var $success_msg = $button.attr('data-success-message');
     var $error_msg = $button.attr('data-error-message');