Fix rate limit of 600 requests per minutes on Gitlab.com (#106)

This is a simple fix, maximum of 10 requests per seconds = maximum of 600 requests per minutes. It does slow down a bit the report but it prevent a fatal error. It's possible to design a more sophisticated fix but it will be more complex with more states to manage. It would be more prone to bugs. I favor simplicity over maximum speed.

Co-authored-by: Jean-Denis Caron <[email protected]>
Co-authored-by: Ciprian Dobre-Trifan <[email protected]>

Author: 3 people

package.json

@@ -57,6 +57,7 @@
     "request-promise-native": "^1.0.4",
     "shelljs": "^0.8.3",
     "tempfile": "^2.0.0",
+    "throttled-queue": "^1.0.7",
     "underscore": "^1.9.1",
     "xdg-basedir": "^3.0.0",
     "xlsx": "^0.13.5"

src/models/base.js

@@ -2,6 +2,7 @@ const request = require('request-promise-native');
 const url = require('url');
 const async = require('async');
 const crypto = require('crypto');
+const throttle = require('throttled-queue')(10, 1000);
 
 /**
  * base model
@@ -35,7 +36,7 @@ class base {
 
         data.private_token = this.token;
 
-        return new Promise((resolve, reject) => {
+        return new Promise((resolve, reject) => throttle(() => {
             request.post(`${this.url}${path}`, {
                 json: true,
                 body: data,
@@ -49,7 +50,7 @@ class base {
                 if (this.config.get('_createDump')) this.setDump(response, key);
                 resolve(response);
             }).catch(e => reject(e));
-        });
+        }));
     }
 
     /**
@@ -66,7 +67,7 @@ class base {
         path += (path.includes('?') ? '&' : '?') + `private_token=${this.token}`;
         path += `&page=${page}&per_page=${perPage}`;
 
-        return new Promise((resolve, reject) => {
+        return new Promise((resolve, reject) => throttle(() => {
             request(`${this.url}${path}`, {
                 json: true,
                 insecure: this._insecure,
@@ -79,7 +80,7 @@ class base {
                 if (this.config.get('_createDump')) this.setDump(response, key);
                 resolve(response);
             }).catch(e => reject(e));
-        });
+        }));
     }
 
     /**
@@ -190,4 +191,4 @@ class base {
     }
 }
 
-module.exports = base;
+module.exports = base;

yarn.lock

@@ -2429,6 +2429,11 @@ [email protected], text-encoding@^0.6.4:
   version "0.6.4"
   resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.6.4.tgz#e399a982257a276dae428bb92845cb71bdc26d19"
 
+throttled-queue@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/throttled-queue/-/throttled-queue-1.0.7.tgz#da7ed6702941993044a1c5fd2ac3a58582dd2977"
+  integrity sha512-/HT49S7m+NvdyJMoMRzIYlawKjeHn8jEc8TZaGmFi5IBu09hIiU/QoP1zcrB9X2qsVC11PgfRfqd8zEQs7PlEA==
+
 throttleit@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c"