From f6f49da1403a06d93be1b02406b7c7a37898f0d9 Mon Sep 17 00:00:00 2001 From: Jean-Denis Caron Date: Mon, 9 Mar 2020 17:34:56 -0400 Subject: [PATCH 1/2] Fix rate limit of 600 requests per minutes on Gitlab.com This is a simple fix, maximum of 10 requests per seconds = maximum of 600 requests per minutes. It does slow down a bit the report but it prevent a fatal error. It's possible to design a more sophisticated fix but it will be more complex with more states to manage. It would be more prone to bugs. I favor simplicity over maximum speed. --- package.json | 1 + src/models/base.js | 57 ++++++++++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index 8dc55e8..9496a9c 100755 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "request-promise-native": "^1.0.4", "shelljs": "^0.8.3", "tempfile": "^2.0.0", + "throttled-queue": "^1.0.7", "underscore": "^1.9.1", "xdg-basedir": "^3.0.0", "xlsx": "^0.13.5" diff --git a/src/models/base.js b/src/models/base.js index a7ca006..678e000 100755 --- a/src/models/base.js +++ b/src/models/base.js @@ -2,6 +2,9 @@ const request = require('request-promise-native'); const url = require('url'); const async = require('async'); const crypto = require('crypto'); +const throttledQueue = require('throttled-queue'); + +const throttle = throttledQueue(10, 1000); /** * base model @@ -36,19 +39,21 @@ class base { data.private_token = this.token; return new Promise((resolve, reject) => { - request.post(`${this.url}${path}`, { - json: true, - body: data, - insecure: this._insecure, - proxy: this._proxy, - resolveWithFullResponse: true, - headers: { - 'PRIVATE-TOKEN': this.token - } - }).then(response => { - if (this.config.get('_createDump')) this.setDump(response, key); - resolve(response); - }).catch(e => reject(e)); + throttle(() => { + request.post(`${this.url}${path}`, { + json: true, + body: data, + insecure: this._insecure, + proxy: this._proxy, + resolveWithFullResponse: true, + headers: { + 'PRIVATE-TOKEN': this.token + } + }).then(response => { + if (this.config.get('_createDump')) this.setDump(response, key); + resolve(response); + }).catch(e => reject(e)); + }) }); } @@ -67,18 +72,20 @@ class base { path += `&page=${page}&per_page=${perPage}`; return new Promise((resolve, reject) => { - request(`${this.url}${path}`, { - json: true, - insecure: this._insecure, - proxy: this._proxy, - resolveWithFullResponse: true, - headers: { - 'PRIVATE-TOKEN': this.token - } - }).then(response => { - if (this.config.get('_createDump')) this.setDump(response, key); - resolve(response); - }).catch(e => reject(e)); + throttle(() => { + request(`${this.url}${path}`, { + json: true, + insecure: this._insecure, + proxy: this._proxy, + resolveWithFullResponse: true, + headers: { + 'PRIVATE-TOKEN': this.token + } + }).then(response => { + if (this.config.get('_createDump')) this.setDump(response, key); + resolve(response); + }).catch(e => reject(e)); + }) }); } From e032d1c8ad44b9341d6e966e48c5c500c013f6dc Mon Sep 17 00:00:00 2001 From: Ciprian Dobre-Trifan Date: Mon, 8 Jun 2020 00:09:07 +0300 Subject: [PATCH 2/2] clean up diff & update yarn - whitepace magic to minimize diff - update yarn.lock file --- src/models/base.js | 68 +++++++++++++++++++++------------------------- yarn.lock | 5 ++++ 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/models/base.js b/src/models/base.js index 678e000..e053700 100755 --- a/src/models/base.js +++ b/src/models/base.js @@ -2,9 +2,7 @@ const request = require('request-promise-native'); const url = require('url'); const async = require('async'); const crypto = require('crypto'); -const throttledQueue = require('throttled-queue'); - -const throttle = throttledQueue(10, 1000); +const throttle = require('throttled-queue')(10, 1000); /** * base model @@ -38,23 +36,21 @@ class base { data.private_token = this.token; - return new Promise((resolve, reject) => { - throttle(() => { - request.post(`${this.url}${path}`, { - json: true, - body: data, - insecure: this._insecure, - proxy: this._proxy, - resolveWithFullResponse: true, - headers: { - 'PRIVATE-TOKEN': this.token - } - }).then(response => { - if (this.config.get('_createDump')) this.setDump(response, key); - resolve(response); - }).catch(e => reject(e)); - }) - }); + return new Promise((resolve, reject) => throttle(() => { + request.post(`${this.url}${path}`, { + json: true, + body: data, + insecure: this._insecure, + proxy: this._proxy, + resolveWithFullResponse: true, + headers: { + 'PRIVATE-TOKEN': this.token + } + }).then(response => { + if (this.config.get('_createDump')) this.setDump(response, key); + resolve(response); + }).catch(e => reject(e)); + })); } /** @@ -71,22 +67,20 @@ class base { path += (path.includes('?') ? '&' : '?') + `private_token=${this.token}`; path += `&page=${page}&per_page=${perPage}`; - return new Promise((resolve, reject) => { - throttle(() => { - request(`${this.url}${path}`, { - json: true, - insecure: this._insecure, - proxy: this._proxy, - resolveWithFullResponse: true, - headers: { - 'PRIVATE-TOKEN': this.token - } - }).then(response => { - if (this.config.get('_createDump')) this.setDump(response, key); - resolve(response); - }).catch(e => reject(e)); - }) - }); + return new Promise((resolve, reject) => throttle(() => { + request(`${this.url}${path}`, { + json: true, + insecure: this._insecure, + proxy: this._proxy, + resolveWithFullResponse: true, + headers: { + 'PRIVATE-TOKEN': this.token + } + }).then(response => { + if (this.config.get('_createDump')) this.setDump(response, key); + resolve(response); + }).catch(e => reject(e)); + })); } /** @@ -197,4 +191,4 @@ class base { } } -module.exports = base; \ No newline at end of file +module.exports = base; diff --git a/yarn.lock b/yarn.lock index 2da1b0d..8dfdeda 100755 --- a/yarn.lock +++ b/yarn.lock @@ -2429,6 +2429,11 @@ text-encoding@0.6.4, text-encoding@^0.6.4: version "0.6.4" resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.6.4.tgz#e399a982257a276dae428bb92845cb71bdc26d19" +throttled-queue@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/throttled-queue/-/throttled-queue-1.0.7.tgz#da7ed6702941993044a1c5fd2ac3a58582dd2977" + integrity sha512-/HT49S7m+NvdyJMoMRzIYlawKjeHn8jEc8TZaGmFi5IBu09hIiU/QoP1zcrB9X2qsVC11PgfRfqd8zEQs7PlEA== + throttleit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c"