Flame_Phoenix Flame_Phoenix - 3 months ago 10
Node.js Question

Underscore.js throttle not working

Objective



Figure what is wrong with my code, or if underscore.js throttle works as it should.

Background



I have a huge list of postal codes in a file, and I am reading those codes and pasting them on console.

I am trying to use Underscore.js throttle() function, however my code stops after two runs (even though I have dozens), and the rest of the values are never printed.

Code



My code is in a very simple NodeJS project. I created a MCVE of the situation I am facing:

"use strict";

//requiremetns
let fs = require('fs');
let readLine = require('readline');
let _ = require('underscore');

//constants
const INPUT_FILE = 'dataset.txt';
const RADIX_CONVERSATION = 10;
const THROTTLE_DELAY = 500;

let init = function() {

let lineReader = readLine.createInterface({
input: fs.createReadStream(INPUT_FILE),
output: process.stdout,
terminal: false
});

let throttledRequestFn = _.throttle(requestFn, THROTTLE_DELAY);

lineReader.on('line', function(line) {
line = line.trim();

if (_.isNaN(parseInt(line, RADIX_CONVERSATION))) {
//Do some stuff
}
else {
throttledRequestFn('mahCountry', line);
}
});
};

let requestFn = function(country, postalCode){
console.log('request for ' + country + ' and postal ' + postalCode + ' done');
return false;
};


init();


Here I first start by reading the file, one line at a time. Then if the line I am reading is a number, I print something, otherwise nothing.

Following is a test file:

Vietnam
000000
100000
160000
170000
180000
200000
220000
230000
240000
250000
260000
270000
280000
290000
300000
310000
320000
330000
350000
360000
380000
390000
400000
410000
420000
430000
440000
460000
480000
510000
520000
530000
550000
560000
570000
580000
590000
600000
620000
630000
640000
650000
660000
670000
700000
790000
800000
810000
820000
830000
840000
850000
860000
870000
880000
890000
900000
910000
920000
930000
940000
950000
960000
970000


Question



The way I see it, my code should make 2 requests per second, with a 500 ms of delay between each one. It should print all the codes in the test file.
However, I never seen anything past the second value! Why is this happening?

Answer

The throttle function is working as intended. From the documentation:

Useful for rate-limiting events that occur faster than you can keep up with.

This means that your wrapped function will probably be called less often than you want.

What you actually want is probably some kind of queue. Underscore doesn't provide one, but the async library does: http://caolan.github.io/async/docs.html#.queue

let fs = require('fs');
let readLine = require('readline');
let _ = require('async');
// import the async library
let async = require('async');

const INPUT_FILE = 'dataset.txt';
const RADIX_CONVERSATION = 10;

// create the queue
var q = async.queue(function(task, callback) {
  // make sure the queue task calls the callback only after the THROTTLE_DELAY
  setTimeout(function () {
    requestFn(task.country, task.postalCode);
    callback();
  }, THROTTLE_DELAY);

}, 1)

q.drain = function () {
  console.log('all items have been processed');
};

let init = function () {
  let lineReader = readLine.createInterface({
    input: fs.createReadStream(INPUT_FILE),
    output: process.stdout,
    terminal: false
  });

  lineReader.on('line', function(line) {
    line = line.trim();

    if (_.isNaN(parseInt(line, RADIX_CONVERSATION))) {
      // Do some stuff
    }
    else {
      // Add the line to the Queue, to be executed later
      q.push({country: 'mahCountry', postalCode: line});
    }
  });
};

let requestFn = function(country, postalCode){
  console.log('request for ' + country + ' and postal ' + postalCode + ' done');
  return false;
};

init();

Notice the use of setTimeout in the function that handles the elements in the queue. That way, you'll still only make one request every 500ms, but will be guaranteed to make all of them.

Comments