Ardzii Ardzii - 1 year ago 114
Node.js Question

Request JSON within a loop

I'm having trouble to surf correctly between objects that are been returned by an API request.

Basically, I have an Array (

tickerArr
with 25 elements that I get from another request) which I use for a
forEach
loop for another request.

Here's what the code looks like:

var request = require('request');
var async = require('async');
var fs = require('fs')

var getTickerList = require('./getTickerList').getTickerList;
var unixTimeStamp = new Date().toISOString();

async function getTickerInfo() {

var tickerArr = await getTickerList;
var arrLength = tickerArr.length;
var init = 0;

console.log(`${arrLength} to process to the batchfile...`);
var getTickerInfo = new Promise((resolve, reject) => {
async.forEach(tickerArr, (item, callback) => {
request({
url:`https://api.kraken.com/0/public/Ticker?pair=${item}`,
json: true
}, (error, response, body) => {
var tickerInfo = JSON.stringify(body.result)
fs.appendFile(`./modules/batches/${unixTimeStamp}batch.json`, tickerInfo, (err) => {
if (err) throw err;
init ++;
var progress = Math.round(init / arrLength * 100);
console.log(`[${progress}%] - ${item} successfully added!`);
});
resolve();
})
});
});
}

getTickerInfo()


Unfortunately, and even if the request works correctly, the objects being returned have a specific path:



  • Error: []

  • Result:


    • Pair(x):


      • a: [1, 2, 3]

      • b: [1, 2, 3] ect...






You can find an exact example of the information being returned from the request here:

{
"error": [],
"result": {
"XXBTZEUR": {
"a": ["2212.31000", "3", "3.000"],
"b": ["2212.21100", "1", "1.000"],
"c": ["2212.31000", "0.15800000"],
"v": ["4999.06419498", "9993.55448658"],
"p": ["2206.04624", "2181.36028"],
"t": [15065, 29524],
"l": ["2167.00000", "2122.92000"],
"h": ["2239.00000", "2239.00000"],
"o": "2184.99000"
}
}
}


The only problem is: I would like to 'transform' the object I get from the request, to another object (just arranging stuff around, most of all: put the 'Pair(x)' attribute as a value for a name key) but since I don't know the pair in advance (the values in my
tickerArray
are enough to make the request but do not apparently correspond to reference the object), I can't access the information contained within result to manipulate it.

Anybody have any idea?

Thanks in advance!

Answer Source

Your question has multiple levels and maybe it's best to break up the task into its constituent parts first.

  1. Calling the Kraken REST API and unwrapping the response
  2. Creating small, composable functions
  3. Transforming an object into a new layout
  4. Writing the results to a file

0. Preface

Since you want to work with async/await, the first thing to know is that this is syntactical sugar for "function returns promise"/"wait for the promise resolution". So this

async function foo() {
    return somePromise();
}

var fooResult = await foo();
// do something with fooResult

and this

function foo() {
    return somePromise();
}

foo().then(function (fooResult) {
    // do something with fooResult
});

are effectively the same thing. (As always, it's a little more complicated than that, but that's the gist of it.)

Since we are working with promises here, the most sensible thing to do is to use libraries that also use promises. request has the request-promise counter-part, so let's use that.

var request = require('request-promise');

1. Calling the Kraken REST API and unwrapping the response

Making an HTTP request and breaking up the response (err and result parts) works the same for all API endpoints, so we can write a function that handles this task.

async function getFromApi(endpoint, params) {
  return request({
    url: "https://api.kraken.com/0/public/" + endpoint,
    qs: params,
    json: true
  }).then(function (data) {
    if (data.err.length) throw new Error(`API error xyz for ${endpoint} with params ...`);
    return data.result;
  });
}

Users of this function will not have to deal with err at all and have access to result directly.

2. Creating small, composable functions

Our getFromApi() function returns a promise, therefore we can re-use it in async wrappers for various API endpoints:

async function getTickerList() {
  return getFromApi("TickerList"); // ...or something like that
}

async function getSingleTicker(pair) {
  return getFromApi("Ticker", {pair: pair}).then(transformTicker);
}

async function getTickerInfo(tickerList) {
  return Promise.all(tickerList.map(getSingleTicker));
}

Note that, like in getFromApi(), we can use Promise#then to modify the overall output of an operation.

getTickerInfo() accepts an array of ticker names. It uses Array#map to run all API requests in parallel, and Promise#all to allow awaiting the overall result. The fairly complex operation of fetching and transforming multiple results in parallel composes into a pretty straight-forward one-liner.

3. Transforming an object into a new layout

transformTicker() is meant to accept an object in form {"XXBTZEUR": {"a": [...], ... }} and return a transformed variant of it.

function transformTicker(ticker) {
  var result = {};

  // ...massage ticker into desired form
  Object.keys(ticker).forEach( pair => {
      Object.keys(ticker[pair]).forEach( k => {
          result[k] = { /* ... */ };
      });
  });

  return result;
}

4. Writing the results to a file

Appending to a JSON file does not work. JSON can only be read and written as a whole. Let's fetch a list of ticker names, the associated tickers and write the results to a file.

var tickerList = await getTickerList();
var tickerInfo = await getTickerInfo(tickerList);
var filename = `./modules/batches/${new Date().toISOString()}batch.json`;

fs.writeFile(filename, JSON.stringify(tickerInfo), (err) => {
  if (err) throw err;
  console.log("Done");
});

You can switch use a promisified version (see Bluebird#promisifyAll) of the fs module, since the plain version breaks the nice async/await semantics again by requiring a continuation callback.

Recommended from our users: Dynamic Network Monitoring from WhatsUp Gold from IPSwitch. Free Download