Erez Erez - 3 months ago 14
Node.js Question

node js - multi tasking for each item in array

I am trying to implement a way to upload files asynchronously.
I have a process I want to apply to every item of my array.
I am taking the name of each item, call a API to get additinal information about it, then I am sending it to a text to speech utility, and upload the resulting

wav
file to a S3 instance.

I can't find a way to do this asynchronously, and wait for all of them to finish.

I can do it in serie, but it take lots of time (12 minutes for 30 files (2mb each file)).

I tried to implement a asynchronous way, which takes around 5 minutes (7 minutes less), but I fear the problem is on the net line?

Function to apply to each item:

function doAll(c, lan, country, fileName, callback){
getNews(c, lan)
.then(function(newsResults){
getWavFile(newsResults, lan, fileName)
.then(function(wavResults){
uploadToS3(country,lan,fileName)
.then(function(s3Results){
return callback("done");
}, function(s3err){
console.log('s3 error: ',s3err);
return callback("done");
})
}, function(waverr){
console.log('wav error: ',waverr);
})
}, function(newserr){
console.log('news error: ',newserr);
})
}


Array example :

var arr = [
{
_id: '5769369ba2d42fd82ca4d851',
Name: 'Sports',
Order: 1,
Color: 'White',
Description: 'ספורט',
UpdatedDate: '2016-07-28T07:44:47.906Z',
CreatedDate: '2016-06-21T12:44:11.468Z',
Country: 'IL',
Langs: [
{
Name: 'Sports',
IsoCode: 'en',
Url: 'SportsJSON',
_id: '576b93486c7a9ff025275836'
},
{
Name: 'ספורט',
IsoCode: 'iw',
Url: 'HebrewSportsJSON',
_id: '576be6ad56126ccc25852613'
}
]
},
{
_id: '576bf4eb28176a3e5ce15afa',
Name: 'Top Stories',
Description: 'הכותרות',
Color: 'ww',
Order: 1,
UpdatedDate: '2016-07-10T12:01:26.713Z',
CreatedDate: '2016-06-23T14:40:43.435Z',
Country: 'IL',
Langs: [
{
Name: 'כותרות',
Url: 'HebrewTopStoriesJSON',
IsoCode: 'iw',
_id: '576bf52228176a3e5ce15afb'
},
{
Name: 'Top Stories',
IsoCode: 'en',
Url: 'TopStoriesJSON',
_id: '576bf94d28176a3e5ce15afd'
}
]
},
{
_id: '5756d5d6c4a3dfe478b16aa2',
Description: 'Nation Channel',
Order: 1,
Color: 'blue',
Code: 'Nation',
Name: 'Nation',
UpdatedDate: '2016-06-24T22:23:07.198Z',
CreatedDate: '2016-06-07T14:10:30.699Z',
Country: 'US',
Langs: [
{
Name: 'Nation',
IsoCode: 'en',
Url: 'NationJson',
_id: '576db2cb28176a3e5ce15b02'
}
]
}
]


My asynchronous way:

var array = [] // see the example how array look like
var newArray= [];
console.log('start uploading files time:', new Date());
for (var i = 0; i < array.length; i++) {
var list = array[i].Langs;
for (var j= 0; j < list.length; j++) {
var c = list[j];
var lan = convertIsoCode(c.IsoCode);
var fileName = array[i].Name + "_" + lan;
var country = array[i].Country;
doAll(c,lan,country,fileName, function(){
newArray.push(array[i]);
if (array.length == newArray.length) {
console.log('done');
defer.resolve('done');
}
})
}

}


EDIT:

I tried to do it with
async.each
and
async.parallel
, but didn't succeed, can anyone show me the right way to implement it?

Answer

Removed newArray since you don't need it for anything useful, it was wasting CPU time and was a horrendous way of tracking what was done. A simple counter would have done the tricks.

Gone ES6 since it's 2016. Also added semi colon because you were using them inconstitently.

Also, doAll is not a meaningful name.

'use strict';

const async = require('async');

let array = [/*data*/];

console.log('START ' + (new Date()));
//Asynchronously iterate throught the array
async.each(array, (item, callback) => {
    //item is your array[i]
    async.each(item.Langs, (lang, callback) => {
        //lang is your array[i].Langs[j]
        let lan = convertIsoCode(item.IsoCode),
            fileName = item.Name + '_' + lan,
            country = item.Country;

        //Apply your functions
        getNews(c, lan).then((newsResults) => {
            getWavFile(newsResults, lan, fileName).then((wavResults) => {
                uploadToS3(country,lan,fileName).then((s3Results) => {
                    //Everything is OK, callback without error
                    callback();
                }, (s3err) => {
                    //Raise the error
                    callback(s3err);
                });
            }, (waverr) => {
                console.log('wav error: ',waverr);
                //Raise the error
                callback(waverr);
            });
        }, (newserr) => {
            console.log('news error: ',newserr);
            //Raise the error
            callback(newserr);
        });
    }, (error) => {
        callback(error);
    });
}, (error) => {
    //If a error was raised, everything pending will be aborted and the error will be displayed
    if(error) {
        console.log(error);
    //Else, just report it did fine
    } else {
        console.log('OK');
    }
});