Hisni Fazlija Hisni Fazlija - 7 months ago 55
Javascript Question

How do I write a synchronous while loop using node.js and promises

I would like to call a function which takes data for every student for a specific time interval and repeat the same process until it reaches some condition.

The current code it seems to make parallel calls to the function iterateThruAllStudents();

My code looks like this:

var startDate = new Date("March 16, 2016 00:00:00"); //Start from February

var fromTimestamp = null;
var toTimestamp = null;
var today = new Date();
var todayTimestamp = new Date(today).getTime() / 1000;

async.whilst(
function () {
return fromTimestamp < todayTimestamp;
},
function (callback) {
console.log(startDate);
fromTimestamp = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 5);
toTimestamp = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(fromTimestamp, toTimestamp);
callback(null, startDate);
},
function (err, n) {
console.log('finished for ' + n);
}
);

function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}

var counter = 0;
async.eachSeries(students, function iteratee(student, callback) {
if (student.worksnap.user != null) {
var worksnapOptions = {
hostname: 'api.worksnaps.com',
path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic ' + auth_hash
},
method: 'GET'
};

getTimeEntries(worksnapOptions)
.then(function (response) { //callback invoked on deferred.resolve
return convertXMLToJson(response);
}).then(function (timeEntries) {
console.log('convert xml to json');
var isEmpty = _.isEmpty(timeEntries); // true
if (isEmpty) {
callback(null);
}
return saveTimeEntry(timeEntries);
}).then(function (response) {
counter++;
console.log('all timeEntries for one student finished....Student: ' + student.worksnap.user.user_id + ' Student Counter: ' + counter);
callback(null);
});
} else {
callback(null);
}
});
});
}

function convertXMLToJson(response) {
var deferred = Q.defer();
parser.parseString(response, function (err, results) {
if (err) {
deferred.reject(err);
}
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
deferred.resolve(timeEntries);
});
return deferred.promise;
}

function saveTimeEntry(timeEntries) {
var deferred = Q.defer();
_.forEach(timeEntries.time_entry, function (item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
deferred.reject(err);
}
student.worksnap.timeEntries.push(item);
student.save(function (err) {
if (err) {
deferred.reject(err);
} else {
//console.log(item);
}
});

});
});
deferred.resolve('finished saving timeEntries for one student...');

return deferred.promise;
}

function getTimeEntries(requestOptions) {
//create a deferred object from Q
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var deferred = Q.defer();
var req = http.request(requestOptions, function (response) {
//set the response encoding to parse json string
response.setEncoding('utf8');
var responseData = '';
//append data to responseData variable on the 'data' event emission
response.on('data', function (data) {
responseData += data;
});
//listen to the 'end' event
response.on('end', function () {
//resolve the deferred object with the response
console.log('http call finished');
deferred.resolve(responseData);
});
});

//listen to the 'error' event
req.on('error', function (err) {
//if an error occurs reject the deferred
console.log('inside On error.');
console.log(err);
deferred.reject(err);
});
req.end();
//we are returning a promise object
//if we returned the deferred object
//deferred object reject and resolve could potentially be modified
//violating the expected behavior of this function
return deferred.promise;
}


Anyone has an idea how to achieve such thing that I can grab data for all students synchronously for each time interval?

Answer

Take a look at the docs for async.eachSeries() (alias of #each()). You'll want to supply a third argument, and call the callback originating from the argument in the second function of async.whilst().

I think the following modifications will do what you need. Take a look for the callback I named done, specifically:

var startDate = new Date("March 16, 2016 00:00:00");  //Start from February

var fromTimestamp = null;
var toTimestamp = null;
var today = new Date();
var todayTimestamp = new Date(today).getTime() / 1000;

async.whilst(
    function () {
        return fromTimestamp < todayTimestamp;
    },
    function (callback) {
        console.log(startDate);
        fromTimestamp = new Date(startDate).getTime() / 1000;
        startDate.setDate(startDate.getDate() + 5);
        toTimestamp = new Date(startDate).getTime() / 1000;
        iterateThruAllStudents(fromTimestamp, toTimestamp, callback);
    },
    function (err, n) {
        console.log('finished for ' + n);
    }
);

function iterateThruAllStudents(from, to, done) {
    Student.find({status: 'student'})
        .populate('user')
        .exec(function (err, students) {
            if (err) {
                throw err;
            }

            var counter = 0;
            async.eachSeries(students, function iteratee(student, callback) {
                if (student.worksnap.user != null) {
                    var worksnapOptions = {
                        hostname: 'api.worksnaps.com',
                        path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
                        headers: {
                            'Authorization': 'Basic ' + auth_hash
                        },
                        method: 'GET'
                    };

                    getTimeEntries(worksnapOptions)
                        .then(function (response) { //callback invoked on deferred.resolve
                            return convertXMLToJson(response);
                        }).then(function (timeEntries) {
                            console.log('convert xml to json');
                            var isEmpty = _.isEmpty(timeEntries); // true
                            if (isEmpty) {
                                callback(null);
                            }
                            return saveTimeEntry(timeEntries);
                        }).then(function (response) {
                            counter++;
                            console.log('all timeEntries for one student finished....Student: ' + student.worksnap.user.user_id + ' Student Counter: ' + counter);
                            callback(null);
                        });
                } else {
                    callback(null);
                }
            }, function eachSeriesFinished(err) {
                if (err)
                    return done(err);
                return done(null, to);
            });
        });
}

function convertXMLToJson(response) {
    var deferred = Q.defer();
    parser.parseString(response, function (err, results) {
        if (err) {
            deferred.reject(err);
        }
        var json_string = JSON.stringify(results.time_entries);
        var timeEntries = JSON.parse(json_string);
        deferred.resolve(timeEntries);
    });
    return deferred.promise;
}

function saveTimeEntry(timeEntries) {
    var deferred = Q.defer();
    _.forEach(timeEntries.time_entry, function (item) {
        Student.findOne({
                'worksnap.user.user_id': item.user_id[0]
            })
            .populate('user')
            .exec(function (err, student) {
                if (err) {
                    deferred.reject(err);
                }
                student.worksnap.timeEntries.push(item);
                student.save(function (err) {
                    if (err) {
                        deferred.reject(err);
                    } else {
                        //console.log(item);
                    }
                });

            });
    });
    deferred.resolve('finished saving timeEntries for one student...');

    return deferred.promise;
}

function getTimeEntries(requestOptions) {
    //create a deferred object from Q
    process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
    var deferred = Q.defer();
    var req = http.request(requestOptions, function (response) {
        //set the response encoding to parse json string
        response.setEncoding('utf8');
        var responseData = '';
        //append data to responseData variable on the 'data' event emission
        response.on('data', function (data) {
            responseData += data;
        });
        //listen to the 'end' event
        response.on('end', function () {
            //resolve the deferred object with the response
            console.log('http call finished');
            deferred.resolve(responseData);
        });
    });

    //listen to the 'error' event
    req.on('error', function (err) {
        //if an error occurs reject the deferred
        console.log('inside On error.');
        console.log(err);
        deferred.reject(err);
    });
    req.end();
    //we are returning a promise object
    //if we returned the deferred object
    //deferred object reject and resolve could potentially be modified
    //violating the expected behavior of this function
    return deferred.promise;
}

Side note, if you were open to it, using something like co or async/await would simplify this code a lot, in my opinion.