direxit direxit - 2 months ago 6
Node.js Question

fs.readFile inside async.waterfall doesn't return data

After much research and tests, I am stucked into my waterfall :

async.waterfall([
//first 5 functions are going fine and passing expected vars to the next function the function below is the pb
// eg adSetId var comes just fine from previous function
function(imgName,adsetId,callback) {
fs.readFile('./shopifyImages/'+imgName , 'base64' ,function(err,imgData) {
callback(null,imgData,adsetId);
});
}
]);


imgData
is always returning empty value.
There is no error from the function and its working fine ouside of the waterfall (
fs.readFile
works as expected), some
console.log
gives empty
imgData
as well.

FULL CODE

async.waterfall([
//GET AD ACCOUNT FOR USER
function(callback) {
var sqlAd_act = vsprintf('select * from fbTokens where userId=%s', [roomIDCook.userId]);
connection.query(sqlAd_act).then(function(rows) {
//console.log('rows '+rows);
callback(null, rows[0].ad_act);
});
},
//SET CAMPAIGN
function(ad_act, callback) {
fb.api('/act_' + ad_act + '/campaigns', 'post', {
'status': 'PAUSED',
'name': 'GENIE 2'
}, function(data) {
//console.log('data: ' + util.inspect(data))
callback(null, data.id, ad_act);
});
},
//SET ADSET
function(campaignId, ad_act, callback) {
console.log('ad account ' + ad_act);
console.log('campaignId ' + campaignId);
fb.api('/act_' + ad_act + '/adsets', 'post', {
'daily_budget': 500,
'start_time': '2017-02-18T20:11:25+0000',
'end_time': '2017-03-25T20:11:25+0000',
'name': 'new adset',
'optimization_goal': 'LINK_CLICKS',
'objective': 'LINK_CLICKS',
'campaign_id': campaignId,
'status': 'PAUSED',
'billing_event': 'LINK_CLICKS',
'bid_amount': 2,
'targeting': {
"geo_locations": {
"countries": ["US"]
},
"publisher_platforms": ["facebook"]
}
},
function(res) {
if (!res || res.error) {
console.log(!res ? 'error occurred' : util.inspect(res.error));
return;
}
//console.log(util.inspect(res));
callback(null, res.id)
});
},
// CREATE THE AD FOR THE ADSET
function(adsetId, callback) {
// set img
var imgUri = 'https://cdn.shopify.com/s/files/1/1336/5343/products/9_b75df43c-1660-4aeb-89b7-418df1a9853c_1024x1024.jpg?v=1466985077';
var DOWNLOAD_DIR = 'shopifyImages/';
var imgName = 'TEST2.jpg';
console.log('adsetId ' + adsetId);
// download img
request.head(imgUri, function(err, res, body) {
console.log('content-type:', res.headers['content-type']);
console.log('content-length:', res.headers['content-length']);
request(imgUri).pipe(fs.createWriteStream(DOWNLOAD_DIR + imgName));
console.log('adsetId ' + adsetId);
callback(null, imgName, adsetId);
});
// read img for facebook
},
function(imgName, adsetId, callback) {
fs.readFile('./shopifyImages/' + imgName, 'base64', function(err, imgData) {
console.log(util.inspect(err));
console.log(util.inspect(imgData));
callback(null, imgData, adsetId);
});
},
function(imgData, adsetId, callback) {
console.log(imgData);
console.log('adsetId ' + adsetId);
}
], function(error, c) {
console.log(c);
});

Answer

Because streams are asynchronous, you have a timing issue. You are doing this:

request(imgUri).pipe(fs.createWriteStream(DOWNLOAD_DIR + imgName));
callback(null, imgName, adsetId);

But, you are not waiting for the streams to complete before calling the callback() and going on to the next step where you expect the imgName file to already exist. So, you are trying to read the newly created file before it has finished being written.

You need to register an event handler for that stream so you know when that stream has finished writing and then and only then should you call the callback(null, imgName, adsetId) to go onto the next step.

I'm not a streams whiz so there might be a more elegant way to do this, but here's one solution:

// CREATE THE AD FOR THE ADSET
function(adsetId, callback) {
    // set img
    var imgUri = 'https://cdn.shopify.com/s/files/1/1336/5343/products/9_b75df43c-1660-4aeb-89b7-418df1a9853c_1024x1024.jpg?v=1466985077';
    var DOWNLOAD_DIR = 'shopifyImages/';
    var imgName = 'TEST2.jpg';
    console.log('adsetId ' + adsetId);
    // download img
    request.head(imgUri, function(err, res, body) {
        console.log('content-type:', res.headers['content-type']);
        console.log('content-length:', res.headers['content-length']);
        console.log('adsetId ' + adsetId);
        var outputStream = fs.createWriteStream(DOWNLOAD_DIR + imgName);
        outputStream.on('finish', function() {
            callback(null, imgName, adsetId);
        });
        outputStream.on('error', function(err) {
            callback(err);
        });
        request(imgUri).pipe(outputStream);
    });

FYI, it's conceptually odd to write data to a file in one step and then in the next step read it back into memory. Quite inefficient. Why not just read it the first time into memory or if you still want it in a file, then write it to a file, but retain in memory the data you wrote?