Kw3si Kw3si - 3 months ago 22
Linux Question

Node.js - exec command for tar files works correctly first time, but produces corrupted tar contents upon subsequent execution

I am building a web app with Node.js, and I am at a point where I need to produce a tar archive of a directory of PDFs. The application is running on a VM running Ubuntu 14.04 server. My code to do this is shown below:

function tarDirectory(path, token, callback) {
var exec = require('child_process').exec;
var cmd = 'cd ' + path + ' && tar -cvf genericName-' + token + '.tar' + ' ' + token;

exec(cmd, function(error, stdout, stderr) {
console.log(stdout);
console.log(stderr);
if (error) {
console.error(error);
}
if(callback) callback();
});
}


and this tarDirectory function is called by the following code:

router.post('/files/generate', function(req, res, next) {
IDList = req.body['IDs[]'];
token = req.body['token'];

// if just a single file being generated
if (typeof req.body['IDs[]'] === "string"){
filehelper.generateFile(IDList[0], req.app.locals.site.basedir + "temp/", token);
}
// if multiple files being generated
else {
IDList.forEach(function(id) {
filehelper.generateFile(id, req.app.locals.site.basedir + "temp/", token);
});
}
filehelper.tarDirectory(req.app.locals.site.basedir + "temp/", token, res.end);
});


The code expects a post request with dynamic data that is generated by a button click in my web app, and will then create files based on the data and tar it into a directory. This all works fine and good... the first time. When I click the button for the first time in a while, the tar is produced, and when I open it, the client-side PDFs are identical to the ones on the server. When I click again within an hour or so, though, I receive a tar file, but when I open the archive and unpack it, the PDFs are all corrupted and about half the expected byte size. I am at a loss here... I had a suspicion that it might be related to improper handling of stream closing, but I'm not sure.

This is the code that generates the PDFs into a directory, which is then tarred after the generation:

function generateFile(id, path, token) {
var dirPath = path + token;
var filePath = path + token + "/file" + id + ".pdf";

console.log("creating file for: " + id);

try{
fs.statSync(dirPath).isDirectory();
} catch (err) {
fs.mkdirSync(dirPath);
}
// start the file pdf generation
file = new PDFDocument();
output = fs.createWriteStream(filePath);
output.on('close', function(){
return;
});

file.pipe(output);

// handle the intricacies of the file generation
file.text("file" + id + ".pdf");

// end the file
file.end();
}

Answer
  1. Is everything okay with pdf files, before compressing?
  2. In your generateFile function you have WriteStream, which is async. But, you calling this function as sync., and start .tar compression without waiting before pdf generation be complete, which may cause this issue.
  3. As recommendation: try to wrap generateFile with promise, or iterate async., and start compression only after all files generation completed.

Example with bluebird:

var Promise = require('bluebird');

function generateFile(id, path, token) {
  return new Promise(function(resolve, reject) {
  var dirPath = path + token;
  var filePath = path + token + "/file" + id + ".pdf";

  console.log("creating file for: " + id);

  try{
    fs.statSync(dirPath).isDirectory();
  } catch (err) {
    fs.mkdirSync(dirPath);
  }
  // start the file pdf generation
  file = new PDFDocument();
  output = fs.createWriteStream(filePath);
  output.on('close', function(){
    return resolve();
  });

  output.on('error', function(error) {
    return reject(error);
  });

  file.pipe(output);

  // handle the intricacies of the file generation
  file.text("file" + id + ".pdf");

  // end the file
  file.end();
  });
}

Pdfs generation and compressing.

 var Promise = require('bluebird');

    ....

    //IDList.forEach(function(id) {
    //      filehelper.generateFile(id, req.app.locals.site.basedir + "temp/", //token);});

    //replace with

    Promise.map(IDList, function(id) {
      return filehelper.generateFile(id, req.app.locals.site.basedir + "temp/", token);
    })
    .then(function() {
    //all files are ready, start compressing
    })
    .catch(function(error) {
    //we have error
    });
Comments