Loourr Loourr - 3 months ago 75
Javascript Question

Download file from url and upload it to AWS S3 without saving - node.js

I'm writing an application which downloads images from a url and then uploads it to an S3 bucket using the aws-sdk.

Perviously I was just downloading images and saving them to disk like this.

request.head(url, function(err, res, body){

request(url).pipe(fs.createWriteStream(image_path));

});


And then uploading the images to AWS S3 like this

fs.readFile(image_path, function(err, data){
s3.client.putObject({
Bucket: 'myBucket',
Key: image_path,
Body: data
ACL:'public-read'
}, function(err, resp) {
if(err){
console.log("error in s3 put object cb");
} else {
console.log(resp);
console.log("successfully added image to s3");
}
});
});


But I would like to skip the part where I save the image to disk. Is there some way I can
pipe
the response from
request(url)
to a variable and then upload that?

Answer

Heres some code to do it in coffee script

    options = {
        uri: uri
        encoding: 'binary'
    }
    request(options, (error, response, body) =>
        if error? or response.statusCode isnt 200 
            console.log "failed to get image"
            console.log error
        else 
            body = new Buffer(body, 'binary')

            @s3.putObject({
                Body: body
                Key: path
                Bucket: 'bucket_name'
            }, (error, data) =>
                if error?
                    console.log "error downloading image to s3"
                else 
                    console.log "success uploading to s3"
            )           
    )

based on Armadillo Jim's comment the a simplified version would look like this

    options = {
        uri: uri
        encoding: null
    }
    request(options, (error, response, body) =>
        if error? or response.statusCode isnt 200 
            console.log "failed to get image"
            console.log error
        else 
            @s3.putObject({
                Body: body
                Key: path
                Bucket: 'bucket_name'
            }, (error, data) =>
                if error?
                    console.log "error downloading image to s3"
                else 
                    console.log "success uploading to s3"
            )           
    )

passing null for encoding causes the body to get returned as a buffer.

And in javascript it looks like:

    var options = {
        uri: uri,
        encoding: null
    };
    request(options, function(error, response, body) {
        if (error || response.statusCode !== 200) { 
            console.log("failed to get image");
            console.log(error);
        } else {
            s3.putObject({
                Body: body,
                Key: path,
                Bucket: 'bucket_name'
            }, function(error, data) { 
                if (error) {
                    console.log("error downloading image to s3");
                } else {
                    console.log("success uploading to s3");
                }
            }); 
        }   
    });