Brad Brad - 15 days ago 8
Node.js Question

child_process stream backpressure

I'm using

with Node.js and piping that stream through a few other transform streams, eventually through a
node-brake
stream to limit the data rate. The braking stream seems to have no effect, and in fact data eventually is lost at the end of a long chain.

execStream('some-external-binary').pipe(transform1).pipe(transform2).pipe(brake(1024))


What I think is happening is that the
child_process
STDOUT
stream (within
exec-stream
) is not pausing, and therefore buffers fill until data is lost.

Do
child_process
streams behave this way? Is there any way to get backpressure to work properly with
child_process
streams?

Answer

I'm not familiar enough with exec-stream and node-brake to understand all the avenues for data-loss.

However, I did a little experiment to see if node-brake has a back-pressure effect, which you mentioned could be a potential area for data-loss.

File is also hosted on Gist.

###
Created for http://stackoverflow.com/questions/34982953/child-process-stream-backpressure
Please pardon the CoffeeScript, but I couldn't stand to extend stream.Transform in native JavaScript.
###
fs = require("fs")
execStream = require("exec-stream")
brake = require("brake")

file = fs.createWriteStream("tmp.txt")

class Double extends require("stream").Transform
  _transform: (chunk, enc, cb) ->
    @_last ?= Date.now()
    @_called ?= []
    @_called.push Date.now() - @_last
    @_last = Date.now()
    @push chunk.toString() + chunk.toString()
    cb()

class UpperCase extends require("stream").Transform
  _transform: (chunk, enc, cb) ->
    @push chunk.toString().toUpperCase()
    cb()

sum = (nums) ->
  o = 0
  o += i for i in nums
  o

doTest = (size) ->

  transform1 = new Double()
  transform2 = new UpperCase()
  transform3 = new Double()

  execStream("dd", ["if=/dev/urandom", "bs=1024", "count=1"])
    .pipe(transform1)
    .pipe(transform2)
    .pipe(brake(size))
    .pipe(transform3)
    .pipe(file)

  file.on "finish", ->
    fs.stat "tmp.txt", (err, stats) ->
      throw err if err
      called1 = transform1._called
      averagePreBrake = sum(called1) / called1.length
      called2 = transform3._called
      averagePostBrake = sum(called2) / called2.length
      console.log """
        Generated with brake(#{size}): #{stats.size}
        Average time between transformations pre-brake:  #{averagePreBrake}ms
        Average time between transformations post-brake: #{averagePostBrake}ms
      """

doTest 1024
doTest 256

Results of the process are below.

stdout from above Coffeescript

I notice that there are no gaps between transformations before the brake. However, the brake disrupts transformations later down the line. Given this data, I suspect that node-brake has no back-pressure effect.

[Screenshot of console output generated by my sh2png utility]