Jared Jared - 3 months ago 108
JSON Question

Why can't Elasticsearch bulk API parse my JSON?

I have this JSON-like data that I am

PUT
ing via a HTTP request to an Elasticsearch API /_bulk endpoint:

{"update":{"_id":1,"_type":"myType","_index":"myIndex"}}
{"doc":{"id":1,"name":"Foo"},"doc_as_upsert":true}
{"update":{"_id":2,"_type":"myType","_index":"myIndex"}}
{"doc":{"id":2,"name":"Bar"},"doc_as_upsert":true}


When I
PUT
the data via Postman (a Google Chrome app) it works successfully.

When I
PUT
the data via my Node.js script (a
request()
call), I get this error:

{
"error" : {
"root_cause" : [ {
"type" : "parse_exception",
"reason" : "Failed to derive xcontent"
} ],
"type" : "parse_exception",
"reason" : "Failed to derive xcontent"
},
"status" : 400
}


I'm sending (what I assume is) the exact same data both ways but only Postman is working. I am ending each line with a \n character (including the last one) and I believe my formatting is correct.

Is there something I am doing wrong?

Edit: the Node code (simplified):

var sourceData = JSON.parse(fs.readFileSync('test-data.json', 'utf8')),

sourceData.forEach(function(data) {
var docid = data.id;

updates.push(JSON.stringify({
update: {
_id: docid,
_type: config.elasticsearch.type,
_index: config.elasticsearch.index
}
}));

updates.push(JSON.stringify({
doc: data,
doc_as_upsert: true
}));
});

updates = updates.join("\n") + "\n";

request({
uri: 'http://my-endpoint/_bulk',
method: 'POST',
data: updates,
proxy: 'http://' + config.proxy.host + ':' + config.proxy.port
}, function() { ... });

Val Val
Answer

You need to use the body parameter to send the updates, not data (which is for multipart requests)

request({
    uri: 'http://my-endpoint/_bulk',
    method: 'POST',
    body: updates,               // <---- change this
    proxy: 'http://' + config.proxy.host + ':' + config.proxy.port
}, function() { ... });