iskandarblue iskandarblue - 1 month ago 12
Node.js Question

error running AGStoShapefile.js

I have successfully installed the AGStoShapefile, which is a node.js script that converts ESRI Geoservices to geojson and shapefile formats.

The github repository is here https://github.com/tannerjt/AGStoShapefile
and the specific .js file which I am having the issue with is https://github.com/tannerjt/AGStoShapefile/blob/master/AGStoSHP.js

By default, the script takes an input txt file with services and outputs .shp, .json files. This services.txt is located in the AGStoShapefile directory, and in the terminal the script runs perfectly until line 154, at which point the following error appears:

node AGStoSHP.js
info: Number of features for service: 225
info: Getting chunks of 100 features...
info: query -> 0 out of 225
info: query -> 100 out of 225
info: query -> 200 out of 225
info: Number of features for service: 8258
info: Getting chunks of 100 features...
info: query -> 0 out of 8258
info: query -> 100 out of 8258
info: query -> 200 out of 8258
info: query -> 300 out of 8258
info: query -> 400 out of 8258
info: query -> 500 out of 8258
info: query -> 600 out of 8258
info: query -> 700 out of 8258
info: query -> 800 out of 8258
info: query -> 900 out of 8258


it continues...

info: query -> 7100 out of 8258
info: query -> 7200 out of 8258
info: query -> 7300 out of 8258
info: query -> 7400 out of 8258
info: query -> 7500 out of 8258
info: query -> 7600 out of 8258
info: query -> 7700 out of 8258
info: query -> 7800 out of 8258
info: query -> 7900 out of 8258
info: query -> 8000 out of 8258
info: query -> 8100 out of 8258
info: query -> 8200 out of 8258
info: all requests settled
info: creating WildFire json
info: Creating Esri JSON
info: Creating GeoJSON
info: Creating Shapefile
events.js:154
throw er; // Unhandled 'error' event
^

Error: ENOENT: no such file or directory, open './output/WildFire.json'
at Error (native)


Any idea why the output files are not being created? I am fairly new to node.js

Here is the AGStoSHP.js file:

var ogr2ogr = require('ogr2ogr');
var esri2geo = require('esri2geo');
var q = require('q');
var request = q.nfbind(require('request'));
var objectstream = require('objectstream');
var fs = require('fs');
var queryString = require('query-string');
var winston = require('winston');

// Setup logging with winston
winston.level = 'debug';
// winston.add(winston.transports.File, {filename: './logfile.log'});

// ./mixin.js
// merge user query params with default
var mixin = require('./mixin');

var serviceFile = process.argv[2] || 'services.txt';
var outDir = process.argv[3] || './output/';
if(outDir[outDir.length - 1] !== '/') {
outDir += '/';
}

// Make request to each service
fs.readFile(serviceFile, function (err, data) {
if (err) {
winston.info(err);
throw err;
}
data.toString().split('\n').forEach(function (service) {
var service = service.split('|');
if(service[0].split('').length == 0) return;
var baseUrl = getBaseUrl(service[0].trim()) + '/query';

var reqQS = {
where: '1=1',
returnIdsOnly: true,
f: 'json'
};
var userQS = getUrlVars(service[0].trim());
// mix one obj with another
var qs = mixin(userQS, reqQS);
var qs = queryString.stringify(qs);
var url = decodeURIComponent(getBaseUrl(baseUrl) + '/query/?' + qs);

request({
url : url,
method : 'GET',
json : true
}, function (err, response, body) {
var err = err || body.error;
if(err) {
winston.info(err);
throw err;
}
requestService(service[0].trim(), service[1].trim(), body.objectIds);
});
})
});

// Resquest JSON from AGS
function requestService(serviceUrl, serviceName, objectIds) {
objectIds.sort();
winston.info('Number of features for service: ', objectIds.length);
winston.info('Getting chunks of 100 features...');
var requests = [];

for(var i = 0; i < Math.ceil(objectIds.length / 100); i++) {
var ids = [];
if ( ((i + 1) * 100) < objectIds.length ) {
ids = objectIds.slice(i * 100, (i * 100) + 100);
} else {
ids = objectIds.slice(i * 100, objectIds[objectIds.length]);
}

if(ids[0] !== undefined) {
winston.info('query ->', (i * 100) , 'out of', objectIds.length);
} else {
winston.info('wait for requests to settle...');
continue;
}

// we need these query params
var reqQS = {
objectIds : ids.join(','),
geometryType : 'esriGeometryEnvelope',
returnGeometry : true,
returnIdsOnly : false,
outFields : '*',
outSR : '4326',
f : 'json'
};
// user provided query params
var userQS = getUrlVars(serviceUrl);
// mix one obj with another
var qs = mixin(userQS, reqQS);
var qs = queryString.stringify(qs);
var url = decodeURIComponent(getBaseUrl(serviceUrl) + '/query/?' + qs);
var r = request({
url : url,
method : 'GET',
json : true
});

requests.push(r);
};

q.allSettled(requests).then(function (results) {
winston.info('all requests settled');
var allFeatures;
for(var i = 0; i < results.length; i++) {
if(i == 0) {
allFeatures = results[i].value[0].body;
} else {
allFeatures.features = allFeatures.features.concat(results[i].value[0].body.features);
}
}
winston.info('creating', serviceName, 'json');
var json = allFeatures;

//esri json
winston.info('Creating Esri JSON');
var stream = fs.createWriteStream(outDir + serviceName + '.json');
var objstream = objectstream.createSerializeStream(stream);
objstream.write(json);
objstream.end();

//geojson
winston.info('Creating GeoJSON');
var stream = fs.createWriteStream(outDir + serviceName + '.geojson');
var objstream = objectstream.createSerializeStream(stream);
esri2geo(json, function (err, data) {
if(err) {
throw(err);
winston.info('Error converting esri json to geojson');
return;
}
objstream.write(data);
objstream.end();
winston.info('Creating Shapefile');
//shapefile
var shapefile = ogr2ogr(data)
.format('ESRI Shapefile')
.options(['-nln', serviceName])
.skipfailures();
shapefile.stream().pipe(fs.createWriteStream(outDir + serviceName + '.zip'));
});

}).catch(function (err) {
winston.info(err);
throw err;
});
}


//http://stackoverflow.com/questions/4656843/jquery-get-querystring-from-url
function getUrlVars(url) {
var vars = {}, hash;
var hashes = url.slice(url.indexOf('?') + 1).split('&');
for(var i = 0; i < hashes.length; i++)
{
hash = hashes[i].split('=');
vars[hash[0].toString()] = hash[1];
}
return vars;
}

// get base url for query
function getBaseUrl(url) {
// remove any query params
var url = url.split("?")[0];
if((/\/$/ig).test(url)) {
url = url.substring(0, url.length - 1);
}
return url;
}

Answer

I am the maintainer of that GitHub repository. The script assumes the output folder is already created. You may try creating a folder called output, or manually specifying your own output location and input services text file like so:

node AGStoSHP.js services.txt ./myotherfolder/