Elad Benda2 Elad Benda2 - 2 months ago 12
HTTP Question

What parallelism in my code might cause "Timeout waiting for connection from pool"

I have this java http client code:

private void init(String[] args) {
setServerVersion(args);
commonParallelGenerator.setInputFlavor();
commonParallelGenerator.setNumberOfThreads("25"); //machine has 31 cores.
}


...



Report report = requestsList
.parallelStream()
.map(request -> freshResultsGenerator.getResponse(request, e2EResultLongBL))

...


//uses this http client eventually:

import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.inject.Inject;


public class RoutingUrlHttpClient implements IRoutingUrlHttpClient {

final static Logger logger = Logger.getLogger(RoutingUrlHttpClient.class);


private IRoutingResponseFromStringFetcher routingResponseParser;
private IRoutingResponseConverter routingResponseConverter;
private IUrlUtils urlUtils;
private CloseableHttpClient client;
private ILogUtils logUtils;


@Inject
@Singleton
public RoutingUrlHttpClient(IRoutingResponseFromStringFetcher routingResponseParser,
IRoutingResponseConverter routingResponseConverter, IUrlUtils urlUtils,
ILogUtils logUtils) {
this.routingResponseParser = routingResponseParser;
this.routingResponseConverter = routingResponseConverter;
this.urlUtils = urlUtils;
this.logUtils = logUtils;

RequestConfig requestConfig = RequestConfig.custom()
//time till handshake
.setConnectTimeout(40 * 1000)
//happens when you have a pool of connections and they are all busy, not allowing the connection
// manager to give you one connection to make the request.
.setConnectionRequestTimeout(40 * 1000)
//time till response
.setSocketTimeout(40 * 1000)
.build();
client = HttpClientBuilder
.create()
.setDefaultRequestConfig(requestConfig)
.build();
}

private CompleteRoutingResponseDtoWrapper sendRoutingRequestString(int numberOfTriesLeft,
String routingRequestUrl) {
routingRequestUrl = urlUtils.getHttpUrl(routingRequestUrl);

CompleteRoutingResponseDtoWrapper answer = new CompleteRoutingResponseDtoWrapper();
CloseableHttpResponse response = null;
try {

logger.debug("before sending http");

Stopwatch stopWatch = Stopwatch.createStarted();
response = client.execute(new HttpGet(routingRequestUrl));
stopWatch.stop();
// String latencyMsg = "after sending http. client-latency: "+stopWatch.elapsed(TimeUnit.MILLISECONDS) +" server-latency: "+response.getHeaders("Latency")[0].getValue();

logUtils.addLongToLongStatisticCollector("http.client.latency", (int)stopWatch.elapsed(TimeUnit.MILLISECONDS));
logUtils.addLongToLongStatisticCollector("http.server.latency", Integer.parseInt(response.getHeaders("Latency")[0].getValue()));

answer = analyzeStatusCodeAndMsgBody(numberOfTriesLeft, routingRequestUrl, answer, response, stopWatch);


} catch (Exception e) {
// e.printStackTrace();
// System.out.println(e.getMessage());
answer.errorMsg = e.getMessage();
answer.latency = null;
}
handleNullResponse(answer);
return answer;
}


I often get error:

Timeout waiting for connection from pool


when I GET a request from the idea,

but the navigation works well when I navigate throw the browser

Where do you think I should investigate for bottle neck?

Answer

The problem was that I haven't closed the httpClient

            //try-with-resources
            try (CloseableHttpClient client = HttpClientBuilder
                    .create()
                    .setDefaultRequestConfig(requestConfig)
                    .build()){
                response = client.execute(new HttpGet(routingRequestUrl));
            stopWatch.stop();
//            String latencyMsg = "after sending http. client-latency: "+stopWatch.elapsed(TimeUnit.MILLISECONDS) +" server-latency: "+response.getHeaders("Latency")[0].getValue();

            logUtils.addLongToLongStatisticCollector("http.client.latency", (int)stopWatch.elapsed(TimeUnit.MILLISECONDS));
            logUtils.addLongToLongStatisticCollector("http.server.latency", Integer.parseInt(response.getHeaders("Latency")[0].getValue()));

            answer = analyzeStatusCodeAndMsgBody(numberOfTriesLeft, routingRequestUrl, answer, response, stopWatch);
            }
Comments