Brokes Brokes - 21 days ago 5
HTTP Question

Multiple threads Go for HTTP get

I'm working with Go in an API for the bus frequency of my city, but i'm a little bit stuck on the threads when i try to make HTTP Get to many urls.
Without concurrency, the programs takes over 16 minutes to complete the 1500 url calls to take the HTTP status code, and i was trying to use the concurrency, but after reading many posts i don't understand how goroutines work...

The idea is to make ONE function and change the number of requests, like here:

go getBusPostStatus(600, 800)


But i'm completely stucked on that...

Here is the code:

package main

import (
"fmt"
"net/http"
"strconv"
"time"
)
var i int = 0
var convStr string
var message = make(chan string)

/*func main(){
for i = 0; i < 1500; i++ {
z = strconv.Itoa(i)
url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + z
resp, err := http.Get(url)
if err != nil {
fmt.Println("Houston, we've got problems")
}else{
if resp.StatusCode == 200{
fmt.Println("OK: El poste "+z+" existe")
}else{
fmt.Println("WARN: El poste "+z+" NO existe")
}
}
}
}*/

//Return 2 houndred posts
func returnTH(c chan string){
for i = 0; i < 200; i++ {
convStr = strconv.Itoa(i)
url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
resp, err := http.Get(url)
if err != nil {
fmt.Println("Houston, we've got problems")
}else{
if resp.StatusCode == 200{
//fmt.Println("OK: El poste "+z+" existe")
c <- "OK: The bus post "+convStr+" exists"
}else{
//fmt.Println("WARN: El poste "+z+" NO existe")
c <- "WARN: The bus post "+convStr+" does not exist"
}
}
}
}
func returnFH(z chan string){
for i = 201; i < 400; i++ {
convStr = strconv.Itoa(i)
url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
resp, err := http.Get(url)
if err != nil {
fmt.Println("Houston, we've got problems")
}else{
if resp.StatusCode == 200{
//fmt.Println("OK: El poste "+z+" existe")
z <- "OK: The bus post "+convStr+" exists"
}else{
//fmt.Println("WARN: El poste "+z+" NO existe")
z <- "WARN: The bus post "+convStr+" does not exist"
}
}
}
}

func threadPrint(c, z chan string){
for {
threadOne := <- c
threadTwo := <- z
fmt.Println(threadOne)
fmt.Println(threadTwo)
}
}
func main(){
start := time.Now()
var c chan string = make(chan string)
var z chan string = make(chan string)
//for i = 0; i < 1500; i++{
go returnTH(c)
go returnFH(z)
go threadPrint(c,z)
/*go getBusPostStatus(400, 600)
go getBusPostStatus(600, 800)
go getBusPostStatus(800, 1000)
go getBusPostStatus(1000, 1200)
go getBusPostStatus(1200, 1400)
go getBusPostStatus(1400, 1500)*/
//}
timeExec:= time.Since(start)
fmt.Println("Time to exec code = ", timeExec)

/*var input string
fmt.Scanln(&input)
fmt.Println("done")*/
}


Many thanks in advance!!

Answer

Following is a simplified example code which requests 100 times concurrently and prints results, using goroutine and channel. Hope this code helps.

package main

import (
    "fmt"
    "math/rand"
    "time"
)

func main() {
    rep := 100
    results := make(chan string)

    // Use goroutine to send multiple time-consuming jobs to the channel.
    for i := 0; i < rep; i++ {
        go func(num int) {
            results <- mockHTTPRequest(num)
        }(i)
    }

    // Receive results from the channel and use them.
    for i := 0; i < rep; i++ {
        fmt.Println(<-results)
    }
}

func mockHTTPRequest(num int) string {
    timeDelay := rand.Intn(5000)
    time.Sleep(time.Duration(timeDelay) * time.Millisecond)
    if timeDelay%2 == 0 {
        return fmt.Sprintf("OK: The bus post %v exists", num)
    }
    return fmt.Sprintf("WARN: The bus post %v does not exist", num)
}

You can run this code on https://play.golang.org/p/RR34roRIl4 .