Topto Topto - 3 days ago 7
Python Question

Facing issues selecting 2nd item from suggestion drop-down

I've written a script in python with selenium to scrape the city & zip from google maps using reverse lookup. It is working good so far. The only requirement I can't get fulfilled is: after putting an address in the search box sometimes few suggestions come up like a drop-down to choose. It by default chooses the first one. If I wish to go for the second one from the suggestion, I get stuck.

Here is the script I'm trying with:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

search_street = ['10123 HARDISON','1006 ELEANOR ST']

def find_address(driver, wait):
for item in range(0,2):
driver.get("https://maps.google.com/maps?q=Houston")
wait = WebDriverWait(driver, 10)
input_val = wait.until(EC.presence_of_element_located((By.NAME, "q")))
input_val.clear()
input_val.send_keys(search_street[item])
driver.find_element_by_id('searchbox-searchbutton').click()

# wait.until(EC.presence_of_element_located((By.XPATH, "//div[@id='sbse6']")))
# driver.find_elements_by_xpath("//div[@id='sbse6']")[2].click()

wait.until(EC.presence_of_element_located((By.XPATH, "//div[@class='section-hero-header-description']")))
element = driver.find_element_by_xpath("//div[@class='section-hero-header-description']")
name = driver.execute_script("return arguments[0].childNodes[3].textContent", element).strip()
print(name)

if __name__ == '__main__':
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
try:
find_address(driver, wait)
finally:
driver.quit()


Html element for the suggestion drop-down:

<div class="sbdd_b" style=""><div class="suggestions"><ul class="sbsb_b" role="listbox"><li role="presentation" class="sbsb_c " dir="ltr" style="text-align: left;"><div role="option" id="sbse5"><div class="sbqs_a"></div><div class="sbqs_c"><div jstcache="746" class="suggest" jsan="7.suggest"><div jstcache="747" jsinstance="*0" class="suggest-text-layout"><div jstcache="748" class="suggest-left-cell"><div role="presentation" jstcache="749" class="suggest-icon-container maps-sprite-suggest-place-pin" jsan="7.suggest-icon-container,7.maps-sprite-suggest-place-pin,0.role"></div><img role="presentation" jstcache="750" class="suggest-activity" style="display:none"><span jstcache="751" style="display:none"><span jstcache="752" class="suggest-query" jsan="7.suggest-query"></span> nearby</span><span jstcache="753" class="suggest-query" jsan="7.suggest-query,t-fmS2vATOQZY"><span jstcache="764" jsinstance="0" class=" suggest-bold" jsan="7.suggest-bold">10123 Valley Park Dr</span><span jstcache="764" jsinstance="*1" class="">ive</span></span> <span jstcache="754"><span jstcache="764" jsinstance="*0">Houston, TX</span></span></div><div class="suggest-right-cell"><button jstcache="755" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">set location</button><button jstcache="756" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">edit</button><span jstcache="757" style="display:none"><div class="suggest-ad-creative-annotation">Ad</div><span jstcache="758"></span></span></div></div></div></div></div></li><li role="presentation" class="sbsb_c " dir="ltr" style="text-align: left;"><div role="option" id="sbse6"><div class="sbqs_a"></div><div class="sbqs_c"><div jstcache="746" class="suggest" jsan="7.suggest"><div jstcache="747" jsinstance="*0" class="suggest-text-layout"><div jstcache="748" class="suggest-left-cell"><div role="presentation" jstcache="749" class="suggest-icon-container maps-sprite-suggest-place-pin" jsan="7.suggest-icon-container,7.maps-sprite-suggest-place-pin,0.role"></div><img role="presentation" jstcache="750" class="suggest-activity" style="display:none"><span jstcache="751" style="display:none"><span jstcache="752" class="suggest-query" jsan="7.suggest-query"></span> nearby</span><span jstcache="753" class="suggest-query" jsan="7.suggest-query,t-fmS2vATOQZY"><span jstcache="764" jsinstance="0" class=" suggest-bold" jsan="7.suggest-bold">10123 Valley Park Dr</span><span jstcache="764" jsinstance="*1" class="">ive</span></span> <span jstcache="754"><span jstcache="764" jsinstance="*0">Garland, TX</span></span></div><div class="suggest-right-cell"><button jstcache="755" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">set location</button><button jstcache="756" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">edit</button><span jstcache="757" style="display:none"><div class="suggest-ad-creative-annotation">Ad</div><span jstcache="758"></span></span></div></div></div></div></div></li><li role="presentation" class="sbsb_c " dir="ltr" style="text-align: left;"><div role="option" id="sbse7"><div class="sbqs_a"></div><div class="sbqs_c"><div jstcache="746" class="suggest" jsan="7.suggest"><div jstcache="747" jsinstance="*0" class="suggest-text-layout"><div jstcache="748" class="suggest-left-cell"><div role="presentation" jstcache="749" class="suggest-icon-container maps-sprite-suggest-place-pin" jsan="7.suggest-icon-container,7.maps-sprite-suggest-place-pin,0.role"></div><img role="presentation" jstcache="750" class="suggest-activity" style="display:none"><span jstcache="751" style="display:none"><span jstcache="752" class="suggest-query" jsan="7.suggest-query"></span> nearby</span><span jstcache="753" class="suggest-query" jsan="7.suggest-query,t-fmS2vATOQZY"><span jstcache="764" jsinstance="0" class=" suggest-bold" jsan="7.suggest-bold">10123 Valley Park Dr</span><span jstcache="764" jsinstance="*1" class="">ive</span></span> <span jstcache="754"><span jstcache="764" jsinstance="*0">Garden Ridge, TX</span></span></div><div class="suggest-right-cell"><button jstcache="755" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">set location</button><button jstcache="756" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">edit</button><span jstcache="757" style="display:none"><div class="suggest-ad-creative-annotation">Ad</div><span jstcache="758"></span></span></div></div></div></div></div></li><li role="presentation" class="sbsb_c " dir="ltr" style="text-align: left;"><div role="option" id="sbse8"><div class="sbqs_a"></div><div class="sbqs_c"><div jstcache="746" class="suggest" jsan="7.suggest"><div jstcache="747" jsinstance="*0" class="suggest-text-layout"><div jstcache="748" class="suggest-left-cell"><div role="presentation" jstcache="749" class="suggest-icon-container maps-sprite-suggest-place-pin" jsan="7.suggest-icon-container,7.maps-sprite-suggest-place-pin,0.role"></div><img role="presentation" jstcache="750" class="suggest-activity" style="display:none"><span jstcache="751" style="display:none"><span jstcache="752" class="suggest-query" jsan="7.suggest-query"></span> nearby</span><span jstcache="753" class="suggest-query" jsan="7.suggest-query,t-fmS2vATOQZY"><span jstcache="764" jsinstance="0" class=" suggest-bold" jsan="7.suggest-bold">10123 Valley Park Dr</span><span jstcache="764" jsinstance="*1" class="">ive</span></span> <span jstcache="754"><span jstcache="764" jsinstance="*0">San Antonio, TX</span></span></div><div class="suggest-right-cell"><button jstcache="755" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">set location</button><button jstcache="756" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">edit</button><span jstcache="757" style="display:none"><div class="suggest-ad-creative-annotation">Ad</div><span jstcache="758"></span></span></div></div></div></div></div></li><li role="presentation" class="sbsb_c " dir="ltr" style="text-align: left;"><div role="option" id="sbse9"><div class="sbqs_a"></div><div class="sbqs_c"><div jstcache="746" class="suggest" jsan="7.suggest"><div jstcache="747" jsinstance="*0" class="suggest-text-layout"><div jstcache="748" class="suggest-left-cell"><div role="presentation" jstcache="749" class="suggest-icon-container maps-sprite-suggest-place-pin" jsan="7.suggest-icon-container,7.maps-sprite-suggest-place-pin,0.role"></div><img role="presentation" jstcache="750" class="suggest-activity" style="display:none"><span jstcache="751" style="display:none"><span jstcache="752" class="suggest-query" jsan="7.suggest-query"></span> nearby</span><span jstcache="753" class="suggest-query" jsan="7.suggest-query,t-fmS2vATOQZY"><span jstcache="764" jsinstance="0" class=" suggest-bold" jsan="7.suggest-bold">10123 Valley Park Dr</span><span jstcache="764" jsinstance="*1" class="">ive</span></span> <span jstcache="754"><span jstcache="764" jsinstance="*0">Temple, TX</span></span></div><div class="suggest-right-cell"><button jstcache="755" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">set location</button><button jstcache="756" jsaction="omnibox.editAlias" class="blue-button-text suggest-edit-link" style="display:none">edit</button><span jstcache="757" style="display:none"><div class="suggest-ad-creative-annotation">Ad</div><span jstcache="758"></span></span></div></div></div></div></div></li></ul></div></div>


The suggestion drop-down I'm talking about is this:
image_link

Btw, the xpath with which I was trying to select the second option from suggestion drop-down is also given in my script which is grayed out at this moment.

Answer Source

I couldn't do it with an xpath quite possibly because I found that the page changed very quickly and elements became stale very rapidly. However I can do it with classes:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from time import sleep

search_street = ['10123 HARDISON','1006 ELEANOR ST']

def find_address(driver, wait):
    for item in range(0,2):
        driver.get("https://maps.google.com/maps?q=Houston")
        wait = WebDriverWait(driver, 10)
        input_val = wait.until(EC.presence_of_element_located((By.NAME, "q")))
        input_val.clear()
        input_val.send_keys(search_street[item])

        wait.until(EC.presence_of_element_located((By.CLASS_NAME, "suggest")))
        sleep(5) # To prevent stale element reference as the text changes.
        for element in driver.find_elements_by_class_name("suggest"):
            print (element.text)

if __name__ == '__main__':
    driver = webdriver.Chrome()
    wait = WebDriverWait(driver, 10)
    try:
        find_address(driver, wait)
    finally:
        driver.quit()

Outputs:

10123 Hardison Lane Houston, TX
10123 Hardison Lane Abilene, TX
10123 Hardison Road Woodburn, KY
10123 Hardison Road Charlotte, NC
10123 Hardison Road Columbia, TN
1006 Eleanor Street College Station, TX
1006 Eleanor Street Houston, TX
1006 Eleanor Street Victoria, TX
1006 Eleanor Street La Marque, TX
1006 Eleanor Street Brenham, TX

You can execute a click event on your chosen element like this:

driver.find_elements_by_class_name("suggest")[2].click()