user2543622 user2543622 - 7 months ago 294
Python Question

parsing google news using beautiful soup python

I have python code as below. It searches a google news page and prints hyperlinks and titles for each news. My problem is that googlenews groups news that are similar in one bucket and below script prints only 1st news in each bucket. How can I print all new from all buckets?

from bs4 import BeautifulSoup
import requests

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}

#r = requests.get('http://www.aflcio.org/Legislation-and-Politics/Legislative-Alerts', headers=headers)
r = requests.get('https://www.google.com/search?q=%22eric+bledsoe%22&tbm=nws&tbs=qdr:d', headers=headers)
r = requests.get('https://www.google.com/search?q=%22lebron+james%22&tbm=nws&tbs=qdr:y', headers=headers)

soup = BeautifulSoup(r.text, "html.parser")

letters = soup.find_all("div", class_="_cnc")
#print soup.prettify()
#print letters
print type(letters)
print len(letters)
print("\n")

for x in range(0, len(letters)):
print x
print letters[x].a["href"]


print("\n")

letters2 = soup.find_all("a", class_="l _HId")
for x in range(0, len(letters2)):
print x
print letters2[x].get_text()

print ("\n----------content")
#print letters[0]


By bucketing news I mean that in the below image, first few news are grouped together. The news "LeBron James compares one of his teammates to Denn" is part of another group.

enter image description here

Answer

I'm not sure what you mean by bucket? If you mean you're trying to parse multiple websites then I can tell you you're overwriting r by sending it multiple news requests.get()

Here's a loop to process all the URLs you have in the urls array.

import bs4
import requests

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}


urls = ["https://www.google.com/search?q=%22eric+bledsoe%22&tbm=nws&tbs=qdr:d",
        "https://www.google.com/search?q=%22lebron+james%22&tbm=nws&tbs=qdr:y"]

ahrefs = []
titles = []

for url in urls:
    req = requests.get(url, headers=headers)
    soup = bs4.BeautifulSoup(req.text, "html.parser")

    #you don't even have to process the div container
    #just go strait to <a> and using indexing get "href"
    #headlines
    ahref  = [a["href"] for a in soup.find_all("a", class_="_HId")]
    #"buckets"
    ahref += [a["href"] for a in soup.find_all("a", class_="_sQb")]
    ahrefs.append(ahref)

    #or get_text() will return the array inside the hyperlink
    #the title you want
    title =  [a.get_text() for a in soup.find_all("a", class_="_HId")]
    title += [a.get_text() for a in soup.find_all("a", class_="_sQb")]
    titles.append(title)

#print(ahrefs)
#print(titles)

My google search for lebron turns up 18 results, including subheadlines, and len(ahrefs[1]) == 18