SHUTTEHFACE SHUTTEHFACE - 6 months ago 18
Python Question

Getting TypeError when trying to concatenate 'str' and 'NoneType' objects

I have the following code:
import requests
from bs4 import BeautifulSoup

def hltvmatch_spider(max_offset):
offset = 0
while offset < max_offset:
url = 'http://www.hltv.org/?pageid=188&offset=' + str(offset)
source_code = requests.get(url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text, 'html.parser')
for link in soup.findAll('a'):
href = "http://www.hltv.org" + link.get('href')
print(href)
offset += 50

hltvmatch_spider(1)


which is returning and error:

href = "http://www.hltv.org" + link.get('href')
TypeError: cannot concatenate 'str' and 'NoneType' objects


Any ideas what I need to change for fix? THANKS FOR THE HELP!!

Answer

You should set href=True so you only get anchors with href attributes, calling .get("href") will return None when the anchor has no href attribute:

for link in soup.findAll('a', href=True):

As I commented getting all anchors is not probably what you want as joining to the base url is not going to work, this will get all the anchor tags from the main content:

cont = soup.select_one("div.covMainBoxContent")

print([a["href"] for a in cont.select("a[href]")])

Which would give you:

['/?pageid=188&matchid=31342', '/?pageid=179&teamid=4411', '/?pageid=179&teamid=5995', '/?pageid=188&eventid=2135', '/?pageid=188&matchid=31343', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6865', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31339', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6865', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31338', '/?pageid=179&teamid=5995', '/?pageid=179&teamid=6736', '/?pageid=188&eventid=2135', '/?pageid=188&matchid=31341', '/?pageid=179&teamid=6620', '/?pageid=179&teamid=6807', '/?pageid=188&eventid=2238', '/?pageid=188&matchid=31340', '/?pageid=179&teamid=6807', '/?pageid=179&teamid=6620', '/?pageid=188&eventid=2238', '/?pageid=188&matchid=31329', '/?pageid=179&teamid=5995', '/?pageid=179&teamid=6736', '/?pageid=188&eventid=2135', '/?pageid=188&matchid=31336', '/?pageid=179&teamid=6998', '/?pageid=179&teamid=4602', '/?pageid=188&eventid=2262', '/?pageid=188&matchid=31334', '/?pageid=179&teamid=4602', '/?pageid=179&teamid=6998', '/?pageid=188&eventid=2262', '/?pageid=188&matchid=31331', '/?pageid=179&teamid=4674', '/?pageid=179&teamid=6133', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31330', '/?pageid=179&teamid=6133', '/?pageid=179&teamid=4674', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31333', '/?pageid=179&teamid=4501', '/?pageid=179&teamid=6686', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31332', '/?pageid=179&teamid=4501', '/?pageid=179&teamid=6686', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31319', '/?pageid=179&teamid=4411', '/?pageid=179&teamid=6615', '/?pageid=188&eventid=2135', '/?pageid=188&matchid=31321', '/?pageid=179&teamid=6133', '/?pageid=179&teamid=5929', '/?pageid=188&eventid=2262', '/?pageid=188&matchid=31320', '/?pageid=179&teamid=5929', '/?pageid=179&teamid=6133', '/?pageid=188&eventid=2262', '/?pageid=188&matchid=31318', '/?pageid=179&teamid=6615', '/?pageid=179&teamid=4411', '/?pageid=188&eventid=2135', '/?pageid=188&matchid=31328', '/?pageid=179&teamid=6222', '/?pageid=179&teamid=6408', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31327', '/?pageid=179&teamid=6408', '/?pageid=179&teamid=6222', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31326', '/?pageid=179&teamid=6621', '/?pageid=179&teamid=6968', '/?pageid=188&eventid=2252', '/?pageid=188&matchid=31325', '/?pageid=179&teamid=6621', '/?pageid=179&teamid=6968', '/?pageid=188&eventid=2252', '/?pageid=188&matchid=31324', '/?pageid=179&teamid=6619', '/?pageid=179&teamid=6785', '/?pageid=188&eventid=2252', '/?pageid=188&matchid=31322', '/?pageid=179&teamid=6619', '/?pageid=179&teamid=6785', '/?pageid=188&eventid=2252', '/?pageid=188&matchid=31317', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6407', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31316', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6407', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31315', '/?pageid=179&teamid=6995', '/?pageid=179&teamid=7009', '/?pageid=188&eventid=2253', '/?pageid=188&matchid=31306', '/?pageid=179&teamid=6995', '/?pageid=179&teamid=7009', '/?pageid=188&eventid=2253', '/?pageid=188&matchid=31314', '/?pageid=179&teamid=4501', '/?pageid=179&teamid=6686', '/?pageid=188&eventid=2262', '/?pageid=188&matchid=31310', '/?pageid=179&teamid=6686', '/?pageid=179&teamid=4501', '/?pageid=188&eventid=2262', '/?pageid=188&matchid=31304', '/?pageid=179&teamid=6889', '/?pageid=179&teamid=6994', '/?pageid=188&eventid=2253', '/?pageid=188&matchid=31302', '/?pageid=179&teamid=6994', '/?pageid=179&teamid=6889', '/?pageid=188&eventid=2253', '/?pageid=188&matchid=31313', '/?pageid=179&teamid=6865', '/?pageid=179&teamid=4688', '/?pageid=188&eventid=2255', '/?pageid=188&matchid=31312', '/?pageid=179&teamid=4688', '/?pageid=179&teamid=6865', '/?pageid=188&eventid=2255', '/?pageid=188&matchid=31311', '/?pageid=179&teamid=4688', '/?pageid=179&teamid=6865', '/?pageid=188&eventid=2255', '/?pageid=188&matchid=31309', '/?pageid=179&teamid=4602', '/?pageid=179&teamid=6408', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31308', '/?pageid=179&teamid=4602', '/?pageid=179&teamid=6408', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31307', '/?pageid=179&teamid=4602', '/?pageid=179&teamid=6408', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31305', '/?pageid=179&teamid=6133', '/?pageid=179&teamid=4548', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31303', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6133', '/?pageid=188&eventid=2254', '/?pageid=188&matchid=31294', '/?pageid=179&teamid=4869', '/?pageid=179&teamid=6137', '/?pageid=188&eventid=2176', '/?pageid=188&matchid=31293', '/?pageid=179&teamid=6137', '/?pageid=179&teamid=4869', '/?pageid=188&eventid=2176', '/?pageid=188&matchid=31292', '/?pageid=179&teamid=4869', '/?pageid=179&teamid=6137', '/?pageid=188&eventid=2176', '/?pageid=188&matchid=31291', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6407', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31290', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6407', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31289', '/?pageid=179&teamid=4548', '/?pageid=179&teamid=6407', '/?pageid=188&eventid=2232', '/?pageid=188&matchid=31301', '/?pageid=179&teamid=5996', '/?pageid=179&teamid=6981', '/?pageid=188&eventid=2273', '/?pageid=188&matchid=31300', '/?pageid=179&teamid=5996', '/?pageid=179&teamid=6981', '/?pageid=188&eventid=2273', '/?pageid=188&matchid=31299', '/?pageid=179&teamid=5996', '/?pageid=179&teamid=6869', '/?pageid=188&eventid=2273', '/?pageid=188&matchid=31298', '/?pageid=179&teamid=5996', '/?pageid=179&teamid=6869', '/?pageid=188&eventid=2273', '/?pageid=188&matchid=31297', '/?pageid=179&teamid=6981', '/?pageid=179&teamid=6792', '/?pageid=188&eventid=2273']
Comments