tadm123 tadm123 - 29 days ago 19
Python Question

Multi-thread in Python

I'm following the book "Automate the boring tasks with Python" and I'm trying to create a progrma that downloads multiple comics from

http://xkcd.com

simultaneously, but has ran into some problems. I'm copying the exact same program as it is on the book.

Here's my code:

# multidownloadXkcd.py - Downloads XKCD comics using multiple threads.

import requests, os ,bs4, threading

os.chdir('c:\\users\\patty\\desktop')
os.makedirs('xkcd', exist_ok=True) # store comics on ./xkcd

def downloadXkcd(startComic, endComic):
for urlNumber in range(startComic, endComic):
#Download the page
print('Downloading page http://xkcd.com/%s...' %(urlNumber))
res = requests.get('http://xkcd.com/%s' % (urlNumber))
res.raise_for_status()

soup= bs4.BeautifulSoup(res.text, "html.parser")

#Find the URL of the comic image.
comicElem = soup.select('#comic img')
if comicElem == []:
print('Could not find comic image.')
else:
comicUrl = comicElem[0].get('src')
#Download the image.
print('Downloading image %s...' % (comicUrl))
res = requests.get(comicUrl, "html.parser")
res.raise_for_status()

#Save the image to ./xkcd.
imageFile = open(os.path.join('xkcd', os.path.basename(comicUrl)), 'wb')
for chunk in res.iter_content(100000):
imageFile.write(chunk)
imageFile.close()

downloadThreads = [] # a list of all the Thread objects
for i in range(0,1400, 100): # loops 14 times, creates 14 threads
downloadThread = threading.Thread(target=downloadXkcd, args=(i, i + 99))
downloadThreads.append(downloadThread)
downloadThread.start()

# Wait for all threads to end.
for downloadThread in downloadThreads:
downloadThread.join()
print('Done.')


I'm getting the following exception:

Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Python\Python35\lib\threading.py", line 914, in _bootstrap_inner
self.run()
File "C:\Python\Python35\lib\threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\PATTY\PycharmProjects\CH15_TASKS\practice.py", line 13, in downloadXkcd
res.raise_for_status()
File "C:\Python\Python35\lib\site-packages\requests\models.py", line 862, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http://xkcd.com/0
Exception in thread Thread-2:
Traceback (most recent call last):
File "C:\Python\Python35\lib\threading.py", line 914, in _bootstrap_inner
self.run()
File "C:\Python\Python35\lib\threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\PATTY\PycharmProjects\CH15_TASKS\practice.py", line 25, in downloadXkcd
res = requests.get(comicUrl, "html.parser")
File "C:\Python\Python35\lib\site-packages\requests\api.py", line 70, in get
return request('get', url, params=params, **kwargs)
File "C:\Python\Python35\lib\site-packages\requests\api.py", line 56, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python\Python35\lib\site-packages\requests\sessions.py", line 461, in request
prep = self.prepare_request(req)
File "C:\Python\Python35\lib\site-packages\requests\sessions.py", line 394, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "C:\Python\Python35\lib\site-packages\requests\models.py", line 294, in prepare
self.prepare_url(url, params)
File "C:\Python\Python35\lib\site-packages\requests\models.py", line 354, in prepare_url
raise MissingSchema(error)
requests.exceptions.MissingSchema: Invalid URL '//imgs.xkcd.com/comics/family_circus.jpg': No schema supplied. Perhaps you meant http:////imgs.xkcd.com/comics/family_circus.jpg?


It's says that the URL is invalid but whenever I copy paste that url into the webrowser it seems to be valid. Anyone know how would I fix this? Thanks

Answer

Yea , as @spectras said , just because your url fixes your url it doesn't mean that is valid. Try using a "http://www." before it and try to see if its working.

Comments