Mona Jalal Mona Jalal - 1 year ago 65
HTTP Question

image downloading gets stuck on a link that doesn't exists anymore

So I am not sure how to handle this situation. It almost works on many other broken links but not this one:

import datetime
import praw
import re
import urllib
import requests
from bs4 import BeautifulSoup

sub = 'dog'
imgurUrlPattern = re.compile(r'(*))(\?.*)?')
r = praw.Reddit(user_agent = "download all images from a subreddit",
user_site = "lamiastella")
already_done = []
#checkWords = ['', 'jpg', 'png',]
check_words = ['jpg', 'jpeg', 'png']

subreddit = r.get_subreddit(sub)
for submission in subreddit.get_top_from_all(limit=10000):
#for submission in subreddit.get_hot(limit=10000):
is_image = any(string in submission.url for string in check_words)
print '[LOG] Getting url: ' + submission.url
if not in already_done and is_image:
if submission.url.endswith('/'):
modified_url = submission.url[:len(submission.url)-1]
urllib.urlretrieve(modified_url, '/home/jalal/computer_vision/image_retrieval/images/' +'%y-%m-%d-%s') + modified_url[-5:])
except Exception as e:
urllib.urlretrieve(submission.url, '/home/jalal/computer_vision/image_retrieval/images/' +'%y-%m-%d-%s') + submission.url[-5:])
except Exception as e:

print '[LOG] Done Getting ' + submission.url
print('{0}: {1}'.format('submission id is',
elif '' in submission.url and not (submission.url.endswith('gif')
or submission.url.endswith('webm')
or submission.url.endswith('mp4')
or submission.url.endswith('all')
or '#' in submission.url
or '/a/' in submission.url):
# This is an Imgur page with a single image.
html_source = requests.get(submission.url).text # download the image's page
soup = BeautifulSoup(html_source, "lxml")
image_url ='img')[0]['src']
if image_url.startswith('//'):
# if no schema is supplied in the url, prepend 'http:' to it
image_url = 'http:' + image_url
image_id = image_url[image_url.rfind('/') + 1:image_url.rfind('.')]
urllib.urlretrieve(image_url, '/home/jalal/computer_vision/image_retrieval/images/' + 'imgur_'+'%y-%m-%d-%s') + submission.url[-9:0])
elif '' in submission.url:
html_source = requests.get(submission.url).text
soup = BeautifulSoup(html_source, "lxml")
instagram_url = soup.find('meta', {"property":"og:image"})['content']
urllib.urlretrieve(instagram_url, '/home/jalal/computer_vision/image_retrieval/images/' + 'instagram_'+'%y-%m-%d-%s') + '.jpg')

I get stuck at a link
and have to CTL+C it:

[LOG] Done Getting
submission id is: 1fv70j
[LOG] Getting url:
[LOG] Done Getting
submission id is: 1dof3o
[LOG] Getting url:
^CTraceback (most recent call last):
File "", line 35, in <module>
urllib.urlretrieve(submission.url, '/home/jalal/computer_vision/image_retrieval/images/' +'%y-%m-%d-%s') + submission.url[-5:])
File "/usr/lib/python2.7/", line 98, in urlretrieve
return opener.retrieve(url, filename, reporthook, data)
File "/usr/lib/python2.7/", line 245, in retrieve
fp =, data)
File "/usr/lib/python2.7/", line 213, in open
return getattr(self, name)(url)
File "/usr/lib/python2.7/", line 350, in open_http
File "/usr/lib/python2.7/", line 1053, in endheaders
File "/usr/lib/python2.7/", line 897, in _send_output
File "/usr/lib/python2.7/", line 859, in send
File "/usr/lib/python2.7/", line 836, in connect
self.timeout, self.source_address)
File "/usr/lib/python2.7/", line 566, in create_connection
File "/usr/lib/python2.7/", line 228, in meth
return getattr(self._sock,name)(*args)

Please suggest fixes for this.

I used something like:

image_file = urllib2.urlopen(modified_url)
with open('/home/jalal/computer_vision/image_retrieval/images/' +'%y-%m-%d-%s') + modified_url[-5:], 'wb') as output_image:

and still get stuck for the particular link.

Answer Source

Use urlopen with timeout argument:

>>> import urllib2
>>> modified_url = ''
>>> try:
...     image_file = urllib2.urlopen(modified_url, timeout=5)
... except urllib2.URLError:
...     print 'could not download :('
could not download :(
Recommended from our users: Dynamic Network Monitoring from WhatsUp Gold from IPSwitch. Free Download