diff --git a/geomagio/Util.py b/geomagio/Util.py index a72f6a05bd8fd2fe826e6f1d80a7b7494eb833d5..cd92c247278bf34c53ee36bf523c929be33720a5 100644 --- a/geomagio/Util.py +++ b/geomagio/Util.py @@ -1,7 +1,9 @@ -import urllib2 +import pycurl import numpy import os from obspy.core import Stats, Trace +from StringIO import StringIO +import sys class ObjectView(object): @@ -103,7 +105,7 @@ def get_intervals(starttime, endtime, size=86400, align=True, trim=False): return intervals -def read_url(url): +def read_url(url, connect_timeout=15, max_redirects=5, timeout=300): """Open and read url contents. Parameters @@ -121,15 +123,24 @@ def read_url(url): urllib2.URLError if any occurs """ - response = urllib2.urlopen(url) content = None + curl = pycurl.Curl() + out = StringIO() try: - content = response.read() - except urllib2.URLError, e: - print e.reason - raise + curl.setopt(pycurl.FOLLOWLOCATION, 1) + curl.setopt(pycurl.MAXREDIRS, max_redirects) + curl.setopt(pycurl.CONNECTTIMEOUT, connect_timeout) + curl.setopt(pycurl.TIMEOUT, timeout) + curl.setopt(pycurl.NOSIGNAL, 1) + curl.setopt(pycurl.URL, url) + curl.setopt(pycurl.WRITEFUNCTION, out.write) + curl.perform() + content = out.getvalue() + except Exception as e: + print >> sys.stderr, "Error reading url: " + str(e) + print >> sys.stderr, url finally: - response.close() + curl.close() return content diff --git a/setup.py b/setup.py index 84ef2911c87c6b82a32c55c9aaa286b5052151c1..3d803c6d97f332977defc8809e6433ef742e459c 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ setup( 'numpy', 'matplotlib', 'scipy', - 'obspy' + 'obspy', + 'pycurl' ], scripts=[ 'bin/geomag.py'