From 4d2985ffe455d2b940ec02d353c932cb73fd6b86 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Wed, 11 May 2016 12:07:28 -0600
Subject: [PATCH] Update Util.read_url to use pycurl
---
geomagio/Util.py | 27 +++++++++++++++++++--------
setup.py | 3 ++-
2 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/geomagio/Util.py b/geomagio/Util.py
index a72f6a05..cd92c247 100644
--- a/geomagio/Util.py
+++ b/geomagio/Util.py
@@ -1,7 +1,9 @@
-import urllib2
+import pycurl
import numpy
import os
from obspy.core import Stats, Trace
+from StringIO import StringIO
+import sys
class ObjectView(object):
@@ -103,7 +105,7 @@ def get_intervals(starttime, endtime, size=86400, align=True, trim=False):
return intervals
-def read_url(url):
+def read_url(url, connect_timeout=15, max_redirects=5, timeout=300):
"""Open and read url contents.
Parameters
@@ -121,15 +123,24 @@ def read_url(url):
urllib2.URLError
if any occurs
"""
- response = urllib2.urlopen(url)
content = None
+ curl = pycurl.Curl()
+ out = StringIO()
try:
- content = response.read()
- except urllib2.URLError, e:
- print e.reason
- raise
+ curl.setopt(pycurl.FOLLOWLOCATION, 1)
+ curl.setopt(pycurl.MAXREDIRS, max_redirects)
+ curl.setopt(pycurl.CONNECTTIMEOUT, connect_timeout)
+ curl.setopt(pycurl.TIMEOUT, timeout)
+ curl.setopt(pycurl.NOSIGNAL, 1)
+ curl.setopt(pycurl.URL, url)
+ curl.setopt(pycurl.WRITEFUNCTION, out.write)
+ curl.perform()
+ content = out.getvalue()
+ except Exception as e:
+ print >> sys.stderr, "Error reading url: " + str(e)
+ print >> sys.stderr, url
finally:
- response.close()
+ curl.close()
return content
diff --git a/setup.py b/setup.py
index 84ef2911..3d803c6d 100644
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,8 @@ setup(
'numpy',
'matplotlib',
'scipy',
- 'obspy'
+ 'obspy',
+ 'pycurl'
],
scripts=[
'bin/geomag.py'
--
GitLab