Merge pull request #168 from ldecicco-USGS/master

Change WQP requests to zip as default

DESCRIPTION

 Package: dataRetrieval
 Type: Package
 Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data
-Version: 2.4.5
-Date: 2015-12-09
+Version: 2.4.6
+Date: 2016-01-26
 Authors@R: c( person("Robert", "Hirsch", role = c("aut"),
     email = "rhirsch@usgs.gov"),
     person("Laura", "DeCicco", role = c("aut","cre"),

R/constructNWISURL.r

@@ -222,6 +222,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
 #' retrieval for the earliest possible record.
 #' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
 #' retrieval for the latest possible record.
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @keywords data import WQP web service
 #' @return url string
 #' @export
@@ -233,7 +234,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
 #' url_wqp <- constructWQPURL(paste("USGS",siteNumber,sep="-"),
 #'            c('01075','00029','00453'),
 #'            startDate,endDate)
-constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){
+constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate,zip=TRUE){
   
   multipleSites <- length(siteNumber) > 1
   multiplePcodes <- length(parameterCd)>1
@@ -267,6 +268,11 @@ constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){
   }
   
   url <- paste0(url,"&sorted=no&mimeType=tsv")
+  
+  if(zip){
+    url <- paste0(url,"&zip=yes")
+  }
+  
   return(url)
 
 }

R/getWebServiceData.R

@@ -20,9 +20,12 @@
 #' rawData <- getWebServiceData(obs_url)
 #' }
 getWebServiceData <- function(obs_url, ...){
+  
   possibleError <- tryCatch({
     h <- basicHeaderGatherer()
-    returnedDoc <- getURI(obs_url, headerfunction = h$update, ...)      
+    
+    returnedDoc <- getURI(obs_url, headerfunction = h$update, 
+                          useragent = paste("dataRetrieval",packageVersion("dataRetrieval"),sep="/"), ...)      
   }, warning = function(w) {
     warning(w, "with url:", obs_url)
   }, error = function(e) {

R/importRDB1.r

@@ -185,7 +185,9 @@ importRDB1 <- function(obs_url, asDateTime=TRUE, convertType = TRUE, tz=""){
       
       if("tz_cd" %in% header.names){
         date.time.cols <- which(sapply(readr.data, function(x) inherits(x, "POSIXct")))
-        readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE)
+        if(length(date.time.cols) > 0){
+          readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE)
+        }
       }
       
       if("DATE" %in% header.names){

R/importWQP.R

@@ -5,7 +5,7 @@
 #' Imports data from the Water Quality Portal based on a specified url.
 #' 
 #' @param obs_url character URL to Water Quality Portal#' @keywords data import USGS web service
-#' @param zip logical used to request the data in a zip format (TRUE)
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @param tz character to set timezone attribute of datetime. Default is an empty quote, which converts the 
 #' datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column).
 #' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
@@ -34,13 +34,14 @@
 #' rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '')
 #' 
 #' rawSample <- importWQP(rawSampleURL)
-#' url2 <- paste0(rawSampleURL,"&zip=yes")
-#' rawSample2 <- importWQP(url2, zip=TRUE)
+#' 
+#' rawSampleURL_noZip <- constructWQPURL('USGS-01594440','01075', '', '', FALSE)
+#' rawSample2 <- importWQP(rawSampleURL_noZip, zip=FALSE)
 #' 
 #' STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')
 #' STORETdata <- importWQP(STORETex)
 #' }
-importWQP <- function(obs_url, zip=FALSE, tz=""){
+importWQP <- function(obs_url, zip=TRUE, tz=""){
   
   if(tz != ""){
     tz <- match.arg(tz, c("America/New_York","America/Chicago",

R/readWQPdata.R

@@ -4,8 +4,7 @@
 #' because it allows for other agencies rather than the USGS.  
 #'
 #' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options
-#' @param zip logical to request data via downloading zip file. Default set to FALSE. The overhead associated with 
-#' downloading and un-zipping only improves performance for large data returns.
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @keywords data import WQP web service
 #' @return A data frame with at least the following columns:
 #' \tabular{lll}{ 
@@ -98,7 +97,7 @@
 #' 
 #'                         
 #' }
-readWQPdata <- function(..., zip=FALSE){
+readWQPdata <- function(..., zip=TRUE){
   
   matchReturn <- list(...)
 
@@ -158,7 +157,7 @@ readWQPdata <- function(..., zip=FALSE){
   retval <- importWQP(urlCall,zip=zip, tz=tz)
   
   if(!all(is.na(retval))){
-    siteInfo <- whatWQPsites(...)
+    siteInfo <- whatWQPsites(...,zip=zip)
     
     siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName,
                                  agency_cd=siteInfo$OrganizationIdentifier,

R/whatWQPsites.R

@@ -4,6 +4,7 @@
 #' Arguments to the function should be based on \url{www.waterqualitydata.us/webservices_documentation.jsp}
 #'
 #' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @keywords data import WQP web service
 #' @return A data frame with at least the following columns:
 #' \tabular{lll}{ 
@@ -55,9 +56,9 @@
 #' 
 #' type <- "Stream"
 #' sites <- whatWQPsites(countycode="US:55:025",siteType=type)
-#' data <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
+#' lakeSites <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
 #' }
-whatWQPsites <- function(...){
+whatWQPsites <- function(...,zip=TRUE){
 
   matchReturn <- list(...)
   
@@ -94,12 +95,17 @@ whatWQPsites <- function(...){
   
   
   baseURL <- "http://www.waterqualitydata.us/Station/search?"
-  urlCall <- paste(baseURL,
+  urlCall <- paste0(baseURL,
                urlCall,
-               "&mimeType=tsv&sorted=no",sep = "")
+               "&mimeType=tsv&sorted=no")
+  if(zip){
+    urlCall <- paste0(urlCall,"&zip=yes")
+  }
+  
+  retval <- importWQP(urlCall, zip=zip)
   
-  retval <- importWQP(urlCall)
-  retval$queryTime <- Sys.time()
+  attr(retval, "queryTime") <- Sys.time()
+  attr(retval, "url") <- urlCall
   
   return(retval)
   

man/constructWQPURL.Rd

@@ -4,7 +4,7 @@
 \alias{constructWQPURL}
 \title{Construct WQP url for data retrieval}
 \usage{
-constructWQPURL(siteNumber, parameterCd, startDate, endDate)
+constructWQPURL(siteNumber, parameterCd, startDate, endDate, zip = TRUE)
 }
 \arguments{
 \item{siteNumber}{string or vector of strings USGS site number.  This is usually an 8 digit number}
@@ -16,6 +16,8 @@ retrieval for the earliest possible record.}
 
 \item{endDate}{character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
 retrieval for the latest possible record.}
+
+\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
 }
 \value{
 url string

man/importWQP.Rd

@@ -4,12 +4,12 @@
 \alias{importWQP}
 \title{Basic Water Quality Portal Data parser}
 \usage{
-importWQP(obs_url, zip = FALSE, tz = "")
+importWQP(obs_url, zip = TRUE, tz = "")
 }
 \arguments{
 \item{obs_url}{character URL to Water Quality Portal#' @keywords data import USGS web service}
 
-\item{zip}{logical used to request the data in a zip format (TRUE)}
+\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
 
 \item{tz}{character to set timezone attribute of datetime. Default is an empty quote, which converts the 
 datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column).
@@ -31,8 +31,9 @@ Imports data from the Water Quality Portal based on a specified url.
 rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '')
 
 rawSample <- importWQP(rawSampleURL)
-url2 <- paste0(rawSampleURL,"&zip=yes")
-rawSample2 <- importWQP(url2, zip=TRUE)
+
+rawSampleURL_noZip <- constructWQPURL('USGS-01594440','01075', '', '', FALSE)
+rawSample2 <- importWQP(rawSampleURL_noZip, zip=FALSE)
 
 STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')
 STORETdata <- importWQP(STORETex)

man/readWQPdata.Rd

@@ -4,11 +4,10 @@
 \alias{readWQPdata}
 \title{General Data Import from Water Quality Portal}
 \usage{
-readWQPdata(..., zip = FALSE)
+readWQPdata(..., zip = TRUE)
 }
 \arguments{
-\item{zip}{logical to request data via downloading zip file. Default set to FALSE. The overhead associated with 
-downloading and un-zipping only improves performance for large data returns.}
+\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
 
 \item{\dots}{see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options}
 }

man/whatWQPsites.Rd

@@ -4,9 +4,11 @@
 \alias{whatWQPsites}
 \title{Site Data Import from Water Quality Portal}
 \usage{
-whatWQPsites(...)
+whatWQPsites(..., zip = TRUE)
 }
 \arguments{
+\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
+
 \item{\dots}{see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options}
 }
 \value{
@@ -62,7 +64,7 @@ site1 <- whatWQPsites(siteid="USGS-01594440")
 
 type <- "Stream"
 sites <- whatWQPsites(countycode="US:55:025",siteType=type)
-data <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
+lakeSites <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
 }
 }
 \keyword{WQP}

tests/testthat/tests_imports.R

@@ -128,8 +128,8 @@ test_that("External WQP tests", {
   rawSample <- importWQP(rawSampleURL)
   expect_is(rawSample$ActivityStartDateTime, 'POSIXct')
   
-  url2 <- paste0(rawSampleURL,"&zip=yes")
-  rawSample2 <- suppressWarnings(importWQP(url2, TRUE))
+  url2 <- constructWQPURL('USGS-01594440','01075', '', '', zip = FALSE)
+  rawSample2 <- suppressWarnings(importWQP(url2, FALSE))
   expect_is(rawSample2$ActivityStartDateTime, 'POSIXct')
   
   STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')