Commit e671e03c authored by Laura A DeCicco's avatar Laura A DeCicco
Browse files

Merge pull request #168 from ldecicco-USGS/master

Change WQP requests to zip as default
parents 95243b70 66a85d25
Package: dataRetrieval
Type: Package
Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data
Version: 2.4.5
Date: 2015-12-09
Version: 2.4.6
Date: 2016-01-26
Authors@R: c( person("Robert", "Hirsch", role = c("aut"),
email = "rhirsch@usgs.gov"),
person("Laura", "DeCicco", role = c("aut","cre"),
......
......@@ -222,6 +222,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
#' retrieval for the earliest possible record.
#' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the latest possible record.
#' @param zip logical to request data via downloading zip file. Default set to TRUE.
#' @keywords data import WQP web service
#' @return url string
#' @export
......@@ -233,7 +234,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
#' url_wqp <- constructWQPURL(paste("USGS",siteNumber,sep="-"),
#' c('01075','00029','00453'),
#' startDate,endDate)
constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){
constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate,zip=TRUE){
multipleSites <- length(siteNumber) > 1
multiplePcodes <- length(parameterCd)>1
......@@ -267,6 +268,11 @@ constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){
}
url <- paste0(url,"&sorted=no&mimeType=tsv")
if(zip){
url <- paste0(url,"&zip=yes")
}
return(url)
}
......@@ -20,9 +20,12 @@
#' rawData <- getWebServiceData(obs_url)
#' }
getWebServiceData <- function(obs_url, ...){
possibleError <- tryCatch({
h <- basicHeaderGatherer()
returnedDoc <- getURI(obs_url, headerfunction = h$update, ...)
returnedDoc <- getURI(obs_url, headerfunction = h$update,
useragent = paste("dataRetrieval",packageVersion("dataRetrieval"),sep="/"), ...)
}, warning = function(w) {
warning(w, "with url:", obs_url)
}, error = function(e) {
......
......@@ -185,7 +185,9 @@ importRDB1 <- function(obs_url, asDateTime=TRUE, convertType = TRUE, tz=""){
if("tz_cd" %in% header.names){
date.time.cols <- which(sapply(readr.data, function(x) inherits(x, "POSIXct")))
readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE)
if(length(date.time.cols) > 0){
readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE)
}
}
if("DATE" %in% header.names){
......
......@@ -5,7 +5,7 @@
#' Imports data from the Water Quality Portal based on a specified url.
#'
#' @param obs_url character URL to Water Quality Portal#' @keywords data import USGS web service
#' @param zip logical used to request the data in a zip format (TRUE)
#' @param zip logical to request data via downloading zip file. Default set to TRUE.
#' @param tz character to set timezone attribute of datetime. Default is an empty quote, which converts the
#' datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column).
#' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
......@@ -34,13 +34,14 @@
#' rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '')
#'
#' rawSample <- importWQP(rawSampleURL)
#' url2 <- paste0(rawSampleURL,"&zip=yes")
#' rawSample2 <- importWQP(url2, zip=TRUE)
#'
#' rawSampleURL_noZip <- constructWQPURL('USGS-01594440','01075', '', '', FALSE)
#' rawSample2 <- importWQP(rawSampleURL_noZip, zip=FALSE)
#'
#' STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')
#' STORETdata <- importWQP(STORETex)
#' }
importWQP <- function(obs_url, zip=FALSE, tz=""){
importWQP <- function(obs_url, zip=TRUE, tz=""){
if(tz != ""){
tz <- match.arg(tz, c("America/New_York","America/Chicago",
......
......@@ -4,8 +4,7 @@
#' because it allows for other agencies rather than the USGS.
#'
#' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options
#' @param zip logical to request data via downloading zip file. Default set to FALSE. The overhead associated with
#' downloading and un-zipping only improves performance for large data returns.
#' @param zip logical to request data via downloading zip file. Default set to TRUE.
#' @keywords data import WQP web service
#' @return A data frame with at least the following columns:
#' \tabular{lll}{
......@@ -98,7 +97,7 @@
#'
#'
#' }
readWQPdata <- function(..., zip=FALSE){
readWQPdata <- function(..., zip=TRUE){
matchReturn <- list(...)
......@@ -158,7 +157,7 @@ readWQPdata <- function(..., zip=FALSE){
retval <- importWQP(urlCall,zip=zip, tz=tz)
if(!all(is.na(retval))){
siteInfo <- whatWQPsites(...)
siteInfo <- whatWQPsites(...,zip=zip)
siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName,
agency_cd=siteInfo$OrganizationIdentifier,
......
......@@ -4,6 +4,7 @@
#' Arguments to the function should be based on \url{www.waterqualitydata.us/webservices_documentation.jsp}
#'
#' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options
#' @param zip logical to request data via downloading zip file. Default set to TRUE.
#' @keywords data import WQP web service
#' @return A data frame with at least the following columns:
#' \tabular{lll}{
......@@ -55,9 +56,9 @@
#'
#' type <- "Stream"
#' sites <- whatWQPsites(countycode="US:55:025",siteType=type)
#' data <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
#' lakeSites <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
#' }
whatWQPsites <- function(...){
whatWQPsites <- function(...,zip=TRUE){
matchReturn <- list(...)
......@@ -94,12 +95,17 @@ whatWQPsites <- function(...){
baseURL <- "http://www.waterqualitydata.us/Station/search?"
urlCall <- paste(baseURL,
urlCall <- paste0(baseURL,
urlCall,
"&mimeType=tsv&sorted=no",sep = "")
"&mimeType=tsv&sorted=no")
if(zip){
urlCall <- paste0(urlCall,"&zip=yes")
}
retval <- importWQP(urlCall, zip=zip)
retval <- importWQP(urlCall)
retval$queryTime <- Sys.time()
attr(retval, "queryTime") <- Sys.time()
attr(retval, "url") <- urlCall
return(retval)
......
......@@ -4,7 +4,7 @@
\alias{constructWQPURL}
\title{Construct WQP url for data retrieval}
\usage{
constructWQPURL(siteNumber, parameterCd, startDate, endDate)
constructWQPURL(siteNumber, parameterCd, startDate, endDate, zip = TRUE)
}
\arguments{
\item{siteNumber}{string or vector of strings USGS site number. This is usually an 8 digit number}
......@@ -16,6 +16,8 @@ retrieval for the earliest possible record.}
\item{endDate}{character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
retrieval for the latest possible record.}
\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
}
\value{
url string
......
......@@ -4,12 +4,12 @@
\alias{importWQP}
\title{Basic Water Quality Portal Data parser}
\usage{
importWQP(obs_url, zip = FALSE, tz = "")
importWQP(obs_url, zip = TRUE, tz = "")
}
\arguments{
\item{obs_url}{character URL to Water Quality Portal#' @keywords data import USGS web service}
\item{zip}{logical used to request the data in a zip format (TRUE)}
\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
\item{tz}{character to set timezone attribute of datetime. Default is an empty quote, which converts the
datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column).
......@@ -31,8 +31,9 @@ Imports data from the Water Quality Portal based on a specified url.
rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '')
rawSample <- importWQP(rawSampleURL)
url2 <- paste0(rawSampleURL,"&zip=yes")
rawSample2 <- importWQP(url2, zip=TRUE)
rawSampleURL_noZip <- constructWQPURL('USGS-01594440','01075', '', '', FALSE)
rawSample2 <- importWQP(rawSampleURL_noZip, zip=FALSE)
STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')
STORETdata <- importWQP(STORETex)
......
......@@ -4,11 +4,10 @@
\alias{readWQPdata}
\title{General Data Import from Water Quality Portal}
\usage{
readWQPdata(..., zip = FALSE)
readWQPdata(..., zip = TRUE)
}
\arguments{
\item{zip}{logical to request data via downloading zip file. Default set to FALSE. The overhead associated with
downloading and un-zipping only improves performance for large data returns.}
\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
\item{\dots}{see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options}
}
......
......@@ -4,9 +4,11 @@
\alias{whatWQPsites}
\title{Site Data Import from Water Quality Portal}
\usage{
whatWQPsites(...)
whatWQPsites(..., zip = TRUE)
}
\arguments{
\item{zip}{logical to request data via downloading zip file. Default set to TRUE.}
\item{\dots}{see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options}
}
\value{
......@@ -62,7 +64,7 @@ site1 <- whatWQPsites(siteid="USGS-01594440")
type <- "Stream"
sites <- whatWQPsites(countycode="US:55:025",siteType=type)
data <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
lakeSites <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
}
}
\keyword{WQP}
......
......@@ -128,8 +128,8 @@ test_that("External WQP tests", {
rawSample <- importWQP(rawSampleURL)
expect_is(rawSample$ActivityStartDateTime, 'POSIXct')
url2 <- paste0(rawSampleURL,"&zip=yes")
rawSample2 <- suppressWarnings(importWQP(url2, TRUE))
url2 <- constructWQPURL('USGS-01594440','01075', '', '', zip = FALSE)
rawSample2 <- suppressWarnings(importWQP(url2, FALSE))
expect_is(rawSample2$ActivityStartDateTime, 'POSIXct')
STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment