Commit d3ff929c authored by Laura A DeCicco's avatar Laura A DeCicco Committed by GitHub
Browse files

Merge pull request #265 from ldecicco-USGS/nwisChanges

Nwis changes
parents 21cd62e7 162239fa
Package: dataRetrieval Package: dataRetrieval
Type: Package Type: Package
Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data
Version: 2.5.11 Version: 2.5.12
Date: 2016-08-02 Date: 2016-08-08
Authors@R: c( person("Robert", "Hirsch", role = c("aut"), Authors@R: c( person("Robert", "Hirsch", role = c("aut"),
email = "rhirsch@usgs.gov"), email = "rhirsch@usgs.gov"),
person("Laura", "DeCicco", role = c("aut","cre"), person("Laura", "DeCicco", role = c("aut","cre"),
...@@ -37,7 +37,7 @@ Imports: ...@@ -37,7 +37,7 @@ Imports:
stats, stats,
utils, utils,
dplyr, dplyr,
readr (>= 0.1.1.9000) readr (>= 1.0.0)
Suggests: Suggests:
xtable, xtable,
knitr, knitr,
......
dataRetrieval 2.5.12
==========
* Response to readr 1.0.0 updates
* Added NWIS Water Use data
* Added NWIS Statistical Service data
* Added message on updates to NWIS data on startup message
* Automatically retries web service calls when status code is greater than 400
* More integrated use of pkg.env to set base URLs
dataRetrieval 2.5.7 dataRetrieval 2.5.7
========== ==========
* HEAD requests can be returned in readWQP* functions. This can give some indication on amount of data available. * HEAD requests can be returned in readWQP* functions. This can give some indication on amount of data available.
......
pkg.env <- new.env() pkg.env <- new.env()
.onLoad = function(libname, pkgname){ .onLoad = function(libname, pkgname){
suppressMessages(setAccess('public')) suppressMessages(setAccess('public'))
options(Access.dataRetrieval = NULL) options(Access.dataRetrieval = NULL)
} }
\ No newline at end of file
.onAttach = function(libname, pkgname){
packageStartupMessage("Recent changes to NWIS data may affect dataRetrieval output.
Please see http://help.waterdata.usgs.gov/news/061016
for more information.")
}
...@@ -97,9 +97,10 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate ...@@ -97,9 +97,10 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
searchCriteria <- paste(searchCriteria, "multiple_parameter_cds", sep=",") searchCriteria <- paste(searchCriteria, "multiple_parameter_cds", sep=",")
searchCriteria <- paste("list_of_search_criteria",searchCriteria,sep="=") searchCriteria <- paste("list_of_search_criteria",searchCriteria,sep="=")
baseURL <- "http://nwis.waterdata.usgs.gov/nwis/qwdata"
baseURL <- drURL("qwdata")
url <- paste(baseURL,siteNumber,sep="?") url <- paste0(baseURL,siteNumber)
url <- paste(url, pCodes,searchCriteria, url <- paste(url, pCodes,searchCriteria,
"group_key=NONE&sitefile_output_format=html_table&column_name=agency_cd", "group_key=NONE&sitefile_output_format=html_table&column_name=agency_cd",
"column_name=site_no&column_name=station_nm&inventory_output=0&rdb_inventory_output=file", "column_name=site_no&column_name=station_nm&inventory_output=0&rdb_inventory_output=file",
...@@ -107,48 +108,48 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate ...@@ -107,48 +108,48 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
"format=rdb&rdb_qw_attributes=0&date_format=YYYY-MM-DD", "format=rdb&rdb_qw_attributes=0&date_format=YYYY-MM-DD",
"rdb_compression=value", sep = "&") "rdb_compression=value", sep = "&")
if(expanded){ if(expanded){
url <- paste0(url,"&qw_sample_wide=0") url <- appendDrURL(url,qw_sample_wide="0")
url <- gsub("rdb_qw_attributes=0","rdb_qw_attributes=expanded",url) url <- gsub("rdb_qw_attributes=0","rdb_qw_attributes=expanded",url)
} else { } else {
url <- paste0(url,"&qw_sample_wide=separated_wide") url <- appendDrURL(url,qw_sample_wide="separated_wide")
} }
if (nzchar(startDate)) { if (nzchar(startDate)) {
url <- paste(url,"&begin_date=",startDate,sep="") url <- appendDrURL(url,begin_date=startDate)
} }
if (nzchar(endDate)) { if (nzchar(endDate)) {
url <- paste(url,"&end_date=",endDate,sep="") url <- appendDrURL(url,end_date=endDate)
} }
}, },
rating = { rating = {
ratingType <- match.arg(ratingType, c("base", "corr", "exsa")) ratingType <- match.arg(ratingType, c("base", "corr", "exsa"))
url <- paste0("http://waterdata.usgs.gov/nwisweb/get_ratings?site_no=", url <- drURL("rating", site_no=siteNumber,file_type=ratingType)
siteNumber, "&file_type=", ratingType)
}, },
peak = { peak = {
url <- paste0("http://nwis.waterdata.usgs.gov/usa/nwis/peak/?site_no=", siteNumber, url <- drURL("peak", site_no=siteNumber,
"&range_selection=date_range&format=rdb") range_selection="date_range",
format="rdb")
if (nzchar(startDate)) { if (nzchar(startDate)) {
url <- paste0(url,"&begin_date=",startDate) url <- appendDrURL(url,begin_date=startDate)
} }
if(nzchar(endDate)){ if(nzchar(endDate)){
url <- paste0(url, "&end_date=", endDate) url <- appendDrURL(url, end_date=endDate)
} }
}, },
meas = { meas = {
url <- paste0("http://waterdata.usgs.gov/nwis/measurements?site_no=", siteNumber, url <- drURL("measurements", site_no=siteNumber,
"&range_selection=date_range") range_selection="date_range")
if (nzchar(startDate)) { if (nzchar(startDate)) {
url <- paste0(url,"&begin_date=",startDate) url <- appendDrURL(url,begin_date=startDate)
} }
if(nzchar(endDate)){ if(nzchar(endDate)){
url <- paste0(url, "&end_date=", endDate) url <- appendDrURL(url, end_date=endDate)
} }
if(expanded){ if(expanded){
url <- paste0(url,"&format=rdb_expanded") url <- appendDrURL(url,format="rdb_expanded")
} else { } else {
url <- paste0(url,"&format=rdb") url <- appendDrURL(url,format="rdb")
} }
}, },
...@@ -172,16 +173,18 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate ...@@ -172,16 +173,18 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
} }
statType <- paste(statType,collapse=",") statType <- paste(statType,collapse=",")
parameterCd <- paste(parameterCd,collapse=",") parameterCd <- paste(parameterCd,collapse=",")
url <- paste0("http://waterservices.usgs.gov/nwis/stat/?format=rdb&sites=",siteNumber, url <- drURL("stat", sites=siteNumber,
"&statType=",statType,"&statReportType=",statReportType,"&parameterCd=",parameterCd) statType=statType,
statReportType=statReportType,
parameterCd=parameterCd)
if (nzchar(startDate)) { if (nzchar(startDate)) {
url <- paste0(url,"&startDT=",startDate) url <- appendDrURL(url,startDT=startDate)
} }
if (nzchar(endDate)) { if (nzchar(endDate)) {
url <- paste0(url,"&endDT=",endDate) url <- appendDrURL(url,endDT=endDate)
} }
if (!grepl("(?i)daily",statReportType)){ if (!grepl("(?i)daily",statReportType)){
url <- paste0(url,"&missingData=off") url <- appendDrURL(url,missingData="off")
} }
}, },
......
...@@ -258,6 +258,9 @@ importRDB1 <- function(obs_url, asDateTime=TRUE, convertType = TRUE, tz=""){ ...@@ -258,6 +258,9 @@ importRDB1 <- function(obs_url, asDateTime=TRUE, convertType = TRUE, tz=""){
attr(readr.data, "header") <- attr(doc, "header") attr(readr.data, "header") <- attr(doc, "header")
} }
if("spec" %in% names(attributes(readr.data))){
attr(readr.data, "spec") <- NULL
}
return(readr.data) return(readr.data)
......
...@@ -450,8 +450,12 @@ readNWISgwl <- function (siteNumbers,startDate="",endDate="", convertType = TRUE ...@@ -450,8 +450,12 @@ readNWISgwl <- function (siteNumbers,startDate="",endDate="", convertType = TRUE
#' @importFrom dplyr left_join #' @importFrom dplyr left_join
#' @examples #' @examples
#' \dontrun{ #' \dontrun{
#' x1 <- readNWISstat(siteNumbers=c("02319394"),
#' parameterCd=c("00060"),
#' statReportType="annual")
#'
#' #all the annual mean discharge data for two sites #' #all the annual mean discharge data for two sites
#' x <- readNWISstat(siteNumbers=c("02319394","02171500"), #' x2 <- readNWISstat(siteNumbers=c("02319394","02171500"),
#' parameterCd=c("00010","00060"), #' parameterCd=c("00010","00060"),
#' statReportType="annual") #' statReportType="annual")
#' #'
......
...@@ -41,7 +41,7 @@ access = match.arg(access, c('public','internal','cooperator','USGS')) ...@@ -41,7 +41,7 @@ access = match.arg(access, c('public','internal','cooperator','USGS'))
} }
pkg.env$site = "http://waterservices.usgs.gov/nwis/site/" pkg.env$site = "http://waterservices.usgs.gov/nwis/site/"
pkg.env$iv = "http://nwis.waterservices.usgs.gov/nwis/iv/" pkg.env$iv = "http://waterservices.usgs.gov/nwis/iv/"
pkg.env$dv = "http://waterservices.usgs.gov/nwis/dv/" pkg.env$dv = "http://waterservices.usgs.gov/nwis/dv/"
pkg.env$gwlevels = "http://waterservices.usgs.gov/nwis/gwlevels/" pkg.env$gwlevels = "http://waterservices.usgs.gov/nwis/gwlevels/"
pkg.env$measurements = "http://waterdata.usgs.gov/nwis/measurements/" pkg.env$measurements = "http://waterdata.usgs.gov/nwis/measurements/"
...@@ -49,7 +49,7 @@ access = match.arg(access, c('public','internal','cooperator','USGS')) ...@@ -49,7 +49,7 @@ access = match.arg(access, c('public','internal','cooperator','USGS'))
pkg.env$rating = "http://waterdata.usgs.gov/nwisweb/get_ratings/" pkg.env$rating = "http://waterdata.usgs.gov/nwisweb/get_ratings/"
pkg.env$qwdata = "http://nwis.waterdata.usgs.gov/nwis/qwdata" pkg.env$qwdata = "http://nwis.waterdata.usgs.gov/nwis/qwdata"
pkg.env$stat = "http://waterservices.usgs.gov/nwis/stat/" pkg.env$stat = "http://waterservices.usgs.gov/nwis/stat/"
options(Access.dataRetrieval = access) options(Access.dataRetrieval = access)
} }
......
...@@ -683,7 +683,7 @@ specificCond <- readWQPqw('WIDNR_WQX-10032762', ...@@ -683,7 +683,7 @@ specificCond <- readWQPqw('WIDNR_WQX-10032762',
A tool for finding NWIS characteristic names can be found at: A tool for finding NWIS characteristic names can be found at:
\url{http://www.waterqualitydata.us/public_srsnames.jsp} \url{http://www.waterqualitydata.us/public_srsnames/}
\FloatBarrier \FloatBarrier
......
...@@ -53,8 +53,12 @@ See \url{http://waterservices.usgs.gov/rest/Statistics-Service.html} for more in ...@@ -53,8 +53,12 @@ See \url{http://waterservices.usgs.gov/rest/Statistics-Service.html} for more in
} }
\examples{ \examples{
\dontrun{ \dontrun{
x1 <- readNWISstat(siteNumbers=c("02319394"),
parameterCd=c("00060"),
statReportType="annual")
#all the annual mean discharge data for two sites #all the annual mean discharge data for two sites
x <- readNWISstat(siteNumbers=c("02319394","02171500"), x2 <- readNWISstat(siteNumbers=c("02319394","02171500"),
parameterCd=c("00010","00060"), parameterCd=c("00010","00060"),
statReportType="annual") statReportType="annual")
......
...@@ -18,13 +18,23 @@ test_that("Unit value data returns correct types", { ...@@ -18,13 +18,23 @@ test_that("Unit value data returns correct types", {
timeZoneChange <- renameNWISColumns(timeZoneChange) timeZoneChange <- renameNWISColumns(timeZoneChange)
expect_is(rawData$dateTime, 'POSIXct') expect_is(rawData$dateTime, 'POSIXct')
expect_is(rawData$Flow_Inst, 'numeric') expect_is(rawData$Flow_Inst, 'numeric')
expect_that(attr(rawData, "url"), equals( # expect_that(attr(rawData, "url"), equals(
"http://nwis.waterservices.usgs.gov/nwis/iv/?site=05114000&format=waterml,1.1&ParameterCd=00060&startDT=2014-10-10&endDT=2014-10-10") # "http://nwis.waterservices.usgs.gov/nwis/iv/?site=05114000&format=waterml,1.1&ParameterCd=00060&startDT=2014-10-10&endDT=2014-10-10")
) # )
# #First switchover to standard time: # #First switchover to standard time:
# expect_that(as.numeric(timeZoneChange[which(timeZoneChange$tz_cd == "America/Chicago")[1],"dateTime"]), # expect_that(as.numeric(timeZoneChange[which(timeZoneChange$tz_cd == "America/Chicago")[1],"dateTime"]),
# equals(as.numeric(as.POSIXct("2013-11-03 01:00:00", tz="UTC")+60*60*6))) # equals(as.numeric(as.POSIXct("2013-11-03 01:00:00", tz="UTC")+60*60*6)))
site <- "04087170"
pCode <- "63680"
startDate <- "2012-07-10"
endDate <- "2012-07-17"
dd_2 <- readNWISuv(site, pCode, startDate, endDate)
expect_true(all(names(dd_2) %in% c("agency_cd","site_no",
"dateTime","X_.YSI.6136.UP._63680_00011",
"X_YSI.6136.DOWN_63680_00011","X_.YSI.6136.UP._63680_00011_cd",
"X_YSI.6136.DOWN_63680_00011_cd","tz_cd")))
}) })
context("Peak, rating, meas, site") context("Peak, rating, meas, site")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment