Merge pull request #265 from ldecicco-USGS/nwisChanges

Nwis changes

DESCRIPTION

 Package: dataRetrieval
 Type: Package
 Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data
-Version: 2.5.11
-Date: 2016-08-02
+Version: 2.5.12
+Date: 2016-08-08
 Authors@R: c( person("Robert", "Hirsch", role = c("aut"),
     email = "rhirsch@usgs.gov"),
     person("Laura", "DeCicco", role = c("aut","cre"),
@@ -37,7 +37,7 @@ Imports:
     stats,
     utils,
     dplyr,
-    readr (>= 0.1.1.9000)
+    readr (>= 1.0.0)
 Suggests:
     xtable,
     knitr,

NEWS

+dataRetrieval 2.5.12
+==========
+* Response to readr 1.0.0 updates
+* Added NWIS Water Use data
+* Added NWIS Statistical Service data
+* Added message on updates to NWIS data on startup message
+* Automatically retries web service calls when status code is greater than 400
+* More integrated use of pkg.env to set base URLs
+
 dataRetrieval 2.5.7
 ==========
 * HEAD requests can be returned in readWQP* functions. This can give some indication on amount of data available.

R/AAA.R

 pkg.env <- new.env()
+
 .onLoad = function(libname, pkgname){
   suppressMessages(setAccess('public'))
   options(Access.dataRetrieval = NULL)
-}
\ No newline at end of file
+}
+
+.onAttach = function(libname, pkgname){
+  packageStartupMessage("Recent changes to NWIS data may affect dataRetrieval output. 
+Please see http://help.waterdata.usgs.gov/news/061016 
+for more information.")  
+}

R/constructNWISURL.r

@@ -97,9 +97,10 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
              
              searchCriteria <- paste(searchCriteria, "multiple_parameter_cds", sep=",")
              searchCriteria <- paste("list_of_search_criteria",searchCriteria,sep="=")
-             baseURL <- "http://nwis.waterdata.usgs.gov/nwis/qwdata"
+
+             baseURL <- drURL("qwdata")
              
-             url <- paste(baseURL,siteNumber,sep="?")
+             url <- paste0(baseURL,siteNumber)
              url <- paste(url, pCodes,searchCriteria,
                           "group_key=NONE&sitefile_output_format=html_table&column_name=agency_cd",
                           "column_name=site_no&column_name=station_nm&inventory_output=0&rdb_inventory_output=file",
@@ -107,48 +108,48 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
                           "format=rdb&rdb_qw_attributes=0&date_format=YYYY-MM-DD",
                           "rdb_compression=value", sep = "&")
              if(expanded){
-               url <- paste0(url,"&qw_sample_wide=0")
+               url <- appendDrURL(url,qw_sample_wide="0")
                url <- gsub("rdb_qw_attributes=0","rdb_qw_attributes=expanded",url)
              } else {
-               url <- paste0(url,"&qw_sample_wide=separated_wide")
+               url <- appendDrURL(url,qw_sample_wide="separated_wide")
              }
              
              if (nzchar(startDate)) {
-               url <- paste(url,"&begin_date=",startDate,sep="")
+               url <- appendDrURL(url,begin_date=startDate)
              }
              
              if (nzchar(endDate)) {
-               url <- paste(url,"&end_date=",endDate,sep="")
+               url <- appendDrURL(url,end_date=endDate)
              }
            },
         rating = {
           ratingType <- match.arg(ratingType, c("base", "corr", "exsa"))
-          url <- paste0("http://waterdata.usgs.gov/nwisweb/get_ratings?site_no=",
-                siteNumber, "&file_type=", ratingType)
+          url <- drURL("rating", site_no=siteNumber,file_type=ratingType)
         },
         peak = {
-          url <- paste0("http://nwis.waterdata.usgs.gov/usa/nwis/peak/?site_no=", siteNumber,
-                "&range_selection=date_range&format=rdb")
+          url <- drURL("peak", site_no=siteNumber,
+                       range_selection="date_range",
+                       format="rdb")
           if (nzchar(startDate)) {
-            url <- paste0(url,"&begin_date=",startDate)
-          }
+            url <- appendDrURL(url,begin_date=startDate)
+          } 
           if(nzchar(endDate)){
-            url <- paste0(url, "&end_date=", endDate)
+            url <- appendDrURL(url, end_date=endDate)
           }
         },
         meas = {
-          url <- paste0("http://waterdata.usgs.gov/nwis/measurements?site_no=", siteNumber,
-                "&range_selection=date_range")
+          url <- drURL("measurements", site_no=siteNumber,
+                       range_selection="date_range")
           if (nzchar(startDate)) {
-            url <- paste0(url,"&begin_date=",startDate)
+            url <- appendDrURL(url,begin_date=startDate)
           }
           if(nzchar(endDate)){
-            url <- paste0(url, "&end_date=", endDate)
+            url <- appendDrURL(url, end_date=endDate)
           }
           if(expanded){
-            url <- paste0(url,"&format=rdb_expanded")
+            url <- appendDrURL(url,format="rdb_expanded")
           } else {
-            url <- paste0(url,"&format=rdb")
+            url <- appendDrURL(url,format="rdb")
           }
 
         },
@@ -172,16 +173,18 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
           }
           statType <- paste(statType,collapse=",")
           parameterCd <- paste(parameterCd,collapse=",")
-          url <- paste0("http://waterservices.usgs.gov/nwis/stat/?format=rdb&sites=",siteNumber,
-                        "&statType=",statType,"&statReportType=",statReportType,"&parameterCd=",parameterCd)
+          url <- drURL("stat", sites=siteNumber,
+                       statType=statType,
+                       statReportType=statReportType,
+                       parameterCd=parameterCd)
           if (nzchar(startDate)) {
-            url <- paste0(url,"&startDT=",startDate)
+            url <- appendDrURL(url,startDT=startDate)
           }
           if (nzchar(endDate)) {
-            url <- paste0(url,"&endDT=",endDate)
+            url <- appendDrURL(url,endDT=endDate)
           }
           if (!grepl("(?i)daily",statReportType)){
-            url <- paste0(url,"&missingData=off")
+            url <- appendDrURL(url,missingData="off")
           }
           
         },

R/importRDB1.r

@@ -258,6 +258,9 @@ importRDB1 <- function(obs_url, asDateTime=TRUE, convertType = TRUE, tz=""){
     attr(readr.data, "header") <- attr(doc, "header")
   }
 
+  if("spec" %in% names(attributes(readr.data))){
+    attr(readr.data, "spec") <- NULL
+  }
   
   return(readr.data)
   

R/readNWISunit.r

@@ -450,8 +450,12 @@ readNWISgwl <- function (siteNumbers,startDate="",endDate="", convertType = TRUE
 #' @importFrom dplyr left_join
 #' @examples
 #' \dontrun{
+#' x1 <- readNWISstat(siteNumbers=c("02319394"),
+#'                   parameterCd=c("00060"),
+#'                   statReportType="annual") 
+#' 
 #' #all the annual mean discharge data for two sites
-#' x <- readNWISstat(siteNumbers=c("02319394","02171500"),
+#' x2 <- readNWISstat(siteNumbers=c("02319394","02171500"),
 #'                   parameterCd=c("00010","00060"),
 #'                   statReportType="annual")
 #' 

R/setAccess.R

@@ -41,7 +41,7 @@ access = match.arg(access, c('public','internal','cooperator','USGS'))
   }
   
   pkg.env$site = "http://waterservices.usgs.gov/nwis/site/"
-  pkg.env$iv = "http://nwis.waterservices.usgs.gov/nwis/iv/"
+  pkg.env$iv = "http://waterservices.usgs.gov/nwis/iv/"
   pkg.env$dv =  "http://waterservices.usgs.gov/nwis/dv/"
   pkg.env$gwlevels = "http://waterservices.usgs.gov/nwis/gwlevels/"
   pkg.env$measurements = "http://waterdata.usgs.gov/nwis/measurements/"
@@ -49,7 +49,7 @@ access = match.arg(access, c('public','internal','cooperator','USGS'))
   pkg.env$rating = "http://waterdata.usgs.gov/nwisweb/get_ratings/"
   pkg.env$qwdata = "http://nwis.waterdata.usgs.gov/nwis/qwdata"
   pkg.env$stat = "http://waterservices.usgs.gov/nwis/stat/"
-  
+
   options(Access.dataRetrieval = access)
 }
 

inst/doc/dataRetrieval.Rnw

@@ -683,7 +683,7 @@ specificCond <- readWQPqw('WIDNR_WQX-10032762',
 
 A tool for finding NWIS characteristic names can be found at: 
 
-\url{http://www.waterqualitydata.us/public_srsnames.jsp}
+\url{http://www.waterqualitydata.us/public_srsnames/}
 
 \FloatBarrier
 

man/readNWISstat.Rd

@@ -53,8 +53,12 @@ See \url{http://waterservices.usgs.gov/rest/Statistics-Service.html} for more in
 }
 \examples{
 \dontrun{
+x1 <- readNWISstat(siteNumbers=c("02319394"),
+                  parameterCd=c("00060"),
+                  statReportType="annual") 
+
 #all the annual mean discharge data for two sites
-x <- readNWISstat(siteNumbers=c("02319394","02171500"),
+x2 <- readNWISstat(siteNumbers=c("02319394","02171500"),
                   parameterCd=c("00010","00060"),
                   statReportType="annual")
 

tests/testthat/tests_userFriendly_fxns.R

@@ -18,13 +18,23 @@ test_that("Unit value data returns correct types", {
   timeZoneChange <- renameNWISColumns(timeZoneChange)
   expect_is(rawData$dateTime, 'POSIXct')
   expect_is(rawData$Flow_Inst, 'numeric')
-  expect_that(attr(rawData, "url"), equals(
-      "http://nwis.waterservices.usgs.gov/nwis/iv/?site=05114000&format=waterml,1.1&ParameterCd=00060&startDT=2014-10-10&endDT=2014-10-10")
-  )
+  # expect_that(attr(rawData, "url"), equals(
+  #     "http://nwis.waterservices.usgs.gov/nwis/iv/?site=05114000&format=waterml,1.1&ParameterCd=00060&startDT=2014-10-10&endDT=2014-10-10")
+  # )
 #   #First switchover to standard time:
 #   expect_that(as.numeric(timeZoneChange[which(timeZoneChange$tz_cd == "America/Chicago")[1],"dateTime"]),
 #               equals(as.numeric(as.POSIXct("2013-11-03 01:00:00", tz="UTC")+60*60*6)))
   
+  site <- "04087170"
+  pCode <- "63680"
+  startDate <- "2012-07-10"
+  endDate <- "2012-07-17"
+  dd_2 <- readNWISuv(site, pCode, startDate, endDate)
+  expect_true(all(names(dd_2) %in% c("agency_cd","site_no",                   
+                                 "dateTime","X_.YSI.6136.UP._63680_00011",   
+                                 "X_YSI.6136.DOWN_63680_00011","X_.YSI.6136.UP._63680_00011_cd",
+                                 "X_YSI.6136.DOWN_63680_00011_cd","tz_cd")))
+  
 })
 
 context("Peak, rating, meas, site")