readNWISunit.r 30.6 KB
Newer Older
1
#' Instantaneous value data retrieval from USGS (NWIS)
2
#'
Laura A DeCicco's avatar
Laura A DeCicco committed
3
4
5
6
#' Imports data from NWIS web service. This function gets the data from here: \url{https://waterservices.usgs.gov/}
#' A list of parameter codes can be found here: \url{https://nwis.waterdata.usgs.gov/nwis/pmcodes/}
#' A list of statistic codes can be found here: \url{https://nwis.waterdata.usgs.gov/nwis/help/?read_file=stat&format=table}.
#' More information on the web service can be found here: \url{https://waterservices.usgs.gov/rest/IV-Service.html}.
7
#'
8
9
#' @param siteNumbers character USGS site number (or multiple sites).  This is usually an 8 digit number
#' @param parameterCd character USGS parameter code.  This is usually an 5 digit number.
10
#' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
11
#' retrieval for the earliest possible record. Simple date arguments are specified in local time.
Laura A DeCicco's avatar
Laura A DeCicco committed
12
#' See more information here: \url{https://waterservices.usgs.gov/rest/IV-Service.html}.
13
#' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
14
#' retrieval for the latest possible record. Simple date arguments are specified in local time.
Laura A DeCicco's avatar
Laura A DeCicco committed
15
#' See more information here: \url{https://waterservices.usgs.gov/rest/IV-Service.html}.
16
17
#' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the 
#' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column.
18
#' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
19
20
21
#' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu",
#' "America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla". See also  \code{OlsonNames()} 
#' for more information on time zones.
22
#' @keywords data import USGS web service
Laura A DeCicco's avatar
Laura A DeCicco committed
23
24
25
#' @return A data frame with the following columns:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
26
27
28
29
#' agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr
#' site_no \tab character \tab The USGS site number \cr
#' dateTime \tab POSIXct \tab The date and time of the value converted to UTC \cr 
#' tz_cd \tab character \tab The time zone code for dateTime \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
30
31
32
#' code \tab character \tab Any codes that qualify the corresponding value\cr
#' value \tab numeric \tab The numeric value for the parameter \cr
#' }
Laura A DeCicco's avatar
Laura A DeCicco committed
33
#' Note that code and value are repeated for the parameters requested. The names are of the form: 
Laura A DeCicco's avatar
Laura A DeCicco committed
34
35
36
37
38
39
#' X_D_P_S, where X is literal, 
#' D is an option description of the parameter, 
#' P is the parameter code, 
#' and S is the statistic code (if applicable).
#' 
#' There are also several useful attributes attached to the data frame:
Laura A DeCicco's avatar
Laura A DeCicco committed
40
#' \tabular{lll}{
Laura A DeCicco's avatar
Laura A DeCicco committed
41
42
43
44
45
46
47
48
49
#' Name \tab Type \tab Description \cr
#' url \tab character \tab The url used to generate the data \cr
#' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr
#' variableInfo \tab data.frame \tab A data frame containing information on the requested parameters \cr
#' statisticInfo \tab data.frame \tab A data frame containing information on the requested statistics on the data \cr
#' queryTime \tab POSIXct \tab The time the data was returned \cr
#' }
#' 
#' @seealso \code{\link{renameNWISColumns}}, \code{\link{importWaterML1}}
50
51
#' @export
#' @examples
52
#' site_id <- '05114000'
Laura A DeCicco's avatar
Laura A DeCicco committed
53
#' parameterCd <- '00060'
54
55
#' startDate <- "2014-10-10"
#' endDate <- "2014-10-10"
56
#' \donttest{
Laura A DeCicco's avatar
Laura A DeCicco committed
57
#' 
58
#' rawData <- readNWISuv(site_id,parameterCd,startDate,endDate)
59
#' 
60
61
#' rawData_today <- readNWISuv(site_id, parameterCd, Sys.Date(),Sys.Date())
#' 
62
#' timeZoneChange <- readNWISuv(c('04024430','04024000'),parameterCd,
Laura A DeCicco's avatar
Laura A DeCicco committed
63
#'          "2013-11-03","2013-11-03")
64
#'  
65
#' centralTime <- readNWISuv(site_id,parameterCd,
66
67
68
69
#'                            "2014-10-10T12:00", "2014-10-10T23:59",
#'                            tz="America/Chicago")
#' 
#' # Adding 'Z' to the time indicates to the web service to call the data with UTC time:
70
#' GMTdata <- readNWISuv(site_id,parameterCd,
71
#'                            "2014-10-10T00:00Z", "2014-10-10T23:59Z")
Laura A DeCicco's avatar
Laura A DeCicco committed
72
#' 
73
#' }
74
readNWISuv <- function (siteNumbers,parameterCd,startDate="",endDate="", tz="UTC"){  
75
  
76
77
78
79
80
81
82
  if(as.character(startDate) == "" || (as.Date(startDate) <= Sys.Date()-120)){
    service <- "iv"
  } else {
    service <- "iv_recent"
  }
  
  url <- constructNWISURL(siteNumbers,parameterCd,startDate,endDate,service,format="xml")
83

84
  data <- importWaterML1(url,asDateTime=TRUE,tz=tz)
85
  
86
  return (data)
Laura A DeCicco's avatar
Laura A DeCicco committed
87
}
88

89
#' Peak flow data from USGS (NWIS)
90
#' 
Laura A DeCicco's avatar
Laura A DeCicco committed
91
#' Reads peak flow from NWISweb. Data is retrieved from \url{https://waterdata.usgs.gov/nwis}.
92
#' In some cases, the specific date of the peak data is not know. This function will default to
Laura A DeCicco's avatar
Laura A DeCicco committed
93
94
#' converting complete dates to a "Date" object, and converting incomplete dates to "NA". If those incomplete dates are
#' needed, set the `asDateTime` argument to FALSE. No dates will be converted to R Date objects.
95
#' 
96
#' @param siteNumbers character USGS site number(or multiple sites).  This is usually an 8 digit number.
97
98
99
100
#' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the earliest possible record.
#' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the latest possible record.
101
102
#' @param asDateTime logical default to \code{TRUE}. When \code{TRUE}, the peak_dt column is converted
#' to a Date object, and incomplete dates are removed. When \code{FALSE}, no columns are removed, but no dates are converted.
Laura A DeCicco's avatar
Laura A DeCicco committed
103
104
#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates, datetimes,
#' numerics based on a standard algorithm. If false, everything is returned as a character
105
106
107
108
109
#' @return A data frame with the following columns:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
#' agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr
#' site_no \tab character \tab The USGS site number \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
110
111
112
113
114
115
#' peak_dt \tab Date \tab Date of peak streamflow \cr
#' peak_tm \tab character \tab Time of peak streamflow as character \cr
#' peak_va \tab numeric \tab Annual peak streamflow value in cfs \cr
#' peak_cd \tab character \tab Peak Discharge-Qualification codes (see \code{comment} for more information) \cr
#' gage_ht \tab numeric \tab Gage height for the associated peak streamflow in feet \cr
#' gage_ht_cd \tab character \tab Gage height qualification codes \cr
116
117
#' year_last_pk \tab numeric \tab Peak streamflow reported is the highest since this year \cr
#' ag_dt \tab Date \tab Date of maximum gage-height for water year (if not concurrent with peak) \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
118
#' ag_tm \tab character \tab Time of maximum gage-height for water year (if not concurrent with peak) \cr
119
#' ag_gage_ht \tab numeric \tab maximum Gage height for water year in feet (if not concurrent with peak) \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
120
#' ag_gage_ht_cd \tab character \tab maximum Gage height code \cr
121
122
123
124
125
126
127
128
#' }
#' 
#' There are also several useful attributes attached to the data frame:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
#' url \tab character \tab The url used to generate the data \cr
#' queryTime \tab POSIXct \tab The time the data was returned \cr
#' comment \tab character \tab Header comments from the RDB file \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
129
#' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr
130
#' }
Laura A DeCicco's avatar
Laura A DeCicco committed
131
#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}}
132
133
#' @export
#' @examples
134
#' site_ids <- c('01594440','040851325')
135
#' \donttest{
136
137
#' data <- readNWISpeak(site_ids)
#' data2 <- readNWISpeak(site_ids, asDateTime=FALSE)
Laura A DeCicco's avatar
Laura A DeCicco committed
138
139
#' stations<-c("06011000")
#' peakdata<-readNWISpeak(stations,convertType=FALSE)
140
#' }
Laura A DeCicco's avatar
Laura A DeCicco committed
141
readNWISpeak <- function (siteNumbers,startDate="",endDate="", asDateTime=TRUE, convertType = TRUE){  
142
  
Laura A DeCicco's avatar
Laura A DeCicco committed
143
  # Doesn't seem to be a peak xml service
Laura A DeCicco's avatar
Laura A DeCicco committed
144
  url <- constructNWISURL(siteNumbers,NA,startDate,endDate,"peak")
145
  
Laura A DeCicco's avatar
Laura A DeCicco committed
146
  data <- importRDB1(url, asDateTime=asDateTime, convertType = convertType)
Laura A DeCicco's avatar
Laura A DeCicco committed
147
  
Laura A DeCicco's avatar
Laura A DeCicco committed
148
  if(nrow(data) > 0){
Laura A DeCicco's avatar
Laura A DeCicco committed
149
    if(asDateTime & convertType){
Laura A DeCicco's avatar
Laura A DeCicco committed
150
      
151
      if("peak_dt" %in% names(data)){
David Watkins's avatar
David Watkins committed
152
        if(any(nchar(as.character(data$peak_dt)) <= 7, na.rm = TRUE) | any(grepl("[0-9]*-[0-9]*-00",data$peak_dt), na.rm = TRUE)){
Laura A DeCicco's avatar
Laura A DeCicco committed
153
          stop("Not all dates could be converted to Date object. Use convertType=FALSE to retrieve the raw text")
154
155
        } else {
          data$peak_dt <- as.Date(data$peak_dt, format="%Y-%m-%d")
Laura A DeCicco's avatar
Laura A DeCicco committed
156
        }
Doug Friedman's avatar
Doug Friedman committed
157
        if(anyNA(data$peak_dt)){
David Watkins's avatar
David Watkins committed
158
159
          message("Some dates could not be converted to a valid date, and were returned as NA")
        }
Laura A DeCicco's avatar
Laura A DeCicco committed
160
      }
161
162
163
      
      badDates <- which(grepl("[0-9]*-[0-9]*-00",data$peak_dt))
 
Laura A DeCicco's avatar
Laura A DeCicco committed
164
      if("ag_dt" %in% names(data))  data$ag_dt <- as.Date(data$ag_dt, format="%Y-%m-%d")
165
    }
166

167
    
Laura A DeCicco's avatar
Laura A DeCicco committed
168
    siteInfo <- readNWISsite(siteNumbers)
Laura A DeCicco's avatar
Laura A DeCicco committed
169
170
171
172
    siteInfo <- merge(x = unique(data[,c("agency_cd","site_no")]), 
                      y = siteInfo,
                      by=c("agency_cd","site_no"), 
                      all.x = TRUE)
Laura A DeCicco's avatar
Laura A DeCicco committed
173
174
175
176
177
178
179
    
    attr(data, "siteInfo") <- siteInfo
    attr(data, "variableInfo") <- NULL
    attr(data, "statisticInfo") <- NULL
  }    
    return (data)
  
180
181
}

182
#' Rating table for an active USGS streamgage retrieval
183
#' 
Laura A DeCicco's avatar
Laura A DeCicco committed
184
#' Reads current rating table for an active USGS streamgage from NWISweb. 
Laura A DeCicco's avatar
Laura A DeCicco committed
185
#' Data is retrieved from \url{https://waterdata.usgs.gov/nwis}.
186
#' 
187
188
#' @param siteNumber character USGS site number.  This is usually an 8 digit number
#' @param type character can be "base", "corr", or "exsa"
Laura A DeCicco's avatar
Laura A DeCicco committed
189
190
#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates, datetimes,
#' numerics based on a standard algorithm. If false, everything is returned as a character
191
192
193
194
195
196
197
198
199
200
#' @return A data frame. If \code{type} is "base," then the columns are
#'INDEP, typically the gage height, in feet; DEP, typically the streamflow,
#'in cubic feet per second; and STOR, where "*" indicates that the pair are
#'a fixed point of the rating curve. If \code{type} is "exsa," then an
#'additional column, SHIFT, is included that indicates the current shift in
#'the rating for that value of INDEP. If \code{type} is "corr," then the
#'columns are INDEP, typically the gage height, in feet; CORR, the correction
#'for that value; and CORRINDEP, the corrected value for CORR.\cr
#'If \code{type} is "base," then the data frame has an attribute called "RATING"
#'that describes the rating curve is included.
201
202
203
204
205
206
207
208
209
210
211
#'
#' There are also several useful attributes attached to the data frame:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
#' url \tab character \tab The url used to generate the data \cr
#' queryTime \tab POSIXct \tab The time the data was returned \cr
#' comment \tab character \tab Header comments from the RDB file \cr
#' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr
#' RATING \tab character \tab Rating information \cr
#' }
#'
212
213
#' @note Not all active USGS streamgages have traditional rating curves that
#'relate flow to stage.
Laura A DeCicco's avatar
Laura A DeCicco committed
214
#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}}
215
216
#' @export
#' @examples
217
#' site_id <- '01594440'
218
#' \donttest{
219
#' data <- readNWISrating(site_id, "base")
220
#' attr(data, "RATING")
221
#' }
Laura A DeCicco's avatar
Laura A DeCicco committed
222
readNWISrating <- function (siteNumber,type="base",convertType = TRUE){  
223
224
  
  # No rating xml service 
225
  url <- constructNWISURL(siteNumber,service="rating",ratingType = type)
226
    
Laura A DeCicco's avatar
Laura A DeCicco committed
227
  data <- importRDB1(url, asDateTime=FALSE, convertType = convertType)
228
  
229
230
231
232
233
  if("current_rating_nu" %in% names(data)){
    intColumns <- intColumns[!("current_rating_nu" %in% names(data)[intColumns])]
    data$current_rating_nu <- gsub(" ", "", data$current_rating_nu)
  }
  
Laura A DeCicco's avatar
Laura A DeCicco committed
234
235
236
237
238
239
240
241
242
  if(nrow(data) > 0){
    if(type == "base") {
      Rat <- grep("//RATING ", comment(data), value=TRUE, fixed=TRUE)
      Rat <- sub("# //RATING ", "", Rat)
      Rat <- scan(text=Rat, sep=" ", what="")
      attr(data, "RATING") <- Rat
    }
    
    siteInfo <- readNWISsite(siteNumber)
243

Laura A DeCicco's avatar
Laura A DeCicco committed
244
245
246
    attr(data, "siteInfo") <- siteInfo
    attr(data, "variableInfo") <- NULL
    attr(data, "statisticInfo") <- NULL
247
248
  }
  
249
250
251
  return (data)
}

252
#'Surface-water measurement data retrieval from USGS (NWIS)
253
#'
Laura A DeCicco's avatar
Laura A DeCicco committed
254
255
#'Reads surface-water measurement data from NWISweb. Data is retrieved from \url{https://waterdata.usgs.gov/nwis}.
#'See \url{https://waterdata.usgs.gov/usa/nwis/sw} for details about surface water.
256
#'
257
#' @param siteNumbers character USGS site number (or multiple sites).  This is usually an 8 digit number
258
259
260
261
#' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the earliest possible record.
#' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the latest possible record.
262
263
#' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the 
#' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column.
264
#' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
265
266
267
#' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu",
#' "America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla". See also  \code{OlsonNames()} 
#' for more information on time zones.
268
#' @param expanded logical. Whether or not (TRUE or FALSE) to call the expanded data.
Laura A DeCicco's avatar
Laura A DeCicco committed
269
270
#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates, datetimes,
#' numerics based on a standard algorithm. If false, everything is returned as a character
271
272
273
274
275
#' @return A data frame with at least the following columns:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
#' agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr
#' site_no \tab character \tab The USGS site number \cr
276
#' measurement_dt \tab POSIXct \tab The date and time (in POSIXct) of the measurement. Unless specified
277
278
#' with the tz parameter, this is converted to UTC. If the measurement_dt column is an incomplete, a measurement_dt_date and
#' measurement_dt_time column are added to the returned data frame.   \cr
279
#' tz_cd \tab character \tab The time zone code for the measurement_dt column \cr
280
#' }
281
#'  
Laura A DeCicco's avatar
Laura A DeCicco committed
282
#' See \url{https://waterdata.usgs.gov/usa/nwis/sw} for details about surface water, and 
Laura A DeCicco's avatar
Laura A DeCicco committed
283
#' \url{https://waterdata.usgs.gov/nwis/help?output_formats_help}
Laura A DeCicco's avatar
Laura A DeCicco committed
284
#' for help on the columns and codes.
Laura A DeCicco's avatar
Laura A DeCicco committed
285
#' 
286
287
288
289
290
291
#' There are also several useful attributes attached to the data frame:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
#' url \tab character \tab The url used to generate the data \cr
#' queryTime \tab POSIXct \tab The time the data was returned \cr
#' comment \tab character \tab Header comments from the RDB file \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
292
#' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr
293
#' tz_cd_reported \tab The originally reported time zone \cr
294
#' }
Laura A DeCicco's avatar
Laura A DeCicco committed
295
#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}}
296
297
#' @export
#' @examples
298
#' site_ids <- c('01594440','040851325')
299
#' \donttest{
300
#' data <- readNWISmeas(site_ids)
Laura A DeCicco's avatar
Laura A DeCicco committed
301
#' Meas05316840 <- readNWISmeas("05316840")
302
#' Meas05316840.ex <- readNWISmeas("05316840",expanded=TRUE)
Laura A DeCicco's avatar
Laura A DeCicco committed
303
#' Meas07227500.ex <- readNWISmeas("07227500",expanded=TRUE)
Laura A DeCicco's avatar
Laura A DeCicco committed
304
#' Meas07227500.exRaw <- readNWISmeas("07227500",expanded=TRUE, convertType = FALSE)
305
#' }
306
readNWISmeas <- function (siteNumbers,startDate="",endDate="", tz="UTC", expanded=FALSE, convertType = TRUE){  
307
  
Laura A DeCicco's avatar
Laura A DeCicco committed
308
  # Doesn't seem to be a WaterML1 format option
309
  url <- constructNWISURL(siteNumbers,NA,startDate,endDate,"meas", expanded = expanded)
310
  
Laura A DeCicco's avatar
Laura A DeCicco committed
311
  data <- importRDB1(url,asDateTime=TRUE,tz=tz, convertType = convertType)
312
  
Laura A DeCicco's avatar
Laura A DeCicco committed
313
314
315
316
317
  if(nrow(data) > 0){
    if("diff_from_rating_pc" %in% names(data)){
      data$diff_from_rating_pc <- as.numeric(data$diff_from_rating_pc)
    }
    
Laura A DeCicco's avatar
Laura A DeCicco committed
318
319
320
    url <- attr(data, "url")
    comment <- attr(data, "comment")
    queryTime <- attr(data, "queryTime")
jsta's avatar
jsta committed
321
    header <- attr(data, "headerInfo")
Laura A DeCicco's avatar
Laura A DeCicco committed
322
    
Laura A DeCicco's avatar
Laura A DeCicco committed
323
324
    if(convertType){
      data$measurement_dateTime <- data$measurement_dt
325
      data$measurement_dt <- suppressWarnings(as.Date(data$measurement_dateTime))
Laura A DeCicco's avatar
Laura A DeCicco committed
326
327
328
329
330
331
332
333
334
335
      data$measurement_tm <- strftime(data$measurement_dateTime, "%H:%M")
      data$measurement_tm[is.na(data$tz_cd_reported)] <- ""
      indexDT <- which("measurement_dt" == names(data))
      indexTZ <- which("tz_cd" == names(data))
      indexTM <- which("measurement_tm" == names(data))
      indexTZrep <- which("tz_cd_reported" == names(data))
      newOrder <- c(1:indexDT,indexTM,indexTZrep,c((indexDT+1):ncol(data))[!(c((indexDT+1):ncol(data)) %in% c(indexTZrep,indexTM,indexTZ))],indexTZ)
  
      data <- data[,newOrder]      
    }
Laura A DeCicco's avatar
Laura A DeCicco committed
336
337


Laura A DeCicco's avatar
Laura A DeCicco committed
338
    siteInfo <- readNWISsite(siteNumbers)
Laura A DeCicco's avatar
Laura A DeCicco committed
339
340
341
342
    siteInfo <- merge(x = unique(data[,c("agency_cd","site_no")]), 
                      y = siteInfo,
                      by=c("agency_cd","site_no"), 
                      all.x = TRUE)
Laura A DeCicco's avatar
Laura A DeCicco committed
343
344
345
346
347
    attr(data, "url") <- url
    attr(data, "comment") <- comment
    attr(data, "queryTime") <- queryTime
    attr(data, "header") <- header
    
Laura A DeCicco's avatar
Laura A DeCicco committed
348
349
350
    attr(data, "siteInfo") <- siteInfo
    attr(data, "variableInfo") <- NULL
    attr(data, "statisticInfo") <- NULL    
351
352
  }
  
353
354
  return (data)
}
355

356
#' Groundwater level measurements retrieval from USGS (NWIS)
357
#'
Laura A DeCicco's avatar
Laura A DeCicco committed
358
#' Reads groundwater level measurements from NWISweb. Mixed date/times come back from the service 
Laura A DeCicco's avatar
Laura A DeCicco committed
359
#' depending on the year that the data was collected. See \url{https://waterdata.usgs.gov/usa/nwis/gw}
360
361
#' for details about groundwater. By default the returned dates are converted to date objects, unless convertType
#' is specified as FALSE. Sites with non-standard date formats (i.e. lacking a day) can be affected (see examples).
Laura A DeCicco's avatar
Laura A DeCicco committed
362
#' See \url{https://waterservices.usgs.gov/rest/GW-Levels-Service.html} for more information.
363
#' 
364
#' @param siteNumbers character USGS site number (or multiple sites).  This is usually an 8 digit number
365
366
367
368
#' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the earliest possible record.
#' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the latest possible record.
369
#' @param parameterCd character USGS parameter code.  This is usually an 5 digit number. Default is "".
Laura A DeCicco's avatar
Laura A DeCicco committed
370
371
#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates, datetimes,
#' numerics based on a standard algorithm. If false, everything is returned as a character
372
373
#' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the 
#' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column.
Laura A DeCicco's avatar
Laura A DeCicco committed
374
#' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
375
376
377
#' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu",
#' "America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla". See also  \code{OlsonNames()} 
#' for more information on time zones.
Laura A DeCicco's avatar
Laura A DeCicco committed
378
379
380
#' @return A data frame with the following columns:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
381
382
#' agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr
#' site_no \tab character \tab The USGS site number \cr
383
384
385
386
387
388
389
390
#' site_tp_cd \tab character \tab Site type code \cr 
#' lev_dt \tab Date \tab Date level measured\cr
#' lev_tm \tab character \tab Time level measured \cr
#' lev_tz_cd \tab character \tab Time datum \cr
#' lev_va \tab numeric \tab Water level value in feet below land surface\cr
#' sl_lev_va \tab numeric \tab Water level value in feet above specific vertical datum \cr
#' lev_status_cd \tab character \tab The status of the site at the time the water level was measured \cr
#' lev_agency_cd \tab character \tab The agency code of the person measuring the water level \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
391
392
393
#' }
#' 
#' There are also several useful attributes attached to the data frame:
Laura A DeCicco's avatar
Laura A DeCicco committed
394
#' \tabular{lll}{
Laura A DeCicco's avatar
Laura A DeCicco committed
395
396
397
#' Name \tab Type \tab Description \cr
#' url \tab character \tab The url used to generate the data \cr
#' queryTime \tab POSIXct \tab The time the data was returned \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
398
#' comment \tab character \tab Header comments from the RDB file \cr
David Watkins's avatar
typo    
David Watkins committed
399
#' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr
Laura A DeCicco's avatar
Laura A DeCicco committed
400
401
#' }
#' 
Laura A DeCicco's avatar
Laura A DeCicco committed
402
#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}}
403
404
#' @export
#' @examples
405
#' site_id <- "434400121275801"
406
#' \donttest{
407
#' data <- readNWISgwl(site_id)
Laura A DeCicco's avatar
Sites.    
Laura A DeCicco committed
408
#' sites <- c("434400121275801", "375907091432201")
409
#' data2 <- readNWISgwl(site_id, '','')
Laura A DeCicco's avatar
Laura A DeCicco committed
410
#' data3 <- readNWISgwl("420125073193001", '','')
411
412
#' #handling of data where date has no day
#' data4 <- readNWISgwl("425957088141001", startDate = "1980-01-01") 
413
414
415
#' 
#' data5 <- readNWISgwl("263819081585801", parameterCd = "72019")
#' 
416
#' }
417
418
419
readNWISgwl <- function(siteNumbers, startDate = "", endDate = "",
                        parameterCd = NA,
                        convertType = TRUE, tz = "UTC"){  
420
  
421
422
  url <- constructNWISURL(siteNumbers, parameterCd, startDate,endDate,"gwlevels",format="rdb")
  data <- importRDB1(url, asDateTime= TRUE, convertType = convertType, tz=tz)
Laura A DeCicco's avatar
Laura A DeCicco committed
423

Laura A DeCicco's avatar
Laura A DeCicco committed
424
  if(nrow(data) > 0 && !all(is.na(data$lev_dt))){
David Watkins's avatar
David Watkins committed
425
426
    if(convertType){
      #check that the date includes a day, based on date string length
427
      if(any(nchar(as.character(data$lev_dt)) <= 7) | any(grepl("[0-9]*-[0-9]*-00",data$lev_dt))){
Laura A DeCicco's avatar
Laura A DeCicco committed
428
        message("Not all dates were converted to Date object. Returning raw text for date columns.")
429
430
      } else {
        data$lev_dt <- as.Date(data$lev_dt)
David Watkins's avatar
David Watkins committed
431
      }
David Watkins's avatar
David Watkins committed
432
    }
Laura A DeCicco's avatar
Laura A DeCicco committed
433
    siteInfo <- readNWISsite(siteNumbers)
Laura A DeCicco's avatar
Laura A DeCicco committed
434
435
436
437
    siteInfo <- merge(x = unique(data[,c("agency_cd","site_no")]), 
                      y = siteInfo,
                      by=c("agency_cd","site_no"), 
                      all.x = TRUE)
Laura A DeCicco's avatar
Laura A DeCicco committed
438
439
440
    attr(data, "siteInfo") <- siteInfo
  }
    
441
442
443
  return (data)
}

444
445
446
#' Site statistics retrieval from USGS (NWIS) 
#' 
#' Retrieves site statistics from the USGS Statistics Web Service beta.  
Laura A DeCicco's avatar
Laura A DeCicco committed
447
#' See \url{https://waterservices.usgs.gov/rest/Statistics-Service.html} for more information.
448
#' 
449
#' @param siteNumbers character USGS site number (or multiple sites).  This is usually an 8 digit number.
450
#' @param parameterCd character USGS parameter code.  This is usually a 5 digit number.  
451
452
453
454
455
456
457
458
#' @param startDate character starting date for data retrieval in the form YYYY, YYYY-MM, or YYYY-MM-DD. Dates cannot 
#' be more specific than the statReportType, i.e. startDate for monthly statReportTypes cannot include days, and annual
#' statReportTypes cannot include days or months.  Months and days are optional for the daily statReportType. 
#' Default is "" which indicates retrieval for the earliest possible record.  For daily data, this indicates the 
#' start of the period the statistics will be computed over.
#' @param endDate character ending date for data retrieval in the form YYYY, YYYY-MM, or YYYY-MM-DD. Default is "" 
#' which indicates retrieval for the latest possible record.  For daily data, this indicates the end of the period 
#' the statistics will be computed over.  The same restrictions as startDate apply.  
459
460
461
462
463
#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to
#' numerics based on a standard algorithm. Years, months, and days (if appliccable) are also returned as numerics
#' in separate columns.  If convertType is false, everything is returned as a character.
#' @param statReportType character time division for statistics: daily, monthly, or annual.  Default is daily.
#' Note that daily provides statistics for each calendar day over the specified range of water years, i.e. no more than 366
464
#' data points will be returned for each site/parameter.  Use readNWISdata or readNWISdv for daily averages. 
David Watkins's avatar
David Watkins committed
465
466
#' Also note that 'annual' returns statistics for the calendar year.  Use readNWISdata for water years. Monthly and yearly 
#' provide statistics for each month and year within the range indivually.
467
#' @param statType character type(s) of statistics to output for daily values.  Default is mean, which is the only
468
#' option for monthly and yearly report types. See the statistics service documentation 
Laura A DeCicco's avatar
Laura A DeCicco committed
469
#' at \url{https://waterservices.usgs.gov/rest/Statistics-Service.html} for a full list of codes.  
470
471
472
#' @return A data frame with the following columns:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
473
474
475
#' agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr
#' site_no \tab character \tab The USGS site number \cr
#' parameter_cd \tab character \tab The USGS parameter code \cr
476
#' 
477
478
479
480
481
#' Other columns will be present depending on statReportType and statType
#' }
#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}}
#' @export
#' @examples
482
#' \donttest{
483
484
485
486
#' x1 <- readNWISstat(siteNumbers=c("02319394"),
#'                   parameterCd=c("00060"),
#'                   statReportType="annual") 
#' 
David Watkins's avatar
David Watkins committed
487
#' #all the annual mean discharge data for two sites
488
#' x2 <- readNWISstat(siteNumbers=c("02319394","02171500"),
489
490
#'                   parameterCd=c("00010","00060"),
#'                   statReportType="annual")
David Watkins's avatar
David Watkins committed
491
492
493
#' 
#' #Request p25, p75, and mean values for temperature and discharge for the 2000s
#' #Note that p25 and p75 were not available for temperature, and return NAs
494
495
496
497
498
#' x <- readNWISstat(siteNumbers=c("02171500"),
#'                   parameterCd=c("00010","00060"),
#'                   statReportType="daily",
#'                   statType=c("mean","median"),
#'                   startDate="2000",endDate="2010")
David Watkins's avatar
David Watkins committed
499
#' }
500
readNWISstat <- function(siteNumbers, parameterCd, startDate = "", endDate = "", convertType = TRUE, 
501
                          statReportType = "daily", statType = "mean"){
502

503
504
505
506
507
508
509
510
  #check for NAs in site numbers
  if(any(is.na(siteNumbers))){
    siteNumbers <- siteNumbers[!is.na(siteNumbers)]
    if(length(siteNumbers)==0){
      stop("siteNumbers was all NAs")
    }
    warning("NAs were passed in siteNumbers; they were ignored")
  }
511
  url <- constructNWISURL(siteNumbers,parameterCd,startDate,endDate,service = "stat",format = "rdb", 
512
                          statType = statType, statReportType = statReportType)
513
  data <- importRDB1(url,asDateTime=TRUE, convertType = convertType)
514
  
515
  siteInfo <- readNWISsite(siteNumbers)
Jerry Martin's avatar
Jerry Martin committed
516
517
  
  if(nrow(data) > 0){
Laura A DeCicco's avatar
Laura A DeCicco committed
518
519
520
521
    siteInfo <- merge(x = unique(data[,c("agency_cd","site_no")]), 
                      y = siteInfo,
                      by=c("agency_cd","site_no"), 
                      all.x = TRUE)
Jerry Martin's avatar
Jerry Martin committed
522
523
  }
  
524
525
  attr(data, "siteInfo") <- siteInfo
  
526
  return (data)
David Watkins's avatar
David Watkins committed
527
528
529
530
}

#' Water use data retrieval from USGS (NWIS)
#' 
Laura A DeCicco's avatar
Laura A DeCicco committed
531
#' Retrieves water use data from USGS Water Use Data for the Nation.  See \url{https://waterdata.usgs.gov/nwis/wu} for 
David Watkins's avatar
David Watkins committed
532
533
#' more information.  All available use categories for the supplied arguments are retrieved. 
#' 
534
#' @param stateCd could be character (full name, abbreviation, id), or numeric (id). Only one is accepted per query.  
David Watkins's avatar
David Watkins committed
535
536
537
538
#' @param countyCd could be character (name, with or without "County", or "ALL"), numeric (id), or code{NULL}, which will 
#' return state or national data depending on the stateCd argument.  \code{ALL} may also be supplied, which will return data 
#' for every county in a state. Can be a vector of counties in the same state.  
#' @param years integer Years for data retrieval. Must be years ending in 0 or 5. Default is all available years.
539
#' @param categories character categories of water use.  Defaults to \code{ALL}.  Specific categories must be supplied as two-
David Watkins's avatar
David Watkins committed
540
541
#' letter abbreviations as seen in the URL when using the NWIS water use web interface.  Note that 
#' there are different codes for national and state level data.  
David Watkins's avatar
David Watkins committed
542
543
544
#' @param convertType logical defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to
#' numerics based on a standard algorithm. Years, months, and days (if appliccable) are also returned as numerics
#' in separate columns.  If convertType is false, everything is returned as a character.
David Watkins's avatar
David Watkins committed
545
546
547
#' @param transform logical only intended for use with national data.  Defaults to \code{FALSE}, with data being returned as 
#' presented by the web service.  If \code{TRUE}, data will be transformed and returned with column names, which will reformat
#' national data to be similar to state data.  
David Watkins's avatar
David Watkins committed
548
549
550
#' @return A data frame with at least the year of record, and all available statistics for the given geographic parameters.
#' County and state fields will be included as appropriate.
#' 
David Watkins's avatar
David Watkins committed
551
552
#' @export
#' @examples 
553
#' \donttest{
David Watkins's avatar
David Watkins committed
554
#' #All data for a county
David Watkins's avatar
David Watkins committed
555
#' allegheny <- readNWISuse(stateCd = "Pennsylvania",countyCd = "Allegheny")
David Watkins's avatar
David Watkins committed
556
557
#' 
#' #Data for an entire state for certain years
David Watkins's avatar
David Watkins committed
558
#' ohio <- readNWISuse(years=c(2000,2005,2010),stateCd = "OH", countyCd = NULL)
David Watkins's avatar
David Watkins committed
559
560
#' 
#' #Data for an entire state, county by county
David Watkins's avatar
David Watkins committed
561
#' pr <- readNWISuse(years=c(2000,2005,2010),stateCd = "PR",countyCd="ALL")
David Watkins's avatar
David Watkins committed
562
#' 
David Watkins's avatar
David Watkins committed
563
564
#' #All national-scale data, transforming data frame to named columns from named rows
#' national <- readNWISuse(stateCd = NULL, countyCd = NULL, transform = TRUE)
David Watkins's avatar
David Watkins committed
565
566
#' 
#' #Washington, DC data
David Watkins's avatar
David Watkins committed
567
568
569
570
#' dc <- readNWISuse(stateCd = "DC",countyCd = NULL)
#' 
#' #data for multiple counties, with different input formatting
#' paData <- readNWISuse(stateCd = "42",countyCd = c("Allegheny County", "BUTLER", 1, "031"))
David Watkins's avatar
David Watkins committed
571
#' 
572
573
#' #retrieving two specific categories for an entire state
#' ks <- readNWISuse(stateCd = "KS", countyCd = NULL, categories = c("IT","LI"))
David Watkins's avatar
David Watkins committed
574
#' }
575
readNWISuse <- function(stateCd, countyCd, years = "ALL", categories = "ALL", convertType = TRUE, transform = FALSE){
David Watkins's avatar
David Watkins committed
576
 
David Watkins's avatar
David Watkins committed
577
  countyID <- NULL
Laura A DeCicco's avatar
Laura A DeCicco committed
578
579
  countyCd <- countyCd[countyCd != ""]
  
Laura A DeCicco's avatar
Laura A DeCicco committed
580
  if(exists("countyCd") && !is.null(countyCd) ){
Laura A DeCicco's avatar
Laura A DeCicco committed
581
582

    if(!any(toupper(countyCd) == "ALL")){
Laura A DeCicco's avatar
Laura A DeCicco committed
583
584
585
586
      for(c in countyCd){
        code <- countyCdLookup(state = stateCd, county = c, outputType = "id")
        countyID <- c(countyID,code)
      }
Laura A DeCicco's avatar
Laura A DeCicco committed
587
    } else {
Laura A DeCicco's avatar
Laura A DeCicco committed
588
      countyID <- toupper(countyID)
David Watkins's avatar
David Watkins committed
589
    }
David Watkins's avatar
David Watkins committed
590
  }
David Watkins's avatar
David Watkins committed
591
  
592
593
594
595
  years <- .capitalALL(years)
  categories <- .capitalALL(categories)
  
  url <- constructUseURL(years,stateCd,countyID,categories)
Laura A DeCicco's avatar
Laura A DeCicco committed
596
  returned_data <- importRDB1(url,convertType=convertType)  
David Watkins's avatar
David Watkins committed
597
  
David Watkins's avatar
David Watkins committed
598
  #for total country data arriving in named rows
599
  if(transform){
Laura A DeCicco's avatar
Laura A DeCicco committed
600
601
    cmmnt <- comment(returned_data)
    returned_data <- t(returned_data)
Laura A DeCicco's avatar
Laura A DeCicco committed
602
    colnames(returned_data) <- returned_data[1,]
Laura A DeCicco's avatar
Laura A DeCicco committed
603
604
605
606
    returned_data <- as.data.frame(returned_data[-1,],stringsAsFactors=FALSE)
    returned_data <- cbind(Year=as.integer(substr(rownames(returned_data),2,5)),returned_data)
    rownames(returned_data) <- NULL
    comment(returned_data) <- cmmnt
Laura A DeCicco's avatar
Laura A DeCicco committed
607
608
609
    if(!all(is.null(stateCd)) && all(nchar(stateCd) != 0)){
      warning("transform = TRUE is only intended for national data")
      }
David Watkins's avatar
David Watkins committed
610
  }
Laura A DeCicco's avatar
Laura A DeCicco committed
611
  return(returned_data)
David Watkins's avatar
David Watkins committed
612
}
613
614
615
616
617
618
619

.capitalALL <- function(input){
  if(any(grepl("(?i)all",input))){
    input <- toupper(input)
  }
  return(input)
}