Commit 9b02ff35 authored by Laura A DeCicco's avatar Laura A DeCicco
Browse files

Merge pull request #117 from ldecicco-USGS/master

More tz and state code updates.
parents 9ea4c666 d09cefdb
......@@ -25,6 +25,7 @@ export(readWQPdata)
export(readWQPqw)
export(renameNWISColumns)
export(stateCd)
export(stateCdLookup)
export(whatNWISdata)
export(whatNWISsites)
export(whatWQPsites)
......
......@@ -117,15 +117,22 @@ importWQP <- function(obs_url, zip=FALSE, tz=""){
retval$ActivityStartDateTime <- with(retval, as.POSIXct(paste(ActivityStartDate, ActivityStartTime.Time),format="%Y-%m-%d %H:%M:%S", tz = "UTC"))
retval$ActivityStartDateTime <- retval$ActivityStartDateTime + timeZoneStart*60*60
retval$ActivityStartDateTime <- as.POSIXct(retval$ActivityStartDateTime)
attr(retval$ActivityStartDateTime, "tzone") <- "UTC"
if(tz != ""){
attr(retval$ActivityStartDateTime, "tzone") <- tz
} else {
attr(retval$ActivityStartDateTime, "tzone") <- "UTC"
}
}
if(any(!is.na(timeZoneEnd))){
retval$ActivityEndDateTime <- with(retval, as.POSIXct(paste(ActivityEndDate, ActivityEndTime.Time),format="%Y-%m-%d %H:%M:%S", tz = "UTC"))
retval$ActivityEndDateTime <- retval$ActivityEndDateTime + timeZoneEnd*60*60
retval$ActivityEndDateTime <- as.POSIXct(retval$ActivityEndDateTime)
attr(retval$ActivityEndDateTime, "tzone") <- "UTC"
if(tz != ""){
attr(retval$ActivityEndDateTime, "tzone") <- tz
} else {
attr(retval$ActivityEndDateTime, "tzone") <- "UTC"
}
}
if(all(is.na(retval$ActivityEndDateTime))){
......
......@@ -101,6 +101,15 @@ readNWISdata <- function(service="dv", ...){
format <- "waterml,1.1"
baseURL <- "http://waterservices.usgs.gov/nwis/"
if("stateCd" %in% names(values)){
values["stateCd"] <- stateCdLookup(values["stateCd"], "postal")
}
if("statecode" %in% names(values)){
values["statecode"] <- stateCdLookup(values["statecode"], "postal")
names(values)[names(values) == "statecode"] <- "stateCd"
}
if(service == "iv"){
baseURL <- "http://nwis.waterservices.usgs.gov/nwis/"
} else if (service == "qwdata"){
......@@ -134,8 +143,9 @@ readNWISdata <- function(service="dv", ...){
"America/Denver","America/Los_Angeles",
"America/Anchorage","America/Honolulu",
"America/Jamaica","America/Managua",
"America/Phoenix","America/Metlakatla")
"America/Phoenix","America/Metlakatla","UTC")
tz <- match.arg(tz, rTZ)
if("UTC" == tz) tz <- ""
}
values <- values[!(names(values) %in% "tz")]
} else {
......@@ -199,7 +209,42 @@ readNWISdata <- function(service="dv", ...){
})
}
return(retval)
}
#' State code look up
#'
#' Function to simplify finding state and state code definitions. Used in \code{readNWISdata}
#' and \code{readWQPdata}.
#'
#' @param input could be character (full name, abbreviation, id), or numeric (id)
#' @param outputType character can be "postal","fullName","tableIndex", or "id".
#' @export
#' @examples
#' fullName <- stateCdLookup("wi", "fullName")
#' abbriev <- stateCdLookup("Wisconsin", "postal")
#' id <- stateCdLookup("WI", "id")
#' name <- stateCdLookup(55, "fullName")
#' index <- stateCdLookup("WI", "tableIndex")
#' stateCd[index,]
stateCdLookup <- function(input, outputType="postal"){
outputType <- match.arg(outputType, c("postal","fullName","tableIndex","id"))
if(is.numeric(input) | !is.na(suppressWarnings(as.numeric(input)))){
input <- which(input == as.numeric(stateCd$STATE))
} else if(nchar(input) == 2){
input <- which(tolower(input) == tolower(stateCd$STUSAB))
} else {
input <- which(tolower(input) == tolower(stateCd$STATE_NAME))
}
retVal <- switch(outputType,
postal = stateCd$STUSAB[input],
fullName = stateCd$STATE_NAME[input],
tableIndex = input,
id = stateCd$STATE[input]
)
return(retval)
return(retVal)
}
......@@ -102,10 +102,6 @@ readWQPdata <- function(...){
values <- sapply(matchReturn, function(x) URLencode(as.character(paste(eval(x),collapse=";",sep=""))))
values <- gsub(",","%2C",values)
values <- gsub("%20","+",values)
values <- gsub(":","%3A",values)
if("bBox" %in% names(values)){
values['bBox'] <- gsub(pattern = ";", replacement = ",", x = values['bBox'])
}
......@@ -115,6 +111,41 @@ readWQPdata <- function(...){
names(values)[names(values) == "siteNumber"] <- "siteid"
names(values)[names(values) == "siteNumbers"] <- "siteid"
if("statecode" %in% names(values)){
stCd <- values["statecode"]
if(!grepl("US:",stCd)){
values["statecode"] <- paste0("US:",stateCdLookup(stCd, "id"))
}
}
if("stateCd" %in% names(values)){
stCd <- values["stateCd"]
if(!grepl("US:",stCd)){
values["stateCd"] <- paste0("US:",stateCdLookup(stCd, "id"))
}
names(values)[names(values) == "stateCd"] <- "statecode"
}
if("tz" %in% names(values)){
tz <- values["tz"]
if(tz != ""){
rTZ <- c("America/New_York","America/Chicago",
"America/Denver","America/Los_Angeles",
"America/Anchorage","America/Honolulu",
"America/Jamaica","America/Managua",
"America/Phoenix","America/Metlakatla","UTC")
tz <- match.arg(tz, rTZ)
if("UTC" == tz) tz <- ""
}
values <- values[!(names(values) %in% "tz")]
} else {
tz <- ""
}
values <- gsub(",","%2C",values)
values <- gsub("%20","+",values)
values <- gsub(":","%3A",values)
urlCall <- paste(paste(names(values),values,sep="="),collapse="&")
......@@ -123,7 +154,7 @@ readWQPdata <- function(...){
urlCall,
"&mimeType=tsv")
retval <- importWQP(urlCall,FALSE)
retval <- importWQP(urlCall,FALSE, tz=tz)
if(!all(is.na(retval))){
siteInfo <- whatWQPsites(...)
......
......@@ -14,6 +14,10 @@
#' retrieval for the earliest possible record. Date arguments are always specified in local time.
#' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
#' retrieval for the latest possible record. Date arguments are always specified in local time.
#' @param tz character to set timezone attribute of datetime. Default is an empty quote, which converts the
#' datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column).
#' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
#' "America/Anchorage","America/Honolulu","America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla"
#' @keywords data import USGS web service
#' @return A data frame with at least the following columns:
#' \tabular{lll}{
......@@ -104,10 +108,10 @@
#' rawPHsites <- readWQPqw(c('USGS-05406450', 'USGS-05427949','WIDNR_WQX-133040'), 'pH','','')
#' nwisEx <- readWQPqw('USGS-04024000',c('34247','30234','32104','34220'),'','2012-12-20')
#' }
readWQPqw <- function(siteNumbers,parameterCd,startDate="",endDate=""){
readWQPqw <- function(siteNumbers,parameterCd,startDate="",endDate="",tz=""){
url <- constructWQPURL(siteNumbers,parameterCd,startDate,endDate)
retval <- importWQP(url)
retval <- importWQP(url, tz = tz)
pcodeCheck <- all(nchar(parameterCd) == 5) & all(!is.na(suppressWarnings(as.numeric(parameterCd))))
......
......@@ -61,6 +61,10 @@ whatWQPsites <- function(...){
values <- sapply(matchReturn, function(x) URLencode(as.character(paste(eval(x),collapse=";",sep=""))))
if("tz" %in% names(values)){
values <- values[!(names(values) %in% "tz")]
}
values <- gsub(",","%2C",values)
values <- gsub("%20","+",values)
values <- gsub(":","%3A",values)
......
`dataRetrieval`
=============
Linux: [![travis](https://travis-ci.org/USGS-R/dataRetrieval.svg?branch=master)](https://travis-ci.org/USGS-R/dataRetrieval)
Windows: [![Build status](https://ci.appveyor.com/api/projects/status/msanha92b500grr7?svg=true)](https://ci.appveyor.com/project/ldecicco-USGS/dataretrieval-787)
Retrieval functions for USGS and EPA hydrologic and water quality data.
A fairly large overhaul of the functions was done on November 31, 2014. Feedback from initial users was incorporated. For questions, bug reports, and suggestions please create an Issue here:
Linux: [![travis](https://travis-ci.org/USGS-R/dataRetrieval.svg?branch=master)](https://travis-ci.org/USGS-R/dataRetrieval)
[dataRetrieval Issues page](https://github.com/USGS-R/dataRetrieval/issues)
Windows: [![Build status](https://ci.appveyor.com/api/projects/status/msanha92b500grr7?svg=true)](https://ci.appveyor.com/project/ldecicco-USGS/dataretrieval-787)
##Package Installation
To install the dataRetrieval package, you must be using R 3.0 or greater and run the following command:
```R
install.packages("dataRetrieval")
```
## Function Overview
......@@ -20,21 +20,24 @@ Web service retrieval functions:
| -------------| ------------------------|:-------------|
|`readNWISdata` | `...`, `service` | NWIS data using user-specified queries |
|`readNWISdv` | `Common 3`, `parameterCd`, `statCd` | NWIS daily data with `Common` query |
|`readNWISqw` | `Common 3`, `parameterCd`, `expanded` | NWIS water quality data with `Common` query |
|`readNWISuv` | `Common 3`, `parameterCd` | NWIS instantaneous data with `Common` query |
|`readNWISqw` | `Common 3`, `parameterCd`, `expanded`,`reshape`,`tz` | NWIS water quality data with `Common` query |
|`readNWISuv` | `Common 3`, `parameterCd`,`tz` | NWIS instantaneous data with `Common` query |
|`readNWISpCode` | `parameterCd` | NWIS parameter code information |
|`readNWISgwl` | `Common 3` | NWIS groundwater level data with `Common` query |
|`readNWISpeak` | `Common 3` | NWIS peak flow data with `Common` query |
|`readNWISmeas` | `Common 3` | NWIS surface-water measurement data with `Common` query |
|`readNWISpeak` | `Common 3`,`asDateTime` | NWIS peak flow data with `Common` query |
|`readNWISmeas` | `Common 3`, `tz` | NWIS surface-water measurement data with `Common` query |
|`readNWISrating` | `siteNumber`, `type` | NWIS rating table for an active USGS streamgage |
|`readNWISsite` | `siteNumber` | NWIS site information |
|`whatNWISsites` | `...` | NWIS site search using user-specified queries |
|`whatNWISdata` | `siteNumber`, `service` | NWIS data availability, including period of record and count |
|`readWQPdata` | `...` | WQP data using user-specified queries |
|`readWQPqw` | `Common 3` | WQP data with `Common 3` query and either parameter code or characteristic name|
|`readWQPqw` | `Common 3`, `parameterCd`**,`tz` | WQP data with `Common 3` query and either parameter code or characteristic name|
|`whatWQPsites` | `...` | WQP site search using user-specified queries |
* `Common 3` = siteNumber, startDate, endDate
* `...` = user-defined arguments.
**`parameterCd` in WQP function can be either USGS parameter codes, or EPA characteristic names
##Reporting bugs
......@@ -49,69 +52,13 @@ egret_comments@usgs.gov
Additionally, to subscribe to an email list concerning updates to these R packages, please send a request to egret_comments@usgs.gov.
##Package Installation
To install the dataRetrieval package, you must be using R 3.0 or greater and run the following command:
```R
install.packages("dataRetrieval")
```
##Version updates
##Latest Version Updates
---------------
###dataRetrieval 2.1.0
* Improved error handling
* Moved parameterCdFile and pcodeToName data to sysdata.rda to assure functions using that data are available in other packages.
* Fixed WaterML2 qualifier bug.
###dataRetrieval 2.0.1
* Improved help files and vignette documentation
* Added siteInfo and variableInfo attributes to returned data frames
* Removed any obsolete functions
###dataRetrieval 2.0.0
* Changing naming convention. Migrated `EGRET` specific retrievals to `EGRET`:
Moving `EGRET` specific functions to `EGRET` (version 2.0.0 and greater):
|Information Source | Meta Data | Data |
| -------------| -------------| ------------- |:-------------|
|NWIS | `readNWISInfo` | `readNWISSample` |
| | | `readNWISDaily` |
| Water Quality Portal | `readWQPInfo`| `readWQPSample` |
| User-supplied files | `readUserInfo` | `readUserDaily`|
| | | `readUserSample` |
* Added back WaterML2 parsing tool
* Added specific groundwater, rating, peak, and surfacewater measurement functions
* Attached metadata attributes to returned dataframes
###dataRetrieval 1.4.0
* Changed naming convention:
###dataRetrieval 1.3.3
* Updated getNWISSiteInfo to retrieve multiple site file datasets at once using a vector of siteNumbers as input argument.
* Updated error-handling for Web service calls. More information is returned when errors happen
* Added some basic processing to Water Quality Portal raw data retrievals. Date columns are returned as Date objects, value columns are numeric, and a column is created from the date/time/timezone columns that is POSIXct.
* Added very generalized NWIS and WQP retrieval functions (getNWISData, getNWISSites, getGeneralWQPData, and whatWQPsites) which allow the user to use any argument available on the Web service platform.
###dataRetrieval 1.3.2
* Deprecated getQWData, updated readWQPdata to take either parameter code or characteristic name.
* Changed the name of raw data retrievals to: readNWISqw, getNWISunitData, getNWISdvData, and getWQPqwData (from: readNWISqw, retrieveUnitNWISData, retrieveNWISData, getRawQWData)
* Added NA warning to getDVData function
* Updated mergeReport to allow for Sample data with different measurements taken on the same day
###dataRetrieval 2.2.0
* Changed the output of tz_cd (in all *NWIS functions) to the timezone that is reported in the data frame. Before this version, it was the time zone reported by NWIS. This was confusing however since the dateTime columns default to UTC.
* Minor bug fixes
* Help file updates
* Improved query options in readNWISdata and readWQPdata functions.
##Disclaimer
......
......@@ -4,7 +4,7 @@
\alias{readWQPqw}
\title{Raw Data Import for Water Quality Portal}
\usage{
readWQPqw(siteNumbers, parameterCd, startDate = "", endDate = "")
readWQPqw(siteNumbers, parameterCd, startDate = "", endDate = "", tz = "")
}
\arguments{
\item{siteNumbers}{character site number. This needs to include the full agency code prefix.}
......@@ -17,6 +17,11 @@ retrieval for the earliest possible record. Date arguments are always specified
\item{endDate}{character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
retrieval for the latest possible record. Date arguments are always specified in local time.}
\item{tz}{character to set timezone attribute of datetime. Default is an empty quote, which converts the
datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column).
Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
"America/Anchorage","America/Honolulu","America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla"}
}
\value{
A data frame with at least the following columns:
......
% Generated by roxygen2 (4.1.0): do not edit by hand
% Please edit documentation in R/readNWISdata.r
\name{stateCdLookup}
\alias{stateCdLookup}
\title{State code look up}
\usage{
stateCdLookup(input, outputType = "postal")
}
\arguments{
\item{input}{could be character (full name, abbreviation, id), or numeric (id)}
\item{outputType}{character can be "postal","fullName","tableIndex", or "id".}
}
\description{
Function to simplify finding state and state code definitions. Used in \code{readNWISdata}
and \code{readWQPdata}.
}
\examples{
fullName <- stateCdLookup("wi", "fullName")
abbriev <- stateCdLookup("Wisconsin", "postal")
id <- stateCdLookup("WI", "id")
name <- stateCdLookup(55, "fullName")
index <- stateCdLookup("WI", "tableIndex")
stateCd[index,]
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment