diff --git a/DESCRIPTION b/DESCRIPTION index 837afddca5026e69ffcfd1c171313ca5914f1bc2..87d236f174c88f1ab09f1c6e8e7f56aed8822904 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,6 @@ Package: dataRetrieval Type: Package Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data Version: 2.1.2 -Date: 2015-02-01 Authors@R: c( person("Robert", "Hirsch", role = c("aut"), email = "rhirsch@usgs.gov"), person("Laura", "DeCicco", role = c("aut","cre"), @@ -12,10 +11,8 @@ Authors@R: c( person("Robert", "Hirsch", role = c("aut"), Description: Collection of functions to help retrieve U.S. Geological Survey (USGS) and U.S. Environmental Protection Agency (EPA) water quality and hydrology data from web services. USGS web services are discovered from the - National Water Information Services (NWIS) tools found here: - http://waterservices.usgs.gov/ Both EPA and USGS water quality data are - obtained from the Water Quality Portal found here: - http://www.waterqualitydata.us/portal.jsp + National Water Information System (NWIS) tools. Both EPA and USGS water + quality data are obtained from the Water Quality Portal. License: CC0 Copyright: This software is in the public domain because it contains materials that originally came from the United States Geological Survey, an agency of diff --git a/NAMESPACE b/NAMESPACE index 9d2fec6ac6c646d667c54ef1485b5ff89316ba8a..c629998dd9d03b1841d91c9d0b2cb66c4fe91a84 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -8,6 +8,8 @@ export(importRDB1) export(importWQP) export(importWaterML1) export(importWaterML2) +export(pCodeToName) +export(parameterCdFile) export(readNWISdata) export(readNWISdv) export(readNWISgwl) diff --git a/NEWS b/NEWS index f7124ef34f07919a36aa65000f3094735a86aefe..d1d99297e16b9c84a8a1ddfa2bda0de7bc3b280d 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ dataRetrieval 2.1.1 * Removed tests that call external web services, server upgrades have been causing false negative results * Added functionality to readWQPdata to allow R objects as arguments rather than just character strings * Added parameterCd='all' option for qw calls. +* Exposed sysdata constants dataRetrieval 2.0.2 diff --git a/R/importWQP.R b/R/importWQP.R index 14f5b5ebac6df26fceafc2c638c7ef7c958d227d..93766b48788ce800c7d2f8f9366f51441aae40be 100644 --- a/R/importWQP.R +++ b/R/importWQP.R @@ -30,6 +30,14 @@ #' } importWQP <- function(obs_url, zip=FALSE, tz=""){ + if(tz != ""){ + tz <- match.arg(tz, c("America/New_York","America/Chicago", + "America/Denver","America/Los_Angeles", + "America/Anchorage","America/Honolulu", + "America/Jamaica","America/Managua", + "America/Phoenix","America/Metlakatla")) + } + if(zip){ h <- basicHeaderGatherer() httpHEAD(obs_url, headerfunction = h$update) @@ -49,7 +57,6 @@ importWQP <- function(obs_url, zip=FALSE, tz=""){ if(headerInfo['status'] == "200"){ doc <- unzip(temp) - unlink(temp) } else { unlink(temp) @@ -61,14 +68,6 @@ importWQP <- function(obs_url, zip=FALSE, tz=""){ headerInfo <- attr(doc, "headerInfo") } - if(tz != ""){ - tz <- match.arg(tz, c("America/New_York","America/Chicago", - "America/Denver","America/Los_Angeles", - "America/Anchorage","America/Honolulu", - "America/Jamaica","America/Managua", - "America/Phoenix","America/Metlakatla")) - } - numToBeReturned <- as.numeric(headerInfo["Total-Result-Count"]) if (!is.na(numToBeReturned) & numToBeReturned != 0){ @@ -132,7 +131,10 @@ importWQP <- function(obs_url, zip=FALSE, tz=""){ if(all(is.na(retval$ActivityEndDateTime))){ retval$ActivityEndDateTime <- NULL } - + if(zip){ + unlink(doc) + } + return(retval) } else { diff --git a/R/readNWISdata.r b/R/readNWISdata.r index 8bcd4a151106c1c0b76d0267f9202a8a65963583..c66e7f7f53375a93c5c36fd3f55ef5599fe6f0b2 100644 --- a/R/readNWISdata.r +++ b/R/readNWISdata.r @@ -4,7 +4,7 @@ #' Arguments to the function should be based on \url{http://waterservices.usgs.gov} service calls. #' #' @param service string. Possible values are "iv" (for instantaneous), "dv" (for daily values), "gwlevels" -#' (for groundwater levels) +#' (for groundwater levels), and "site" (for site service) #' @param \dots see \url{http://waterservices.usgs.gov/rest/Site-Service.html#Service} for a complete list of options #' @keywords data import NWIS web service #' @return A data frame with the following columns: @@ -45,12 +45,22 @@ #' multiSite <- readNWISdata(sites=c("04025000","04072150"), service="iv", parameterCd="00010") #' #Not empty: #' multiSite <- readNWISdata(sites=c("04025500","040263491"), service="iv", parameterCd="00060") +#' bBoxEx <- readNWISdata(bBox=c(-83,36.5,-81,38.5), parameterCd="00010") +#' startDate <- as.Date("2013-10-01") +#' endDate <- as.Date("2014-09-30") +#' waterYear <- readNWISdata(bBox=c(-83,36.5,-81,38.5), parameterCd="00010", +#' service="dv", startDate=startDate, endDate=endDate) +#' siteInfo <- readNWISdata(stateCd="WI", parameterCd="00010",hasDataTypeCd="iv", service="site") #' } readNWISdata <- function(service="dv", ...){ matchReturn <- list(...) - match.arg(service, c("dv","iv","gwlevels")) + match.arg(service, c("dv","iv","gwlevels","site", "uv")) + + if(service == "uv"){ + service <- "iv" + } if(length(service) > 1){ stop("Only one service call allowed.") @@ -60,22 +70,35 @@ readNWISdata <- function(service="dv", ...){ names(values)[names(values) == "startDate"] <- "startDT" names(values)[names(values) == "endDate"] <- "endDT" - + names(values)[names(values) == "siteNumber"] <- "sites" + names(values)[names(values) == "siteNumbers"] <- "sites" + urlCall <- paste(paste(names(values),values,sep="="),collapse="&") format <- "waterml,1.1" + baseURL <- "http://waterservices.usgs.gov/nwis/" if(service == "iv"){ baseURL <- "http://nwis.waterservices.usgs.gov/nwis/" - } else { - baseURL <- "http://waterservices.usgs.gov/nwis/" + } + + if(service == "site"){ + format <- "rdb" } baseURL <- paste0(baseURL,service,"/?format=",format,"&") urlCall <- paste0(baseURL,urlCall) - retval <- importWaterML1(urlCall, asDateTime = ("iv" == service)) - + if(service == "site"){ + retval <- importRDB1(urlCall, asDateTime = FALSE, qw = FALSE) + } else { + retval <- importWaterML1(urlCall, asDateTime = ("iv" == service)) + if("dv" == service){ + retval$dateTime <- as.Date(retval$dateTime) + retval$tz_cd <- NULL + names(retval)[names(retval) == "dateTime"] <- "Date" + } + } return(retval) } diff --git a/R/readNWISdv.r b/R/readNWISdv.r index 28f0ae7d221b00489f83a0725c54f5c12c323e4a..75dac0d02dc562c35a669ae8cafb5bc7c4bd6a99 100644 --- a/R/readNWISdv.r +++ b/R/readNWISdv.r @@ -1,8 +1,7 @@ #' Raw Data Import for USGS NWIS Data #' #' Imports data from NWIS web service. This function gets the data from here: \url{http://waterservices.usgs.gov/} -#' A list of parameter codes can be found here: \url{http://help.waterdata.usgs.gov/codes-and-parameters/parameters} -#' A list of statistic codes can be found here: \url{http://help.waterdata.usgs.gov/code/stat_code_query?fmt=html} +#' Information on parameter and statistics codes can be found here: \url{http://help.waterdata.usgs.gov} #' #' @param siteNumber character USGS site number. This is usually an 8 digit number. Multiple sites can be requested with a character vector. #' @param parameterCd character of USGS parameter code(s). This is usually an 5 digit number. diff --git a/R/readNWISpCode.r b/R/readNWISpCode.r index 9be32b5c5e29b331c3a6c5bffc66d19caa46f06c..fcf012601bde710666f023e94d36a0f5cf4806fc 100644 --- a/R/readNWISpCode.r +++ b/R/readNWISpCode.r @@ -3,8 +3,8 @@ #' Imports data from NWIS about meaured parameter based on user-supplied parameter code or codes. #' This function gets the data from here: \url{http://nwis.waterdata.usgs.gov/nwis/pmcodes} #' -#' @param parameterCd character of USGS parameter codes (or multiple parameter codes). These are 5 digit number codes -#' that can be found here: \url{http://help.waterdata.usgs.gov/codes-and-parameters/parameters}. To get a +#' @param parameterCd character of USGS parameter codes (or multiple parameter codes). These are 5 digit number codes, +#' more information can be found here: \url{http://help.waterdata.usgs.gov/}. To get a #' complete list of all current parameter codes in the USGS, use "all" as the input. #' @keywords data import USGS web service #' @return parameterData data frame with the following information: diff --git a/R/readNWISqw.r b/R/readNWISqw.r index ed5301debfe53c800f7b0a533ab87f730b3c397f..540658d60520caa223fcf564f19957694a92ddfa 100644 --- a/R/readNWISqw.r +++ b/R/readNWISqw.r @@ -4,8 +4,30 @@ #' A list of parameter codes can be found here: \url{http://nwis.waterdata.usgs.gov/nwis/pmcodes/} #' A list of statistic codes can be found here: \url{http://nwis.waterdata.usgs.gov/nwis/help/?read_file=stat&format=table} #' +#' @details Valid parameter code groups are "All," or group codes: +#'\tabular{ll}{ +#'Code \tab Description\cr +#'INF \tab Information \cr +#'PHY \tab Physical \cr +#'INM \tab Inorganics, Major, Metals (major cations) \cr +#'INN \tab Inorganics, Major, Non-metals (major anions) \cr +#'NUT \tab Nutrient \cr +#'MBI \tab Microbiological \cr +#'BIO \tab Biological \cr +#'IMN \tab Inorganics, Minor, Non-metals \cr +#'IMM \tab Inorganics, Minor, Metals \cr +#'TOX \tab Toxicity \cr +#'OPE \tab Organics, pesticide \cr +#'OPC \tab Organics, PCBs \cr +#'OOT \tab Organics, other \cr +#'RAD \tab Radiochemical \cr +#'SED \tab Sediment \cr +#'POP \tab Population/community \cr +#'} +#' #' @param siteNumbers character of USGS site numbers. This is usually an 8 digit number -#' @param parameterCd character of USGS parameter code(s). This is usually an 5 digit number. Can also be "all". +#' @param parameterCd character that contains the code for a parameter +#' group, or a character vector of 5-digit parameter codes. See \bold{Details}. #' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates #' retrieval for the earliest possible record. #' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates @@ -58,14 +80,28 @@ #' startDate,endDate,reshape=TRUE) #' parameterCd <- "all" #' rawNWISall <- readNWISqw(siteNumbers,parameterCd, -#' startDate,endDate,reshape=TRUE) +#' startDate,endDate) +#' pgroup <- c("NUT") +#' rawNWISNutrients <- readNWISqw(siteNumbers,pgroup, +#' startDate,endDate) +#' groups <- c("NUT","OPE") +#' rawNWISNutOpe <- readNWISqw(siteNumbers,groups, +#' startDate,endDate) +#' rawNWISOpe <- readNWISqw(siteNumbers,"OPE", +#' startDate,endDate) #' } readNWISqw <- function (siteNumbers,parameterCd,startDate="",endDate="", expanded=TRUE,reshape=FALSE,tz=""){ - if(any(parameterCd == "all")){ - siteNumbers <- paste(siteNumbers, collapse=",") + pgrp <- c("INF", "PHY", "INM", "INN", "NUT", "MBI", "BIO", "IMM", "IMN", "TOX", + "OPE", "OPC", "OOT", "RAD", "XXX", "SED", "POP") + + + + + if(any(parameterCd == "all") | any(parameterCd == "All") ){ + siteNumbers <- paste(siteNumbers, collapse=",") url <- paste0("http://nwis.waterdata.usgs.gov/nwis/qwdata?multiple_site_no=", siteNumbers, "&sort_key=site_no&group_key=NONE&inventory_output=0", "&begin_date=", startDate, "&end_date=", endDate, @@ -73,6 +109,16 @@ readNWISqw <- function (siteNumbers,parameterCd,startDate="",endDate="", "&radio_parm_cds=all_parm_cds&qw_attributes=0&format=rdb", "&qw_sample_wide=0&rdb_qw_attributes=expanded&date_format=YYYY-MM-DD", "&rdb_compression=value&list_of_search_criteria=multiple_site_no") + } else if (all(parameterCd %in% pgrp)){ + siteNumbers <- paste(siteNumbers, collapse=",") + groups <- paste(parameterCd, collapse=",") + url <- paste0("http://nwis.waterdata.usgs.gov/nwis/qwdata?multiple_site_no=", siteNumbers, + "&sort_key=site_no&group_key=NONE&inventory_output=0", + "&begin_date=", startDate, "&end_date=", endDate, + "&TZoutput=0¶m_group=", groups, + "&qw_attributes=0&format=rdb", + "&qw_sample_wide=0&rdb_qw_attributes=expanded&date_format=YYYY-MM-DD", + "&rdb_compression=value&list_of_search_criteria=multiple_site_no") } else { url <- constructNWISURL(siteNumbers, @@ -80,36 +126,40 @@ readNWISqw <- function (siteNumbers,parameterCd,startDate="",endDate="", startDate, endDate,"qw",expanded=expanded) } + + data <- importRDB1(url,asDateTime=TRUE, qw=TRUE, tz = tz) originalHeader <- comment(data) + parameterCd <- unique(data$parm_cd) - if(reshape & expanded){ - columnsToMelt <- c("agency_cd","site_no","sample_dt","sample_tm", - "sample_end_dt","sample_end_tm","sample_start_time_datum_cd","tm_datum_rlbty_cd", - "parm_cd","startDateTime","endDateTime","coll_ent_cd", "medium_cd","project_cd", - "aqfr_cd","tu_id","body_part_id", "hyd_cond_cd", "samp_type_cd", - "hyd_event_cd","sample_lab_cm_tx") - columnsToMelt <- columnsToMelt[columnsToMelt %in% names(data)] - dataWithPcodes <- data[data$parm_cd != "",] - if(sum(data$parm_cd != "") > 0){ - warning("Data returned without pCodes, will not be included in reshape") + if(reshape){ + if(expanded){ + columnsToMelt <- c("agency_cd","site_no","sample_dt","sample_tm", + "sample_end_dt","sample_end_tm","sample_start_time_datum_cd","tm_datum_rlbty_cd", + "parm_cd","startDateTime","endDateTime","coll_ent_cd", "medium_cd","project_cd", + "aqfr_cd","tu_id","body_part_id", "hyd_cond_cd", "samp_type_cd", + "hyd_event_cd","sample_lab_cm_tx") + columnsToMelt <- columnsToMelt[columnsToMelt %in% names(data)] + dataWithPcodes <- data[data$parm_cd != "",] + if(sum(data$parm_cd == "") > 0){ + warning("Some or all data returned without pCodes, those data will not be included in reshape") + } + longDF <- melt(dataWithPcodes, columnsToMelt) + wideDF <- dcast(longDF, ... ~ variable + parm_cd ) + wideDF[,grep("_va_",names(wideDF))] <- sapply(wideDF[,grep("_va_",names(wideDF))], function(x) as.numeric(x)) + pCodesReturned <- unique(dataWithPcodes$parm_cd) + groupByPCode <- as.vector(sapply(pCodesReturned, function(x) grep(x, names(wideDF)) )) + data <- wideDF[,c(1:length(columnsToMelt)-1,groupByPCode)] + comment(data) <- originalHeader + } else { + warning("Reshape can only be used with expanded data. Reshape request will be ignored.") } - longDF <- melt(dataWithPcodes, columnsToMelt) - wideDF <- dcast(longDF, ... ~ variable + parm_cd ) - wideDF[,grep("_va_",names(wideDF))] <- sapply(wideDF[,grep("_va_",names(wideDF))], function(x) as.numeric(x)) - pCodesReturned <- unique(dataWithPcodes$parm_cd) - groupByPCode <- as.vector(sapply(pCodesReturned, function(x) grep(x, names(wideDF)) )) - data <- wideDF[,c(1:length(columnsToMelt)-1,groupByPCode)] - comment(data) <- originalHeader - - } - - if(reshape & !expanded){ - warning("Reshape can only be used with expanded data. Reshape request will be ignored.") } + parameterCd <- parameterCd[parameterCd != ""] siteInfo <- readNWISsite(siteNumbers) + varInfo <- readNWISpCode(parameterCd) attr(data, "siteInfo") <- siteInfo diff --git a/R/readWQPdata.R b/R/readWQPdata.R index 9b8ecffdf23563f74dc36cf7640cc6977c452511..38ad83dece9825140a709380a8e756ee0455572c 100644 --- a/R/readWQPdata.R +++ b/R/readWQPdata.R @@ -111,6 +111,9 @@ readWQPdata <- function(...){ } values <- checkWQPdates(values) + + names(values)[names(values) == "siteNumber"] <- "siteid" + names(values)[names(values) == "siteNumbers"] <- "siteid" urlCall <- paste(paste(names(values),values,sep="="),collapse="&") diff --git a/R/renameColumns.R b/R/renameColumns.R index e98ec7e39a94c1c26cba2fc1928f196c3488a8f2..7163c1c5c1976b1e45d3fc385e84d4ff4bb11da4 100644 --- a/R/renameColumns.R +++ b/R/renameColumns.R @@ -20,7 +20,7 @@ #'form of the name must be like pXXXXX, where XXXXX is the parameter code. #' @return A dataset like \code{data} with selected columns renamed. #' @note The following statistics codes are converted by \code{renameNWISColumns}. See -#'\url{http://help.waterdata.usgs.gov/stat_cd_nm} for information about USGS statistics codes. +#'\url{http://help.waterdata.usgs.gov} for more information about USGS codes. #'\describe{ #'\item{00001}{Maximum value, suffix: Max} #'\item{00002}{Minimum value, suffix: Min} diff --git a/R/tabbedDataRetrievals.R b/R/tabbedDataRetrievals.R index d21e64cf7a946c8ecc6411f2b045c01eaeeeffb1..21d56157fd985360ac522af98b77405c22e52f77 100644 --- a/R/tabbedDataRetrievals.R +++ b/R/tabbedDataRetrievals.R @@ -3,8 +3,6 @@ #' \tabular{ll}{ #' Package: \tab dataRetrieval\cr #' Type: \tab Package\cr -#' Version: \tab 2.1.1\cr -#' Date: \tab 2015-01-29\cr #' License: \tab Unlimited for this package, dependencies have more restrictive licensing.\cr #' Copyright: \tab This software is in the public domain because it contains materials #' that originally came from the United States Geological Survey, an agency of @@ -24,3 +22,59 @@ #' @keywords USGS, web services NULL +#' List of USGS parameter codes +#' +#' Complete list of USGS parameter codes as of November 7, 2014. The data was pulled from +#' \url{http://nwis.waterdata.usgs.gov/nwis/pmcodes/pmcodes?radio_pm_search=param_group&pm_group=All+--+include+all+parameter+groups& +#' format=rdb&show=parameter_group_nm&show=parameter_nm&show=casrn&show=srsname&show=parameter_units} +#' +#' @name parameterCdFile +#' @return parameterData data frame with information about USGS parameters. +#' +#' \tabular{lll}{ +#' Name \tab Type \tab Description\cr +#' parameter_cd \tab character \tab 5-digit USGS parameter code \cr +#' parameter_group_nm \tab character \tab USGS parameter group name\cr +#' parameter_nm \tab character \tab USGS parameter name\cr +#' casrn \tab character \tab Chemical Abstracts Service (CAS) Registry Number\cr +#' srsname \tab character \tab Substance Registry Services Name\cr +#' parameter_units \tab character \tab Parameter units\cr +#' } +#' +#' +#'@docType data +#'@export parameterCdFile +#'@keywords datasets +#'@examples +#'head(parameterCdFile[,1:2]) +NULL + +#' Data to convert USGS parameter code to characteristic names +#' +#' Data pulled from Water Quality Portal on November 25, 2014. The data was pulled from +#' \url{http://www.waterqualitydata.us/public_srsnames?mimeType=json}. +#' +#' @name pCodeToName +#' @return pCodeToName data frame with information about USGS parameters and how they +#' relate to characteristic names (useful for WQP requests). +#' +#' \tabular{lll}{ +#' Name \tab Type \tab Description\cr +#' parm_cd \tab character \tab 5-digit USGS parameter code \cr +#' description \tab character \tab Parameter description\cr +#' characteristicname \tab character \tab Characteristic Name \cr +#' measureunitcode \tab character \tab Parameter units\cr +#' resultsamplefraction \tab character \tab Result sample fraction text\cr +#' resulttemperaturebasis \tab character \tab Temperature basis information\cr +#' resultstatisticalbasis \tab character \tab Statistical basis\cr +#' resulttimebasis \tab character \tab Time basis\cr +#' resultweightbasis \tab character \tab Weight basis\cr +#' resultparticlesizebasis \tab character \tab Particle size basis\cr +#' last_rev_dt \tab character \tab Latest revision of information\cr +#' } +#' @docType data +#' @export pCodeToName +#' @keywords USGS parameterCd +#' @examples +#' head(pCodeToName[,1:2]) +NULL diff --git a/inst/doc/dataRetrieval.Rnw b/inst/doc/dataRetrieval.Rnw index db83a5395845c930fdd89759c0f463a19464652f..40390f31350c030097b6860cf987b404139a1cdb 100644 --- a/inst/doc/dataRetrieval.Rnw +++ b/inst/doc/dataRetrieval.Rnw @@ -1,7 +1,7 @@ %\VignetteIndexEntry{Introduction to the dataRetrieval package} %\VignetteEngine{knitr::knitr} %\VignetteDepends{} -%\VignetteSuggests{xtable} +%\VignetteSuggests{xtable, testthat} %\VignetteImports{XML, RCurl, reshape2,lubridate,httr,plyr} %\VignettePackage{dataRetrieval} diff --git a/inst/doc/dataRetrieval.pdf b/inst/doc/dataRetrieval.pdf index ce2096d7943bcd383c5c89656f293707be2033d9..ab667863f3c7b1d7dc1461293661f11ca2655827 100644 Binary files a/inst/doc/dataRetrieval.pdf and b/inst/doc/dataRetrieval.pdf differ diff --git a/man/dataRetrieval-package.Rd b/man/dataRetrieval-package.Rd index 4756b365b8ea5c2ad5b75d8aa323ac8c5ea14e26..ac30a455984718e4b1a7a0ea1645109878136a08 100644 --- a/man/dataRetrieval-package.Rd +++ b/man/dataRetrieval-package.Rd @@ -8,8 +8,6 @@ \tabular{ll}{ Package: \tab dataRetrieval\cr Type: \tab Package\cr -Version: \tab 2.1.1\cr -Date: \tab 2015-01-29\cr License: \tab Unlimited for this package, dependencies have more restrictive licensing.\cr Copyright: \tab This software is in the public domain because it contains materials that originally came from the United States Geological Survey, an agency of diff --git a/man/pCodeToName.Rd b/man/pCodeToName.Rd new file mode 100644 index 0000000000000000000000000000000000000000..45af51e616854a9d46d831943cc6beefa80f9851 --- /dev/null +++ b/man/pCodeToName.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2 (4.1.0): do not edit by hand +% Please edit documentation in R/tabbedDataRetrievals.R +\docType{data} +\name{pCodeToName} +\alias{pCodeToName} +\title{Data to convert USGS parameter code to characteristic names} +\value{ +pCodeToName data frame with information about USGS parameters and how they +relate to characteristic names (useful for WQP requests). + +\tabular{lll}{ +Name \tab Type \tab Description\cr +parm_cd \tab character \tab 5-digit USGS parameter code \cr +description \tab character \tab Parameter description\cr +characteristicname \tab character \tab Characteristic Name \cr +measureunitcode \tab character \tab Parameter units\cr +resultsamplefraction \tab character \tab Result sample fraction text\cr +resulttemperaturebasis \tab character \tab Temperature basis information\cr +resultstatisticalbasis \tab character \tab Statistical basis\cr +resulttimebasis \tab character \tab Time basis\cr +resultweightbasis \tab character \tab Weight basis\cr +resultparticlesizebasis \tab character \tab Particle size basis\cr +last_rev_dt \tab character \tab Latest revision of information\cr +} +} +\description{ +Data pulled from Water Quality Portal on November 25, 2014. The data was pulled from +\url{http://www.waterqualitydata.us/public_srsnames?mimeType=json}. +} +\examples{ +head(pCodeToName[,1:2]) +} +\keyword{USGS} +\keyword{parameterCd} + diff --git a/man/parameterCdFile.Rd b/man/parameterCdFile.Rd new file mode 100644 index 0000000000000000000000000000000000000000..516692698af0cb7b415eeddae6f559be41184e6c --- /dev/null +++ b/man/parameterCdFile.Rd @@ -0,0 +1,29 @@ +% Generated by roxygen2 (4.1.0): do not edit by hand +% Please edit documentation in R/tabbedDataRetrievals.R +\docType{data} +\name{parameterCdFile} +\alias{parameterCdFile} +\title{List of USGS parameter codes} +\value{ +parameterData data frame with information about USGS parameters. + +\tabular{lll}{ +Name \tab Type \tab Description\cr +parameter_cd \tab character \tab 5-digit USGS parameter code \cr +parameter_group_nm \tab character \tab USGS parameter group name\cr +parameter_nm \tab character \tab USGS parameter name\cr +casrn \tab character \tab Chemical Abstracts Service (CAS) Registry Number\cr +srsname \tab character \tab Substance Registry Services Name\cr +parameter_units \tab character \tab Parameter units\cr +} +} +\description{ +Complete list of USGS parameter codes as of November 7, 2014. The data was pulled from +\url{http://nwis.waterdata.usgs.gov/nwis/pmcodes/pmcodes?radio_pm_search=param_group&pm_group=All+--+include+all+parameter+groups& +format=rdb&show=parameter_group_nm&show=parameter_nm&show=casrn&show=srsname&show=parameter_units} +} +\examples{ +head(parameterCdFile[,1:2]) +} +\keyword{datasets} + diff --git a/man/readNWISdata.Rd b/man/readNWISdata.Rd index 1dd9353be964fa927edcf90db982534e2cd12574..0973fbecfb02319caf1232fa5e565576c1bac84e 100644 --- a/man/readNWISdata.Rd +++ b/man/readNWISdata.Rd @@ -8,7 +8,7 @@ readNWISdata(service = "dv", ...) } \arguments{ \item{service}{string. Possible values are "iv" (for instantaneous), "dv" (for daily values), "gwlevels" -(for groundwater levels)} +(for groundwater levels), and "site" (for site service)} \item{\dots}{see \url{http://waterservices.usgs.gov/rest/Site-Service.html#Service} for a complete list of options} } @@ -53,6 +53,12 @@ dataTempUnit <- readNWISdata(sites="03086500", service="iv", parameterCd="00010" multiSite <- readNWISdata(sites=c("04025000","04072150"), service="iv", parameterCd="00010") #Not empty: multiSite <- readNWISdata(sites=c("04025500","040263491"), service="iv", parameterCd="00060") +bBoxEx <- readNWISdata(bBox=c(-83,36.5,-81,38.5), parameterCd="00010") +startDate <- as.Date("2013-10-01") +endDate <- as.Date("2014-09-30") +waterYear <- readNWISdata(bBox=c(-83,36.5,-81,38.5), parameterCd="00010", + service="dv", startDate=startDate, endDate=endDate) +siteInfo <- readNWISdata(stateCd="WI", parameterCd="00010",hasDataTypeCd="iv", service="site") } } \seealso{ diff --git a/man/readNWISdv.Rd b/man/readNWISdv.Rd index b6df68c40a14eeb947b6779d13a0c480620c1287..1e4e3716e9a041abac690472cb1becd6e549cd83 100644 --- a/man/readNWISdv.Rd +++ b/man/readNWISdv.Rd @@ -48,8 +48,7 @@ queryTime \tab POSIXct \tab The time the data was returned \cr } \description{ Imports data from NWIS web service. This function gets the data from here: \url{http://waterservices.usgs.gov/} -A list of parameter codes can be found here: \url{http://help.waterdata.usgs.gov/codes-and-parameters/parameters} -A list of statistic codes can be found here: \url{http://help.waterdata.usgs.gov/code/stat_code_query?fmt=html} +Information on parameter and statistics codes can be found here: \url{http://help.waterdata.usgs.gov} } \examples{ siteNumber <- '04085427' diff --git a/man/readNWISpCode.Rd b/man/readNWISpCode.Rd index 30570954baf32d83ffb60b9703a7428942ccd2d1..47fca5f54bc6c20ded296366285d18efb37a9025 100644 --- a/man/readNWISpCode.Rd +++ b/man/readNWISpCode.Rd @@ -7,8 +7,8 @@ readNWISpCode(parameterCd) } \arguments{ -\item{parameterCd}{character of USGS parameter codes (or multiple parameter codes). These are 5 digit number codes -that can be found here: \url{http://help.waterdata.usgs.gov/codes-and-parameters/parameters}. To get a +\item{parameterCd}{character of USGS parameter codes (or multiple parameter codes). These are 5 digit number codes, +more information can be found here: \url{http://help.waterdata.usgs.gov/}. To get a complete list of all current parameter codes in the USGS, use "all" as the input.} } \value{ diff --git a/man/readNWISqw.Rd b/man/readNWISqw.Rd index bfc69ee85b7f69a98bd47a7a3ed08b0a7a84e9a3..ad074eb6a05f7b1eb5cf36268e4db7e4f2aa2077 100644 --- a/man/readNWISqw.Rd +++ b/man/readNWISqw.Rd @@ -10,7 +10,8 @@ readNWISqw(siteNumbers, parameterCd, startDate = "", endDate = "", \arguments{ \item{siteNumbers}{character of USGS site numbers. This is usually an 8 digit number} -\item{parameterCd}{character of USGS parameter code(s). This is usually an 5 digit number. Can also be "all".} +\item{parameterCd}{character that contains the code for a parameter +group, or a character vector of 5-digit parameter codes. See \bold{Details}.} \item{startDate}{character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates retrieval for the earliest possible record.} @@ -60,6 +61,28 @@ Imports data from NWIS web service. This function gets the data from here: \url{ A list of parameter codes can be found here: \url{http://nwis.waterdata.usgs.gov/nwis/pmcodes/} A list of statistic codes can be found here: \url{http://nwis.waterdata.usgs.gov/nwis/help/?read_file=stat&format=table} } +\details{ +Valid parameter code groups are "All," or group codes: +\tabular{ll}{ +Code \tab Description\cr +INF \tab Information \cr +PHY \tab Physical \cr +INM \tab Inorganics, Major, Metals (major cations) \cr +INN \tab Inorganics, Major, Non-metals (major anions) \cr +NUT \tab Nutrient \cr +MBI \tab Microbiological \cr +BIO \tab Biological \cr +IMN \tab Inorganics, Minor, Non-metals \cr +IMM \tab Inorganics, Minor, Metals \cr +TOX \tab Toxicity \cr +OPE \tab Organics, pesticide \cr +OPC \tab Organics, PCBs \cr +OOT \tab Organics, other \cr +RAD \tab Radiochemical \cr +SED \tab Sediment \cr +POP \tab Population/community \cr +} +} \examples{ siteNumbers <- c('04024430','04024000') startDate <- '2010-01-01' @@ -71,7 +94,15 @@ rawNWISqwDataReshaped <- readNWISqw(siteNumbers,parameterCd, startDate,endDate,reshape=TRUE) parameterCd <- "all" rawNWISall <- readNWISqw(siteNumbers,parameterCd, - startDate,endDate,reshape=TRUE) + startDate,endDate) +pgroup <- c("NUT") +rawNWISNutrients <- readNWISqw(siteNumbers,pgroup, + startDate,endDate) +groups <- c("NUT","OPE") +rawNWISNutOpe <- readNWISqw(siteNumbers,groups, + startDate,endDate) +rawNWISOpe <- readNWISqw(siteNumbers,"OPE", + startDate,endDate) } } \seealso{ diff --git a/man/renameNWISColumns.Rd b/man/renameNWISColumns.Rd index 28cdad5914270c4f53ccb6e1bef767eebae25aaf..aeec583a1f7121459d55937d8b75cb63dc88f1ca 100644 --- a/man/renameNWISColumns.Rd +++ b/man/renameNWISColumns.Rd @@ -45,7 +45,7 @@ to rename those columns. } \note{ The following statistics codes are converted by \code{renameNWISColumns}. See -\url{http://help.waterdata.usgs.gov/stat_cd_nm} for information about USGS statistics codes. +\url{http://help.waterdata.usgs.gov} for more information about USGS codes. \describe{ \item{00001}{Maximum value, suffix: Max} \item{00002}{Minimum value, suffix: Min} diff --git a/tests/testthat.R b/tests/testthat.R index fcb82a13f71a2897348d13b16221f325151311d3..17c246c0a75eb05ad0d6cbf33d2aba5a436d57df 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -1,3 +1,3 @@ library(testthat) library(dataRetrieval) -test_check("dataRetrieval") \ No newline at end of file +test_check("dataRetrieval") diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R new file mode 100644 index 0000000000000000000000000000000000000000..1a8beaa6dad526544e7fc5f017fb8c711b4ad033 --- /dev/null +++ b/tests/testthat/tests_general.R @@ -0,0 +1,41 @@ +context("General functions") + +test_that("General NWIS retrievals working", { + testthat::skip_on_cran() + + multiSite <- readNWISdata(sites=c("04025500","040263491"), service="iv", + parameterCd="00060") + expect_is(multiSite$dateTime, 'POSIXct') + + bBoxEx <- readNWISdata(bBox=c(-83,36.5,-81,38.5), parameterCd="00010") + expect_that(length(unique(bBoxEx$site_no)) > 1, is_true()) + + startDate <- as.Date("2013-10-01") + endDate <- as.Date("2014-09-30") + waterYear <- readNWISdata(bBox=c(-83,36.5,-81,38.5), parameterCd="00010", + service="dv", startDate=startDate, endDate=endDate) + expect_is(waterYear$Date, 'Date') + + siteInfo <- readNWISdata(stateCd="WI", parameterCd="00010",hasDataTypeCd="iv", + service="site") + +}) + + +test_that("General WQP retrievals working", { + testthat::skip_on_cran() + + nameToUse <- "pH" + pHData <- readWQPdata(siteid="USGS-04024315",characteristicName=nameToUse) + expect_is(pHData$ActivityStartDateTime, 'POSIXct') + + pHDataExpanded2 <- readWQPdata(bBox=c(-90.10,42.67,-88.64,43.35), + characteristicName=nameToUse) + expect_is(pHDataExpanded2$ActivityStartDateTime, 'POSIXct') + + startDate <- as.Date("2013-01-01") + nutrientDaneCounty <- readWQPdata(countycode="US:55:025",startDate=startDate, + characteristicType="Nutrient") + expect_is(nutrientDaneCounty$ActivityStartDateTime, 'POSIXct') + +}) \ No newline at end of file diff --git a/tests/testthat/tests_imports.R b/tests/testthat/tests_imports.R index 53ef4bc9a74b0b3dd0926defa218f06546539127..ea5fe0488ef78d792ded1a5f9b163225c6b7aa93 100644 --- a/tests/testthat/tests_imports.R +++ b/tests/testthat/tests_imports.R @@ -116,3 +116,21 @@ test_that("External importWaterML1 test", { expect_that(length(unique(inactiveAndActive$site_no)) == 1, is_true()) }) + +context("importWQP_noCRAN") + +test_that("External WQP tests", { + testthat::skip_on_cran() + + rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '') + rawSample <- importWQP(rawSampleURL) + expect_is(rawSample$ActivityStartDateTime, 'POSIXct') + + url2 <- paste0(rawSampleURL,"&zip=yes") + rawSample2 <- suppressWarnings(importWQP(url2, TRUE)) + expect_is(rawSample2$ActivityStartDateTime, 'POSIXct') + + STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '') + STORETdata <- importWQP(STORETex) + expect_is(STORETdata$ActivityStartDateTime, 'POSIXct') +}) \ No newline at end of file diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index 71034dd23d79d0238e1ad3a7d34ebae353bda0fc..53f5384ad009cc8f9e7a582dc2100c427141083d 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -27,7 +27,7 @@ test_that("Unit value data returns correct types", { }) -context("Peak, rating, meas") +context("Peak, rating, meas, site") test_that("peak, rating curves, surface-water measurements", { testthat::skip_on_cran() @@ -44,5 +44,79 @@ test_that("peak, rating curves, surface-water measurements", { siteNumbers <- c('01594440','040851325') data <- readNWISmeas(siteNumbers) expect_is(data$agency_cd, 'character') + + siteINFO <- readNWISsite('05114000') + expect_is(siteINFO$agency_cd, 'character') + + siteINFOMulti <- readNWISsite(c('05114000','09423350')) + expect_that(nrow(siteINFOMulti) == 2, is_true()) + +}) + +context("qw") + +test_that("NWIS qw tests", { + testthat::skip_on_cran() + siteNumbers <- c('04024430','04024000') + startDate <- '2010-01-01' + endDate <- '' + parameterCd <- c('34247','30234','32104','34220') + + rawNWISqwData <- readNWISqw(siteNumbers,parameterCd,startDate,endDate) + expect_is(rawNWISqwData$startDateTime, 'POSIXct') + + rawNWISqwDataReshaped <- readNWISqw(siteNumbers,parameterCd, + startDate,endDate,reshape=TRUE) + expect_is(rawNWISqwDataReshaped$startDateTime, 'POSIXct') + + parameterCd <- "all" + rawNWISall <- readNWISqw(siteNumbers,parameterCd, + startDate,"2011-01-01",reshape=TRUE) + expect_is(rawNWISall$startDateTime, 'POSIXct') + + pgroup <- c("NUT") + rawNWISNutrients <- readNWISqw(siteNumbers,pgroup, + startDate,endDate) + expect_is(rawNWISNutrients$startDateTime, 'POSIXct') + + groups <- c("NUT","OPE") + rawNWISNutOpe <- readNWISqw(siteNumbers,groups, + startDate,endDate) + expect_is(rawNWISNutOpe$startDateTime, 'POSIXct') + }) +context("dv") + +test_that("NWIS dv tests", { + testthat::skip_on_cran() + + siteNumber <- '04085427' + startDate <- '2012-01-01' + endDate <- '2012-06-30' + pCode <- '00060' + + rawDailyQ <- readNWISdv(siteNumber,pCode, startDate, endDate) + expect_is(rawDailyQ$Date, 'Date') + + rawDailyQAndTempMeanMax <- readNWISdv(siteNumber,c('00010','00060'), + startDate, endDate, statCd=c('00001','00003')) + expect_that(length(grep("00060", names(rawDailyQAndTempMeanMax))) >= 2 & + length(grep("00010", names(rawDailyQAndTempMeanMax))) >= 2, is_true()) + + + rawDailyMultiSites<- readNWISdv(c("01491000","01645000"),c('00010','00060'), + startDate, endDate, statCd=c('00001','00003')) + expect_that(length(unique(rawDailyMultiSites$site_no)) > 1, is_true()) + + # Site with no data: + x <- readNWISdv("10258500","00060", "2014-09-08", "2014-09-14") + expect_that(sum(is.na(x$X_00060_00003)) > 0, is_true()) + + site <- "05212700" + notActive <- readNWISdv(site, "00060", "2014-01-01","2014-01-07") + expect_that(nrow(notActive) == 0, is_true()) +}) + + + diff --git a/vignettes/dataRetrieval.Rnw b/vignettes/dataRetrieval.Rnw index db83a5395845c930fdd89759c0f463a19464652f..40390f31350c030097b6860cf987b404139a1cdb 100644 --- a/vignettes/dataRetrieval.Rnw +++ b/vignettes/dataRetrieval.Rnw @@ -1,7 +1,7 @@ %\VignetteIndexEntry{Introduction to the dataRetrieval package} %\VignetteEngine{knitr::knitr} %\VignetteDepends{} -%\VignetteSuggests{xtable} +%\VignetteSuggests{xtable, testthat} %\VignetteImports{XML, RCurl, reshape2,lubridate,httr,plyr} %\VignettePackage{dataRetrieval}