diff --git a/DESCRIPTION b/DESCRIPTION index d19c8ff286e86d683c02cb84173dd394652ff737..f536f2a802b47c66b3573798bd8c56ec62b40db6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -35,7 +35,8 @@ Imports: stats, utils, dplyr, - readr (>= 0.1.1.9000) + readr (>= 0.1.1.9000), + httr Suggests: xtable, knitr, diff --git a/R/readWQPdata.R b/R/readWQPdata.R index 9d52553de7375124a40ecdbb4c7b5503a82ac317..208fc11f4d31eb786319774866bbac1b7343cec0 100644 --- a/R/readWQPdata.R +++ b/R/readWQPdata.R @@ -4,7 +4,8 @@ #' because it allows for other agencies rather than the USGS. #' #' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options -#' @param zip logical to request data via downloading zip file. Default set to FALSE. +#' @param zip logical to request data via downloading zip file. Default set to FALSE. +#' @param querySummary logical to request the number of sites and results a particular query would return before downloading the data. #' @keywords data import WQP web service #' @return A data frame with at least the following columns: #' \tabular{lll}{ @@ -97,7 +98,7 @@ #' #' #' } -readWQPdata <- function(..., zip=FALSE){ +readWQPdata <- function(..., zip=FALSE, querySummary=FALSE){ matchReturn <- list(...) @@ -154,53 +155,59 @@ readWQPdata <- function(..., zip=FALSE){ if(zip) urlCall <- paste0(urlCall,"&zip=yes") - retval <- importWQP(urlCall,zip=zip, tz=tz) - - if(!all(is.na(retval))){ - siteInfo <- whatWQPsites(...,zip=zip) + if(querySummary){ + queryHEAD <- HEAD(urlCall) + retquery <- headers(queryHEAD) + return(retquery) + } else { + retval <- importWQP(urlCall,zip=zip, tz=tz) - siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName, - agency_cd=siteInfo$OrganizationIdentifier, - site_no=siteInfo$MonitoringLocationIdentifier, - dec_lat_va=siteInfo$LatitudeMeasure, - dec_lon_va=siteInfo$LongitudeMeasure, - hucCd=siteInfo$HUCEightDigitCode, + if(!all(is.na(retval))){ + siteInfo <- whatWQPsites(...,zip=zip) + + siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName, + agency_cd=siteInfo$OrganizationIdentifier, + site_no=siteInfo$MonitoringLocationIdentifier, + dec_lat_va=siteInfo$LatitudeMeasure, + dec_lon_va=siteInfo$LongitudeMeasure, + hucCd=siteInfo$HUCEightDigitCode, + stringsAsFactors=FALSE) + + siteInfo <- cbind(siteInfoCommon, siteInfo) + + retvalVariableInfo <- retval[,c("CharacteristicName","USGSPCode", + "ResultMeasure.MeasureUnitCode","ResultSampleFractionText")] + retvalVariableInfo <- unique(retvalVariableInfo) + + variableInfo <- data.frame(characteristicName=retval$CharacteristicName, + parameterCd=retval$USGSPCode, + param_units=retval$ResultMeasure.MeasureUnitCode, + valueType=retval$ResultSampleFractionText, stringsAsFactors=FALSE) - - siteInfo <- cbind(siteInfoCommon, siteInfo) - - retvalVariableInfo <- retval[,c("CharacteristicName","USGSPCode", - "ResultMeasure.MeasureUnitCode","ResultSampleFractionText")] - retvalVariableInfo <- unique(retvalVariableInfo) - - variableInfo <- data.frame(characteristicName=retval$CharacteristicName, - parameterCd=retval$USGSPCode, - param_units=retval$ResultMeasure.MeasureUnitCode, - valueType=retval$ResultSampleFractionText, - stringsAsFactors=FALSE) - - if(any(!is.na(variableInfo$parameterCd))){ - pcodes <- unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]) - pcodes <- pcodes["" != pcodes] - paramINFO <- readNWISpCode(pcodes) - names(paramINFO)["parameter_cd" == names(paramINFO)] <- "parameterCd" - pCodeToName <- pCodeToName - varExtras <- pCodeToName[pCodeToName$parm_cd %in% unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]),] - names(varExtras)[names(varExtras) == "parm_cd"] <- "parameterCd" - variableInfo <- merge(variableInfo, varExtras, by="parameterCd", all = TRUE) - variableInfo <- merge(variableInfo, paramINFO, by="parameterCd", all = TRUE) - variableInfo <- unique(variableInfo) + if(any(!is.na(variableInfo$parameterCd))){ + pcodes <- unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]) + pcodes <- pcodes["" != pcodes] + paramINFO <- readNWISpCode(pcodes) + names(paramINFO)["parameter_cd" == names(paramINFO)] <- "parameterCd" + + pCodeToName <- pCodeToName + varExtras <- pCodeToName[pCodeToName$parm_cd %in% unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]),] + names(varExtras)[names(varExtras) == "parm_cd"] <- "parameterCd" + variableInfo <- merge(variableInfo, varExtras, by="parameterCd", all = TRUE) + variableInfo <- merge(variableInfo, paramINFO, by="parameterCd", all = TRUE) + variableInfo <- unique(variableInfo) + } + + attr(retval, "siteInfo") <- siteInfo + attr(retval, "variableInfo") <- variableInfo + attr(retval, "url") <- urlCall + attr(retval, "queryTime") <- Sys.time() + + return(retval) + } else { + message("The following url returned no data:\n") + message(urlCall) } - - attr(retval, "siteInfo") <- siteInfo - attr(retval, "variableInfo") <- variableInfo - attr(retval, "url") <- urlCall - attr(retval, "queryTime") <- Sys.time() - - return(retval) - } else { - message("The following url returned no data:\n") - message(urlCall) } } \ No newline at end of file diff --git a/R/readWQPqw.r b/R/readWQPqw.r index 684fd1a0b7358c86eb478e91f720b66a16ad05f9..e42e9ee545dd17ddb5e89ff3f34ccd2c01e5a274 100644 --- a/R/readWQPqw.r +++ b/R/readWQPqw.r @@ -108,68 +108,93 @@ #' rawPHsites <- readWQPqw(c('USGS-05406450', 'USGS-05427949','WIDNR_WQX-133040'), 'pH','','') #' nwisEx <- readWQPqw('USGS-04024000',c('34247','30234','32104','34220'),'','2012-12-20') #' } -readWQPqw <- function(siteNumbers,parameterCd,startDate="",endDate="",tz=""){ +readWQPqw <- function(siteNumbers,parameterCd,startDate="",endDate="",tz="", querySummary=FALSE){ url <- constructWQPURL(siteNumbers,parameterCd,startDate,endDate) - retval <- importWQP(url, tz = tz) - pcodeCheck <- all(nchar(parameterCd) == 5) & all(!is.na(suppressWarnings(as.numeric(parameterCd)))) - - if (nzchar(startDate)){ - startDate <- format(as.Date(startDate), format="%m-%d-%Y") - } - - if (nzchar(endDate)){ - endDate <- format(as.Date(endDate), format="%m-%d-%Y") - } - - if(pcodeCheck){ - siteInfo <- whatWQPsites(siteid=paste0(siteNumbers,collapse=";"), - pCode=paste0(parameterCd,collapse=";"), - startDateLo=startDate, startDateHi=endDate) + if(querySummary){ + queryHEAD <- HEAD(url) + retquery <- headers(queryHEAD) + return(retquery) } else { - siteInfo <- whatWQPsites(siteid=paste0(siteNumbers,collapse=";"), - characteristicName=URLencode(paste0(parameterCd,collapse=";")), - startDateLo=startDate, startDateHi=endDate) - } + retval <- importWQP(url, tz = tz) - siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName, - agency_cd=siteInfo$OrganizationIdentifier, - site_no=siteInfo$MonitoringLocationIdentifier, - dec_lat_va=siteInfo$LatitudeMeasure, - dec_lon_va=siteInfo$LongitudeMeasure, - hucCd=siteInfo$HUCEightDigitCode, - stringsAsFactors=FALSE) - - siteInfo <- cbind(siteInfoCommon, siteInfo) - - - variableInfo <- data.frame(characteristicName=retval$CharacteristicName, - parameterCd=retval$USGSPCode, - param_units=retval$ResultMeasure.MeasureUnitCode, - valueType=retval$ResultSampleFractionText, - stringsAsFactors=FALSE) - variableInfo <- unique(variableInfo) - - if(any(!is.na(variableInfo$parameterCd))){ - pcodes <- unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]) - pcodes <- pcodes["" != pcodes] - paramINFO <- readNWISpCode(pcodes) - names(paramINFO)["parameter_cd" == names(paramINFO)] <- "parameterCd" + siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName, + agency_cd=siteInfo$OrganizationIdentifier, + site_no=siteInfo$MonitoringLocationIdentifier, + dec_lat_va=siteInfo$LatitudeMeasure, + dec_lon_va=siteInfo$LongitudeMeasure, + hucCd=siteInfo$HUCEightDigitCode, + stringsAsFactors=FALSE) + + siteInfo <- cbind(siteInfoCommon, siteInfo) - pCodeToName <- pCodeToName - varExtras <- pCodeToName[pCodeToName$parm_cd %in% unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]),] - names(varExtras)[names(varExtras) == "parm_cd"] <- "parameterCd" - variableInfo <- merge(variableInfo, varExtras, by="parameterCd", all = TRUE) - variableInfo <- merge(variableInfo, paramINFO, by="parameterCd", all = TRUE) + + variableInfo <- data.frame(characteristicName=retval$CharacteristicName, + parameterCd=retval$USGSPCode, + param_units=retval$ResultMeasure.MeasureUnitCode, + valueType=retval$ResultSampleFractionText, + stringsAsFactors=FALSE) variableInfo <- unique(variableInfo) + + if(any(!is.na(variableInfo$parameterCd))){ + pcodes <- unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]) + pcodes <- pcodes["" != pcodes] + paramINFO <- readNWISpCode(pcodes) + names(paramINFO)["parameter_cd" == names(paramINFO)] <- "parameterCd" + + if (nzchar(endDate)){ + endDate <- format(as.Date(endDate), format="%m-%d-%Y") + } + + if(pcodeCheck){ + siteInfo <- whatWQPsites(siteid=paste0(siteNumbers,collapse=";"), + pCode=paste0(parameterCd,collapse=";"), + startDateLo=startDate, startDateHi=endDate) + } else { + siteInfo <- whatWQPsites(siteid=paste0(siteNumbers,collapse=";"), + characteristicName=URLencode(paste0(parameterCd,collapse=";")), + startDateLo=startDate, startDateHi=endDate) + } + + siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName, + agency_cd=siteInfo$OrganizationIdentifier, + site_no=siteInfo$MonitoringLocationIdentifier, + dec_lat_va=siteInfo$LatitudeMeasure, + dec_lon_va=siteInfo$LongitudeMeasure, + hucCd=siteInfo$HUCEightDigitCode, + stringsAsFactors=FALSE) + + siteInfo <- cbind(siteInfoCommon, siteInfo) + + + variableInfo <- data.frame(characteristicName=retval$CharacteristicName, + parameterCd=retval$USGSPCode, + param_units=retval$ResultMeasure.MeasureUnitCode, + valueType=retval$ResultSampleFractionText, + stringsAsFactors=FALSE) + variableInfo <- unique(variableInfo) + + if(any(variableInfo$parameterCd != "")){ + pcodes <- unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]) + pcodes <- pcodes["" != pcodes] + paramINFO <- readNWISpCode(pcodes) + names(paramINFO)["parameter_cd" == names(paramINFO)] <- "parameterCd" + + pCodeToName <- pCodeToName + varExtras <- pCodeToName[pCodeToName$parm_cd %in% unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]),] + names(varExtras)[names(varExtras) == "parm_cd"] <- "parameterCd" + variableInfo <- merge(variableInfo, varExtras, by="parameterCd", all = TRUE) + variableInfo <- merge(variableInfo, paramINFO, by="parameterCd", all = TRUE) + variableInfo <- unique(variableInfo) + } + + attr(retval, "siteInfo") <- siteInfo + attr(retval, "variableInfo") <- variableInfo + attr(retval, "url") <- url + attr(retval, "queryTime") <- Sys.time() + + return(retval) + } } - - attr(retval, "siteInfo") <- siteInfo - attr(retval, "variableInfo") <- variableInfo - attr(retval, "url") <- url - attr(retval, "queryTime") <- Sys.time() - - return(retval) - } diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 9a966864f0b7cade16aa267718862da4b7a12204..813c2c8e7d995cef524db84ed33c7a3fff7b8111 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -53,3 +53,18 @@ test_that("General WQP retrievals working", { characteristicType="Nutrient") expect_is(nutrientDaneCounty$ActivityStartDateTime, 'POSIXct') }) + +test_that("WQP head query retrieval working", { + testthat::skip_on_cran() + # testthat::skip_on_cran() + # Bring back when WQP is back + nameToUse <- "pH" + pHDataQueryResults <- readWQPdata(siteid="USGS-04024315", + characteristicName=nameToUse, + querySummary=TRUE) + expect_false(is.null(pHDataQueryResults$`total-site-count`)) + expect_is(pHDataQueryResults$`total-site-count`, 'character') + expect_false(is.null(pHDataQueryResults$`total-result-count`)) + expect_is(pHDataQueryResults$`total-result-count`, 'character') + +})