Added some easy checks to the water quality data to assure that the number of...

Added some easy checks to the water quality data to assure that the number of samples returned matched what the water quality portal said should return.

DESCRIPTION

@@ -56,7 +56,8 @@ Collate:
     'padVariable.r'
 Depends:
     R (>= 2.15.0),
-    XML
+    XML,
+    RCurl
 Imports:
     zoo
 Suggests:

R/getRawQWData.r

@@ -21,8 +21,13 @@
 getRawQWData <- function(siteNumber,ParameterCd,StartDate,EndDate,interactive=TRUE){
 
   url <- constructNWISURL(siteNumber,ParameterCd,StartDate,EndDate,"wqp")
-  
+  h <- basicHeaderGatherer()
+  doc <- getURI(url, headerfunction = h$update)
+  numToBeReturned <- as.numeric(h$value()["Total-Result-Count"])
   suppressWarnings(retval <- read.delim(url, header = TRUE, quote="\"", dec=".", sep='\t', colClasses=c('character'), fill = TRUE))
+  actualNumReturned <- nrow(retval)
+  
+  if(actualNumReturned != numToBeReturned) warning(numToBeReturned, " sample results were expected, ", actualNumReturned, " were returned")
   
   return(retval)
 }
\ No newline at end of file

R/getWQPData.r

@@ -45,23 +45,15 @@ getWQPData <- function(siteNumber,characteristicName,StartDate,EndDate,interacti
                "&startDateHi=",
                EndDate,
                "&countrycode=US&mimeType=tsv",sep = "")
+  h <- basicHeaderGatherer()
+  doc <- getURI(url, headerfunction = h$update)
+  numToBeReturned <- as.numeric(h$value()["Total-Result-Count"])
   
   suppressWarnings(retval <- read.delim(url, header = TRUE, quote="\"", dec=".", sep='\t', colClasses=c('character'), fill = TRUE))
   
-  qualifier <- ifelse((
-    (
-      retval$ResultDetectionConditionText == "Not Detected"
-      & length(grep("Lower", retval$DetectionQuantitationLimitTypeName)) > 0
-    )
-    | 
-      (
-        retval$ResultMeasureValue < retval$DetectionQuantitationLimitMeasure.MeasureValue
-        & retval$ResultValueTypeName == "Actual"
-      )
-  ),
-                      "<",
-                      ""
-  )
+  qualifier <- ifelse((retval$ResultDetectionConditionText == "Not Detected" | 
+                         retval$ResultDetectionConditionText == "Detected Not Quantified" |
+                         retval$ResultMeasureValue < retval$DetectionQuantitationLimitMeasure.MeasureValue),"<","")
   
   correctedData<-ifelse((nchar(qualifier)==0),retval$ResultMeasureValue,retval$DetectionQuantitationLimitMeasure.MeasureValue)
   test <- data.frame(retval$CharacteristicName)
@@ -70,6 +62,9 @@ getWQPData <- function(siteNumber,characteristicName,StartDate,EndDate,interacti
   test$dateTime <- as.Date(retval$ActivityStartDate, "%Y-%m-%d")
   
   originalLength <- nrow(test)
+  
+  if(originalLength != numToBeReturned) warning(numToBeReturned, " sample results were expected, ", originalLength, " were returned")
+  
   test$qualifier <- qualifier
   test$value <- as.numeric(correctedData)
   
@@ -85,5 +80,6 @@ getWQPData <- function(siteNumber,characteristicName,StartDate,EndDate,interacti
   data <- reshape(test, idvar="dateTime", timevar = "CharacteristicName", direction="wide")    
   data$dateTime <- format(data$dateTime, "%Y-%m-%d")
   data$dateTime <- as.Date(data$dateTime)
+  
   return(data)
 }
\ No newline at end of file

man/getWaterML1Data.Rd

@@ -9,14 +9,14 @@
   retrieval}
 }
 \value{
-  Daily a data frame containing columns 'Date' and
-  'Discharge'
+  mergedDF a data frame containing columns agency, site,
+  dateTime, values, and remark codes for all requested
+  combinations
 }
 \description{
   This function accepts a url parameter that already
   contains the desired NWIS site, parameter code,
-  statistic, startdate and enddate. It returns a data frame
-  containing "Date" and "Discharge"
+  statistic, startdate and enddate.
 }
 \examples{
 url <- "http://waterservices.usgs.gov/nwis/dv/?format=waterml,1.1&sites="