readWQPdata.R 15.9 KB
Newer Older
1
#' General Data Import from Water Quality Portal
Laura A DeCicco's avatar
Laura A DeCicco committed
2
#'
David Watkins's avatar
David Watkins committed
3
#' Imports data from Water Quality Portal web service. This function gets the data from here: \url{https://www.waterqualitydata.us}.
4
#' because it allows for other agencies rather than the USGS.  
Laura A DeCicco's avatar
Laura A DeCicco committed
5
#'
6
#' @param \dots see \url{https://www.waterqualitydata.us/webservices_documentation} for a complete list of options. A list of arguments can also be supplied. 
Laura A DeCicco's avatar
Laura A DeCicco committed
7
#' @param querySummary logical to ONLY return the number of records and unique sites that will be returned from this query. This argument is not supported via the combined list from the \dots argument
8
9
10
11
12
13
#' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the 
#' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column.
#' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
#' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu",
#' "America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla". See also  \code{OlsonNames()} 
#' for more information on time zones.
Laura A DeCicco's avatar
Laura A DeCicco committed
14
#' @param ignore_attributes logical to choose to ignore fetching site and parameter attributes. Default is \code{FALSE}.
Laura A DeCicco's avatar
Laura A DeCicco committed
15
#' @keywords data import WQP web service
16
17
18
19
20
21
22
23
24
25
#' @return A data frame with at least the following columns:
#' \tabular{lll}{ 
#' Name \tab Type \tab Description \cr
#' OrganizationIdentifier \tab character \tab  A designator used to uniquely identify a unique business establishment within a context.\cr
#' OrganizationFormalName \tab character \tab  The legal designator (i.e. formal name) of an organization.\cr
#' ActivityIdentifier \tab character \tab	Designator that uniquely identifies an activity within an organization.\cr
#' ActivityTypeCode \tab character \tab	The text describing the type of activity.\cr
#' ActivityMediaName \tab character \tab	Name or code indicating the environmental medium where the sample was taken.\cr
#' ActivityMediaSubdivisionName \tab character \tab	Name or code indicating the environmental matrix as a subdivision of the sample media.\cr
#' ActivityStartDate \tab character \tab	The calendar date on which the field activity is started.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
26
27
#' ActivityStartTime.Time \tab character \tab	The time of day that is reported when the field activity began, based on a 24-hour timescale.\cr
#' ActivityStartTime.TimeZoneCode \tab character \tab	The time zone for which the time of day is reported. Any of the longitudinal divisions of the earth's surface in which a standard time is kept.\cr
28
#' ActivityEndDate \tab character \tab	The calendar date when the field activity is completed.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
29
30
31
32
#' ActivityEndTime.Time \tab character \tab	The time of day that is reported when the field activity ended, based on a 24-hour timescale.\cr
#' ActivityEndTime.TimeZoneCode \tab character \tab	The time zone for which the time of day is reported. Any of the longitudinal divisions of the earth's surface in which a standard time is kept.\cr
#' ActivityDepthHeightMeasure.MeasureValue \tab character \tab	A measurement of the vertical location (measured from a reference point) at which an activity occurred. Measure value is given in the units stored in ActivityDepthHeightMeasure.MeasureUnitCode.\cr
#' ActivityDepthHeightMeasure.MeasureUnitCode \tab character \tab	The code that represents the unit for measuring the item.\cr
33
#' ActivityDepthAltitudeReferencePointText \tab character \tab	The reference used to indicate the datum or reference used to establish the depth/altitude of an activity.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
34
35
36
37
#' ActivityTopDepthHeightMeasure.MeasureValue \tab character \tab	A measurement of the upper vertical location of a vertical location range (measured from a reference point) at which an activity occurred. Measure value is given in the units stored in ActivityTopDepthHeightMeasure.MeasureUnitCode.\cr
#' ActivityTopDepthHeightMeasure.MeasureUnitCode \tab character \tab	The code that represents the unit for measuring the item.\cr
#' ActivityBottomDepthHeightMeasure.MeasureValue \tab character \tab	A measurement of the lower vertical location of a vertical location range (measured from a reference point) at which an activity occurred. Measure value is given in the units stored in ActivityBottomDepthHeightMeasure.MeasureUnitCode.\cr
#' ActivityBottomDepthHeightMeasure.MeasureUnitCode \tab character \tab	The code that represents the unit for measuring the item.\cr
38
39
40
41
42
43
44
#' ProjectIdentifier \tab character \tab 	A designator used to uniquely identify a data collection project within a context of an organization.\cr
#' ActivityConductingOrganizationText \tab character \tab	A name of the Organization conducting an activity.\cr
#' MonitoringLocationIdentifier \tab character \tab	A designator used to describe the unique name, number, or code assigned to identify the monitoring location.\cr
#' ActivityCommentText \tab character \tab	General comments concerning the activity.\cr
#' SampleAquifer * \tab character \tab 	A code that designates the aquifer associated with groundwater samples.\cr
#' HydrologicCondition * \tab character \tab 	Hydrologic condition is the hydrologic condition that is represented by the sample collected (i.e. ? normal, falling, rising, peak stage).\cr
#' HydrologicEvent * \tab character \tab 	A hydrologic event that is represented by the sample collected (i.e. - storm, drought, snowmelt).\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
45
46
47
#' SampleCollectionMethod.MethodIdentifier\tab character \tab 	The identification number or code assigned by the method publisher.\cr
#' SampleCollectionMethod.MethodIdentifierContext \tab character \tab	Identifies the source or data system that created or defined the identifier.\cr
#' SampleCollectionMethod.MethodName \tab character \tab	The title that appears on the method from the method publisher.\cr
48
49
50
51
#' SampleCollectionEquipmentName \tab character \tab	The name for the equipment used in collecting the sample.\cr
#' ResultDetectionConditionText \tab character \tab	The textual descriptor of a result.\cr
#' CharacteristicName \tab character \tab	The object, property, or substance which is evaluated or enumerated by either a direct field measurement, a direct field observation, or by laboratory analysis of material collected in the field.\cr
#' ResultSampleFractionText \tab character \tab	The text name of the portion of the sample associated with results obtained from a physically-partitioned sample.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
52
#' ResultMeasureValue \tab numeric \tab	The reportable measure of the result for the chemical, microbiological or other characteristic being analyzed. Measure value is given in the units stored in ResultMeasure.MeasureUnitCode.\cr
53
#' MeasureQualifierCode \tab character \tab	A code used to identify any qualifying issues that affect the results.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
54
#' ResultMeasure.MeasureUnitCode \tab character \tab	The code that represents the unit for measuring the item.\cr
55
56
57
58
59
60
61
62
63
64
#' ResultStatusIdentifier \tab character \tab	Indicates the acceptability of the result with respect to QA/QC criteria.\cr
#' StatisticalBaseCode \tab character \tab	The code for the method used to calculate derived results.\cr
#' ResultValueTypeName \tab character \tab	A name that qualifies the process which was used in the determination of the result value (e.g., actual, estimated, calculated).\cr
#' ResultWeightBasisText \tab character \tab	The name that represents the form of the sample or portion of the sample which is associated with the result value (e.g., wet weight, dry weight, ash-free dry weight).\cr
#' ResultTimeBasisText \tab character \tab	The period of time (in days) over which a measurement was made. For example, BOD can be measured as 5 day or 20 day BOD.\cr
#' ResultTemperatureBasisText \tab character \tab	The name that represents the controlled temperature at which the sample was maintained during analysis, e.g. 25 deg BOD analysis.\cr
#' ResultParticleSizeBasisText \tab character \tab	User defined free text describing the particle size class for which the associated result is defined.\cr
#' PrecisionValue \tab character \tab	A measure of mutual agreement among individual measurements of the same property usually under prescribed similar conditions.\cr
#' ResultCommentText \tab character \tab	Free text with general comments concerning the result.\cr
#' USGSPCode * \tab character \tab 	5-digit number used in the US Geological Survey computerized data system, National Water Information System (NWIS), to uniquely identify a specific constituent.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
65
66
#' ResultDepthHeightMeasure.MeasureValue + \tab character \tab 	A measurement of the vertical location (measured from a reference point) at which a result occurred.\cr
#' ResultDepthHeightMeasure.MeasureUnitCode + \tab character \tab	The code that represents the unit for measuring the item.\cr
67
68
69
#' ResultDepthAltitudeReferencePointText + \tab character \tab 	The reference used to indicate the datum or reference used to establish the depth/altitude of a result.\cr
#' SubjectTaxonomicName \tab character \tab	The name of the organism from which a tissue sample was taken.\cr
#' SampleTissueAnatomyName  * \tab character \tab 	The name of the anatomy from which a tissue sample was taken.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
70
71
72
#' ResultAnalyticalMethod.MethodIdentifier \tab character \tab	The identification number or code assigned by the method publisher.\cr
#' ResultAnalyticalMethod.MethodIdentifierContext \tab character \tab	Identifies the source or data system that created or defined the identifier.\cr
#' ResultAnalyticalMethod.MethodName \tab character \tab	The title that appears on the method from the method publisher.\cr
73
74
75
76
77
#' MethodDescriptionText * \tab character \tab 	A brief summary that provides general information about the method.\cr
#' LaboratoryName \tab character \tab	The name of Lab responsible for the result.\cr
#' AnalysisStartDate \tab character \tab	The calendar date on which the analysis began.\cr
#' ResultLaboratoryCommentText \tab character \tab	Remarks which further describe the laboratory procedures which produced the result.\cr
#' DetectionQuantitationLimitTypeName \tab character \tab	Text describing the type of detection or quantitation level used in the analysis of a characteristic.\cr
Laura A DeCicco's avatar
Laura A DeCicco committed
78
79
#' DetectionQuantitationLimitMeasure.MeasureValue \tab numeric \tab	Constituent concentration that, when processed through the complete method, produces a signal that is statistically different from a blank. Measure value is given in the units stored in DetectionQuantitationLimitMeasure.MeasureUnitCode.\cr
#' DetectionQuantitationLimitMeasure.MeasureUnitCode \tab character \tab	The code that represents the unit for measuring the item.\cr
80
81
82
83
84
85
#' PreparationStartDate \tab character \tab	The calendar date when the preparation/extraction of the sample for analysis began.\cr
#' ActivityStartDateTime \tab POSIXct \tab Activity start date and time converted to POSIXct UTC.\cr
#' ActivityEndDateTime \tab POSIXct \tab Activity end date and time converted to POSIXct UTC.\cr
#' }
#' * = elements only in NWIS
#' + = elements only in STORET
Laura A DeCicco's avatar
Laura A DeCicco committed
86
87
88
89
90
91
92
93
94
#' 
#' There are also several useful attributes attached to the data frame:
#' \tabular{lll}{
#' Name \tab Type \tab Description \cr
#' url \tab character \tab The url used to generate the data \cr
#' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr
#' variableInfo \tab data.frame \tab A data frame containing information on the requested parameters \cr
#' queryTime \tab POSIXct \tab The time the data was returned \cr
#' }
Laura A DeCicco's avatar
Laura A DeCicco committed
95
96
#' @export
#' @examples
97
#' \donttest{
98
#' nameToUse <- "pH"
Laura A DeCicco's avatar
Laura A DeCicco committed
99
#' pHData <- readWQPdata(siteid="USGS-04024315",characteristicName=nameToUse)
Laura A DeCicco's avatar
Laura A DeCicco committed
100
101
#' pHData_summary <- readWQPdata(bBox=c(-90.10,42.67,-88.64,43.35),
#'      characteristicName=nameToUse, querySummary=TRUE)
Laura A DeCicco's avatar
Laura A DeCicco committed
102
#' startDate <- as.Date("2013-01-01")
103
104
105
106
107
108
109
110
#' secchi.names = c("Depth, Secchi disk depth", 
#'                  "Depth, Secchi disk depth (choice list)", 
#'                  "Secchi Reading Condition (choice list)", 
#'                  "Water transparency, Secchi disc")
#' args <- list('startDateLo' = startDate, 
#'              'startDateHi' = "2013-12-31", 
#'               statecode="WI", 
#'               characteristicName=secchi.names)
Laura A DeCicco's avatar
Laura A DeCicco committed
111
#' 
112
113
114
115
116
117
118
119
120
#' wqp.data <- readWQPdata(args)   
#' 
#' args_2 <- list('startDateLo' = startDate, 
#'              'startDateHi' = "2013-12-31", 
#'               statecode="WI", 
#'               characteristicName=secchi.names,
#'               querySummary=TRUE)
#'
#' wqp.summary <- readWQPdata(args_2) 
121
122
123
124
#' 
#' arg_3 <- list('startDateLo' = startDate, 
#'              'startDateHi' = "2013-12-31")
#' arg_4 <- list(statecode="WI", 
Laura A DeCicco's avatar
Laura A DeCicco committed
125
126
#'               characteristicName=secchi.names)
#' wqp.summary <- readWQPdata(arg_3, arg_4, querySummary=TRUE)
Laura A DeCicco's avatar
Laura A DeCicco committed
127
128
129
#' wqp.summary_WI <- readWQPdata(arg_3, statecode="WI", 
#'                               characteristicName=secchi.names, 
#'                               querySummary=TRUE)
Lindsay R Carr's avatar
Lindsay R Carr committed
130
131
#'                               
#' # querying by county
132
133
134
#' dailyLexingtonVA <- readWQPdata(statecode = "Virginia", 
#'                                 countycode="Lexington", 
#'                                 parameterCd = "00010") 
Laura A DeCicco's avatar
Laura A DeCicco committed
135
#'                                 
136
#' }
Laura A DeCicco's avatar
Laura A DeCicco committed
137
138
readWQPdata <- function(..., querySummary=FALSE, tz="UTC", 
                        ignore_attributes = FALSE){
139
  
140
  tz <- match.arg(tz, OlsonNames())
Laura A DeCicco's avatar
Laura A DeCicco committed
141
  
Laura A DeCicco's avatar
Laura A DeCicco committed
142
143
144
  valuesList <- readWQPdots(...)
  
  service <- valuesList$service
145
  
Laura A DeCicco's avatar
Laura A DeCicco committed
146
147
148
149
150
  values <- sapply(valuesList$values, function(x) URLencode(x, reserved = TRUE))

  baseURL <- drURL(service, arg.list=values)

  baseURL <- appendDrURL(baseURL, mimeType = "tsv")
Laura A DeCicco's avatar
Laura A DeCicco committed
151

Lindsay R Carr's avatar
Lindsay R Carr committed
152
  if(querySummary){
Laura A DeCicco's avatar
Laura A DeCicco committed
153
    retquery <- getQuerySummary(baseURL)
Lindsay R Carr's avatar
Lindsay R Carr committed
154
155
    return(retquery)
  } else {
156
  
Laura A DeCicco's avatar
Laura A DeCicco committed
157
    retval <- importWQP(baseURL, zip = values["zip"] == "yes", tz=tz)
158
    
Laura A DeCicco's avatar
Laura A DeCicco committed
159
    if(!all(is.na(retval)) & !ignore_attributes){
160
      
Laura A DeCicco's avatar
Laura A DeCicco committed
161
      siteInfo <- whatWQPsites(..., service = "Station")
Lindsay R Carr's avatar
Lindsay R Carr committed
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
      
      siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName,
                                   agency_cd=siteInfo$OrganizationIdentifier,
                                   site_no=siteInfo$MonitoringLocationIdentifier,
                                   dec_lat_va=siteInfo$LatitudeMeasure,
                                   dec_lon_va=siteInfo$LongitudeMeasure,
                                   hucCd=siteInfo$HUCEightDigitCode,
                                   stringsAsFactors=FALSE)
      
      siteInfo <- cbind(siteInfoCommon, siteInfo)
      
      retvalVariableInfo <- retval[,c("CharacteristicName","USGSPCode",
                                      "ResultMeasure.MeasureUnitCode","ResultSampleFractionText")]
      retvalVariableInfo <- unique(retvalVariableInfo)
      
      variableInfo <- data.frame(characteristicName=retval$CharacteristicName,
                                 parameterCd=retval$USGSPCode,
                                 param_units=retval$ResultMeasure.MeasureUnitCode,
                                 valueType=retval$ResultSampleFractionText,
181
                                 stringsAsFactors=FALSE)
182
      
Doug Friedman's avatar
Doug Friedman committed
183
      if(!anyNA(variableInfo$parameterCd)){
Lindsay R Carr's avatar
Lindsay R Carr committed
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
        pcodes <- unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)])
        pcodes <- pcodes["" != pcodes]
        paramINFO <- readNWISpCode(pcodes)
        names(paramINFO)["parameter_cd" == names(paramINFO)] <- "parameterCd"
        
        pCodeToName <- pCodeToName
        varExtras <- pCodeToName[pCodeToName$parm_cd %in% unique(variableInfo$parameterCd[!is.na(variableInfo$parameterCd)]),]
        names(varExtras)[names(varExtras) == "parm_cd"] <- "parameterCd"
        variableInfo <- merge(variableInfo, varExtras, by="parameterCd", all = TRUE)
        variableInfo <- merge(variableInfo, paramINFO, by="parameterCd", all = TRUE)
        variableInfo <- unique(variableInfo)
      }
      
      attr(retval, "siteInfo") <- siteInfo
      attr(retval, "variableInfo") <- variableInfo
Laura A DeCicco's avatar
Laura A DeCicco committed
199
      
Lindsay R Carr's avatar
Lindsay R Carr committed
200
    } else {
201
202
      if(!ignore_attributes){
        message("The following url returned no data:\n")
Laura A DeCicco's avatar
Laura A DeCicco committed
203
        message(baseURL)        
204
205
      }

206
    }
207
    attr(retval, "queryTime") <- Sys.time()
Laura A DeCicco's avatar
Laura A DeCicco committed
208
    attr(retval, "url") <- baseURL
Laura A DeCicco's avatar
Laura A DeCicco committed
209
210
    
    return(retval)
211
  }
212
}