From d2b90116a752f2d1b77b1410d809ed3e2e030742 Mon Sep 17 00:00:00 2001 From: ncatolico Date: Tue, 13 Feb 2024 10:29:45 -0700 Subject: [PATCH 01/11] adding more schemas --- .../aquatroll200/aquatroll200_calibrated.avsc | 6 +- .../aquatroll200_cond_corrected.avsc | 8 +- .../aquatroll200/aquatroll200_log_data.avsc | 105 ++++++++++++++++++ .../aquatroll200/aquatroll200_log_flags.avsc | 61 ++++++++++ .../leveltroll500_calibrated.avsc | 4 +- .../leveltroll500_calibrated_wo_source.avsc | 4 +- .../leveltroll500/leveltroll500_log_data.avsc | 84 ++++++++++++++ .../leveltroll500_log_flags.avsc | 51 +++++++++ 8 files changed, 312 insertions(+), 11 deletions(-) create mode 100644 avro_schemas/aquatroll200/aquatroll200_log_data.avsc create mode 100644 avro_schemas/aquatroll200/aquatroll200_log_flags.avsc create mode 100644 avro_schemas/leveltroll500/leveltroll500_log_data.avsc create mode 100644 avro_schemas/leveltroll500/leveltroll500_log_flags.avsc diff --git a/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc b/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc index e169ad3..909a6af 100644 --- a/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc +++ b/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc @@ -34,7 +34,7 @@ "float" ], "default": null, - "doc": "Calibrated pressure of groundwater", + "doc": "Calibrated pressure of water", "__neon_units": "kilopascal", "__neon_stream_id": "0" }, @@ -55,7 +55,7 @@ "float" ], "default": null, - "doc": "Calibrated temperature in groundwater", + "doc": "Calibrated temperature in water", "__neon_units": "celsius", "__neon_stream_id": "2" }, @@ -76,7 +76,7 @@ "float" ], "default": null, - "doc": "Calibrated specific conductance in groundwater", + "doc": "Calibrated specific conductance in water", "__neon_units": "microsiemensPerCentimeter", "__neon_stream_id": "4" }, diff --git a/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc b/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc index 3c50a69..34b004e 100644 --- a/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc +++ b/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc @@ -34,7 +34,7 @@ "float" ], "default": null, - "doc": "Calibrated pressure of groundwater", + "doc": "Calibrated pressure of water", "__neon_units": "kilopascal", "__neon_stream_id": "0" }, @@ -55,7 +55,7 @@ "float" ], "default": null, - "doc": "Calibrated temperature in groundwater", + "doc": "Calibrated temperature in water", "__neon_units": "celsius", "__neon_stream_id": "2" }, @@ -76,7 +76,7 @@ "float" ], "default": null, - "doc": "Calibrated raw conductivity in groundwater", + "doc": "Calibrated raw conductivity in water", "__neon_units": "microsiemensPerCentimeter", "__neon_stream_id": "4" }, @@ -97,7 +97,7 @@ "float" ], "default": null, - "doc": "Calibrated specific conductance in groundwater", + "doc": "Calibrated specific conductance in water", "__neon_units": "microsiemensPerCentimeter" }, { diff --git a/avro_schemas/aquatroll200/aquatroll200_log_data.avsc b/avro_schemas/aquatroll200/aquatroll200_log_data.avsc new file mode 100644 index 0000000..28762d0 --- /dev/null +++ b/avro_schemas/aquatroll200/aquatroll200_log_data.avsc @@ -0,0 +1,105 @@ +{ + "type": "record", + "name": "aquatroll200_log_data", + "namespace": "org.neonscience.schema.dp0p", + "doc": "Log data for the Aqua Troll 520 sensor", + "__version": "1.0", + "__neon_parts": [ + "0317730000" + ], + "fields": [ + { + "name": "source_id", + "type": "string", + "doc": "Source serial number or MAC address" + }, + { + "name": "site_id", + "type": "string", + "doc": "NEON site identifier" + }, + { + "name": "readout_time", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + }, + "doc": "Timestamp of readout expressed in milliseconds since epoch", + "__neon_units": "millisecond" + }, + { + "name": "pressure", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Raw pressure of water", + "__neon_units": "psi", + "__neon_stream_id": "0" + }, + { + "name": "pressure_data_quality", + "type": [ + "null", + "uint16" + ], + "default": null, + "doc": "Data quality id from sensor for pressure represented by a 16 bit unsigned integer", + "__neon_stream_id": "1" + }, + { + "name": "temperature", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Raw temperature in water", + "__neon_units": "celsius", + "__neon_stream_id": "2" + }, + { + "name": "temperature_data_quality", + "type": [ + "null", + "uint16" + ], + "default": null, + "doc": "Data quality code from sensor for temperature represented by a 16 bit unsigned integer", + "__neon_stream_id": "3" + }, + { + "name": "conductivity", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Raw specific conductance in water", + "__neon_units": "microsiemensPerCentimeter", + "__neon_stream_id": "4" + }, + { + "name": "conductivity_data_quality", + "type": [ + "null", + "uint16" + ], + "default": null, + "doc": "Data quality code from sensor for conductivity represented by a 16 bit unsigned integer", + "__neon_stream_id": "5" + }, + { + "name": "internal_battery", + "type": [ + "null", + "uint16" + ], + "default": null, + "doc": "Internal battery voltage in millivolts represented by a 16 bit unsigned integer", + "__neon_units": "millivolt", + "__neon_stream_id": "6" + } + ] +} diff --git a/avro_schemas/aquatroll200/aquatroll200_log_flags.avsc b/avro_schemas/aquatroll200/aquatroll200_log_flags.avsc new file mode 100644 index 0000000..7112ac0 --- /dev/null +++ b/avro_schemas/aquatroll200/aquatroll200_log_flags.avsc @@ -0,0 +1,61 @@ +{ + "type": "record", + "name": "aquatroll200_log_flags", + "namespace": "org.neonscience.schema.dp0p", + "doc": "Log data flags for the Aqua Troll 200 sensor", + "__version": "1.0", + "__neon_parts": [ + "0317730000" + ], + "fields": [ + { + "name": "readout_time", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + }, + "doc": "Timestamp of readout expressed in milliseconds since epoch", + "__neon_units": "millisecond" + }, + { + "name": "pressureLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "logDataDateQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "temperatureLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "conductivityLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the conductivity log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + } + ] +} diff --git a/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc b/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc index 03ca439..fe4698e 100644 --- a/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc +++ b/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc @@ -34,7 +34,7 @@ "float" ], "default": null, - "doc": "Calibrated pressure of groundwater", + "doc": "Calibrated pressure of water", "__neon_units": "kilopascal", "__neon_stream_id": "0" }, @@ -55,7 +55,7 @@ "float" ], "default": null, - "doc": "Calibrated temperature in groundwater", + "doc": "Calibrated temperature in water", "__neon_units": "celsius", "__neon_stream_id": "2" }, diff --git a/avro_schemas/leveltroll500/leveltroll500_calibrated_wo_source.avsc b/avro_schemas/leveltroll500/leveltroll500_calibrated_wo_source.avsc index 4c9eb0d..950a36b 100644 --- a/avro_schemas/leveltroll500/leveltroll500_calibrated_wo_source.avsc +++ b/avro_schemas/leveltroll500/leveltroll500_calibrated_wo_source.avsc @@ -19,7 +19,7 @@ "float" ], "default": null, - "doc": "Calibrated pressure of groundwater", + "doc": "Calibrated pressure of water", "__neon_units": "kilopascal", "__neon_stream_id": "0" }, @@ -40,7 +40,7 @@ "float" ], "default": null, - "doc": "Calibrated temperature in groundwater", + "doc": "Calibrated temperature in water", "__neon_units": "celsius", "__neon_stream_id": "2" }, diff --git a/avro_schemas/leveltroll500/leveltroll500_log_data.avsc b/avro_schemas/leveltroll500/leveltroll500_log_data.avsc new file mode 100644 index 0000000..bc0a2ca --- /dev/null +++ b/avro_schemas/leveltroll500/leveltroll500_log_data.avsc @@ -0,0 +1,84 @@ +{ + "type": "record", + "name": "leveltroll500_log_data", + "namespace": "org.neonscience.schema.dp0p", + "doc": "Log data for the Level Troll 500 sensor", + "__version": "1.0", + "__neon_parts": [ + "0317680000" + ], + "fields": [ + { + "name": "source_id", + "type": "string", + "doc": "Source serial number or MAC address" + }, + { + "name": "site_id", + "type": "string", + "doc": "NEON site identifier" + }, + { + "name": "readout_time", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + }, + "doc": "Timestamp of readout expressed in milliseconds since epoch", + "__neon_units": "millisecond" + }, + { + "name": "pressure", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Raw pressure of water", + "__neon_units": "psi", + "__neon_stream_id": "0" + }, + { + "name": "pressure_data_quality", + "type": [ + "null", + "uint16" + ], + "default": null, + "doc": "Data quality id from sensor for pressure represented by a 16 bit unsigned integer", + "__neon_stream_id": "1" + }, + { + "name": "temperature", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Raw temperature in water", + "__neon_units": "celsius", + "__neon_stream_id": "2" + }, + { + "name": "temperature_data_quality", + "type": [ + "null", + "uint16" + ], + "default": null, + "doc": "Data quality code from sensor for temperature represented by a 16 bit unsigned integer", + "__neon_stream_id": "3" + }, + { + "name": "internal_battery", + "type": [ + "null", + "uint16" + ], + "default": null, + "doc": "Internal battery voltage in millivolts represented by a 16 bit unsigned integer", + "__neon_units": "millivolt", + "__neon_stream_id": "6" + } + ] +} diff --git a/avro_schemas/leveltroll500/leveltroll500_log_flags.avsc b/avro_schemas/leveltroll500/leveltroll500_log_flags.avsc new file mode 100644 index 0000000..ffe6ef4 --- /dev/null +++ b/avro_schemas/leveltroll500/leveltroll500_log_flags.avsc @@ -0,0 +1,51 @@ +{ + "type": "record", + "name": "leveltroll500_log_flags", + "namespace": "org.neonscience.schema.dp0p", + "doc": "Log data flags for the Level Troll 500 sensor", + "__version": "1.0", + "__neon_parts": [ + "0317680000" + ], + "fields": [ + { + "name": "readout_time", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + }, + "doc": "Timestamp of readout expressed in milliseconds since epoch", + "__neon_units": "millisecond" + }, + { + "name": "pressureLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "logDataDateQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "temperatureLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + } + ] +} From d0aeb0ccc342badd16eda59fdb771c72decb58c2 Mon Sep 17 00:00:00 2001 From: ncatolico Date: Tue, 5 Mar 2024 13:44:41 -0700 Subject: [PATCH 02/11] empty files for log flags --- ...l200_location_year-month-day_logFlags.parquet | Bin 0 -> 1554 bytes ...l500_location_year-month-day_logFlags.parquet | Bin 0 -> 1554 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 empty_files/aquatroll200/flags/aquatroll200_location_year-month-day_logFlags.parquet create mode 100644 empty_files/leveltroll500/flags/leveltroll500_location_year-month-day_logFlags.parquet diff --git a/empty_files/aquatroll200/flags/aquatroll200_location_year-month-day_logFlags.parquet b/empty_files/aquatroll200/flags/aquatroll200_location_year-month-day_logFlags.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f07b03791bb127b3a1e403ac2f25831f9d9ac772 GIT binary patch literal 1554 zcmd5-OK;jh5Z+X6R9W;;DMTRUkV|rDlXz`NNN;On12J}jW%E*{${5SU0qh!xF{l34 z{*ulvkl;M-UGOYB-#4@KlDQ)$K1E0QTrt02EaYO(8HS7bulfDN%jXUe{hXpk4uRZ~ zzDBV(n=N9m8TwUc?zGhRm|w#Ito!j(DOQqN!)F^8EfaY8x!U|zTc5Daly1dFLN@n#&pOZjJ|CY3xo^{$*|yCh7{Dd~@k&z)6x!re#f}jovJS?t^VzY?@OW zasp+8+F$4|>>nt9$DVn0>o;RpG_X5q>)Sfob`O|ULhT_bo6Zl+W1Y?utk$y=MP*6L zEWdRXD1-5!ciG-D1nvjkb3IVuICA3O3vYhrM$wrQ$6<2(=*6@9aC(eO=lnTeq%X{; NFB*PA=nno3{sBt#kT(DT literal 0 HcmV?d00001 diff --git a/empty_files/leveltroll500/flags/leveltroll500_location_year-month-day_logFlags.parquet b/empty_files/leveltroll500/flags/leveltroll500_location_year-month-day_logFlags.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f07b03791bb127b3a1e403ac2f25831f9d9ac772 GIT binary patch literal 1554 zcmd5-OK;jh5Z+X6R9W;;DMTRUkV|rDlXz`NNN;On12J}jW%E*{${5SU0qh!xF{l34 z{*ulvkl;M-UGOYB-#4@KlDQ)$K1E0QTrt02EaYO(8HS7bulfDN%jXUe{hXpk4uRZ~ zzDBV(n=N9m8TwUc?zGhRm|w#Ito!j(DOQqN!)F^8EfaY8x!U|zTc5Daly1dFLN@n#&pOZjJ|CY3xo^{$*|yCh7{Dd~@k&z)6x!re#f}jovJS?t^VzY?@OW zasp+8+F$4|>>nt9$DVn0>o;RpG_X5q>)Sfob`O|ULhT_bo6Zl+W1Y?utk$y=MP*6L zEWdRXD1-5!ciG-D1nvjkb3IVuICA3O3vYhrM$wrQ$6<2(=*6@9aC(eO=lnTeq%X{; NFB*PA=nno3{sBt#kT(DT literal 0 HcmV?d00001 From bc16da4d442153f9ddb3533bccdc178a94b203ab Mon Sep 17 00:00:00 2001 From: ncatolico Date: Wed, 6 Mar 2024 13:41:28 -0700 Subject: [PATCH 03/11] change qf to integer type --- ...0_location_year-month-day_logFlags.parquet | Bin 1554 -> 1574 bytes ...0_location_year-month-day_logFlags.parquet | Bin 1554 -> 1574 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/empty_files/aquatroll200/flags/aquatroll200_location_year-month-day_logFlags.parquet b/empty_files/aquatroll200/flags/aquatroll200_location_year-month-day_logFlags.parquet index f07b03791bb127b3a1e403ac2f25831f9d9ac772..4413c8d4362edc1d31073af8df751dc2516d8aac 100644 GIT binary patch delta 503 zcmbQlvy6u`z%j^BjX_3~V4Sjb=6g(A7!_TT9f2SNM7XAdNe~ef08~*9WCVIn-ppcE?^y-pTYzMo zJVB&q2$b*Y>5fk2j$pTebp?aT zKu4!QklTP3Cp)@;gq)CE8R!eP#S5wwOaTe7IuEa$$nc=Dl!&0T6!#p-k|>~3r*x17 XksulZ0v&B9-(WT3s9D+c981LfD+eXd?|2*(*jrcVzoY6O6gEs z;>@t8bN9i0oUk;i%Lt(uG!7lkX0thG_35B&xQ3=~cMrxvRdm>H=h$TvhjGLGM|y%1 za-#0LO2i^w5~p^JZB{sgqi>Nc#A5Rm`oyWyH4$M0Q~(0(w*=!wWSZ`M?AusaEZNY!#b%Ytn9* z1-sJg(&19qdxBafFhJPU;caC=YJ`AEqWw;(sQwxy4eG&0aPxV}Yrmt8%wjv&EmFtq mh%q9B0m1Y6evaa&j=`Vj%*|MilQB4-df(w>)D9yAzy1TpAB92y diff --git a/empty_files/leveltroll500/flags/leveltroll500_location_year-month-day_logFlags.parquet b/empty_files/leveltroll500/flags/leveltroll500_location_year-month-day_logFlags.parquet index f07b03791bb127b3a1e403ac2f25831f9d9ac772..4413c8d4362edc1d31073af8df751dc2516d8aac 100644 GIT binary patch delta 503 zcmbQlvy6u`z%j^BjX_3~V4Sjb=6g(A7!_TT9f2SNM7XAdNe~ef08~*9WCVIn-ppcE?^y-pTYzMo zJVB&q2$b*Y>5fk2j$pTebp?aT zKu4!QklTP3Cp)@;gq)CE8R!eP#S5wwOaTe7IuEa$$nc=Dl!&0T6!#p-k|>~3r*x17 XksulZ0v&B9-(WT3s9D+c981LfD+eXd?|2*(*jrcVzoY6O6gEs z;>@t8bN9i0oUk;i%Lt(uG!7lkX0thG_35B&xQ3=~cMrxvRdm>H=h$TvhjGLGM|y%1 za-#0LO2i^w5~p^JZB{sgqi>Nc#A5Rm`oyWyH4$M0Q~(0(w*=!wWSZ`M?AusaEZNY!#b%Ytn9* z1-sJg(&19qdxBafFhJPU;caC=YJ`AEqWw;(sQwxy4eG&0aPxV}Yrmt8%wjv&EmFtq mh%q9B0m1Y6evaa&j=`Vj%*|MilQB4-df(w>)D9yAzy1TpAB92y From a631793d6824e19ae6aa4906d5a7d26b97d98d3b Mon Sep 17 00:00:00 2001 From: ncatolico Date: Mon, 11 Mar 2024 12:45:51 -0600 Subject: [PATCH 04/11] avro updates --- .../aquatroll200/aquatroll200_log_flags.avsc | 4 +- ...cal_aquatroll200_dp01_quality_metrics.avsc | 120 ++++++++++++++++++ ...quatroll200_dp01_quality_metrics_inst.avsc | 39 ++++++ .../leveltroll500_log_flags.avsc | 4 +- ..._dp01_aquatroll200_specific_ucrt_inst.avsc | 2 +- ...dp01_leveltroll500_specific_ucrt_inst.avsc | 2 +- ...cal_aquatroll200_dp01_quality_metrics.avsc | 120 ++++++++++++++++++ ...quatroll200_dp01_quality_metrics_inst.avsc | 39 ++++++ ...al_leveltroll500_dp01_quality_metrics.avsc | 90 +++++++++++++ ...veltroll500_dp01_quality_metrics_inst.avsc | 30 +++++ 10 files changed, 444 insertions(+), 6 deletions(-) diff --git a/avro_schemas/aquatroll200/aquatroll200_log_flags.avsc b/avro_schemas/aquatroll200/aquatroll200_log_flags.avsc index 7112ac0..6065649 100644 --- a/avro_schemas/aquatroll200/aquatroll200_log_flags.avsc +++ b/avro_schemas/aquatroll200/aquatroll200_log_flags.avsc @@ -34,7 +34,7 @@ "int8" ], "default": null, - "doc": "Quality flag for the log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "doc": "Quality flag for edited log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, { @@ -44,7 +44,7 @@ "int8" ], "default": null, - "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "doc": "Quality flag for the temperature log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, { diff --git a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc index 3379fd8..5c2ecbc 100644 --- a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc +++ b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc @@ -740,6 +740,126 @@ "doc": "Quality metric that summarizes when the persistence test could not be run over the averaging period, as a percent", "__neon_units": "percent" }, + { + "name": "gWatElevLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "gWatElevLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "gWatElevLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the pressure log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDatePassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the edited log data readout time test over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDateFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the edited log data readout time test over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDateNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the edited log data readout time test could not be run, as a percent", + "__neon_units": "percent" + }, + { + "name": "gWatTempLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "gWatTempLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "gWatTempLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the temperature log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "gWatSCondLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "gWatSCondLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "gWatSCondLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the specific conductivity log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, { "name": "missingTempPassQM", "type": [ diff --git a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index 774eccf..7b2ea7a 100644 --- a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -260,6 +260,45 @@ "doc": "Quality flag for the persistence test detailed in NEON.DOC.011081 (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, + { + "name": "gWatElevLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "logDataDateQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for edited log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "gWatTempLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the temperature log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "_ + { + "name": "gWatSCondLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the specific conductiviry log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, { "name": "missingTempQF", "type": [ diff --git a/avro_schemas/leveltroll500/leveltroll500_log_flags.avsc b/avro_schemas/leveltroll500/leveltroll500_log_flags.avsc index ffe6ef4..3ffc45c 100644 --- a/avro_schemas/leveltroll500/leveltroll500_log_flags.avsc +++ b/avro_schemas/leveltroll500/leveltroll500_log_flags.avsc @@ -34,7 +34,7 @@ "int8" ], "default": null, - "doc": "Quality flag for the log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "doc": "Quality flag for edited log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, { @@ -44,7 +44,7 @@ "int8" ], "default": null, - "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "doc": "Quality flag for the temperature log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" } ] diff --git a/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_aquatroll200_specific_ucrt_inst.avsc b/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_aquatroll200_specific_ucrt_inst.avsc index 6340989..54bd531 100644 --- a/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_aquatroll200_specific_ucrt_inst.avsc +++ b/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_aquatroll200_specific_ucrt_inst.avsc @@ -61,7 +61,7 @@ "__neon_units": "microsiemensPerCentimeter" }, { - "name": "surfacewaterColumnHeightNonSysUncert", + "name": "surfacewaterColumnNonSysUncert", "type": [ "null", "float" diff --git a/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_leveltroll500_specific_ucrt_inst.avsc b/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_leveltroll500_specific_ucrt_inst.avsc index c29fabd..1c0e8e6 100644 --- a/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_leveltroll500_specific_ucrt_inst.avsc +++ b/avro_schemas/surfacewaterPhysical/stats_ucrt_grp_avro_schemas/surfacewaterPhysical_dp01_leveltroll500_specific_ucrt_inst.avsc @@ -51,7 +51,7 @@ "__neon_units": "meter" }, { - "name": "surfacewaterColumnHeightNonSysUncert", + "name": "surfacewaterColumnNonSysUncert", "type": [ "null", "float" diff --git a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc index b1c46c5..b5f13ab 100644 --- a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc +++ b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc @@ -740,6 +740,126 @@ "doc": "Quality metric that summarizes when the persistence test could not be run over the averaging period, as a percent", "__neon_units": "percent" }, + { + "name": "sWatElevLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "sWatElevLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "sWatElevLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the pressure log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDatePassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the edited log data readout time test over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDateFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the edited log data readout time test over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDateNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the edited log data readout time test could not be run, as a percent", + "__neon_units": "percent" + }, + { + "name": "sWatTempLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "sWatTempLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "sWatTempLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the temperature log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "sWatSCondLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "sWatSCondLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "sWatSCondLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the specific conductivity log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, { "name": "missingTempPassQM", "type": [ diff --git a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index 497726b..0e25687 100644 --- a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -260,6 +260,45 @@ "doc": "Quality flag for the persistence test detailed in NEON.DOC.011081 (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, + { + "name": "surfacewaterColumnHeightLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "logDataDateQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for edited log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "sWatTempLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the temperature log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "_ + { + "name": "sWatSpecCondLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the specific conductiviry log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, { "name": "missingTempQF", "type": [ diff --git a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics.avsc b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics.avsc index f457573..2af17d6 100644 --- a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics.avsc +++ b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics.avsc @@ -500,6 +500,96 @@ "doc": "Quality metric that summarizes when the persistence test could not be run over the averaging period, as a percent", "__neon_units": "percent" }, + { + "name": "sWatElevLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "sWatElevLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "sWatElevLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the pressure log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDatePassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the edited log data readout time test over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDateFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the edited log data readout time test over the averaging period, as a percent", + "__neon_units": "percent" + }, + { + "name": "logDataDateNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the edited log data readout time test could not be run, as a percent", + "__neon_units": "percent" + }, + { + "name": "sWatTempLogDataPassQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the passed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "__neon_units": "percent" + }, + { + "name": "sWatTempLogDataFailQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric that summarizes the failed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "__neon_units": "percent" + }, + { + "name": "sWatTempLogDataNAQM", + "type": [ + "null", + "float" + ], + "default": null, + "doc": "Quality metric for when the temperature log data summary could not be run over the averaging period, as a percent", + "__neon_units": "percent" + }, { "name": "zeroPressurePassQM", "type": [ diff --git a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics_inst.avsc b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics_inst.avsc index 7c477b8..c097f49 100644 --- a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_leveltroll500_dp01_quality_metrics_inst.avsc @@ -180,6 +180,36 @@ "doc": "Quality flag for the persistence test detailed in NEON.DOC.011081 (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, + { + "name": "surfacewaterColumnHeightLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the pressure log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "logDataDateQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for edited log data readout time (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, + { + "name": "sWatTempLogDataQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the temperature log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, { "name": "zeroPressureQF", "type": [ From 1341340f1629cb4129b17d87dbb311f737b17b32 Mon Sep 17 00:00:00 2001 From: ncatolico Date: Mon, 11 Mar 2024 14:30:20 -0600 Subject: [PATCH 05/11] reorder --- ...cal_aquatroll200_dp01_quality_metrics.avsc | 60 +++++++++---------- ...quatroll200_dp01_quality_metrics_inst.avsc | 20 +++---- ...cal_aquatroll200_dp01_quality_metrics.avsc | 60 +++++++++---------- ...quatroll200_dp01_quality_metrics_inst.avsc | 20 +++---- 4 files changed, 80 insertions(+), 80 deletions(-) diff --git a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc index 5c2ecbc..4048998 100644 --- a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc +++ b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics.avsc @@ -741,153 +741,153 @@ "__neon_units": "percent" }, { - "name": "gWatElevLogDataPassQM", + "name": "missingTempPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "doc": "Quality metric that summarizes the passed outcomes of the missing temperature test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "gWatElevLogDataFailQM", + "name": "missingTempFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "doc": "Quality metric that summarizes the failed outcomes of the missing temperature test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "gWatElevLogDataNAQM", + "name": "missingTempNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the pressure log data summary could not be run over the averaging period, as a percent", + "doc": "Quality metric that summarizes when the missing temperature test could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "logDataDatePassQM", + "name": "gWatElevLogDataPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the edited log data readout time test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the passed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", "__neon_units": "percent" }, { - "name": "logDataDateFailQM", + "name": "gWatElevLogDataFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the edited log data readout time test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the failed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", "__neon_units": "percent" }, { - "name": "logDataDateNAQM", + "name": "gWatElevLogDataNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the edited log data readout time test could not be run, as a percent", + "doc": "Quality metric for when the pressure log data summary could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "gWatTempLogDataPassQM", + "name": "logDataDatePassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "doc": "Quality metric that summarizes the passed outcomes of the edited log data readout time test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "gWatTempLogDataFailQM", + "name": "logDataDateFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "doc": "Quality metric that summarizes the failed outcomes of the edited log data readout time test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "gWatTempLogDataNAQM", + "name": "logDataDateNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the temperature log data summary could not be run over the averaging period, as a percent", + "doc": "Quality metric for when the edited log data readout time test could not be run, as a percent", "__neon_units": "percent" }, { - "name": "gWatSCondLogDataPassQM", + "name": "gWatTempLogDataPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "doc": "Quality metric that summarizes the passed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", "__neon_units": "percent" }, { - "name": "gWatSCondLogDataFailQM", + "name": "gWatTempLogDataFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "doc": "Quality metric that summarizes the failed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", "__neon_units": "percent" }, { - "name": "gWatSCondLogDataNAQM", + "name": "gWatTempLogDataNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the specific conductivity log data summary could not be run over the averaging period, as a percent", + "doc": "Quality metric for when the temperature log data summary could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "missingTempPassQM", + "name": "gWatSCondLogDataPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the missing temperature test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the passed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", "__neon_units": "percent" }, { - "name": "missingTempFailQM", + "name": "gWatSCondLogDataFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the missing temperature test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the failed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", "__neon_units": "percent" }, { - "name": "missingTempNAQM", + "name": "gWatSCondLogDataNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes when the missing temperature test could not be run over the averaging period, as a percent", + "doc": "Quality metric for when the specific conductivity log data summary could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { diff --git a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index 7b2ea7a..86d7023 100644 --- a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -260,6 +260,16 @@ "doc": "Quality flag for the persistence test detailed in NEON.DOC.011081 (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, + { + "name": "missingTempQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the missing temperature test (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, { "name": "gWatElevLogDataQF", "type": [ @@ -299,16 +309,6 @@ "doc": "Quality flag for the specific conductiviry log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, - { - "name": "missingTempQF", - "type": [ - "null", - "int8" - ], - "default": null, - "doc": "Quality flag for the missing temperature test (1=fail, 0=pass, -1=NA (i.e., could not be run))", - "__neon_units": "NA" - }, { "name": "zeroPressureQF", "type": [ diff --git a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc index b5f13ab..b2fcd92 100644 --- a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc +++ b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics.avsc @@ -741,153 +741,153 @@ "__neon_units": "percent" }, { - "name": "sWatElevLogDataPassQM", + "name": "missingTempPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "doc": "Quality metric that summarizes the passed outcomes of the missing temperature test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "sWatElevLogDataFailQM", + "name": "missingTempFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "doc": "Quality metric that summarizes the failed outcomes of the missing temperature test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "sWatElevLogDataNAQM", + "name": "missingTempNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the pressure log data summary could not be run over the averaging period, as a percent", + "doc": "Quality metric that summarizes when the missing temperature test could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "logDataDatePassQM", + "name": "sWatElevLogDataPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the edited log data readout time test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the passed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", "__neon_units": "percent" }, { - "name": "logDataDateFailQM", + "name": "sWatElevLogDataFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the edited log data readout time test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the failed outcomes of the pressure log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", "__neon_units": "percent" }, { - "name": "logDataDateNAQM", + "name": "sWatElevLogDataNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the edited log data readout time test could not be run, as a percent", + "doc": "Quality metric for when the pressure log data summary could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "sWatTempLogDataPassQM", + "name": "logDataDatePassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "doc": "Quality metric that summarizes the passed outcomes of the edited log data readout time test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "sWatTempLogDataFailQM", + "name": "logDataDateFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "doc": "Quality metric that summarizes the failed outcomes of the edited log data readout time test over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "sWatTempLogDataNAQM", + "name": "logDataDateNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the temperature log data summary could not be run over the averaging period, as a percent", + "doc": "Quality metric for when the edited log data readout time test could not be run, as a percent", "__neon_units": "percent" }, { - "name": "sWatSCondLogDataPassQM", + "name": "sWatTempLogDataPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", + "doc": "Quality metric that summarizes the passed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", "__neon_units": "percent" }, { - "name": "sWatSCondLogDataFailQM", + "name": "sWatTempLogDataFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", + "doc": "Quality metric that summarizes the failed outcomes of the temperature log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", "__neon_units": "percent" }, { - "name": "sWatSCondLogDataNAQM", + "name": "sWatTempLogDataNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric for when the specific conductivity log data summary could not be run over the averaging period, as a percent", + "doc": "Quality metric for when the temperature log data summary could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { - "name": "missingTempPassQM", + "name": "sWatSCondLogDataPassQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the passed outcomes of the missing temperature test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the passed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=No log data were used for gap filling)", "__neon_units": "percent" }, { - "name": "missingTempFailQM", + "name": "sWatSCondLogDataFailQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes the failed outcomes of the missing temperature test over the averaging period, as a percent", + "doc": "Quality metric that summarizes the failed outcomes of the specific conductivity log data gap filling test over the averaging period, as a percent (100=log data were used for gap filling 100 percent of the data points)", "__neon_units": "percent" }, { - "name": "missingTempNAQM", + "name": "sWatSCondLogDataNAQM", "type": [ "null", "float" ], "default": null, - "doc": "Quality metric that summarizes when the missing temperature test could not be run over the averaging period, as a percent", + "doc": "Quality metric for when the specific conductivity log data summary could not be run over the averaging period, as a percent", "__neon_units": "percent" }, { diff --git a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index 0e25687..1f008ea 100644 --- a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -260,6 +260,16 @@ "doc": "Quality flag for the persistence test detailed in NEON.DOC.011081 (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, + { + "name": "missingTempQF", + "type": [ + "null", + "int8" + ], + "default": null, + "doc": "Quality flag for the missing temperature test (1=fail, 0=pass, -1=NA (i.e., could not be run))", + "__neon_units": "NA" + }, { "name": "surfacewaterColumnHeightLogDataQF", "type": [ @@ -299,16 +309,6 @@ "doc": "Quality flag for the specific conductiviry log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", "__neon_units": "NA" }, - { - "name": "missingTempQF", - "type": [ - "null", - "int8" - ], - "default": null, - "doc": "Quality flag for the missing temperature test (1=fail, 0=pass, -1=NA (i.e., could not be run))", - "__neon_units": "NA" - }, { "name": "zeroPressureQF", "type": [ From caec493224db3a90aaa5c93e5e95e1170dca92ee Mon Sep 17 00:00:00 2001 From: ncatolico Date: Tue, 19 Mar 2024 10:42:13 -0600 Subject: [PATCH 06/11] typo --- ...ndwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc | 3 ++- ...cewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index 86d7023..65e501d 100644 --- a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -298,7 +298,8 @@ ], "default": null, "doc": "Quality flag for the temperature log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", - "_ + "__neon_units": "NA" + }, { "name": "gWatSCondLogDataQF", "type": [ diff --git a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index 1f008ea..a3a4c95 100644 --- a/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/surfacewaterPhysical/surfacewaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -298,7 +298,8 @@ ], "default": null, "doc": "Quality flag for the temperature log data (1=fail, 0=pass, -1=NA (i.e., could not be run))", - "_ + "__neon_units": "NA" + }, { "name": "sWatSpecCondLogDataQF", "type": [ From 0b245da584730adc81d6ecef327d7f68895a992f Mon Sep 17 00:00:00 2001 From: ncatolico Date: Tue, 19 Mar 2024 11:46:50 -0600 Subject: [PATCH 07/11] add empty files for aquatroll log data --- ...0_location_year-month-day_logFlags.parquet | Bin 1574 -> 1873 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/empty_files/aquatroll200/flags/aquatroll200_location_year-month-day_logFlags.parquet b/empty_files/aquatroll200/flags/aquatroll200_location_year-month-day_logFlags.parquet index 4413c8d4362edc1d31073af8df751dc2516d8aac..308181b2c0ef53f3a9bbc919c70b76d4ade38f44 100644 GIT binary patch literal 1873 zcmds2+fLg+5Z$7RRu+Azgi#{#z)N{5)N3aYD)F$+4cFKZ*A65?svO61FmaMP&W#V~ zH}&&6vks91q3Uy;WG{1OXV1(WE7#1Thh$%TpBK0Dxt-W|yTeKHBe{=!ac7U<=a6VS z1jc=Fi#vTB#r}9aiT%rAP;nEtQG3nvLqGKZFaLF~-NcVZe(Way5>oVrgYLvj?x*+3 z;uQko+kC%Ba@%9C=SQw65b^z|t@Yf9xCN${nWXl0-l=UP%M{wo2`ZQ0&)@xT#Z48l zM|KIxf6l#$bHBEjpa}AbT#yujo}HFTC7~!dcI^meg$k69ML{kWUi_#epMV1#rbPu# z4cy2d3-kn*OBF&ucOTs3ltzar_)D|bri54kjli_;`2l|s} zFy<~Kx5F&!>gH=IyWH=0U+^ihGGnX(KYVn?3QP@Hjj@2018!5*HGGD~>I$y}0OKAW z9);r?V>1Q-%sdFpj8$;puId6;Y5>QoRo$2k7iOqQM#az#`$4&?1ch?w=*^~8zqAA0 zw2gT=tS)CqH$q1$hZ;2*oUVBgW}OwWNKEv0J*?;CIa-ChRrc zi<-T5{fu^NXOeqE`w&+)!YTz$b8|MD0siPI%|+XpPur5!>(*EU)aK>LoOU9MwiIjBjrx-R{7%f(W(zYI zQp1KEAs-`?5AY2;@|<>VijX(24^-K~QU}-@UUUunqw;=W;~2BK2%r&V-o&)&X+IHG z@w^Igix1r!Q*v+HqwYmNn*-toN(tTBcO2B}a<8%;?%5EB^gdjrr2bMD?i5}_lvHk9 zM7^hdL*q*0Nvxy``a)}c)hDcRA8HK7$`hX@zjEmrzcF5z;b-CAyZF}k6!Uq{5Q3{` z_~KkF^^LYwQQ_qQyv5WRGZ27ca=z%p=4fFLIAreQe$TZCE_I9#?<36ofg|VVBzG`! sQYIrcE>@X18%yH%)K) z3d-Dgt*}?Vj87nluOT?!*t&~4Be zk@U2x+U&EmC+#M{zDcd6-KXhL)>2ckU{+f0y4MSu27PMtxzwCnF6~=({x0y8EGlMm zE1BV*9h&M2`&Gb!FgOdSBklhGQP*vG2T!JK5WiE7b)31FXM?4Npkpc0N9O!){)@RUd2-^5-M-3xw>0 Gqx=F-V#=Wa From 86b8c5ad100428fed4db52c9b3dec809f8cbb266 Mon Sep 17 00:00:00 2001 From: ncatolico Date: Tue, 19 Mar 2024 15:51:02 -0600 Subject: [PATCH 08/11] fix typo --- ...oundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc | 1 - 1 file changed, 1 deletion(-) diff --git a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index 65e501d..ba314c9 100644 --- a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -381,7 +381,6 @@ "doc": "Quality flag indicating whether a data product has passed or failed an overall assessment of its quality, detailed in NEON.DOC.001113 (1=fail, 0=pass)", "__neon_units": "NA" }, - , { "name": "gWatSCondAlphaQF", "type": [ From 2f8f420f9200a139158c897c17e8ca446b078e37 Mon Sep 17 00:00:00 2001 From: ncatolico Date: Tue, 19 Mar 2024 16:54:57 -0600 Subject: [PATCH 09/11] one more --- ...oundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc | 1 - 1 file changed, 1 deletion(-) diff --git a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc index ba314c9..33aeece 100644 --- a/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc +++ b/avro_schemas/groundwaterPhysical/groundwaterPhysical_aquatroll200_dp01_quality_metrics_inst.avsc @@ -350,7 +350,6 @@ "doc": "Quality flag indicating whether a data product has passed or failed an overall assessment of its quality, detailed in NEON.DOC.001113 (1=fail, 0=pass)", "__neon_units": "NA" }, - , { "name": "gWatTempAlphaQF", "type": [ From 2a41d750ce0186c9caf31decde8bc1e3188dc126 Mon Sep 17 00:00:00 2001 From: ncatolico Date: Wed, 20 Mar 2024 14:12:05 -0600 Subject: [PATCH 10/11] allow null --- avro_schemas/aquatroll200/aquatroll200_log_data.avsc | 5 ++++- avro_schemas/leveltroll500/leveltroll500_log_data.avsc | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/avro_schemas/aquatroll200/aquatroll200_log_data.avsc b/avro_schemas/aquatroll200/aquatroll200_log_data.avsc index 28762d0..a605a02 100644 --- a/avro_schemas/aquatroll200/aquatroll200_log_data.avsc +++ b/avro_schemas/aquatroll200/aquatroll200_log_data.avsc @@ -15,7 +15,10 @@ }, { "name": "site_id", - "type": "string", + "type": [ + "null", + "string" + ], "doc": "NEON site identifier" }, { diff --git a/avro_schemas/leveltroll500/leveltroll500_log_data.avsc b/avro_schemas/leveltroll500/leveltroll500_log_data.avsc index bc0a2ca..0b7aa4a 100644 --- a/avro_schemas/leveltroll500/leveltroll500_log_data.avsc +++ b/avro_schemas/leveltroll500/leveltroll500_log_data.avsc @@ -15,7 +15,10 @@ }, { "name": "site_id", - "type": "string", + "type": [ + "null", + "string" + ], "doc": "NEON site identifier" }, { From a16b36af25342d02c1268cd17fa0dbefc582aa04 Mon Sep 17 00:00:00 2001 From: ncatolico Date: Mon, 1 Apr 2024 13:45:04 -0600 Subject: [PATCH 11/11] adding null site option --- avro_schemas/aquatroll200/aquatroll200_calibrated.avsc | 5 ++++- avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc | 5 ++++- avro_schemas/leveltroll500/leveltroll500_calibrated.avsc | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc b/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc index 909a6af..1bc65a3 100644 --- a/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc +++ b/avro_schemas/aquatroll200/aquatroll200_calibrated.avsc @@ -15,7 +15,10 @@ }, { "name": "site_id", - "type": "string", + "type": [ + "null", + "string" + ], "doc": "NEON site identifier" }, { diff --git a/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc b/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc index 34b004e..aee2a26 100644 --- a/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc +++ b/avro_schemas/aquatroll200/aquatroll200_cond_corrected.avsc @@ -10,7 +10,10 @@ "fields": [ { "name": "source_id", - "type": "string", + "type": [ + "null", + "string" + ], "doc": "Source serial number or MAC address" }, { diff --git a/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc b/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc index fe4698e..25cb3aa 100644 --- a/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc +++ b/avro_schemas/leveltroll500/leveltroll500_calibrated.avsc @@ -15,7 +15,10 @@ }, { "name": "site_id", - "type": "string", + "type": [ + "null", + "string" + ], "doc": "NEON site identifier" }, {