-
Notifications
You must be signed in to change notification settings - Fork 1
/
NASISpedons-Compare_FGDBschema_against_NASISreport.py
217 lines (164 loc) · 9.12 KB
/
NASISpedons-Compare_FGDBschema_against_NASISreport.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
#-------------------------------------------------------------------------------
# Name: NASISpedons_compareFGDBschema_against_NASISreport.py
# Purpose: This script will compare the table and fields from the input FGDB
# to the tables and fields from the 'WEB Analysis PC MAIN URL Export'
# NASIS report that follows the NASIS 7.1 Schema.
# It is primarily for Jason Nemecek to adjust the pedon reports so that
# they match the 7.1 schema.
#
# Author: Adolfo.Diaz
#
# Created: 04/08/2016
# Last Modified: 9/28/2016
# Copyright: (c) Adolfo.Diaz 2018
#-------------------------------------------------------------------------------
## ===================================================================================
class ExitError(Exception):
pass
## ===================================================================================
def AddMsgAndPrint(msg, severity=0):
# prints message to screen if run as a python script
# Adds tool message to the geoprocessor
#
#Split the message on \n first, so that if it's multiple lines, a GPMessage will be added for each line
try:
print msg
#for string in msg.split('\n'):
#Add a geoprocessing message (in case this is run as a tool)
if severity == 0:
arcpy.AddMessage(msg)
elif severity == 1:
arcpy.AddWarning(msg)
elif severity == 2:
arcpy.AddError(msg)
except:
pass
## ===================================================================================
def errorMsg():
try:
exc_type, exc_value, exc_traceback = sys.exc_info()
theMsg = "\t" + traceback.format_exception(exc_type, exc_value, exc_traceback)[1] + "\n\t" + traceback.format_exception(exc_type, exc_value, exc_traceback)[-1]
AddMsgAndPrint(theMsg,2)
except:
AddMsgAndPrint("Unhandled error in errorMsg method", 2)
pass
# ===============================================================================================================
def splitThousands(someNumber):
# will determine where to put a thousands seperator if one is needed.
# Input is an integer. Integer with or without thousands seperator is returned.
try:
return re.sub(r'(\d{3})(?=\d)', r'\1,', str(someNumber)[::-1])[::-1]
except:
errorMsg()
return someNumber
# =========================================================== Main Body =============================================================================
# Import modules
import sys, string, os, traceback, urllib, re, arcpy
from arcpy import env
if __name__ == '__main__':
try:
pedonID = arcpy.GetParameterAsText(0)
sampleURL = "https://nasis.sc.egov.usda.gov/NasisReportsWebSite/limsreport.aspx?report_name=WEB_AnalysisPC_MAIN_URL_EXPORT&pedonid_list=36186"
#sampleURL = "https://nasis.sc.egov.usda.gov/NasisReportsWebSite/limsreport.aspx?report_name=WEB_AnalysisPC_MAIN_URL_EXPORT&pedonid_list=" + str(pedonID)
pedonGDB = os.path.dirname(sys.argv[0]) + os.sep + 'NasisPedonsTemplate_NEW.gdb'
theReport = urllib.urlopen(sampleURL)
bFieldNameRecord = False # boolean that marks the starting point of the table and attributes.
""" --------------------------------- Parse the report to isolate the NASIS tables and fields -------------------------"""
nasisDict = dict() # collection of table and field names from the URL report {'siteobs': ['seqnum','obsdate','obsdatekind']
tempTableName = "" # name of the current table
# iterate through the report until a valid record is found
for theValue in theReport:
fieldNames = list()
theValue = theValue.strip() # remove whitespace characters
# Iterating through the report
if bFieldNameRecord:
nasisDict[tempTableName] = (theValue.split('|'))
tempTableName = ""
bFieldNameRecord = False
elif theValue.find('@begin') > -1:
bFieldNameRecord = True
tempTableName = theValue[theValue.find('@begin')+7:]
else:
continue
del bFieldNameRecord,tempTableName
if not len(nasisDict):
raise ExitError,"\n\nNo records returned for pedon ID: " + str(pedonID)
""" --------------------------------- Collect FGDB tables and fields -------------------------"""
arcpy.env.workspace = pedonGDB
pedonGDBTables = arcpy.ListTables('*')
if arcpy.ListFeatureClasses('site'):pedonGDBTables.append('site') # Site is not a table so we will manually add it.
missingTables = list()
""" ---------------------------------------- Begin the Comparison between NASIS URL and FGDB -----------------------------"""
discrepancies = 0
for nasisTable in nasisDict.keys():
# Skip any Metadata table
if nasisTable.find("Metadata") > -1: continue
# Check if NASIS table exists in Pedon FGDB; log and continue if it doesn't
if nasisTable not in pedonGDBTables:
missingTables.append(nasisTable)
continue
else:
pedonGDBTables.remove(nasisTable)
gdbTableFields = [f.name for f in arcpy.ListFields(pedonGDB + os.sep + nasisTable)] # List of GDB table fields
nasisFields = [value for value in nasisDict.get(nasisTable)] # List of NASIS table fields
fieldsMissingGDB = list() # List of NASIS fields missing from FGDB
# remove OBJECTID from gdbTableFields list
if 'OBJECTID' in gdbTableFields:
gdbTableFields.remove('OBJECTID')
if 'SHAPE' in gdbTableFields:
gdbTableFields.remove('SHAPE')
gdbTableFieldsTotal = len(gdbTableFields) # number of fields in the FGDB table
nasisFieldsTotal = len(nasisFields) # number of fields in the NASIS table
if gdbTableFieldsTotal == nasisFieldsTotal:
sameNumOfFields = True
else:
sameNumOfFields = False
# Check for NASIS fields missing in the GDB
for nasisField in nasisFields:
if not str(nasisField) in gdbTableFields:
fieldsMissingGDB.append(nasisField)
else:
gdbTableFields.remove(nasisField)
""" -----------------------------------------------Report any tabular field problems to user -------------------------------"""
AddMsgAndPrint("\n=============================================================================================",0)
tablePrinted = False
if not len(fieldsMissingGDB) and not len(gdbTableFields):
AddMsgAndPrint(nasisTable + " Table:",0)
AddMsgAndPrint("\tAll NASIS report fields match FGDB")
continue
# Notify user of missing fields
if len(fieldsMissingGDB):
AddMsgAndPrint(nasisTable + " Table:",2)
AddMsgAndPrint("\tThe following NASIS report fields do NOT exist in the FGDB table:",1)
AddMsgAndPrint("\t\t" + str(fieldsMissingGDB))
tablePrinted = True
discrepancies += len(fieldsMissingGDB)
if len(gdbTableFields):
if not tablePrinted:
AddMsgAndPrint(nasisTable + " Table:",2)
AddMsgAndPrint("\n\tThe following FGDB fields do NOT exist in the NASIS report:",1)
AddMsgAndPrint("\t\t" + str(gdbTableFields))
discrepancies += len(gdbTableFields)
del gdbTableFields,nasisFields,gdbTableFieldsTotal,nasisFieldsTotal,fieldsMissingGDB,tablePrinted
""" ---------------------------------------------------- Report any missing tables to the user ----------------------------------"""
if len(missingTables):
AddMsgAndPrint("\n=============================================================================================",0)
AddMsgAndPrint("The following NASIS report Tables do NOT exist in the FGDB:",2)
missingTables.sort()
AddMsgAndPrint("\t" + str(missingTables))
discrepancies += len(missingTables)
if len(pedonGDBTables):
AddMsgAndPrint("\n=============================================================================================",0)
AddMsgAndPrint("The following FGDB Tables do NOT exist in the NASIS report:",2)
pedonGDBTables.sort()
AddMsgAndPrint("\t" + str(pedonGDBTables))
discrepancies += len(pedonGDBTables)
else:
AddMsgAndPrint("\n\n")
if discrepancies:
AddMsgAndPrint("\nTotal # of discrepancies between NASIS report and FGDB: " + str(splitThousands(discrepancies)) + "\n\n",2)
else:
AddMsgAndPrint("\nThere are no discrepancies between NASIS report and FGDB....CONGRATULATIONS!\n\n",0)
del missingTables,pedonGDBTables
except:
errorMsg()