Mercurial > repos > greg > validate_temperature_data

comparison validate_temperature_data.py @ 0:26415eac98c3 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author greg
date Tue, 27 Nov 2018 11:08:20 -0500
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26415eac98c3
1 #!/usr/bin/env python
2 import argparse
3 import datetime
4 import decimal
5 import re
6 import shutil
7 import sys
8
9 parser = argparse.ArgumentParser()
10 parser.add_argument('--data_type', dest='data_type', default=None, help='Temperature data type, normals or actuals')
11 parser.add_argument('--input_actuals', dest='input_actuals', default=None, help='Daily actuals temperature data')
12 parser.add_argument('--input_normals', dest='input_normals', default=None, help='30 year normals temperature data')
13 parser.add_argument('--output', dest='output', help='Output dataset'),
14 args = parser.parse_args()
15
16 ACTUALS_HEADER = "LATITUDE,LONGITUDE,DATE,DOY,TMIN,TMAX"
17 NORMALS_HEADER = "stationid,latitude,longitude,elev_m,name,st,mmdd,doy,tmin,tmax"
18
19 def add_error_msg(accumulated_msgs, msg):
20 return "%s\n%s" % (accumulated_msgs, msg)
21
22
23 def empty_value(line_no, label, accumulated_msgs):
24 return add_error_msg(accumulated_msgs, "The required %s value is missing on line %d." % (label, line_no))
25
26
27 def stop_error(msg):
28 sys.exit(msg)
29
30
31 def validate_date_string(line_no, date_string, accumulated_msgs):
32 try:
33 datetime.datetime.strptime(date_string, '%Y-%m-%d')
34 return accumulated_msgs
35 except ValueError:
36 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect date format (%s must be YYYY-MM-DD)." % (line_no, date_string))
37
38
39 def validate_decimal(line_no, decimal_string, accumulated_msgs, label):
40 try:
41 decimal.Decimal(decimal_string)
42 return accumulated_msgs
43 except Exception:
44 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect %s decimal value (%s)." % (line_no, label, decimal_string))
45
46
47 def validate_integer(line_no, integer_string, accumulated_msgs, label):
48 if integer_string.isdigit():
49 return accumulated_msgs
50 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect %s integer value (%s)." % (line_no, label, integer_string))
51
52
53 def validate_mmdd(line_no, mmdd, accumulated_msgs):
54 try:
55 datetime.datetime.strptime(mmdd, '%m-%d')
56 return accumulated_msgs
57 except ValueError:
58 # Handle Feb 29.
59 items = mmdd.split("-")
60 try:
61 month = int(items[0])
62 day = int(items[1])
63 if month == 2 and day == 29:
64 return accumulated_msgs
65 except Exception:
66 # Error message accumulated below.
67 pass
68 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect date format (%s must be mm-dd)." % (line_no, mmdd))
69
70
71 accumulated_msgs = ""
72 last_doy = 0
73 # Parse the input file, skipping the header, and validating
74 # that each data line consists of 31 comma-separated items.
75 if args.data_type == "normals":
76 input_file = args.input_normals
77 num_normals_rows = 0
78 else:
79 input_file = args.input_actuals
80 with open(input_file, "r") as ih:
81 for i, line in enumerate(ih):
82 line = line.rstrip("\r\n")
83 items = line.split(",")
84 if args.data_type == "normals":
85 num_normals_rows += 1
86 if i == 0:
87 if line != NORMALS_HEADER:
88 accumulated_msgs = add_error_msg(accumulated_msgs, "The header is invalid, must be %s" % NORMALS_HEADER)
89 continue
90 if i > 367:
91 accumulated_msgs = add_error_msg(accumulated_msgs, "The input file contains more than 367 lines (must be 1 header line and 366 data lines).")
92 stop_error(accumulated_msgs)
93 if len(items) != 10:
94 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains %s columns, (must be 10)." % (i, len(items)))
95 stop_error(accumulated_msgs)
96 stationid = items[0].strip()
97 if len(stationid) == 0:
98 accumulated_msgs = empty_value(i, "stationid", accumulated_msgs)
99 latitude = items[1].strip()
100 accumulated_msgs = validate_decimal(i, latitude, accumulated_msgs, "latitude")
101 longitude = items[2].strip()
102 accumulated_msgs = validate_decimal(i, longitude, accumulated_msgs, "longitude")
103 elev_m = items[3].strip()
104 accumulated_msgs = validate_decimal(i, elev_m, accumulated_msgs, "elev_m")
105 name = items[4].strip()
106 if len(name) == 0:
107 accumulated_msgs = empty_value(i, "name", accumulated_msgs)
108 st = items[5].strip()
109 if len(st) == 0:
110 accumulated_msgs = empty_value(i, "st", accumulated_msgs)
111 mmdd = items[6].strip()
112 accumulated_msgs = validate_mmdd(i, mmdd, accumulated_msgs)
113 doy = items[7].strip()
114 accumulated_msgs = validate_integer(i, doy, accumulated_msgs, "doy")
115 # Make sure the DOY values are consecutive.
116 try:
117 if int(doy) != (last_doy + 1):
118 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains a DOY (%s) that is not conexcutive (previous DOY is %d)." % (i, doy, last_doy))
119 stop_error(accumulated_msgs)
120 else:
121 last_doy += 1
122 except Exception:
123 # The error for an invalid integer was captured above.
124 pass
125 tmin = items[8].strip()
126 accumulated_msgs = validate_decimal(i, tmin, accumulated_msgs, "tmin")
127 tmax = items[9].strip()
128 accumulated_msgs = validate_decimal(i, tmax, accumulated_msgs, "tmax")
129 else:
130 if i == 0:
131 if line != ACTUALS_HEADER:
132 accumulated_msgs = add_error_msg(accumulated_msgs, "The header is invalid, must be %s" % ACTUALS_HEADER)
133 continue
134 if i > 367:
135 accumulated_msgs = add_error_msg(accumulated_msgs, "The input file contains more than 367 lines (must be 1 header line and no more than 366 data lines).")
136 stop_error(accumulated_msgs)
137 if len(items) != 6:
138 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains %s columns, (must be 6)." % (i, len(items)))
139 stop_error(accumulated_msgs)
140 latitude = items[0].strip()
141 accumulated_msgs = validate_decimal(i, latitude, accumulated_msgs, "LATITUDE")
142 longitude = items[1].strip()
143 accumulated_msgs = validate_decimal(i, longitude, accumulated_msgs, "LONGITUDE")
144 date_string = items[2].strip()
145 accumulated_msgs = validate_date_string(i, date_string, accumulated_msgs)
146 doy = items[3].strip()
147 accumulated_msgs = validate_integer(i, doy, accumulated_msgs, "doy")
148 # Make sure the DOY values are consecutive.
149 if i == 1:
150 last_doy = int(doy)
151 else:
152 try:
153 if int(doy) != (last_doy + 1):
154 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains a DOY (%s) that is not conexcutive (previous DOY is %d)." % (i, doy, last_doy))
155 stop_error(accumulated_msgs)
156 else:
157 last_doy += 1
158 except Exception:
159 # The error for an invalid integer was captured above.
160 pass
161 tmin = items[4].strip()
162 accumulated_msgs = validate_decimal(i, tmin, accumulated_msgs, "tmin")
163 tmax = items[5].strip()
164 accumulated_msgs = validate_decimal(i, tmax, accumulated_msgs, "tmax")
165 if args.data_type == "normals" and num_normals_rows != 367:
166 accumulated_msgs = add_error_msg(accumulated_msgs, "The input file contains %d rows, (must be 367)." % num_normals_rows)
167
168 if len(accumulated_msgs) > 0:
169 stop_error(accumulated_msgs)
170
171 shutil.copyfile(input_file, args.output)