annotate validate_affy_metadata.py @ 2:72e79ed7ca41 draft

Uploaded
author greg
date Tue, 17 Sep 2019 12:45:57 -0400
parents b5f5c3d0f349
children 7f95329251f0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
80d672b3e6dd Uploaded
greg
parents:
diff changeset
1 #!/usr/bin/env python
80d672b3e6dd Uploaded
greg
parents:
diff changeset
2 """
80d672b3e6dd Uploaded
greg
parents:
diff changeset
3 Validate the metadata file associated with Affymetrix 96 well plate data.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
4 """
80d672b3e6dd Uploaded
greg
parents:
diff changeset
5 import argparse
80d672b3e6dd Uploaded
greg
parents:
diff changeset
6 import datetime
80d672b3e6dd Uploaded
greg
parents:
diff changeset
7 import decimal
80d672b3e6dd Uploaded
greg
parents:
diff changeset
8 import re
80d672b3e6dd Uploaded
greg
parents:
diff changeset
9 import shutil
80d672b3e6dd Uploaded
greg
parents:
diff changeset
10 import sys
80d672b3e6dd Uploaded
greg
parents:
diff changeset
11
80d672b3e6dd Uploaded
greg
parents:
diff changeset
12 parser = argparse.ArgumentParser()
80d672b3e6dd Uploaded
greg
parents:
diff changeset
13 parser.add_argument('--input', dest='input', help='Metadata file for Affymetrix 96 well plate data')
80d672b3e6dd Uploaded
greg
parents:
diff changeset
14 parser.add_argument('--output', dest='output', help='Output dataset'),
80d672b3e6dd Uploaded
greg
parents:
diff changeset
15 args = parser.parse_args()
80d672b3e6dd Uploaded
greg
parents:
diff changeset
16
80d672b3e6dd Uploaded
greg
parents:
diff changeset
17 EMAIL_MAX_LEN = 255
80d672b3e6dd Uploaded
greg
parents:
diff changeset
18 VALID_EMAIL_RE = re.compile("[^@]+@[^@]+\.[^@]+")
80d672b3e6dd Uploaded
greg
parents:
diff changeset
19
80d672b3e6dd Uploaded
greg
parents:
diff changeset
20
80d672b3e6dd Uploaded
greg
parents:
diff changeset
21 def add_error_msg(accumulated_msgs, msg):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
22 return "%s\n%s" % (accumulated_msgs, msg)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
23
80d672b3e6dd Uploaded
greg
parents:
diff changeset
24
80d672b3e6dd Uploaded
greg
parents:
diff changeset
25 def empty_value(line_no, label, accumulated_msgs):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
26 return add_error_msg(accumulated_msgs, "The required %s value is missing on line %d." % (label, line_no))
80d672b3e6dd Uploaded
greg
parents:
diff changeset
27
80d672b3e6dd Uploaded
greg
parents:
diff changeset
28
80d672b3e6dd Uploaded
greg
parents:
diff changeset
29 def stop_error(msg):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
30 sys.exit(msg)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
31
80d672b3e6dd Uploaded
greg
parents:
diff changeset
32
80d672b3e6dd Uploaded
greg
parents:
diff changeset
33 def string_as_boolean_string(string):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
34 if str(string).lower() in ['true', 'yes', 'on', '1']:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
35 return 'True'
80d672b3e6dd Uploaded
greg
parents:
diff changeset
36 else:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
37 return 'False'
80d672b3e6dd Uploaded
greg
parents:
diff changeset
38
80d672b3e6dd Uploaded
greg
parents:
diff changeset
39
80d672b3e6dd Uploaded
greg
parents:
diff changeset
40 def validate_date_string(line_no, date_string, column, accumulated_msgs):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
41 if len(date_string) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
42 return accumulated_msgs
80d672b3e6dd Uploaded
greg
parents:
diff changeset
43 try:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
44 datetime.datetime.strptime(date_string, '%Y-%m-%d')
80d672b3e6dd Uploaded
greg
parents:
diff changeset
45 return accumulated_msgs
80d672b3e6dd Uploaded
greg
parents:
diff changeset
46 except ValueError:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
47 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect date format (%s must be YYYY-MM-DD) for column %s." % (line_no, date_string, column))
80d672b3e6dd Uploaded
greg
parents:
diff changeset
48
80d672b3e6dd Uploaded
greg
parents:
diff changeset
49
80d672b3e6dd Uploaded
greg
parents:
diff changeset
50 def validate_decimal(line_no, decimal_string, column, accumulated_msgs):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
51 try:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
52 decimal.Decimal(decimal_string)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
53 return accumulated_msgs
80d672b3e6dd Uploaded
greg
parents:
diff changeset
54 except Exception:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
55 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect decimal value (%s) for column %s." % (line_no, decimal_string, column))
80d672b3e6dd Uploaded
greg
parents:
diff changeset
56
80d672b3e6dd Uploaded
greg
parents:
diff changeset
57
80d672b3e6dd Uploaded
greg
parents:
diff changeset
58 def validate_email(line_no, email, accumulated_msgs):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
59 if not (VALID_EMAIL_RE.match(email)):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
60 return add_error_msg(accumulated_msgs, "Line %d contains an invalid email address (%s). " % (line_no, email))
80d672b3e6dd Uploaded
greg
parents:
diff changeset
61 elif len(email) > EMAIL_MAX_LEN:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
62 return add_error_msg(accumulated_msgs, "Line %d contains an email address (%) that is longer than the maximum length, %d characters." % (line_no, email))
80d672b3e6dd Uploaded
greg
parents:
diff changeset
63 return accumulated_msgs
80d672b3e6dd Uploaded
greg
parents:
diff changeset
64
80d672b3e6dd Uploaded
greg
parents:
diff changeset
65
2
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
66 def validate_integer(line_no, integer_string, column, accumulated_msgs):
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
67 try:
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
68 integer_string.isdigit()
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
69 return accumulated_msgs
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
70 except Exception:
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
71 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect integer value (%s) for column %s." % (line_no, integer_string, column))
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
72
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
73
0
80d672b3e6dd Uploaded
greg
parents:
diff changeset
74 accumulated_msgs = ""
80d672b3e6dd Uploaded
greg
parents:
diff changeset
75 # Parse the input file, skipping the header, and validating
80d672b3e6dd Uploaded
greg
parents:
diff changeset
76 # that each data line consists of 31 comma-separated items.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
77 with open(args.input, "r") as ih:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
78 for i, line in enumerate(ih):
80d672b3e6dd Uploaded
greg
parents:
diff changeset
79 if i == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
80 # Skip the header.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
81 continue
80d672b3e6dd Uploaded
greg
parents:
diff changeset
82 # Keep 1-based line value for error messages.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
83 line_no = i + 1
80d672b3e6dd Uploaded
greg
parents:
diff changeset
84 line = line.rstrip("\r\n")
80d672b3e6dd Uploaded
greg
parents:
diff changeset
85 items = line.split("\t")
80d672b3e6dd Uploaded
greg
parents:
diff changeset
86 if len(items) != 32:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
87 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains %s columns, (must be 32)." % (line_no, len(items)))
80d672b3e6dd Uploaded
greg
parents:
diff changeset
88 stop_error(accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
89 # Required and validated.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
90 # Required.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
91 user_specimen_id = items[0]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
92 if len(user_specimen_id) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
93 accumulated_msgs = empty_value(line_no, "user_specimen_id", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
94 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
95 field_call = items[1]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
96 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
97 bcoral_genet_id = items[2]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
98 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
99 bsym_genet_id = items[3]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
100 # Required.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
101 reef = items[4]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
102 if len(reef) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
103 accumulated_msgs = empty_value(line_no, "reef", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
104 # Required.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
105 region = items[5]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
106 if len(region) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
107 accumulated_msgs = empty_value(line_no, "region", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
108 # Required and validated.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
109 latitude = items[6]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
110 accumulated_msgs = validate_decimal(line_no, latitude, "latitude", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
111 # Required and validated.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
112 longitude = items[7]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
113 accumulated_msgs = validate_decimal(line_no, longitude, "longitude", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
114 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
115 geographic_origin = items[8]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
116 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
117 colony_location = items[9]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
118 depth = items[10]
2
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
119 # If depth has a value, then it must be integer.
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
120 if len(depth) > 0:
72e79ed7ca41 Uploaded
greg
parents: 1
diff changeset
121 accumulated_msgs = validate_integer(line_no, depth, "depth", accumulated_msgs)
0
80d672b3e6dd Uploaded
greg
parents:
diff changeset
122 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
123 disease_resist = items[11]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
124 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
125 bleach_resist = items[12]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
126 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
127 mortality = items[13]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
128 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
129 tle = items[14]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
130 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
131 spawning = string_as_boolean_string(items[15])
80d672b3e6dd Uploaded
greg
parents:
diff changeset
132 # Required.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
133 collector_last_name = items[16]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
134 if len(collector_last_name) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
135 accumulated_msgs = empty_value(line_no, "collector_last_name", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
136 # Required.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
137 collector_first_name = items[17]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
138 if len(collector_first_name) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
139 accumulated_msgs = empty_value(line_no, "collector_first_name", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
140 # Required.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
141 org = items[18]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
142 if len(org) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
143 accumulated_msgs = empty_value(line_no, "org", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
144 # Required and validated.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
145 collection_date = items[19]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
146 accumulated_msgs = validate_date_string(line_no, collection_date, "collection_date", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
147 # Required and validated.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
148 contact_email = items[20]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
149 accumulated_msgs = validate_email(line_no, contact_email, accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
150 # Required.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
151 seq_facility = items[21]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
152 if len(seq_facility) == 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
153 accumulated_msgs = empty_value(line_no, "seq_facility", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
154 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
155 array_version = items[22]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
156 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
157 public = string_as_boolean_string(items[23])
80d672b3e6dd Uploaded
greg
parents:
diff changeset
158 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
159 public_after_date = items[24]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
160 accumulated_msga = validate_date_string(line_no, public_after_date, "public_after_date", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
161 # Required and validated.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
162 sperm_motility = items[25]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
163 accumulated_msgs = validate_decimal(line_no, sperm_motility, "sperm_motility", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
164 # Required and validated.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
165 healing_time = items[26]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
166 accumulated_msgs = validate_decimal(line_no, healing_time, "healing_time", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
167 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
168 dna_extraction_method = items[27]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
169 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
170 dna_concentration = items[28]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
171 # If dna_concentration has a value, then it must be decimal.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
172 if len(dna_concentration) > 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
173 accumulated_msgs = validate_decimal(line_no, dna_concentration, "dna_concentration", accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
174 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
175 registry_id = items[29]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
176 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
177 result_folder_name = items[30]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
178 # Optional.
80d672b3e6dd Uploaded
greg
parents:
diff changeset
179 plate_barcode = items[31]
80d672b3e6dd Uploaded
greg
parents:
diff changeset
180
80d672b3e6dd Uploaded
greg
parents:
diff changeset
181
80d672b3e6dd Uploaded
greg
parents:
diff changeset
182 if len(accumulated_msgs) > 0:
80d672b3e6dd Uploaded
greg
parents:
diff changeset
183 stop_error(accumulated_msgs)
80d672b3e6dd Uploaded
greg
parents:
diff changeset
184
80d672b3e6dd Uploaded
greg
parents:
diff changeset
185 shutil.copyfile(args.input, args.output)