annotate Matrix_Validations.py @ 1:f1bcd79cd923 draft default tip

Uploaded

1 '''
2 Created on Jun 7, 2017 modified Feb2018
3 
4 @author: Bob Brown and cjacoby
5 '''
6  
7 import sys, traceback, argparse
8 import numpy as np
9 import os
10 from Matrix_Validate_import import reader, Labeler
11 
12 #Define The Four Arguments Used in the Program
13 def get_args():
14     parser = argparse.ArgumentParser()
15     parser.add_argument('input_file_txt', help='tab delimited text file input matrix(include .txt in name)')
16     parser.add_argument('replacement', type=str, help='Choose Replacement for Missing Value. Valid Choices are strings: "Mean" or "Zero"')
17     parser.add_argument('axes', type=str, help='Choose Axes to Normalize On (Either "Row" or "Column"')
18     parser.add_argument('output_file_txt' ,help='tab delimited text file output name (include .txt in name)')
19     args = parser.parse_args()
20     return args
21 
22 
23 #Define Function to Replace Null Values with Row Mean
24 def nan_replacer_mean_rows(matrix):
25 
26     nonNumCnt= 0
27     nanCnt   = 0   #valid NANs are "NA","N/A","-","?"
28 
29     #Loop Replacing all Null Values with Row Mean
30     for i in range(0,len(matrix)):
31         temp_mean = np.nanmean(matrix[i])
32         for j in range(0,len(matrix[0])):
33             #if matrix[i][j] == "NA": #np.isnan(matrix[i][j]) == True:
34             if np.isnan(matrix[i][j]) == True:
35                 matrix[i][j] = temp_mean     
36                 nanCnt = nanCnt + 1
37     return matrix, nonNumCnt, nanCnt
38 
39 #Define Function to Replace Null Values with Column Mean
40 def nan_replacer_mean_columns(matrix):
41 
42     nonNumCnt= 0
43     nanCnt   = 0   #valid NANs are "NA","N/A","-","?"
44     
45     #Loop Replacing all Null Values with Column Mean
46     for i in range(0,len(matrix[0])):
47         col = [row[i] for row in matrix]
48         temp_mean = np.nanmean(col)
49         for j in range(0,len(matrix)):
50             #if matrix[i][j] == "NA": #elif np.isnan(matrix[j][i]) == True:
51             if np.isnan(matrix[j][i]) == True:
52                 matrix[j][i] = temp_mean  
53                 nanCnt = nanCnt + 1   
54     
55     return matrix, nonNumCnt, nanCnt
56 
57 #Define Function to Replace Null Values with Zero (axis orientation is irrelevant)
58 def nan_replacer_zero(matrix):
59 
60     nonNumCnt= 0
61     nanCnt   = 0   #valid NANs are "NA","N/A","-","?"
62     
63     #Loop Replacing all Null Values with Row Range
64     for i in range(0,len(matrix)):
65         for j in range(0,len(matrix[0])):
66             #if matrix[i][j] =="NA":
67             if np.isnan(matrix[i][j]) == True:
68                matrix[i][j] = 0
69 
70     return matrix, nonNumCnt, nanCnt
71 
72 #Define Function to Re-Label Output Matrix
73 #!!!! not needed no output matrix from Validate tool
74 def OLD_labeler(matrix, og_cols, og_rows, output_file_txt): 
75     #Write Data to Specified Text File Output
76     with open(output_file_txt,'w') as f:
77         f.write("Use original input file for further processing\n")
78     f.close()
79 #        f.write("")
80 #         for k in range(0,len(og_cols)):
81 #                 f.write('\t' + str(og_cols[k]))
82 #         f.write('\n')
83 #         for i in range(0,len(og_rows)):
84 #                 f.write(og_rows[i])
85 #                 for j in range(0,len(matrix[0])):
86 #                         f.write('\t' + format(matrix[i][j]))
87 #                 f.write('\n') 
88     
89 #Main Function
90 def main():
91     args = get_args()
92     #print(args)
93     #sys.stdout.write(str(args))
94     #sys.stdout.write( '\nValid NAN identifiers are "NA","N/A","-", and "?"')
95     
96     matrix,og_cols,og_rows = reader(args.input_file_txt)
97 
98 #     if nonNumCnt > 0:
99 #         print('\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
00 #         #sys.stderr.write('\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
01 #         if nanCnt > 0:
02 #             print('\nWARNING Matrix has '+str(nanCnt)+'  that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
03 #         sys.exit(-1)
04 #     else:
05 #         if nanCnt > 0:
06 #             print('\nWARNING Matrix has NO unknown non-numbers in matrix, but contains '+str(nanCnt)+' that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
07 #         else:
08 #             print('Matrix is Good-to-Go -- all numbers in data area. ')
09 
10     #with open(args.output_file_txt,'w') as f:
11     #    f.write("Use original input file for further processing\n")
12     #f.close()
13     #sys.exit(0)
14     
15 # TODO !!!!!  Below if MDA decides to use it  TURNED OFF FOR NOW
16 # TODO !!!!!  Below if MDA decides to use it  TURNED OFF FOR NOW
17 
18     if args.replacement == "Mean":
19         if args.axes == "Row":
20             matrix, nonNumCnt, nanCnt = nan_replacer_mean_rows(matrix)
21             Labeler(matrix,og_cols,og_rows,args.output_file_txt)
22             #OLD_labeler(matrix, og_cols, og_rows, args.output_file_txt)
23             #print('Mean,Row')
24             if nonNumCnt > 0:
25                 print('ERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
26                 sys.stderr.write('ERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
27                 if nanCnt > 0:
28                     print('WARNING Matrix has '+str(nanCnt)+'  that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
29                 sys.exit(-1)
30             else:
31                 if nanCnt > 0:
32                     print('\nWARNING Matrix has '+str(nanCnt)+'  that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
33                 else:
34                     print('\nMatrix is Good-to-Go -- all numbers in matrix. ')
35                 sys.exit(0)
36         elif args.axes == "Column":
37             matrix, nonNumCnt, nanCnt = nan_replacer_mean_columns(matrix)
38             Labeler(matrix,og_cols,og_rows,args.output_file_txt)
39             #OLD_labeler(matrix, og_cols, og_rows, args.output_file_txt)
40             #print('Mean,Column')
41             if nonNumCnt > 0:
42                 print('\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
43                 sys.stderr.write('\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
44                 if nanCnt > 0:
45                     print('\nWARNING Matrix has '+str(nanCnt)+'  that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
46                 sys.exit(-1)
47             else:
48                 if nanCnt > 0:
49                     print('\nWARNING Matrix has '+str(nanCnt)+'  that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
50                 else:
51                     print('\nMatrix is Good-to-Go -- all numbers in matrix. ')
52                 sys.exit(0)
53         else:
54             print('Mean, but given Invalid Axis= '+str(args.axes))
55             sys.stderr.write('Mean, but given Invalid Axis= '+str(args.axes))
56     elif args.replacement == "Zero":
57         matrix, nonNumCnt, nanCnt = nan_replacer_zero(matrix)
58         Labeler(matrix,og_cols,og_rows,args.output_file_txt)
59         #OLD_labeler(matrix, og_cols, og_rows, args.output_file_txt)
60         if nonNumCnt > 0:
61             print('\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
62             sys.stderr.write('\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = '+str(nonNumCnt)+ ',  %.2f' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+'%' )
63             if nanCnt > 0:
64                 print('\nWARNING Matrix has '+str(nanCnt)+'  that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
65             sys.exit(-1)
66         else:
67             if nanCnt > 0:
68                 print('\nWARNING Matrix has '+str(nanCnt)+'  that is  %.2f' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+'% known NAN identifiers')
69             else:
70                 print('\nMatrix is Good-to-Go -- all numbers in matrix. ')
71             sys.exit(0)
72     else:
73         print('zero, but given Invalid Axis= '+str(args.axes))
74         sys.stderr.write('zero, but given Invalid Axis= '+str(args.axes))
75         sys.exit(-2)
76 
77        
78 if __name__ == '__main__':
79     main()
80     print("done")