annotate Matrix_Validate_import.py @ 1:f1bcd79cd923 draft default tip

Uploaded

1 '''
2 Created on Jun 7, 2017 modified Feb2018
3 
4 @author: cjacoby and Bob Brown
5 '''
6  
7 import sys, traceback, argparse
8 import numpy as np
9 import os
10 #import matplotlib.pyplot as plt
11 #import matplotlib.pyplot as plt; plt.rcdefaults()
12 
13 # Define the Reading Function Which Pulls the Data from a .txt file
14 def reader(input_file_txt, create_plot= False):
15     #Read Matrix, Preserving String Values for Headers first row and first column (both minus first cell) 
16     #Read Matrix, Converting all values to Float for Data Processing    
17         
18     f = open(input_file_txt, "rU")
19 
20     #print( 'Valid NAN identifiers are: empty cells, cells with blanks,"NA","N/A","-", and "?"')
21 
22     column_labels = []
23     row_labels = []
24     matrix  = []
25     firstLine= True
26     
27     line = f.readline()
28 
29 #    "NA","N/A","-","?","NAN","NaN","Na","na","n/a","null",EMPTY/Null, SPACE (blank char) 
30 
31     nanList    = ["", " ","NAN", "NA", "N/A", "-","?"]
32     binCatDict = {"":0, " ":0, "Text":0, "NA":0, "-":0,"NAN":0, "N/A":0,"?":0}
33     row       = 0
34     nanCnt    = 0
35     nonNumCnt = 0
36     
37     while line:
38         line = line.strip("\n")
39         line = line.split('\t')
40 
41         row += 1
42         
43         if firstLine: 
44             lengthRow = len(line)
45             column_labels   = line[1:]
46         else:
47             if lengthRow != len(line):
48                # print("\nERROR matrix row lengths unequal for row 0 and row "+str(row)+"\n" )
49                 sys.exit(-1)
50             
51             temp  = []
52 #            column= 0
53             row_labels.append(str(line[0]))
54            
55             #for item in line[1:]:  use enumerate
56             for column, item in enumerate(line[1:],1):
57 #                column += 1
58                 try:
59                     temp.append(float(item))
60                 except ValueError:
61                     temp.append(np.nan)
62                     itemUC= item.upper()
63                     
64                     if itemUC in nanList:
65                         nanCnt += 1
66                         binCatDict[itemUC]= binCatDict[itemUC]+1
67                        # print( 'Legit nans= ',str(item))
68                     else:
69                         if nonNumCnt == 0:  sys.stderr.write("Start List of up to first 50 Invalid cell values \n")
70                         nonNumCnt +=1
71                         if nonNumCnt < 50:  sys.stderr.write("At row_column= "+str(row)+"_"+str(column)+' invalid data cell value '+ item+"\n")
72                        
73             matrix.append(temp)
74             
75         line = f.readline()
76         firstLine= False
77                    
78     #sys.stdout.write("\n\n")
79     f.close()
80     binCatDict["Text"]= nonNumCnt
81 
82 # plot results of NAN counts above
83 
84     binCat = ["null", "blank", 'hyphen', '?','NA','N/A' ,'NAN', 'text']
85     orderDict= {0:"", 1:"", 2:'-', 3:'?',4:'NA',  5:'N/A' ,6:'NAN', 7:'Text'}
86 #TODO verify dict orde for data    
87         #print("> key value  =",key, str(value))
88 
89     if create_plot:    
90         numBins = len(binCat)
91         binWidth = 1
92         bins     = []
93         binData  = []
94         
95         for key in sorted(orderDict):
96             value= binCatDict[orderDict[key]]   # place items on chart in order and with data value for item
97             if value < 1:
98                 binData.append(value+0.01)
99             else:
00                 binData.append(value)
01 
02         #"""
03         for j in range(numBins):
04             bins.append(j*binWidth)
05     #ttps://pythonspot.com/matplotlib-bar-chart/
06         y_pos = np.arange(numBins)
07         plt.yticks(y_pos, binCat)
08         plt.title("Distribution of NAN types (UPPER & lower & MiXeD case combined)")
09         plt.ylabel('NAN Types')
10         plt.xlabel('Occurrences')
11         #plt.legend()
12         plt.barh(y_pos, binData, align='center', alpha=0.5)
13     
14         fig, ax = plt.subplots(num=1, figsize=(8,3))
15         ax.set_title("Data Cell Counts of Not A Number (NAN) Types")
16         #ax.bar(center,bins, align='center', width=width)
17         #ax.bar(center, hist, align='center', width=width)
18         #ax.set_xticks(bins)
19     #    fig.savefig("/Users/bobbrown/Desktop/Matrix-tools-Test-output/NAN-plot.png")
20         
21     #    fig, ax = plt.subplots(num=1, figsize=(8,3))
22     #    fig.savefig("/Users/bobbrown/Desktop/Matrix-tools-Test-output/hist-out.png")
23         
24         plt.show()
25         #"""
26 
27 #after plot error?
28     x,y=np.shape(matrix)
29     if nanCnt > 0: print("WARNING -- Found "+str(nanCnt)+" Valid Non-numbers. Their percent of total matrix data cell values = "+str((100*nanCnt)/(x*y))+"% ")
30     if nonNumCnt > 0:  sys.exit(-1)
31     #print ("reader output:")
32     #print (matrix)
33     #print (column_labels)
34     #print(row_labels)
35     return matrix,column_labels,row_labels
36 
37 #----------------------------------------------------------------------
38 # Verify Matrix A  column_labels match Matrix B row_labels in name and order for A*B 
39 def MatchLabels(column_labels,row_labels):
40      
41         if len(column_labels) != len(row_labels):
42             sys.err("ERROR 1st matrix column count "+str(len(column_labels))+" not equal 2nd Matrix number row count "+str(len(row_labels))+"\n" )
43         else:
44             cnt= 0
45             for k in range(0,len(column_labels)):
46                 if column_labels[k] != row_labels[k] and cnt < 20:
47                     cnt += 1
48                     #sys.err("ERROR At column & row position "+str(k)+" Matrix 1 column value "+str(column_labels)+" not equal 2nd Matrix row value "+str(row_labels)+"\n" )
49             
50             if cnt > 0: 
51                 sys.exit(-11)
52 #----------------------------------------------------------------------
53 # restores row and column labels in ouput
54 def Labeler(matrix,column_labels,row_labels,output_file_txt):
55     #print("matrix length: " + str(len(matrix)))
56     #print("row labels length: " + str(len(row_labels)))
57     #print("col labels length: " +str(len(column_labels)))
58     #Define Null Sets For Col and Row Headers
59     with open(output_file_txt,'w') as f:
60         f.write("")
61         for k in range(0,len(column_labels)):
62                 f.write('\t' + str(column_labels[k]) )
63         f.write('\n')
64         #for i in range(0,len(row_labels)):
65         for i in range(0,len(matrix)):
66                 f.write(str(row_labels[i]) )
67                 #print("matrix["+str(i)+"] length:" + str(len(matrix[i])))
68                 for j in range(0,len(matrix[0])):
69                         f.write('\t' + format(matrix[i][j]))
70                 f.write('\n')
71 
72 
73 #----------------------------------------------------------------------
74 if __name__ == '__main__':
75     input_file_txt = str(sys.argv[1])
76                                  
77     matrix,column_labels,row_labels = reader(input_file_txt)
78     print("Done")
79