| Previous changeset 0:7f12c81e2083 (2018-11-20) |
|
Commit message:
Uploaded |
|
added:
Matrix_Filters.py Matrix_Filters.xml Matrix_Manipulation.sh Matrix_Manipulation.xml Matrix_Multiply.py Matrix_Multiply.xml Matrix_Statistics.py Matrix_Statistics.xml Matrix_Transformations.py Matrix_Transformations.xml Matrix_Validate_import.py Matrix_Validations.py Matrix_Validations.xml Split_ExcelTabs_IntoFiles.py Split_ExcelTabs_IntoFiles.xml TestOutFile.txt Test_input_into_file.py Test_input_into_file.xml bar_chart_plot.py |
|
removed:
Matrix_Manipulation_Tool_Shed.zip |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Filters.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Filters.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| b'@@ -0,0 +1,694 @@\n+\'\'\'\n+Created on Jun 7, 2017 updated Feb2018\n+\n+@author: rbrown and cjacoby\n+\'\'\'\n+\n+import sys, traceback, argparse\n+import numpy as np\n+from Matrix_Validate_import import reader, Labeler\n+import math\n+#import matplotlib.pyplot as plt\n+\n+#Define argparse Function\n+def get_args():\n+ parser = argparse.ArgumentParser()\n+ parser.add_argument(\'input_file_txt\', help=\'tab delimited text file input matrix(include .txt in name)\')\n+ parser.add_argument(\'choice\',type=str, help=\'Variance Filter Method (Variance or Range)\')\n+ parser.add_argument(\'thresh\', help=\'Thershold for Variance Filtering\')\n+ parser.add_argument(\'axes\', help=\'Axes to Filter on (Either Row or Column\')\n+ parser.add_argument(\'output_file_txt\', help=\'tab delimited text file output name (include .txt in name)\')\n+ args = parser.parse_args()\n+ return args\n+\n+def Range_Filter_Row(matrix,thresh,row_header_list,column_header_list):\n+ #Create Null Set of Filtered Row(Populated Later)\n+ deletes = []\n+ minVal = +9999999\n+ maxVal = -99999\n+ #Loop to Determine Which Rows have sub-Threshold Range\n+ for i in range(0,len(matrix)):\n+ temp_range = np.max(matrix[i][0::]) - np.min(matrix[i][0::])\n+\n+ if temp_range < minVal: minVal = temp_range\n+ elif temp_range > maxVal: maxVal = temp_range\n+\n+ if temp_range <= float(thresh):\n+ deletes = np.append(deletes,[i],0)\n+ \n+ #Delete Rows sub-Threshold Rows \n+ matrix = np.delete(matrix,deletes,0)\n+ filter_rows = np.delete(row_header_list,deletes,0)\n+ filter_cols = column_header_list\n+ return matrix, filter_rows, filter_cols,len(deletes),minVal,maxVal\n+\n+def Range_Filter_Col(matrix,thresh,row_header_list,column_header_list):\n+ #Create Null Set of Filtered Row(Populated Later)\n+ deletes = []\n+ minVal = +9999999\n+ maxVal = -99999\n+ #Loop to Determine Which Rows have sub-Threshold Variance\n+ for i in range(0,len(matrix[0])):\n+ \n+ temp_range = np.max([row[i] for row in matrix]) - np.min([row[i] for row in matrix]) \n+\n+ if temp_range < minVal: minVal = temp_range\n+ elif temp_range > maxVal: maxVal = temp_range\n+ \n+ #print(temp_stdev)\n+ if temp_range <= float(thresh):\n+ deletes = np.append(deletes,[i],0)\n+ print(deletes)\n+\n+ #Delete Rows sub-Threshold Rows \n+ matrix = np.delete(matrix,deletes,1)\n+ filter_rows = row_header_list\n+ filter_cols = np.delete(column_header_list,deletes,0)\n+ #np.savetxt(\'testtest.txt\',matrix,delimiter=\'\\t\')\n+ \n+ return matrix, filter_rows, filter_cols,len(deletes),minVal,maxVal\n+\n+#Define Function Which Deletes Sub-Threshold Rows\n+def Variance_Percent_Filter_row(matrix,cutoff,row_header_list,column_header_list, create_plot= False):\n+# if create a plot then DO NOT remove DATA only print diagram of variance ranges !!!\n+\n+# temp_stdev = np.var(matrix[i][1::])\n+ #cutoff is the percentile rank of the variance values\n+ cutoff= int(cutoff)/100.0\n+ if cutoff > 0.99 or cutoff < .01:\n+ sys.stderr.write( "ERROR illegal cutoff value= "+str(cutoff*100)+" allowed values 1 to 99")\n+ sys.exit(-8)\n+ \n+ deletes = []\n+ varianceDict = {}\n+ minVal = +9999999\n+ maxVal = -99999\n+ \n+ #Loop to Determine Which Rows have sub-Threshold Variance\n+ for i in range(len(matrix)):\n+ vector = []\n+ for p in range(len(matrix[0])):\n+ if not math.isnan(matrix[i][p]): \n+ vector.append(matrix[i][p])\n+ \n+ #temp_stdev = np.var(matrix[:,i])\n+ if len(vector) > 1:\n+ temp_stdev = np.var(vector)\n+ else:\n+ temp_stdev = 0.0\n+ \n+ if temp_stdev < minVal: \n+ minVal = temp_stdev\n+ elif temp_stdev > maxVal: \n+ maxVal = temp_stdev\n+\n+ if temp_stdev not in varianceDict:\n+ varianceDict[temp_stdev] = [i]\n+ else:\n+ '..b'tered out \'+str(delCnt)+\' rows using MAD maximum value > \'+str(threshold))\n+ elif args.axes == "Column":\n+ if args.choice == "MADpercent": threshold= len(column_header_list)*threshold/100.0\n+\n+ matrix, filter_rows, filter_cols,delCnt,maxVal = Col_Value_MAD(matrix,threshold,row_header_list,column_header_list)\n+ Labeler(matrix,filter_cols,filter_rows,args.output_file_txt)\n+ if delCnt < 1:\n+ print(\'\\nNO Filtering occurred for columns using MAD < \'+str(threshold)+ \' by columns. Matrix columns MAD maximum value= %.2f\' % maxVal)\n+ sys.stderr.write(\'\\nFiltering out columns using MAD < \'+str(threshold)+ \' by columns. Matrix columns MAD maximum value= %.2f\' % maxVal)\n+ sys.exit(-1)\n+ else: \n+ print(\'\\nFiltered out \'+str(delCnt)+\' columns using MAD maximum value > \'+str(threshold))\n+#NANlimit\n+ elif args.choice == "NANlimit" or args.choice == "NANpercent": \n+ maxNANs= int(args.thresh)\n+ val= \' \'\n+ if args.choice == "NANpercent":\n+ n,m = np.shape(matrix)\n+ maxNANs= int(int(args.thresh)*n/100)\n+ val= \'%\' \n+ if args.axes == "Row":\n+ matrix, filter_rows, filter_cols,delCnt, maxFoundNANs = NAN_Filter_Row(matrix,nanList,maxNANs,row_header_list,column_header_list)\n+ Labeler(matrix,filter_cols,filter_rows,args.output_file_txt)\n+ if delCnt < 1:\n+ print(\'\\nNO Filtering occurred for rows using NAN limit = or > \'+str(args.thresh)+val+ \' by row. Matrix row max NAN count is =\' + str(maxFoundNANs ))\n+ sys.stderr.write(\'\\nNO Filtering out rows using NAN limit = or > \'+str(args.thresh)+val+ \' by row. Matrix row max NAN count is =\' + str(maxFoundNANs ))\n+ sys.exit(-1)\n+ else: \n+ print(\'\\nFiltered out \'+str(delCnt)+\' rows using NAN limit = or > \'+str(args.thresh)+val)\n+ elif args.axes == "Column":\n+ matrix, filter_rows, filter_cols,delCnt, maxFoundNANs = NAN_Filter_Column(matrix, nanList, maxNANs, row_header_list, column_header_list)\n+ Labeler(matrix,filter_cols,filter_rows,args.output_file_txt)\n+ if delCnt < 1:\n+ print(\'\\nNO Filtering occurred for columns using NAN limit = or > \'+str(args.thresh)+val+ \' by columns. Matrix columns max NAN count is = \'+ str(maxFoundNANs))\n+ sys.stderr.write(\'\\nNO Filtering out columns using NAN limit = or > \'+str(args.thresh)+val+ \' by columns. Matrix columns max NAN count is = \'+ str(maxFoundNANs))\n+ sys.exit(-1)\n+ else: \n+ print(\'\\nFiltered out \'+str(delCnt)+\' columns using NAN limit = or > \'+str(args.thresh)+val )\n+\n+# elif args.choice == "covariance":\n+# if args.axes == "Row":\n+# matrix, filter_rows, filter_cols = CoVariance_Percent_Filter_row(matrix,args.thresh,row_header_list,column_header_list)\n+# Labeler(matrix,filter_rows,filter_cols,args.output_file_txt)\n+# print(\'Covariance_Filter on row\')\n+# elif args.axes == "Column":\n+# matrix, filter_rows, filter_cols = CoVariance_Percent_Filter_col(matrix,args.thresh,row_header_list,column_header_list)\n+# Labeler(matrix,filter_rows,filter_cols,args.output_file_txt)\n+# print(\'Covariance_Filter on column\')\n+ else:\n+ print(\'Invalid Axes = \'+str(args.axes))\n+ sys.exit(-1)\n+ else:\n+ print("Invalid Filter Choice = "+str(args.choice))\n+ sys.exit(-2) \n+ \n+ \n+ except Exception as err:\n+ traceback.print_exc()\n+ sys.exit(-3)\n+\n+if __name__ == \'__main__\':\n+ main()\n+ print("\\ndone")\n+ sys.exit(0)\n' |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Filters.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Filters.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,58 @@ +<?xml version='1.1' encoding='utf-8'?> +<tool id="Matrix_Filters" name="Matrix Filters" version="2.1.12"> + <description> rows or columns based on specified threshold</description> + <command interpreter="python">Matrix_Filters.py '$p_input' '$extra.choice' '$extra.thresh' '$axes' '$output_file'</command> + <inputs> + <param name="p_input" type="data" format="tabular" label="Input Matrix" /> + + <conditional name="extra"> + <param name="choice" type="select" label="Choose Metric to Filter By"> + <option value="LowerLimit">Remove row/column if any cell is less than threshold value</option> + <option value="UpperLimit">Remove row/column if any cell is greater than threshold value</option> + <option value="NANnumber">Remove row/column if number NAN cells exceeds threshold value </option> + <option value="NANpercent">Remove row/column if NAN cells exceeds threshold percent</option> + <option value="VarianceValue">Remove row/column if Variance is below threshold value</option> + <option value="VarianceCount">Remove row/column if Variance is not in top row/column count</option> + <option value="VariancePercent">Remove row/column if Variance is not in top percent</option> + <option value="MADcount">Remove row/column if Median Absolute Deviation (MAD) is not in top row/column count</option> + <option value="MADpercent">Remove row/column if Median Absolute Deviation (MAD) is not in top percent</option> +<!-- <option value="Covariance">Remove row/column if covariance exceeds threshold value</option> + <option value="Median">Remove row/column if Median is less than threshold value</option> + <option value="MADvalue">Remove row/column if Median Absolute Deviation (MAD) is below threshold value</option> + <option value="Mean">Remove row/column if Mean is less than threshold value</option> + <option value="Quantile">Remove row/column if less than Quantile_Cutoff</option> + <option value="GeneList">Keep only these genes</option> + <option value="Histogram">View filter histogram</option> + --> + </param> + <when value="LowerLimit"> + <param name="thresh" type="float" size="5" value="0.01" label="Absolute minimum value" help="Lowest value allowed or row/col removed."/> + </when> + <when value="UpperLimit"> + <param name="thresh" type="float" size="5" value="0.9" label="Absolute maximum value" help="Highest value allowed or row/col removed."/> + </when> + <when value="NANnumber"> + <param name="thresh" type="float" size="5" value="50" label="Maximum number NANs" help="More NANs per row/col removed."/> + </when> + <when value="NANpercent"> + <param name="thresh" type="float" size="5" value="20" label="Percent maximum" help="Above percent rows/cols removed."/> + </when> + <when value="Variance"> + <param name="thresh" type="float" size="5" value="90" label="Percentile minimum" help="Below percentile rows/cols removed."/> + </when> + </conditional> + <!-- + <param name="thresh" type="float" size="4" value=".05" label="Filter Threshold (Value above/below row/column will be Removed)" /> +--> + <param name="axes" type="select" label="Choose Axes to Filter on (Rows or Columns)"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + + + </inputs> + <outputs> + <data name="output_file" format="tabular" /> + </outputs> + + </tool> \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Manipulation.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Manipulation.sh Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| @@ -0,0 +1,42 @@ +#echo "1: " $1 # tool directory +#echo "2: " $2 # manipulation option +#echo "3: " $3 # input file +#echo "4: " $4 # output file +#echo "5: " $5 # choice +#echo "6: " $6 # thresh +#echo "7: " $7 # axis +#echo "8: " $8 # transpose +#echo "9: " $9 # input2 +#echo "10: " ${10} # offsetvalue +#echo "11: " ${11} # scalevalue +#echo "12: " ${12} +#echo "13: " ${13} +#echo "14: " ${14} +#echo "15: " ${15} +#echo "16: " ${16} + +#echo "tool directory is: " $1 +if [ "$2" = "Matrix_Filters" ]; then + echo "filter chosen" + #python $__tool_directory__/Matrix_Filters.py '$p_input '${manipulation.extra.choice}' '${manipulation.extra.thresh}' '${manipulation.axis}' '$output_file' + python $1/Matrix_Filters.py $3 $5 $6 $7 $4 +elif [ "$2" = "Matrix_Multiply" ]; then + echo "multiply chosen" + #python '$__tool_directory__/Matrix_Multiply.py' '$p_input' '${manipulation.extra.transpose}' '${manipulation.extra.input2}' '${manipulation.extra.choice}' '$output_file' + python $1/Matrix_Multiply.py $3 $8 $9 $5 $4 +elif [ "$2" = "Matrix_Statistics" ]; then + echo "statistics chosen" + #python '$__tool_directory__/Matrix_Statistics.py' '$p_input' '$choice' '$cutoff' '$axis' '$out_file' + python $1/Matrix_Statistics.py $3 $5 $6 $7 $4 +elif [ "$2" = "Matrix_Transformations" ]; then + echo "transform chosen" + #python '$__tool_directory__/Matrix_Transformations.py' '$p_input' '$choice' '$axis' '$scalevalue' '$offsetvalue' '$output_file' + python $1/Matrix_Transformations.py $3 $5 $7 ${11} ${10} $4 +elif [ "$2" = "Matrix_Validations" ]; then + echo "validations chosen" + #python '$__tool_directory__/Matrix_Validations.py' '$p_input' '${manipulation.extra.choice}' '${manipulation.extra.axis}' '$output_file' + python $1/Matrix_Validations.py $3 $5 $7 $4 +else + echo "no valid choice made" +fi + |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Manipulation.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Manipulation.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| b'@@ -0,0 +1,406 @@\n+<?xml version=\'1.1\' encoding=\'utf-8\'?>\n+<tool id="Matrix_Manipulation" name="Matrix Manipulation" version="1.2.4">\n+\t <description> Data manipulation before heatmap creation</description>\n+\n+\t<command interpreter="bash" detect_errors="aggressive">\n+\t\t$__tool_directory__/Matrix_Manipulation.sh \'$__tool_directory__\' \'${manipulation.option}\' \'$p_input\' \'$output_file\' \n+\t\t\t\'${manipulation.extra.choice}\' \'${manipulation.extra.thresh}\' \'${manipulation.extra.axis}\' \n+\t\t\t\'${manipulation.extra.transpose}\' \'${manipulation.extra.input2}\' \'${manipulation.extra.offsetvalue}\' \'${manipulation.extra.scalevalue}\'\n+\t\t\t\n+\t</command>\n+\t\n+\t <inputs>\n+\t<param name="p_input" type="data" format="tabular" label="Input Matrix" />\n+\t<conditional name="manipulation">\n+\t\t<param name="option" type="select" label="Choose Manipulation Method">\n+\t\t\t<option value="Matrix_Validations">Missing or Invalid Data</option>\n+\t\t\t<option value="Matrix_Filters">Filter Data</option>\n+\t\t\t<option value="Matrix_Transformations">Transform Data</option>\n+\t\t\t<option value="Matrix_Multiply">Multiply or Correlate Matrices</option>\n+\t\t\t<!-- <option value="Matrix_Statistics">View metadata about Heat Map Matrix</option> -->\n+\t\t\t<!-- <option value="Split_ExcelTabs_IntoFiles">Split Excel tabs into separate tab delimited files</option> -->\n+\t\t</param>\n+\t\n+\t\n+\t\t\t<when value="Matrix_Validations">\n+\t\t\t <conditional name="extra">\n+\t\t\t\t \t<param name="choice" type="select" label="Replace Invalid Value with:">\n+\t\t\t\t \t\t<option value="Mean">Mean</option>\n+\t\t\t\t \t\t<option value="Zero">Zero</option>\n+\t\t\t\t \t</param>\n+\t\t\t\t\t<when value="Mean">\n+\t\t\t\t \t\t<param name="axis" type="select" label="Choose Axis:">\n+\t\t\t\t\t\t\t<option value="Row">Row</option>\n+\t\t\t\t \t\t\t<option value="Column">Column</option>\n+\t\t\t\t \t\t</param>\n+\t\t\t\t \t\t\n+\t\t\t\t \t\t<param name="offsetvalue" type="text" size="10" hidden="true" value="0"/>\n+\t\t\t\t \t\t<param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/>\n+\t\t\t\t \t\t<param name="thresh" type="float" size="4" value=".05" hidden="true" />\n+\t\t\t\t \t\t<param name="transpose" type="text" size="10" hidden="true" value="n" />\n+\t\t\t\t \t\t<param name="input2" type="data" hidden="true" value="$input1" />\n+\t\t\t\t </when>\n+\t\t\t \t\t<when value="Zero">\n+\t\t\t\t \t\t<param name="axis" type="text" size="10" hidden="true" value="Row"/>\n+\t\t\t\t \t\t\n+\t\t\t\t \t\t<param name="offsetvalue" type="text" size="10" hidden="true" value="0"/>\n+\t\t\t\t \t\t<param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/>\n+\t\t\t\t \t\t<param name="thresh" type="float" size="4" value=".05" hidden="true" />\n+\t\t\t\t \t\t<param name="transpose" type="text" size="10" hidden="true" value="n" />\n+\t\t\t\t \t\t<param name="input2" type="data" hidden="true" value="$input1" />\n+\t\t\t\t </when>\n+\t\t\t </conditional>\n+\t\t \t</when>\n+\t\n+\t\n+\t\n+\t\t\t<when value="Matrix_Filters">\n+\t\t\t <conditional name="extra">\n+\t\t\t <param name="choice" type="select" label="Filter Criteria:">\n+\t\t\t \t\t<option value="LowerLimit">Remove row/column if any cell is less than threshold value</option>\n+\t\t\t \t\t<option value="UpperLimit">Remove row/column if any cell is greater than threshold value</option>\n+\t\t\t \t\t<option value="NANlimit">Remove row/column if number of invalid cells exceeds threshold value </option>\n+\t\t\t \t\t<option value="NANpercent">Remove row/column if invalid cells exceeds threshold percent</option>\n+\t\t\t \t\t<!-- <option value="VarianceValue">Remove row/column if Variance is below threshold value</option> -->\n+\t\t\t \t\t<option value="VarianceCount">Remove row/column if Variance is not in top row/column count</option>\n+\t\t\t \t\t<option value="VariancePercent">Remove row/column if Variance is not in top percent</option>\n+\t\t\t \t\t<!-- <option value="MADcount">Remove row/column if Median Absolute Deviation (MAD) is not in top row/column count</option>\n+\t\t\t \t\t<option value="MADpercent">Remove row/column if Median Absolute Deviation (MAD) is not in top percent</option> -->\n+\t\t\t<!-- \t<option value="Covariance">Remove row/column if covarian'..b'\t \t\t<param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/>\n+\t\t\t\t\t \t<param name="thresh" type="float" size="4" value=".05" hidden="true" />\n+\t\t\t\t\t \t<param name="axis" type="text" size="10" hidden="true" value="Row"/>\n+\t\t\t\t\t </when>\n+\t\t\t\t \t <when value="Corr2Matrices">\n+\t\t\t\t\t \t<param name="transpose" type="select" label="transpose matrix?">\n+\t\t\t\t\t\t <option value="y">Transpose matrix</option>\n+\t\t\t\t\t\t <option value="n" selected="true">Do Not transpose matrix</option>\n+\t\t\t\t\t \t</param> \t\n+\t\t\t\t\t \t<param name="input2" type="data" format="tabular" label="Second Input Matrix size NxP" />\n+\t\t\t\t\t \t\n+\t\t\t\t\t \t<param name="offsetvalue" type="text" size="10" hidden="true" value="0"/>\n+\t\t\t\t \t\t<param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/>\n+\t\t\t\t\t \t<param name="thresh" type="float" size="4" value=".05" hidden="true" />\n+\t\t\t\t\t \t<param name="axis" type="text" size="10" hidden="true" value="Row"/>\n+\t\t\t\t\t </when>\n+\t\t\t\t \t <when value="Corr1Matrix">\n+\t\t\t\t\t \t<param name="transpose" type="text" size="10" hidden="true" value="n" />\n+\t\t\t\t\t \t<param name="input2" type="data" hidden="true" value="$input1" />\n+\t\t\t\t\t \t\n+\t\t\t\t\t \t<param name="offsetvalue" type="text" size="10" hidden="true" value="0"/>\n+\t\t\t\t \t\t<param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/>\n+\t\t\t\t\t \t<param name="thresh" type="float" size="4" value=".05" hidden="true" />\n+\t\t\t\t\t \t<param name="axis" type="text" size="10" hidden="true" value="Row"/>\n+\t\t\t\t\t </when>\n+\t\t\t </conditional>\n+\t\t \t</when>\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+\t\t \t<when value="Matrix_Statistics">\n+\t\t\t \t<conditional name="extra">\n+\t\t\t\t <param name="choice" type="select" label="Choose Metric to Filter By">\n+\t\t\t\t \t\t<option value="Histogram">Plot info about either matrix row/column</option>\n+\t\t\t\t \t</param>\t\n+\t\t\t\t \t<when value = "Histogram">\n+\t\t\t\t\t \t<param name="thresh" type="float" size="4" value=".05" label="Filter Threshold (Value above/below row/column will be Removed)" />\n+\t\t\t\t\t \t<param name="axis" type="select" label="Choose axis to Filter on (Rows or Columns)">\n+\t\t\t\t\t \t\t<option value="Row">Row</option>\n+\t\t\t\t\t \t\t<option value="Column">Column</option>\n+\t\t\t\t\t \t</param>\t\n+\t\t\t\t\t \t\n+\t\t\t\t\t \t<param name="offsetvalue" type="text" size="10" hidden="true" value="0"/>\n+\t\t\t\t \t\t<param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/>\n+\t\t\t\t\t \t<param name="transpose" type="text" size="10" hidden="true" value="n" />\n+\t\t\t\t\t \t<param name="input2" type="data" hidden="true" value="$input1" />\n+\t\t\t\t \t</when>\n+\t\t\t \t</conditional>\n+\t\t \t</when>\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+\t\t \t<!-- <when value="Split_ExcelTabs_IntoFiles">\n+\t\t \t\t<description>into separate tab delimited files</description>\n+\t\t\t\t\n+\t\t\t\t \t<param name="input" type="data" format="txt" label="Input Excel.xls or Excel.xlsx file"> </param>\n+\t\t\t\t \t\t\n+\t\t\t\t <conditional name="extra">\n+\t\t\t\t \t <param name="choice" type="select" label="Select tab" help="Select tab position or name">\n+\t\t\t\t\t\t<option value="tab_number">Tab number</option>\n+\t\t\t\t \t </param> \t\n+\t\t\t\t\n+\t\t\t\t\'$extra.tabname\'\n+\t\t\t\t\t\t<option value="tab_name">Tab name</option>\n+\t\t\t\t\t\t<option value="all">All tabs in file</option>\n+\t\t\t\t \t\t<option value="1">First tab only</option>\n+\t\t\t\t \t </param> \t\n+\t\t\t\t <when value="tab_name">\n+\t\t\t\t\t\t<param name="tabname" type="text" size="20" value="" label="Tab Name" help="Spreadsheet Tab Name">\n+\t\t\t\t\t\t</param>\n+\t\t\t\t </when>\n+\t\t\t\t\n+\t\t\t\t <when value="tab_number">\n+\t\t\t\t\t\t<param name="tabnumber" type="text" size="2" value="1" label="Tab position" help="Integer Tab position from left">\n+\t\t\t\t\t\t</param>\n+\t\t\t\t </when>\n+\t\t\t\t </conditional>\n+\t\t\t\t\n+\t\t\t\t <outputs>\n+\t\t\t\t \t<data name="output_file" format="tabular" label="Output File Name" help="Name based on Input file name and tab name"/>\n+\t\t\t\t </outputs>\n+\t\t \t</when> -->\n+\t\t \t\n+\t\t \t\n+\t\t \t\n+ \t</conditional>\n+ \t</inputs>\n+ \t<outputs>\n+\t \t<data name="output_file" format="tabular" />\n+\t </outputs>\n+ </tool>\n\\ No newline at end of file\n' |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Manipulation_Tool_Shed.zip |
| b |
| Binary file Matrix_Manipulation_Tool_Shed.zip has changed |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Multiply.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Multiply.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| @@ -0,0 +1,173 @@ +''' +Created on March 6, 2018 + +@author: Bob Brown based on John Weinstein's algorithm +''' + +import os +import re +import shutil +import traceback +import sys, traceback, argparse +import numpy as np +import warnings +#import scipy.stats as ss +from Matrix_Validate_import import reader, Labeler, MatchLabels +import math +warnings.filterwarnings('error') + +# John Weinsteins algorithm by bob brown https://discover.nci.nih.gov/CorrelateMatrices/help.do +#http://www.blog.pythonlibrary.org/2014/04/30/reading-excel-spreadsheets-with-python-and-xlrd/ + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('input_file1', help='text file input matrix(include .txt in name)') + parser.add_argument('transpose', type=str, help='transpose matrix 1?') + parser.add_argument('input_file2', help='text file input matrix(include .txt in name)') + parser.add_argument('choice', type=str, help='Choose Normalization Method: 1 = Z-score, 2 = Mean Centered, 3 = log2, 4= rank') +# parser.add_argument('scaleValue', help='optional scaling factor for matrix)') + parser.add_argument('out_fileName', help='text file output matrix(include .txt in name)') + args = parser.parse_args() + if args.transpose == "": args.transpose = 'n' + return args + + +def Matrix_Multiply(matrix1, matrix2): + + try: +#TODO handle NANs + + matrixOut= np.dot(matrix1, matrix2) + + + except Exception as err: + traceback.print_exc() + sys.exit(-5) + + return(matrixOut) + + +#CorrelateMatrices correlation acorss 2 martices https://discover.nci.nih.gov/CorrelateMatrices/home.do +def Correlate_Matrices(matrix1, matrix2): + + #try: + # Leave both matrices as size axn and bxn and treat a is column and b as row + #matrix1T = Transpose(matrix1) + +#TODO handle NANs + numRows1,numColumns1= np.shape(matrix1) + + numRows2,numColumns2= np.shape(matrix2) + matrixOut= [] + + if numColumns1 != numRows2: + print("ERROR number columns Matrix 1 ", str(numColumns1), " not equal number rows for Matrix 2 ",str(numRows2)) + sys.exit(-1) +#TODO need to look for NANs?? + + for i in range(numRows1): + vectorM1 = matrix1[i][:] + meanVec1 = np.nanmean(vectorM1) + varStdDev1 = np.nanstd(vectorM1, ddof=1) + lowStdDev1 = False + #if equals zero + if abs(varStdDev1) < .000001: + print("ERROR Variance value almost zero", str(varStdDev1), " for Matrix 1 Row ",str(i+1)) + lowStdDev1= True + correlationRow= [] + + for j in range(numColumns2): + vectorM2 = [] + for t in range(numRows2): + vectorM2.append(matrix2[t][j]) + meanVec2 = np.nanmean(vectorM2) + varStdDev2 = np.nanstd(vectorM2, ddof=1) + lowStdDev2= False + #if equals zero + if abs(varStdDev2) < .000001: + print("ERROR Variance value almost zero", str(varStdDev2), " for Matrix 2 Column ",str(j+1)) + lowStdDev2= True + + covarStdDev12= 0 + + if not lowStdDev1 and not lowStdDev2: + #try: + for pos in range(len(vectorM1)): + covarStdDev12 += ((vectorM1[pos]-meanVec1)/varStdDev1)*((vectorM2[pos]-meanVec2)/varStdDev2) +# bottom= (numColumns1 -1)*(varStdDev1*varStdDev2) +# correlationRow.append( covarStdDev12/bottom) + correlationRow.append( covarStdDev12/(numColumns1 -1)) + #except: bad value because of NAN or other + else: + correlationRow.append("divide by 0") # cannot calculate correlation var too small + + matrixOut.append(correlationRow) + +# except Exception as err: +# traceback.print_exc() +# sys.exit(-6) + + return(matrixOut) + +#---------------------------------------------------------------------- +def Transpose(in_mat): + out_mat = [] + numRows,numColumns= np.shape(in_mat) + + for i in range(numColumns): + temp= [] + for j in range(numRows): + temp.append(in_mat[j][i]) + out_mat.append(temp) + #print( str(out_mat)) + return out_mat + + +#---------------------------------------------------------------------- +if __name__ == "__main__": + +# input_file1 = "/Users/bobbrown/Desktop/Gene-by-var.txt" +# input_file2 = "/Users/bobbrown/Desktop/var-by-sample.txt" +# out_fileName = "/Users/bobbrown/Desktop/MatixMult-1-2-Out.txt" +# selection = "MatrixMultiply" +#TODO address NANs ??? + + try: + args = get_args() + selection= args.choice + + matrix1,column_labels1,row_labels1 = reader(args.input_file1) # to be transposed later + matrix2,column_labels2,row_labels2 = reader(args.input_file2) + + + if args.transpose == 'y' or args.input_file1 == args.input_file2: + matrix1 = Transpose(matrix1) + print("\n>>>NOTICE Transposed first matrix so matrix 1 columns = Matrix 2 number rows ") + temp = row_labels1 #swap labels for output matrix + row_labels1 = column_labels1 #swap labels for output matrix + column_labels1= temp #swap labels for output matrix + + MatchLabels(column_labels1,row_labels2) # verfiy labels and their order match + + if len(column_labels1) != len(row_labels2): + print("\n>>> ERROR attempting to multiple Matrices of incompatible dimensions ") + print("First Matrix is "+str(len(row_labels1))+" by "+str(len(column_labels1))+" where second Matrix is "+str(len(og_row2))+" by "+str(len(column_labels2))+"\n") + print("Matrices must have dimensions AxB and BxC. A can equal C (square matrices)") + sys.exit(-1) + + if selection == "MatrixMultiply": + matrixOut= Matrix_Multiply(matrix1, matrix2 ) + + elif selection == "Corr2Matrices" or selection == "Corr1Matrix": + matrixOut = Correlate_Matrices(matrix1, matrix2) + + Labeler(matrixOut,column_labels2,row_labels1,args.out_fileName) + + print("Matrix Multiply "+str(len(row_labels1))+" by "+str(len(column_labels1))+" Matrix 1 by "+str(len(row_labels2))+" by "+str(len(column_labels2))+" matrix 2") + print("Output Matrix dimensions are "+str(len(row_labels1))+" by "+str(len(column_labels2))+"\n") + + except Exception as err: + traceback.print_exc() + sys.exit(-3) + + sys.exit(0) \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Multiply.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Multiply.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,52 @@ +<?xml version='1.1' encoding='utf-8'?> +<tool id="Matrix_Multiply" name="Matrix Compose" version="1.2.4"> + <description> one matrix using one or two matrices</description> + <command interpreter="python">Matrix_Multiply.py '$extra.input1' 'extra.$transpose' 'extra.$input2' '$extra.choice' '$output_file'</command> + + <inputs> + <conditional name="extra"> + <param name="choice" type="select" label="Choose composition type"> + <option value="MatrixMultiply">Matrix Multiplication</option> + <option value="Corr2Matrices">Correlate two matrices</option> + <option value="Corr1Matrix">Correlate one matrix with itself</option> + </param> + + <when value="MatrixMultiply"> + <param name="input1" type="data" format="tabular" label="First Input Matrix size NxM" /> + <param name="transpose" type="select" label="transpose matrix?"> + <option value="y">Transpose matrix</option> + <option value="n" selected="true">Do Not transpose matrix</option> + </param> + <param name="input2" type="data" format="tabular" label="Second Input Matrix size NxP" /> + </when> + <when value="Corr2Matrices"> + <param name="input1" type="data" format="tabular" label="First Input Matrix size NxM" /> + <param name="transpose" type="select" label="transpose matrix?"> + <option value="y">Transpose matrix</option> + <option value="n" selected="true">Do Not transpose matrix</option> + </param> + <param name="input2" type="data" format="tabular" label="Second Input Matrix size NxP" /> + </when> + <when value="Corr1Matrix"> + <param name="input1" type="data" format="tabular" label="Input Matrix" /> + <param name="transpose" type="text" size="10" hidden="true" value="n" /> + <param name="input2" type="data" hidden="true" value="$input1" /> + </when> + </conditional> + + +<!-- <param name="thresh" type="float" size="4" value=".05" label="Filter Threshold (Axes with Metric below Threshold will be Removed)" /> + <param name="transpose" type="select" multiple="false" display="checkboxes" label="transpose first matrix?"> + + <param name="axes" type="select" label="Chose Axes to Filter on (Rows or Columns)"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + --> + </inputs> + + <outputs> + <data name="output_file" format="tabular" /> + </outputs> + + </tool> \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Statistics.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Statistics.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| @@ -0,0 +1,145 @@ +''' +Created on Feb2018 + +@author: bob brown +''' + +import sys, traceback, argparse +import numpy as np +from Matrix_Validate_import import reader +#import matplotlib.pyplot as plt +from Matrix_Filters import Variance_Percent_Filter_row, Variance_Percent_Filter_col + +#Define argparse Function +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('input_file_txt', help='tab delimited text file input matrix(include .txt in name)') + parser.add_argument('choice',type=str, help='Variance Filter Method (Variance or Range)') + parser.add_argument('thresh', help='Thershold for Variance Filtering') + parser.add_argument('axes', help='Axes to Filter on (Either Row or Column') + parser.add_argument('output_file_txt', help='tab delimited text file output name (include .txt in name)') + args = parser.parse_args() + return args + + +#Define Function Which Labels Rows/Columns on Output +def labeler(matrix,filter_rows,filter_cols,output_file_txt): + + #Write Data to Specified Text File Output + with open(output_file_txt,'w') as f: + f.write("") + for k in range(0,len(filter_cols)): + f.write('\t' + filter_cols[k]) + f.write('\n') + for i in range(0,len(filter_rows)): + f.write(filter_rows[i]) + for j in range(0,len(matrix[0])): + f.write('\t' + format(matrix[i][j])) + f.write('\n') + + +def Histo(matrix): + numBins= 20 + data = [] +# numRow,numCol= np.shape(matrix) + for i in range(len(matrix[0])): + data.append(np.nanmean([row[i] for row in matrix])) + +# print(str(np.nanmean([row[i] for row in matrix]))) + +#https://stackoverflow.com/questions/5328556/histogram-matplotlib + #bins = [0, 40, 60, 75, 90, 110, 125, 140, 160, 200] + minBin = int(min(data)-0.5) + maxBin = int(max(data)+0.5) + binWidth = float(maxBin-minBin)/numBins + bins= [] + """ + for j in range(numBins): + bins.append(minBin+ j*binWidth) + #bins= 20 + n, bins, patches = plt.hist(data,bins, normed=False) + #n, bins, patches = plt.hist(data,bins, normed=1, color='green') + #hist, bins = np.histogram(data, bins=bins) + width = np.diff(bins) + center = (minBin + bins[1:]) / 2 + + cm = plt.cm.get_cmap('RdYlBu_r') + #col = (n-n.min())/(n.max()-n.min()) + for c, p in zip(bins, patches): + plt.setp( p, 'facecolor', cm(c/numBins)) + fig, ax = plt.subplots(num=1, figsize=(8,3)) + ax.set_title("Distribution of Column Means") + #ax.bar(center,bins, align='center', width=width) + #ax.bar(center, hist, align='center', width=width) + #ax.set_xticks(bins) +# fig.savefig("/Users/bobbrown/Desktop/Matrix-tools-Test-output/Column_Mean_Histogram.png") + + plt.show() + """ + return() + +#========== test create variable number output files in Galaxy +def CreateFiles(output_file_info): + + for i in range(3): + fd= open( output_file_info, 'w') + fd.write('File number = '+ str(i)+"\n") + fd.close() + + return() + +#================== + + #Define Main Function +def main(): + try: + args = get_args() + #sys.stdout.write(str(args)+"\n") + nanList= ["NAN", "NA", "N/A", "-","?","nan", "na", "n/a"] + + matrix, og_cols,og_rows = reader(args.input_file_txt) + #old_reader matrix, og_rows, og_cols = reader(args.input_file_txt) +# if float(args.thresh) < 0.000001: +# print('Invalid negative threshold chosen = '+str(args.thresh)+" choose positive value") +# sys.exit(-4) + + if args.choice == "Histogram": + Histo(matrix) + elif args.choice == "CreateFiles": + CreateFiles(args.output_file_info) + + elif args.choice == "Variance": + if args.axes == "Row": + matrix, filter_rows, filter_cols,delCnt,minVal,maxVal = Variance_Percent_Filter_row(matrix,1,og_rows,og_cols,True) + labeler(matrix,filter_rows,filter_cols,args.output_file_txt) +# if delCnt < 1: +# print('\nNO Filtering occurred for rows using variance < '+str(args.thresh)+ ' by row. Matrix row minimum variance= %.2f' % minVal+' and maximum variance= %.2f' % maxVal) +# sys.stderr.write('\nFiltering out rows using variance < '+str(args.thresh)+ ' removed '+str(delCnt)+' rows') +# sys.exit(-1) +# else: +# print('\nFiltering out rows using variance < '+str(args.thresh)+ ' removed '+str(delCnt)+' rows') + elif args.axes == "Column": + matrix, filter_rows, filter_cols,delCnt,minVal,maxVal = Variance_Percent_Filter_col(matrix,1,og_rows,og_cols,True) + labeler(matrix,filter_rows,filter_cols,args.output_file_txt) +# if delCnt < 1: +# print('\nNO Filtering occurred for columns using variance < '+str(args.thresh)+ ' by columns. Matrix columns minimum variance= %.2f' % minVal+' and maximum variance= %.2f' % maxVal) +# sys.stderr.write('\nFiltering out rows using variance < '+str(args.thresh)+ ' removed '+str(delCnt)+' rows') +# sys.exit(-1) +# else: +# print('\nFiltering out columns using variance < '+str(args.thresh)+ ' removed '+str(delCnt)+' columns') + else: + print('Invalid Axes = '+str(args.axes)) + sys.exit(-1) + else: + print("Invalid Filter Choice = "+str(args.choice)) + sys.exit(-2) + + + except Exception as err: + traceback.print_exc() + sys.exit(-3) + +if __name__ == '__main__': + main() + print("\nFini") + sys.exit(0) \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Statistics.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Statistics.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,23 @@ +<?xml version='1.1' encoding='utf-8'?> +<tool id="Matrix_Statistics" name="Matrix Statistics" version="1.1.5" force_history_refresh="True"> + <description>View metadata about Heat Map Matrix</description> + <command interpreter="python">Matrix_Statistics.py '$input' '$choice' '$cutoff' '$axes' '$out_file'</command> + <inputs> + <param name="choice" type="select" label="Choose Metric to Filter By"> + <option value="Histogram">Plot info about either matrix row/column</option> + </param> + + <param name="input" type="data" format="tabular" label="Heat Map Matrix" /> + + <param name="cutoff" type="float" size="4" value=".05" label="Filter Threshold (Value above/below row/column will be Removed)" /> + + <param name="axes" type="select" label="Choose Axes to Filter on (Rows or Columns)"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </inputs> + <outputs> + <data name="out_file" format="tabular" /> + </outputs> + + </tool> \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Transformations.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Transformations.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| b'@@ -0,0 +1,301 @@\n+\'\'\'\n+Created on Jun 6, 2017 updated Feb 2018\n+\n+@author: cjacoby and Bob Brown\n+\'\'\'\n+import os\n+import sys, traceback, argparse\n+import numpy as np\n+from numpy import size, array\n+import warnings\n+from Matrix_Validate_import import reader\n+#import scipy.stats as ss\n+warnings.filterwarnings(\'error\')\n+\n+#Define argparse Function\n+def get_args():\n+ parser = argparse.ArgumentParser()\n+ parser.add_argument(\'input_file_txt\', help=\'text file input matrix(include .txt in name)\')\n+ parser.add_argument(\'choice\', type=str, help=\'Choose normalization Method: 1 = Z-score, 2 = Mean Centered, 3 = log2, 4= rank\')\n+ parser.add_argument(\'axes\', type=str, help=\'Choose Axis to normalize On (Row or Column)\')\n+ parser.add_argument(\'scalevalue\', help=\'optional scaling factor for matrix)\')\n+ parser.add_argument(\'offsetvalue\', help=\'optional offset for matrix\')\n+ parser.add_argument(\'output_file_txt\', help=\'text file output matrix(include .txt in name)\')\n+ args = parser.parse_args()\n+ return args\n+\n+\n+def Zscore_row(matrix):\n+\n+ #Loop To Perform Z-Score normalization\n+ for i in range(0,len(matrix)):\n+ temp_mean = np.nanmean(matrix[i])\n+ temp_stdev = np.nanstd(matrix[i],ddof=1)\n+ for j in range(0,len(matrix[0])):\n+ matrix[i][j] = (matrix[i][j]-temp_mean)/temp_stdev\n+ return(matrix)\n+\n+#Define Z-Score normalization Function\n+def Zscore_col(matrix):\n+\n+ #Loop To Perform Z-Score normalization\n+ for i in range(len(matrix[0])):\n+# matrix[:][i] = [scaleValue*x+offset for x in matrix[i]] \n+ temp_mean = np.nanmean([row[i] for row in matrix])\n+ temp_stdev = np.nanstd([row[i] for row in matrix],ddof=1)\n+ #Probably Should Have if statement checking if stdev equals zero, although this implies the data is already Z-score normalized\n+ for j in range(len(matrix)):\n+ matrix[j][i] = (matrix[j][i]-temp_mean)/temp_stdev\n+ return(matrix)\n+\n+\n+#Define Mean Centered or Median centered normalization Function\n+def MeanMedianCenter_row(matrix,type):\n+\n+ \n+ #Loop To Perform mean or median center\n+ for i in range(0,len(matrix)):\n+ if type == "mean": \n+ temp_type = np.nanmean(matrix[i][1::])\n+ else:\n+ temp_type = np.nanmedian(matrix[i][1::])\n+ \n+ for j in range(0,len(matrix[0])):\n+ matrix[i][j] = (matrix[i][j]-temp_type)\n+ return(matrix)\n+\n+\n+#Define mean or median\n+def MeanMedianCenter_col(matrix,type):\n+\n+ #Loop To Perform mean or median center\n+ for i in range(0,len(matrix[0])):\n+ if type == "mean": \n+ temp_type = np.nanmean([row[i] for row in matrix])\n+ else:\n+ temp_type = np.nanmedian([row[i] for row in matrix])\n+ #Probably Should Have if statement checking if stdev equals zero, although this implies the data is already Z-score normalized\n+ for j in range(0,len(matrix)):\n+ matrix[j][i] = (matrix[j][i]-temp_type)\n+ return(matrix)\n+\n+#Divide by sum of the Row Function\n+def Divide_By_Sum_row(matrix):\n+ \n+ #Loop To Perform mean or median center\n+ numRow,numCol= np.shape(matrix)\n+ \n+ for i in range(numRow):\n+ sumValue = sum(matrix[i][:])\n+\n+ #if equals zero\n+ if abs(sumValue) > .0001:\n+ for j in range(numCol):\n+ matrix[i][j] = matrix[i][j]/sumValue\n+ else: \n+ print("ERROR Cannot divide by Sum almost zero", str(sumValue), " for Row ",str(i+1))\n+ return(matrix)\n+\n+\n+#Divide by sum of the Column Function\n+def Divide_By_Sum_col(matrix):\n+\n+ #Loop To Perform mean or median center\n+ numRow,numCol= np.shape(matrix)\n+ \n+ for i in range(numCol):\n+ sumValue= 0\n+\n+ #if equals zero\n+ if abs(sumValue) > .0001:\n+ for j in range(numRow):\n+ matrix[j][i] = (matrix[j][i]/sumValue)\n+ else: \n+ print("ERROR Cannot divide by Sum almost zero", str(sumValue), '..b' matrix = MeanMedianCenter_row(matrix,"median")\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("median-center by row")\n+ elif args.axes == "Column":\n+ matrix = MeanMedianCenter_col(matrix,"median")\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("median-center by column")\n+ else:\n+ print("meancenter, invalid axis")\n+ elif args.choice == "add_offset":\n+ if args.axes == "Row":\n+ #offset = -100 #!!!! TODO REMOVE AND ADD WHEN clause to xml to get value \n+ matrix = ScaleOffset_row(matrix,1.0,offsetValue)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("offset of "+str(offsetValue)+" by row")\n+ elif args.axes == "Column":\n+ matrix = ScaleOffset_col(matrix,1.0,offsetValue)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("offset of "+str(offsetValue)+" by column")\n+ else:\n+ print("offset"+str(offsetValue)+" invalid axis -not row or column")\n+ elif args.choice == "scale":\n+ if args.axes == "Row":\n+ #scaleValue = 1000 #!!!! TODO REMOVE AND ADD WHEN clause to xml to get value\n+ matrix = ScaleOffset_row(matrix,scaleValue,0.0)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("scaling "+str(scaleValue)+" by row")\n+ elif args.axes == "Column":\n+ matrix = ScaleOffset_col(matrix,scaleValue,0.0)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("scaling "+str(scaleValue)+" by column")\n+ else:\n+ print("scaling "+str(scaleValue)+" invalid axis")\n+ elif args.choice == "transpose":\n+ matrix = Transpose(matrix) #issue using same matrix? \n+ labeler(matrix,og_rows,og_cols,args.output_file_txt) #swapped row&col labels\n+ print("transpose mxn matrix to nxm size")\n+ elif args.choice == "ln_normalization":\n+ matrix = Convert2Logs(matrix,"log2",offsetValue)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("log2 plus "+str(offsetValue)+" normalization for all values")\n+ elif args.choice == "log_normalization":\n+ matrix = Convert2Logs(matrix,"log10",offsetValue)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("log10 normalization for all values")\n+ elif args.choice == "rank":\n+ if args.axes == "Row":\n+ matrix = Rankdata_ByRow(matrix)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("performed rank normalization by row")\n+ elif args.axes == "Column":\n+ matrix = Rankdata_ByColumn(matrix)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("performed rank normalization by column")\n+ else:\n+ print("rank, invalid axis")\n+ elif args.choice == "divide_by_sum":\n+ if args.axes == "Row":\n+ matrix = Divide_By_Sum_row(matrix)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("performed divide row N values by row N\'s sum")\n+ elif args.axes == "Column":\n+ matrix = Divide_By_Sum_col(matrix)\n+ labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ print("performed divide column N values by column N\'s sum")\n+ else:\n+ print("divide_by_sum, invalid axis")\n+\n+ else:\n+ print("Invalid normalization Choice")\n+ \n+ except Exception as err:\n+ traceback.print_exc()\n+ sys.exit(1)\n+\n+\n+if __name__ == \'__main__\':\n+ main()\n+ print("Done")\n' |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Transformations.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Transformations.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,106 @@ +<?xml version='1.1' encoding='utf-8'?> +<tool id="Matrix_Transformations" name="Matrix Transformations" version="2.2"> + <description> by Rows, Columns, All by method</description> + <command interpreter="python">Matrix_Transformations.py '$p_input' '$extra.choice' '$extra.axes' '$extra.scalevalue' '$extra.offsetvalue' '$output_file'</command> + + <inputs> + <conditional name="extra"> + <param name="choice" type="select" label="Choose Normalization Method"> + <option value="mean_center_normalization">Mean Centered Normalization</option> + <option value="median_center_normalization">Median Centered Normalization</option> + <option value="ln_normalization">Log2 Normalization w offset</option> + <option value="log_normalization">Log10 Normalization w offset</option> + <option value="z_score_normalization">Z-Score Normalization</option> + <option value="transpose">Matrix Transpose</option> + <option value="add_offset">Add Offset Value</option> + <option value="scale">Scale by Multiple</option> + <option value="divide_by_sum">Divide by Sum</option> + </param> + <when value="add_offset"> + <param name="offsetvalue" type="text" size="10" value="0.0" label="Matrix Values offset" help="Add value to each Matrix element."/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="text" size="10" hidden="true" value="neither"/> + </when> + <when value="scale"> + <param name="scalevalue" type="text" size="10" value="1.0" label="Matrix Value scaling" help="multiply value with each Matrix element."/> + <param name="offsetvalue" type="text" size="10" hidden="true" value="0"/> + <param name="axes" type="text" size="10" hidden="true" value="neither"/> + </when> + <when value="ln_normalization"> + <param name="offsetvalue" type="text" size="10" value="0.0" label="Matrix Values offset" help="Calculate log2 for each Matrix element then add offset."/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="select" label="Choose Axis"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </when> + <when value="log_normalization"> + <param name="offsetvalue" type="text" size="10" value="0.0" label="Matrix Values offset" help="Calculate log10 for each Matrix element then add offset."/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="select" label="Choose Axis"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </when> + <when value="z_score_normalization"> + <param name="offsetvalue" type="text" size="10" hidden="true" value="0"/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="select" label="Choose Axis"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </when> + <when value="mean_center_normalization"> + <param name="offsetvalue" type="text" size="10" hidden="true" value="0"/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="select" label="Choose Axis"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </when> + <when value="median_center_normalization"> + <param name="offsetvalue" type="text" size="10" hidden="true" value="0"/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="select" label="Choose Axis"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </when> + <when value="transpose"> + <param name="offsetvalue" type="text" size="10" hidden="true" value="0"/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="text" size="10" hidden="true" value="neither"/> + </when> + <when value="divide_by_sum"> + <param name="offsetvalue" type="text" size="10" hidden="true" value="0"/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + <param name="axes" type="select" label="Choose Axis"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </when> + </conditional> + + <param name="p_input" type="data" format="tabular" label="Input Matrix"> </param> + +<!-- need scipy!!!! + <option value="rank">Rank Normalization</option> + <when value="rank"> + <param name="offsetvalue" type="text" size="10" hidden="true" value="0"/> + <param name="scalevalue" type="text" size="10" hidden="true" value="1.0"/> + </when> + + <param name="axes" type="select" label="Choose Axis"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> +--> + </inputs> + + + + <outputs> + <data name="output_file" format="tabular" /> + </outputs> + + </tool> \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Validate_import.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Validate_import.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| @@ -0,0 +1,179 @@ +''' +Created on Jun 7, 2017 modified Feb2018 + +@author: cjacoby and Bob Brown +''' + +import sys, traceback, argparse +import numpy as np +import os +#import matplotlib.pyplot as plt +#import matplotlib.pyplot as plt; plt.rcdefaults() + +# Define the Reading Function Which Pulls the Data from a .txt file +def reader(input_file_txt, create_plot= False): + #Read Matrix, Preserving String Values for Headers first row and first column (both minus first cell) + #Read Matrix, Converting all values to Float for Data Processing + + f = open(input_file_txt, "rU") + + #print( 'Valid NAN identifiers are: empty cells, cells with blanks,"NA","N/A","-", and "?"') + + column_labels = [] + row_labels = [] + matrix = [] + firstLine= True + + line = f.readline() + +# "NA","N/A","-","?","NAN","NaN","Na","na","n/a","null",EMPTY/Null, SPACE (blank char) + + nanList = ["", " ","NAN", "NA", "N/A", "-","?"] + binCatDict = {"":0, " ":0, "Text":0, "NA":0, "-":0,"NAN":0, "N/A":0,"?":0} + row = 0 + nanCnt = 0 + nonNumCnt = 0 + + while line: + line = line.strip("\n") + line = line.split('\t') + + row += 1 + + if firstLine: + lengthRow = len(line) + column_labels = line[1:] + else: + if lengthRow != len(line): + # print("\nERROR matrix row lengths unequal for row 0 and row "+str(row)+"\n" ) + sys.exit(-1) + + temp = [] +# column= 0 + row_labels.append(str(line[0])) + + #for item in line[1:]: use enumerate + for column, item in enumerate(line[1:],1): +# column += 1 + try: + temp.append(float(item)) + except ValueError: + temp.append(np.nan) + itemUC= item.upper() + + if itemUC in nanList: + nanCnt += 1 + binCatDict[itemUC]= binCatDict[itemUC]+1 + # print( 'Legit nans= ',str(item)) + else: + if nonNumCnt == 0: sys.stderr.write("Start List of up to first 50 Invalid cell values \n") + nonNumCnt +=1 + if nonNumCnt < 50: sys.stderr.write("At row_column= "+str(row)+"_"+str(column)+' invalid data cell value '+ item+"\n") + + matrix.append(temp) + + line = f.readline() + firstLine= False + + #sys.stdout.write("\n\n") + f.close() + binCatDict["Text"]= nonNumCnt + +# plot results of NAN counts above + + binCat = ["null", "blank", 'hyphen', '?','NA','N/A' ,'NAN', 'text'] + orderDict= {0:"", 1:"", 2:'-', 3:'?',4:'NA', 5:'N/A' ,6:'NAN', 7:'Text'} +#TODO verify dict orde for data + #print("> key value =",key, str(value)) + + if create_plot: + numBins = len(binCat) + binWidth = 1 + bins = [] + binData = [] + + for key in sorted(orderDict): + value= binCatDict[orderDict[key]] # place items on chart in order and with data value for item + if value < 1: + binData.append(value+0.01) + else: + binData.append(value) + + #""" + for j in range(numBins): + bins.append(j*binWidth) + #ttps://pythonspot.com/matplotlib-bar-chart/ + y_pos = np.arange(numBins) + plt.yticks(y_pos, binCat) + plt.title("Distribution of NAN types (UPPER & lower & MiXeD case combined)") + plt.ylabel('NAN Types') + plt.xlabel('Occurrences') + #plt.legend() + plt.barh(y_pos, binData, align='center', alpha=0.5) + + fig, ax = plt.subplots(num=1, figsize=(8,3)) + ax.set_title("Data Cell Counts of Not A Number (NAN) Types") + #ax.bar(center,bins, align='center', width=width) + #ax.bar(center, hist, align='center', width=width) + #ax.set_xticks(bins) + # fig.savefig("/Users/bobbrown/Desktop/Matrix-tools-Test-output/NAN-plot.png") + + # fig, ax = plt.subplots(num=1, figsize=(8,3)) + # fig.savefig("/Users/bobbrown/Desktop/Matrix-tools-Test-output/hist-out.png") + + plt.show() + #""" + +#after plot error? + x,y=np.shape(matrix) + if nanCnt > 0: print("WARNING -- Found "+str(nanCnt)+" Valid Non-numbers. Their percent of total matrix data cell values = "+str((100*nanCnt)/(x*y))+"% ") + if nonNumCnt > 0: sys.exit(-1) + #print ("reader output:") + #print (matrix) + #print (column_labels) + #print(row_labels) + return matrix,column_labels,row_labels + +#---------------------------------------------------------------------- +# Verify Matrix A column_labels match Matrix B row_labels in name and order for A*B +def MatchLabels(column_labels,row_labels): + + if len(column_labels) != len(row_labels): + sys.err("ERROR 1st matrix column count "+str(len(column_labels))+" not equal 2nd Matrix number row count "+str(len(row_labels))+"\n" ) + else: + cnt= 0 + for k in range(0,len(column_labels)): + if column_labels[k] != row_labels[k] and cnt < 20: + cnt += 1 + #sys.err("ERROR At column & row position "+str(k)+" Matrix 1 column value "+str(column_labels)+" not equal 2nd Matrix row value "+str(row_labels)+"\n" ) + + if cnt > 0: + sys.exit(-11) +#---------------------------------------------------------------------- +# restores row and column labels in ouput +def Labeler(matrix,column_labels,row_labels,output_file_txt): + #print("matrix length: " + str(len(matrix))) + #print("row labels length: " + str(len(row_labels))) + #print("col labels length: " +str(len(column_labels))) + #Define Null Sets For Col and Row Headers + with open(output_file_txt,'w') as f: + f.write("") + for k in range(0,len(column_labels)): + f.write('\t' + str(column_labels[k]) ) + f.write('\n') + #for i in range(0,len(row_labels)): + for i in range(0,len(matrix)): + f.write(str(row_labels[i]) ) + #print("matrix["+str(i)+"] length:" + str(len(matrix[i]))) + for j in range(0,len(matrix[0])): + f.write('\t' + format(matrix[i][j])) + f.write('\n') + + +#---------------------------------------------------------------------- +if __name__ == '__main__': + input_file_txt = str(sys.argv[1]) + + matrix,column_labels,row_labels = reader(input_file_txt) + print("Done") + |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Validations.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Validations.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| b'@@ -0,0 +1,180 @@\n+\'\'\'\n+Created on Jun 7, 2017 modified Feb2018\n+\n+@author: Bob Brown and cjacoby\n+\'\'\'\n+ \n+import sys, traceback, argparse\n+import numpy as np\n+import os\n+from Matrix_Validate_import import reader, Labeler\n+\n+#Define The Four Arguments Used in the Program\n+def get_args():\n+ parser = argparse.ArgumentParser()\n+ parser.add_argument(\'input_file_txt\', help=\'tab delimited text file input matrix(include .txt in name)\')\n+ parser.add_argument(\'replacement\', type=str, help=\'Choose Replacement for Missing Value. Valid Choices are strings: "Mean" or "Zero"\')\n+ parser.add_argument(\'axes\', type=str, help=\'Choose Axes to Normalize On (Either "Row" or "Column"\')\n+ parser.add_argument(\'output_file_txt\' ,help=\'tab delimited text file output name (include .txt in name)\')\n+ args = parser.parse_args()\n+ return args\n+\n+\n+#Define Function to Replace Null Values with Row Mean\n+def nan_replacer_mean_rows(matrix):\n+\n+ nonNumCnt= 0\n+ nanCnt = 0 #valid NANs are "NA","N/A","-","?"\n+\n+ #Loop Replacing all Null Values with Row Mean\n+ for i in range(0,len(matrix)):\n+ temp_mean = np.nanmean(matrix[i])\n+ for j in range(0,len(matrix[0])):\n+ #if matrix[i][j] == "NA": #np.isnan(matrix[i][j]) == True:\n+ if np.isnan(matrix[i][j]) == True:\n+ matrix[i][j] = temp_mean \n+ nanCnt = nanCnt + 1\n+ return matrix, nonNumCnt, nanCnt\n+\n+#Define Function to Replace Null Values with Column Mean\n+def nan_replacer_mean_columns(matrix):\n+\n+ nonNumCnt= 0\n+ nanCnt = 0 #valid NANs are "NA","N/A","-","?"\n+ \n+ #Loop Replacing all Null Values with Column Mean\n+ for i in range(0,len(matrix[0])):\n+ col = [row[i] for row in matrix]\n+ temp_mean = np.nanmean(col)\n+ for j in range(0,len(matrix)):\n+ #if matrix[i][j] == "NA": #elif np.isnan(matrix[j][i]) == True:\n+ if np.isnan(matrix[j][i]) == True:\n+ matrix[j][i] = temp_mean \n+ nanCnt = nanCnt + 1 \n+ \n+ return matrix, nonNumCnt, nanCnt\n+\n+#Define Function to Replace Null Values with Zero (axis orientation is irrelevant)\n+def nan_replacer_zero(matrix):\n+\n+ nonNumCnt= 0\n+ nanCnt = 0 #valid NANs are "NA","N/A","-","?"\n+ \n+ #Loop Replacing all Null Values with Row Range\n+ for i in range(0,len(matrix)):\n+ for j in range(0,len(matrix[0])):\n+ #if matrix[i][j] =="NA":\n+ if np.isnan(matrix[i][j]) == True:\n+ matrix[i][j] = 0\n+\n+ return matrix, nonNumCnt, nanCnt\n+\n+#Define Function to Re-Label Output Matrix\n+#!!!! not needed no output matrix from Validate tool\n+def OLD_labeler(matrix, og_cols, og_rows, output_file_txt): \n+ #Write Data to Specified Text File Output\n+ with open(output_file_txt,\'w\') as f:\n+ f.write("Use original input file for further processing\\n")\n+ f.close()\n+# f.write("")\n+# for k in range(0,len(og_cols)):\n+# f.write(\'\\t\' + str(og_cols[k]))\n+# f.write(\'\\n\')\n+# for i in range(0,len(og_rows)):\n+# f.write(og_rows[i])\n+# for j in range(0,len(matrix[0])):\n+# f.write(\'\\t\' + format(matrix[i][j]))\n+# f.write(\'\\n\') \n+ \n+#Main Function\n+def main():\n+ args = get_args()\n+ #print(args)\n+ #sys.stdout.write(str(args))\n+ #sys.stdout.write( \'\\nValid NAN identifiers are "NA","N/A","-", and "?"\')\n+ \n+ matrix,og_cols,og_rows = reader(args.input_file_txt)\n+\n+# if nonNumCnt > 0:\n+# print(\'\\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\' )\n+# #sys.stderr.write(\'\\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\''..b' if nonNumCnt > 0:\n+ print(\'ERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\' )\n+ sys.stderr.write(\'ERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\' )\n+ if nanCnt > 0:\n+ print(\'WARNING Matrix has \'+str(nanCnt)+\' that is %.2f\' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+\'% known NAN identifiers\')\n+ sys.exit(-1)\n+ else:\n+ if nanCnt > 0:\n+ print(\'\\nWARNING Matrix has \'+str(nanCnt)+\' that is %.2f\' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+\'% known NAN identifiers\')\n+ else:\n+ print(\'\\nMatrix is Good-to-Go -- all numbers in matrix. \')\n+ sys.exit(0)\n+ elif args.axes == "Column":\n+ matrix, nonNumCnt, nanCnt = nan_replacer_mean_columns(matrix)\n+ Labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ #OLD_labeler(matrix, og_cols, og_rows, args.output_file_txt)\n+ #print(\'Mean,Column\')\n+ if nonNumCnt > 0:\n+ print(\'\\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\' )\n+ sys.stderr.write(\'\\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\' )\n+ if nanCnt > 0:\n+ print(\'\\nWARNING Matrix has \'+str(nanCnt)+\' that is %.2f\' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+\'% known NAN identifiers\')\n+ sys.exit(-1)\n+ else:\n+ if nanCnt > 0:\n+ print(\'\\nWARNING Matrix has \'+str(nanCnt)+\' that is %.2f\' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+\'% known NAN identifiers\')\n+ else:\n+ print(\'\\nMatrix is Good-to-Go -- all numbers in matrix. \')\n+ sys.exit(0)\n+ else:\n+ print(\'Mean, but given Invalid Axis= \'+str(args.axes))\n+ sys.stderr.write(\'Mean, but given Invalid Axis= \'+str(args.axes))\n+ elif args.replacement == "Zero":\n+ matrix, nonNumCnt, nanCnt = nan_replacer_zero(matrix)\n+ Labeler(matrix,og_cols,og_rows,args.output_file_txt)\n+ #OLD_labeler(matrix, og_cols, og_rows, args.output_file_txt)\n+ if nonNumCnt > 0:\n+ print(\'\\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\' )\n+ sys.stderr.write(\'\\nERROR Matrix has non-numbers that are non-NAN identifiers in matrix. Total and percent unknown strings found = \'+str(nonNumCnt)+ \', %.2f\' % (100.0*nonNumCnt/(1.0*len(og_cols)*len(og_rows)))+\'%\' )\n+ if nanCnt > 0:\n+ print(\'\\nWARNING Matrix has \'+str(nanCnt)+\' that is %.2f\' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+\'% known NAN identifiers\')\n+ sys.exit(-1)\n+ else:\n+ if nanCnt > 0:\n+ print(\'\\nWARNING Matrix has \'+str(nanCnt)+\' that is %.2f\' % (100.0*nanCnt/(1.0*len(og_cols)*len(og_rows)))+\'% known NAN identifiers\')\n+ else:\n+ print(\'\\nMatrix is Good-to-Go -- all numbers in matrix. \')\n+ sys.exit(0)\n+ else:\n+ print(\'zero, but given Invalid Axis= \'+str(args.axes))\n+ sys.stderr.write(\'zero, but given Invalid Axis= \'+str(args.axes))\n+ sys.exit(-2)\n+\n+ \n+if __name__ == \'__main__\':\n+ main()\n+ print("done")\n' |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Matrix_Validations.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Matrix_Validations.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,40 @@ +<?xml version='1.1' encoding='utf-8'?> +<tool id="Matrix_Validations" name="Matrix Data Validation" version="2.2"> + <description> Locate and identify non-numbers</description> + <command interpreter="python">Matrix_Validations.py '$p_input' 'Zero' 'Row' '$output_file'</command> + <inputs> + <param name="p_input" type="data" format="tabular" label="Input Matrix" /> + + <!-- + <conditional name="extra"> + <param name="replacement" type="select" label="Choose Replacement Type for Non-Numeric Data Cells"> + <option value="Mean">Mean</option> + <option value="Zero">Zero</option> + </param> + <when value="Mean"> + <param name="axes" type="select" label="Choose Axes To Calculate Cell Replacement Value"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + </when> + <when value="Zero"> + <param name="axes" type="text" size="10" hidden="true" value="Row"/> + </when> + </conditional> +--> +<!-- replace by when above to place zero values doesn't matter if by row or column + <param name="axes" type="select" label="Choose Axes To Calculate Cell Replacement Value"> + <option value="Row">Row</option> + <option value="Column">Column</option> + </param> + --> + + + </inputs> + + <outputs> + <data name="output_file" format="txt" /> + </outputs> + + </tool> + \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Split_ExcelTabs_IntoFiles.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Split_ExcelTabs_IntoFiles.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| b'@@ -0,0 +1,290 @@\n+import sys\n+import os\n+#import MySQLdb\n+#import config\n+import subprocess\n+import re\n+import shutil\n+import traceback\n+#import xlsxwriter\n+import xlrd\n+\n+#http://www.blog.pythonlibrary.org/2014/04/30/reading-excel-spreadsheets-with-python-and-xlrd/\n+\n+def File_From_Tab(infileName, outfileName, tabName,tabNumber):\n+ """\n+ Open and read an Excel file\n+ """\n+ book = xlrd.open_workbook(infileName)\n+ # print number of sheets\n+ #print book.nsheets\n+ \n+ # print sheet names\n+ tabList= book.sheet_names()\n+ #print tabList\n+ #print book.sheet_names()\n+ if tabName == "" and (tabNumber <1 or tabNumber > len(tabList)):\n+ sys.stderr.write("\\n>>>ERROR illegal tab number "+str(tabNumber)+" input when no tab name was specified\\n")\n+ sys.stderr.write("\\n>>>Allowed tab numbers, or tab names, for this file with "+str(len(tabList))+" total tabs are:")\n+ \n+ for i in range(len(tabList)):\n+ sys.stderr.write("\\n>>> tab number "+str(i+1)+" is named "+str(tabList[i]))\n+ sys.exit(-1)\n+\n+ if tabName != "": # use name instead of tab number\n+ found = False\n+ i = 0\n+ while (i < len(tabList)) and not found:\n+ i += 1\n+ if tabName == str(tabList[i-1]):\n+ tabNumber = i\n+ found = True\n+ if not found:\n+ sys.stderr("\\n>>> ERROR -- Input Tab name "+tabName+" was not found\\n")\n+ sys.exit(-1)\n+ # get the first worksheet\n+ #first_sheet = book.sheet_by_index(0)\n+ worksheet = book.sheet_by_index(tabNumber-1)\n+ \n+ outFile = open(outfileName+str(tabList[tabNumber-1]+".tsv"), \'w\')\n+ \n+ #https://stackoverflow.com/questions/14944623/python-xrld-read-rows-and-columns\n+ #workbook = xlrd.open_workbook(\'my_workbook.xls\')\n+ #worksheet = workbook.sheet_by_name(\'Sheet1\')\n+ num_rows = worksheet.nrows - 1\n+ num_cells = worksheet.ncols - 1\n+ curr_row = -1\n+ while curr_row < num_rows:\n+ curr_row += 1\n+ row = worksheet.row(curr_row)\n+ \n+ if curr_row == 0:\n+ endOfLine= False\n+ allRowNumCols= len(row)\n+ i= len(row)-1\n+ # find length of matrix and covariates using first row \n+ # Cell Types: 0=Empty, 1=Text, 2=Number, 3=Date, 4=Boolean, 5=Error, 6=Blank\n+ while i <= len(row)-1 and not endOfLine:\n+ cell_type = worksheet.cell_type(curr_row, i)\n+ #temp = str(worksheet.cell_value(curr_row, i))\n+ #print( " pos and cell type row one ",cell_type, i)\n+ \n+ if cell_type == 0 or cell_type == 6:\n+ allRowNumCols -= 1\n+ i -= 1\n+ else:\n+ endOfLine= True\n+ \n+ if allRowNumCols < 5:\n+ sys.stderr.write("\\nERROR First row number of columns= "+str(allRowNumCols)+" is too short, so all rows will be ignored\\n")\n+ sys.exit(-1)\n+ elif curr_row == 0: \n+ sys.stdout.write("\\nALL Rows must all have the same number of columns as the First row\'s number columns = "+ str(allRowNumCols) +"\\n")\n+ \n+ temp= \'\'\n+ rowLen= 0\n+ endOfLine= False\n+ \n+ while rowLen < allRowNumCols and not endOfLine:\n+ temp += str(worksheet.cell_value(curr_row, rowLen))+"\\t"\n+ #temp += str(row[rowLen])+"\\t"\n+ rowLen += 1\n+ \n+ temp = temp[:-1]+"\\n"\n+ #print \'Row:\', curr_row, len(row), rowLen\n+ outFile.write(temp) #TODO check if rows are all same length\n+ \n+ sys.stdout.write("File created with "+str(curr_row)+" rows and "+str(allRowNumCols)+" columns\\n")\n+# curr_cell = -1\n+# while curr_cell < num_cells:\n+# curr_cell += 1\n+# # Cell Types: 0=Empty, 1=Text, 2=Number, 3=Date, 4=Boolean, 5=Error, 6=Blank\n+# cell_type = worksheet.cell_type(curr_row, curr_cell)\n+# cell_value = worksheet.cell_value(curr_row, curr_cell)\n+# pri'..b'ine)\n+# toks = line.split(",")\n+# line_num += 1\n+# if line_num == 1:\n+# line += "1,CF2"\n+# new_lines_normLinear_with_cf2.append(line)\n+# titles = toks\n+# continue\n+# pos_rf = int(toks[titles.index(\'Order\')])\n+# line += "," + str(dict_cf2_values[pos_rf])\n+# new_lines_normLinear_with_cf2.append(line)\n+# rf_normLinear.close()\n+# # rename the old normLinear file\n+# os.rename(norm_linear_path, os.path.join(path_to_dir, \'before_cf2_NormLinear.csv\'))\n+# \n+# # write new normLinear with cf2\n+# wf_new_normLinear = open(norm_linear_path, \'w\')\n+# for line_writing in new_lines_normLinear_with_cf2:\n+# wf_new_normLinear.write(line_writing + "\\n")\n+# wf_new_normLinear.close()\n+# except Exception, err_write_normLinear_with_cf2_values:\n+# errors = str(repr(err_write_normLinear_with_cf2_values))\n+# \n+# return errors\n+# \n+# \n+# # This function constructs the worksheet for each tab in the excel file for a report\n+# # It puts these things in this order:\n+# # 1. Title of the tab\n+# # 2. Header for the tab\n+# # 3. Content of the tab\n+# def construct_worksheet_for_xlsx(worksheet, header_path, title_top_of_tab, tab_input_path):\n+# \n+# reload(sys) \n+# sys.setdefaultencoding(\'utf8\')\n+# errors = ""\n+# \n+# try:\n+# # Write the title at the top of the tab\n+# worksheet.write(0,0,title_top_of_tab)\n+# \n+# # Variable to keep track of the rows\n+# row_num = 1\n+# \n+# # Write the header stuff\n+# header_file = open(header_path, \'rU\')\n+# for head_line in header_file:\n+# head_line = strip_new_line_from_right_side(head_line)\n+# head_toks = head_line.split(",")\n+# col_num = 0\n+# for tok in head_toks:\n+# worksheet.write(row_num, col_num, tok)\n+# col_num += 1\n+# row_num += 1\n+# \n+# # Write the content stuff\n+# tab_input_file = open(tab_input_path, \'rU\')\n+# for tab_line in tab_input_file:\n+# tab_line = strip_new_line_from_right_side(tab_line)\n+# tab_toks = tab_line.split(",")\n+# col_num = 0\n+# for tok in tab_toks:\n+# tok = tok.decode(\'iso-8859-1\').encode(\'utf-8\')\n+# worksheet.write(row_num, col_num, tok)\n+# col_num += 1\n+# row_num += 1 \n+# \n+# header_file.close()\n+# tab_input_file.close()\n+# except Exception, e:\n+# errors = errors + "\\n\\nAn error occurred while constructing the "+title_top_of_tab+" tab for the excel file.\\n"\n+# errors = errors + "The error was :\\n\\t" + str(e) + "\\n\\n"\n+# try:\n+# header_file.close()\n+# tab_input_file.close()\n+# except NameError:\n+# x = 5\n+# \n+ return errors\n+\n+#----------------------------------------------------------------------\n+if __name__ == "__main__":\n+ \n+ #try:\n+ if len(sys.argv) > 4:\n+ infileName = \'"\'+sys.argv[1]+\'"\'\n+ tabName = \'"\'+sys.argv[2]+\'"\' \n+ tabNumber = 0 \n+ if tabName == \'\': tabNumber = int(sys.argv[3]) \n+ outfileName = \'"\'+sys.argv[4]+\'"\' #TODO Later multiple outputs one per tab\n+ \n+ sys.stdout.write( "\\nInput parameters ",str(sys.argv[1:4]),"\\n" )\n+ \n+ #infileName = "/Users/bobbrown/Desktop/01_Gordon_Mills__Zhiyong_Ding.xlsx"\n+ #outfileName= "/Users/bobbrown/Desktop/01_Gordon_Mills__Zhiyong_Ding-Tab-Out-"\n+ #tabName ="NormLog2"\n+ #tabName =""\n+ #tabNumber= 10\n+ \n+ status= File_From_Tab(infileName, outfileName, tabName, tabNumber )\n+ #except\n+ #sys.exit(-1)\n+ \n+ sys.exit(0)\n\\ No newline at end of file\n' |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Split_ExcelTabs_IntoFiles.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Split_ExcelTabs_IntoFiles.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,35 @@ +<?xml version='1.1' encoding='utf-8'?> +<tool id="Split_ExcelTabs_IntoFiles" name="Split Excel Tabs" version="1.0"> + <description>into separate tab delimited files</description> + <command interpreter="python">Split_ExcelTabs_IntoFiles.py '$input' '' '$extra.tabnumber' '$output_file'</command> + + <inputs> + <param name="input" type="data" format="txt" label="Input Excel.xls or Excel.xlsx file"> </param> + + <conditional name="extra"> + <param name="choice" type="select" label="Select tab" help="Select tab position or name"> + <option value="tab_number">Tab number</option> + </param> +<!-- +'$extra.tabname' + <option value="tab_name">Tab name</option> + <option value="all">All tabs in file</option> + <option value="1">First tab only</option> + </param> + <when value="tab_name"> + <param name="tabname" type="text" size="20" value="" label="Tab Name" help="Spreadsheet Tab Name"> + </param> + </when> +--> + <when value="tab_number"> + <param name="tabnumber" type="text" size="2" value="1" label="Tab position" help="Integer Tab position from left"> + </param> + </when> + </conditional> + </inputs> + + <outputs> + <data name="output_file" format="tabular" label="Output File Name" help="Name based on Input file name and tab name"/> + </outputs> + + </tool> \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 TestOutFile.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/TestOutFile.txt Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,1 @@ +output from input= TestInFile \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Test_input_into_file.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Test_input_into_file.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +#Created on Jule 23, 2018 + +# @author: Bob Brown + +import sys +import os + +def main(): + + # Grab the inputs from the Galaxy xml interface and write to a file that is passed to the program + # Not each of the parameters as separate command line variables. +# ab_gene_name_for_header={} +# ab_rrid_for_header={} + dir= "/Users/bobbrown/Desktop/junk/" + accepted_extensions = ["csv", "tsv"] + filenames = [fn for fn in os.listdir(dir) if fn.split(".")[-1] in accepted_extensions] + for f in filenames: + print("filename= "+f) + os.remove(dir+f) + + sys.exit(0) + + ab_gene_name_for_header={'abc':'geneName'} + ab_rrid_for_header={'abc':'rrid123'} + line= 'abc,123\n' + + pos= line.find(",") + ABname= line[0:pos] + ABnewName= ABname+ "|"+ab_gene_name_for_header[ABname]+"|"+ab_rrid_for_header[ABname] + line= ABnewName+line[pos:] + line= line.replace(',','\t') + sys.exit(0) +# try: + print(' \n starting Test program read params from file stored in tools dir. Arguments=') + print(str(sys.argv[1:])+'\n') + + if False: + infileName = sys.argv[1] + # directory = sys.argv[2] + directory = '/Users/bobbrown/Desktop/' + outfileName = sys.argv[3] #use later + # outfile = sys.argv[6] + + #sys.stdout.write + + # ifile= open(directory+"/"+infileName,'rU') + ifile= open(directory+infileName,'rU') + ofile= open(directory+outfileName,'w') + # ofile= open('/Users/bobbrown/Desktop/TestOutFileVarParams.txt','w') + + cnt= 0 + # for param in range(2,len(sys.argv)): + # cnt +=1 + # ofile.write("param "+str(cnt)+"= "+param+"\n") + + + for param in ifile: + cnt +=1 + ofile.write("param "+str(cnt)+"= "+param) + + ifile.close() + + ofile.close() + + print('Fini -- rows read = '+str(cnt)+'\n') + +# except : +# print('Error>>> ') + + return +## +## + +if __name__ == '__main__': main() + #sys.exit(0) \ No newline at end of file |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 Test_input_into_file.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Test_input_into_file.xml Tue Nov 27 14:20:40 2018 -0500 |
| b |
| @@ -0,0 +1,61 @@ +<?xml version='1.1' encoding='utf-8'?> +<tool id="Test-input-into-file" name="Test-File" version="1.0.0" force_history_refresh="True"> + <description> Read xml params into a file</description> + +<command interpreter="python">Test-input-into-file.py $inputfile "$__tool_dir__" $outputfile + #for $op in $test_param1 + ${op.discrete_fields1.Text_Fields1} + #end for + "$EndofVarParam1" + "$Covariate_Type" + "$EndofVarParam2" + #for $op in $test_param3 + ${op.discrete_fields3.Text_Fields3} + #end for + "$EndofVarParam3" + </command> +<!-- #in=$__tool_dir__/$inputfile + #inputfile="$__tool_dir__"$inputfile + +type Describes the parameter type - each different type as different semantics + and the tool form widget is different. Currently valid parameter types are: + text, integer, float, boolean, genomebuild, select, color, data_column, hidden, hidden_data, baseurl, file, ftpfile, data, data_collection, library_data, drill_down. The definition of supported parameter + types as defined in the parameter_types dictionary in /lib/galaxy/tools/parameters/basic.py. + --> + <stdio> + <exit_code range="1:" level="fatal" /> + </stdio> + <inputs> + <data format="txt" name="inputfile"/> + <repeat name="test_param1" title="Select 1 or more Covariate Clinical Categories"> + <conditional name="discrete_fields1"> + <param name="Text_Fields1" type="select" label="Covariate Bar Choices" help=""> + <option value='cat_EMPTY' selected="true">Click cursor in middle of this text to see choices</option> + <option value='cat_SampleTypeCode'>Sample Cohort</option> + <option value='cat_gender'>Gender</option> + <option value='cat_race'>Race</option> + </param> + </conditional> + </repeat> + <param name="EndofVarParam1" type="hidden" value="paramend1" </param> + <param name="Covariate_Type" type="select" value="" label="Type of Covariate Bars" help="Choose Sample or Gene covariate bar files"> + <option value="S" selected="true">Participant-Sample</option> + <option value="G" >Gene</option> + </param> + <param name="EndofVarParam2" type="hidden" value="paramend2" </param> + <repeat name="test_param3" title="Select 1 or more Covariate Clinical Categories"> + <conditional name="discrete_fields3"> + <param name="Text_Fields3" type="select" label="Covariate Bar Choices" help=""> + <option value='cat_EMPTY' selected="true">Click cursor in middle of this text to see choices</option> + <option value='cat_SampleTypeCode'>Sample Cohort</option> + <option value='cat_gender'>Gender</option> + <option value='cat_race'>Race</option> + </param> + </conditional> + </repeat> + <param name="EndofVarParam3" type="hidden" value="paramend1" </param> + </inputs> + <outputs> + <data format="txt" name="outputfile" label="${tool.name} on ${on_string}"/> + </outputs> +</tool> |
| b |
| diff -r 7f12c81e2083 -r f1bcd79cd923 bar_chart_plot.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bar_chart_plot.py Tue Nov 27 14:20:40 2018 -0500 |
| [ |
| @@ -0,0 +1,140 @@ +#!/usr/bin/env python +""" +histogram_gnuplot.py <datafile> <xtic column> <column_list> <title> <ylabel> <yrange_min> <yrange_max> <grath_file> +a generic histogram builder based on gnuplot backend + + data_file - tab delimited file with data + xtic_column - column containing labels for x ticks [integer, 0 means no ticks] + column_list - comma separated list of columns to plot + title - title for the entire histrogram + ylabel - y axis label + yrange_max - minimal value at the y axis (integer) + yrange_max - maximal value at the y_axis (integer) + to set yrange to autoscaling assign 0 to yrange_min and yrange_max + graph_file - file to write histogram image to + img_size - as X,Y pair in pixels (e.g., 800,600 or 600,800 etc.) + + + This tool required gnuplot and gnuplot.py + +anton nekrutenko | anton@bx.psu.edu +""" + +import string +import sys +import tempfile + +import Gnuplot +import Gnuplot.funcutils + +assert sys.version_info[:2] >= (2, 4) + + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit() + + +def main(tmpFileName): + skipped_lines_count = 0 + skipped_lines_index = [] + gf = open(tmpFileName, 'w') + + try: + in_file = open(sys.argv[1], 'r') + xtic = int(sys.argv[2]) + col_list = string.split(sys.argv[3], ",") + title = 'set title "' + sys.argv[4] + '"' + ylabel = 'set ylabel "' + sys.argv[5] + '"' + ymin = sys.argv[6] + ymax = sys.argv[7] + img_file = sys.argv[8] + img_size = sys.argv[9] + except: + stop_err("Check arguments\n") + + try: + int(col_list[0]) + except: + stop_err('You forgot to set columns for plotting\n') + + for i, line in enumerate(in_file): + valid = True + line = line.rstrip('\r\n') + if line and not line.startswith('#'): + row = [] + try: + fields = line.split('\t') + for col in col_list: + row.append(str(float(fields[int(col) - 1]))) + except: + valid = False + skipped_lines_count += 1 + skipped_lines_index.append(i) + else: + valid = False + skipped_lines_count += 1 + skipped_lines_index.append(i) + + if valid and xtic > 0: + row.append(fields[xtic - 1]) + elif valid and xtic == 0: + row.append(str(i)) + + if valid: + gf.write('\t'.join(row)) + gf.write('\n') + + if skipped_lines_count < i: + # Prepare 'using' clause of plot statement + g_plot_command = ' ' + + # Set the first column + if xtic > 0: + g_plot_command = "'%s' using 1:xticlabels(%s) ti 'Column %s', " % (tmpFileName, str(len(row)), col_list[0]) + else: + g_plot_command = "'%s' using 1 ti 'Column %s', " % (tmpFileName, col_list[0]) + + # Set subsequent columns + for i in range(1, len(col_list)): + g_plot_command += "'%s' using %s t 'Column %s', " % (tmpFileName, str(i + 1), col_list[i]) + + g_plot_command = g_plot_command.rstrip(', ') + + yrange = 'set yrange [' + ymin + ":" + ymax + ']' + + try: + g = Gnuplot.Gnuplot() + g('reset') + g('set boxwidth 0.9 absolute') + g('set style fill solid 1.00 border -1') + g('set style histogram clustered gap 5 title offset character 0, 0, 0') + g('set xtics border in scale 1,0.5 nomirror rotate by 90 offset character 0, 0, 0') + g('set key invert reverse Left outside') + if xtic == 0: + g('unset xtics') + g(title) + g(ylabel) + g_term = 'set terminal png tiny size ' + img_size + g(g_term) + g_out = 'set output "' + img_file + '"' + if ymin != ymax: + g(yrange) + g(g_out) + g('set style data histograms') + g.plot(g_plot_command) + except: + stop_err("Gnuplot error: Data cannot be plotted") + else: + sys.stderr.write('Column(s) %s of your dataset do not contain valid numeric data' % sys.argv[3]) + + if skipped_lines_count > 0: + sys.stdout.write('\nWARNING. You dataset contain(s) %d invalid lines starting with line #%d. These lines were skipped while building the graph.\n' % (skipped_lines_count, skipped_lines_index[0] + 1)) + + +if __name__ == "__main__": + # The tempfile initialization is here because while inside the main() it seems to create a condition + # when the file is removed before gnuplot has a chance of accessing it + gp_data_file = tempfile.NamedTemporaryFile('w') + Gnuplot.gp.GnuplotOpts.default_term = 'png' + main(gp_data_file.name) |