Mercurial > repos > devteam > correlation
changeset 0:24e01abf9e34 draft default tip
Imported from capsule None
| author | devteam |
|---|---|
| date | Mon, 28 Jul 2014 11:55:23 -0400 |
| parents | |
| children | |
| files | cor.py cor.xml kendall.png pearson.png spearman.png test-data/cor.tabular test-data/cor_out.txt tool_dependencies.xml |
| diffstat | 8 files changed, 218 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cor.py Mon Jul 28 11:55:23 2014 -0400 @@ -0,0 +1,88 @@ +#!/usr/bin/env python +#Greg Von Kuster +""" +Calculate correlations between numeric columns in a tab delim file. +usage: %prog infile output.txt columns method +""" + +import sys +from rpy import * + +def stop_err(msg): + sys.stderr.write(msg) + sys.exit() + +def main(): + method = sys.argv[4] + assert method in ( "pearson", "kendall", "spearman" ) + + try: + columns = map( int, sys.argv[3].split( ',' ) ) + except: + stop_err( "Problem determining columns, perhaps your query does not contain a column of numerical data." ) + + matrix = [] + skipped_lines = 0 + first_invalid_line = 0 + invalid_value = '' + invalid_column = 0 + + for i, line in enumerate( file( sys.argv[1] ) ): + valid = True + line = line.rstrip('\n\r') + + if line and not line.startswith( '#' ): + # Extract values and convert to floats + row = [] + for column in columns: + column -= 1 + fields = line.split( "\t" ) + if len( fields ) <= column: + valid = False + else: + val = fields[column] + if val.lower() == "na": + row.append( float( "nan" ) ) + else: + try: + row.append( float( fields[column] ) ) + except: + valid = False + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = i+1 + invalid_value = fields[column] + invalid_column = column+1 + else: + valid = False + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = i+1 + + if valid: + matrix.append( row ) + + if skipped_lines < i: + try: + out = open( sys.argv[2], "w" ) + except: + stop_err( "Unable to open output file" ) + + # Run correlation + try: + value = r.cor( array( matrix ), use="pairwise.complete.obs", method=method ) + except Exception, exc: + out.close() + stop_err("%s" %str( exc )) + for row in value: + print >> out, "\t".join( map( str, row ) ) + out.close() + + if skipped_lines > 0: + msg = "..Skipped %d lines starting with line #%d. " %( skipped_lines, first_invalid_line ) + if invalid_value and invalid_column > 0: + msg += "Value '%s' in column %d is not numeric." % ( invalid_value, invalid_column ) + print msg + +if __name__ == "__main__": + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cor.xml Mon Jul 28 11:55:23 2014 -0400 @@ -0,0 +1,101 @@ +<tool id="cor2" name="Correlation" version="1.0.0"> + <description>for numeric columns</description> + <requirements> + <requirement type="package" version="1.0.3">rpy</requirement> + </requirements> + <command interpreter="python">cor.py $input1 $out_file1 $numeric_columns $method</command> + <inputs> + <param format="tabular" name="input1" type="data" label="Dataset" help="Dataset missing? See TIP below"/> + <param name="numeric_columns" label="Numerical columns" type="data_column" numerical="True" multiple="True" data_ref="input1" help="Multi-select list - hold the appropriate key while clicking to select multiple columns" /> + <param name="method" type="select" label="Method"> + <option value="pearson">Pearson</option> + <option value="kendall">Kendall rank</option> + <option value="spearman">Spearman rank</option> + </param> + </inputs> + <outputs> + <data format="txt" name="out_file1" /> + </outputs> + <tests> + <!-- + Test a tabular input with the first line being a comment without a # character to start + --> + <test> + <param name="input1" value="cor.tabular" /> + <param name="numeric_columns" value="2,3" /> + <param name="method" value="pearson" /> + <output name="out_file1" file="cor_out.txt" /> + </test> + </tests> + <help> + +.. class:: infomark + +**TIP:** If your data is not TAB delimited, use *Text Manipulation->Convert* + +.. class:: warningmark + +Missing data ("nan") removed from each pairwise comparison + +----- + +**Syntax** + +This tool computes the matrix of correlation coefficients between numeric columns. + +- All invalid, blank and comment lines are skipped when performing computations. The number of skipped lines is displayed in the resulting history item. + +- **Pearson's Correlation** reflects the degree of linear relationship between two variables. It ranges from +1 to -1. A correlation of +1 means that there is a perfect positive linear relationship between variables. The formula for Pearson's correlation is: + + .. image:: pearson.png + + where n is the number of items + +- **Kendall's rank correlation** is used to measure the degree of correspondence between two rankings and assessing the significance of this correspondence. The formula for Kendall's rank correlation is: + + .. image:: kendall.png + + where n is the number of items, and P is the sum. + +- **Spearman's rank correlation** assesses how well an arbitrary monotonic function could describe the relationship between two variables, without making any assumptions about the frequency distribution of the variables. The formula for Spearman's rank correlation is + + .. image:: spearman.png + + where D is the difference between the ranks of corresponding values of X and Y, and N is the number of pairs of values. + +----- + +**Example** + +- Input file:: + + #Person Height Self Esteem + 1 68 4.1 + 2 71 4.6 + 3 62 3.8 + 4 75 4.4 + 5 58 3.2 + 6 60 3.1 + 7 67 3.8 + 8 68 4.1 + 9 71 4.3 + 10 69 3.7 + 11 68 3.5 + 12 67 3.2 + 13 63 3.7 + 14 62 3.3 + 15 60 3.4 + 16 63 4.0 + 17 65 4.1 + 18 67 3.8 + 19 63 3.4 + 20 61 3.6 + +- Computing the correlation coefficients between columns 2 and 3 of the above file (using Pearson's Correlation), the output is:: + + 1.0 0.730635686279 + 0.730635686279 1.0 + + So the correlation for our twenty cases is .73, which is a fairly strong positive relationship. + </help> +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/cor.tabular Mon Jul 28 11:55:23 2014 -0400 @@ -0,0 +1,21 @@ +Person Height Self Esteem +1 68 4.1 +2 71 4.6 +3 62 3.8 +4 75 4.4 +5 58 3.2 +6 60 3.1 +7 67 3.8 +8 68 4.1 +9 71 4.3 +1 69 3.7 +1 68 3.5 +1 67 3.2 +1 63 3.7 +1 62 3.3 +1 60 3.4 +1 63 4.0 +1 65 4.1 +1 67 3.8 +1 63 3.4 +2 61 3.6 \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/cor_out.txt Mon Jul 28 11:55:23 2014 -0400 @@ -0,0 +1,2 @@ +1.0 0.730635686279 +0.730635686279 1.0
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Mon Jul 28 11:55:23 2014 -0400 @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="rpy" version="1.0.3"> + <repository changeset_revision="82170c94ca7c" name="package_rpy_1_0_3" owner="devteam" toolshed="http://toolshed.g2.bx.psu.edu" /> + </package> +</tool_dependency>