|
0
|
1 <tool id="ROC Curve" name="Plot ROC Curve">
|
|
4
|
2 <description>using kmerSVM predictions</description>
|
|
0
|
3 <command interpreter="sh">r_wrapper.sh $script_file</command>
|
|
|
4 <inputs>
|
|
|
5 <param format="tabular" name="cvpred_data" type="data" label="CV Predictions"/>
|
|
|
6 </inputs>
|
|
|
7 <outputs>
|
|
|
8 <data format="png" name="roccurve.png" from_work_dir="roccurve.png" />
|
|
|
9 </outputs>
|
|
|
10
|
|
|
11 <configfiles>
|
|
|
12 <configfile name="script_file">
|
|
|
13
|
|
|
14 rm(list = objects() )
|
|
|
15
|
|
|
16 ########## plot ROC and PR-Curve #########
|
|
|
17 roccurve <- function(x) {
|
|
|
18 sink(NULL,type="message")
|
|
|
19 options(warn=-1)
|
|
|
20 suppressMessages(suppressWarnings(library('ROCR')))
|
|
|
21 svmresult <- data.frame(x)
|
|
|
22 colnames(svmresult) <- c("Seqid","Pred","Label", "CV")
|
|
|
23
|
|
|
24 linewd <- 1
|
|
|
25 wd <- 4
|
|
|
26 ht <- 4
|
|
|
27 fig.nrows <- 1
|
|
|
28 fig.ncols <- 1
|
|
|
29 pt <- 10
|
|
|
30 cex.general <- 1
|
|
|
31 cex.lab <- 0.9
|
|
|
32 cex.axis <- 0.9
|
|
|
33 cex.main <- 1.2
|
|
|
34 cex.legend <- 0.8
|
|
|
35
|
|
|
36 png("roccurve.png", width=wd*fig.ncols, height=ht*fig.nrows, unit="in", res=100)
|
|
|
37
|
|
|
38 par(xaxs="i", yaxs="i", mar=c(3.5,3.5,2,2)+0.1, mgp=c(2,0.8,0), mfrow=c(fig.nrows, fig.ncols))
|
|
|
39
|
|
|
40 CVs <- unique(svmresult[["CV"]])
|
|
|
41 preds <- list()
|
|
|
42 labs <- list()
|
|
|
43 auc <- c()
|
|
|
44 for(i in 1:length(CVs)) {
|
|
|
45 preds[i] <- subset(svmresult, CV==(i-1), select=c(Pred))
|
|
|
46 labs[i] <- subset(svmresult, CV==(i-1), select=c(Label))
|
|
|
47 }
|
|
|
48
|
|
|
49 pred <- prediction(preds, labs)
|
|
|
50 perf_roc <- performance(pred, 'tpr', 'fpr')
|
|
|
51 perf_auc <- performance(pred, 'auc')
|
|
|
52
|
|
|
53 avgauc <- 0
|
|
|
54
|
|
|
55 for(j in 1:length(CVs)) {
|
|
|
56 avgauc <- avgauc + perf_auc@y.values[[j]]
|
|
|
57 }
|
|
|
58
|
|
|
59 avgauc <- avgauc/length(CVs)
|
|
|
60
|
|
|
61 plot(perf_roc, colorize=T, main="ROC curve", spread.estimate="stderror",
|
|
|
62 xlab="1-Specificity", ylab="Sensitivity", cex.lab=1.2)
|
|
|
63 text(0.2, 0.1, paste("AUC=", format(avgauc, digits=3, nsmall=3)))
|
|
|
64
|
|
|
65 dev.off()
|
|
|
66 }
|
|
|
67
|
|
|
68 ############## main function #################
|
|
|
69 d <- read.table("${cvpred_data}")
|
|
|
70
|
|
|
71 roccurve(d)
|
|
|
72
|
|
|
73 </configfile>
|
|
|
74 </configfiles>
|
|
|
75
|
|
|
76 <help>
|
|
|
77
|
|
|
78 **Note**
|
|
|
79
|
|
|
80 This tool is based on the ROCR library. If you use this tool please cite:
|
|
|
81
|
|
|
82 Tobias Sing, Oliver Sander, Niko Beerenwinkel, Thomas Lengauer.
|
|
|
83 ROCR: visualizing classifier performance in R.
|
|
|
84 Bioinformatics 21(20):3940-3941 (2005).
|
|
|
85
|
|
|
86 ----
|
|
|
87
|
|
|
88 **What it does**
|
|
|
89
|
|
|
90 Takes as input cross-validation predictions and calculates ROC Curve and its area under curve (AUC).
|
|
|
91
|
|
|
92 ----
|
|
|
93
|
|
|
94 **Results**
|
|
|
95
|
|
|
96 ROC Curve: Receiver Operating Characteristic Curve. Compares true positive rate (sensitivity) to false positive rate (1 - specificity).
|
|
|
97
|
|
|
98 Area Under the ROC Curve (AUC): Probability that of a randomly selected positive/negative pair, the positive will be scored more highly by the trained SVM than a negative.
|
|
|
99
|
|
|
100 .. class:: infomark
|
|
|
101
|
|
|
102 ROC curves can be inaccurate if there is a large skew in class distribution. For more information see:
|
|
|
103
|
|
|
104 Jesse Davis, Mark Goadrich.
|
|
|
105 The Relationship Between Precision-Recall and ROC Curves.
|
|
|
106 Proceedings of the 23rd Annual Internation Conference on Machine Learning.
|
|
|
107 Pittsburgh, PA, 2006.
|
|
|
108
|
|
|
109 <!--
|
|
|
110 **Example**
|
|
|
111
|
|
|
112 .. image:: ./static/images/sample_roc_chen.png
|
|
|
113 -->
|
|
|
114 </help>
|
|
|
115 </tool>
|
