ROCR/ 0000755 0001762 0000144 00000000000 13653304235 011023 5 ustar ligges users ROCR/NAMESPACE 0000644 0001762 0000144 00000001061 13653005516 012240 0 ustar ligges users # Generated by roxygen2: do not edit by hand
S3method(plot,performance)
export(performance)
export(prediction)
exportClasses(performance)
exportClasses(prediction)
exportMethods(plot)
import(graphics)
import(methods)
import(stats)
importFrom(grDevices,chull)
importFrom(grDevices,rainbow)
importFrom(grDevices,xy.coords)
importFrom(graphics,par)
importFrom(graphics,plot.default)
importFrom(graphics,plot.xy)
importFrom(stats,approxfun)
importFrom(stats,chisq.test)
importFrom(stats,median)
importFrom(stats,sd)
importFrom(stats,uniroot)
ROCR/demo/ 0000755 0001762 0000144 00000000000 13644317760 011756 5 ustar ligges users ROCR/demo/ROCR.R 0000644 0001762 0000144 00000022613 13644317760 012652 0 ustar ligges users ## -----------------------------------------------------------------------------------
## Demo file for ROCR; start with 'demo(ROCR)'
## -----------------------------------------------------------------------------------
# if(dev.cur() <= 1) get(getOption("device"))()
if(dev.cur() <= 1) dev.new()
opar <- par(ask = interactive() &&
(.Device %in% c("X11", "GTK", "gnome", "windows","quartz")))
data(ROCR.hiv)
pp <- ROCR.hiv$hiv.svm$predictions
ll <- ROCR.hiv$hiv.svm$labels
par(mfrow=c(2,2))
pred<- prediction(pp, ll)
perf <- performance(pred, "tpr", "fpr")
plot(perf, avg= "threshold", colorize=TRUE, lwd= 3,
main= "With ROCR you can produce standard plots like ROC curves ...")
plot(perf, lty=3, col="grey78", add=TRUE)
perf <- performance(pred, "prec", "rec")
plot(perf, avg= "threshold", colorize=TRUE, lwd= 3,
main= "... Precision/Recall graphs ...")
plot(perf, lty=3, col="grey78", add=TRUE)
perf <- performance(pred, "sens", "spec")
plot(perf, avg= "threshold", colorize=TRUE, lwd= 3,
main="... Sensitivity/Specificity plots ...")
plot(perf, lty=3, col="grey78", add=TRUE)
perf <- performance(pred, "lift", "rpp")
plot(perf, avg= "threshold", colorize=TRUE, lwd= 3,
main= "... and Lift charts.")
plot(perf, lty=3, col="grey78", add=TRUE)
# ------------------------------------------------------------------------------------
data(ROCR.xval)
pp <- ROCR.xval$predictions
ll <- ROCR.xval$labels
pred <- prediction(pp,ll)
perf <- performance(pred,'tpr','fpr')
par(mfrow=c(2,2))
plot(perf, colorize=TRUE, lwd=2,main='ROC curves from 10-fold cross-validation')
plot(perf, avg='vertical', spread.estimate='stderror',lwd=3,main='Vertical averaging + 1 standard error',col='blue')
plot(perf, avg='horizontal', spread.estimate='boxplot',lwd=3,main='Horizontal averaging + boxplots',col='blue')
plot(perf, avg='threshold', spread.estimate='stddev',lwd=2, main='Threshold averaging + 1 standard deviation',colorize=TRUE)
# ------------------------------------------------------------------------------------
data(ROCR.hiv)
pp.unnorm <- ROCR.hiv$hiv.svm$predictions
ll <- ROCR.hiv$hiv.svm$labels
# normalize predictions to 0..1
v <- unlist(pp.unnorm)
pp <- lapply(pp.unnorm, function(run) {approxfun(c(min(v), max(v)), c(0,1))(run)})
par(mfrow=c(2,2))
pred<- prediction(pp, ll)
perf <- performance(pred, "tpr", "fpr")
plot(perf, avg= "threshold", colorize=TRUE, lwd= 3,
coloraxis.at=seq(0,1,by=0.2),
main= "ROC curve")
plot(perf, col="gray78", add=TRUE)
plot(perf, avg= "threshold", colorize=TRUE, colorkey=FALSE,lwd= 3,
main= "ROC curve",add=TRUE)
perf <- performance(pred, "acc")
plot(perf, avg= "vertical", spread.estimate="boxplot", lwd=3,col='blue',
show.spread.at= seq(0.1, 0.9, by=0.1),
main= "Accuracy across the range of possible cutoffs")
plot(performance(pred, "cal", window.size= 10),
avg="vertical",
main= "How well are the probability predictions calibrated?")
plot(0,0,type="n", xlim= c(0,1), ylim=c(0,7),
xlab="Cutoff", ylab="Density",
main="How well do the predictions separate the classes?")
for (runi in 1:length(pred@predictions)) {
lines(density(pred@predictions[[runi]][pred@labels[[runi]]=="-1"]), col= "red")
lines(density(pred@predictions[[runi]][pred@labels[[runi]]=="1"]), col="green")
}
#---------------------------------------------------------------------
par(mfrow= c(2,2))
# ...you can freely combine performance measures (pcmiss,lift)
data(ROCR.xval)
pred <- prediction(ROCR.xval$predictions, ROCR.xval$labels)
perf <- performance(pred,"pcmiss","lift")
# plot(perf, colorize=TRUE)
plot(perf, colorize=TRUE, print.cutoffs.at=seq(0,1,by=0.1), text.adj=c(1.2,1.2), avg="threshold", lwd=3,
main= "You can freely combine performance measures ...")
data(ROCR.simple)
pred <- prediction(ROCR.simple$predictions, ROCR.simple$labels)
perf <- performance(pred,"tpr","fpr")
plot(perf, colorize=TRUE, colorkey.pos="top", print.cutoffs.at=seq(0,1,by=0.1), text.cex=1,
text.adj=c(1.2, 1.2), lwd=2)
# ... cutoff stacking
data(ROCR.xval)
pred <- prediction(ROCR.xval$predictions, ROCR.xval$labels)
perf <- performance(pred,"tpr","fpr")
plot(perf,
print.cutoffs.at=seq(0,1,by=0.2),
text.cex=0.8,
text.y=lapply(as.list(seq(0,0.5,by=0.05)), function(x) { rep(x,length(perf@x.values[[1]])) } ),
col= as.list(terrain.colors(10)),
text.col= as.list(terrain.colors(10)),
points.col= as.list(terrain.colors(10)),
main= "Cutoff stability")
# .... no functional dependencies needed, truly parametrized curve
data(ROCR.xval)
pred <- prediction(ROCR.xval$predictions, ROCR.xval$labels)
perf <- performance(pred,"acc","lift")
plot(perf, colorize=TRUE, main="Truly parametrized curves")
plot(perf, colorize=TRUE, print.cutoffs.at=seq(0,1,by=0.1), add=TRUE, text.adj=c(1.2, 1.2), avg="threshold", lwd=3)
# --------------------------------------------------------------------
# (Expected cost) curve + ROC convex hull
par(mfrow=c(1,2))
data(ROCR.hiv)
plot(0,0,xlim=c(0,1),ylim=c(0,1),xlab='Probability cost function',
ylab="Normalized expected cost",
main='HIV data: Expected cost curve (Drummond & Holte)')
pred<-prediction(ROCR.hiv$hiv.nn$predictions,ROCR.hiv$hiv.nn$labels)
lines(c(0,1),c(0,1))
lines(c(0,1),c(1,0))
perf1 <- performance(pred,'fpr','fnr')
for (i in 1:length(perf1@x.values)) {
for (j in 1:length(perf1@x.values[[i]])) {
lines(c(0,1),c(perf1@y.values[[i]][j], perf1@x.values[[i]][j]),col=rev(terrain.colors(10))[i],lty=3)
}
}
perf<-performance(pred,'ecost')
plot(perf,lwd=1.5,xlim=c(0,1),ylim=c(0,1),add=TRUE)
# RCH
data(ROCR.simple)
ROCR.simple$labels[ROCR.simple$predictions >= 0.7 & ROCR.simple$predictions < 0.85] <- 0
#as.numeric(!labels[predictions >= 0.6 & predictions < 0.85])
pred <- prediction(ROCR.simple$predictions, ROCR.simple$labels)
perf <- performance(pred,'tpr','fpr')
plot(perf, main="ROC curve with concavities (suboptimal) and ROC convex hull (Fawcett)")
perf1 <- performance(pred,'rch')
plot(perf1,add=TRUE,col='red',lwd=2)
#---------------------------------------------------------------------
# (plotting cutoff vs. measure)
data(ROCR.hiv)
pp <- ROCR.hiv$hiv.svm$predictions
ll <- ROCR.hiv$hiv.svm$labels
measures <- c('tpr','fpr','acc','err','rec','sens','fnr','tnr','spec',
'ppv','prec','npv','fall','miss','pcfall','pcmiss',
'phi','mat','mi','chisq','odds','lift','f')
## Don't be surprised by the decreased cutoff regions produced by 'odds ratio'.
## Cf. ?performance for details.
pred <- prediction(pp, ll)
par(mfrow=c(5,5))
for (measure in measures) {
perf <- performance(pred, measure)
plot(perf,avg="vertical",spread.estimate="boxplot")
}
#---------------------------------------------------------------------
measures <- c('tpr','err','prec','phi','mi','chisq','odds','lift','f')
par(mfrow=c(6,6))
for (i in 1:(length(measures)-1)) {
for (j in (i+1):length(measures)) {
perf <- performance(pred, measures[i], measures[j])
plot(perf, avg="threshold", colorize=TRUE)
}
}
#---------------------------------------------------------------------
data(ROCR.hiv)
pp <- ROCR.hiv$hiv.svm$predictions
ll <- ROCR.hiv$hiv.svm$labels
data(ROCR.xval)
pp <- ROCR.xval$predictions
ll <- ROCR.xval$labels
pred <- prediction(pp, ll)
par(mfrow=c(3,3))
perf <- performance(pred, "odds", "fpr")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "phi", "err")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "f", "err")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "f", "ppv")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "mat", "ppv")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "npv", "ppv")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "acc", "phi")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "lift", "phi")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "f", "phi")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "mi", "phi")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "chisq", "phi")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "acc", "mi")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "fall", "odds")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "tpr", "lift")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "fall", "lift")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "npv", "f")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "prec", "f")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
perf <- performance(pred, "tpr", "f")
plot(perf, colorize=TRUE)
plot(perf, avg="threshold", lwd=2, add=TRUE)
par(opar)
ROCR/demo/00Index 0000644 0001762 0000144 00000000077 13644317760 013114 0 ustar ligges users ROCR demonstrates some of the graphical capabilities of ROCR
ROCR/README.md 0000644 0001762 0000144 00000010470 13647572625 012321 0 ustar ligges users # ROCR
[](https://github.com/ipa-tys/ROCR/actions?query=workflow:R-CMD-check)
[](https://CRAN.r-project.org/package=ROCR)
[](https://codecov.io/gh/ipa-tys/ROCR)
*visualizing classifier performance in R, with only 3 commands*

### Please support our work by citing the ROCR article in your publications:
***Sing T, Sander O, Beerenwinkel N, Lengauer T. [2005]
ROCR: visualizing classifier performance in R.
Bioinformatics 21(20):3940-1.***
Free full text:
http://bioinformatics.oxfordjournals.org/content/21/20/3940.full
[
](https://www.mpi-inf.mpg.de/home/)
`ROCR` was originally developed at the [Max Planck Institute for Informatics](https://www.mpi-inf.mpg.de/home/)
## Introduction
`ROCR` (with obvious pronounciation) is an R package for evaluating and visualizing classifier performance. It is...
- ...easy to use: adds only three new commands to R.
- ...flexible: integrates tightly with R's built-in graphics facilities.
- ...powerful: Currently, 28 performance measures are implemented, which can be freely combined to form parametric curves such as ROC curves, precision/recall curves, or lift curves. Many options such as curve averaging (for cross-validation or bootstrap), augmenting the averaged curves by standard error bar or boxplots, labeling cutoffs to the curve, or coloring curves according to cutoff.
### Performance measures that `ROCR` knows:
Accuracy, error rate, true positive rate, false positive rate, true negative rate, false negative rate, sensitivity, specificity, recall, positive predictive value, negative predictive value, precision, fallout, miss, phi correlation coefficient, Matthews correlation coefficient, mutual information, chi square statistic, odds ratio, lift value, precision/recall F measure, ROC convex hull, area under the ROC curve, precision/recall break-even point, calibration error, mean cross-entropy, root mean squared error, SAR measure, expected cost, explicit cost.
### `ROCR` features:
ROC curves, precision/recall plots, lift charts, cost curves, custom curves by freely selecting one performance measure for the x axis and one for the y axis, handling of data from cross-validation or bootstrapping, curve averaging (vertically, horizontally, or by threshold), standard error bars, box plots, curves that are color-coded by cutoff, printing threshold values on the curve, tight integration with Rs plotting facilities (making it easy to adjust plots or to combine multiple plots), fully customizable, easy to use (only 3 commands).
## Installation of `ROCR`
The most straightforward way to install and use `ROCR` is to install it from
`CRAN` by starting `R` and using the `install.packages` function:
```
install.packages("ROCR")
```
Alternatively you can install it from command line using the tar ball like this:
```
R CMD INSTALL ROCR_*.tar.gz
```
## Getting started
from withing R ...
```
library(ROCR)
demo(ROCR)
help(package=ROCR)
```
## Examples
Using ROCR's 3 commands to produce a simple ROC plot:
```
pred <- prediction(predictions, labels)
perf <- performance(pred, measure = "tpr", x.measure = "fpr")
plot(perf, col=rainbow(10))
```
## Documentation
- The Reference Manual found [here](https://CRAN.r-project.org/package=ROCR)
- Slide deck for a tutorial talk (feel free to re-use for teaching, but please give appropriate credits and write us an email) [[PPT](https://raw.githubusercontent.com/ipa-tys/ROCR/rocr-images/ROCR_Talk_Tobias_Sing.ppt)]
- A few pointers to the literature on classifier evaluation
## Contact
Questions, comments, and suggestions are very welcome. Open an issue on GitHub and we can discuss. We are also interested in seeing how ROCR is used in publications. Thus, if you have prepared a paper using ROCR we'd be happy to know.
ROCR/data/ 0000755 0001762 0000144 00000000000 13644317760 011743 5 ustar ligges users ROCR/data/ROCR.xval.rda 0000644 0001762 0000144 00000050121 13644317760 014150 0 ustar ligges users ŽX7](v 6&آ؊-*("at.,:}~,u{S3s9g̜/.hi#Cڵ[1O-:
̛k[ԩv_9oƜ8ߜ7\;7$
]Ny6(K%^~@)s)y9}Bl'"O)ϨIj
rz6>Rc(-䴅Sg|z~LJJ|BނS(|#[ÁrMXB:>2˹Ot"*ψTbҏ(E,Ne't _ZJombMJsF'v9W7;Kywo1SH8N; T&e_a(I 'U<@Jt:M&^1;Q 0:ݎ!o\Wx_)U%sqO#:L-)cQɱQn-M1tĹGgkcP&MyN
/=p^gA~&؞!(?%rql&muriO~6qCY鄧D2oK[$xޤ,)~c۰/wj#MRʳEe,ȝ,\My+!_#'\r
Bt Zt*] x4£MXgyDX{R}3e3Qb^T_!XGz cyM _1vS~s;֗Bbd
t%W7XZAoŠ#џy+[3nKӁExR,)l}qj)\[rZ'z)pJ~I
A~A~Z4^͢*/N|)}oHp+2jiu$sr9LBrUk^z5IW[aډhӰ;`;cΦ,L3̏uƟh#f?.5/pGOD]8""I7/<7]fB>yzM / o}kOE;9$;eYC|^{n-~!t\:QK3~CyyNrk}ӂ| ź2qsN~53%WK]I-8'(|[Fi?-%z'K7;($za=SP36iի6'#;w̢h7M/@=ZzBoY}@ +YܐHRB}3͌:Cb~ݱokO2';SRIzs
zbztx;
_!~'(1~UQ 'qQ&ԾDa%Q;6ʅ<Iey0haRh@#aB
dc'ֺO^@1efO^o_V./Av/yƽ;wHE ta5Mѝ3ǟ
C">1^P^ ]R)X}I}p(Ҥ ?7RŏB\8S]41ط(f":F ;/wW;=*!=;y`.sS;{{*/95?v$M!sd`]K=-?#mĒྭ3
tr}D픿P.dQ (&>=MZdBt _x1/?o]
aLכx!{Quʇ< Ž
ا3nK~;"z9F)M-꜏b4#3rHNDʝNlGXJo͒`ziׇুW0E<rat| _ZߞC&)-~|
瘴kJ H.ɳ -'2z,fLV`tHzՌۨ^wf^⭇2&I.1'̆{{x;;EߛҘ?RĘ:9aXf$rSԣuwA$JW5|p1_p_KpsBJ!on4r̈́I4xt|gg<qP;hf իm/zQXgR/
rw?f9 W@oR_{؝9ɱ
_1yŏ}1b7*Tݷ /&igjfcJYR9:R2˃/97h(W5N]^~-;M.O2|yz20A&SZ{uI>mK左Qo
ASh?!vo7rz7g#>ѡi !s8eN}D/GNEm