predieval {predieval}

Calculating measures for calibration for benefit for a prediction model

Description

This function calculates a series of measures to assess decision accuracy, discrimination for benefit, and calibration for benefit of a prediction model.

Usage

predieval(
  repeats = 50,
  Ngroups = 10,
  X,
  treat,
  Y,
  predicted.treat.1,
  predicted.treat.0,
  type = "continuous",
  bootstraps = 500
)

Arguments

repeats

The number of repetitions for the algorithm.

Ngroups

The number of groups to split the data.

X

A dataframe with patient covariates.

treat

A vector with the treatment assignment. This must be 0 (for control treatment) or 1 (for active treatment).

Y

The observed outcome. For binary outcomes this should be 0 or 1

predicted.treat.1

A vector with the model predictions for each patient, under the active treatment. For the case of a binary outcome this should be probabilities of an event.

predicted.treat.0

A vector with the model predictions for each patient, under the control treatment. For the case of a binary outcome this should be probabilities of an event.

type

The type of the outcome, "binary" or "continuous".

bootstraps

The number of bootstrap samples to be used for calculating confidence intervals.

Value

A table with all estimated measures of performance.

Examples

 # continuous outcome
 dat0=simcont(500)$dat
 head(dat0)
 # Randomly shuffle the data
 dat<-dat0[sample(nrow(dat0)),]
 # Create random folds
 dat$folds <- cut(seq(1,nrow(dat)),breaks=10,labels=FALSE)

 # Obtain out-of-sample predictions
 dat.out.CV<-list()
 for (i in 1:10){
   dat.in.CV=dat[dat$folds!=i,]
   dat.out.CV[[i]]=dat[dat$folds==i,]
   dat1<-dat.out.CV[[i]]; dat1$t=1
   dat0<-dat.out.CV[[i]]; dat0$t=0
   m1=lm(data=dat.in.CV, y.observed~x1*t+x2*t)
   dat.out.CV[[i]]$predict.treat.1=predict(newdata=dat1, m1)# predictions in treatment
   dat.out.CV[[i]]$predict.treat.0=predict(newdata=dat0, m1)# predicions in control
 }

 dat.CV=dat.out.CV[[1]]
 for (i in 2:10){  dat.CV=rbind(dat.CV,dat.out.CV[[i]])}

 # assess model performance
 predieval(repeats=20, Ngroups=c(5:10),
             X=dat.CV[,c("x1", "x2","x3")],
             Y=dat.CV$y.observed,
             predicted.treat.1 = dat.CV$predict.treat.1,
             predicted.treat.0 = dat.CV$predict.treat.0,
             treat=dat.CV$t, type="continuous")


 # binary outcome
 dat0=simbinary(500)$dat
 head(dat0)

 # Randomly shuffle the data
 dat<-dat0[sample(nrow(dat0)),]
 # Create random folds
 dat$folds <- cut(seq(1,nrow(dat)),breaks=10,labels=FALSE)

 dat.out.CV<-list()
 for (i in 1:10){
   dat.in.CV=dat[dat$folds!=i,]
   dat.out.CV[[i]]=dat[dat$folds==i,]
   dat1<-dat.out.CV[[i]]; dat1$t=1
   dat0<-dat.out.CV[[i]]; dat0$t=0
   glm1=glm(y.observed~(x1+x2+x3)*t, data=dat.in.CV, family = binomial(link = "logit"))
   dat.out.CV[[i]]$predict.treat.1=predict(newdata=dat1, glm1)# predictions in treatment
   dat.out.CV[[i]]$predict.treat.0=predict(newdata=dat0, glm1)# predicions in control
 }

 dat.CV=dat.out.CV[[1]]
 for (i in 2:10){  dat.CV=rbind(dat.CV,dat.out.CV[[i]])}


 predieval(repeats=20, Ngroups=c(5:10), X=dat.CV[,c("x1", "x2","x3")],
             Y=dat.CV$y.observed,
             predicted.treat.1 = expit(dat.CV$predict.treat.1),
             predicted.treat.0 = expit(dat.CV$predict.treat.0),
             treat=dat.CV$t, type="binary",bootstraps = 50)