Trying out model calibration with probably

[dchiu911]

library(probably)

mod1 <- mod_two_step_full$hybrid_rf_s1
mod2 <- mod_two_step_full$hybrid_svm_s2

cal_pred1 <-
  augment(mod1, new_data = train_calib) %>%
  mutate(truth = factor(hist_gr)) %>%
  select(matches(".pred"), truth)

cal_pred1 %>%
  mutate(.class_pred = make_two_class_pred(
    estimate = .pred_HGSC,
    levels = levels(truth),
    buffer = 0.01
  )) %>%
  mutate(.pred_fct = as.factor(.class_pred)) %>%
  filter(is.na(.pred_fct))

cal_pred2 <-
  augment(mod2, new_data = train_calib) %>%
  mutate(truth = factor(revHist)) %>%
  select(matches(".pred"), truth) %>%
  filter(truth != "HGSC") %>%
  droplevels()

cal_pred2 %>%
  mutate(.class_pred = make_class_pred(
    estimate = .pred_CCOC, .pred_ENOC, .pred_LGSC, .pred_MUC,
    levels = levels(truth),
    min_prob = 0.5
  )) %>%
  count(.class_pred)

smoothed_iso1 <- cal_estimate_isotonic(conf_pred1, truth = class)
smoothed_iso2 <- cal_estimate_multinomial(conf_pred2, truth = class, smooth = FALSE)

conf_pred1 <-
  augment(mod1, new_data = conf_data) %>%
  mutate(class = factor(ifelse(
    conf_class != "HGSC", "non-HGSC", conf_class
  )))

conf_pred2 <-
  augment(mod2, new_data = conf_data) %>%
  mutate(class = factor(conf_class))

conf_pred1 %>%
  cal_apply(smoothed_iso1) %>%
  cal_plot_windowed(truth = class, estimate = .pred_HGSC)

conf_pred2 %>%
  cal_apply(smoothed_iso2) %>%
  cal_plot_windowed(truth = class)

conf_pred2 %>%
  cal_apply(smoothed_iso2) %>%
  arrange(desc(.pred_MUC))


conf_pred1 %>%
  cal_apply(smoothed_iso1, pred_class = .pred_class) %>%
  mutate(.pred_class_new = factor(ifelse(.pred_HGSC >= 0.5, "HGSC", "non-HGSC"))) %>%
  filter(.pred_class != .pred_class_new)
  # conf_mat(class, .pred_class_new)

conf_pred2 %>%
  cal_apply(smoothed_iso2) %>%
  arrange(desc(.pred_MUC))