diff --git a/NAMESPACE b/NAMESPACE index 3405d737..0429f1b5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -145,6 +145,7 @@ export(pareto_k_ids) export(pareto_k_influence_values) export(pareto_k_table) export(pareto_k_values) +export(plot_loo_difference) export(pointwise) export(print_dims) export(pseudobma_weights) diff --git a/R/loo_difference_plot.R b/R/loo_difference_plot.R new file mode 100644 index 00000000..dc60f3f5 --- /dev/null +++ b/R/loo_difference_plot.R @@ -0,0 +1,213 @@ +#' Compare models across domains +#' +#' The LOO difference plot shows how the ELPD of two different models +#' changes when a predictor is varied. This can be useful for identifying +#' opportunities for model stacking or expansion. Pointwise differences +#' are computed as `loo_1 - loo_2`, so positive values indicate better +#' predictive performance for `loo_1`. +#' +#' @param y A vector of observations. +#' @param loo_1,loo_2 Objects returned by [loo()]. +#' @param group An optional grouping variable with the same length as `y`. +#' Points are colored according to group membership. +#' @param size,alpha Point size and opacity passed to [ggplot2::geom_point()]. +#' @param jitter Amount of horizontal jitter passed to +#' [ggplot2::position_jitter()]. +#' @param sort_by_group If `TRUE`, observations are ordered by `group` +#' and the x-axis is replaced by a sequential index. The supplied `y` values +#' are therefore not used as x coordinates. Plotting by index can be useful +#' when categories have very different sample sizes. To control the group +#' order, supply `group` as a factor with levels in the desired order. +#' @param label_threshold Optional nonnegative threshold for labeling +#' observations. Observations for which the absolute pointwise ELPD +#' difference exceeds this value are labeled. If `NULL`, no observations +#' are labeled. +#' @param labels Optional vector of labels with the same length as `y`, used +#' for observations selected by `label_threshold`. If `NULL`, observation +#' indices are used. +#' +#' @template bayesvis-reference +#' +#' @return A [ggplot2::ggplot()] object. +#' +#' @examples +#' # Artificial example +#' log_lik <- example_loglik_matrix() +#' shift <- seq(-0.5, 0.5, length.out = ncol(log_lik)) +#' log_lik_2 <- sweep(log_lik, 2, shift, FUN = "+") +#' +#' loo_1 <- loo(log_lik) +#' loo_2 <- loo(log_lik_2) +#' +#' plot_loo_difference( +#' seq_len(ncol(log_lik)), +#' loo_1, +#' loo_2 +#' ) +#' +#' # Label observations with large pointwise ELPD differences +#' plot_loo_difference( +#' seq_len(ncol(log_lik)), +#' loo_1, +#' loo_2, +#' label_threshold = 0.3 +#' ) +#' +#' # Create interspersed groups, then sort them in the plot +#' group <- rep(c("A", "A", "A", "B"), length.out = ncol(log_lik)) +#' +#' plot_loo_difference( +#' seq_len(ncol(log_lik)), +#' loo_1, +#' loo_2, +#' group = group, +#' sort_by_group = TRUE +#' ) +#' +#' @export +plot_loo_difference <- + function( + y, + loo_1, + loo_2, + group = NULL, + size = 1, + alpha = 1, + jitter = 0, + sort_by_group = FALSE, + label_threshold = NULL, + labels = NULL + ) { + if (!requireNamespace("ggplot2", quietly = TRUE)) { + stop( + "Please install 'ggplot2' to use `plot_loo_difference()`.", + call. = FALSE + ) + } + + checkmate::assert_flag(sort_by_group) + loo_compare_checks(nlist(loo_1, loo_2)) + + # elpd_diffs(a, b) computes b - a + elpd_diff <- elpd_diffs(loo_2, loo_1) + + checkmate::assert_atomic_vector( + y, + len = length(elpd_diff) + ) + + if (!is.null(group)) { + checkmate::assert_atomic_vector( + group, + len = length(y), + any.missing = FALSE + ) + } + + if (!is.null(labels)) { + checkmate::assert_atomic_vector( + labels, + len = length(y) + ) + + if (is.null(label_threshold)) { + stop( + "`label_threshold` must be supplied when `labels` is supplied.", + call. = FALSE + ) + } + } + + if (!is.null(label_threshold)) { + checkmate::assert_number( + label_threshold, + lower = 0, + finite = TRUE + ) + } + + if (!is.null(label_threshold) && is.null(labels)) { + labels <- seq_along(y) + } + + if (sort_by_group) { + if (is.null(group)) { + stop( + "`group` must be supplied when `sort_by_group = TRUE`.", + call. = FALSE + ) + } + + ordering <- order(group) + elpd_diff <- elpd_diff[ordering] + group <- group[ordering] + + if (!is.null(labels)) { + labels <- labels[ordering] + } + + y <- seq_along(elpd_diff) + } + + plot_data <- data.frame( + y = y, + elpd_diff = elpd_diff + ) + + if (!is.null(group)) { + plot_data$group <- factor(group) + } + + if (!is.null(label_threshold)) { + plot_data$labels <- ifelse( + abs(plot_data$elpd_diff) > label_threshold, + as.character(labels), + "" + ) + } + + jitter_position <- ggplot2::position_jitter( + width = jitter, + height = 0, + seed = 1 + ) + + plot <- ggplot2::ggplot( + data = plot_data, + mapping = ggplot2::aes(x = y, y = elpd_diff) + ) + + ggplot2::geom_hline(yintercept = 0) + + ggplot2::labs( + x = if (sort_by_group) "Index" else NULL, + y = "Pointwise ELPD Difference (loo_1 - loo_2)" + ) + + if (is.null(group)) { + plot <- plot + + ggplot2::geom_point( + position = jitter_position, + alpha = alpha, + size = size + ) + } else { + plot <- plot + + ggplot2::geom_point( + ggplot2::aes(color = group), + position = jitter_position, + alpha = alpha, + size = size + ) + + ggplot2::labs(color = "Group") + } + + if (!is.null(label_threshold)) { + plot <- plot + + ggplot2::geom_text( + ggplot2::aes(label = labels), + position = jitter_position, + vjust = -0.5 + ) + } + + plot + } diff --git a/_pkgdown.yml b/_pkgdown.yml index 0a216a02..9acc5235 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -95,6 +95,7 @@ reference: - loo_model_weights - stacking_weights - pseudobma_weights + - plot_loo_difference - title: Helper functions for K-fold CV contents: - kfold_split_random diff --git a/man/plot_loo_difference.Rd b/man/plot_loo_difference.Rd new file mode 100644 index 00000000..e628913c --- /dev/null +++ b/man/plot_loo_difference.Rd @@ -0,0 +1,100 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/loo_difference_plot.R +\name{plot_loo_difference} +\alias{plot_loo_difference} +\title{Compare models across domains} +\usage{ +plot_loo_difference( + y, + loo_1, + loo_2, + group = NULL, + size = 1, + alpha = 1, + jitter = 0, + sort_by_group = FALSE, + label_threshold = NULL, + labels = NULL +) +} +\arguments{ +\item{y}{A vector of observations.} + +\item{loo_1, loo_2}{Objects returned by \code{\link[=loo]{loo()}}.} + +\item{group}{An optional grouping variable with the same length as \code{y}. +Points are colored according to group membership.} + +\item{size, alpha}{Point size and opacity passed to \code{\link[ggplot2:geom_point]{ggplot2::geom_point()}}.} + +\item{jitter}{Amount of horizontal jitter passed to +\code{\link[ggplot2:position_jitter]{ggplot2::position_jitter()}}.} + +\item{sort_by_group}{If \code{TRUE}, observations are ordered by \code{group} +and the x-axis is replaced by a sequential index. The supplied \code{y} values +are therefore not used as x coordinates. Plotting by index can be useful +when categories have very different sample sizes. To control the group +order, supply \code{group} as a factor with levels in the desired order.} + +\item{label_threshold}{Optional nonnegative threshold for labeling +observations. Observations for which the absolute pointwise ELPD +difference exceeds this value are labeled. If \code{NULL}, no observations +are labeled.} + +\item{labels}{Optional vector of labels with the same length as \code{y}, used +for observations selected by \code{label_threshold}. If \code{NULL}, observation +indices are used.} +} +\value{ +A \code{\link[ggplot2:ggplot]{ggplot2::ggplot()}} object. +} +\description{ +The LOO difference plot shows how the ELPD of two different models +changes when a predictor is varied. This can be useful for identifying +opportunities for model stacking or expansion. Pointwise differences +are computed as \code{loo_1 - loo_2}, so positive values indicate better +predictive performance for \code{loo_1}. +} +\examples{ +# Artificial example +log_lik <- example_loglik_matrix() +shift <- seq(-0.5, 0.5, length.out = ncol(log_lik)) +log_lik_2 <- sweep(log_lik, 2, shift, FUN = "+") + +loo_1 <- loo(log_lik) +loo_2 <- loo(log_lik_2) + +plot_loo_difference( + seq_len(ncol(log_lik)), + loo_1, + loo_2 +) + +# Label observations with large pointwise ELPD differences +plot_loo_difference( + seq_len(ncol(log_lik)), + loo_1, + loo_2, + label_threshold = 0.3 +) + +# Create interspersed groups, then sort them in the plot +group <- rep(c("A", "A", "A", "B"), length.out = ncol(log_lik)) + +plot_loo_difference( + seq_len(ncol(log_lik)), + loo_1, + loo_2, + group = group, + sort_by_group = TRUE +) + +} +\references{ +Gabry, J. , Simpson, D. , Vehtari, A. , Betancourt, M. and +Gelman, A. (2019), Visualization in Bayesian workflow. +\emph{J. R. Stat. Soc. A}, 182: 389-402. doi:10.1111/rssa.12378 +(\href{https://rss.onlinelibrary.wiley.com/doi/full/10.1111/rssa.12378}{journal version}, +\href{https://arxiv.org/abs/1709.01449}{preprint arXiv:1709.01449}, +\href{https://github.com/jgabry/bayes-vis-paper}{code on GitHub}) +} diff --git a/tests/testthat/test-loo_difference_plot.R b/tests/testthat/test-loo_difference_plot.R new file mode 100644 index 00000000..7e294f55 --- /dev/null +++ b/tests/testthat/test-loo_difference_plot.R @@ -0,0 +1,125 @@ +skip_if_not_installed("ggplot2") + +log_lik <- example_loglik_matrix() +shift <- seq(-0.5, 0.5, length.out = ncol(log_lik)) +log_lik_2 <- sweep(log_lik, 2, shift, FUN = "+") + +loo_1 <- loo(log_lik) +loo_2 <- loo(log_lik_2) +y <- seq_len(ncol(log_lik)) + +test_that("plot_loo_difference returns pointwise ELPD differences", { + p <- plot_loo_difference(y, loo_1, loo_2) + + expect_s3_class(p, "ggplot") + expect_equal(p$data$y, y) + expect_equal(p$data$elpd_diff, -shift) +}) + +test_that("plot_loo_difference sorts observations and labels by group", { + group <- factor( + rep(c("A", "B"), length.out = length(y)), + levels = c("B", "A") + ) + labels <- paste0("obs", y) + ordering <- order(group) + + p <- plot_loo_difference( + y, + loo_1, + loo_2, + group = group, + sort_by_group = TRUE, + label_threshold = 0.3, + labels = labels + ) + + expect_equal(p$data$y, seq_along(y)) + expect_equal(p$data$elpd_diff, -shift[ordering]) + expect_equal( + as.character(p$data$group), + as.character(group[ordering]) + ) + expect_equal( + p$data$labels, + ifelse( + abs(shift[ordering]) > 0.3, + labels[ordering], + "" + ) + ) +}) + +test_that("plot_loo_difference uses observation indices as default labels", { + p <- plot_loo_difference( + y, + loo_1, + loo_2, + label_threshold = 0.3 + ) + + expect_equal( + p$data$labels, + ifelse( + abs(shift) > 0.3, + as.character(seq_along(y)), + "" + ) + ) +}) + +test_that("plot_loo_difference checks observation-level arguments", { + expect_error( + plot_loo_difference(y[-1], loo_1, loo_2) + ) + + expect_error( + plot_loo_difference( + y, + loo_1, + loo_2, + group = rep("A", length(y) - 1) + ) + ) + + expect_error( + plot_loo_difference( + y, + loo_1, + loo_2, + label_threshold = -1 + ) + ) + + expect_error( + plot_loo_difference( + y, + loo_1, + loo_2, + label_threshold = 0.3, + labels = y[-1] + ) + ) + + expect_error( + plot_loo_difference( + y, + loo_1, + loo_2, + labels = y + ), + "`label_threshold` must be supplied when `labels` is supplied.", + fixed = TRUE + ) + + expect_error( + plot_loo_difference( + y, + loo_1, + loo_2, + sort_by_group = TRUE + ), + "`group` must be supplied", + fixed = TRUE + ) +})