diff --git a/.ci/atime/bisect-7907.R b/.ci/atime/bisect-7907.R new file mode 100644 index 0000000000..a2ff3e68ce --- /dev/null +++ b/.ci/atime/bisect-7907.R @@ -0,0 +1,23 @@ +## This script is for running the git bisect analysis in PR7907, which +## revealed two slowdowns in fread(). +pkg.path <- "~/R/data.table" +Test <- "fread N=cols regression" +library(data.table) +if(FALSE){ + ## if you want to start the bisect over from scratch, first remove + ## all historical versions of data.table installed by atime: + atime::atime_versions_remove("data.table") +} +## These are the commits visualized on +## https://tdhock.github.io/2026-10-15-bisect-performance-pr7907 +(bisect.dt <- rowwiseDT( + old=, new=, label=, expected=, stage=, + "Fast", "Slow", "Parent7370", "?", "discovery", + "PR7406", "Slow", "PR7370", "?", "discovery", + "Fast", "PR7383", "PR4164", "?", "discovery", + "PR7375", "Slow", "PR7370", "PR7370", "final", + "Fast", "Parent7370", "PR7375", "PR7375", "final")) +for(bisect.i in 1:nrow(bisect.dt)){ + print(bisect.row <- bisect.dt[bisect.i]) + bisect.row[, atime::bisect(pkg.path, Test, old, new)] +} diff --git a/.ci/atime/bisect-animint.R b/.ci/atime/bisect-animint.R new file mode 100644 index 0000000000..8ebae965d8 --- /dev/null +++ b/.ci/atime/bisect-animint.R @@ -0,0 +1,148 @@ +## This script is for visualizing results computed in bisect-7907.R +library(animint2) +library(data.table) + +Test.dir <- "bisect/fread_N_cols_regression/Fast-Slow" +(results.wide <- data.table( + csv=Sys.glob(file.path(Test.dir, "csv", "*")) +)[ +, fread(csv), by=csv +][ +, rank := rank(time) +][order(rank)][, let( + Rank = factor(rank, labels=format(time)), + Status = factor(status) +)][]) +(results.long <- melt( + results.wide, + measure.vars=measure(variable, expr.name, sep="."))) + +details.dt.list <- list() +dot.dt.list <- list() +common.dt.list <- list() +for(result.RData in Sys.glob(file.path(Test.dir, "RData", "*"))){ + (objs <- load(result.RData)) + commit <- sub(".RData", "", basename(result.RData)) + details <- tres.or.status$measurements[, .( + commit, expr.name, N, median, min, max + )] + common.dt.list[[commit]] <- details[, .( + versions=.N + ), by=N][ + versions==max(versions) + ][.N, data.table(commit, N)] + details.dt.list[[commit]] <- details + dot.dt.list[[commit]] <- data.table( + commit, + tpred$prediction) +} +(details.dt <- rbindlist(details.dt.list)) +(dot.dt <- rbindlist(dot.dt.list)) +(common.dt <- rbindlist(common.dt.list)) + +limit.dt <- data.table(seconds=tpred$seconds.limit) +animint( + overview=ggplot()+ + ggtitle("Overview, select commit")+ + theme_bw()+ + theme_animint(height=600)+ + geom_tallrect(aes( + xmin=rank-0.5, xmax=rank+0.5, + key=rank, + fill=Status), + color=NA, + data=results.wide)+ + geom_line(aes( + rank, value, color=expr.name, group=expr.name), + data=results.long)+ + facet_grid(variable ~ ., scales="free")+ + theme(axis.text.x=element_text(angle=30, hjust=1))+ + geom_label_aligned(aes( + rank, value, label=expr.name, color=expr.name), + data=results.long[rank==max(rank)], + hjust=0, + alignment="vertical")+ + scale_fill_manual(values=c( + "0"="grey", + "1"="white"))+ + scale_y_continuous("")+ + scale_x_continuous("commit rank over time")+ + geom_tallrect(aes( + xmin=rank-0.5, xmax=rank+0.5), + clickSelects="commit", + alpha=1, + alpha_off=0, + color="black", + color_off=NA, + fill=NA, + data=results.wide), + details=ggplot()+ + ggtitle("Details for selected commit")+ + theme_bw()+ + theme(legend.position="none")+ + theme_animint(width=600, height=600, last_in_row=TRUE)+ + geom_hline(aes( + yintercept=seconds), + color="grey", + data=limit.dt)+ + geom_text(aes( + 0, seconds*1.1, + label=sprintf( + "time limit = %s seconds", + paste(seconds) + )), + hjust=0, + color="grey50", + data=limit.dt)+ + geom_vline(aes( + xintercept=N, + key=1), + data=common.dt, + color="grey", + showSelected="commit")+ + geom_text(aes( + N, 0, label=sprintf("largest common N=%s", paste(N)), + key=1), + data=common.dt, + hjust=1, + color="grey50", + showSelected="commit")+ + geom_text(aes( + 0, 0, label=commit, + key=1), + hjust=0, + vjust=0, + data=common.dt, + showSelected="commit")+ + geom_ribbon(aes( + N, ymin=min, ymax=max, + key=expr.name, fill=expr.name, group=expr.name), + showSelected=c("expr.name","commit"), + alpha=0.5, + color=NA, + data=details.dt)+ + geom_line(aes( + N, median, + key=expr.name, color=expr.name, group=expr.name), + showSelected=c("expr.name","commit"), + data=details.dt)+ + geom_point(aes( + N, unit.value, + key=expr.name, color=expr.name), + showSelected=c("expr.name","commit"), + fill="white", + data=dot.dt)+ + geom_label_aligned(aes( + N, unit.value, label=expr.name, + key=expr.name, color=expr.name), + showSelected=c("expr.name","commit"), + alignment="horizontal", + vjust=-0.1, + alpha=0.5, + data=dot.dt)+ + scale_x_log10()+ + scale_y_log10("seconds"), + duration=list( + commit=1000) +) + diff --git a/.ci/atime/tests.R b/.ci/atime/tests.R index 2d342ded67..f8b58a073a 100644 --- a/.ci/atime/tests.R +++ b/.ci/atime/tests.R @@ -1,3 +1,9 @@ +bisect.restore.fun <- function(tinfo){ + ## necessary to avoid failed checkout when bisecting some old commits + ## https://github.com/Rdatatable/data.table/issues/7909 + cat("", file=file.path(tinfo$checkout.path, ".gitattributes")) + gert::git_restore(".", repo=tinfo$checkout.path) +} pval.thresh <- 0.001 # to reduce false positives. # Test case adapted from https://github.com/Rdatatable/data.table/issues/6105#issue-2268691745 which is where the issue was reported. @@ -260,6 +266,27 @@ test.list <- atime::atime_test_list( } }, + "fread N=cols regression" = atime::atime_test( + N = unique(as.integer(2^seq(0, 20, by=0.25))), # finer N grid for git bisect. + setup={ + set.seed(1) + n.rows <- 100 + input.vec <- rnorm(n.rows*N) + input.mat <- matrix(input.vec, n.rows, N) + input.dt <- data.table(input.mat) + input.csv <- tempfile() + data.table::fwrite(input.dt, input.csv) + }, + seconds.limit=0.1, + Fast="1.14.8", + Slow="1.18.6", + bisect.commits=c( + PR7375="59f966cf6839a0494e42ed1ab98616eea03dcb36", #first fread slowdown to intermediate performance, added comment.char arg + PR7383="67db7f7fb33b99da8cc5dc575714924b96d0f0fe", #intermediate performance, using this as new in bisect with old=1.14.8 yields PR#7375. + Parent7370="1685a3b47d48f323afeae589545f7cbb7717ec28", #intermediate performance, using this as old in bisect with new=1.18.6 yields PR#7370. + PR7370="df7fa8071c6b818d24e47cc1a84a6dc0491550d6"), #second slowdown to same performance as Slow, sep detection single column quoted + expr=data.table::fread(input.csv, showProgress = FALSE, nThread=1)), + # Constant overhead improvement https://github.com/Rdatatable/data.table/pull/6925 # Test case adapted from https://github.com/Rdatatable/data.table/pull/7022#discussion_r2107900643 "fread disk overhead improved in #6925" = atime::atime_test(