Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 112 additions & 2 deletions R/data.table.R
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ replace_dot_alias = function(e) {
}
}

"[.data.table" = function (x, i, j, by, keyby, with=TRUE, nomatch=getOption("datatable.nomatch", NA), mult="all", roll=FALSE, rollends=if (roll=="nearest") c(TRUE,TRUE) else if (roll>=0) c(FALSE,TRUE) else c(TRUE,FALSE), which=FALSE, .SDcols, verbose=getOption("datatable.verbose"), allow.cartesian=getOption("datatable.allow.cartesian"), drop=NULL, on=NULL)
"[.data.table" = function (x, i, j, by, keyby, with=TRUE, nomatch=getOption("datatable.nomatch", NA), mult="all", roll=FALSE, rollends=if (roll=="nearest") c(TRUE,TRUE) else if (roll>=0) c(FALSE,TRUE) else c(TRUE,FALSE), which=FALSE, .SDcols, verbose=getOption("datatable.verbose"), allow.cartesian=getOption("datatable.allow.cartesian"), drop=NULL, on=NULL, having)
{
# ..selfcount <<- ..selfcount+1 # in dev, we check no self calls, each of which doubles overhead, or could
# test explicitly if the caller is [.data.table (even stronger test. TO DO.)
Expand Down Expand Up @@ -1373,10 +1373,13 @@ replace_dot_alias = function(e) {
} else return(base::`-.POSIXt`(e1, e2))
}

do_having = !missing(having)

if (byjoin) {
# The groupings come instead from each row of the i data.table.
# Much faster for a few known groups vs a 'by' for all followed by a subset
if (!is.data.table(i)) stop("logical error. i is not data.table, but mult='all' and 'by'=.EACHI")
if (do_having) stop("having argument is not supported with 'by' = .EACHI") ## could change to allow .N as it may be help for non-equi joins
byval = i
bynames = if (missing(on)) head(key(x),length(leftcols)) else names(on)
allbyvars = NULL
Expand All @@ -1399,7 +1402,7 @@ replace_dot_alias = function(e) {
} else {
# Find the groups, using 'byval' ...
if (missingby) stop("Internal error: by= is missing") # nocov

if (length(byval) && length(byval[[1L]])) {
if (!bysameorder && isFALSE(byindex)) {
if (verbose) {last.started.at=proc.time();cat("Finding groups using forderv ... ");flush.console()}
Expand Down Expand Up @@ -1454,6 +1457,67 @@ replace_dot_alias = function(e) {
len__ = uniqlengths(f__, xnrow)
# TO DO: combine uniqlist and uniquelengths into one call. Or, just set len__ to NULL when dogroups infers that.
if (verbose) { cat(timetaken(last.started.at),"\n"); flush.console() }

}

if (do_having) {

having_sub = substitute(having)
av_having = all.vars(having_sub)

if (!length(av_having <- setdiff(av_having, ".N"))) { # only .N has been assigned and no need to go to gForce
lgl_ans = eval(having_sub, envir = list(.N = len__))
} else {
.HavingEnv = new.env()
.HavingEnv$.N = .HavingEnv$len__ = len__
.HavingEnv$o__ = o__
.HavingEnv$f__ = f__
.HavingEnv$irows = irows

av_having = intersect(av_having, names_x)

for (av in av_having) {
assign(av, x[[av]], .HavingEnv)
}
lgl_ans = group_eval_tree(having_sub, .HavingEnv, use.GForce = TRUE)
}

if (verbose) sprintf("Of the %d original groups, %d evaluated to TRUE in having", length(f__), sum(lgl_ans))

## We will short circuit:
## dt[, .SD, by, having]
## dt[, V1, by, having]
## dt[, .(V1), by, having]
if (jsub == as.name(".SD")) {
cols = chmatch(sdvars, names_x)
subset_only = TRUE
} else if ((jsub_is_name <- is.name(jsub)) ||
jsub %iscall% "list") {
cols = chmatch(all.vars(if (jsub_is_name) jsub else jsub[-1L], functions = TRUE), names_x)
subset_only = !anyNA(cols)
} else subset_only = FALSE

inds = .Call(Cvecseq_having, f__, len__, lgl_ans, retGrpArg = !subset_only, o__)

if (subset_only) {
irows = if (is.null(irows)) inds else irows[inds]
return(setDT(c(lapply(byval, function(vec) .Call(CsubsetVector, vec, inds)), .Call(CsubsetDT, x, irows, cols))))
} else {
if (!is.null(inds)) {
## TODO make this more efficient
## likely need to reconsider vecseq_having as this portion is causing performance issues
len__ = attr(inds, "grplen", exact = TRUE)
f__ = attr(inds, "starts", exact = TRUE)
o_o_ = order(inds)
sorted_o = inds[o_o_]
irows = if (is.null(irows)) sorted_o else sort(irows[inds])
byval = lapply(byval, function(vec) .Call(CsubsetVector, vec, sorted_o)) ##dogroups need this; gforce could use original_f__[lgl_ans] instead of this
o__ = order(o_o_) ##gforce needs these; dogroups would be better off with inds
attr(o__, "maxgrpn") <- attr(inds, "maxgrpn", exact = TRUE)
} else {
do_having = FALSE # if is.null(inds) that means there were no subsets. This is to account that we didn't change anything.
}
}
}
} else {
f__=NULL
Expand Down Expand Up @@ -3109,3 +3173,49 @@ isReallyReal = function(x) {
names(on) = xCols
return(list(on = on, ops = idx_op))
}

group_eval_tree = function(e, env, init = TRUE, one_per_group = TRUE, use.GForce = TRUE) {
## Recursive function to determine what can ignore groupings or have gforce optimizations.

## Let's say e = sum(x) > global_var
## where
## -- x has been assigned to our environment env
## -- global_var exists in the scope of env but not assigned (i.e., env$global_var does not work)

## This function will continually evaluate the expression and
## when it sees x, it will determine that there is no need to immediately evaluate it.
## Instead, we can wait to evaluate x until we're back to sum(x). Therefore,
## this function would return expression x for this iteration.

## On the other hand, global_var will not be seen directly in the env. Therefore,
## we attempt to eval to see if it existed up a level from the original env assignment.
## The function would either return the value of global_var or, if not found, error.

if (length(e) == 1L) {
if (is.atomic(e) || exists(as.character(e), env)) {
ans = e
} else {
ans = eval(e, env)
}
} else {
fx = e[[1L]]
is_GForced = FALSE
if (use.GForce && as.character(fx) %chin% gfuns) {
e[[1L]] = as.name(paste0("g", e[[1L]]))
e[[-1L]] = group_eval_tree(e[[-1L]], env, FALSE, one_per_group, use.GForce)
ans = gforce(env, e, env$o__, env$f__, env$len__, env$irows)[[1L]]
} else if (is.primitive(eval(fx))) {
## basically we have white-listed functions. Still need to verify approach
## but we want to allow +, -, *, etc., to be run without grouping
e[-1] = lapply(e[-1], group_eval_tree, env, FALSE, one_per_group, use.GForce)
if (init)
ans = eval(e, env)
else
ans = e
} else {
stop('function is not supported') ## TODO make a simplified doGroups
# ans = do.call(as.character(fx), lapply(e[-1L], group_eval_tree, env, FALSE, one_per_group, use.GForce), envir = env)
}
}
return(ans)
}
25 changes: 25 additions & 0 deletions inst/tests/tests.Rraw
Original file line number Diff line number Diff line change
Expand Up @@ -16853,3 +16853,28 @@ A = data.table(A=c(complex(real = 1:3, imaginary=c(0, -1, 1)), NaN))
test(2138.3, rbind(A,B), data.table(A=c(as.character(A$A), B$A)))
A = data.table(A=as.complex(rep(NA, 5)))
test(2138.4, rbind(A,B), data.table(A=c(as.character(A$A), B$A)))

# test having: FR #788 PR #4412
dt = data.table(grp = c(1L, 2L, 1L, 2L), x = letters[sample(4L)])
test(2139.01, dt[, .SD, by = grp, having = .N > 0L], dt[, .SD, grp])
dt = data.table(grp = c(2L, 1L, 2L, 1L), x = letters[sample(4L)])
test(2139.02, dt[, .SD, by = grp, having = .N > 0L], dt[, .SD, grp])

n = 1e5
grps = 1e4
dt = data.table(x = sample(grps, n, TRUE), y = runif(n))
test(2139.03, dt[, .SD, by = x, having = .N > 5L], dt[dt[, .I[.N > 5L], by = x]$V1])
#outside_var = 5L
#test(2139.04, dt[, .SD, by = x, having = .N > outside_var], dt[dt[, .I[.N > 5L], by = x]$V1]) ## this works in normal global environment but fails here.
test(2139.05, dt[, y, by = x, having = .N > 5L], dt[dt[, .I[.N > 5L], by = x]$V1])
test(2139.051, dt[, .(y), by = x, having = .N > 5L], dt[dt[, .I[.N > 5L], by = x]$V1])
test(2139.052, dt[, sum(y), by = x, having = .N > 5L], dt[, if (.N > 5L) sum(y), by = x])
test(2139.053, dt[, .(y, sum(y)), by = x, having = .N > 5L], dt[, if (.N > 5L) .(y, sum(y)), by = x])
test(2139.06, dt[, .SD, by = x, having = which.max(y) > 3L], error = "function is not supported")
test(2139.061, dt[, .SD, by = x, having = max(y) > 0.9], dt[dt[, .I[max(y) > 0.9], by = x]$V1])
dt <- as.data.table(iris)

test(2139.07, dt[, .SD, by = Species, having = .N > 40L], dt[, .SD, by = Species])
test(2139.08, dt[, .SD, by = Species, having = .N < 40L], dt[0L, .SD, by = Species])
test(2139.09, dt[-(1:5), .SD, by = Species, having = .N > 44L], dt[-(1:5), .SD, by = Species])
test(2139.10, dt[-(1:5), .SD, by = Species, having = .N > 45L], dt[-(1:50), .SD, by = Species])
4 changes: 3 additions & 1 deletion man/data.table.Rd
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ data.table(\dots, keep.rownames=FALSE, check.names=FALSE, key=NULL, stringsAsFac
.SDcols,
verbose = getOption("datatable.verbose"), # default: FALSE
allow.cartesian = getOption("datatable.allow.cartesian"), # default: FALSE
drop = NULL, on = NULL)
drop = NULL, on = NULL, having)
}
\arguments{
\item{\dots}{ Just as \code{\dots} in \code{\link{data.frame}}. Usual recycling rules are applied to vectors of different lengths to create a list of equal length vectors.}
Expand Down Expand Up @@ -170,6 +170,8 @@ data.table(\dots, keep.rownames=FALSE, check.names=FALSE, key=NULL, stringsAsFac
}
See examples as well as \href{../doc/datatable-secondary-indices-and-auto-indexing.html}{\code{vignette("datatable-secondary-indices-and-auto-indexing")}}.
}

\item{having}{ \code{FALSE} Expression which evaluates to a logical vector of length 1 for each group. Currently only GForce functions are optimized.}
}
\details{
\code{data.table} builds on base \R functionality to reduce 2 types of time:\cr
Expand Down
1 change: 1 addition & 0 deletions src/data.table.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ extern SEXP sym_index;
extern SEXP sym_BY;
extern SEXP sym_starts, char_starts;
extern SEXP sym_maxgrpn;
extern SEXP sym_grplen;
extern SEXP sym_colClassesAs;
extern SEXP sym_verbose;
extern SEXP SelfRefSymbol;
Expand Down
4 changes: 4 additions & 0 deletions src/init.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ SEXP sym_index;
SEXP sym_BY;
SEXP sym_starts, char_starts;
SEXP sym_maxgrpn;
SEXP sym_grplen;
SEXP sym_colClassesAs;
SEXP sym_verbose;
SEXP SelfRefSymbol;
Expand Down Expand Up @@ -119,6 +120,7 @@ SEXP lock();
SEXP unlock();
SEXP islockedR();
SEXP allNAR();
SEXP vecseq_having();

// .Externals
SEXP fastmean();
Expand Down Expand Up @@ -211,6 +213,7 @@ R_CallMethodDef callMethods[] = {
{"CfrollapplyR", (DL_FUNC) &frollapplyR, -1},
{"CtestMsgR", (DL_FUNC) &testMsgR, -1},
{"C_allNAR", (DL_FUNC) &allNAR, -1},
{"Cvecseq_having", (DL_FUNC) &vecseq_having, -1},
{NULL, NULL, 0}
};

Expand Down Expand Up @@ -339,6 +342,7 @@ void attribute_visible R_init_datatable(DllInfo *info)
sym_index = install("index");
sym_BY = install(".BY");
sym_maxgrpn = install("maxgrpn");
sym_grplen = install("grplen");
sym_colClassesAs = install("colClassesAs");
sym_verbose = install("datatable.verbose");
SelfRefSymbol = install(".internal.selfref");
Expand Down
118 changes: 118 additions & 0 deletions src/vecseq.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,121 @@ SEXP vecseq(SEXP x, SEXP len, SEXP clamp)
return(ans);
}

SEXP vecseq_having(SEXP x, SEXP len, SEXP having, SEXP retGrpArg, SEXP o__) {

if (!isInteger(x)) error(_("x must be an integer vector"));
if (!isInteger(len)) error(_("len must be an integer vector"));
if (!isLogical(having)) error(_("having must be a logical vector"));
if (!isInteger(o__)) error(_("o__ must be an integer vector"));
//if (!isInteger(irows)) error(_("irows must be an integer vector"));
//if (!isLogical(isGforce)) error(_("isGforce must be a logical vector"));
if (!isLogical(retGrpArg)) error(_("retGrpArg must be a logical vector"));

if (LENGTH(x) != LENGTH(len)) error(_("x and len must be the same length"));
if (LENGTH(x) != LENGTH(having)) error(_("x and having must be the same length"));
if (LENGTH(retGrpArg) > 1) error(_("retGrpArg must be a logical vector of length 1"));

const int *ihaving = INTEGER(having);
const int *ix = INTEGER(x);
const int *ilen = INTEGER(len), nlen=LENGTH(len);

int reslen = 0;
int total_true = 0;

for (int i = 0; i < nlen; ++i) {
if (ihaving[i]) {
reslen += ilen[i];
total_true++;
}
}

if (reslen == 0) return(allocVector(INTSXP, 0));

bool retGrp = LOGICAL(retGrpArg)[0] == TRUE;
if (total_true == nlen && retGrp) return(R_NilValue); //no need to recalculate if all groups are returned from having

SEXP ans = PROTECT(allocVector(INTSXP, reslen));
int *ians = INTEGER(ans);

if (retGrp) {
// should return a list to minimize allocations
// We have the following allocations
// - f__ DONE
// - len__ DONE
// - max_grp DONE

// TODO
// - vec_seq_result_o = order(ans)
// - irows = if (is.null(irows)) vec_seq_result_o else sort(irows[ans])
// - byval = lapply(byval, `[`, ans[vec_seq_result_o])
// - o__ = order(vec_seq_result_o)

SEXP tt, vv;

setAttrib(ans, sym_starts, tt = allocVector(INTSXP, total_true));
setAttrib(ans, sym_grplen, vv = allocVector(INTSXP, total_true));

int *ss = INTEGER(tt);
int *ww = INTEGER(vv);

int k = 0;
int l = 0;
int total = 1;
int maxgrpn = 0;
if (LENGTH(o__)) {
const int *io__ = INTEGER(o__);
for (int i=0; i<nlen; ++i) {
if (!ihaving[i]) continue;
int thisx = ix[i] - 1;
int elem = ilen[i];
ss[l] = total;
total += elem;
ww[l] = elem;
l++;
if (elem > maxgrpn) maxgrpn = elem;
for (int j = thisx; j < (thisx + ilen[i]); j++) {
ians[k++] = io__[j];
}
}
} else {
for (int i=0; i<nlen; ++i) {
if (!ihaving[i]) continue;
int thisx = ix[i];
int elem = ilen[i];
ss[l] = total;
total += elem;
ww[l] = elem;
l++;
if (elem > maxgrpn) maxgrpn = elem;
for (int j = thisx; j < (thisx + ilen[i]); j++) {
ians[k++] = j;
}
}
}
//R_orderVector1(ians, total_true, ans, true, false);
setAttrib(ans, sym_maxgrpn, ScalarInteger(maxgrpn));
} else {
int k = 0;
if (LENGTH(o__)) {
const int *io__ = INTEGER(o__);
for (int i=0; i<nlen; ++i) {
if (!ihaving[i]) continue;
int thisx = ix[i] - 1;
for (int j = thisx; j < (thisx + ilen[i]); j++) {
ians[k++] = io__[j];
}
}
} else {
for (int i=0; i<nlen; ++i) {
if (!ihaving[i]) continue;
int thisx = ix[i];
for (int j = thisx; j < (thisx + ilen[i]); j++) {
ians[k++] = j;
}
}
}
}

UNPROTECT(1);
return(ans);
}