From 90772b8b09e50a7d2991a57ff4ff031d065babda Mon Sep 17 00:00:00 2001 From: Joshua <143388422+joshuablac@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:21:07 +0100 Subject: [PATCH] Fix NULL handling in VAR, STDEV, VARP, STDEVP, STD and STDDEV for #2481 VARP (and STDEVP, STD, STDDEV through it) counted NULL as 0, so VARP over [1, 2, 3, 4, NULL] gave 2 instead of 1.25. VAR and STDEV only skipped null, so a row with a missing value (undefined) turned every variance aggregate into NULL. An all-NULL group gave 0. All six aggregates now skip NULL values (null, undefined or NaN) and return NULL (undefined) when there is nothing to aggregate. The sample variance (VAR, STDEV) is NULL for fewer than two values, as in SQL. The duplicate definition of STDEV is removed. Co-Authored-By: Claude Opus 5.5 --- src/55functions.js | 52 +++++++++++++++++++++-------------------- test/test2481.js | 58 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 25 deletions(-) create mode 100644 test/test2481.js diff --git a/src/55functions.js b/src/55functions.js index 5a11ee5fd3..d9db9f0576 100644 --- a/src/55functions.js +++ b/src/55functions.js @@ -482,14 +482,19 @@ alasql.aggr.QUART3 = function (v, s, stage) { return alasql.aggr.QUART(v, s, stage, 3); }; -// Standard deviation +// NULL values (null, undefined or NaN) are skipped by the variance aggregates (#2481) +var isAggrNull = function (v) { + return v == null || (typeof v === 'number' && isNaN(v)); +}; + +// Sample variance alasql.aggr.VAR = function (v, s, stage) { if (stage === 1) { // Initialise sum, sum of squares, and count - return v === null ? {sum: 0, sumSq: 0, count: 0} : {sum: v, sumSq: v * v, count: 1}; + return isAggrNull(v) ? {sum: 0, sumSq: 0, count: 0} : {sum: v, sumSq: v * v, count: 1}; } else if (stage === 2) { // Update sum, sum of squares, and count - if (v !== null) { + if (!isAggrNull(v)) { s.sum += v; s.sumSq += v * v; s.count++; @@ -500,60 +505,57 @@ alasql.aggr.VAR = function (v, s, stage) { // This avoids the need to store and iterate over all values if (s.count > 1) { return (s.sumSq - (s.sum * s.sum) / s.count) / (s.count - 1); - } else { - // Handling for cases with less than 2 values (variance is undefined or zero) - return 0; } + // Sample variance needs at least two values, otherwise it is NULL + return undefined; } }; +// Sample standard deviation alasql.aggr.STDEV = function (v, s, stage) { if (stage === 1 || stage === 2) { return alasql.aggr.VAR(v, s, stage); - } else { - return Math.sqrt(alasql.aggr.VAR(v, s, stage)); - } -}; - -alasql.aggr.STDEV = function (v, s, stage) { - if (stage === 1 || stage === 2) { - return alasql.aggr.VAR(v, s, stage); - } else { - return Math.sqrt(alasql.aggr.VAR(v, s, stage)); } + const variance = alasql.aggr.VAR(v, s, stage); + return variance === undefined ? undefined : Math.sqrt(variance); }; +// Population variance alasql.aggr.VARP = function (value, accumulator, stage) { if (stage === 1) { // Initialise accumulator with count, sum, and sum of squares - return {count: 1, sum: value, sumSq: value * value}; + return isAggrNull(value) + ? {count: 0, sum: 0, sumSq: 0} + : {count: 1, sum: value, sumSq: value * value}; } else if (stage === 2) { // Update accumulator - accumulator.count++; - accumulator.sum += value; - accumulator.sumSq += value * value; + if (!isAggrNull(value)) { + accumulator.count++; + accumulator.sum += value; + accumulator.sumSq += value * value; + } return accumulator; } else { - // Final stage: Calculate variance + // Final stage: Calculate variance, NULL if no values were aggregated if (accumulator.count > 0) { const mean = accumulator.sum / accumulator.count; const variance = accumulator.sumSq / accumulator.count - mean * mean; return variance; - } else { - return 0; // Return 0 variance if no values were aggregated } + return undefined; } }; +// Population standard deviation alasql.aggr.STD = alasql.aggr.STDDEV = alasql.aggr.STDEVP = function (v, s, stage) { if (stage == 1 || stage == 2) { return alasql.aggr.VARP(v, s, stage); - } else { - return Math.sqrt(alasql.aggr.VARP(v, s, stage)); } + const variance = alasql.aggr.VARP(v, s, stage); + return variance === undefined ? undefined : Math.sqrt(variance); }; alasql._aggrOriginal = alasql.aggr; diff --git a/test/test2481.js b/test/test2481.js new file mode 100644 index 0000000000..feb478efd2 --- /dev/null +++ b/test/test2481.js @@ -0,0 +1,58 @@ +if (typeof exports === 'object') { + var assert = require('assert'); + var alasql = require('..'); +} + +let testId = '2481'; // VAR / STDEV / VARP / STDEVP / STD / STDDEV and NULL values + +describe(`Test ${testId} - variance aggregates skip NULL values`, function () { + const sql = + 'SELECT VAR(a) AS v, STDEV(a) AS s, VARP(a) AS vp, STDEVP(a) AS sp, STD(a) AS sd, STDDEV(a) AS sdd FROM ?'; + const clean = [{a: 1}, {a: 2}, {a: 3}, {a: 4}]; + const expected = alasql(sql, [clean]); + + it('A) Results for values without NULL', function () { + assert.deepStrictEqual(expected, [ + { + v: 5 / 3, + s: Math.sqrt(5 / 3), + vp: 1.25, + sp: Math.sqrt(1.25), + sd: Math.sqrt(1.25), + sdd: Math.sqrt(1.25), + }, + ]); + }); + + it('B) NULL values are skipped', function () { + assert.deepStrictEqual(alasql(sql, [[...clean, {a: null}]]), expected); + assert.deepStrictEqual(alasql(sql, [[{a: null}, ...clean]]), expected); + }); + + it('C) Missing values are skipped', function () { + assert.deepStrictEqual(alasql(sql, [[...clean, {}]]), expected); + assert.deepStrictEqual(alasql(sql, [[{}, ...clean]]), expected); + }); + + it('D) Only NULL values give NULL (undefined)', function () { + assert.deepStrictEqual(alasql(sql, [[{a: null}, {}]]), [ + {v: undefined, s: undefined, vp: undefined, sp: undefined, sd: undefined, sdd: undefined}, + ]); + }); + + it('E) A single value: sample variance is NULL, population variance is 0', function () { + assert.deepStrictEqual(alasql(sql, [[{a: 5}, {a: null}]]), [ + {v: undefined, s: undefined, vp: 0, sp: 0, sd: 0, sdd: 0}, + ]); + }); + + it('F) Per group', function () { + const res = alasql('SELECT g, VARP(a) AS vp, VAR(a) AS v FROM ? GROUP BY g', [ + [{g: 1, a: 2}, {g: 1, a: 4}, {g: 1}, {g: 2, a: null}], + ]); + assert.deepStrictEqual(res, [ + {g: 1, vp: 1, v: 2}, + {g: 2, vp: undefined, v: undefined}, + ]); + }); +});