diff --git a/src/55functions.js b/src/55functions.js index 5a11ee5fd3..d9db9f0576 100644 --- a/src/55functions.js +++ b/src/55functions.js @@ -482,14 +482,19 @@ alasql.aggr.QUART3 = function (v, s, stage) { return alasql.aggr.QUART(v, s, stage, 3); }; -// Standard deviation +// NULL values (null, undefined or NaN) are skipped by the variance aggregates (#2481) +var isAggrNull = function (v) { + return v == null || (typeof v === 'number' && isNaN(v)); +}; + +// Sample variance alasql.aggr.VAR = function (v, s, stage) { if (stage === 1) { // Initialise sum, sum of squares, and count - return v === null ? {sum: 0, sumSq: 0, count: 0} : {sum: v, sumSq: v * v, count: 1}; + return isAggrNull(v) ? {sum: 0, sumSq: 0, count: 0} : {sum: v, sumSq: v * v, count: 1}; } else if (stage === 2) { // Update sum, sum of squares, and count - if (v !== null) { + if (!isAggrNull(v)) { s.sum += v; s.sumSq += v * v; s.count++; @@ -500,60 +505,57 @@ alasql.aggr.VAR = function (v, s, stage) { // This avoids the need to store and iterate over all values if (s.count > 1) { return (s.sumSq - (s.sum * s.sum) / s.count) / (s.count - 1); - } else { - // Handling for cases with less than 2 values (variance is undefined or zero) - return 0; } + // Sample variance needs at least two values, otherwise it is NULL + return undefined; } }; +// Sample standard deviation alasql.aggr.STDEV = function (v, s, stage) { if (stage === 1 || stage === 2) { return alasql.aggr.VAR(v, s, stage); - } else { - return Math.sqrt(alasql.aggr.VAR(v, s, stage)); - } -}; - -alasql.aggr.STDEV = function (v, s, stage) { - if (stage === 1 || stage === 2) { - return alasql.aggr.VAR(v, s, stage); - } else { - return Math.sqrt(alasql.aggr.VAR(v, s, stage)); } + const variance = alasql.aggr.VAR(v, s, stage); + return variance === undefined ? undefined : Math.sqrt(variance); }; +// Population variance alasql.aggr.VARP = function (value, accumulator, stage) { if (stage === 1) { // Initialise accumulator with count, sum, and sum of squares - return {count: 1, sum: value, sumSq: value * value}; + return isAggrNull(value) + ? {count: 0, sum: 0, sumSq: 0} + : {count: 1, sum: value, sumSq: value * value}; } else if (stage === 2) { // Update accumulator - accumulator.count++; - accumulator.sum += value; - accumulator.sumSq += value * value; + if (!isAggrNull(value)) { + accumulator.count++; + accumulator.sum += value; + accumulator.sumSq += value * value; + } return accumulator; } else { - // Final stage: Calculate variance + // Final stage: Calculate variance, NULL if no values were aggregated if (accumulator.count > 0) { const mean = accumulator.sum / accumulator.count; const variance = accumulator.sumSq / accumulator.count - mean * mean; return variance; - } else { - return 0; // Return 0 variance if no values were aggregated } + return undefined; } }; +// Population standard deviation alasql.aggr.STD = alasql.aggr.STDDEV = alasql.aggr.STDEVP = function (v, s, stage) { if (stage == 1 || stage == 2) { return alasql.aggr.VARP(v, s, stage); - } else { - return Math.sqrt(alasql.aggr.VARP(v, s, stage)); } + const variance = alasql.aggr.VARP(v, s, stage); + return variance === undefined ? undefined : Math.sqrt(variance); }; alasql._aggrOriginal = alasql.aggr; diff --git a/test/test2481.js b/test/test2481.js new file mode 100644 index 0000000000..feb478efd2 --- /dev/null +++ b/test/test2481.js @@ -0,0 +1,58 @@ +if (typeof exports === 'object') { + var assert = require('assert'); + var alasql = require('..'); +} + +let testId = '2481'; // VAR / STDEV / VARP / STDEVP / STD / STDDEV and NULL values + +describe(`Test ${testId} - variance aggregates skip NULL values`, function () { + const sql = + 'SELECT VAR(a) AS v, STDEV(a) AS s, VARP(a) AS vp, STDEVP(a) AS sp, STD(a) AS sd, STDDEV(a) AS sdd FROM ?'; + const clean = [{a: 1}, {a: 2}, {a: 3}, {a: 4}]; + const expected = alasql(sql, [clean]); + + it('A) Results for values without NULL', function () { + assert.deepStrictEqual(expected, [ + { + v: 5 / 3, + s: Math.sqrt(5 / 3), + vp: 1.25, + sp: Math.sqrt(1.25), + sd: Math.sqrt(1.25), + sdd: Math.sqrt(1.25), + }, + ]); + }); + + it('B) NULL values are skipped', function () { + assert.deepStrictEqual(alasql(sql, [[...clean, {a: null}]]), expected); + assert.deepStrictEqual(alasql(sql, [[{a: null}, ...clean]]), expected); + }); + + it('C) Missing values are skipped', function () { + assert.deepStrictEqual(alasql(sql, [[...clean, {}]]), expected); + assert.deepStrictEqual(alasql(sql, [[{}, ...clean]]), expected); + }); + + it('D) Only NULL values give NULL (undefined)', function () { + assert.deepStrictEqual(alasql(sql, [[{a: null}, {}]]), [ + {v: undefined, s: undefined, vp: undefined, sp: undefined, sd: undefined, sdd: undefined}, + ]); + }); + + it('E) A single value: sample variance is NULL, population variance is 0', function () { + assert.deepStrictEqual(alasql(sql, [[{a: 5}, {a: null}]]), [ + {v: undefined, s: undefined, vp: 0, sp: 0, sd: 0, sdd: 0}, + ]); + }); + + it('F) Per group', function () { + const res = alasql('SELECT g, VARP(a) AS vp, VAR(a) AS v FROM ? GROUP BY g', [ + [{g: 1, a: 2}, {g: 1, a: 4}, {g: 1}, {g: 2, a: null}], + ]); + assert.deepStrictEqual(res, [ + {g: 1, vp: 1, v: 2}, + {g: 2, vp: undefined, v: undefined}, + ]); + }); +});