Fix NULL handling in VAR, STDEV, VARP, STDEVP, STD and STDDEV (part of #2481)

src/55functions.js

@@ -482,14 +482,19 @@ alasql.aggr.QUART3 = function (v, s, stage) {
 	return alasql.aggr.QUART(v, s, stage, 3);
 };
 
-// Standard deviation
+// NULL values (null, undefined or NaN) are skipped by the variance aggregates (#2481)
+var isAggrNull = function (v) {
+	return v == null || (typeof v === 'number' && isNaN(v));
+};
+
+// Sample variance
 alasql.aggr.VAR = function (v, s, stage) {
 	if (stage === 1) {
 		// Initialise sum, sum of squares, and count
-		return v === null ? {sum: 0, sumSq: 0, count: 0} : {sum: v, sumSq: v * v, count: 1};
+		return isAggrNull(v) ? {sum: 0, sumSq: 0, count: 0} : {sum: v, sumSq: v * v, count: 1};
 	} else if (stage === 2) {
 		// Update sum, sum of squares, and count
-		if (v !== null) {
+		if (!isAggrNull(v)) {
 			s.sum += v;
 			s.sumSq += v * v;
 			s.count++;
@@ -500,60 +505,57 @@ alasql.aggr.VAR = function (v, s, stage) {
 		// This avoids the need to store and iterate over all values
 		if (s.count > 1) {
 			return (s.sumSq - (s.sum * s.sum) / s.count) / (s.count - 1);
-		} else {
-			// Handling for cases with less than 2 values (variance is undefined or zero)
-			return 0;
 		}
+		// Sample variance needs at least two values, otherwise it is NULL
+		return undefined;
 	}
 };
 
+// Sample standard deviation
 alasql.aggr.STDEV = function (v, s, stage) {
 	if (stage === 1 || stage === 2) {
 		return alasql.aggr.VAR(v, s, stage);
-	} else {
-		return Math.sqrt(alasql.aggr.VAR(v, s, stage));
-	}
-};
-
-alasql.aggr.STDEV = function (v, s, stage) {
-	if (stage === 1 || stage === 2) {
-		return alasql.aggr.VAR(v, s, stage);
-	} else {
-		return Math.sqrt(alasql.aggr.VAR(v, s, stage));
 	}
+	const variance = alasql.aggr.VAR(v, s, stage);
+	return variance === undefined ? undefined : Math.sqrt(variance);
 };
 
+// Population variance
 alasql.aggr.VARP = function (value, accumulator, stage) {
 	if (stage === 1) {
 		// Initialise accumulator with count, sum, and sum of squares
-		return {count: 1, sum: value, sumSq: value * value};
+		return isAggrNull(value)
+			? {count: 0, sum: 0, sumSq: 0}
+			: {count: 1, sum: value, sumSq: value * value};
 	} else if (stage === 2) {
 		// Update accumulator
-		accumulator.count++;
-		accumulator.sum += value;
-		accumulator.sumSq += value * value;
+		if (!isAggrNull(value)) {
+			accumulator.count++;
+			accumulator.sum += value;
+			accumulator.sumSq += value * value;
+		}
 		return accumulator;
 	} else {
-		// Final stage: Calculate variance
+		// Final stage: Calculate variance, NULL if no values were aggregated
 		if (accumulator.count > 0) {
 			const mean = accumulator.sum / accumulator.count;
 			const variance = accumulator.sumSq / accumulator.count - mean * mean;
 			return variance;
-		} else {
-			return 0; // Return 0 variance if no values were aggregated
 		}
+		return undefined;
 	}
 };
 
+// Population standard deviation
 alasql.aggr.STD =
 	alasql.aggr.STDDEV =
 	alasql.aggr.STDEVP =
 		function (v, s, stage) {
 			if (stage == 1 || stage == 2) {
 				return alasql.aggr.VARP(v, s, stage);
-			} else {
-				return Math.sqrt(alasql.aggr.VARP(v, s, stage));
 			}
+			const variance = alasql.aggr.VARP(v, s, stage);
+			return variance === undefined ? undefined : Math.sqrt(variance);
 		};
 
 alasql._aggrOriginal = alasql.aggr;

test/test2481.js

@@ -0,0 +1,58 @@
+if (typeof exports === 'object') {
+	var assert = require('assert');
+	var alasql = require('..');
+}
+
+let testId = '2481'; // VAR / STDEV / VARP / STDEVP / STD / STDDEV and NULL values
+
+describe(`Test ${testId} - variance aggregates skip NULL values`, function () {
+	const sql =
+		'SELECT VAR(a) AS v, STDEV(a) AS s, VARP(a) AS vp, STDEVP(a) AS sp, STD(a) AS sd, STDDEV(a) AS sdd FROM ?';
+	const clean = [{a: 1}, {a: 2}, {a: 3}, {a: 4}];
+	const expected = alasql(sql, [clean]);
+
+	it('A) Results for values without NULL', function () {
+		assert.deepStrictEqual(expected, [
+			{
+				v: 5 / 3,
+				s: Math.sqrt(5 / 3),
+				vp: 1.25,
+				sp: Math.sqrt(1.25),
+				sd: Math.sqrt(1.25),
+				sdd: Math.sqrt(1.25),
+			},
+		]);
+	});
+
+	it('B) NULL values are skipped', function () {
+		assert.deepStrictEqual(alasql(sql, [[...clean, {a: null}]]), expected);
+		assert.deepStrictEqual(alasql(sql, [[{a: null}, ...clean]]), expected);
+	});
+
+	it('C) Missing values are skipped', function () {
+		assert.deepStrictEqual(alasql(sql, [[...clean, {}]]), expected);
+		assert.deepStrictEqual(alasql(sql, [[{}, ...clean]]), expected);
+	});
+
+	it('D) Only NULL values give NULL (undefined)', function () {
+		assert.deepStrictEqual(alasql(sql, [[{a: null}, {}]]), [
+			{v: undefined, s: undefined, vp: undefined, sp: undefined, sd: undefined, sdd: undefined},
+		]);
+	});
+
+	it('E) A single value: sample variance is NULL, population variance is 0', function () {
+		assert.deepStrictEqual(alasql(sql, [[{a: 5}, {a: null}]]), [
+			{v: undefined, s: undefined, vp: 0, sp: 0, sd: 0, sdd: 0},
+		]);
+	});
+
+	it('F) Per group', function () {
+		const res = alasql('SELECT g, VARP(a) AS vp, VAR(a) AS v FROM ? GROUP BY g', [
+			[{g: 1, a: 2}, {g: 1, a: 4}, {g: 1}, {g: 2, a: null}],
+		]);
+		assert.deepStrictEqual(res, [
+			{g: 1, vp: 1, v: 2},
+			{g: 2, vp: undefined, v: undefined},
+		]);
+	});
+});