From 54c697dc9080acce68fd53481f6e5a876a2f18f6 Mon Sep 17 00:00:00 2001 From: DT1000101 Date: Sun, 2 Aug 2026 17:03:38 +0100 Subject: [PATCH 1/3] Add Floaty CSV support and fail clearly on unknown CSV headers. Floaty CSV exports were misrouted through the Float Control parser, which flooded the console and hung on the loading screen. Detect Floaty CSV, map it like Floaty JSON, and show an error for unrecognised formats. Co-authored-by: Cursor --- README.md | 2 +- src/components/Picker.svelte | 5 +- src/components/View.svelte | 27 +++- src/components/View.ts | 2 +- src/lib/parse/__fixtures__/floaty.csv | 4 + src/lib/parse/float-control.ts | 7 + src/lib/parse/floaty.test.ts | 32 +++- src/lib/parse/floaty.ts | 216 ++++++++++++++++++++------ src/lib/parse/index.test.ts | 16 ++ src/lib/parse/index.ts | 34 ++-- 10 files changed, 282 insertions(+), 63 deletions(-) create mode 100644 src/lib/parse/__fixtures__/floaty.csv diff --git a/README.md b/README.md index 09cc7c0..0955342 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Now you'll have `Float View` all the time, even offline! ## How do I use it? 1. Get on your board and record a ride with [Float Control] or [Floaty] -2. Export your ride data (Float Control will put it in a `.csv.zip`, Floaty in a `.json`) +2. Export your ride data (Float Control will put it in a `.csv.zip`, Floaty in a `.json` or `.csv`) 3. Load it up in (if it's zipped, unzip it first!) 4. Enjoy! diff --git a/src/components/Picker.svelte b/src/components/Picker.svelte index 01cbb5d..9d9331e 100644 --- a/src/components/Picker.svelte +++ b/src/components/Picker.svelte @@ -32,7 +32,10 @@ Float Control
  • an exported CSV file from VESC Tool
    • -
  • an exported JSON file from Floaty
    • +
  • + an exported JSON or CSV file from + Floaty +
  • ... or drag and drop a supported file onto this window!
    • err instanceof ParseError); for (const err of results.errors) { if (err instanceof FloatControlLimitedError) { banners.push({ text: err.message, kind: 'warning' }); @@ -147,12 +148,23 @@ if (err instanceof ParseError) { console.error(err, err.cause); - alert( - `An error occurred when parsing ride, displayed data may be incomplete or incorrect! (${err.message})`, - ); } } + // Fatal: nothing usable to show — surface the error and return to the file picker. + if (results.data.length === 0 && parseErrors.length > 0) { + alert(`Could not load ride:\n\n${parseErrors.map((err) => err.message).join('\n')}`); + file = undefined; + source = DataSource.None; + return; + } + + if (parseErrors.length > 0) { + alert( + `An error occurred when parsing ride, displayed data may be incomplete or incorrect! (${parseErrors.map((err) => err.message).join('; ')})`, + ); + } + rows = results.data; // initialize visibility and trimming to full range for the newly loaded ride visibleFromMap = new Array(rows.length).fill(true); @@ -160,7 +172,14 @@ trimEnd = rows.length ? rows.length - 1 : 0; selectedIndex = 0; - stats = computeStats(rows, pointsOfInterest); + stats = computeStats(rows, findPointsOfInterest(rows)); + }) + .catch((error) => { + clearTimeout(timer); + console.error(error); + alert(`Could not load ride:\n\n${error instanceof Error ? error.message : String(error)}`); + file = undefined; + source = DataSource.None; }) .finally(() => (loading = false)); } diff --git a/src/components/View.ts b/src/components/View.ts index 011c0d8..5fdd535 100644 --- a/src/components/View.ts +++ b/src/components/View.ts @@ -112,7 +112,7 @@ export function computeStats(rows: RowWithIndex[], pois: PointOfInterest[]): Rid highestFieldWeakeningCurrent, highestTempMotor, highestTempController, - totalDistanceMeters: rows[rows.length - 1]!.distance, + totalDistanceMeters: rows.at(-1)?.distance ?? 0, }; } diff --git a/src/lib/parse/__fixtures__/floaty.csv b/src/lib/parse/__fixtures__/floaty.csv new file mode 100644 index 0000000..ac924e6 --- /dev/null +++ b/src/lib/parse/__fixtures__/floaty.csv @@ -0,0 +1,4 @@ +timestamp,speed,dutyCycle,batteryVolts,batteryPercent,batteryCurrent,motorCurrent,motorTemp,controllerTemp,tripDistance,lifeDistance,remainingDistance,rollAngle,pitchAngle,truePitchAngle,inputTilt,throttle,ampHours,wattHours,state,switchState,setpointAdjustmentType,faultCode,adc1,adc2,sessionId,altitude,latitude,longitude,accuracy,gpsSpeed,gpsTimestamp +105,0.4,0.03,81.9,0.92,0.2,9.9,18,18,0,592.211,0,-4,1,0,0,0,0,0,1,1,0,0,3.1,0.08,00000000-0000-0000-0000-000000000000,122.16287420969456,-1.0,1.5,4.55257009550728,0.23000000417232513,110.0522 +115,0.7,0.04,81.8,0.92,0.4,14,21,18,0.5,592.211,0,-4,0,-1,0,0,0,0,1,1,2,0,3.1,0.08,00000000-0000-0000-0000-000000000000,123.54580882564187,-1.1,1.6,4.552570096263577,0.769999980926508,120.0503 +125,,,,0.92,,,0,0,0,592.211,0,-4,0,-1,0,0,0,0,1,3,2,0,3.1,3.08,00000000-0000-0000-0000-000000000000,123.54580882564187,-1.1,1.6,4.552570096263577,0.769999980926508,120.0503 diff --git a/src/lib/parse/float-control.ts b/src/lib/parse/float-control.ts index 7e1e477..f375a99 100644 --- a/src/lib/parse/float-control.ts +++ b/src/lib/parse/float-control.ts @@ -5,6 +5,13 @@ import { attachIndex } from '../misc'; import { RowKey, State, type Row, type RowWithIndex, Units } from './types'; import { FloatControlLimitedError, ParseError } from './errors'; +export function looksLikeFloatControlCsv(headerLine: string): boolean { + return headerLine + .split(',') + .map((header) => header.trim()) + .some((header) => header in floatControlKeyMap); +} + const transformHeader = (header: string) => { const key = floatControlKeyMap[header as FloatControlRawHeader]; if (!key && !Object.values(RowKey).includes(header as RowKey)) { diff --git a/src/lib/parse/floaty.test.ts b/src/lib/parse/floaty.test.ts index 1128a00..96df0a8 100644 --- a/src/lib/parse/floaty.test.ts +++ b/src/lib/parse/floaty.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from 'vitest'; -import { parseFloatyJson } from './floaty'; +import { parseFloatyCsv, parseFloatyJson } from './floaty'; import floatyJsonString from './__fixtures__/floaty.json?raw'; import floatyJson from './__fixtures__/floaty.json'; +import floatyCsv from './__fixtures__/floaty.csv?raw'; describe(parseFloatyJson.name, () => { test('maps gps locations to logs', async () => { @@ -57,3 +58,32 @@ describe(parseFloatyJson.name, () => { expect(data[2]!.distance).toBe(data[1]!.distance); }); }); + +describe(parseFloatyCsv.name, () => { + test('maps floaty csv rows including inline gps', async () => { + const { data, units, source, errors } = await parseFloatyCsv(floatyCsv); + expect(errors).toEqual([]); + expect(source).toBe('floaty'); + expect(units).toEqual('metric'); + expect(data).toHaveLength(3); + + expect(data[0]!.speed).toBe(0.4); + expect(data[0]!.duty).toBe(3); + expect(data[0]!.voltage).toBe(81.9); + expect(data[0]!.gps_latitude).toBe(-1.0); + expect(data[0]!.gps_longitude).toBe(1.5); + expect(data[0]!.time).toBe(0); + expect(data[1]!.time).toBe(0.01); + }); + + test('backfills empty csv cells like floaty json nulls', async () => { + const { data } = await parseFloatyCsv(floatyCsv); + expect(data[2]!.current_battery).toBe(data[1]!.current_battery); + expect(data[2]!.voltage).toBe(data[1]!.voltage); + expect(data[2]!.duty).toBe(data[1]!.duty); + expect(data[2]!.speed).toBe(data[1]!.speed); + expect(data[2]!.temp_mosfet).toBe(data[1]!.temp_mosfet); + expect(data[2]!.temp_motor).toBe(data[1]!.temp_motor); + expect(data[2]!.distance).toBe(data[1]!.distance); + }); +}); diff --git a/src/lib/parse/floaty.ts b/src/lib/parse/floaty.ts index 248f906..f510ae4 100644 --- a/src/lib/parse/floaty.ts +++ b/src/lib/parse/floaty.ts @@ -1,61 +1,74 @@ +import csv from 'papaparse'; + import { type ParseResult } from './index'; import { attachIndex } from '../misc'; import { FloatyJsonSchema, type ZFloatyJson, type ZLocation, type ZLog } from './floaty.types'; import { DataSource, stateCodeMap, Units, type Row } from './types'; import { ParseError } from './errors'; -function rowsFromFloatyJson(json: ZFloatyJson): Row[] { - const rows: Row[] = []; - // NOTE: sometimes Floaty doesn't record values, and seems to just put `null` (or 0) in its logs. - // When it does, we backtrack until we find the last known value for it. - const findValue = (index: number, key: keyof ZLog, floatyEmptyValue?: unknown): number => { - const current = json.logs[index]![key]; - if (current !== null && (floatyEmptyValue === undefined || current !== floatyEmptyValue)) { - return current; - } +/** Headers that uniquely identify a Floaty CSV export (vs Float Control / VESC Tool). */ +const FLOATY_CSV_MARKERS = ['dutyCycle', 'batteryVolts', 'tripDistance'] as const; - let i = index - 1; - while (i > 0) { - const value = json.logs[i]![key]; - if (value !== null && (floatyEmptyValue === undefined || value !== floatyEmptyValue)) { - return value; - } +export function looksLikeFloatyCsv(headerLine: string): boolean { + const headers = new Set(headerLine.split(',').map((header) => header.trim())); + return FLOATY_CSV_MARKERS.every((marker) => headers.has(marker)); +} - i--; +/** + * NOTE: sometimes Floaty doesn't record values, and seems to just put `null` (or 0) in its logs. + * When it does, we backtrack until we find the last known value for it. + */ +function findValue(logs: ZLog[], index: number, key: keyof ZLog, floatyEmptyValue?: unknown): number { + const current = logs[index]![key]; + if (current !== null && (floatyEmptyValue === undefined || current !== floatyEmptyValue)) { + return current as number; + } + + let i = index - 1; + while (i > 0) { + const value = logs[i]![key]; + if (value !== null && (floatyEmptyValue === undefined || value !== floatyEmptyValue)) { + return value as number; } - return 0; - }; + i--; + } - const map = (log: ZLog, location: ZLocation, index: number): Row => { - const state_raw = findValue(index, 'state'); - return { - adc1: findValue(index, 'adc1'), - adc2: findValue(index, 'adc2'), - ah: findValue(index, 'ampHours'), - altitude: location.altitude, - current_battery: findValue(index, 'batteryCurrent'), - current_motor: findValue(index, 'motorCurrent'), - distance: findValue(index, 'tripDistance', 0), - duty: findValue(index, 'dutyCycle') * 100, - gps_accuracy: location.accuracy, - gps_latitude: location.latitude, - gps_longitude: location.longitude, - motor_fault: findValue(index, 'faultCode'), - pitch: findValue(index, 'pitchAngle'), - roll: findValue(index, 'rollAngle'), - speed: findValue(index, 'speed'), - state_raw, - state: stateCodeMap[state_raw] ?? '??', - temp_mosfet: findValue(index, 'controllerTemp', 0), - temp_motor: findValue(index, 'motorTemp', 0), - time: (log.timestamp - json.startTime) / 1000, - true_pitch: findValue(index, 'truePitchAngle'), - voltage: findValue(index, 'batteryVolts'), - wh: findValue(index, 'wattHours'), - }; + return 0; +} + +function mapFloatyLog(logs: ZLog[], location: ZLocation, index: number, startTime: number): Row { + const log = logs[index]!; + const state_raw = findValue(logs, index, 'state'); + return { + adc1: findValue(logs, index, 'adc1'), + adc2: findValue(logs, index, 'adc2'), + ah: findValue(logs, index, 'ampHours'), + altitude: location.altitude, + current_battery: findValue(logs, index, 'batteryCurrent'), + current_motor: findValue(logs, index, 'motorCurrent'), + distance: findValue(logs, index, 'tripDistance', 0), + duty: findValue(logs, index, 'dutyCycle') * 100, + gps_accuracy: location.accuracy, + gps_latitude: location.latitude, + gps_longitude: location.longitude, + motor_fault: findValue(logs, index, 'faultCode'), + pitch: findValue(logs, index, 'pitchAngle'), + roll: findValue(logs, index, 'rollAngle'), + speed: findValue(logs, index, 'speed'), + state_raw, + state: stateCodeMap[state_raw] ?? '??', + temp_mosfet: findValue(logs, index, 'controllerTemp', 0), + temp_motor: findValue(logs, index, 'motorTemp', 0), + time: (log.timestamp - startTime) / 1000, + true_pitch: findValue(logs, index, 'truePitchAngle'), + voltage: findValue(logs, index, 'batteryVolts'), + wh: findValue(logs, index, 'wattHours'), }; +} +function rowsFromFloatyJson(json: ZFloatyJson): Row[] { + const rows: Row[] = []; const { logs, locations } = json; let locationIdx = 0; for (let i = 0; i < logs.length; ++i) { @@ -65,12 +78,123 @@ function rowsFromFloatyJson(json: ZFloatyJson): Row[] { location = locations[++locationIdx]!; } - rows.push(map(log, location, i)); + rows.push(mapFloatyLog(logs, location, i, json.startTime)); } return rows; } +const optionalNumber = (value: string | undefined): number | null => { + if (value === undefined || value === '') { + return null; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +}; + +const requiredNumber = (value: string | undefined, fallback = 0): number => { + const parsed = optionalNumber(value); + return parsed === null ? fallback : parsed; +}; + +type FloatyCsvRow = Record; + +function csvRowToLog(row: FloatyCsvRow): ZLog { + return { + adc1: optionalNumber(row.adc1), + adc2: optionalNumber(row.adc2), + ampHours: optionalNumber(row.ampHours), + batteryCurrent: optionalNumber(row.batteryCurrent), + batteryPercent: optionalNumber(row.batteryPercent), + batteryVolts: optionalNumber(row.batteryVolts), + controllerTemp: optionalNumber(row.controllerTemp), + dutyCycle: optionalNumber(row.dutyCycle), + faultCode: optionalNumber(row.faultCode), + inputTilt: optionalNumber(row.inputTilt), + lifeDistance: optionalNumber(row.lifeDistance), + motorCurrent: optionalNumber(row.motorCurrent), + motorTemp: optionalNumber(row.motorTemp), + pitchAngle: optionalNumber(row.pitchAngle), + remainingDistance: optionalNumber(row.remainingDistance), + rollAngle: optionalNumber(row.rollAngle), + setpointAdjustmentType: optionalNumber(row.setpointAdjustmentType), + speed: optionalNumber(row.speed), + state: optionalNumber(row.state), + switchState: optionalNumber(row.switchState), + throttle: optionalNumber(row.throttle), + timestamp: requiredNumber(row.timestamp), + tripDistance: optionalNumber(row.tripDistance), + truePitchAngle: optionalNumber(row.truePitchAngle), + wattHours: optionalNumber(row.wattHours), + }; +} + +function csvRowToLocation(row: FloatyCsvRow): ZLocation { + return { + timestamp: requiredNumber(row.gpsTimestamp ?? row.timestamp), + altitude: requiredNumber(row.altitude), + latitude: requiredNumber(row.latitude), + longitude: requiredNumber(row.longitude), + accuracy: requiredNumber(row.accuracy), + speed: requiredNumber(row.gpsSpeed), + }; +} + +function rowsFromFloatyCsv(rawRows: FloatyCsvRow[]): Row[] { + const logs = rawRows.map(csvRowToLog); + const startTime = logs[0]?.timestamp ?? 0; + return rawRows.map((raw, index) => mapFloatyLog(logs, csvRowToLocation(raw), index, startTime)); +} + +export async function parseFloatyCsv(input: string | File): Promise { + const text = typeof input === 'string' ? input : await input.text(); + + return new Promise((resolve) => { + csv.parse(text, { + header: true, + skipEmptyLines: true, + complete: (results) => { + try { + if (results.data.length === 0) { + resolve({ + source: DataSource.Floaty, + data: [], + units: Units.Metric, + errors: [new ParseError('Floaty CSV contained no rows!', results.errors)], + }); + return; + } + + resolve({ + source: DataSource.Floaty, + data: attachIndex(rowsFromFloatyCsv(results.data)), + units: Units.Metric, + errors: results.errors.length + ? [new ParseError('Failed to parse Floaty CSV properly!', results.errors)] + : [], + }); + } catch (error) { + resolve({ + source: DataSource.Floaty, + data: [], + units: Units.Metric, + errors: [new ParseError('Failed to parse Floaty CSV!', error)], + }); + } + }, + error: (error: Error) => { + resolve({ + source: DataSource.Floaty, + data: [], + units: Units.Metric, + errors: [new ParseError('Failed to parse Floaty CSV!', error)], + }); + }, + }); + }); +} + export async function parseFloatyJson(input: string | File): Promise { try { const json = JSON.parse(typeof input === 'string' ? input : await input.text()); diff --git a/src/lib/parse/index.test.ts b/src/lib/parse/index.test.ts index 1b1a5c8..0dcbbe3 100644 --- a/src/lib/parse/index.test.ts +++ b/src/lib/parse/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from 'vitest'; import fcMetricCsv from './__fixtures__/fc_metric.csv?raw'; +import floatyCsv from './__fixtures__/floaty.csv?raw'; import vescMetricCsv from './__fixtures__/vesc_metric.csv?raw'; import { parse } from './index'; @@ -28,4 +29,19 @@ describe(parse.name, () => { const result = await parse(new MockFile([fcMetricCsv], 'fc.csv', { type: 'text/csv' })); expect(result.source).toBe('float_control'); }); + + test('routes Floaty CSV to Floaty parser', async () => { + const result = await parse(new MockFile([floatyCsv], 'floaty.csv', { type: 'text/csv' })); + expect(result.source).toBe('floaty'); + expect(result.errors).toEqual([]); + expect(result.data).toHaveLength(3); + }); + + test('rejects unrecognised CSV headers with a parse error', async () => { + const result = await parse(new MockFile(['foo,bar\n1,2\n'], 'unknown.csv', { type: 'text/csv' })); + expect(result.source).toBe('none'); + expect(result.data).toEqual([]); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]!.message).toMatch(/Unrecognised CSV headers/); + }); }); diff --git a/src/lib/parse/index.ts b/src/lib/parse/index.ts index 4e47911..e93fdd4 100644 --- a/src/lib/parse/index.ts +++ b/src/lib/parse/index.ts @@ -1,7 +1,7 @@ import * as fflate from 'fflate'; -import { parseFloatControlCsv } from './float-control'; -import { parseFloatyJson } from './floaty'; +import { parseFloatControlCsv, looksLikeFloatControlCsv } from './float-control'; +import { looksLikeFloatyCsv, parseFloatyCsv, parseFloatyJson } from './floaty'; import { parseVescToolCsv } from './vesc-tool'; import { DataSource, Units, type RowWithIndex } from './types'; import { ParseError } from './errors'; @@ -56,18 +56,34 @@ export async function parse(file: File): Promise { const semicolonCount = (firstLine.match(/;/g) ?? []).length; const commaCount = (firstLine.match(/,/g) ?? []).length; - // Heuristic: VESC Tool exports are semicolon-delimited, while Float Control uses commas. - // If this does not look like VESC Tool, we fall back to Float Control parsing. + // Heuristic: VESC Tool exports are semicolon-delimited, while Float Control / Floaty use commas. if (semicolonCount > commaCount) { return await parseVescToolCsv(text); } - const parsed = await parseFloatControlCsv(text); + if (looksLikeFloatyCsv(firstLine)) { + return await parseFloatyCsv(text); + } + + if (looksLikeFloatControlCsv(firstLine)) { + const parsed = await parseFloatControlCsv(text); + return { + source: DataSource.FloatControl, + data: parsed.csv.data, + units: parsed.units, + errors: parsed.errors, + }; + } + return { - source: DataSource.FloatControl, - data: parsed.csv.data, - units: parsed.units, - errors: parsed.errors, + source: DataSource.None, + data: [], + units: Units.Metric, + errors: [ + new ParseError('Unrecognised CSV headers. Expected a Float Control, Floaty, or VESC Tool export.', { + header: firstLine, + }), + ], }; } From fae19129777ef91d34033d42d583db2c215dd2db Mon Sep 17 00:00:00 2001 From: acheronfail Date: Sat, 29 Aug 2026 21:36:31 +0930 Subject: [PATCH 2/3] Fix Floaty backfill from first row --- src/lib/parse/floaty.test.ts | 14 ++++++++++++++ src/lib/parse/floaty.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/lib/parse/floaty.test.ts b/src/lib/parse/floaty.test.ts index 96df0a8..ddff7f2 100644 --- a/src/lib/parse/floaty.test.ts +++ b/src/lib/parse/floaty.test.ts @@ -86,4 +86,18 @@ describe(parseFloatyCsv.name, () => { expect(data[2]!.temp_motor).toBe(data[1]!.temp_motor); expect(data[2]!.distance).toBe(data[1]!.distance); }); + + test('backfills the second row from the first row', async () => { + const header = + 'timestamp,speed,dutyCycle,batteryVolts,tripDistance,latitude,longitude,altitude,accuracy,gpsSpeed,gpsTimestamp'; + const input = `${header}\n1000,1,0.1,80,1,1,2,3,4,5,1000\n1010,,,,,1,2,3,4,5,1010`; + const { data } = await parseFloatyCsv(input); + + expect(data[1]).toMatchObject({ + speed: 1, + duty: 10, + voltage: 80, + distance: 1, + }); + }); }); diff --git a/src/lib/parse/floaty.ts b/src/lib/parse/floaty.ts index f510ae4..2b9d5c4 100644 --- a/src/lib/parse/floaty.ts +++ b/src/lib/parse/floaty.ts @@ -25,7 +25,7 @@ function findValue(logs: ZLog[], index: number, key: keyof ZLog, floatyEmptyValu } let i = index - 1; - while (i > 0) { + while (i >= 0) { const value = logs[i]![key]; if (value !== null && (floatyEmptyValue === undefined || value !== floatyEmptyValue)) { return value as number; From fae1ecf0caf05b4641b7741216ee2309ab0d4b4d Mon Sep 17 00:00:00 2001 From: acheronfail Date: Sat, 29 Aug 2026 21:30:39 +0930 Subject: [PATCH 3/3] Harden CSV format detection --- src/lib/parse/csv-format.ts | 53 ++++++++++++++++++++++++++++++++++ src/lib/parse/float-control.ts | 16 ++++------ src/lib/parse/floaty.ts | 10 ++----- src/lib/parse/index.test.ts | 39 +++++++++++++++++++++++++ src/lib/parse/index.ts | 42 ++++++++++++--------------- 5 files changed, 117 insertions(+), 43 deletions(-) create mode 100644 src/lib/parse/csv-format.ts diff --git a/src/lib/parse/csv-format.ts b/src/lib/parse/csv-format.ts new file mode 100644 index 0000000..b1c99c2 --- /dev/null +++ b/src/lib/parse/csv-format.ts @@ -0,0 +1,53 @@ +import csv from 'papaparse'; + +import { floatControlKeyMap, FloatControlRawHeader } from './float-control.types'; +import { RowKey } from './types'; + +export enum CsvFormat { + FloatControl = 'float_control', + Floaty = 'floaty', + VescTool = 'vesc_tool', + Unknown = 'unknown', +} + +const FLOATY_SIGNATURE = ['timestamp', 'dutyCycle', 'batteryVolts', 'tripDistance'] as const; +const FLOAT_CONTROL_SIGNATURE = [RowKey.Time, RowKey.Speed, RowKey.Duty] as const; +const VESC_TOOL_SIGNATURE = ['ms_today', 'input_voltage', 'duty_cycle'] as const; +const rowKeys = new Set(Object.values(RowKey)); + +export const cleanCsvHeader = (header: string): string => header.replace(/^\uFEFF/, '').trim(); + +export function normalizeFloatControlHeader(header: string): string { + const cleaned = cleanCsvHeader(header); + if (Object.hasOwn(floatControlKeyMap, cleaned)) { + return floatControlKeyMap[cleaned as FloatControlRawHeader]; + } + + return cleaned; +} + +export const isNormalizedRowKey = (header: string): boolean => rowKeys.has(header); + +const containsAll = (headers: Set, signature: readonly string[]): boolean => + signature.every((header) => headers.has(header)); + +export function detectCsvFormat(text: string): CsvFormat { + const parsed = csv.parse(text, { + preview: 1, + skipEmptyLines: true, + }); + if (parsed.errors.length > 0 || parsed.data.length === 0) { + return CsvFormat.Unknown; + } + + const headers = parsed.data[0]!.map((header) => cleanCsvHeader(String(header))); + const rawHeaders = new Set(headers); + const normalizedHeaders = new Set(headers.map(normalizeFloatControlHeader)); + const matches = [ + containsAll(rawHeaders, FLOATY_SIGNATURE) && CsvFormat.Floaty, + containsAll(normalizedHeaders, FLOAT_CONTROL_SIGNATURE) && CsvFormat.FloatControl, + containsAll(rawHeaders, VESC_TOOL_SIGNATURE) && CsvFormat.VescTool, + ].filter((format): format is CsvFormat => format !== false); + + return matches.length === 1 ? matches[0]! : CsvFormat.Unknown; +} diff --git a/src/lib/parse/float-control.ts b/src/lib/parse/float-control.ts index f375a99..6fb4255 100644 --- a/src/lib/parse/float-control.ts +++ b/src/lib/parse/float-control.ts @@ -1,24 +1,18 @@ import csv, { type ParseResult } from 'papaparse'; -import { floatControlKeyMap, FloatControlRawHeader } from './float-control.types'; +import { FloatControlRawHeader } from './float-control.types'; import demoCsv from '../../assets/demo.csv?raw'; import { attachIndex } from '../misc'; import { RowKey, State, type Row, type RowWithIndex, Units } from './types'; import { FloatControlLimitedError, ParseError } from './errors'; - -export function looksLikeFloatControlCsv(headerLine: string): boolean { - return headerLine - .split(',') - .map((header) => header.trim()) - .some((header) => header in floatControlKeyMap); -} +import { isNormalizedRowKey, normalizeFloatControlHeader } from './csv-format'; const transformHeader = (header: string) => { - const key = floatControlKeyMap[header as FloatControlRawHeader]; - if (!key && !Object.values(RowKey).includes(header as RowKey)) { + const normalized = normalizeFloatControlHeader(header); + if (normalized === header.trim() && !isNormalizedRowKey(normalized)) { console.warn('Unknown header found in CSV file', { header }); } - return key ?? header; + return normalized; }; const parseFloatValue = (input: string): number => { diff --git a/src/lib/parse/floaty.ts b/src/lib/parse/floaty.ts index 2b9d5c4..f103c79 100644 --- a/src/lib/parse/floaty.ts +++ b/src/lib/parse/floaty.ts @@ -5,14 +5,7 @@ import { attachIndex } from '../misc'; import { FloatyJsonSchema, type ZFloatyJson, type ZLocation, type ZLog } from './floaty.types'; import { DataSource, stateCodeMap, Units, type Row } from './types'; import { ParseError } from './errors'; - -/** Headers that uniquely identify a Floaty CSV export (vs Float Control / VESC Tool). */ -const FLOATY_CSV_MARKERS = ['dutyCycle', 'batteryVolts', 'tripDistance'] as const; - -export function looksLikeFloatyCsv(headerLine: string): boolean { - const headers = new Set(headerLine.split(',').map((header) => header.trim())); - return FLOATY_CSV_MARKERS.every((marker) => headers.has(marker)); -} +import { cleanCsvHeader } from './csv-format'; /** * NOTE: sometimes Floaty doesn't record values, and seems to just put `null` (or 0) in its logs. @@ -154,6 +147,7 @@ export async function parseFloatyCsv(input: string | File): Promise csv.parse(text, { header: true, skipEmptyLines: true, + transformHeader: cleanCsvHeader, complete: (results) => { try { if (results.data.length === 0) { diff --git a/src/lib/parse/index.test.ts b/src/lib/parse/index.test.ts index 0dcbbe3..0142964 100644 --- a/src/lib/parse/index.test.ts +++ b/src/lib/parse/index.test.ts @@ -37,6 +37,45 @@ describe(parse.name, () => { expect(result.data).toHaveLength(3); }); + test('routes normalized Float Control headers to the Float Control parser', async () => { + const normalizedCsv = 'time,state,distance,speed,duty\n0,riding,0,1,2\n'; + const result = await parse(new MockFile([normalizedCsv], 'normalized.csv', { type: 'text/csv' })); + expect(result.source).toBe('float_control'); + expect(result.data).toHaveLength(1); + }); + + test('handles quoted Floaty headers with a byte-order mark', async () => { + const quotedFloatyCsv = `\uFEFF"timestamp","speed","dutyCycle","batteryVolts","tripDistance"\n1000,1,0.1,80,1\n`; + const result = await parse(new MockFile([quotedFloatyCsv], 'quoted-floaty.csv', { type: 'text/csv' })); + expect(result.source).toBe('floaty'); + expect(result.errors).toEqual([]); + expect(result.data).toHaveLength(1); + expect(result.data[0]).toMatchObject({ time: 0, speed: 1, duty: 10, voltage: 80, distance: 1 }); + }); + + test('does not classify a CSV from one coincidentally matching header', async () => { + const result = await parse(new MockFile(['State,foo\nriding,1\n'], 'coincidental.csv', { type: 'text/csv' })); + expect(result.source).toBe('none'); + expect(result.data).toEqual([]); + expect(result.errors).toHaveLength(1); + }); + + test('rejects unrecognised semicolon-delimited CSV headers', async () => { + const result = await parse(new MockFile(['foo;bar\n1;2\n'], 'unknown.csv', { type: 'text/csv' })); + expect(result.source).toBe('none'); + expect(result.data).toEqual([]); + expect(result.errors).toHaveLength(1); + }); + + test('rejects headers that ambiguously match multiple formats', async () => { + const ambiguousCsv = + 'timestamp,dutyCycle,batteryVolts,tripDistance,ms_today,input_voltage,duty_cycle\n1,0.1,80,1,1,80,0.1\n'; + const result = await parse(new MockFile([ambiguousCsv], 'ambiguous.csv', { type: 'text/csv' })); + expect(result.source).toBe('none'); + expect(result.data).toEqual([]); + expect(result.errors).toHaveLength(1); + }); + test('rejects unrecognised CSV headers with a parse error', async () => { const result = await parse(new MockFile(['foo,bar\n1,2\n'], 'unknown.csv', { type: 'text/csv' })); expect(result.source).toBe('none'); diff --git a/src/lib/parse/index.ts b/src/lib/parse/index.ts index e93fdd4..6461121 100644 --- a/src/lib/parse/index.ts +++ b/src/lib/parse/index.ts @@ -1,10 +1,11 @@ import * as fflate from 'fflate'; -import { parseFloatControlCsv, looksLikeFloatControlCsv } from './float-control'; -import { looksLikeFloatyCsv, parseFloatyCsv, parseFloatyJson } from './floaty'; +import { parseFloatControlCsv } from './float-control'; +import { parseFloatyCsv, parseFloatyJson } from './floaty'; import { parseVescToolCsv } from './vesc-tool'; import { DataSource, Units, type RowWithIndex } from './types'; import { ParseError } from './errors'; +import { CsvFormat, detectCsvFormat } from './csv-format'; export interface ParseResult { data: RowWithIndex[]; @@ -52,27 +53,20 @@ export async function parse(file: File): Promise { if (file.type === SupportedMimeTypes.Csv || lowerName.endsWith('.csv')) { const text = await file.text(); - const firstLine = text.split(/\r?\n/, 1)[0] ?? ''; - const semicolonCount = (firstLine.match(/;/g) ?? []).length; - const commaCount = (firstLine.match(/,/g) ?? []).length; - - // Heuristic: VESC Tool exports are semicolon-delimited, while Float Control / Floaty use commas. - if (semicolonCount > commaCount) { - return await parseVescToolCsv(text); - } - - if (looksLikeFloatyCsv(firstLine)) { - return await parseFloatyCsv(text); - } - - if (looksLikeFloatControlCsv(firstLine)) { - const parsed = await parseFloatControlCsv(text); - return { - source: DataSource.FloatControl, - data: parsed.csv.data, - units: parsed.units, - errors: parsed.errors, - }; + switch (detectCsvFormat(text)) { + case CsvFormat.Floaty: + return await parseFloatyCsv(text); + case CsvFormat.FloatControl: { + const parsed = await parseFloatControlCsv(text); + return { + source: DataSource.FloatControl, + data: parsed.csv.data, + units: parsed.units, + errors: parsed.errors, + }; + } + case CsvFormat.VescTool: + return await parseVescToolCsv(text); } return { @@ -81,7 +75,7 @@ export async function parse(file: File): Promise { units: Units.Metric, errors: [ new ParseError('Unrecognised CSV headers. Expected a Float Control, Floaty, or VESC Tool export.', { - header: firstLine, + header: text.split(/\r?\n/, 1)[0] ?? '', }), ], };