Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ function units(payload) {

// Report the token breakdown of a payload and where the tokens are going.
export function analyzePayload(payload, { pricePerMTok = 3, counter = estimateTokens, top = 10 } = {}) {
requireNonNegativeNumber("pricePerMTok", pricePerMTok);
requireNonNegativeNumber("top", top, { integer: true });
const us = units(payload).map((u) => ({ ...u, tokens: u.fixedTokens ?? counter(u.text) }));
const total = us.reduce((a, u) => a + u.tokens, 0);
const byKind = {}, byRole = {};
Expand Down
8 changes: 8 additions & 0 deletions test/basic.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,13 @@ test("units rejects non-array payload.messages with TypeError", () => {
});



test("analyzePayload validates pricePerMTok and top", () => {
const payload = { messages: [{ role: "user", content: "hello" }] };
assert.throws(() => analyzePayload(payload, { pricePerMTok: -5 }), RangeError);
assert.throws(() => analyzePayload(payload, { top: -1 }), RangeError);
assert.throws(() => analyzePayload(payload, { top: 1.5 }), RangeError);
});
test("analyzePayload charges image_url blocks a flat documented estimate", () => {
const short = analyzePayload({ messages: [{ role: "user", content: [
{ type: "image_url", image_url: { url: "https://example.com/a.png" } },
Expand All @@ -186,6 +193,7 @@ test("analyzePayload charges image_url blocks a flat documented estimate", () =>
assert.equal(long.byKind.image_url, 85);
assert.equal(short.totalTokens, long.totalTokens);
});

test("estimateTokens keeps CJK text within the documented reference tolerance", () => {
const text = "漢".repeat(200);
const referenceTokens = 200;
Expand Down
Loading