Skip to content

feat(bench): add raw Workshop scenarios and a Tier 1 matrix example #462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Teakowa merged 1 commit into from
Sep 30, 2026
Merged

feat(bench): add raw Workshop scenarios and a Tier 1 matrix example #462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions benchmarks/agent/matrix.example.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"agents": [
{"id": "example-model", "cmd": "BENCH_MODEL=example-model python3 benchmarks/agent/adapters/claude_code.py"}
],
"cells": [
{"wright": "none", "knowledge": "none", "network": "off"},
{"wright": "bin", "knowledge": "none", "network": "off"},
{"wright": "bin+skill", "knowledge": "none", "network": "off"},
{"wright": "none", "knowledge": "web", "network": "on"},
{"wright": "none", "knowledge": "wiki", "network": "off"}
],
"trials": 5,
"parallel": 2,
"seed": 1,
"options": {"skill_dir": "../skills/skills/wright", "wiki_dir": "path/to/pinned/wiki", "env_pass": ["HOME", "ANTHROPIC_API_KEY"]}
}
268 changes: 268 additions & 0 deletions benchmarks/agent/scenarios/ana-paintball-ws/negative/no-raycast/mode.ws
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
settings
{
main
{
Mode Name: "Ana Paintball"
Description: "Every Ana rifle shot kills. Scoped shots pierce. Sleep darts + melee kill (and cost the victim a point). 5/10/15 streaks, nuke at 15, dart-only finish at 24. First to 25."
}
modes
{
Deathmatch
{
enabled maps
{
Black Forest
Château Guillard
Eichenwalde
Kanezaka
}
Score To Win: 25
}
}
}

variables {
global:
0: pierceEye
1: pierceDir
2: pierceWall
3: pierceCandidates
4: pierceTarget
5: pierceIdx
6: pierceAlong
7: pierceOnRay
player:
0: streak
1: nukeActive
2: nukeFired
}

rule ("Force Ana") {
event {
Ongoing - Each Player;
All;
All;
}
conditions {
Has Spawned(Event Player) == True;
(Event Player).nukeActive == False;
}
actions {
Start Forcing Player To Be Hero(Event Player, Hero(Ana));
}
}

rule ("Rifle hit kills") {
event {
Player Dealt Damage;
All;
All;
}
conditions {
Event Ability == Button(Primary Fire);
Attacker != Victim;
(Attacker).nukeActive == False;
Score Of(Attacker) < 24;
}
actions {
Kill(Victim, Attacker);
"Scoped: also kill every other enemy on the line of fire up to the first wall.\nNo waits in here, so the shared scratch globals cannot interleave between shots."
If(Is Firing Secondary(Attacker));
Set Global Variable(pierceEye, Eye Position(Attacker));
Set Global Variable(pierceDir, Facing Direction Of(Attacker));
Set Global Variable(pierceWall, 200);
Set Global Variable(pierceCandidates, All Living Players(All Teams));
For Global Variable(pierceIdx, 0, Count Of(Global.pierceCandidates), 1);
Set Global Variable(pierceTarget, Value In Array(Global.pierceCandidates, Global.pierceIdx));
Skip If(Compare(Global.pierceTarget, ==, Attacker), 6);
Set Global Variable(pierceAlong, Dot Product(Subtract(Add(Position Of(Global.pierceTarget), Vector(0, 0.9, 0)), Global.pierceEye), Global.pierceDir));
Skip If(Or(Compare(Global.pierceAlong, <, 0), Compare(Global.pierceAlong, >, Global.pierceWall)), 4);
Set Global Variable(pierceOnRay, Add(Global.pierceEye, Multiply(Global.pierceDir, Global.pierceAlong)));
If(Compare(Min(Min(Distance Between(Add(Position Of(Global.pierceTarget), Vector(0, 0.3, 0)), Global.pierceOnRay), Distance Between(Add(Position Of(Global.pierceTarget), Vector(0, 0.9, 0)), Global.pierceOnRay)), Distance Between(Eye Position(Global.pierceTarget), Global.pierceOnRay)), <=, 0.6));
Kill(Global.pierceTarget, Attacker);
End;
//__label_continue_2__:
//__label_continue_3__:
End;
}
}

rule ("Melee kills sleeper") {
event {
Player Dealt Damage;
All;
All;
}
conditions {
Event Ability == Button(Melee);
Attacker != Victim;
Has Status(Victim, Asleep) == True;
}
actions {
Kill(Victim, Attacker);
Modify Player Score(Victim, -1);
}
}

rule ("Death bookkeeping") {
event {
Player Died;
All;
All;
}
actions {
Set Player Variable(Event Player, streak, 0);
If((Event Player).nukeActive);
Set Player Variable(Event Player, nukeActive, False);
Set Player Variable(Event Player, nukeFired, False);
}
}

rule ("Count kill") {
event {
Player Died;
All;
All;
}
conditions {
Attacker != Null;
Attacker != Event Player;
}
actions {
Modify Player Variable(Attacker, streak, Add, 1);
If(Compare((Attacker).streak, ==, 5));
Big Message(All Players(All Teams), Custom String("{0} is on a RAMPAGE! 5 kills in a row", Attacker));
Else If(Compare((Attacker).streak, ==, 10));
Big Message(All Players(All Teams), Custom String("{0} is UNSTOPPABLE! 10 kills in a row", Attacker));
Else If(Compare((Attacker).streak, ==, 15));
Big Message(All Players(All Teams), Custom String("{0} is GODLIKE! 15 kills in a row - NUCLEAR STRIKE incoming", Attacker));
}
}

rule ("Grant nuke") {
event {
Ongoing - Each Player;
All;
All;
}
conditions {
(Event Player).streak == 15;
(Event Player).nukeActive == False;
Is Alive(Event Player) == True;
}
actions {
Set Player Variable(Event Player, nukeActive, True);
Set Player Variable(Event Player, nukeFired, False);
Start Forcing Player To Be Hero(Event Player, Hero(Doomfist));
Wait(0.25, Ignore Condition);
Set Primary Fire Enabled(Event Player, False);
Set Secondary Fire Enabled(Event Player, False);
Set Ability 1 Enabled(Event Player, False);
Set Ability 2 Enabled(Event Player, False);
Set Melee Enabled(Event Player, False);
Set Ultimate Charge(Event Player, 100);
Big Message(Event Player, Custom String("NUCLEAR STRIKE READY - use your ultimate!"));
}
}

rule ("Nuke fired") {
event {
Ongoing - Each Player;
All;
All;
}
conditions {
(Event Player).nukeActive != False;
Is Using Ultimate(Event Player) == True;
}
actions {
Set Player Variable(Event Player, nukeFired, True);
}
}

rule ("Nuke kill wins") {
event {
Player Died;
All;
All;
}
conditions {
Attacker != Null;
Attacker != Event Player;
(Attacker).nukeActive != False;
Or((Attacker).nukeFired, Compare(Event Ability, ==, Button(Ultimate))) == True;
}
actions {
Declare Player Victory(Attacker);
}
}

rule ("Nuke spent") {
event {
Ongoing - Each Player;
All;
All;
}
conditions {
(Event Player).nukeActive != False;
(Event Player).nukeFired != False;
Is Using Ultimate(Event Player) == False;
}
actions {
"Let late meteor kills register before handing the rifle back."
Wait(1.5, Abort When False);
Set Player Variable(Event Player, nukeActive, False);
Set Player Variable(Event Player, nukeFired, False);
Start Forcing Player To Be Hero(Event Player, Hero(Ana));
Wait(0.25, Ignore Condition);
Set Primary Fire Enabled(Event Player, True);
Set Secondary Fire Enabled(Event Player, True);
Set Ability 1 Enabled(Event Player, True);
Set Ability 2 Enabled(Event Player, True);
Set Melee Enabled(Event Player, True);
}
}

rule ("Dart-only at 24") {
event {
Ongoing - Each Player;
All;
All;
}
conditions {
Has Spawned(Event Player) == True;
}
actions {
While(True);
Set Primary Fire Enabled(Event Player, And(Compare(Score Of(Event Player), <, 24), Not((Event Player).nukeActive)));
Wait(0.25, Ignore Condition);
End;
}
}

rule ("Fast dart at 24") {
event {
Ongoing - Each Player;
All;
All;
}
conditions {
Score Of(Event Player) >= 24;
Ability Cooldown(Event Player, Button(Ability 1)) > 2.5;
}
actions {
Set Ability Cooldown(Event Player, Button(Ability 1), 2.5);
}
}

rule ("HUD") {
event {
Ongoing - Each Player;
All;
All;
}
actions {
Create HUD Text(Event Player, Custom String("Streak {0}", (Event Player).streak), Null, Null, Left, 0, Color(White), Null, Null, Visible To and String, Default Visibility);
Create HUD Text(If-Then-Else(Compare(Score Of(Event Player), >=, 24), Event Player, Null), Null, Custom String("Rifle OFF - land a sleep dart, then melee"), Null, Left, 1, Null, Color(Red), Null, Visible To and String, Default Visibility);
}
}
13 changes: 13 additions & 0 deletions benchmarks/agent/scenarios/ana-paintball-ws/prompt.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
Create an Ana Paintball game mode for the Overwatch Workshop. Write it as raw Workshop script (the text format the Workshop editor imports and exports) and save it as `mode.ws` in the current directory (a single file).

Every shot must kill instantly.

- Maps: Black Forest, Château Guillard, Eichenwalde, Kanezaka
- First to 25 score wins
- Rampage: display big messages when a player gets 5, 10, and 15 kills in a row
- At 15 kills in a row a player gets a "nuclear strike" (Doomfist's ultimate). If they kill someone with it, they instantly win
- Scoped shots must pierce enemies
- The sleep dart doesn't kill instantly (or at all), but meleeing a sleeping enemy kills them and the killed player loses one point
- A player at 24 score has the rifle disabled; they must kill someone with the sleep dart (cooldown reduced to 2.5s)

When you are done, say briefly what you verified and what you could not verify.
Loading
Loading