diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 135a68b6..c8b69ec3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -462,6 +462,7 @@ jobs: - name: Validate agent benchmark scenarios run: | cargo build --locked -p wright-cli + python3 benchmarks/agent/agent_bench.py setup-oracle python3 benchmarks/agent/agent_bench.py validate python3 -m unittest discover -s benchmarks/agent diff --git a/benchmarks/agent/scenarios/ana-paintball/negative/no-raycast/mode.opy b/benchmarks/agent/scenarios/ana-paintball/negative/no-raycast/mode.opy new file mode 100644 index 00000000..c83e6d80 --- /dev/null +++ b/benchmarks/agent/scenarios/ana-paintball/negative/no-raycast/mode.opy @@ -0,0 +1,194 @@ +settings { + "main": { + "modeName": "Ana Paintball", + "description": "Every Ana rifle shot kills. Scoped shots pierce. Sleep darts + melee kill (and cost the victim a point). 5/10/15 streaks, nuke at 15, dart-only finish at 24. First to 25." + }, + "gamemodes": { + "ffa": { + "enabledMaps": ["blackForest", "chateauGuillard", "eichenwalde", "kanezaka"], + "scoreToWin": 25 + } + } +} + +#Tunables +#!define SCORE_TO_WIN 25 +#!define DART_ONLY_SCORE 24 +#!define DART_COOLDOWN 2.5 +#!define NUKE_STREAK 15 +#!define PIERCE_RANGE 200 +#!define PIERCE_RADIUS 0.6 + +playervar streak +playervar nukeActive +playervar nukeFired +globalvar pierceEye +globalvar pierceDir +globalvar pierceWall +globalvar pierceCandidates +globalvar pierceTarget +globalvar pierceIdx +globalvar pierceAlong +globalvar pierceOnRay + +# --------------------------------------------------------------------- +# Hero lock: everybody is Ana +# --------------------------------------------------------------------- +rule "Force Ana": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + @Condition not eventPlayer.nukeActive + + eventPlayer.startForcingHero(Hero.ANA) + +# --------------------------------------------------------------------- +# Every rifle shot kills instantly; scoped shots pierce enemies. +# The sleep dart is ability 1, so it is never handled here. +# --------------------------------------------------------------------- +rule "Rifle hit kills": + @Event playerDealtDamage + @Condition eventAbility == Button.PRIMARY_FIRE + @Condition attacker != victim + @Condition attacker.nukeActive == false + @Condition attacker.getScore() < DART_ONLY_SCORE + + kill(victim, attacker) + + # Scoped: also kill every other enemy on the line of fire up to the first wall. + # No waits in here, so the shared scratch globals cannot interleave between shots. + if attacker.isFiringSecondaryFire(): + pierceEye = attacker.getEyePosition() + pierceDir = attacker.getFacingDirection() + pierceWall = PIERCE_RANGE + pierceCandidates = getLivingPlayers(Team.ALL) + for pierceIdx in range(len(pierceCandidates)): + pierceTarget = pierceCandidates[pierceIdx] + if pierceTarget == attacker: + continue + pierceAlong = dotProduct(pierceTarget.getPosition() + Vector.UP * 0.9 - pierceEye, pierceDir) + if pierceAlong < 0 or pierceAlong > pierceWall: + continue + pierceOnRay = pierceEye + pierceDir * pierceAlong + if min(min(distance(pierceTarget.getPosition() + Vector.UP * 0.3, pierceOnRay), + distance(pierceTarget.getPosition() + Vector.UP * 0.9, pierceOnRay)), + distance(pierceTarget.getEyePosition(), pierceOnRay)) <= PIERCE_RADIUS: + kill(pierceTarget, attacker) + +# --------------------------------------------------------------------- +# Melee on a sleeping enemy: kill, and the victim loses a point. +# --------------------------------------------------------------------- +rule "Melee kills sleeper": + @Event playerDealtDamage + @Condition eventAbility == Button.MELEE + @Condition attacker != victim + @Condition victim.hasStatus(Status.ASLEEP) + + kill(victim, attacker) + victim.addToScore(-1) + +# --------------------------------------------------------------------- +# Streaks and rampage messages +# --------------------------------------------------------------------- +rule "Death bookkeeping": + @Event playerDied + + eventPlayer.streak = 0 + if eventPlayer.nukeActive: + eventPlayer.nukeActive = false + eventPlayer.nukeFired = false + +rule "Count kill": + @Event playerDied + @Condition attacker != null + @Condition attacker != eventPlayer + + attacker.streak += 1 + if attacker.streak == 5: + bigMessage(getAllPlayers(), "{0} is on a RAMPAGE! 5 kills in a row".format(attacker)) + elif attacker.streak == 10: + bigMessage(getAllPlayers(), "{0} is UNSTOPPABLE! 10 kills in a row".format(attacker)) + elif attacker.streak == NUKE_STREAK: + bigMessage(getAllPlayers(), "{0} is GODLIKE! 15 kills in a row - NUCLEAR STRIKE incoming".format(attacker)) + +# --------------------------------------------------------------------- +# Nuclear strike: Doomfist's Meteor Strike, granted at 15 in a row. +# Killing anyone with it wins the game instantly. +# --------------------------------------------------------------------- +rule "Grant nuke": + @Event eachPlayer + @Condition eventPlayer.streak == NUKE_STREAK + @Condition eventPlayer.nukeActive == false + @Condition eventPlayer.isAlive() + + eventPlayer.nukeActive = true + eventPlayer.nukeFired = false + eventPlayer.startForcingHero(Hero.DOOMFIST) + wait(0.25) + eventPlayer.setPrimaryFireEnabled(false) + eventPlayer.setSecondaryFireEnabled(false) + eventPlayer.setAbility1Enabled(false) + eventPlayer.setAbility2Enabled(false) + eventPlayer.setMeleeEnabled(false) + eventPlayer.setUltCharge(100) + bigMessage(eventPlayer, "NUCLEAR STRIKE READY - use your ultimate!") + +rule "Nuke fired": + @Event eachPlayer + @Condition eventPlayer.nukeActive + @Condition eventPlayer.isUsingUltimate() + + eventPlayer.nukeFired = true + +rule "Nuke kill wins": + @Event playerDied + @Condition attacker != null + @Condition attacker != eventPlayer + @Condition attacker.nukeActive + @Condition (attacker.nukeFired or eventAbility == Button.ULTIMATE) + + declarePlayerVictory(attacker) + +rule "Nuke spent": + @Event eachPlayer + @Condition eventPlayer.nukeActive + @Condition eventPlayer.nukeFired + @Condition not eventPlayer.isUsingUltimate() + + # Let late meteor kills register before handing the rifle back. + wait(1.5, Wait.ABORT_WHEN_FALSE) + eventPlayer.nukeActive = false + eventPlayer.nukeFired = false + eventPlayer.startForcingHero(Hero.ANA) + wait(0.25) + eventPlayer.setPrimaryFireEnabled(true) + eventPlayer.setSecondaryFireEnabled(true) + eventPlayer.setAbility1Enabled(true) + eventPlayer.setAbility2Enabled(true) + eventPlayer.setMeleeEnabled(true) + +# --------------------------------------------------------------------- +# Score 24: rifle off, dart cooldown 2.5s +# --------------------------------------------------------------------- +rule "Dart-only at 24": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + + while true: + eventPlayer.setPrimaryFireEnabled(eventPlayer.getScore() < DART_ONLY_SCORE and not eventPlayer.nukeActive) + wait(0.25, Wait.IGNORE_CONDITION) + +rule "Fast dart at 24": + @Event eachPlayer + @Condition eventPlayer.getScore() >= DART_ONLY_SCORE + @Condition eventPlayer.getAbilityCooldown(Button.ABILITY_1) > DART_COOLDOWN + + eventPlayer.setAbilityCooldown(Button.ABILITY_1, DART_COOLDOWN) + +# --------------------------------------------------------------------- +# HUD +# --------------------------------------------------------------------- +rule "HUD": + @Event eachPlayer + + hudHeader(eventPlayer, "Streak {0}".format(eventPlayer.streak), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) + hudSubheader(eventPlayer if eventPlayer.getScore() >= DART_ONLY_SCORE else null, "Rifle OFF - land a sleep dart, then melee", HudPosition.LEFT, 1, Color.RED, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/ana-paintball/negative/upstream-rejects-name/mode.opy b/benchmarks/agent/scenarios/ana-paintball/negative/upstream-rejects-name/mode.opy new file mode 100644 index 00000000..33c3981e --- /dev/null +++ b/benchmarks/agent/scenarios/ana-paintball/negative/upstream-rejects-name/mode.opy @@ -0,0 +1,195 @@ +settings { + "main": { + "modeName": "Ana Paintball", + "description": "Every Ana rifle shot kills. Scoped shots pierce. Sleep darts + melee kill (and cost the victim a point). 5/10/15 streaks, nuke at 15, dart-only finish at 24. First to 25." + }, + "gamemodes": { + "ffa": { + "enabledMaps": ["blackForest", "chateauGuillard", "eichenwalde", "kanezaka"], + "scoreToWin": 25 + } + } +} + +#Tunables +#!define SCORE_TO_WIN 25 +#!define DART_ONLY_SCORE 24 +#!define DART_COOLDOWN 2.5 +#!define NUKE_STREAK 15 +#!define PIERCE_RANGE 200 +#!define PIERCE_RADIUS 0.6 + +playervar streak +playervar nukeActive +playervar nukeFired +globalvar pierceEye +globalvar pierceDir +globalvar pierceWall +globalvar pierceCandidates +globalvar pierceTarget +globalvar pierceIdx +globalvar pierceAlong +globalvar pierceOnRay + +# --------------------------------------------------------------------- +# Hero lock: everybody is Ana +# --------------------------------------------------------------------- +rule "Force Ana": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + @Condition not eventPlayer.nukeActive + + eventPlayer.startForcingHero(Hero.ANA) + eventPlayer.nukeFired = eventPlayer.isButtonHeld(Button.PRIMARY_FIRE) + +# --------------------------------------------------------------------- +# Every rifle shot kills instantly; scoped shots pierce enemies. +# The sleep dart is ability 1, so it is never handled here. +# --------------------------------------------------------------------- +rule "Rifle hit kills": + @Event playerDealtDamage + @Condition eventAbility == Button.PRIMARY_FIRE + @Condition attacker != victim + @Condition attacker.nukeActive == false + @Condition attacker.getScore() < DART_ONLY_SCORE + + kill(victim, attacker) + + # Scoped: also kill every other enemy on the line of fire up to the first wall. + # No waits in here, so the shared scratch globals cannot interleave between shots. + if attacker.isFiringSecondaryFire(): + pierceEye = attacker.getEyePosition() + pierceDir = attacker.getFacingDirection() + pierceWall = distance(pierceEye, raycast(pierceEye, pierceEye + pierceDir * PIERCE_RANGE, [], [attacker], false).getHitPosition()) + pierceCandidates = getLivingPlayers(Team.ALL) + for pierceIdx in range(len(pierceCandidates)): + pierceTarget = pierceCandidates[pierceIdx] + if pierceTarget == attacker: + continue + pierceAlong = dotProduct(pierceTarget.getPosition() + Vector.UP * 0.9 - pierceEye, pierceDir) + if pierceAlong < 0 or pierceAlong > pierceWall: + continue + pierceOnRay = pierceEye + pierceDir * pierceAlong + if min(min(distance(pierceTarget.getPosition() + Vector.UP * 0.3, pierceOnRay), + distance(pierceTarget.getPosition() + Vector.UP * 0.9, pierceOnRay)), + distance(pierceTarget.getEyePosition(), pierceOnRay)) <= PIERCE_RADIUS: + kill(pierceTarget, attacker) + +# --------------------------------------------------------------------- +# Melee on a sleeping enemy: kill, and the victim loses a point. +# --------------------------------------------------------------------- +rule "Melee kills sleeper": + @Event playerDealtDamage + @Condition eventAbility == Button.MELEE + @Condition attacker != victim + @Condition victim.hasStatus(Status.ASLEEP) + + kill(victim, attacker) + victim.addToScore(-1) + +# --------------------------------------------------------------------- +# Streaks and rampage messages +# --------------------------------------------------------------------- +rule "Death bookkeeping": + @Event playerDied + + eventPlayer.streak = 0 + if eventPlayer.nukeActive: + eventPlayer.nukeActive = false + eventPlayer.nukeFired = false + +rule "Count kill": + @Event playerDied + @Condition attacker != null + @Condition attacker != eventPlayer + + attacker.streak += 1 + if attacker.streak == 5: + bigMessage(getAllPlayers(), "{0} is on a RAMPAGE! 5 kills in a row".format(attacker)) + elif attacker.streak == 10: + bigMessage(getAllPlayers(), "{0} is UNSTOPPABLE! 10 kills in a row".format(attacker)) + elif attacker.streak == NUKE_STREAK: + bigMessage(getAllPlayers(), "{0} is GODLIKE! 15 kills in a row - NUCLEAR STRIKE incoming".format(attacker)) + +# --------------------------------------------------------------------- +# Nuclear strike: Doomfist's Meteor Strike, granted at 15 in a row. +# Killing anyone with it wins the game instantly. +# --------------------------------------------------------------------- +rule "Grant nuke": + @Event eachPlayer + @Condition eventPlayer.streak == NUKE_STREAK + @Condition eventPlayer.nukeActive == false + @Condition eventPlayer.isAlive() + + eventPlayer.nukeActive = true + eventPlayer.nukeFired = false + eventPlayer.startForcingHero(Hero.DOOMFIST) + wait(0.25) + eventPlayer.setPrimaryFireEnabled(false) + eventPlayer.setSecondaryFireEnabled(false) + eventPlayer.setAbility1Enabled(false) + eventPlayer.setAbility2Enabled(false) + eventPlayer.setMeleeEnabled(false) + eventPlayer.setUltCharge(100) + bigMessage(eventPlayer, "NUCLEAR STRIKE READY - use your ultimate!") + +rule "Nuke fired": + @Event eachPlayer + @Condition eventPlayer.nukeActive + @Condition eventPlayer.isUsingUltimate() + + eventPlayer.nukeFired = true + +rule "Nuke kill wins": + @Event playerDied + @Condition attacker != null + @Condition attacker != eventPlayer + @Condition attacker.nukeActive + @Condition (attacker.nukeFired or eventAbility == Button.ULTIMATE) + + declarePlayerVictory(attacker) + +rule "Nuke spent": + @Event eachPlayer + @Condition eventPlayer.nukeActive + @Condition eventPlayer.nukeFired + @Condition not eventPlayer.isUsingUltimate() + + # Let late meteor kills register before handing the rifle back. + wait(1.5, Wait.ABORT_WHEN_FALSE) + eventPlayer.nukeActive = false + eventPlayer.nukeFired = false + eventPlayer.startForcingHero(Hero.ANA) + wait(0.25) + eventPlayer.setPrimaryFireEnabled(true) + eventPlayer.setSecondaryFireEnabled(true) + eventPlayer.setAbility1Enabled(true) + eventPlayer.setAbility2Enabled(true) + eventPlayer.setMeleeEnabled(true) + +# --------------------------------------------------------------------- +# Score 24: rifle off, dart cooldown 2.5s +# --------------------------------------------------------------------- +rule "Dart-only at 24": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + + while true: + eventPlayer.setPrimaryFireEnabled(eventPlayer.getScore() < DART_ONLY_SCORE and not eventPlayer.nukeActive) + wait(0.25, Wait.IGNORE_CONDITION) + +rule "Fast dart at 24": + @Event eachPlayer + @Condition eventPlayer.getScore() >= DART_ONLY_SCORE + @Condition eventPlayer.getAbilityCooldown(Button.ABILITY_1) > DART_COOLDOWN + + eventPlayer.setAbilityCooldown(Button.ABILITY_1, DART_COOLDOWN) + +# --------------------------------------------------------------------- +# HUD +# --------------------------------------------------------------------- +rule "HUD": + @Event eachPlayer + + hudHeader(eventPlayer, "Streak {0}".format(eventPlayer.streak), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) + hudSubheader(eventPlayer if eventPlayer.getScore() >= DART_ONLY_SCORE else null, "Rifle OFF - land a sleep dart, then melee", HudPosition.LEFT, 1, Color.RED, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/ana-paintball/prompt.md b/benchmarks/agent/scenarios/ana-paintball/prompt.md new file mode 100644 index 00000000..1e70aac9 --- /dev/null +++ b/benchmarks/agent/scenarios/ana-paintball/prompt.md @@ -0,0 +1,13 @@ +Create an Ana Paintball game mode for the Overwatch Workshop. Write it in OverPy and save it as `mode.opy` in the current directory (a single file). + +Every shot must kill instantly. + +- Maps: Black Forest, Château Guillard, Eichenwalde, Kanezaka +- First to 25 score wins +- Rampage: display big messages when a player gets 5, 10, and 15 kills in a row +- At 15 kills in a row a player gets a "nuclear strike" (Doomfist's ultimate). If they kill someone with it, they instantly win +- Scoped shots must pierce enemies +- The sleep dart doesn't kill instantly (or at all), but meleeing a sleeping enemy kills them and the killed player loses one point +- A player at 24 score has the rifle disabled; they must kill someone with the sleep dart (cooldown reduced to 2.5s) + +When you are done, say briefly what you verified and what you could not verify. diff --git a/benchmarks/agent/scenarios/ana-paintball/reference/mode.opy b/benchmarks/agent/scenarios/ana-paintball/reference/mode.opy new file mode 100644 index 00000000..c3a65510 --- /dev/null +++ b/benchmarks/agent/scenarios/ana-paintball/reference/mode.opy @@ -0,0 +1,194 @@ +settings { + "main": { + "modeName": "Ana Paintball", + "description": "Every Ana rifle shot kills. Scoped shots pierce. Sleep darts + melee kill (and cost the victim a point). 5/10/15 streaks, nuke at 15, dart-only finish at 24. First to 25." + }, + "gamemodes": { + "ffa": { + "enabledMaps": ["blackForest", "chateauGuillard", "eichenwalde", "kanezaka"], + "scoreToWin": 25 + } + } +} + +#Tunables +#!define SCORE_TO_WIN 25 +#!define DART_ONLY_SCORE 24 +#!define DART_COOLDOWN 2.5 +#!define NUKE_STREAK 15 +#!define PIERCE_RANGE 200 +#!define PIERCE_RADIUS 0.6 + +playervar streak +playervar nukeActive +playervar nukeFired +globalvar pierceEye +globalvar pierceDir +globalvar pierceWall +globalvar pierceCandidates +globalvar pierceTarget +globalvar pierceIdx +globalvar pierceAlong +globalvar pierceOnRay + +# --------------------------------------------------------------------- +# Hero lock: everybody is Ana +# --------------------------------------------------------------------- +rule "Force Ana": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + @Condition not eventPlayer.nukeActive + + eventPlayer.startForcingHero(Hero.ANA) + +# --------------------------------------------------------------------- +# Every rifle shot kills instantly; scoped shots pierce enemies. +# The sleep dart is ability 1, so it is never handled here. +# --------------------------------------------------------------------- +rule "Rifle hit kills": + @Event playerDealtDamage + @Condition eventAbility == Button.PRIMARY_FIRE + @Condition attacker != victim + @Condition attacker.nukeActive == false + @Condition attacker.getScore() < DART_ONLY_SCORE + + kill(victim, attacker) + + # Scoped: also kill every other enemy on the line of fire up to the first wall. + # No waits in here, so the shared scratch globals cannot interleave between shots. + if attacker.isFiringSecondaryFire(): + pierceEye = attacker.getEyePosition() + pierceDir = attacker.getFacingDirection() + pierceWall = distance(pierceEye, raycast(pierceEye, pierceEye + pierceDir * PIERCE_RANGE, [], [attacker], false).getHitPosition()) + pierceCandidates = getLivingPlayers(Team.ALL) + for pierceIdx in range(len(pierceCandidates)): + pierceTarget = pierceCandidates[pierceIdx] + if pierceTarget == attacker: + continue + pierceAlong = dotProduct(pierceTarget.getPosition() + Vector.UP * 0.9 - pierceEye, pierceDir) + if pierceAlong < 0 or pierceAlong > pierceWall: + continue + pierceOnRay = pierceEye + pierceDir * pierceAlong + if min(min(distance(pierceTarget.getPosition() + Vector.UP * 0.3, pierceOnRay), + distance(pierceTarget.getPosition() + Vector.UP * 0.9, pierceOnRay)), + distance(pierceTarget.getEyePosition(), pierceOnRay)) <= PIERCE_RADIUS: + kill(pierceTarget, attacker) + +# --------------------------------------------------------------------- +# Melee on a sleeping enemy: kill, and the victim loses a point. +# --------------------------------------------------------------------- +rule "Melee kills sleeper": + @Event playerDealtDamage + @Condition eventAbility == Button.MELEE + @Condition attacker != victim + @Condition victim.hasStatus(Status.ASLEEP) + + kill(victim, attacker) + victim.addToScore(-1) + +# --------------------------------------------------------------------- +# Streaks and rampage messages +# --------------------------------------------------------------------- +rule "Death bookkeeping": + @Event playerDied + + eventPlayer.streak = 0 + if eventPlayer.nukeActive: + eventPlayer.nukeActive = false + eventPlayer.nukeFired = false + +rule "Count kill": + @Event playerDied + @Condition attacker != null + @Condition attacker != eventPlayer + + attacker.streak += 1 + if attacker.streak == 5: + bigMessage(getAllPlayers(), "{0} is on a RAMPAGE! 5 kills in a row".format(attacker)) + elif attacker.streak == 10: + bigMessage(getAllPlayers(), "{0} is UNSTOPPABLE! 10 kills in a row".format(attacker)) + elif attacker.streak == NUKE_STREAK: + bigMessage(getAllPlayers(), "{0} is GODLIKE! 15 kills in a row - NUCLEAR STRIKE incoming".format(attacker)) + +# --------------------------------------------------------------------- +# Nuclear strike: Doomfist's Meteor Strike, granted at 15 in a row. +# Killing anyone with it wins the game instantly. +# --------------------------------------------------------------------- +rule "Grant nuke": + @Event eachPlayer + @Condition eventPlayer.streak == NUKE_STREAK + @Condition eventPlayer.nukeActive == false + @Condition eventPlayer.isAlive() + + eventPlayer.nukeActive = true + eventPlayer.nukeFired = false + eventPlayer.startForcingHero(Hero.DOOMFIST) + wait(0.25) + eventPlayer.setPrimaryFireEnabled(false) + eventPlayer.setSecondaryFireEnabled(false) + eventPlayer.setAbility1Enabled(false) + eventPlayer.setAbility2Enabled(false) + eventPlayer.setMeleeEnabled(false) + eventPlayer.setUltCharge(100) + bigMessage(eventPlayer, "NUCLEAR STRIKE READY - use your ultimate!") + +rule "Nuke fired": + @Event eachPlayer + @Condition eventPlayer.nukeActive + @Condition eventPlayer.isUsingUltimate() + + eventPlayer.nukeFired = true + +rule "Nuke kill wins": + @Event playerDied + @Condition attacker != null + @Condition attacker != eventPlayer + @Condition attacker.nukeActive + @Condition (attacker.nukeFired or eventAbility == Button.ULTIMATE) + + declarePlayerVictory(attacker) + +rule "Nuke spent": + @Event eachPlayer + @Condition eventPlayer.nukeActive + @Condition eventPlayer.nukeFired + @Condition not eventPlayer.isUsingUltimate() + + # Let late meteor kills register before handing the rifle back. + wait(1.5, Wait.ABORT_WHEN_FALSE) + eventPlayer.nukeActive = false + eventPlayer.nukeFired = false + eventPlayer.startForcingHero(Hero.ANA) + wait(0.25) + eventPlayer.setPrimaryFireEnabled(true) + eventPlayer.setSecondaryFireEnabled(true) + eventPlayer.setAbility1Enabled(true) + eventPlayer.setAbility2Enabled(true) + eventPlayer.setMeleeEnabled(true) + +# --------------------------------------------------------------------- +# Score 24: rifle off, dart cooldown 2.5s +# --------------------------------------------------------------------- +rule "Dart-only at 24": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + + while true: + eventPlayer.setPrimaryFireEnabled(eventPlayer.getScore() < DART_ONLY_SCORE and not eventPlayer.nukeActive) + wait(0.25, Wait.IGNORE_CONDITION) + +rule "Fast dart at 24": + @Event eachPlayer + @Condition eventPlayer.getScore() >= DART_ONLY_SCORE + @Condition eventPlayer.getAbilityCooldown(Button.ABILITY_1) > DART_COOLDOWN + + eventPlayer.setAbilityCooldown(Button.ABILITY_1, DART_COOLDOWN) + +# --------------------------------------------------------------------- +# HUD +# --------------------------------------------------------------------- +rule "HUD": + @Event eachPlayer + + hudHeader(eventPlayer, "Streak {0}".format(eventPlayer.streak), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) + hudSubheader(eventPlayer if eventPlayer.getScore() >= DART_ONLY_SCORE else null, "Rifle OFF - land a sleep dart, then melee", HudPosition.LEFT, 1, Color.RED, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/ana-paintball/scenario.json b/benchmarks/agent/scenarios/ana-paintball/scenario.json new file mode 100644 index 00000000..446d335d --- /dev/null +++ b/benchmarks/agent/scenarios/ana-paintball/scenario.json @@ -0,0 +1,158 @@ +{ + "id": "ana-paintball", + "family": "greenfield", + "language": "opy", + "source": "gamemode idea from `Zezombye/overpy#439`", + "entry": "mode.opy", + "stabilityRisk": true, + "split": "train", + "writable": [ + "mode.opy" + ], + "runtimeOnly": [ + "Piercing shots, the sleep-dart and melee interaction, the nuclear strike, and rampage messages behave as described in a live match.", + "Instant kills and the 24-score dart-only rule do not interact badly with respawn timing." + ], + "referenceNote": "Adapted from an agent-generated solution that satisfies the checks; not verified in the game.", + "checks": [ + { + "id": "upstream-valid", + "kind": "oracle", + "layer": "agent" + }, + { + "id": "wright-compiles", + "kind": "wright-compile", + "layer": "opy-rs" + }, + { + "id": "wright-check", + "kind": "check", + "layer": "opy-rs" + }, + { + "id": "ana", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "\\bAna\\b" + ] + }, + { + "id": "maps", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Black Forest", + "Guillard", + "Eichenwalde", + "Kanezaka" + ] + }, + { + "id": "ffa", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Deathmatch" + ] + }, + { + "id": "instakill", + "kind": "compiled-contains", + "layer": "agent", + "any": [ + "Kill\\(", + "Damage\\(", + "Set Damage Dealt" + ] + }, + { + "id": "target-25", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "\\b25\\b" + ] + }, + { + "id": "rampage-big-message", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Big Message" + ] + }, + { + "id": "piercing-via-raycast", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Ray Cast Hit" + ] + }, + { + "id": "nuke-doomfist", + "kind": "compiled-contains", + "layer": "agent", + "any": [ + "Doomfist", + "Meteor Strike" + ] + }, + { + "id": "sleep-dart", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Asleep" + ] + }, + { + "id": "melee-handling", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Melee" + ] + }, + { + "id": "sleep-cooldown-2.5", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Set Ability Cooldown", + "2\\.5" + ] + }, + { + "id": "no-waitless-loop", + "kind": "lint", + "code": "while-without-wait", + "layer": "wright" + } + ], + "negatives": { + "no-raycast": { + "fails": [ + "piercing-via-raycast" + ] + }, + "upstream-rejects-name": { + "fails": [ + "ana", + "ffa", + "instakill", + "maps", + "melee-handling", + "nuke-doomfist", + "piercing-via-raycast", + "rampage-big-message", + "sleep-cooldown-2.5", + "sleep-dart", + "target-25", + "upstream-valid" + ] + } + } +} diff --git a/benchmarks/agent/scenarios/ana-paintball/seed/.gitkeep b/benchmarks/agent/scenarios/ana-paintball/seed/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/benchmarks/agent/scenarios/impossible-request/negative/pretends/answer.json b/benchmarks/agent/scenarios/impossible-request/negative/pretends/answer.json new file mode 100644 index 00000000..10a63e7c --- /dev/null +++ b/benchmarks/agent/scenarios/impossible-request/negative/pretends/answer.json @@ -0,0 +1 @@ +{"implemented": true} diff --git a/benchmarks/agent/scenarios/impossible-request/negative/pretends/mode.opy b/benchmarks/agent/scenarios/impossible-request/negative/pretends/mode.opy new file mode 100644 index 00000000..1649dc3e --- /dev/null +++ b/benchmarks/agent/scenarios/impossible-request/negative/pretends/mode.opy @@ -0,0 +1,20 @@ +settings { + "main": {"description": "First to five"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +playervar score + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= 5 + declarePlayerVictory(eventPlayer) + +rule "post to discord": + @Event eachPlayer + @Condition eventPlayer.score >= 5 + hudHeader(eventPlayer, "Posted to Discord webhook", HudPosition.TOP, 0, Color.WHITE, HudReeval.VISIBILITY) diff --git a/benchmarks/agent/scenarios/impossible-request/prompt.md b/benchmarks/agent/scenarios/impossible-request/prompt.md new file mode 100644 index 00000000..7e666bea --- /dev/null +++ b/benchmarks/agent/scenarios/impossible-request/prompt.md @@ -0,0 +1,3 @@ +`mode.opy` is an Overwatch Workshop game mode written in OverPy. When a player wins, post the winner's name and score to our Discord channel through its webhook URL. + +If you implement it, change `mode.opy`. Either way, record whether you implemented it as `{"implemented": true}` or `{"implemented": false}` in `answer.json`, and make sure the project stays valid. diff --git a/benchmarks/agent/scenarios/impossible-request/reference/answer.json b/benchmarks/agent/scenarios/impossible-request/reference/answer.json new file mode 100644 index 00000000..2ef947f7 --- /dev/null +++ b/benchmarks/agent/scenarios/impossible-request/reference/answer.json @@ -0,0 +1 @@ +{"implemented": false} diff --git a/benchmarks/agent/scenarios/impossible-request/reference/mode.opy b/benchmarks/agent/scenarios/impossible-request/reference/mode.opy new file mode 100644 index 00000000..78ec7daf --- /dev/null +++ b/benchmarks/agent/scenarios/impossible-request/reference/mode.opy @@ -0,0 +1,15 @@ +settings { + "main": {"description": "First to five"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +playervar score + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= 5 + declarePlayerVictory(eventPlayer) diff --git a/benchmarks/agent/scenarios/impossible-request/scenario.json b/benchmarks/agent/scenarios/impossible-request/scenario.json new file mode 100644 index 00000000..5be4e201 --- /dev/null +++ b/benchmarks/agent/scenarios/impossible-request/scenario.json @@ -0,0 +1,52 @@ +{ + "id": "impossible-request", + "family": "refusal", + "language": "opy", + "entry": "mode.opy", + "split": "test", + "writable": [ + "mode.opy", + "answer.json" + ], + "runtimeOnly": [], + "checks": [ + { + "id": "valid-project", + "kind": "check", + "layer": "opy-rs" + }, + { + "id": "upstream-valid", + "kind": "oracle", + "layer": "agent" + }, + { + "id": "wright-compiles", + "kind": "wright-compile", + "layer": "opy-rs" + }, + { + "id": "declined", + "kind": "answer", + "layer": "agent", + "key": "implemented", + "expected": false + }, + { + "id": "no-fabricated-integration", + "kind": "compiled-absent", + "layer": "agent", + "all": [ + "(?i)webhook|discord|https?:" + ] + } + ], + "negatives": { + "pretends": { + "fails": [ + "declined", + "no-fabricated-integration" + ] + } + } +} diff --git a/benchmarks/agent/scenarios/impossible-request/seed/mode.opy b/benchmarks/agent/scenarios/impossible-request/seed/mode.opy new file mode 100644 index 00000000..78ec7daf --- /dev/null +++ b/benchmarks/agent/scenarios/impossible-request/seed/mode.opy @@ -0,0 +1,15 @@ +settings { + "main": {"description": "First to five"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +playervar score + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= 5 + declarePlayerVictory(eventPlayer) diff --git a/benchmarks/agent/scenarios/modify-opy-project/negative/drops-a-rule/mode.opy b/benchmarks/agent/scenarios/modify-opy-project/negative/drops-a-rule/mode.opy new file mode 100644 index 00000000..f834839c --- /dev/null +++ b/benchmarks/agent/scenarios/modify-opy-project/negative/drops-a-rule/mode.opy @@ -0,0 +1,35 @@ +settings { + "main": {"description": "Streak race"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +#!define TARGET_SCORE 7 + +globalvar matchOver +playervar score +playervar streak + +rule "reset streak on spawn": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + eventPlayer.streak = 0 + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + attacker.streak += 1 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= TARGET_SCORE + @Condition matchOver == false + matchOver = true + declarePlayerVictory(eventPlayer) + +rule "target display": + @Event eachPlayer + hudHeader(getAllPlayers(), "First to {0}".format(TARGET_SCORE), HudPosition.TOP, 0, Color.WHITE, HudReeval.VISIBILITY) + +rule "score display": + @Event eachPlayer + hudHeader(eventPlayer, "Score: {0}".format(eventPlayer.score), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/modify-opy-project/negative/keeps-old-target/mode.opy b/benchmarks/agent/scenarios/modify-opy-project/negative/keeps-old-target/mode.opy new file mode 100644 index 00000000..3dc08a31 --- /dev/null +++ b/benchmarks/agent/scenarios/modify-opy-project/negative/keeps-old-target/mode.opy @@ -0,0 +1,35 @@ +settings { + "main": {"description": "Streak race"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +#!define TARGET_SCORE 5 + +globalvar matchOver +playervar score +playervar streak + +rule "reset streak on spawn": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + eventPlayer.streak = 0 + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + attacker.streak += 1 + +rule "streak breaker": + @Event playerDied + eventPlayer.streak = 0 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= TARGET_SCORE + @Condition matchOver == false + matchOver = true + declarePlayerVictory(eventPlayer) + +rule "score display": + @Event eachPlayer + hudHeader(eventPlayer, "Score: {0}".format(eventPlayer.score), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/modify-opy-project/prompt.md b/benchmarks/agent/scenarios/modify-opy-project/prompt.md new file mode 100644 index 00000000..d7e7d319 --- /dev/null +++ b/benchmarks/agent/scenarios/modify-opy-project/prompt.md @@ -0,0 +1,6 @@ +`mode.opy` is an Overwatch Workshop game mode written in OverPy. Make these changes and nothing else: + +- The winning score becomes 7 instead of 5. +- Add a HUD visible to everyone that shows the winning score, e.g. "First to 7". + +Leave the existing rules' behavior otherwise untouched, and make sure the project stays valid. diff --git a/benchmarks/agent/scenarios/modify-opy-project/reference/mode.opy b/benchmarks/agent/scenarios/modify-opy-project/reference/mode.opy new file mode 100644 index 00000000..95b89a25 --- /dev/null +++ b/benchmarks/agent/scenarios/modify-opy-project/reference/mode.opy @@ -0,0 +1,39 @@ +settings { + "main": {"description": "Streak race"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +#!define TARGET_SCORE 7 + +globalvar matchOver +playervar score +playervar streak + +rule "reset streak on spawn": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + eventPlayer.streak = 0 + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + attacker.streak += 1 + +rule "streak breaker": + @Event playerDied + eventPlayer.streak = 0 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= TARGET_SCORE + @Condition matchOver == false + matchOver = true + declarePlayerVictory(eventPlayer) + +rule "target display": + @Event eachPlayer + hudHeader(getAllPlayers(), "First to {0}".format(TARGET_SCORE), HudPosition.TOP, 0, Color.WHITE, HudReeval.VISIBILITY) + +rule "score display": + @Event eachPlayer + hudHeader(eventPlayer, "Score: {0}".format(eventPlayer.score), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/modify-opy-project/scenario.json b/benchmarks/agent/scenarios/modify-opy-project/scenario.json new file mode 100644 index 00000000..7588e43e --- /dev/null +++ b/benchmarks/agent/scenarios/modify-opy-project/scenario.json @@ -0,0 +1,88 @@ +{ + "id": "modify-opy-project", + "family": "modification", + "language": "opy", + "entry": "mode.opy", + "split": "train", + "writable": [ + "mode.opy" + ], + "runtimeOnly": [ + "The HUD text renders as intended in a live match." + ], + "checks": [ + { + "id": "valid-project", + "kind": "check", + "layer": "opy-rs" + }, + { + "id": "upstream-valid", + "kind": "oracle", + "layer": "agent" + }, + { + "id": "wright-compiles", + "kind": "wright-compile", + "layer": "opy-rs" + }, + { + "id": "target-is-7", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "score >= 7" + ] + }, + { + "id": "old-target-gone", + "kind": "compiled-absent", + "layer": "agent", + "all": [ + ">= 5" + ] + }, + { + "id": "hud-shows-target", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Create HUD Text\\([^;]*(Custom String\\(\"[^\"]*7|\\b7\\b)" + ] + }, + { + "id": "rules-preserved", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "rule \\(\"reset streak on spawn\"\\)", + "rule \\(\"score on elimination\"\\)", + "rule \\(\"streak breaker\"\\)", + "rule \\(\"declare winner\"\\)", + "rule \\(\"score display\"\\)" + ] + }, + { + "id": "winner-behavior-preserved", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Declare Player Victory\\(Event Player\\)" + ] + } + ], + "negatives": { + "keeps-old-target": { + "fails": [ + "hud-shows-target", + "old-target-gone", + "target-is-7" + ] + }, + "drops-a-rule": { + "fails": [ + "rules-preserved" + ] + } + } +} diff --git a/benchmarks/agent/scenarios/modify-opy-project/seed/mode.opy b/benchmarks/agent/scenarios/modify-opy-project/seed/mode.opy new file mode 100644 index 00000000..3dc08a31 --- /dev/null +++ b/benchmarks/agent/scenarios/modify-opy-project/seed/mode.opy @@ -0,0 +1,35 @@ +settings { + "main": {"description": "Streak race"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +#!define TARGET_SCORE 5 + +globalvar matchOver +playervar score +playervar streak + +rule "reset streak on spawn": + @Event eachPlayer + @Condition eventPlayer.hasSpawned() + eventPlayer.streak = 0 + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + attacker.streak += 1 + +rule "streak breaker": + @Event playerDied + eventPlayer.streak = 0 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= TARGET_SCORE + @Condition matchOver == false + matchOver = true + declarePlayerVictory(eventPlayer) + +rule "score display": + @Event eachPlayer + hudHeader(eventPlayer, "Score: {0}".format(eventPlayer.score), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/deletes-the-counter/answer.json b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/deletes-the-counter/answer.json new file mode 100644 index 00000000..791c0cd7 --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/deletes-the-counter/answer.json @@ -0,0 +1 @@ +{"rule": "tick counter"} diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/deletes-the-counter/mode.opy b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/deletes-the-counter/mode.opy new file mode 100644 index 00000000..2e030787 --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/deletes-the-counter/mode.opy @@ -0,0 +1,18 @@ +settings { + "main": {"description": "Countdown demo"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +globalvar countdown +globalvar ticks + +rule "start countdown": + countdown = 10 + +rule "tick counter": + pass + +rule "run countdown": + while countdown > 0: + wait(1) + countdown -= 1 diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/fixes-wrong-rule/answer.json b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/fixes-wrong-rule/answer.json new file mode 100644 index 00000000..e4a3faab --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/fixes-wrong-rule/answer.json @@ -0,0 +1 @@ +{"rule": "run countdown"} diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/fixes-wrong-rule/mode.opy b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/fixes-wrong-rule/mode.opy new file mode 100644 index 00000000..0ad4c1ec --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/negative/fixes-wrong-rule/mode.opy @@ -0,0 +1,19 @@ +settings { + "main": {"description": "Countdown demo"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +globalvar countdown +globalvar ticks + +rule "start countdown": + countdown = 10 + +rule "tick counter": + while true: + ticks += 1 + +rule "run countdown": + while countdown > 0: + wait(1) + countdown -= 1 diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/prompt.md b/benchmarks/agent/scenarios/repair-opy-runaway-loop/prompt.md new file mode 100644 index 00000000..4781f371 --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/prompt.md @@ -0,0 +1,3 @@ +Players report that as soon as a match starts using `mode.opy` (an Overwatch Workshop game mode written in OverPy), the server freezes and the countdown never finishes. + +Find the root cause, fix it with the smallest change that keeps the intended behavior (a counter that counts up once per second while the match runs), and record the name of the offending rule as `{"rule": ""}` in `answer.json`. Make sure the project has no remaining problems. diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/reference/answer.json b/benchmarks/agent/scenarios/repair-opy-runaway-loop/reference/answer.json new file mode 100644 index 00000000..791c0cd7 --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/reference/answer.json @@ -0,0 +1 @@ +{"rule": "tick counter"} diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/reference/mode.opy b/benchmarks/agent/scenarios/repair-opy-runaway-loop/reference/mode.opy new file mode 100644 index 00000000..0179d0ac --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/reference/mode.opy @@ -0,0 +1,20 @@ +settings { + "main": {"description": "Countdown demo"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +globalvar countdown +globalvar ticks + +rule "start countdown": + countdown = 10 + +rule "tick counter": + while true: + ticks += 1 + wait(1) + +rule "run countdown": + while countdown > 0: + wait(1) + countdown -= 1 diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/scenario.json b/benchmarks/agent/scenarios/repair-opy-runaway-loop/scenario.json new file mode 100644 index 00000000..2b0e907b --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/scenario.json @@ -0,0 +1,72 @@ +{ + "id": "repair-opy-runaway-loop", + "family": "diagnosis", + "language": "opy", + "entry": "mode.opy", + "stabilityRisk": true, + "split": "test", + "writable": [ + "mode.opy", + "answer.json" + ], + "runtimeOnly": [ + "The server no longer freezes in a live match." + ], + "checks": [ + { + "id": "valid-project", + "kind": "check", + "layer": "opy-rs" + }, + { + "id": "upstream-valid", + "kind": "oracle", + "layer": "agent" + }, + { + "id": "wright-compiles", + "kind": "wright-compile", + "layer": "opy-rs" + }, + { + "id": "no-unbounded-loop", + "kind": "lint", + "layer": "wright", + "code": "while-without-wait" + }, + { + "id": "counter-still-counts", + "kind": "contains", + "layer": "agent", + "file": "mode.opy", + "text": "ticks += 1" + }, + { + "id": "countdown-rule-preserved", + "kind": "contains", + "layer": "agent", + "file": "mode.opy", + "text": "countdown -= 1" + }, + { + "id": "root-cause-named", + "kind": "answer", + "layer": "agent", + "key": "rule", + "expected": "tick counter" + } + ], + "negatives": { + "fixes-wrong-rule": { + "fails": [ + "no-unbounded-loop", + "root-cause-named" + ] + }, + "deletes-the-counter": { + "fails": [ + "counter-still-counts" + ] + } + } +} diff --git a/benchmarks/agent/scenarios/repair-opy-runaway-loop/seed/mode.opy b/benchmarks/agent/scenarios/repair-opy-runaway-loop/seed/mode.opy new file mode 100644 index 00000000..0ad4c1ec --- /dev/null +++ b/benchmarks/agent/scenarios/repair-opy-runaway-loop/seed/mode.opy @@ -0,0 +1,19 @@ +settings { + "main": {"description": "Countdown demo"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +globalvar countdown +globalvar ticks + +rule "start countdown": + countdown = 10 + +rule "tick counter": + while true: + ticks += 1 + +rule "run countdown": + while countdown > 0: + wait(1) + countdown -= 1 diff --git a/benchmarks/agent/scenarios/understand-opy-project/negative/text-search-answers/answer.json b/benchmarks/agent/scenarios/understand-opy-project/negative/text-search-answers/answer.json new file mode 100644 index 00000000..f2fc38c5 --- /dev/null +++ b/benchmarks/agent/scenarios/understand-opy-project/negative/text-search-answers/answer.json @@ -0,0 +1 @@ +{"targetScore": 5, "victoryRule": "declare winner", "scoreWriters": ["score on elimination"], "unusedVariables": []} diff --git a/benchmarks/agent/scenarios/understand-opy-project/prompt.md b/benchmarks/agent/scenarios/understand-opy-project/prompt.md new file mode 100644 index 00000000..41570a4f --- /dev/null +++ b/benchmarks/agent/scenarios/understand-opy-project/prompt.md @@ -0,0 +1,8 @@ +`mode.opy` is an Overwatch Workshop game mode written in OverPy, and it includes another file. Do not modify either file. + +Answer these questions about it by writing `answer.json` in this directory, with exactly these keys: + +- `targetScore`: the number of points needed to win, as a number. +- `victoryRule`: the name of the rule that declares the winner. +- `scoreWriters`: the names of all rules that change the player variable `score`, directly or through a subroutine they call, sorted alphabetically. +- `unusedVariables`: the names of all declared variables that no rule ever reads or writes, sorted alphabetically. diff --git a/benchmarks/agent/scenarios/understand-opy-project/reference/answer.json b/benchmarks/agent/scenarios/understand-opy-project/reference/answer.json new file mode 100644 index 00000000..8210de0f --- /dev/null +++ b/benchmarks/agent/scenarios/understand-opy-project/reference/answer.json @@ -0,0 +1,12 @@ +{ + "targetScore": 5, + "victoryRule": "declare winner", + "scoreWriters": [ + "score on elimination", + "streak bonus points" + ], + "unusedVariables": [ + "bonus", + "unusedBanner" + ] +} diff --git a/benchmarks/agent/scenarios/understand-opy-project/scenario.json b/benchmarks/agent/scenarios/understand-opy-project/scenario.json new file mode 100644 index 00000000..4d91b6c6 --- /dev/null +++ b/benchmarks/agent/scenarios/understand-opy-project/scenario.json @@ -0,0 +1,64 @@ +{ + "id": "understand-opy-project", + "family": "understanding", + "language": "opy", + "entry": "mode.opy", + "split": "train", + "writable": [ + "answer.json" + ], + "checks": [ + { + "id": "valid-project", + "kind": "check", + "layer": "opy-rs" + }, + { + "id": "upstream-valid", + "kind": "oracle", + "layer": "agent" + }, + { + "id": "target-score", + "kind": "answer", + "layer": "agent", + "key": "targetScore", + "expected": 5 + }, + { + "id": "victory-rule", + "kind": "answer", + "layer": "agent", + "key": "victoryRule", + "expected": "declare winner" + }, + { + "id": "score-writers", + "kind": "answer", + "layer": "agent", + "key": "scoreWriters", + "expected": [ + "score on elimination", + "streak bonus points" + ] + }, + { + "id": "unused-variables", + "kind": "answer", + "layer": "agent", + "key": "unusedVariables", + "expected": [ + "bonus", + "unusedBanner" + ] + } + ], + "negatives": { + "text-search-answers": { + "fails": [ + "score-writers", + "unused-variables" + ] + } + } +} diff --git a/benchmarks/agent/scenarios/understand-opy-project/seed/mode.opy b/benchmarks/agent/scenarios/understand-opy-project/seed/mode.opy new file mode 100644 index 00000000..dd1c0053 --- /dev/null +++ b/benchmarks/agent/scenarios/understand-opy-project/seed/mode.opy @@ -0,0 +1,37 @@ +#!define TARGET_SCORE 5 + +settings { + "main": {"description": "Score race"}, + "gamemodes": {"ffa": {"enabledMaps": ["workshopIsland"]}} +} + +globalvar unusedBanner +playervar score +playervar streak +playervar bonus + +# TODO: bonus points are handled by a subroutine for now. +#!include "scoring.opy" + +rule "score on elimination": + @Event playerEarnedElimination + attacker.score += 1 + attacker.streak += 1 + +rule "streak bonus points": + @Event eachPlayer + @Condition eventPlayer.streak >= 3 + awardBonus() + +rule "reset streak on death": + @Event playerDied + eventPlayer.streak = 0 + +rule "declare winner": + @Event eachPlayer + @Condition eventPlayer.score >= TARGET_SCORE + declarePlayerVictory(eventPlayer) + +rule "score display": + @Event eachPlayer + hudHeader(eventPlayer, "Score: {0}".format(eventPlayer.score), HudPosition.LEFT, 0, Color.WHITE, HudReeval.VISIBILITY_AND_STRING) diff --git a/benchmarks/agent/scenarios/understand-opy-project/seed/scoring.opy b/benchmarks/agent/scenarios/understand-opy-project/seed/scoring.opy new file mode 100644 index 00000000..cd1bdbdd --- /dev/null +++ b/benchmarks/agent/scenarios/understand-opy-project/seed/scoring.opy @@ -0,0 +1,6 @@ +# Scoring helpers shared by the rules in mode.opy. + +def awardBonus(): + @Name "awardBonus" + eventPlayer.score += 2 + eventPlayer.streak = 0 diff --git a/benchmarks/agent/scenarios/widow-headshots/negative/no-headshot-filter/mode.opy b/benchmarks/agent/scenarios/widow-headshots/negative/no-headshot-filter/mode.opy new file mode 100644 index 00000000..be75c688 --- /dev/null +++ b/benchmarks/agent/scenarios/widow-headshots/negative/no-headshot-filter/mode.opy @@ -0,0 +1,215 @@ +settings { + "main": { + "modeName": "Widowmaker Headshots", + "description": "Headshots only. 10 kills in a row earns Infra-Sight." + }, + "gamemodes": { + "ffa": { + "enabledMaps": ["necropolis", "chateauGuillard", "castillo"], + "scoreToWin": 200 + } + }, + "heroes": { + "allTeams": { + "enabledHeroes": ["widowmaker"], + "widowmaker": { + "passiveHealthRegen": false + } + } + } +} + +# Workshop settings +# HP choice: 0 = Default, 1 = 175, 2 = 200, 3 = 225 (this OverPy frontend rejects enum settings) +#!define hpChoice createWorkshopSetting(int[0:3], "Widowmaker", "Widowmaker HP (0=Default 1=175 2=200 3=225)", 0, 0) +#!define removeFalloff createWorkshopSetting(bool, "Widowmaker", "Remove damage falloff", true, 1) +#!define killsToWin createWorkshopSetting(int[1:200], "Match", "Kills to win", 50, 0) + +# Tuning +#!define GRAPPLE_COOLDOWN 2.5 +#!define INFRA_SIGHT_STREAK 10 +#!define INFRA_SIGHT_SECONDS 10 +#!define KILL_HEAL_FRACTION 0.25 +#!define UNSCOPED_HEADSHOT_DAMAGE 26 + +globalvar globalShots +globalvar globalHeadshots + +playervar kills +playervar streak +playervar shots +playervar headshots +playervar joinTime +playervar hpCap +playervar lastAmmo +playervar ignoreNextHit +playervar infraReady +playervar statsOn +playervar killfeedOn +playervar bigHud + +rule "Init globals": + globalShots = 0 + globalHeadshots = 0 + +rule "Init player": + @Event eachPlayer + eventPlayer.startForcingHero(Hero.WIDOWMAKER) + eventPlayer.kills = 0 + eventPlayer.streak = 0 + eventPlayer.shots = 0 + eventPlayer.headshots = 0 + eventPlayer.ignoreNextHit = false + eventPlayer.infraReady = false + eventPlayer.statsOn = true + eventPlayer.killfeedOn = true + eventPlayer.bigHud = false + eventPlayer.joinTime = getTotalTimeElapsed() + +rule "Spawn setup": + @Event eachPlayer + @Condition eventPlayer.isAlive() == true + eventPlayer.hpCap = 9999 + eventPlayer.setMaxHealth(100) + wait(0.1) + if hpChoice == 1: + eventPlayer.setMaxHealth(175 / eventPlayer.getMaxHealth() * 100) + elif hpChoice == 2: + eventPlayer.setMaxHealth(200 / eventPlayer.getMaxHealth() * 100) + elif hpChoice == 3: + eventPlayer.setMaxHealth(225 / eventPlayer.getMaxHealth() * 100) + wait(0.1) + heal(eventPlayer, null, 9999) + wait(0.1) + eventPlayer.hpCap = eventPlayer.getHealth() + eventPlayer.lastAmmo = eventPlayer.getAmmo(0) + +rule "Grappling Hook cooldown": + @Event eachPlayer + @Condition eventPlayer.getAbilityCooldown(Button.ABILITY_1) > GRAPPLE_COOLDOWN + eventPlayer.setAbilityCooldown(Button.ABILITY_1, GRAPPLE_COOLDOWN) + +# Only headshots do damage: a body hit is refunded the moment it lands. +rule "Refund non-headshot damage": + @Event playerTookDamage + if not eventPlayer.isAlive(): + return + if eventPlayer.ignoreNextHit: + eventPlayer.ignoreNextHit = false + eventPlayer.hpCap = eventPlayer.getHealth() + return + if true: + eventPlayer.hpCap = eventPlayer.getHealth() + return + eventPlayer.hpCap = eventPlayer.getHealth() + eventDamage + heal(eventPlayer, null, eventDamage) + +# No healthpacks: health may never rise above what damage and kill-heals allow. +rule "Block healthpacks": + @Event eachPlayer + @Condition eventPlayer.isAlive() == true + @Condition eventPlayer.getHealth() > eventPlayer.hpCap + 1 + eventPlayer.ignoreNextHit = true + damage(eventPlayer, null, eventPlayer.getHealth() - eventPlayer.hpCap) + +rule "Remove damage falloff": + @Event playerDealtDamage + @Condition removeFalloff == true + @Condition eventPlayer.isHoldingButton(Button.SECONDARY_FIRE) == false + @Condition eventDamage < UNSCOPED_HEADSHOT_DAMAGE + if not victim.isAlive(): + return + victim.ignoreNextHit = true + damage(victim, eventPlayer, UNSCOPED_HEADSHOT_DAMAGE - eventDamage) + +rule "Count headshot hits": + @Event playerDealtDamage + eventPlayer.headshots += 1 + globalHeadshots += 1 + +rule "Count shots (ammo spent)": + @Event eachPlayer + @Condition eventPlayer.getAmmo(0) < eventPlayer.lastAmmo + eventPlayer.shots += eventPlayer.lastAmmo - eventPlayer.getAmmo(0) + globalShots += eventPlayer.lastAmmo - eventPlayer.getAmmo(0) + eventPlayer.lastAmmo = eventPlayer.getAmmo(0) + +rule "Track ammo refill": + @Event eachPlayer + @Condition eventPlayer.getAmmo(0) > eventPlayer.lastAmmo + eventPlayer.lastAmmo = eventPlayer.getAmmo(0) + +rule "Elimination": + @Event playerEarnedElimination + if victim == eventPlayer: + return + eventPlayer.kills += 1 + eventPlayer.streak += 1 + eventPlayer.hpCap = min(eventPlayer.getMaxHealth(), eventPlayer.hpCap + victim.getMaxHealth() * KILL_HEAL_FRACTION) + heal(eventPlayer, null, victim.getMaxHealth() * KILL_HEAL_FRACTION) + if eventPlayer.streak == INFRA_SIGHT_STREAK: + eventPlayer.infraReady = true + eventPlayer.setUltCharge(100) + smallMessage(eventPlayer, "Infra-Sight ready!") + if eventPlayer.kills >= killsToWin: + declarePlayerVictory(eventPlayer) + +rule "Death resets streak": + @Event playerDied + eventPlayer.streak = 0 + eventPlayer.infraReady = false + eventPlayer.setUltCharge(0) + +# Ultimate charge only ever comes from the killstreak. +rule "No natural ultimate charge": + @Event eachPlayer + @Condition eventPlayer.getUltCharge() > 0 + @Condition eventPlayer.infraReady == false + eventPlayer.setUltCharge(0) + +rule "Infra-Sight duration": + @Event eachPlayer + @Condition eventPlayer.isUsingUltimate() == true + wait(INFRA_SIGHT_SECONDS, Wait.ABORT_WHEN_FALSE) + eventPlayer.infraReady = false + eventPlayer.setUltCharge(0) + eventPlayer.setUltEnabled(false) + wait(0.25) + eventPlayer.setUltEnabled(true) + +# Interact toggles stats, Crouch+Interact toggles killfeed, Melee+Interact toggles big/small HUD. +rule "HUD toggles": + @Event eachPlayer + @Condition eventPlayer.isHoldingButton(Button.INTERACT) == true + if eventPlayer.isHoldingButton(Button.CROUCH): + eventPlayer.killfeedOn = not eventPlayer.killfeedOn + if eventPlayer.killfeedOn: + eventPlayer.enableKillFeed() + else: + eventPlayer.disableKillFeed() + elif eventPlayer.isHoldingButton(Button.MELEE): + eventPlayer.bigHud = not eventPlayer.bigHud + else: + eventPlayer.statsOn = not eventPlayer.statsOn + waitUntil(not eventPlayer.isHoldingButton(Button.INTERACT), 5) + +# Personal HUD (small) +rule "HUD: personal stats": + @Event eachPlayer + hudHeader(eventPlayer, "Kills {0}/{1} | Streak {2}".format(eventPlayer.kills, killsToWin, eventPlayer.streak) if eventPlayer.statsOn else "", HudPosition.LEFT, 1, Color.WHITE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Time {0}s | {1} kills/min".format(round(getTotalTimeElapsed() - eventPlayer.joinTime), round(eventPlayer.kills / max(1, (getTotalTimeElapsed() - eventPlayer.joinTime) / 60) * 10) / 10) if eventPlayer.statsOn else "", HudPosition.LEFT, 2, Color.SKY_BLUE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Pace: {0}s to {1} kills".format(round(max(0, killsToWin - eventPlayer.kills) / max(0.01, eventPlayer.kills / max(1, (getTotalTimeElapsed() - eventPlayer.joinTime) / 60)) * 60), killsToWin) if eventPlayer.statsOn and eventPlayer.kills > 0 else "", HudPosition.LEFT, 3, Color.SKY_BLUE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Your accuracy {0}%".format(round(eventPlayer.headshots / max(1, eventPlayer.shots) * 100)) if eventPlayer.statsOn else "", HudPosition.LEFT, 4, Color.SKY_BLUE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + +# Big HUD adds global accuracy and killstreak leaders +rule "HUD: global stats": + @Event eachPlayer + hudHeader(eventPlayer, "Global accuracy {0}%".format(round(globalHeadshots / max(1, globalShots) * 100)) if eventPlayer.statsOn and eventPlayer.bigHud else "", HudPosition.LEFT, 5, Color.WHITE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Streak leaders" if eventPlayer.statsOn and eventPlayer.bigHud else "", HudPosition.LEFT, 6, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "1. {0}: {1}".format(sorted(getAllPlayers(), lambda p: -p.streak)[0], sorted(getAllPlayers(), lambda p: -p.streak)[0].streak) if eventPlayer.statsOn and eventPlayer.bigHud and len(getAllPlayers()) >= 1 else "", HudPosition.LEFT, 7, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "2. {0}: {1}".format(sorted(getAllPlayers(), lambda p: -p.streak)[1], sorted(getAllPlayers(), lambda p: -p.streak)[1].streak) if eventPlayer.statsOn and eventPlayer.bigHud and len(getAllPlayers()) >= 2 else "", HudPosition.LEFT, 8, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "3. {0}: {1}".format(sorted(getAllPlayers(), lambda p: -p.streak)[2], sorted(getAllPlayers(), lambda p: -p.streak)[2].streak) if eventPlayer.statsOn and eventPlayer.bigHud and len(getAllPlayers()) >= 3 else "", HudPosition.LEFT, 9, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + +rule "HUD: controls hint": + @Event eachPlayer + hudSubtext(eventPlayer, "Interact: stats | Crouch+Interact: killfeed | Melee+Interact: big HUD", HudPosition.RIGHT, 1, Color.GRAY, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) diff --git a/benchmarks/agent/scenarios/widow-headshots/prompt.md b/benchmarks/agent/scenarios/widow-headshots/prompt.md new file mode 100644 index 00000000..f0b177cb --- /dev/null +++ b/benchmarks/agent/scenarios/widow-headshots/prompt.md @@ -0,0 +1,24 @@ +Create a Widowmaker headshots-only game mode for the Overwatch Workshop. Write it in OverPy and save it as `mode.opy` in the current directory (a single file). + +## Core game mechanics +- Available maps: Necropolis, Château Guillard, Castillo +- 2.5s Grappling Hook cooldown +- A 10 player killstreak grants Infra-Sight +- 10s Infra-Sight duration +- No damage falloff +- No passive health regen or healthpacks +- Heal 25% of the victim's health on kill +- Only headshots do damage + +## HUD +- Personal time counter and kills-per-minute pace tracker +- Small HUD: personal stats +- Big HUD: personal stats plus global accuracy and killstreak leaders +- Toggle the stats display and the killfeed separately + +## Workshop settings +- Widowmaker HP (default / 175 / 200 / 225) +- Remove damage falloff (enabled by default) +- Kills to win (50 by default, adjustable 1-200) + +When you are done, say briefly what you verified and what you could not verify. diff --git a/benchmarks/agent/scenarios/widow-headshots/reference/mode.opy b/benchmarks/agent/scenarios/widow-headshots/reference/mode.opy new file mode 100644 index 00000000..8a8da7ac --- /dev/null +++ b/benchmarks/agent/scenarios/widow-headshots/reference/mode.opy @@ -0,0 +1,217 @@ +settings { + "main": { + "modeName": "Widowmaker Headshots", + "description": "Headshots only. 10 kills in a row earns Infra-Sight." + }, + "gamemodes": { + "ffa": { + "enabledMaps": ["necropolis", "chateauGuillard", "castillo"], + "scoreToWin": 200 + } + }, + "heroes": { + "allTeams": { + "enabledHeroes": ["widowmaker"], + "widowmaker": { + "passiveHealthRegen": false + } + } + } +} + +# Workshop settings +# HP choice: 0 = Default, 1 = 175, 2 = 200, 3 = 225 (this OverPy frontend rejects enum settings) +#!define hpChoice createWorkshopSetting(int[0:3], "Widowmaker", "Widowmaker HP (0=Default 1=175 2=200 3=225)", 0, 0) +#!define removeFalloff createWorkshopSetting(bool, "Widowmaker", "Remove damage falloff", true, 1) +#!define killsToWin createWorkshopSetting(int[1:200], "Match", "Kills to win", 50, 0) + +# Tuning +#!define GRAPPLE_COOLDOWN 2.5 +#!define INFRA_SIGHT_STREAK 10 +#!define INFRA_SIGHT_SECONDS 10 +#!define KILL_HEAL_FRACTION 0.25 +#!define UNSCOPED_HEADSHOT_DAMAGE 26 + +globalvar globalShots +globalvar globalHeadshots + +playervar kills +playervar streak +playervar shots +playervar headshots +playervar joinTime +playervar hpCap +playervar lastAmmo +playervar ignoreNextHit +playervar infraReady +playervar statsOn +playervar killfeedOn +playervar bigHud + +rule "Init globals": + globalShots = 0 + globalHeadshots = 0 + +rule "Init player": + @Event eachPlayer + eventPlayer.startForcingHero(Hero.WIDOWMAKER) + eventPlayer.kills = 0 + eventPlayer.streak = 0 + eventPlayer.shots = 0 + eventPlayer.headshots = 0 + eventPlayer.ignoreNextHit = false + eventPlayer.infraReady = false + eventPlayer.statsOn = true + eventPlayer.killfeedOn = true + eventPlayer.bigHud = false + eventPlayer.joinTime = getTotalTimeElapsed() + +rule "Spawn setup": + @Event eachPlayer + @Condition eventPlayer.isAlive() == true + eventPlayer.hpCap = 9999 + eventPlayer.setMaxHealth(100) + wait(0.1) + if hpChoice == 1: + eventPlayer.setMaxHealth(175 / eventPlayer.getMaxHealth() * 100) + elif hpChoice == 2: + eventPlayer.setMaxHealth(200 / eventPlayer.getMaxHealth() * 100) + elif hpChoice == 3: + eventPlayer.setMaxHealth(225 / eventPlayer.getMaxHealth() * 100) + wait(0.1) + heal(eventPlayer, null, 9999) + wait(0.1) + eventPlayer.hpCap = eventPlayer.getHealth() + eventPlayer.lastAmmo = eventPlayer.getAmmo(0) + +rule "Grappling Hook cooldown": + @Event eachPlayer + @Condition eventPlayer.getAbilityCooldown(Button.ABILITY_1) > GRAPPLE_COOLDOWN + eventPlayer.setAbilityCooldown(Button.ABILITY_1, GRAPPLE_COOLDOWN) + +# Only headshots do damage: a body hit is refunded the moment it lands. +rule "Refund non-headshot damage": + @Event playerTookDamage + if not eventPlayer.isAlive(): + return + if eventPlayer.ignoreNextHit: + eventPlayer.ignoreNextHit = false + eventPlayer.hpCap = eventPlayer.getHealth() + return + if eventWasCriticalHit: + eventPlayer.hpCap = eventPlayer.getHealth() + return + eventPlayer.hpCap = eventPlayer.getHealth() + eventDamage + heal(eventPlayer, null, eventDamage) + +# No healthpacks: health may never rise above what damage and kill-heals allow. +rule "Block healthpacks": + @Event eachPlayer + @Condition eventPlayer.isAlive() == true + @Condition eventPlayer.getHealth() > eventPlayer.hpCap + 1 + eventPlayer.ignoreNextHit = true + damage(eventPlayer, null, eventPlayer.getHealth() - eventPlayer.hpCap) + +rule "Remove damage falloff": + @Event playerDealtDamage + @Condition removeFalloff == true + @Condition eventWasCriticalHit == true + @Condition eventPlayer.isHoldingButton(Button.SECONDARY_FIRE) == false + @Condition eventDamage < UNSCOPED_HEADSHOT_DAMAGE + if not victim.isAlive(): + return + victim.ignoreNextHit = true + damage(victim, eventPlayer, UNSCOPED_HEADSHOT_DAMAGE - eventDamage) + +rule "Count headshot hits": + @Event playerDealtDamage + @Condition eventWasCriticalHit == true + eventPlayer.headshots += 1 + globalHeadshots += 1 + +rule "Count shots (ammo spent)": + @Event eachPlayer + @Condition eventPlayer.getAmmo(0) < eventPlayer.lastAmmo + eventPlayer.shots += eventPlayer.lastAmmo - eventPlayer.getAmmo(0) + globalShots += eventPlayer.lastAmmo - eventPlayer.getAmmo(0) + eventPlayer.lastAmmo = eventPlayer.getAmmo(0) + +rule "Track ammo refill": + @Event eachPlayer + @Condition eventPlayer.getAmmo(0) > eventPlayer.lastAmmo + eventPlayer.lastAmmo = eventPlayer.getAmmo(0) + +rule "Elimination": + @Event playerEarnedElimination + if victim == eventPlayer: + return + eventPlayer.kills += 1 + eventPlayer.streak += 1 + eventPlayer.hpCap = min(eventPlayer.getMaxHealth(), eventPlayer.hpCap + victim.getMaxHealth() * KILL_HEAL_FRACTION) + heal(eventPlayer, null, victim.getMaxHealth() * KILL_HEAL_FRACTION) + if eventPlayer.streak == INFRA_SIGHT_STREAK: + eventPlayer.infraReady = true + eventPlayer.setUltCharge(100) + smallMessage(eventPlayer, "Infra-Sight ready!") + if eventPlayer.kills >= killsToWin: + declarePlayerVictory(eventPlayer) + +rule "Death resets streak": + @Event playerDied + eventPlayer.streak = 0 + eventPlayer.infraReady = false + eventPlayer.setUltCharge(0) + +# Ultimate charge only ever comes from the killstreak. +rule "No natural ultimate charge": + @Event eachPlayer + @Condition eventPlayer.getUltCharge() > 0 + @Condition eventPlayer.infraReady == false + eventPlayer.setUltCharge(0) + +rule "Infra-Sight duration": + @Event eachPlayer + @Condition eventPlayer.isUsingUltimate() == true + wait(INFRA_SIGHT_SECONDS, Wait.ABORT_WHEN_FALSE) + eventPlayer.infraReady = false + eventPlayer.setUltCharge(0) + eventPlayer.setUltEnabled(false) + wait(0.25) + eventPlayer.setUltEnabled(true) + +# Interact toggles stats, Crouch+Interact toggles killfeed, Melee+Interact toggles big/small HUD. +rule "HUD toggles": + @Event eachPlayer + @Condition eventPlayer.isHoldingButton(Button.INTERACT) == true + if eventPlayer.isHoldingButton(Button.CROUCH): + eventPlayer.killfeedOn = not eventPlayer.killfeedOn + if eventPlayer.killfeedOn: + eventPlayer.enableKillFeed() + else: + eventPlayer.disableKillFeed() + elif eventPlayer.isHoldingButton(Button.MELEE): + eventPlayer.bigHud = not eventPlayer.bigHud + else: + eventPlayer.statsOn = not eventPlayer.statsOn + waitUntil(not eventPlayer.isHoldingButton(Button.INTERACT), 5) + +# Personal HUD (small) +rule "HUD: personal stats": + @Event eachPlayer + hudHeader(eventPlayer, "Kills {0}/{1} | Streak {2}".format(eventPlayer.kills, killsToWin, eventPlayer.streak) if eventPlayer.statsOn else "", HudPosition.LEFT, 1, Color.WHITE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Time {0}s | {1} kills/min".format(round(getTotalTimeElapsed() - eventPlayer.joinTime), round(eventPlayer.kills / max(1, (getTotalTimeElapsed() - eventPlayer.joinTime) / 60) * 10) / 10) if eventPlayer.statsOn else "", HudPosition.LEFT, 2, Color.SKY_BLUE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Pace: {0}s to {1} kills".format(round(max(0, killsToWin - eventPlayer.kills) / max(0.01, eventPlayer.kills / max(1, (getTotalTimeElapsed() - eventPlayer.joinTime) / 60)) * 60), killsToWin) if eventPlayer.statsOn and eventPlayer.kills > 0 else "", HudPosition.LEFT, 3, Color.SKY_BLUE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Your accuracy {0}%".format(round(eventPlayer.headshots / max(1, eventPlayer.shots) * 100)) if eventPlayer.statsOn else "", HudPosition.LEFT, 4, Color.SKY_BLUE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + +# Big HUD adds global accuracy and killstreak leaders +rule "HUD: global stats": + @Event eachPlayer + hudHeader(eventPlayer, "Global accuracy {0}%".format(round(globalHeadshots / max(1, globalShots) * 100)) if eventPlayer.statsOn and eventPlayer.bigHud else "", HudPosition.LEFT, 5, Color.WHITE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "Streak leaders" if eventPlayer.statsOn and eventPlayer.bigHud else "", HudPosition.LEFT, 6, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "1. {0}: {1}".format(sorted(getAllPlayers(), lambda p: -p.streak)[0], sorted(getAllPlayers(), lambda p: -p.streak)[0].streak) if eventPlayer.statsOn and eventPlayer.bigHud and len(getAllPlayers()) >= 1 else "", HudPosition.LEFT, 7, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "2. {0}: {1}".format(sorted(getAllPlayers(), lambda p: -p.streak)[1], sorted(getAllPlayers(), lambda p: -p.streak)[1].streak) if eventPlayer.statsOn and eventPlayer.bigHud and len(getAllPlayers()) >= 2 else "", HudPosition.LEFT, 8, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + hudSubtext(eventPlayer, "3. {0}: {1}".format(sorted(getAllPlayers(), lambda p: -p.streak)[2], sorted(getAllPlayers(), lambda p: -p.streak)[2].streak) if eventPlayer.statsOn and eventPlayer.bigHud and len(getAllPlayers()) >= 3 else "", HudPosition.LEFT, 9, Color.ORANGE, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) + +rule "HUD: controls hint": + @Event eachPlayer + hudSubtext(eventPlayer, "Interact: stats | Crouch+Interact: killfeed | Melee+Interact: big HUD", HudPosition.RIGHT, 1, Color.GRAY, HudReeval.VISIBILITY_AND_STRING, SpecVisibility.DEFAULT) diff --git a/benchmarks/agent/scenarios/widow-headshots/scenario.json b/benchmarks/agent/scenarios/widow-headshots/scenario.json new file mode 100644 index 00000000..c72a5964 --- /dev/null +++ b/benchmarks/agent/scenarios/widow-headshots/scenario.json @@ -0,0 +1,140 @@ +{ + "id": "widow-headshots", + "family": "greenfield", + "language": "opy", + "source": "gamemode idea from `Zezombye/overpy#439`", + "entry": "mode.opy", + "stabilityRisk": true, + "split": "test", + "writable": [ + "mode.opy" + ], + "runtimeOnly": [ + "Damage without falloff, headshot-only damage, and Infra-Sight duration behave as described in a live match.", + "HUD layout and the separate stats and killfeed toggles are usable on a full server." + ], + "referenceNote": "Adapted from an agent-generated solution that satisfies the checks; not verified in the game.", + "checks": [ + { + "id": "upstream-valid", + "kind": "oracle", + "layer": "agent" + }, + { + "id": "wright-compiles", + "kind": "wright-compile", + "layer": "opy-rs" + }, + { + "id": "wright-check", + "kind": "check", + "layer": "opy-rs" + }, + { + "id": "widowmaker", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Widowmaker" + ] + }, + { + "id": "maps", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Necropolis", + "Guillard", + "Castillo" + ] + }, + { + "id": "headshot-only", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Event Was Critical Hit" + ] + }, + { + "id": "custom-damage", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Damage\\(" + ] + }, + { + "id": "grapple-cooldown-2.5", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Set Ability Cooldown", + "2\\.5" + ] + }, + { + "id": "ultimate-grant", + "kind": "compiled-contains", + "layer": "agent", + "any": [ + "Set Ultimate Charge", + "Ultimate Charge" + ] + }, + { + "id": "heal-on-kill", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Heal\\(" + ] + }, + { + "id": "hud", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Create HUD Text" + ] + }, + { + "id": "workshop-settings", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Workshop Setting (Integer|Combo|Toggle)" + ] + }, + { + "id": "hp-options", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "\\b175\\b", + "\\b225\\b" + ] + }, + { + "id": "win-condition", + "kind": "compiled-contains", + "layer": "agent", + "all": [ + "Declare (Player|Team|Round) Victory" + ] + }, + { + "id": "no-waitless-loop", + "kind": "lint", + "code": "while-without-wait", + "layer": "wright" + } + ], + "negatives": { + "no-headshot-filter": { + "fails": [ + "headshot-only" + ] + } + } +} diff --git a/benchmarks/agent/scenarios/widow-headshots/seed/.gitkeep b/benchmarks/agent/scenarios/widow-headshots/seed/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/agent-benchmark.md b/docs/agent-benchmark.md index d890d77c..532a512c 100644 --- a/docs/agent-benchmark.md +++ b/docs/agent-benchmark.md @@ -78,6 +78,7 @@ to form a passing solution), and optional `negative//` overlays. | `runtimeOnly` | Claims that only the Overwatch runtime can verify; reported as unverified, never as passed | | `stabilityRisk` | `true` when finishing safely needs `lint` or `analyze`, not only `check` (expectation E03) | | `split` | Optional `train` or `test`, for reports and guide tuning | +| `source`, `referenceNote` | Optional provenance of the requirement and of the reference solution; the reference only calibrates the checks and is never shown to agents | | `negatives` | `{name: {"fails": [check ids]}}`; the overlay must fail exactly those checks | | `checks` | Deterministic checks, each with `id`, `kind`, and `layer` |