Failure/foul-magics
1
0
Fork 0
forked from External/mage
Code Pull requests Releases Packages Activity Actions

AI, tests: added stability tests to make sure AI simulations can process errors and freezes (part of #13638, #13766);

Browse source
This commit is contained in:
Oleg Agafonov 2025-06-28 22:51:34 +04:00
parent 85c04bca59
commit c3a0c731d6
12 changed files with 298 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

15
Mage.Server.Plugins/Mage.Player.AI.MA/src/mage/player/ai/ComputerPlayer6.java
View file

@ -114,6 +114,13 @@ public class ComputerPlayer6 extends ComputerPlayer {
this.actionCache = player.actionCache;
}
/**
* Change simulation timeout - used for AI stability tests only
*/
public void setMaxThinkTimeSecs(int maxThinkTimeSecs) {
this.maxThinkTimeSecs = maxThinkTimeSecs;
}
@Override
public ComputerPlayer6 copy() {
return new ComputerPlayer6(this);
@ -431,6 +438,8 @@ public class ComputerPlayer6 extends ComputerPlayer {
* @return
*/
protected Integer addActionsTimed() {
// TODO: all actions added and calculated one by one,
// multithreading do not supported here
// run new game simulation in parallel thread
FutureTask<Integer> task = new FutureTask<>(() -> addActions(root, maxDepth, Integer.MIN_VALUE, Integer.MAX_VALUE));
threadPoolSimulations.execute(task);
@ -446,15 +455,15 @@ public class ComputerPlayer6 extends ComputerPlayer {
}
} catch (TimeoutException | InterruptedException e) {
// AI thinks too long
logger.info("ai simulating - timed out");
logger.warn("AI player thinks too long - " + getName() + " - " + root.game);
task.cancel(true);
} catch (ExecutionException e) {
// game error
logger.error("AI simulation catch game error: " + e, e);
logger.error("AI player catch game error in simulation - " + getName() + " - " + root.game + ": " + e, e);
task.cancel(true);
// real games: must catch and log
// unit tests: must raise again for fast fail
if (this.isTestsMode()) {
if (this.isTestMode() && this.isFastFailInTestMode()) {
throw new IllegalStateException("One of the simulated games raise the error: " + e, e);
}
} catch (Throwable e) {

3
Mage.Server.Plugins/Mage.Player.AI.MA/src/mage/player/ai/ComputerPlayer7.java
View file

@ -142,7 +142,8 @@ public class ComputerPlayer7 extends ComputerPlayer6 {
}
}
} else {
logger.info('[' + game.getPlayer(playerId).getName() + "][pre] Action: skip");
// nothing to choose or freeze/infinite game
logger.info("AI player can't find next action: " + getName());
}
} else {
logger.debug("Next Action exists!");

6
Mage.Server.Plugins/Mage.Player.AI/src/main/java/mage/player/ai/ComputerPlayer.java
View file

@ -52,7 +52,7 @@ public class ComputerPlayer extends PlayerImpl {
protected static final int PASSIVITY_PENALTY = 5; // Penalty value for doing nothing if some actions are available
// debug only: set TRUE to debug simulation's code/games (on false sim thread will be stopped after few secs by timeout)
protected static final boolean COMPUTER_DISABLE_TIMEOUT_IN_GAME_SIMULATIONS = true; // DebugUtil.AI_ENABLE_DEBUG_MODE;
public static final boolean COMPUTER_DISABLE_TIMEOUT_IN_GAME_SIMULATIONS = false; // DebugUtil.AI_ENABLE_DEBUG_MODE;
// AI agents uses game simulation thread for all calcs and it's high CPU consumption
// More AI threads - more parallel AI games can be calculate
@ -64,7 +64,7 @@ public class ComputerPlayer extends PlayerImpl {
// * use yours CPU cores for best performance
// TODO: add server config to control max AI threads (with CPU cores by default)
// TODO: rework AI implementation to use multiple sims calculation instead one by one
final static int COMPUTER_MAX_THREADS_FOR_SIMULATIONS = 1;//DebugUtil.AI_ENABLE_DEBUG_MODE ? 1 : 5;
final static int COMPUTER_MAX_THREADS_FOR_SIMULATIONS = 5;//DebugUtil.AI_ENABLE_DEBUG_MODE ? 1 : 5;
// remember picked cards for better draft choices
@ -104,7 +104,7 @@ public class ComputerPlayer extends PlayerImpl {
@Override
public boolean chooseMulligan(Game game) {
if (hand.size() < 6
|| isTestsMode() // ignore mulligan in tests
|| isTestMode() // ignore mulligan in tests
|| game.getClass().getName().contains("Momir") // ignore mulligan in Momir games
) {
return false;

2
Mage.Server.Plugins/Mage.Player.AIMCTS/src/mage/player/ai/ComputerPlayerMCTS.java
View file

@ -196,7 +196,7 @@ public class ComputerPlayerMCTS extends ComputerPlayer {
} catch (ExecutionException e) {
// real games: must catch and log
// unit tests: must raise again for fast fail
if (this.isTestsMode()) {
if (this.isTestMode() && this.isFastFailInTestMode()) {
throw new IllegalStateException("One of the simulated games raise the error: " + e, e);
}
}

237
Mage.Tests/src/test/java/org/mage/test/AI/basic/SimulationStabilityAITest.java Normal file
View file

@ -0,0 +1,237 @@
package org.mage.test.AI.basic;
import mage.abilities.Ability;
import mage.abilities.common.LimitedTimesPerTurnActivatedAbility;
import mage.abilities.costs.mana.ManaCostsImpl;
import mage.abilities.effects.Effect;
import mage.abilities.effects.OneShotEffect;
import mage.abilities.effects.common.GainLifeEffect;
import mage.constants.Outcome;
import mage.constants.PhaseStep;
import mage.constants.Zone;
import mage.game.Game;
import mage.player.ai.ComputerPlayer;
import mage.player.ai.ComputerPlayer7;
import mage.util.ThreadUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.mage.test.serverside.base.CardTestPlayerBaseWithAIHelps;
/**
* Tests for AI simulation stability (AI must process simulations with freezes or errors)
*
* @author JayDi85
*/
public class SimulationStabilityAITest extends CardTestPlayerBaseWithAIHelps {
@Before
public void prepare() {
// comment it to enable AI code debug
Assert.assertFalse("AI stability tests must be run under release config", ComputerPlayer.COMPUTER_DISABLE_TIMEOUT_IN_GAME_SIMULATIONS);
}
@Test
public void test_GameFreeze_OnlyGoodAbilities() {
removeAllCardsFromLibrary(playerA);
// possible combinations: from 1 to 3 abilities - all fine
addFreezeAbility("good 1", false);
addFreezeAbility("good 2", false);
addFreezeAbility("good 3", false);
// AI must activate all +3 life
aiPlayStep(1, PhaseStep.PRECOMBAT_MAIN, playerA);
setStrictChooseMode(true);
setStopAt(1, PhaseStep.END_TURN);
execute();
assertLife(playerA, 20 + 3);
}
@Test
public void test_GameFreeze_OnlyFreezeAbilities() {
removeAllCardsFromLibrary(playerA);
// possible combinations: from 1 to 3 bad abilities - all bad
addFreezeAbility("freeze 1", true);
addFreezeAbility("freeze 2", true);
addFreezeAbility("freeze 3", true);
// AI can't finish any simulation and do not choose to activate in real game
aiPlayStep(1, PhaseStep.PRECOMBAT_MAIN, playerA);
setStrictChooseMode(true);
setStopAt(1, PhaseStep.END_TURN);
execute();
assertLife(playerA, 20);
}
@Test
@Ignore
// TODO: AI actions simulation do not support multithreading, so whole next action search
// will fail on any problem (enable after new simulation implement)
public void test_GameFreeze_GoodAndFreezeAbilities() {
removeAllCardsFromLibrary(playerA);
// possible combinations: some good chains, some bad chains
addFreezeAbility("good 1", false);
addFreezeAbility("good 2", false);
addFreezeAbility("good 3", false);
addFreezeAbility("freeze 1", true);
// AI must see and filter bad combinations with freeze ability in the chain
// so only 1 + 2 + 3 will give best score and will be chosen for real game
aiPlayStep(1, PhaseStep.PRECOMBAT_MAIN, playerA);
setStrictChooseMode(true);
setStopAt(1, PhaseStep.END_TURN);
execute();
assertLife(playerA, 20 + 3);
}
@Test
public void test_GameError_OnlyGoodAbilities() {
removeAllCardsFromLibrary(playerA);
// possible combinations: from 1 to 3 abilities - all fine
addErrorAbility("good 1", false);
addErrorAbility("good 2", false);
addErrorAbility("good 3", false);
// AI must activate all +3 life
aiPlayStep(1, PhaseStep.PRECOMBAT_MAIN, playerA);
setStrictChooseMode(true);
setStopAt(1, PhaseStep.END_TURN);
execute();
assertLife(playerA, 20 + 3);
}
@Test
public void test_GameError_OnlyErrorAbilities() {
removeAllCardsFromLibrary(playerA);
// it's ok to have error logs in output
// possible combinations: from 1 to 3 bad abilities - all bad
addErrorAbility("error 1", true);
addErrorAbility("error 2", true);
addErrorAbility("error 3", true);
// AI can't finish any simulation and do not choose to activate in real game
aiPlayStep(1, PhaseStep.PRECOMBAT_MAIN, playerA);
setStrictChooseMode(true);
setStopAt(1, PhaseStep.END_TURN);
execute();
assertLife(playerA, 20);
}
@Test
@Ignore
// TODO: AI actions simulation do not support multithreading, so whole next action search
// will fail on any problem (enable after new simulation implement)
public void test_GameError_GoodAndFreezeAbilities() {
removeAllCardsFromLibrary(playerA);
// it's ok to have error logs in output
// possible combinations: some good chains, some bad chains
addErrorAbility("good 1", false);
addErrorAbility("good 2", false);
addErrorAbility("good 3", false);
addErrorAbility("error 1", true);
// AI must see and filter bad combinations with error ability in the chain
// so only 1 + 2 + 3 will give best score and will be chosen for real game
aiPlayStep(1, PhaseStep.PRECOMBAT_MAIN, playerA);
setStrictChooseMode(true);
setStopAt(1, PhaseStep.END_TURN);
execute();
assertLife(playerA, 20 + 3);
}
private void addFreezeAbility(String name, boolean isFreeze) {
// change max think timeout to lower value, so test can work faster
ComputerPlayer7 aiPlayer = (ComputerPlayer7) playerA.getRealPlayer();
aiPlayer.setMaxThinkTimeSecs(1);
Effect effect;
if (isFreeze) {
effect = new GameFreezeEffect();
} else {
effect = new GainLifeEffect(1);
}
effect.setText(name);
addCustomCardWithAbility(name, playerA, new LimitedTimesPerTurnActivatedAbility(effect, new ManaCostsImpl<>("{G}")));
addCard(Zone.BATTLEFIELD, playerA, "Forest", 1);
}
private void addErrorAbility(String name, boolean isError) {
// change error processing, so test can continue simulations after catch error - like a real game
playerA.setFastFailInTestMode(false);
Effect effect;
if (isError) {
effect = new GameErrorEffect();
} else {
effect = new GainLifeEffect(1);
}
effect.setText(name);
addCustomCardWithAbility(name, playerA, new LimitedTimesPerTurnActivatedAbility(effect, new ManaCostsImpl<>("{G}")));
addCard(Zone.BATTLEFIELD, playerA, "Forest", 1);
}
}
class GameFreezeEffect extends OneShotEffect {
GameFreezeEffect() {
super(Outcome.Benefit);
}
private GameFreezeEffect(final GameFreezeEffect effect) {
super(effect);
}
public GameFreezeEffect copy() {
return new GameFreezeEffect(this);
}
public boolean apply(Game game, Ability source) {
// freeze simulation, AI must close sim thread by timeout
System.out.println("apply freeze effect on " + game); // for debug only, show logs from any sim thread
while (true) {
ThreadUtils.sleep(1000);
}
}
}
class GameErrorEffect extends OneShotEffect {
GameErrorEffect() {
super(Outcome.Benefit);
}
private GameErrorEffect(final GameErrorEffect effect) {
super(effect);
}
public GameErrorEffect copy() {
return new GameErrorEffect(this);
}
public boolean apply(Game game, Ability source) {
// error simulation, AI must close error thread, do not use rollback
System.out.println("apply error effect on " + game); // for debug only, show logs from any sim thread
throw new IllegalStateException("Test error");
}
}

23
Mage.Tests/src/test/java/org/mage/test/player/TestPlayer.java
View file

@ -775,7 +775,12 @@ public class TestPlayer implements Player {
AIRealGameControlUntil = endStep; // disable on end step
computerPlayer.priority(game);
actions.remove(action);
computerPlayer.resetPassed(); // remove AI's pass, so runtime/check commands can be executed in same priority
// remove AI's pass, so runtime/check commands can be executed in same priority
// aiPlayStep can cause double priority call, but it's better to have workable checkXXX commands
// (AI will do nothing on second priority call anyway)
if (!actions.isEmpty()) {
computerPlayer.resetPassed();
}
return true;
}
@ -3438,7 +3443,7 @@ public class TestPlayer implements Player {
@Override
public boolean isComputer() {
// all players in unit tests are computers, so it allows testing different logic (Human vs AI)
if (isTestsMode()) {
if (isTestMode()) {
// AIRealGameSimulation = true - full plyable AI
// AIRealGameSimulation = false - choose assisted AI (Human)
return AIRealGameSimulation;
@ -3874,8 +3879,8 @@ public class TestPlayer implements Player {
}
@Override
public boolean isTestsMode() {
return computerPlayer.isTestsMode();
public boolean isTestMode() {
return computerPlayer.isTestMode();
}
@Override
@ -3883,6 +3888,16 @@ public class TestPlayer implements Player {
computerPlayer.setTestMode(value);
}
@Override
public boolean isFastFailInTestMode() {
return computerPlayer.isFastFailInTestMode();
}
@Override
public void setFastFailInTestMode(boolean value) {
computerPlayer.setFastFailInTestMode(value);
}
@Override
public boolean isTopCardRevealed() {
return computerPlayer.isTopCardRevealed();

2
Mage/src/main/java/mage/abilities/AbilityImpl.java
View file

@ -377,7 +377,7 @@ public abstract class AbilityImpl implements Ability {
// unit tests only: it allows to add targets/choices by two ways:
// 1. From cast/activate command params (process it here)
// 2. From single addTarget/setChoice, it's a preferred method for tests (process it in normal choose dialogs like human player)
if (controller.isTestsMode()) {
if (controller.isTestMode()) {
if (!controller.addTargets(this, game)) {
return false;
}

2
Mage/src/main/java/mage/game/GameImpl.java
View file

@ -1782,7 +1782,7 @@ public abstract class GameImpl implements Game {
// count total errors
Player activePlayer = this.getPlayer(getActivePlayerId());
if (activePlayer != null && !activePlayer.isTestsMode()) {
if (activePlayer != null && !activePlayer.isTestMode() && !activePlayer.isFastFailInTestMode()) {
// real game - try to continue
priorityErrorsCount++;
continue;

3
Mage/src/main/java/mage/game/combat/Combat.java
View file

@ -695,7 +695,8 @@ public class Combat implements Serializable, Copyable<Combat> {
// real game: send warning
// test: fast fail
game.informPlayers(controller.getLogName() + ": WARNING - AI can't find good blocker combination and will skip it - report your battlefield to github - " + game.getCombat());
if (controller.isTestsMode()) {
if (controller.isTestMode() && controller.isFastFailInTestMode()) {
// fast fail in tests
// how-to fix: AI code must support failed abilities or use cases
throw new IllegalArgumentException("AI can't find good blocker combination");
}

10
Mage/src/main/java/mage/players/Player.java
View file

@ -79,7 +79,13 @@ public interface Player extends MageItem, Copyable<Player> {
*/
boolean isHuman();
boolean isTestsMode();
boolean isTestMode();
void setTestMode(boolean value);
boolean isFastFailInTestMode();
void setFastFailInTestMode(boolean value);
/**
* Current player is AI. Use it in card's code and all other places.
@ -398,8 +404,6 @@ public interface Player extends MageItem, Copyable<Player> {
*/
void setGameUnderYourControl(Game game, boolean value, boolean fullRestore);
void setTestMode(boolean value);
void setAllowBadMoves(boolean allowBadMoves);
/**

13
Mage/src/main/java/mage/players/PlayerImpl.java
View file

@ -147,6 +147,7 @@ public abstract class PlayerImpl implements Player, Serializable {
protected Set<UUID> inRange = new HashSet<>(); // players list in current range of influence (updates each turn due rules)
protected boolean isTestMode = false;
protected boolean isFastFailInTestMode = false;
protected boolean canGainLife = true;
protected boolean canLoseLife = true;
protected PayLifeCostLevel payLifeCostLevel = PayLifeCostLevel.allAbilities;
@ -4602,7 +4603,7 @@ public abstract class PlayerImpl implements Player, Serializable {
}
@Override
public boolean isTestsMode() {
public boolean isTestMode() {
return isTestMode;
}
@ -4611,6 +4612,16 @@ public abstract class PlayerImpl implements Player, Serializable {
this.isTestMode = value;
}
@Override
public boolean isFastFailInTestMode() {
return isFastFailInTestMode;
}
@Override
public void setFastFailInTestMode(boolean value) {
this.isFastFailInTestMode = value;
}
@Override
public boolean isTopCardRevealed() {
return topCardRevealed;

2
Mage/src/main/java/mage/util/FuzzyTestsUtil.java
View file

@ -58,7 +58,7 @@ public class FuzzyTestsUtil {
return;
}
Player samplePlayer = game.getPlayers().values().stream().findFirst().orElse(null);
if (samplePlayer == null || !samplePlayer.isTestsMode()) {
if (samplePlayer == null || !samplePlayer.isTestMode()) {
return;
}