server: fixed server app freeze on another instance already running, improved threads usage (related to #11285);

This commit is contained in:
Oleg Agafonov 2024-06-23 15:58:25 +04:00
parent f0c38cdb87
commit 7d675de876
21 changed files with 203 additions and 106 deletions

View file

@ -74,8 +74,6 @@ public class MainManagerFactory implements ManagerFactory {
threadExecutor().getServerHealthExecutor().scheduleAtFixedRate(() -> {
try {
//logger.info("---");
//logger.info("Server health check started");
this.tableManager().checkHealth();
this.chatManager().checkHealth();
this.userManager().checkHealth();
@ -83,8 +81,6 @@ public class MainManagerFactory implements ManagerFactory {
} catch (Exception ex) {
logger.fatal("Server health check: catch unknown error - " + ex, ex);
}
//logger.info("Server health check end");
//logger.info("---");
}, SERVER_HEALTH_CHECK_TIMEOUT_MINS, SERVER_HEALTH_CHECK_TIMEOUT_MINS, TimeUnit.MINUTES);
}

View file

@ -5,6 +5,8 @@ import mage.server.managers.UserManager;
import mage.server.record.UserStats;
import mage.server.record.UserStatsRepository;
import mage.server.util.ServerMessagesUtil;
import mage.util.ThreadUtils;
import mage.util.XMageThreadFactory;
import mage.view.UserView;
import org.apache.log4j.Logger;
@ -27,12 +29,16 @@ public class UserManagerImpl implements UserManager {
private static final int USER_CONNECTION_TIMEOUT_SESSION_EXPIRE_AFTER_SECS = 3 * 60; // session expire - remove from all tables and chats (can't reconnect after it)
private static final int USER_CONNECTION_TIMEOUT_REMOVE_FROM_SERVER_SECS = 8 * 60; // removes from users list
private static final int SERVER_USERS_LIST_UPDATE_SECS = 4; // server side updates (client use own timeouts to request users list)
private static final int SERVER_USERS_LIST_UPDATE_SECS = 10; // server side updates (client use own timeouts to request users list)
private static final Logger logger = Logger.getLogger(UserManagerImpl.class);
protected final ScheduledExecutorService expireExecutor = Executors.newSingleThreadScheduledExecutor();
protected final ScheduledExecutorService userListExecutor = Executors.newSingleThreadScheduledExecutor();
protected final ScheduledExecutorService CONNECTION_EXPIRED_EXECUTOR = Executors.newSingleThreadScheduledExecutor(
new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_CONNECTION_EXPIRED_CHECK)
);
protected final ScheduledExecutorService USERS_LIST_REFRESH_EXECUTOR = Executors.newSingleThreadScheduledExecutor(
new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_USERS_LIST_REFRESH)
);
private List<UserView> userInfoList = new ArrayList<>(); // all users list for main room/chat
private int maxUsersOnline = 0;
@ -50,8 +56,8 @@ public class UserManagerImpl implements UserManager {
public void init() {
USER_EXECUTOR = managerFactory.threadExecutor().getCallExecutor();
expireExecutor.scheduleAtFixedRate(this::checkExpired, USER_CONNECTION_TIMEOUTS_CHECK_SECS, USER_CONNECTION_TIMEOUTS_CHECK_SECS, TimeUnit.SECONDS);
userListExecutor.scheduleAtFixedRate(this::updateUserInfoList, SERVER_USERS_LIST_UPDATE_SECS, SERVER_USERS_LIST_UPDATE_SECS, TimeUnit.SECONDS);
CONNECTION_EXPIRED_EXECUTOR.scheduleAtFixedRate(this::checkExpired, USER_CONNECTION_TIMEOUTS_CHECK_SECS, USER_CONNECTION_TIMEOUTS_CHECK_SECS, TimeUnit.SECONDS);
USERS_LIST_REFRESH_EXECUTOR.scheduleAtFixedRate(this::updateUserInfoList, SERVER_USERS_LIST_UPDATE_SECS, SERVER_USERS_LIST_UPDATE_SECS, TimeUnit.SECONDS);
}
@Override

View file

@ -25,6 +25,8 @@ import mage.server.Main;
import mage.server.User;
import mage.server.managers.ManagerFactory;
import mage.util.MultiAmountMessage;
import mage.util.ThreadUtils;
import mage.util.XMageThreadFactory;
import mage.utils.StreamUtils;
import mage.utils.timer.PriorityTimer;
import mage.view.*;
@ -53,7 +55,7 @@ public class GameController implements GameCallback {
private final ExecutorService gameExecutor;
private static final Logger logger = Logger.getLogger(GameController.class);
protected final ScheduledExecutorService joinWaitingExecutor = Executors.newSingleThreadScheduledExecutor();
private ScheduledExecutorService JOIN_WAITING_EXECUTOR = null;
private ScheduledFuture<?> futureTimeout;
private final ManagerFactory managerFactory;
@ -238,13 +240,21 @@ public class GameController implements GameCallback {
}
}
);
joinWaitingExecutor.scheduleAtFixedRate(() -> {
// wait all players
if (JOIN_WAITING_EXECUTOR == null) {
JOIN_WAITING_EXECUTOR = Executors.newSingleThreadScheduledExecutor(
new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_GAME_JOIN_WAITING + " " + game.getId())
);
}
JOIN_WAITING_EXECUTOR.scheduleAtFixedRate(() -> {
try {
sendInfoAboutPlayersNotJoinedYetAndTryToFixIt();
} catch (Exception ex) {
logger.fatal("Send info about player not joined yet:", ex);
}
}, GAME_TIMEOUTS_CHECK_JOINING_STATUS_EVERY_SECS, GAME_TIMEOUTS_CHECK_JOINING_STATUS_EVERY_SECS, TimeUnit.SECONDS);
checkJoinAndStart();
}
@ -347,7 +357,9 @@ public class GameController implements GameCallback {
}
private void sendInfoAboutPlayersNotJoinedYetAndTryToFixIt() {
// runs every 10 secs untill all players join
// TODO: need code and feature review - is it useful? 2024-06-23
// runs every 10 secs until all players join
// runs BEFORE game start, so it's safe to call player.leave here
for (Player player : game.getPlayers().values()) {
if (player.canRespond() && player.isHuman()) {
Optional<User> requestedUser = getUserByPlayerId(player.getId());
@ -412,7 +424,7 @@ public class GameController implements GameCallback {
private void checkJoinAndStart() {
if (isAllJoined()) {
joinWaitingExecutor.shutdownNow();
JOIN_WAITING_EXECUTOR.shutdownNow();
managerFactory.threadExecutor().getCallExecutor().execute(this::startGame);
}
}

View file

@ -11,6 +11,8 @@ import mage.players.PlayerType;
import mage.server.RoomImpl;
import mage.server.User;
import mage.server.managers.ManagerFactory;
import mage.util.ThreadUtils;
import mage.util.XMageThreadFactory;
import mage.view.MatchView;
import mage.view.RoomUsersView;
import mage.view.TableView;
@ -25,18 +27,21 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* @author BetaSteward_at_googlemail.com
* @author BetaSteward_at_googlemail.com, JayDi85
*/
public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable {
private static final Logger LOGGER = Logger.getLogger(GamesRoomImpl.class);
private static final int MAX_FINISHED_TABLES = 50;
private static final int MAX_FINISHED_TABLES = 25;
private static final ScheduledExecutorService UPDATE_EXECUTOR = Executors.newSingleThreadScheduledExecutor();
private static List<TableView> tableView = new ArrayList<>();
private static List<MatchView> matchView = new ArrayList<>();
private static List<RoomUsersView> roomUsersView = new ArrayList<>();
// server's lobby
private static List<TableView> lobbyTables = new ArrayList<>();
private static List<MatchView> lobbyMatches = new ArrayList<>();
private static List<RoomUsersView> lobbyUsers = new ArrayList<>();
private static final ScheduledExecutorService UPDATE_LOBBY_EXECUTOR = Executors.newSingleThreadScheduledExecutor(
new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_LOBBY_REFRESH)
);
private final ManagerFactory managerFactory;
private final ConcurrentHashMap<UUID, Table> tables = new ConcurrentHashMap<>();
@ -44,9 +49,11 @@ public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable {
public GamesRoomImpl(ManagerFactory managerFactory) {
super(managerFactory.chatManager());
this.managerFactory = managerFactory;
UPDATE_EXECUTOR.scheduleAtFixedRate(() -> {
// update lobby's data
UPDATE_LOBBY_EXECUTOR.scheduleAtFixedRate(() -> {
try {
update();
updateLobby();
} catch (Exception e) {
LOGGER.fatal("Games room update error: " + e.getMessage(), e);
}
@ -56,10 +63,11 @@ public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable {
@Override
public List<TableView> getTables() {
return tableView;
return lobbyTables;
}
private void update() {
private void updateLobby() {
// tables and matches
List<Table> allTables = new ArrayList<>(tables.values());
allTables.sort(new TableListSorter());
List<MatchView> matchList = new ArrayList<>();
@ -77,8 +85,10 @@ public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable {
this.removeTable(table.getId());
}
}
tableView = tableList;
matchView = matchList;
lobbyTables = tableList;
lobbyMatches = matchList;
// users
List<UsersView> users = new ArrayList<>();
for (User user : managerFactory.userManager().getUsers()) {
if (user.isOnlineUser()) {
@ -105,7 +115,6 @@ public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable {
}
}
}
users.sort((one, two) -> one.getUserName().compareToIgnoreCase(two.getUserName()));
List<RoomUsersView> roomUserInfo = new ArrayList<>();
roomUserInfo.add(new RoomUsersView(users,
@ -113,12 +122,12 @@ public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable {
managerFactory.threadExecutor().getActiveThreads(managerFactory.threadExecutor().getGameExecutor()),
managerFactory.configSettings().getMaxGameThreads()
));
roomUsersView = roomUserInfo;
lobbyUsers = roomUserInfo;
}
@Override
public List<MatchView> getFinished() {
return matchView;
return lobbyMatches;
}
@Override
@ -190,7 +199,7 @@ public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable {
@Override
public List<RoomUsersView> getRoomUsersInfo() {
return roomUsersView;
return lobbyUsers;
}
}

View file

@ -1,5 +1,7 @@
package mage.server.util;
import mage.util.ThreadUtils;
import mage.util.XMageThreadFactory;
import mage.utils.StreamUtils;
import org.apache.log4j.Logger;
@ -44,8 +46,10 @@ public enum ServerMessagesUtil {
private static final AtomicInteger reconnects = new AtomicInteger(0);
ServerMessagesUtil() {
ScheduledExecutorService updateExecutor = Executors.newSingleThreadScheduledExecutor();
updateExecutor.scheduleAtFixedRate(this::reloadMessages, 5, SERVER_MSG_REFRESH_RATE_SECS, TimeUnit.SECONDS);
ScheduledExecutorService NEWS_MESSAGES_EXECUTOR = Executors.newSingleThreadScheduledExecutor(
new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_NEWS_REFRESH)
);
NEWS_MESSAGES_EXECUTOR.scheduleAtFixedRate(this::reloadMessages, 5, SERVER_MSG_REFRESH_RATE_SECS, TimeUnit.SECONDS);
}
public List<String> getMessages() {

View file

@ -3,6 +3,7 @@ package mage.server.util;
import mage.server.managers.ConfigSettings;
import mage.server.managers.ThreadExecutor;
import mage.util.ThreadUtils;
import mage.util.XMageThreadFactory;
import org.apache.log4j.Logger;
import java.util.concurrent.*;
@ -146,19 +147,3 @@ public class ThreadExecutorImpl implements ThreadExecutor {
}
}
class XMageThreadFactory implements ThreadFactory {
private final String prefix;
XMageThreadFactory(String prefix) {
this.prefix = prefix;
}
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
// default name, but threads can change it (example: on game or tourney start)
thread.setName(prefix + " " + thread.getThreadGroup().getName() + "-" + thread.getId());
return thread;
}
}