Tokens rework:

- added tokens database (now all tokens store in tokens-database.txt, related to #10139);
 - added sets/cards/tokens stats on app's start;

Mage.Client/src/main/java/mage/client/deckeditor/collection/viewer/MageBook.java

@@ -5,10 +5,7 @@ import mage.cards.CardDimensions;
 import mage.cards.ExpansionSet;
 import mage.cards.MageCard;
 import mage.cards.Sets;
-import mage.cards.repository.CardCriteria;
-import mage.cards.repository.CardInfo;
-import mage.cards.repository.CardRepository;
-import mage.cards.repository.ExpansionRepository;
+import mage.cards.repository.*;
 import mage.client.MageFrame;
 import mage.client.cards.BigCard;
 import mage.client.components.HoverButton;
@@ -30,7 +27,6 @@ import mage.game.permanent.token.Token;
 import mage.view.*;
 import org.apache.log4j.Logger;
 import org.mage.card.arcane.ManaSymbols;
-import org.mage.plugins.card.images.CardDownloadData;
 
 import javax.imageio.ImageIO;
 import javax.swing.*;
@@ -46,7 +42,6 @@ import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static java.lang.Math.min;
-import static org.mage.plugins.card.images.DownloadPicturesService.getTokenCardUrls;
 
 /**
  * Card viewer (mage book) with cards and page flipping
@@ -236,20 +231,19 @@ public class MageBook extends JComponent {
         List<Object> res = new ArrayList<>();
 
         // tokens
-        List<CardDownloadData> allTokens = getTokenCardUrls().stream()
-                .filter(token -> token.getSet().equals(currentSet))
-                .filter(token -> token.getAffectedClassName().contains("Token"))
+        List<TokenInfo> allTokens = TokenRepository.instance.getByType(TokenType.TOKEN)
+                .stream()
+                .filter(token -> token.getSetCode().equals(currentSet))
                 .collect(Collectors.toList());
         allTokens.forEach(token -> {
-            String fullClassName = "mage.game.permanent.token." + token.getAffectedClassName();
             try {
-                Class<?> c = Class.forName(fullClassName);
+                Class<?> c = Class.forName(token.getFullClassFileName());
                 Constructor<?> cons = c.getConstructor();
                 Object newToken = cons.newInstance();
                 if (newToken instanceof Token) {
                     ((Token) newToken).setOriginalExpansionSetCode(currentSet);
                     ((Token) newToken).setExpansionSetCodeForImage(currentSet);
-                    ((Token) newToken).setTokenType(token.getType()); // must be called after set code, so it keep the type
+                    ((Token) newToken).setTokenType(token.getImageNumber()); // must be called after set code, so it keep the type
                     res.add(newToken);
                 }
             } catch (Exception e) {
@@ -258,14 +252,13 @@ public class MageBook extends JComponent {
         });
 
         // emblems
-        List<CardDownloadData> allEmblems = getTokenCardUrls().stream()
-                .filter(token -> token.getSet().equals(currentSet))
-                .filter(token -> token.getAffectedClassName().contains("Emblem"))
+        List<TokenInfo> allEmblems = TokenRepository.instance.getByType(TokenType.EMBLEM)
+                .stream()
+                .filter(token -> token.getSetCode().equals(currentSet))
                 .collect(Collectors.toList());
         allEmblems.forEach(token -> {
-            String fullClassName = "mage.game.command.emblems." + token.getAffectedClassName();
             try {
-                Class<?> c = Class.forName(fullClassName);
+                Class<?> c = Class.forName(token.getFullClassFileName());
                 Constructor<?> cons = c.getConstructor();
                 Object newEmblem = cons.newInstance();
                 if (newEmblem instanceof Emblem) {
@@ -278,14 +271,13 @@ public class MageBook extends JComponent {
         });
 
         // planes
-        List<CardDownloadData> allPlanes = getTokenCardUrls().stream()
-                .filter(token -> token.getSet().equals(currentSet))
-                .filter(token -> token.getAffectedClassName().contains("Plane"))
+        List<TokenInfo> allPlanes = TokenRepository.instance.getByType(TokenType.PLANE)
+                .stream()
+                .filter(token -> token.getSetCode().equals(currentSet))
                 .collect(Collectors.toList());
         allPlanes.forEach(token -> {
-            String fullClassName = "mage.game.command.planes." + token.getAffectedClassName();
             try {
-                Class<?> c = Class.forName(fullClassName);
+                Class<?> c = Class.forName(token.getFullClassFileName());
                 Constructor<?> cons = c.getConstructor();
                 Object newPlane = cons.newInstance();
                 if (newPlane instanceof Plane) {
@@ -298,14 +290,13 @@ public class MageBook extends JComponent {
         });
 
         // dungeons
-        List<CardDownloadData> allDungeons = getTokenCardUrls().stream()
-                .filter(token -> token.getSet().equals(currentSet))
-                .filter(token -> token.getAffectedClassName().contains("Dungeon"))
+        List<TokenInfo> allDungeons = TokenRepository.instance.getByType(TokenType.DUNGEON)
+                .stream()
+                .filter(token -> token.getSetCode().equals(currentSet))
                 .collect(Collectors.toList());
         allDungeons.forEach(token -> {
-            String fullClassName = "mage.game.command.dungeons." + token.getAffectedClassName();
             try {
-                Class<?> c = Class.forName(fullClassName);
+                Class<?> c = Class.forName(token.getFullClassFileName());
                 Constructor<?> cons = c.getConstructor();
                 Object newDungeon = cons.newInstance();
                 if (newDungeon instanceof Dungeon) {

Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java

@@ -5,6 +5,7 @@ import mage.cards.Sets;
 import mage.cards.repository.CardCriteria;
 import mage.cards.repository.CardInfo;
 import mage.cards.repository.CardRepository;
+import mage.cards.repository.TokenRepository;
 import mage.client.MageFrame;
 import mage.client.dialog.DownloadImagesDialog;
 import mage.client.dialog.PreferencesDialog;
@@ -536,7 +537,19 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
                 }
             });
 
-            allCardsUrls.addAll(getTokenCardUrls());
+            // tokens
+            TokenRepository.instance.getAllTokens().forEach(token -> {
+                CardDownloadData card = new CardDownloadData(
+                        token.getName(),
+                        token.getSetCode(),
+                        "0",
+                        false,
+                        token.getImageNumber(),
+                        true,
+                        token.getImageFileName()
+                );
+                allCardsUrls.add(card);
+            });
         } catch (Exception e) {
             logger.error(e);
         }

Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java

@@ -16,10 +16,7 @@ import mage.abilities.keyword.*;
 import mage.cards.*;
 import mage.cards.decks.DeckCardLists;
 import mage.cards.decks.importer.DeckImporter;
-import mage.cards.repository.CardCriteria;
-import mage.cards.repository.CardInfo;
-import mage.cards.repository.CardRepository;
-import mage.cards.repository.CardScanner;
+import mage.cards.repository.*;
 import mage.constants.CardType;
 import mage.constants.Rarity;
 import mage.constants.SubType;
@@ -42,8 +39,6 @@ import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.mage.plugins.card.dl.sources.ScryfallImageSupportCards;
-import org.mage.plugins.card.images.CardDownloadData;
-import org.mage.plugins.card.images.DownloadPicturesService;
 import org.reflections.Reflections;
 
 import java.io.IOException;
@@ -1227,30 +1222,30 @@ public class VerifyCardDataTest {
 
 
         // tok file's data
-        List<CardDownloadData> tokFileTokens = DownloadPicturesService.getTokenCardUrls();
+        List<TokenInfo> tokFileTokens = TokenRepository.instance.getAllTokens();
         LinkedHashMap<String, String> tokDataClassesIndex = new LinkedHashMap<>();
         LinkedHashMap<String, String> tokDataNamesIndex = new LinkedHashMap<>();
-        LinkedHashMap<String, List<CardDownloadData>> tokDataTokensBySetIndex = new LinkedHashMap<>();
-        for (CardDownloadData tokData : tokFileTokens) {
+        LinkedHashMap<String, List<TokenInfo>> tokDataTokensBySetIndex = new LinkedHashMap<>();
+        for (TokenInfo tokData : tokFileTokens) {
 
             String searchName;
             String setsList;
 
             // by set
-            List<CardDownloadData> tokensInSet = tokDataTokensBySetIndex.getOrDefault(tokData.getSet(), null);
+            List<TokenInfo> tokensInSet = tokDataTokensBySetIndex.getOrDefault(tokData.getSetCode(), null);
             if (tokensInSet == null) {
                 tokensInSet = new ArrayList<>();
-                tokDataTokensBySetIndex.put(tokData.getSet(), tokensInSet);
+                tokDataTokensBySetIndex.put(tokData.getSetCode(), tokensInSet);
             }
             tokensInSet.add(tokData);
 
             // by class
-            searchName = tokData.getAffectedClassName();
+            searchName = tokData.getFullClassFileName();
             setsList = tokDataClassesIndex.getOrDefault(searchName, "");
             if (!setsList.isEmpty()) {
                 setsList += ",";
             }
-            setsList += tokData.getSet();
+            setsList += tokData.getSetCode();
             tokDataClassesIndex.put(searchName, setsList);
 
             // by name
@@ -1259,7 +1254,7 @@ public class VerifyCardDataTest {
             if (!setsList.isEmpty()) {
                 setsList += ",";
             }
-            setsList += tokData.getSet();
+            setsList += tokData.getSetCode();
             tokDataNamesIndex.put(searchName, setsList);
         }
 
@@ -1302,7 +1297,7 @@ public class VerifyCardDataTest {
         tokDataTokensBySetIndex.forEach((setCode, setTokens) -> {
             if (!allSetCodes.contains(setCode)) {
                 errorsList.add("error, card-pictures-tok.txt contains unknown set code: "
-                        + setCode + " - " + setTokens.stream().map(CardDownloadData::getName).collect(Collectors.joining(", ")));
+                        + setCode + " - " + setTokens.stream().map(TokenInfo::getName).collect(Collectors.joining(", ")));
             }
         });
 
@@ -1323,7 +1318,7 @@ public class VerifyCardDataTest {
         }
         // set uses tokens, but tok data miss it
         setsWithTokens.forEach((setCode, sourceCards) -> {
-            List<CardDownloadData> setTokens = tokDataTokensBySetIndex.getOrDefault(setCode, null);
+            List<TokenInfo> setTokens = tokDataTokensBySetIndex.getOrDefault(setCode, null);
             if (setTokens == null) {
                 // it's not a problem -- just find set's cards without real tokens for image tests
                 // Possible reasons:
@@ -1350,7 +1345,7 @@ public class VerifyCardDataTest {
                             .map(card -> card.getName() + " - " + String.join(", ", card.getRules()))
                             .noneMatch(s -> s.contains(needTokenName))) {
                         warningsList.add("info, tok-data has un-used tokens: "
-                                + token.getSet() + " - " + token.getName());
+                                + token.getSetCode() + " - " + token.getName());
                     }
                 });
             }
@@ -1362,7 +1357,7 @@ public class VerifyCardDataTest {
                 // - outdated set code in tokens database (must be fixed by new set code, another verify check it)
                 // - promo set contains additional tokens for main set (it's ok and must be ignored, example: Saproling in E02)
                 warningsList.add("warning, tok-data has tokens, but real set haven't cards with it: "
-                        + setCode + " - " + setTokens.stream().map(CardDownloadData::getName).collect(Collectors.joining(", ")));
+                        + setCode + " - " + setTokens.stream().map(TokenInfo::getName).collect(Collectors.joining(", ")));
             }
         });
 

Mage/src/main/java/mage/cards/repository/RepositoryUtil.java

@@ -22,10 +22,22 @@ public final class RepositoryUtil {
     public static final boolean CARD_DB_RECREATE_BY_CLIENT_SIDE = true; // re-creates db from client (best performance) or downloads from server on connects (can be slow)
 
     public static void bootstrapLocalDb() {
-        // call local db to init all sets and cards repository (need for correct updates cycle, not on random request)
+        // call local db to init all sets and cards/tokens repository (need for correct updates cycle, not on random request)
         logger.info("Loading database...");
         ExpansionRepository.instance.getContentVersionConstant();
         CardRepository.instance.getContentVersionConstant();
+        TokenRepository.instance.getAllTokens().size();
+
+        // stats
+        int totalCards = CardRepository.instance.findCards(new CardCriteria().nightCard(false)).size()
+                + CardRepository.instance.findCards(new CardCriteria().nightCard(true)).size();
+        logger.info("Database stats:");
+        logger.info(" - sets: " + ExpansionRepository.instance.getAll().size());
+        logger.info(" - cards: " + totalCards);
+        logger.info(" - tokens: " + TokenRepository.instance.getByType(TokenType.TOKEN).size());
+        logger.info(" - emblems: " + TokenRepository.instance.getByType(TokenType.EMBLEM).size());
+        logger.info(" - planes: " + TokenRepository.instance.getByType(TokenType.PLANE).size());
+        logger.info(" - dungeons: " + TokenRepository.instance.getByType(TokenType.DUNGEON).size());
     }
 
     public static boolean isDatabaseObsolete(ConnectionSource connectionSource, String entityName, long version) throws SQLException {

Mage/src/main/java/mage/cards/repository/TokenInfo.java

@@ -0,0 +1,71 @@
+package mage.cards.repository;
+
+/**
+ * Token item for tokens database
+ *
+ * @author JayDi85
+ */
+public class TokenInfo {
+
+    private TokenType tokenType;
+    private String name;
+    private String setCode;
+    private Integer imageNumber = 1; // if one set contains diff images with same name
+
+    private String classFileName;
+
+    private String imageFileName;
+
+    public TokenInfo(TokenType tokenType, String name, String setCode, Integer imageNumber) {
+        this(tokenType, name, setCode, imageNumber, "", "");
+    }
+
+    public TokenInfo(TokenType tokenType, String name, String setCode, Integer imageNumber, String classFileName, String imageFileName) {
+        this.tokenType = tokenType;
+        this.name = name;
+        this.setCode = setCode;
+        this.imageNumber = imageNumber;
+        this.classFileName = classFileName;
+        this.imageFileName = imageFileName;
+    }
+
+    public TokenType getTokenType() {
+        return tokenType;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getImageFileName() {
+        return imageFileName;
+    }
+
+    public String getSetCode() {
+        return setCode;
+    }
+
+    public String getClassFileName() {
+        return classFileName;
+    }
+
+    public Integer getImageNumber() {
+        return imageNumber;
+    }
+
+    public String getFullClassFileName() {
+        String simpleName = classFileName.isEmpty() ? name.replaceAll("[^a-zA-Z0-9]", "") : classFileName;
+        switch (this.tokenType) {
+            case TOKEN:
+                return "mage.game.permanent.token." + simpleName;
+            case EMBLEM:
+                return "mage.game.command.emblems." + simpleName;
+            case PLANE:
+                return "mage.game.command.planes." + simpleName;
+            case DUNGEON:
+                return "mage.game.command.dungeons." + simpleName;
+            default:
+                throw new IllegalStateException("Unknown token type: " + this.tokenType);
+        }
+    }
+}

Mage/src/main/java/mage/cards/repository/TokenRepository.java

@@ -0,0 +1,197 @@
+package mage.cards.repository;
+
+import org.apache.log4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author JayDi85
+ */
+public enum TokenRepository {
+
+    instance;
+
+    private static final Logger logger = Logger.getLogger(TokenRepository.class);
+
+    private ArrayList<TokenInfo> allTokens = new ArrayList<>();
+    private Map<String, List<TokenInfo>> indexByClassName = new HashMap<>();
+    private Map<TokenType, List<TokenInfo>> indexByType = new HashMap<>();
+
+    TokenRepository() {
+    }
+
+    public void init() {
+        allTokens.clear();
+        indexByClassName.clear();
+        indexByType.clear();
+
+        allTokens = loadAllTokens();
+
+        // index
+        allTokens.forEach(token -> {
+            // by class
+            List<TokenInfo> list = indexByClassName.getOrDefault(token.getClassFileName(), null);
+            if (list == null) {
+                list = new ArrayList<>();
+                indexByClassName.put(token.getClassFileName(), list);
+            }
+            list.add(token);
+
+            // by type
+            list = indexByType.getOrDefault(token.getTokenType(), null);
+            if (list == null) {
+                list = new ArrayList<>();
+                indexByType.put(token.getTokenType(), list);
+            }
+            list.add(token);
+        });
+    }
+
+    public List<TokenInfo> getAllTokens() {
+        if (allTokens.isEmpty()) {
+            init();
+        }
+
+        return allTokens;
+    }
+
+    public List<TokenInfo> getByType(TokenType tokenType) {
+        return indexByType.getOrDefault(tokenType, new ArrayList<>());
+    }
+
+    private static ArrayList<TokenInfo> loadAllTokens() throws RuntimeException {
+        // Must load tokens data in strict mode (throw exception on any error)
+        // Try to put verify checks here instead verify tests
+        String dbSource = "tokens-database.txt";
+        ArrayList<TokenInfo> list = new ArrayList<>();
+        InputStream in = TokenRepository.class.getClassLoader().getResourceAsStream(dbSource);
+        if (in == null) {
+            throw new RuntimeException("Tokens database: can't load resource file " + dbSource);
+        }
+
+        List<String> errorsList = new ArrayList<>();
+        try (InputStreamReader input = new InputStreamReader(in);
+             BufferedReader reader = new BufferedReader(input)) {
+            String line = reader.readLine();
+            while (line != null) {
+                try {
+                    line = line.trim();
+                    if (!line.startsWith("|")) {
+                        continue;
+                    }
+
+                    List<String> params = Arrays.stream(line.split("\\|", -1))
+                            .map(String::trim)
+                            .collect(Collectors.toList());
+                    if (params.size() < 5) {
+                        errorsList.add("Tokens database: wrong params count: " + line);
+                        continue;
+                    }
+                    if (!params.get(1).toLowerCase(Locale.ENGLISH).equals("generate")) {
+                        // TODO: remove "generate" from db
+                        errorsList.add("Tokens database: miss generate param: " + line);
+                        continue;
+                    }
+
+                    // image number (uses if one set contains multiple tokens with same name)
+                    int imageNumber = 0;
+                    if (!params.get(4).isEmpty()) {
+                        imageNumber = Integer.parseInt(params.get(4));
+                    }
+
+                    // image file name
+                    String imageFileName = "";
+                    if (params.size() > 5 && !params.get(5).isEmpty()) {
+                        imageFileName = params.get(5);
+                    }
+
+                    // token class name (uses for images search for render)
+                    String tokenClassName = "";
+                    if (params.size() > 7 && !params.get(6).isEmpty()) {
+                        tokenClassName = params.get(6);
+                    }
+                    if (tokenClassName.isEmpty()) {
+                        errorsList.add("Tokens database: miss class name: " + line);
+                        continue;
+                    }
+
+                    // object type
+                    String objectType = params.get(2);
+                    String tokenName = params.get(3);
+                    String setCode = "";
+                    TokenType tokenType = null;
+
+                    // type - token
+                    if (objectType.startsWith("TOK:")) {
+                        setCode = objectType.substring("TOK:".length());
+                        tokenType = TokenType.TOKEN;
+                    }
+
+                    // type - emblem
+                    if (objectType.startsWith("EMBLEM:")) {
+                        setCode = objectType.substring("EMBLEM:".length());
+                        tokenType = TokenType.EMBLEM;
+                        if (!tokenName.startsWith("Emblem ")) {
+                            errorsList.add("Tokens database: emblem's name must start with [Emblem ...] word: " + line);
+                            continue;
+                        }
+                        if (!tokenClassName.endsWith("Emblem")) {
+                            errorsList.add("Tokens database: emblem's class name must ends with [...Emblem] word: " + line);
+                            continue;
+                        }
+                    }
+
+                    // type - plane
+                    if (objectType.startsWith("PLANE:")) {
+                        setCode = objectType.substring("PLANE:".length());
+                        tokenType = TokenType.PLANE;
+                        if (!tokenName.startsWith("Plane - ")) {
+                            errorsList.add("Tokens database: plane's name must start with [Plane - ...] word: " + line);
+                            continue;
+                        }
+                        if (!tokenClassName.endsWith("Plane")) {
+                            errorsList.add("Tokens database: plane's class name must ends with [...Plane] word: " + line);
+                            continue;
+                        }
+                    }
+
+                    // type - dungeon
+                    if (objectType.startsWith("DUNGEON:")) {
+                        setCode = objectType.substring("DUNGEON:".length());
+                        tokenType = TokenType.DUNGEON;
+                        if (!tokenClassName.endsWith("Dungeon")) {
+                            errorsList.add("Tokens database: dungeon's class name must ends with [...Dungeon] word: " + line);
+                            continue;
+                        }
+                    }
+
+                    // type - unknown
+                    if (tokenType == null) {
+                        errorsList.add("Tokens database: unknown line format: " + line);
+                        continue;
+                    }
+
+                    // OK
+                    TokenInfo token = new TokenInfo(tokenType, tokenName, setCode, imageNumber, tokenClassName, imageFileName);
+                    list.add(token);
+                } finally {
+                    line = reader.readLine();
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new RuntimeException("Tokens database: can't read data, unknown error - " + e.getMessage());
+        }
+
+        if (!errorsList.isEmpty()) {
+            errorsList.forEach(logger::error);
+            throw new RuntimeException(String.format("Tokens database: found %d errors, see logs above for details", errorsList.size()));
+        }
+
+        return list;
+    }
+}

Mage/src/main/java/mage/cards/repository/TokenType.java

@@ -0,0 +1,15 @@
+package mage.cards.repository;
+
+/**
+ * XMage's token types for images
+ *
+ * @author JayDi85
+ */
+public enum TokenType {
+
+    TOKEN,
+    EMBLEM,
+    PLANE,
+    DUNGEON
+
+}

Mage/src/main/resources/tokens-database.txt

@@ -11,6 +11,7 @@
 # - use simple name for the emblem like Gideon
 # - all names must be different in one set
 # - xmage do not support various arts for the emblems (e.g. must be one emblem's image per set)
+# - download images must be setup in ScryfallImageSupportTokens
 |Generate|EMBLEM:BFZ|Emblem Gideon|||GideonAllyOfZendikarEmblem|
 |Generate|EMBLEM:BFZ|Emblem Kiora|||KioraMasterOfTheDepthsEmblem|
 |Generate|EMBLEM:BFZ|Emblem Nixilis|||ObNixilisReignitedEmblem|
@@ -114,6 +115,8 @@
 |Generate|EMBLEM:MOC|Emblem Teferi|||TeferisTalentEmblem|
 
 # ALL PLANES
+# Usage hints:
+# - download images must be setup in ScryfallImageSupportTokens
 |Generate|PLANE:PCA|Plane - Academy at Tolaria West|||AcademyAtTolariaWestPlane|
 |Generate|PLANE:PCA|Plane - Agyrem|||AgyremPlane|
 |Generate|PLANE:PCA|Plane - Akoum|||AkoumPlane|
@@ -137,11 +140,15 @@
 |Generate|PLANE:PCA|Plane - Trail of the Mage-Rings|||TrailOfTheMageRingsPlane|
 
 # ALL DUNGEONS
+# Usage hints:
+# - download images must be setup in ScryfallImageSupportTokens
 |Generate|DUNGEON:AFR|Tomb of Annihilation|||TombOfAnnihilationDungeon|
 |Generate|DUNGEON:AFR|Lost Mine of Phandelver|||LostMineOfPhandelverDungeon|
 |Generate|DUNGEON:AFR|Dungeon of the Mad Mage|||DungeonOfTheMadMageDungeon|
 
 # ALL TOKENS
+# Usage hints:
+# - download images must be setup in ScryfallImageSupportTokens
 
 # 10E
 |Generate|TOK:10E|Dragon|||DragonToken2|