Merge pull request #2474 from nigelzor/verify-card-data

Add tests that compare card data with mtgjson

.gitignore

@@ -77,6 +77,11 @@ Mage/target
 Mage.Updater/target
 mage.updater.client/target
 
+# Mage.Verify
+Mage.Verify/target
+Mage.Verify/AllCards.json.zip
+Mage.Verify/AllSets.json.zip
+
 releases
 Utils/author.txt
 .DS_Store

Mage.Verify/pom.xml

@@ -0,0 +1,76 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.mage</groupId>
+        <artifactId>mage-root</artifactId>
+        <version>1.4.16</version>
+    </parent>
+
+    <artifactId>mage-verify</artifactId>
+    <packaging>jar</packaging>
+    <name>Mage Verify</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>mage</artifactId>
+            <version>${mage-version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>mage-sets</artifactId>
+            <version>${mage-version}</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.6.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <type>jar</type>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <configuration>
+                        <argLine>-Dfile.encoding=UTF-8</argLine>
+                    </configuration>
+                </plugin>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-compiler-plugin</artifactId>
+<!--                    <configuration>
+                        <compilerArgument>-Xlint:unchecked</compilerArgument>
+                    </configuration> -->
+                </plugin>
+                <plugin>
+                    <artifactId>maven-resources-plugin</artifactId>
+                    <configuration>
+                        <encoding>UTF-8</encoding>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+        <finalName>mage-verify</finalName>
+    </build>
+
+    <properties/>
+
+</project>

Mage.Verify/src/main/java/mage/verify/JsonCard.java

@@ -0,0 +1,40 @@
+package mage.verify;
+
+import java.util.List;
+
+class JsonCard {
+    public String layout;
+    public String name;
+    public List<String> names; // flip cards
+    public String manaCost;
+    public int cmc;
+    public List<String> colors;
+    public List<String> colorIdentity;
+    public String type;
+    public List<String> supertypes;
+    public List<String> types;
+    public List<String> subtypes;
+    public String text;
+    public String power;
+    public String toughness;
+    public int loyalty;
+    public String imageName;
+    public boolean starter; // only available in boxed sets and not in boosters
+    public int hand; // vanguard
+    public int life; // vanguard
+    public String mciNumber;
+
+    // only available in AllSets.json
+    public String artist;
+    public String flavor;
+    public String id;
+    public int multiverseid;
+    public String rarity;
+    public boolean reserved;
+    public int[] variations;
+    public String number;
+    public String releaseDate; // promos
+    public String border;
+    public String watermark;
+    public boolean timeshifted;
+}

Mage.Verify/src/main/java/mage/verify/JsonSet.java

@@ -0,0 +1,23 @@
+package mage.verify;
+
+import java.util.List;
+import java.util.Map;
+
+class JsonSet {
+    public String name;
+    public String code;
+    public String oldCode;
+    public String gathererCode;
+    public String magicCardsInfoCode;
+    public String[] magicRaritiesCodes;
+    public String releaseDate;
+    public String border;
+    public String type;
+    public List<Object> booster; // [String|[String]]
+    public List<JsonCard> cards;
+    public String block;
+    public boolean onlineOnly;
+    public String mkm_id;
+    public String mkm_name;
+    public Map<String, String> translations;
+}

Mage.Verify/src/main/java/mage/verify/MtgJson.java

@@ -0,0 +1,109 @@
+package mage.verify;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.text.Normalizer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.zip.ZipInputStream;
+
+public class MtgJson {
+    private MtgJson() {}
+
+    private static class CardHolder {
+        private static final Map<String, JsonCard> cards;
+        static {
+            try {
+                cards = loadAllCards();
+                addAliases(cards);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private static class SetHolder {
+        private static final Map<String, JsonSet> sets;
+        static {
+            try {
+                sets = loadAllSets();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private static Map<String, JsonCard> loadAllCards() throws IOException {
+        return readFromZip("AllCards.json.zip", new TypeReference<Map<String, JsonCard>>() {});
+    }
+
+    private static Map<String, JsonSet> loadAllSets() throws IOException {
+        return readFromZip("AllSets.json.zip", new TypeReference<Map<String, JsonSet>>() {});
+    }
+
+    private static <T> T readFromZip(String filename, TypeReference<T> ref) throws IOException {
+        InputStream stream = MtgJson.class.getResourceAsStream(filename);
+        if (stream == null) {
+            File file = new File(filename);
+            if (!file.exists()) {
+                InputStream download = new URL("http://mtgjson.com/json/" + filename).openStream();
+                Files.copy(download, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                System.out.println("Downloaded " + filename + " to " + file.getAbsolutePath());
+            } else {
+                System.out.println("Using " + filename + " from " + file.getAbsolutePath());
+            }
+            stream = new FileInputStream(file);
+        }
+        ZipInputStream zipInputStream = new ZipInputStream(stream);
+        zipInputStream.getNextEntry();
+        return new ObjectMapper().readValue(zipInputStream, ref);
+    }
+
+    public static Map<String, JsonSet> sets() {
+        return SetHolder.sets;
+    }
+
+    public static JsonCard card(String name) {
+        return findReference(CardHolder.cards, name);
+    }
+
+    private static <T> T findReference(Map<String, T> reference, String name) {
+        T ref = reference.get(name);
+        if (ref == null) {
+            name = name.replaceFirst("\\bA[Ee]", "Æ");
+            ref = reference.get(name);
+        }
+        if (ref == null) {
+            name = name.replace("'", "\""); // for Kongming, "Sleeping Dragon" & Pang Tong, "Young Phoenix"
+            ref = reference.get(name);
+        }
+        return ref;
+    }
+
+    private static <T> void addAliases(Map<String, T> reference) {
+        Map<String, String> aliases = new HashMap<>();
+        for (String name : reference.keySet()) {
+            String unaccented = stripAccents(name);
+            if (!name.equals(unaccented)) {
+                aliases.put(name, unaccented);
+            }
+        }
+        for (Map.Entry<String, String> mapping : aliases.entrySet()) {
+            reference.put(mapping.getValue(), reference.get(mapping.getKey()));
+        }
+    }
+
+    private static String stripAccents(String str) {
+        String decomposed = Normalizer.normalize(str, Normalizer.Form.NFKD);
+        return decomposed.replaceAll("[\\p{InCombiningDiacriticalMarks}]", "");
+    }
+
+}

Mage.Verify/src/test/java/mage/verify/VerifyCardDataTest.java

@@ -0,0 +1,234 @@
+package mage.verify;
+
+import mage.ObjectColor;
+import mage.cards.Card;
+import mage.cards.CardImpl;
+import mage.cards.CardSetInfo;
+import mage.cards.ExpansionSet;
+import mage.cards.Sets;
+import mage.cards.SplitCard;
+import mage.cards.basiclands.BasicLand;
+import mage.constants.CardType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class VerifyCardDataTest {
+
+    // right now this is very noisy, and not useful enough to make any assertions on
+    private static final boolean CHECK_SOURCE_TOKENS = false;
+
+    public static List<Card> allCards() {
+        Collection<ExpansionSet> sets = Sets.getInstance().values();
+        List<Card> cards = new ArrayList<>();
+        for (ExpansionSet set : sets) {
+            if (set.isCustomSet()) {
+                continue;
+            }
+            for (ExpansionSet.SetCardInfo setInfo : set.getSetCardInfo()) {
+                cards.add(CardImpl.createCard(setInfo.getCardClass(), new CardSetInfo(setInfo.getName(), set.getCode(),
+                        setInfo.getCardNumber(), setInfo.getRarity(), setInfo.getGraphicInfo())));
+            }
+        }
+        return cards;
+    }
+
+    private void warn(Card card, String message) {
+        System.out.println("Warning: " + message + " for " + card.getName());
+    }
+
+    private void fail(Card card, String category, String message) {
+        failed++;
+        System.out.println("Error: (" + category + ") " + message + " for " + card.getName());
+    }
+
+    private int failed = 0;
+
+    @Test
+    public void verifyCards() throws IOException {
+        for (Card card : allCards()) {
+            Set<String> tokens = findSourceTokens(card.getClass());
+            if (card.isSplitCard()) {
+                check(((SplitCard) card).getLeftHalfCard(), null);
+                check(((SplitCard) card).getRightHalfCard(), null);
+            } else {
+                check(card, tokens);
+            }
+        }
+        if (failed > 0) {
+            Assert.fail(failed + " Errors");
+        }
+    }
+
+    private static final Pattern SHORT_JAVA_STRING = Pattern.compile("(?<=\")[A-Z][a-z]+(?=\")");
+
+    private Set<String> findSourceTokens(Class c) throws IOException {
+        if (!CHECK_SOURCE_TOKENS || BasicLand.class.isAssignableFrom(c)) {
+            return null;
+        }
+        String path = "../Mage.Sets/src/" + c.getName().replace(".", "/") + ".java";
+        try {
+            String source = new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8);
+            Matcher matcher = SHORT_JAVA_STRING.matcher(source);
+            Set<String> tokens = new HashSet<>();
+            while (matcher.find()) {
+                tokens.add(matcher.group());
+            }
+            return tokens;
+        } catch (NoSuchFileException e) {
+            System.out.println("failed to read " + path);
+            return null;
+        }
+    }
+
+    private void check(Card card, Set<String> tokens) {
+        JsonCard ref = MtgJson.card(card.getName());
+        if (ref == null) {
+            warn(card, "Missing card reference");
+            return;
+        }
+        checkAll(card, ref);
+        if (tokens != null) {
+            JsonCard ref2 = null;
+            if (card.isFlipCard()) {
+                ref2 = MtgJson.card(card.getFlipCardName());
+            }
+            for (String token : tokens) {
+                if (!(token.equals(card.getName())
+                        || containsInTypesOrText(ref, token)
+                        || containsInTypesOrText(ref, token.toLowerCase())
+                        || (ref2 != null && (containsInTypesOrText(ref2, token) || containsInTypesOrText(ref2, token.toLowerCase())))
+                )) {
+                    System.out.println("unexpected token " + token + " in " + card);
+                }
+            }
+        }
+    }
+
+    private boolean containsInTypesOrText(JsonCard ref, String token) {
+        return contains(ref.types, token)
+                || contains(ref.subtypes, token)
+                || contains(ref.supertypes, token)
+                || ref.text.contains(token);
+    }
+
+    private boolean contains(Collection<String> options, String value) {
+        return options != null && options.contains(value);
+    }
+
+    private void checkAll(Card card, JsonCard ref) {
+        checkCost(card, ref);
+        checkPT(card, ref);
+        checkSubtypes(card, ref);
+        checkSupertypes(card, ref);
+        checkTypes(card, ref);
+        checkColors(card, ref);
+    }
+
+    private void checkColors(Card card, JsonCard ref) {
+        // gatherer is missing the color indicator on one card:
+        if ("Ulrich, Uncontested Alpha".equals(ref.name)) {
+            return;
+        }
+        Collection<String> expected = ref.colors;
+        ObjectColor color = card.getColor(null);
+        if (expected == null) {
+            expected = Collections.emptyList();
+        }
+        if (expected.size() != color.getColorCount() ||
+                (color.isBlack() && !expected.contains("Black")) ||
+                (color.isBlue() && !expected.contains("Blue")) ||
+                (color.isGreen() && !expected.contains("Green")) ||
+                (color.isRed() && !expected.contains("Red")) ||
+                (color.isWhite() && !expected.contains("White"))) {
+            fail(card, "colors", color + " != " + expected);
+        }
+    }
+
+    private void checkSubtypes(Card card, JsonCard ref) {
+        Collection<String> expected = ref.subtypes;
+        if (expected != null && expected.contains("Urza’s")) {
+            expected = new ArrayList<>(expected);
+            for (ListIterator<String> it = ((List<String>) expected).listIterator(); it.hasNext();) {
+                if (it.next().equals("Urza’s")) {
+                    it.set("Urza's");
+                }
+            }
+        }
+        if (!eqSet(card.getSubtype(null), expected)) {
+            fail(card, "subtypes", card.getSubtype(null) + " != " + expected);
+        }
+    }
+
+    private void checkSupertypes(Card card, JsonCard ref) {
+        Collection<String> expected = ref.supertypes;
+        if (!eqSet(card.getSupertype(), expected)) {
+            fail(card, "supertypes", card.getSupertype() + " != " + expected);
+        }
+    }
+
+    private void checkTypes(Card card, JsonCard ref) {
+        Collection<String> expected = ref.types;
+        List<String> type = new ArrayList<>();
+        for (CardType cardType : card.getCardType()) {
+            type.add(cardType.toString());
+        }
+        if (!eqSet(type, expected)) {
+            fail(card, "types", type + " != " + expected);
+        }
+    }
+
+    private static <T> boolean eqSet(Collection<T> a, Collection<T> b) {
+        if (a == null || a.isEmpty()) {
+            return b == null || b.isEmpty();
+        }
+        return b != null && a.size() == b.size() && a.containsAll(b);
+    }
+
+    private void checkPT(Card card, JsonCard ref) {
+        if (!eqPT(card.getPower().toString(), ref.power) || !eqPT(card.getToughness().toString(), ref.toughness)) {
+            String pt = card.getPower() + "/" + card.getToughness();
+            String expected = ref.power + "/" + ref.toughness;
+            fail(card, "pt", pt + " != " + expected);
+        }
+    }
+
+    private boolean eqPT(String found, String expected) {
+        if (expected == null) {
+            return "0".equals(found);
+        } else {
+            return found.equals(expected) || expected.contains("*");
+        }
+    }
+
+    private void checkCost(Card card, JsonCard ref) {
+        String expected = ref.manaCost;
+        String cost = join(card.getManaCost().getSymbols());
+        if ("".equals(cost)) {
+            cost = null;
+        }
+        if (cost != null) {
+            cost = cost.replaceAll("P\\}", "/P}");
+        }
+        if (!Objects.equals(cost, expected)) {
+            fail(card, "cost", cost + " != " + expected);
+        }
+    }
+
+    private String join(Iterable<?> items) {
+        StringBuilder result = new StringBuilder();
+        for (Object item : items) {
+            result.append(item);
+        }
+        return result.toString();
+    }
+
+}

pom.xml

@@ -61,6 +61,7 @@
         <module>Mage.Tests</module>
         <module>Mage.Updater</module>
         <module>Mage.Stats</module>
+        <module>Mage.Verify</module>
     </modules>
 
     <repositories>