// warning, dot not add os, language and other details
@@ -290,13 +299,18 @@ public class XmageURLConnection {
return "";
}
+ public static InputStream downloadBinary(String resourceUrl) {
+ return downloadBinary(resourceUrl, false);
+ }
+
/**
* Fast download of binary data
*
* @return stream on OK 200 response or null on any other errors
*/
- public static InputStream downloadBinary(String resourceUrl) {
+ public static InputStream downloadBinary(String resourceUrl, boolean downloadAsGZip) {
XmageURLConnection con = new XmageURLConnection(resourceUrl);
+ con.setForceGZipEncoding(downloadAsGZip);
con.startConnection();
if (con.isConnected()) {
try {
diff --git a/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java b/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java
index 9ee537214d0..dd77bf28980 100644
--- a/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java
+++ b/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java
@@ -301,10 +301,9 @@ public final class ManaSymbols {
if (SvgUtils.haveSvgSupport()) {
file = getSymbolFileNameAsSVG(symbol);
if (file.exists()) {
- try {
- InputStream fileStream = new FileInputStream(file);
+ try(InputStream fileStream = Files.newInputStream(file.toPath())) {
image = loadSymbolAsSVG(fileStream, file.getPath(), size, size);
- } catch (FileNotFoundException ignore) {
+ } catch (IOException ignore) {
}
}
}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java
index 7de570322db..3b2ca30e029 100644
--- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java
@@ -4,7 +4,10 @@ import mage.client.util.CardLanguage;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
/**
* @author North, JayDi85
@@ -70,4 +73,8 @@ public interface CardImageSource {
default boolean isTokenImageProvided(String setCode, String cardName, Integer tokenNumber) {
return false;
}
+
+ default void onFinished() {
+ // cleanup temp resources
+ }
}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiBulkData.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiBulkData.java
new file mode 100644
index 00000000000..b4bf322f4c1
--- /dev/null
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiBulkData.java
@@ -0,0 +1,19 @@
+package org.mage.plugins.card.dl.sources;
+
+import java.util.Date;
+
+/**
+ * Scryfall API: bulk file links to download
+ *
+ * API docs: here
+ *
+ * @author JayDi85
+ */
+public class ScryfallApiBulkData {
+
+ public String object;
+ public String type;
+ public Date updated_at;
+ public long size;
+ public String download_uri;
+}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCard.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCard.java
new file mode 100644
index 00000000000..5d9385f48d7
--- /dev/null
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCard.java
@@ -0,0 +1,156 @@
+package org.mage.plugins.card.dl.sources;
+
+import mage.cards.decks.CardNameUtil;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Scryfall API: card info
+ *
+ * API docs: here
+ *
+ * Field values depends on data source, check it on null (as example: bulk data from oracle cards and bulk data from all cards)
+ *
+ * @author JayDi85
+ */
+public class ScryfallApiCard {
+
+ public String name;
+ public String set; // set code
+ public String collector_number;
+ public String lang;
+ public String layout; // card type like adventure, etc, see https://scryfall.com/docs/api/layouts
+ public List card_faces;
+ public String image_status; // missing, placeholder, lowres, or highres_scan
+ public Map image_uris;
+
+ // fast access fields, fills on loading
+ transient public String imageSmall = "";
+ transient public String imageNormal = "";
+ transient public String imageLarge = "";
+
+ // potentially interesting fields, can be used in other places
+ //public UUID oracle_id; // TODO: implement card hint with oracle/cr ruling texts (see Rulings bulk data)
+ //public Integer edhrec_rank; // TODO: use it to rating cards for AI and draft bots
+ //public Object legalities; // TODO: add verify check for bans list
+ //public Boolean full_art; // TODO: add verify check for full art usage in sets
+ //public Object prices; // TODO: add total deck price and table limit by deck's price
+ //public Date released_at; // the date this card was first released.
+ //public String watermark; // background watermark image for some cards
+
+ public void prepareCompatibleData() {
+ if (this.image_uris != null) {
+ this.imageSmall = this.image_uris.getOrDefault("small", "");
+ this.imageNormal = this.image_uris.getOrDefault("normal", "");
+ this.imageLarge = this.image_uris.getOrDefault("large", "");
+ this.image_uris = null;
+ }
+
+ if (this.card_faces != null) {
+ this.card_faces.forEach(ScryfallApiCardFace::prepareCompatibleData);
+ }
+
+ // workaround for adventure card name fix:
+ // - scryfall: Ondu Knotmaster // Throw a Line
+ // - xmage: Ondu Knotmaster
+ if (this.layout.equals("adventure")) {
+ this.name = this.card_faces.get(0).name;
+ }
+
+ // workaround for flip card name and image fixes:
+ // - scryfall: Budoka Pupil // Ichiga, Who Topples Oaks
+ // - xmage: Budoka Pupil
+ if (this.layout.equals("flip")) {
+ if (!this.card_faces.get(0).imageNormal.isEmpty()
+ || !this.card_faces.get(1).imageNormal.isEmpty()) {
+ throw new IllegalArgumentException("Scryfall: unsupported data type, flip parts must not have images data in scryfall "
+ + this.set + " - " + this.collector_number + " - " + this.name);
+ }
+
+ // fix name
+ this.name = this.card_faces.get(0).name;
+
+ // fix image (xmage uses diff cards for flips, but it's same image)
+ // example: https://scryfall.com/card/sok/103/homura-human-ascendant-homuras-essence
+ this.card_faces.get(0).image_uris = this.image_uris;
+ this.card_faces.get(0).imageSmall = this.imageSmall;
+ this.card_faces.get(0).imageNormal = this.imageNormal;
+ this.card_faces.get(0).imageLarge = this.imageLarge;
+ this.card_faces.get(1).image_uris = this.image_uris;
+ this.card_faces.get(1).imageSmall = this.imageSmall;
+ this.card_faces.get(1).imageNormal = this.imageNormal;
+ this.card_faces.get(1).imageLarge = this.imageLarge;
+ }
+
+ // workaround for reversed cards:
+ // - scryfall: Command Tower // Command Tower
+ // - xmage: Command Tower (second side as diff card and direct link image), example: https://scryfall.com/card/rex/26/command-tower-command-tower
+ if (this.layout.equals("reversible_card")) {
+ if (!this.card_faces.get(0).name.equals(this.card_faces.get(1).name)) {
+ throw new IllegalArgumentException("Scryfall: unsupported data type, reversible_card has diff faces "
+ + this.set + " - " + this.collector_number + " - " + this.name);
+ }
+ this.name = this.card_faces.get(0).name;
+ }
+
+ // workaround for non ascii names
+ // - scryfall uses original names like Arna Kennerüd, Skycaptain
+ // - xmage need ascii only names like Arna Kennerud, Skycaptain
+ this.name = CardNameUtil.normalizeCardName(this.name);
+
+ // workaround for non scii card numbers
+ // - scryfall uses unicode numbers for reprints like Chandra Nalaar - dd2 - 34★ https://scryfall.com/card/dd2/34%E2%98%85/
+ // - xmage uses ascii alternative Chandra Nalaar - dd2 - 34*
+ this.collector_number = transformCardNumberFromScryfallToXmage(this.collector_number);
+
+ }
+
+ public static String transformCardNumberFromXmageToScryfall(String cardNumber) {
+ String res = cardNumber;
+ if (res.endsWith("*")) {
+ res = res.substring(0, res.length() - 1) + "★";
+ }
+ if (res.endsWith("+")) {
+ res = res.substring(0, res.length() - 1) + "†";
+ }
+ if (res.endsWith("Ph")) {
+ res = res.substring(0, res.length() - 2) + "Φ";
+ }
+ return res;
+ }
+
+ public static String transformCardNumberFromScryfallToXmage(String cardNumber) {
+ String res = cardNumber;
+ if (res.endsWith("★")) {
+ res = res.substring(0, res.length() - 1) + "*";
+ }
+ if (res.endsWith("†")) {
+ res = res.substring(0, res.length() - 1) + "+";
+ }
+ if (res.endsWith("Φ")) {
+ res = res.substring(0, res.length() - 1) + "Ph";
+ }
+ return res;
+ }
+
+ public String findImage(String imageSize) {
+ // api possible values:
+ // - small
+ // - normal
+ // - large
+ // - png
+ // - art_crop
+ // - border_crop
+ switch (imageSize) {
+ case "small":
+ return this.imageSmall;
+ case "normal":
+ return this.imageNormal;
+ case "large":
+ return this.imageLarge;
+ default:
+ throw new IllegalArgumentException("Unsupported image size: " + imageSize);
+ }
+ }
+}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCardFace.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCardFace.java
new file mode 100644
index 00000000000..44fe09a7766
--- /dev/null
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCardFace.java
@@ -0,0 +1,50 @@
+package org.mage.plugins.card.dl.sources;
+
+import java.util.Map;
+
+/**
+ * Scryfall API: card face info (used for double faced cards)
+ *
+ * API docs: here
+ *
+ *
+ * @author JayDi85
+ */
+public class ScryfallApiCardFace {
+ public String name;
+ public Map image_uris;
+
+ // fast access fields, fills on loading
+ transient public String imageSmall = "";
+ transient public String imageNormal = "";
+ transient public String imageLarge = "";
+
+ public void prepareCompatibleData() {
+ if (this.image_uris != null) {
+ this.imageSmall = this.image_uris.getOrDefault("small", "");
+ this.imageNormal = this.image_uris.getOrDefault("normal", "");
+ this.imageLarge = this.image_uris.getOrDefault("large", "");
+ this.image_uris = null;
+ }
+ }
+
+ public String findImage(String imageSize) {
+ // api possible values:
+ // - small
+ // - normal
+ // - large
+ // - png
+ // - art_crop
+ // - border_crop
+ switch (imageSize) {
+ case "small":
+ return this.imageSmall;
+ case "normal":
+ return this.imageNormal;
+ case "large":
+ return this.imageLarge;
+ default:
+ throw new IllegalArgumentException("Unsupported image size: " + imageSize);
+ }
+ }
+}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java
index c562513d2ae..78671efc9ad 100644
--- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java
@@ -1,22 +1,36 @@
package org.mage.plugins.card.dl.sources;
+import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
+import com.google.gson.stream.JsonReader;
import mage.MageException;
import mage.client.remote.XmageURLConnection;
import mage.client.util.CardLanguage;
import mage.util.JsonUtil;
+import net.java.truevfs.access.TFile;
+import net.java.truevfs.access.TFileInputStream;
+import net.java.truevfs.access.TFileOutputStream;
+import net.java.truevfs.access.TVFS;
import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData;
-import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.zip.GZIPInputStream;
+
+import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
/**
+ * Download: scryfall API support to download images and other data
+ *
* @author JayDi85
*/
public class ScryfallImageSource implements CardImageSource {
@@ -32,6 +46,21 @@ public class ScryfallImageSource implements CardImageSource {
private static final ReentrantLock waitBeforeRequestLock = new ReentrantLock();
private static final int DOWNLOAD_TIMEOUT_MS = 300;
+ // bulk images optimization, how it works
+ // - scryfall limit all calls to API endpoints, but allow direct calls to CDN (scryfall.io) without limitation
+ // - so download and prepare whole scryfall database
+ // - for each downloading card try to find it in bulk database and use direct link from it
+ // - only card images supported (without tokens, emblems, etc)
+ private static final boolean SCRYFALL_BULK_FILES_ENABLED = true; // use bulk data to find direct links instead API calls
+ private static final long SCRYFALL_BULK_FILES_OUTDATE_IN_MILLIS = 7 * 24 * 3600 * 1000; // after 1 week need to download again
+ private static final String SCRYFALL_BULK_FILES_DATABASE_SOURCE_API = "https://api.scryfall.com/bulk-data/all-cards"; // 300 MB in zip
+ private static final boolean SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE = false; // default: false - for faster debug only, ignore write operations
+ // run app with -Dxmage.scryfallEnableBulkData=false to disable bulk data (e.g. for testing api links generation)
+ private static final String SCRYFALL_BULK_FILES_PROPERTY = "xmage.scryfallEnableBulkData";
+
+ private static final List bulkCardsDatabaseAll = new ArrayList<>(); // 100MB memory footprint, cleaning on download stop
+ private static final Map bulkCardsDatabaseDefault = new HashMap<>(); // card/set/number
+
public static ScryfallImageSource getInstance() {
return instance;
}
@@ -117,7 +146,7 @@ public class ScryfallImageSource implements CardImageSource {
// basic cards by api call (redirect to img link)
// example: https://api.scryfall.com/cards/xln/121/en?format=image
if (baseUrl == null) {
- String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId());
+ String cn = ScryfallApiCard.transformCardNumberFromXmageToScryfall(card.getCollectorId());
baseUrl = String.format("https://api.scryfall.com/cards/%s/%s/%s?format=image",
formatSetName(card.getSet(), isToken),
cn,
@@ -132,12 +161,12 @@ public class ScryfallImageSource implements CardImageSource {
// include_variations=true added to deal with the cards that scryfall has marked as variations that seem to sometimes fail
// eg https://api.scryfall.com/cards/4ed/134†?format=image fails
// eg https://api.scryfall.com/cards/4ed/134†?format=image&include_variations=true succeeds
-
}
return new CardImageUrls(baseUrl, alternativeUrl);
}
+ // TODO: delete face code after bulk data implemented?
private String getFaceImageUrl(CardDownloadData card, boolean isToken) throws Exception {
final String defaultCode = CardLanguage.ENGLISH.getCode();
final String localizedCode = languageAliases.getOrDefault(this.getCurrentLanguage(), defaultCode);
@@ -147,14 +176,8 @@ public class ScryfallImageSource implements CardImageSource {
String apiUrl = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId());
if (apiUrl != null) {
- if (apiUrl.endsWith("*/")) {
- apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "★/";
- } else if (apiUrl.endsWith("+/")) {
- apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "†/";
- } else if (apiUrl.endsWith("Ph/")) {
- apiUrl = apiUrl.substring(0, apiUrl.length() - 3) + "Φ/";
- }
// BY DIRECT URL
+ // direct url already contains scryfall compatible card numbers (with unicode chars)
// direct links via hardcoded API path. Used for cards with non-ASCII collector numbers
if (localizedCode.equals(defaultCode)) {
// english only, so can use workaround without loc param (scryfall download first available card)
@@ -168,7 +191,7 @@ public class ScryfallImageSource implements CardImageSource {
} else {
// BY CARD NUMBER
// localized and default
- String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId());
+ String cn = ScryfallApiCard.transformCardNumberFromXmageToScryfall(card.getCollectorId());
needUrls.add(String.format("https://api.scryfall.com/cards/%s/%s/%s",
formatSetName(card.getSet(), isToken),
cn,
@@ -221,9 +244,23 @@ public class ScryfallImageSource implements CardImageSource {
@Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) {
- // prepare download list example (
preparedUrls.clear();
+ // direct images without api calls
+ if (!prepareBulkData(downloadServiceInfo, downloadList)) {
+ return false;
+ }
+ analyseBulkData(downloadServiceInfo, downloadList);
+
+ // second side images need lookup for
+ if (!prepareSecondSideImages(downloadServiceInfo, downloadList)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ private boolean prepareSecondSideImages(DownloadServiceInfo downloadServiceInfo, List downloadList) {
// prepare stats
int needPrepareCount = 0;
int currentPrepareCount = 0;
@@ -235,11 +272,16 @@ public class ScryfallImageSource implements CardImageSource {
updatePrepareStats(downloadServiceInfo, needPrepareCount, currentPrepareCount);
for (CardDownloadData card : downloadList) {
- // need cancel
+ // fast cancel
if (downloadServiceInfo.isNeedCancel()) {
return false;
}
+ // already prepare, e.g. on bulk data
+ if (preparedUrls.containsKey(card)) {
+ continue;
+ }
+
// prepare the back face URL
if (card.isSecondSide()) {
currentPrepareCount++;
@@ -259,10 +301,410 @@ public class ScryfallImageSource implements CardImageSource {
return true;
}
+ private boolean prepareBulkData(DownloadServiceInfo downloadServiceInfo, List downloadList) {
+ boolean isEnabled = SCRYFALL_BULK_FILES_ENABLED;
+ if (System.getProperty(SCRYFALL_BULK_FILES_PROPERTY) != null) {
+ isEnabled = Boolean.parseBoolean(System.getProperty(SCRYFALL_BULK_FILES_PROPERTY));
+ }
+ if (!isEnabled) {
+ return true;
+ }
+
+ // if up to date
+ if (isBulkDataPrepared()) {
+ return true;
+ }
+
+ // NEED TO DOWNLOAD
+
+ // clean
+ TFile bulkTempFile = prepareTempFileForBulkData();
+ if (bulkTempFile.exists()) {
+ try {
+ if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
+ bulkTempFile.rm();
+ TVFS.umount();
+ }
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // find actual download link
+ String s = XmageURLConnection.downloadText(SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
+ if (s.isEmpty()) {
+ logger.error("Can't get bulk info from scryfall api " + SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
+ return false;
+ }
+ ScryfallApiBulkData bulkData = new Gson().fromJson(s, ScryfallApiBulkData.class);
+ if (bulkData == null
+ || bulkData.download_uri == null
+ || bulkData.download_uri.isEmpty()
+ || bulkData.size == 0) {
+ logger.error("Unknown bulk info format from scryfall api " + SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
+ return false;
+ }
+
+ // download
+ if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
+ logger.info("Scryfall: downloading additional data files, size " + bulkData.size / (1024 * 1024) + " MB");
+ String url = bulkData.download_uri;
+ InputStream inputStream = XmageURLConnection.downloadBinary(url, true);
+ OutputStream outputStream = null;
+ try {
+ try {
+ if (inputStream == null) {
+ logger.error("Can't get bulk data from scryfall api " + url);
+ return false;
+ }
+ outputStream = new TFileOutputStream(bulkTempFile);
+
+ byte[] buf = new byte[5 * 1024 * 1024]; // 5 MB buffer
+ int len;
+ long needDownload = bulkData.size / 7; // text -> zip size multiplier
+ long doneDownload = 0;
+ while ((len = inputStream.read(buf)) != -1) {
+ // fast cancel
+ if (downloadServiceInfo.isNeedCancel()) {
+ // stop download and delete current file
+ inputStream.close();
+ outputStream.close();
+ if (bulkTempFile.exists()) {
+ bulkTempFile.rm();
+ }
+ return false;
+ }
+
+ // all fine, can save data part
+ outputStream.write(buf, 0, len);
+
+ doneDownload += len;
+ updateBulkDownloadStats(downloadServiceInfo, doneDownload, needDownload);
+ }
+ outputStream.flush();
+
+ // unpack
+ logger.info("Scryfall: unpacking files...");
+ Path zippedBulkPath = Paths.get(getBulkTempFileName());
+ Path textBulkPath = Paths.get(getBulkStaticFileName());
+ Files.deleteIfExists(textBulkPath);
+ try (GZIPInputStream in = new GZIPInputStream(Files.newInputStream(zippedBulkPath));
+ FileOutputStream out = new FileOutputStream(textBulkPath.toString())) {
+ byte[] buffer = new byte[5 * 1024 * 1024];
+ long needUnpack = bulkData.size; // zip -> text
+ long doneUnpack = 0;
+ while ((len = in.read(buffer)) > 0) {
+ // fast cancel
+ if (downloadServiceInfo.isNeedCancel()) {
+ out.flush();
+ Files.deleteIfExists(textBulkPath);
+ return false;
+ }
+
+ out.write(buffer, 0, len);
+
+ doneUnpack += len;
+ updateBulkUnpackingStats(downloadServiceInfo, doneUnpack, needUnpack);
+ }
+ }
+ logger.info("Scryfall: unpacking done");
+
+ // all fine
+ } finally {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ if (outputStream != null) {
+ outputStream.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("Catch unknown error while download bulk data from scryfall api " + url, e);
+ return false;
+ }
+ }
+
+ return isBulkDataPrepared();
+ }
+
+ private boolean isBulkDataPrepared() {
+
+ // already loaded
+ if (bulkCardsDatabaseAll.size() > 0) {
+ return true;
+ }
+
+ // file not exists
+ Path textBulkPath = Paths.get(getBulkStaticFileName());
+ if (!Files.exists(textBulkPath)) {
+ return false;
+ }
+
+ // file outdated
+ try {
+ if (System.currentTimeMillis() - Files.getLastModifiedTime(textBulkPath).toMillis() > SCRYFALL_BULK_FILES_OUTDATE_IN_MILLIS) {
+ logger.info("Scryfall: bulk files outdated, need to download new");
+ return false;
+ }
+ } catch (IOException ignore) {
+ }
+
+ // bulk files are too big, so must read and parse it in stream mode only
+ // 500k reprints in diff languages
+ Gson gson = new Gson();
+ try (TFileInputStream inputStream = new TFileInputStream(textBulkPath.toFile());
+ InputStreamReader inputReader = new InputStreamReader(inputStream);
+ JsonReader jsonReader = new JsonReader(inputReader)) {
+
+ bulkCardsDatabaseAll.clear();
+ bulkCardsDatabaseDefault.clear();
+
+ jsonReader.beginArray();
+ while (jsonReader.hasNext()) {
+ ScryfallApiCard card = gson.fromJson(jsonReader, ScryfallApiCard.class);
+
+ // prepare data
+ // memory optimization: fewer data, from 1145 MB to 470 MB
+ card.prepareCompatibleData();
+
+ // keep only usefully languages
+ // memory optimization: fewer items, from 470 MB to 96 MB
+ // make sure final list will have one version
+ // example: japan only card must be in final list for any selected languages https://scryfall.com/card/war/180★/
+ String key = String.format("%s/%s/%s", card.name, card.set, card.collector_number);
+ if (!card.lang.equals(this.currentLanguage.getCode())
+ && !card.lang.equals(CardLanguage.ENGLISH.getCode())
+ && bulkCardsDatabaseDefault.containsKey(key)) {
+ continue;
+ }
+
+ // default versions list depends on scryfall bulk data order (en first)
+ bulkCardsDatabaseDefault.put(key, card);
+ bulkCardsDatabaseAll.add(card);
+ }
+ jsonReader.close();
+ return bulkCardsDatabaseAll.size() > 0;
+ } catch (Exception e) {
+ logger.error("Can't read bulk file (possible reason: broken format)");
+ try {
+ // clean up
+ if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
+ Files.deleteIfExists(textBulkPath);
+ }
+ } catch (IOException ignore) {
+ }
+ }
+ return false;
+ }
+
+ private void analyseBulkData(DownloadServiceInfo downloadServiceInfo, List downloadList) {
+ // prepare card indexes
+ // name/set/number/lang - normal cards
+ Map bulkImagesIndexAll = createBulkImagesIndex(downloadServiceInfo, bulkCardsDatabaseAll, true);
+ // name/set/number - default cards
+ Map bulkImagesIndexDefault = createBulkImagesIndex(downloadServiceInfo, bulkCardsDatabaseDefault.values(), false);
+
+ // find good images
+ AtomicInteger statsOldPrepared = new AtomicInteger();
+ AtomicInteger statsDirectLinks = new AtomicInteger();
+ AtomicInteger statsNewPrepared = new AtomicInteger();
+ AtomicInteger statsNotFound = new AtomicInteger();
+ for (CardDownloadData card : downloadList) {
+ // fast cancel
+ if (downloadServiceInfo.isNeedCancel()) {
+ return;
+ }
+
+ // already prepared, e.g. from direct download links or other sources
+ if (preparedUrls.containsKey(card)) {
+ statsOldPrepared.incrementAndGet();
+ continue;
+ }
+
+ // only cards supported
+ if (card.isToken()) {
+ continue;
+ }
+
+ // ignore direct links
+ String directLink = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId());
+ if (directLink != null) {
+ statsDirectLinks.incrementAndGet();
+ continue;
+ }
+
+ String searchingName = card.getName();
+ String searchingSet = card.getSet().toLowerCase(Locale.ENGLISH);
+ String searchingNumber = card.getCollectorId();
+
+ // current language
+ String key = String.format("%s/%s/%s/%s",
+ searchingName,
+ searchingSet,
+ searchingNumber,
+ this.currentLanguage
+ );
+ String link = bulkImagesIndexAll.getOrDefault(key, "");
+
+ // default en language
+ if (link.isEmpty()) {
+ key = String.format("%s/%s/%s/%s",
+ searchingName,
+ searchingSet,
+ searchingNumber,
+ CardLanguage.ENGLISH.getCode()
+ );
+ link = bulkImagesIndexAll.getOrDefault(key, "");
+ }
+
+ // default non en-language
+ if (link.isEmpty()) {
+ key = String.format("%s/%s/%s/",
+ searchingName,
+ searchingSet,
+ searchingNumber);
+ link = bulkImagesIndexDefault.getOrDefault(key, "");
+ }
+
+ if (link.isEmpty()) {
+ // how-to fix:
+ // - make sure name notation compatible, see workarounds in ScryfallApiCard.prepareCompatibleData
+ // - makue sure it's not fake card (xmage can use fake cards instead doubled-images) -- add it to ScryfallImageSupportCards.directDownloadLinks
+ logger.info("Scryfall: bulk image not found (outdated cards data in xmage?): " + key + " ");
+ statsNotFound.incrementAndGet();
+ } else {
+ preparedUrls.put(card, link);
+ statsNewPrepared.incrementAndGet();
+ }
+ }
+
+ logger.info(String.format("Scryfall: bulk optimization result - need cards: %d, already prepared: %d, direct links: %d, new prepared: %d, NOT FOUND: %d",
+ downloadList.size(),
+ statsOldPrepared.get(),
+ statsDirectLinks.get(),
+ statsNewPrepared.get(),
+ statsNotFound.get() // all not founded cards must be fixed
+ ));
+ }
+
+ private Map createBulkImagesIndex(DownloadServiceInfo downloadServiceInfo, Collection sourceList, boolean useLocalization) {
+ Map res = new HashMap<>();
+ for (ScryfallApiCard card : sourceList) {
+ // fast cancel
+ if (downloadServiceInfo.isNeedCancel()) {
+ return res;
+ }
+
+ // main card
+ String image = card.findImage(getImageQuality());
+ if (!image.isEmpty()) {
+ String mainKey = String.format("%s/%s/%s/%s",
+ card.name,
+ card.set,
+ card.collector_number,
+ useLocalization ? card.lang : ""
+ );
+ if (res.containsKey(mainKey)) {
+ // how-to fix: rework whole card number logic in xmage and scryfall
+ throw new IllegalArgumentException("Wrong code usage: scryfall used unique card numbers, but found duplicated: " + mainKey);
+ }
+ res.put(mainKey, image);
+ }
+
+ // faces
+ if (card.card_faces != null) {
+ // hints:
+ // 1. Not all faces has images - example: adventure cards
+ // 2. Some faces contain fake data in second face, search it on scryfall by "set_type:memorabilia"
+ // example: https://scryfall.com/card/aznr/25/clearwater-pathway-clearwater-pathway
+ // 3. Some faces contain diff images of the same card, layout = reversible_card
+ // example: https://scryfall.com/card/rex/26/command-tower-command-tower
+
+ Set usedNames = new HashSet<>();
+ card.card_faces.forEach(face -> {
+ // workaround to ignore fake data, see above about memorabilia
+ if (usedNames.contains(face.name)) {
+ return;
+ }
+ usedNames.add(face.name);
+
+ String faceImage = face.findImage(getImageQuality());
+ if (!faceImage.isEmpty()) {
+ String faceKey = String.format("%s/%s/%s/%s",
+ face.name,
+ card.set,
+ card.collector_number,
+ useLocalization ? card.lang : ""
+ );
+
+ // workaround for flip cards - ignore first face - it already in the index
+ // example: https://scryfall.com/card/sok/103/homura-human-ascendant-homuras-essence
+ if (card.layout.equals("flip")
+ && card.name.equals(face.name)
+ && res.containsKey(faceKey)) {
+ return;
+ }
+
+ if (res.containsKey(faceKey)) {
+ // how-to fix: rework whole card number logic in xmage and scryfall
+ throw new IllegalArgumentException("Wrong code usage: scryfall used unique card numbers, but found duplicated: " + faceKey);
+ }
+ res.put(faceKey, faceImage);
+ }
+ });
+ }
+ }
+
+ return res;
+ }
+
+ private String getBulkTempFileName() {
+ return getImagesDir() + File.separator + "downloading" + File.separator + "scryfall_bulk_cards.json.gz";
+ }
+
+ private String getBulkStaticFileName() {
+ return getImagesDir() + File.separator + "downloading" + File.separator + "scryfall_bulk_cards.json";
+ }
+
+ private TFile prepareTempFileForBulkData() {
+ TFile file = new TFile(getBulkTempFileName());
+ TFile parent = file.getParentFile();
+ if (parent != null) {
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
+ }
+ return file;
+ }
+
+ private void updateBulkDownloadStats(DownloadServiceInfo service, long done, long need) {
+ int doneMb = (int) (done / (1024 * 1024));
+ int needMb = (int) (need / (1024 * 1024));
+ synchronized (service.getSync()) {
+ service.updateProgressMessage(
+ String.format("Step 1 of 3. Downloading additional data... %d of %d MB", doneMb, needMb),
+ doneMb,
+ needMb
+ );
+ }
+ }
+
+ private void updateBulkUnpackingStats(DownloadServiceInfo service, long done, long need) {
+ int doneMb = (int) (done / (1024 * 1024));
+ int needMb = (int) (need / (1024 * 1024));
+ synchronized (service.getSync()) {
+ service.updateProgressMessage(
+ String.format("Step 2 of 3. Unpacking additional files... %d of %d MB", doneMb, needMb),
+ doneMb,
+ needMb
+ );
+ }
+ }
+
private void updatePrepareStats(DownloadServiceInfo service, int need, int current) {
synchronized (service.getSync()) {
service.updateProgressMessage(
- String.format("Preparing download list... %d of %d", current, need),
+ String.format("Step 3 of 3. Preparing download list... %d of %d", current, need),
current,
need
);
@@ -334,7 +776,12 @@ public class ScryfallImageSource implements CardImageSource {
public void doPause(String fullUrl) {
// scryfall recommends 300 ms timeout per each request to API to work under a rate limit
// possible error: 429 Too Many Requests
- // TODO: add diff endpoint supports (api calls with timeout, cdn/direct calls without timeout)
+
+ // cdn source can be safe and called without pause
+ if (fullUrl.contains("scryfall.io")) {
+ return;
+ }
+
waitBeforeRequest();
}
@@ -387,4 +834,18 @@ public class ScryfallImageSource implements CardImageSource {
// only direct tokens from set
return ScryfallImageSupportTokens.findTokenLink(setCode, cardName, tokenNumber) != null;
}
+
+ /**
+ * Image quality
+ */
+ public String getImageQuality() {
+ return "large";
+ }
+
+ @Override
+ public void onFinished() {
+ // cleanup resources
+ bulkCardsDatabaseAll.clear();
+ bulkCardsDatabaseDefault.clear();
+ }
}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceNormal.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceNormal.java
index 04d9a085c13..3d049cb3dde 100644
--- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceNormal.java
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceNormal.java
@@ -11,19 +11,19 @@ import java.util.stream.Collectors;
public class ScryfallImageSourceNormal extends ScryfallImageSource {
private static final ScryfallImageSourceNormal instanceNormal = new ScryfallImageSourceNormal();
-
+
public static ScryfallImageSource getInstance() {
return instanceNormal;
}
private static String innerModifyUrlString(String oneUrl) {
- return oneUrl.replaceFirst("/large/","/normal/").replaceFirst("format=image","format=image&version=normal");
+ return oneUrl.replaceFirst("/large/", "/normal/").replaceFirst("format=image", "format=image&version=normal");
}
private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) {
- List downloadUrls = cardUrls.getDownloadList().stream()
- .map(ScryfallImageSourceNormal::innerModifyUrlString)
- .collect(Collectors.toList());
+ List downloadUrls = cardUrls.getDownloadList().stream()
+ .map(ScryfallImageSourceNormal::innerModifyUrlString)
+ .collect(Collectors.toList());
return new CardImageUrls(downloadUrls);
}
@@ -48,4 +48,8 @@ public class ScryfallImageSourceNormal extends ScryfallImageSource {
return 92f;
}
+ @Override
+ public String getImageQuality() {
+ return "normal";
+ }
}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceSmall.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceSmall.java
index 29d86c97287..a3c54c6c506 100644
--- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceSmall.java
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceSmall.java
@@ -11,19 +11,19 @@ import java.util.stream.Collectors;
public class ScryfallImageSourceSmall extends ScryfallImageSource {
private static final ScryfallImageSourceSmall instanceSmall = new ScryfallImageSourceSmall();
-
+
public static ScryfallImageSource getInstance() {
return instanceSmall;
}
private static String innerModifyUrlString(String oneUrl) {
- return oneUrl.replaceFirst("/large/","/small/").replaceFirst("format=image","format=image&version=small");
+ return oneUrl.replaceFirst("/large/", "/small/").replaceFirst("format=image", "format=image&version=small");
}
private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) {
- List downloadUrls = cardUrls.getDownloadList().stream()
- .map(ScryfallImageSourceSmall::innerModifyUrlString)
- .collect(Collectors.toList());
+ List downloadUrls = cardUrls.getDownloadList().stream()
+ .map(ScryfallImageSourceSmall::innerModifyUrlString)
+ .collect(Collectors.toList());
return new CardImageUrls(downloadUrls);
}
@@ -48,4 +48,8 @@ public class ScryfallImageSourceSmall extends ScryfallImageSource {
return 14f;
}
+ @Override
+ public String getImageQuality() {
+ return "small";
+ }
}
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSupportCards.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSupportCards.java
index 3f68b310cc6..e44d9f2cc6f 100644
--- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSupportCards.java
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSupportCards.java
@@ -681,19 +681,6 @@ public class ScryfallImageSupportCards {
return null;
}
- public static String prepareCardNumber(String xmageCardNumber) {
- if (xmageCardNumber.endsWith("*")) {
- return xmageCardNumber.substring(0, xmageCardNumber.length() - 1) + "★";
- }
- if (xmageCardNumber.endsWith("+")) {
- return xmageCardNumber.substring(0, xmageCardNumber.length() - 1) + "†";
- }
- if (xmageCardNumber.endsWith("Ph")) {
- return xmageCardNumber.substring(0, xmageCardNumber.length() - 2) + "Φ";
- }
- return xmageCardNumber;
- }
-
public static String findDirectDownloadLink(String setCode, String cardName, String cardNumber) {
String key = findDirectDownloadKey(setCode, cardName, cardNumber);
return directDownloadLinks.get(key);
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java b/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java
index ec740414bee..e18c499ab2e 100644
--- a/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java
@@ -66,8 +66,8 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
// protect from wrong data save
// there are possible land images with small sizes, so must research content in check
- private static final int MIN_FILE_SIZE_OF_GOOD_IMAGE = 1024 * 6; // broken
- private static final int MIN_FILE_SIZE_OF_POSSIBLE_BAD_IMAGE = 1024 * 15; // possible broken (need content check)
+ private static final int MIN_FILE_SIZE_OF_GOOD_IMAGE = 1024 * 6; // smaller files will be mark as broken
+ private static final int MIN_FILE_SIZE_OF_POSSIBLE_BAD_IMAGE = 1024 * 8; // smaller files will be checked for possible broken mark (slow)
private final DownloadImagesDialog uiDialog;
private boolean needCancel;
@@ -77,8 +77,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
private List cardsAll;
private List cardsMissing;
private final List cardsDownloadQueue;
- private int missingCardsCount = 0;
- private int missingTokensCount = 0;
private final List selectedSets = new ArrayList<>();
private static CardImageSource selectedSource;
@@ -429,13 +427,13 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
}
private void updateProgressText(int cardCount, int tokenCount) {
- missingTokensCount = 0;
+ int missingTokensCount = 0;
for (CardDownloadData card : cardsMissing) {
if (card.isToken()) {
missingTokensCount++;
}
}
- missingCardsCount = cardsMissing.size() - missingTokensCount;
+ int missingCardsCount = cardsMissing.size() - missingTokensCount;
uiDialog.setCurrentInfo("Missing: " + missingCardsCount + " card images / " + missingTokensCount + " token images");
int imageSum = cardCount + tokenCount;
@@ -446,7 +444,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
} else if (cardIndex == 0) {
statusEnd = "image download NOT STARTED. Please start.";
} else {
- statusEnd = String.format("image downloading... Please wait [%.1f Mb left].", mb);
+ statusEnd = String.format("image downloading... Please wait [%.1f MB left].", mb);
}
updateProgressMessage(String.format("%d of %d (%d cards and %d tokens) %s",
0, imageSum, cardCount, tokenCount, statusEnd
@@ -618,7 +616,8 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
}
});
- if (badContentChecks.get() > 10000) {
+ if (badContentChecks.get() > 1000) {
+ // how-to fix: download full small images and run checks -- make sure it finds fewer files to check
logger.warn("Wrong code usage: too many file content checks (" + badContentChecks.get() + ") - try to decrease min file size");
}
@@ -630,86 +629,93 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
this.cardIndex = 0;
this.resetErrorCount();
- File base = new File(getImagesDir());
- if (!base.exists()) {
- base.mkdir();
- }
-
- int downloadThreadsAmount = Math.max(1, Integer.parseInt((String) uiDialog.getDownloadThreadsCombo().getSelectedItem()));
-
-
- logger.info("Started download of " + cardsDownloadQueue.size() + " images"
- + " from source: " + selectedSource.getSourceName()
- + ", language: " + selectedSource.getCurrentLanguage().getCode()
- + ", threads: " + downloadThreadsAmount);
- updateProgressMessage("Preparing download list...");
- if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) {
- update(0, cardsDownloadQueue.size());
- ExecutorService executor = Executors.newFixedThreadPool(
- downloadThreadsAmount,
- new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_IMAGES_DOWNLOADER, false)
- );
- for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) {
- try {
- CardDownloadData card = cardsDownloadQueue.get(i);
-
- logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')');
-
- CardImageUrls urls;
- if (card.isToken()) {
- if (!"0".equals(card.getCollectorId())) {
- continue;
- }
- urls = selectedSource.generateTokenUrl(card);
- } else {
- urls = selectedSource.generateCardUrl(card);
- }
-
- if (urls == null) {
- String imageRef = selectedSource.getNextHttpImageUrl();
- String fileName = selectedSource.getFileForHttpImage(imageRef);
- if (imageRef != null && fileName != null) {
- imageRef = selectedSource.getSourceName() + imageRef;
- try {
- card.setToken(selectedSource.isTokenSource());
- Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages());
- executor.execute(task);
- } catch (Exception ex) {
- }
- } else if (selectedSource.getTotalImages() == -1) {
- logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')');
- synchronized (sync) {
- update(cardIndex + 1, cardsDownloadQueue.size());
- }
- }
- } else {
- Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size());
- executor.execute(task);
- }
- } catch (Exception ex) {
- logger.error(ex, ex);
- }
- }
-
- executor.shutdown();
- while (!executor.isTerminated()) {
- try {
- TimeUnit.SECONDS.sleep(1);
- } catch (InterruptedException ignore) {
- }
- }
- }
-
-
try {
- TVFS.umount();
- } catch (FsSyncException e) {
- logger.fatal("Couldn't unmount zip files", e);
- JOptionPane.showMessageDialog(null, "Couldn't unmount zip files", "Error", JOptionPane.ERROR_MESSAGE);
+ File base = new File(getImagesDir());
+ if (!base.exists()) {
+ base.mkdir();
+ }
+
+ int downloadThreadsAmount = Math.max(1, Integer.parseInt((String) uiDialog.getDownloadThreadsCombo().getSelectedItem()));
+
+ logger.info("Started download of " + cardsDownloadQueue.size() + " images"
+ + " from source: " + selectedSource.getSourceName()
+ + ", language: " + selectedSource.getCurrentLanguage().getCode()
+ + ", threads: " + downloadThreadsAmount);
+ updateProgressMessage("Preparing download list...");
+ if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) {
+ update(0, cardsDownloadQueue.size());
+ ExecutorService executor = Executors.newFixedThreadPool(
+ downloadThreadsAmount,
+ new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_IMAGES_DOWNLOADER, false)
+ );
+ for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) {
+ try {
+ CardDownloadData card = cardsDownloadQueue.get(i);
+
+ logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')');
+
+ CardImageUrls urls;
+ if (card.isToken()) {
+ if (!"0".equals(card.getCollectorId())) {
+ continue;
+ }
+ urls = selectedSource.generateTokenUrl(card);
+ } else {
+ urls = selectedSource.generateCardUrl(card);
+ }
+
+ if (urls == null) {
+ String imageRef = selectedSource.getNextHttpImageUrl();
+ String fileName = selectedSource.getFileForHttpImage(imageRef);
+ if (imageRef != null && fileName != null) {
+ imageRef = selectedSource.getSourceName() + imageRef;
+ try {
+ card.setToken(selectedSource.isTokenSource());
+ Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages());
+ executor.execute(task);
+ } catch (Exception ex) {
+ }
+ } else if (selectedSource.getTotalImages() == -1) {
+ logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')');
+ synchronized (sync) {
+ update(cardIndex + 1, cardsDownloadQueue.size());
+ }
+ }
+ } else {
+ Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size());
+ executor.execute(task);
+ }
+ } catch (Exception ex) {
+ logger.error(ex, ex);
+ }
+ }
+
+ executor.shutdown();
+ while (!executor.isTerminated()) {
+ try {
+ TimeUnit.SECONDS.sleep(1);
+ } catch (InterruptedException ignore) {
+ }
+ }
+ }
+ } catch (Throwable e) {
+ logger.error("Catch unknown error while downloading: " + e, e);
+ MageFrame.getInstance().showErrorDialog("Catch unknown error while downloading " + e, e);
} finally {
- //
+ // must close all active archives anyway
+ try {
+ TVFS.umount();
+ } catch (FsSyncException e) {
+ logger.fatal("Couldn't unmount zip files " + e, e);
+ MageFrame.getInstance().showErrorDialog("Couldn't unmount zip files " + e, e);
+ }
}
+ // cleanup resources
+ Arrays.stream(DownloadSources.values()).forEach(resource -> {
+ resource.source.onFinished();
+ });
+
// stop
reloadCardsToDownload(uiDialog.getSetsCombo().getSelectedItem().toString());
enableDialogButtons();
@@ -882,16 +888,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
// user cancelled
if (DownloadPicturesService.getInstance().isNeedCancel()) {
// stop download, save current state and exit
- TFile archive = destFile.getTopLevelArchive();
- ///* not need to unmout/close - it's auto action
- if (archive != null && archive.exists()) {
- logger.info("User canceled download. Closing archive file: " + destFile.toString());
- try {
- TVFS.umount(archive);
- } catch (Exception e) {
- logger.error("Can't close archive file: " + e.getMessage(), e);
- }
- }
+ // real archives save will be done on download service finish
try {
TFile.rm(fileTempImage);
} catch (Exception e) {
@@ -946,11 +943,11 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
if (cardIndex < needDownloadCount) {
// downloading
float mb = ((needDownloadCount - lastCardIndex) * selectedSource.getAverageSizeKb()) / 1024;
- updateProgressMessage(String.format("%d of %d image downloading... Please wait [%.1f Mb left].",
+ updateProgressMessage(String.format("%d of %d image downloading... Please wait [%.1f MB left].",
lastCardIndex, needDownloadCount, mb), lastCardIndex, needDownloadCount);
} else {
// finished
- updateProgressMessage("Image download DONE, refreshing stats... Please wait.");
+ updateProgressMessage("Image download DONE, saving last files and refreshing stats... Please wait.");
List downloadedCards = Collections.synchronizedList(new ArrayList<>());
DownloadPicturesService.this.cardsMissing.parallelStream().forEach(cardDownloadData -> {
TFile file = new TFile(CardImageUtils.buildImagePathToCardOrToken(cardDownloadData));
diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java b/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java
index e61012c8019..3ded259a466 100644
--- a/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java
+++ b/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java
@@ -1,28 +1,16 @@
package org.mage.plugins.card.utils;
-import mage.cards.repository.CardInfo;
-import mage.cards.repository.CardRepository;
import mage.cards.repository.TokenRepository;
-import mage.client.MageFrame;
import mage.client.constants.Constants;
import mage.client.dialog.PreferencesDialog;
-import mage.client.remote.XmageURLConnection;
-import mage.remote.Connection;
-import mage.remote.Connection.ProxyType;
import mage.view.CardView;
+import net.java.truevfs.access.TVFS;
+import net.java.truevfs.kernel.spec.FsSyncException;
import org.apache.log4j.Logger;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
import org.mage.plugins.card.images.CardDownloadData;
-import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.URL;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -31,7 +19,6 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Locale;
-import java.util.prefs.Preferences;
public final class CardImageUtils {
@@ -191,19 +178,16 @@ public final class CardImageUtils {
return getImagesDir() + File.separator + "FACE" + File.separator + fixSetNameForWindows(setCode) + File.separator + prepareCardNameForFile(cardName) + ".jpg";
}
- public static Proxy getProxyFromPreferences() {
- Preferences prefs = MageFrame.getPreferences();
- Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None"));
- if (proxyType != ProxyType.NONE) {
- String proxyServer = prefs.get("proxyAddress", "");
- int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0"));
- return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort));
- }
- return null;
- }
-
public static void checkAndFixImageFiles() {
// search broken, temp or outdated files and delete it
+
+ // make sure all archives was closed (e.g. on second call of download dialog)
+ try {
+ TVFS.umount();
+ } catch (FsSyncException e) {
+ LOGGER.fatal("Couldn't unmount zip files on searching broken images " + e, e);
+ }
+
// real images check is slow, so it used on images download only (not here)
Path rootPath = new File(CardImageUtils.getImagesDir()).toPath();
if (!Files.exists(rootPath)) {
diff --git a/Mage.Plugins/Mage.Counter.Plugin/src/main/java/org/mage/plugins/counter/CounterPluginImpl.java b/Mage.Plugins/Mage.Counter.Plugin/src/main/java/org/mage/plugins/counter/CounterPluginImpl.java
index b1d067c3281..5cf3512ef0b 100644
--- a/Mage.Plugins/Mage.Counter.Plugin/src/main/java/org/mage/plugins/counter/CounterPluginImpl.java
+++ b/Mage.Plugins/Mage.Counter.Plugin/src/main/java/org/mage/plugins/counter/CounterPluginImpl.java
@@ -9,13 +9,14 @@ import net.xeoh.plugins.base.annotations.meta.Author;
import org.apache.log4j.Logger;
import java.io.*;
+import java.nio.file.Files;
/**
* Implementation of {@link CounterPlugin}.
- * Stores data in data folder.
- *
- * @version 0.1 14.11.2010 Initial Version
+ * Stores data in data folder.
+ *
* @author nantuko
+ * @version 0.1 14.11.2010 Initial Version
*/
@PluginImplementation
@Author(name = "nantuko")
@@ -69,11 +70,11 @@ public class CounterPluginImpl implements CounterPlugin {
if (data.exists()) {
int prev = 0;
- try(ObjectInputStream ois = new ObjectInputStream(new FileInputStream(data))) {
+ try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(data.toPath()))) {
Object o = ois.readObject();
CounterBean c;
if (o instanceof CounterBean) {
- c = (CounterBean)o;
+ c = (CounterBean) o;
prev = c.getGamesPlayed();
}
} catch (EOFException e) {
@@ -84,10 +85,10 @@ public class CounterPluginImpl implements CounterPlugin {
throw new PluginException(e);
}
- try(ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(data))) {
+ try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(data))) {
synchronized (this) {
CounterBean c = new CounterBean();
- c.setGamesPlayed(prev+1);
+ c.setGamesPlayed(prev + 1);
oos.writeObject(c);
oos.close();
}
@@ -107,12 +108,12 @@ public class CounterPluginImpl implements CounterPlugin {
return 0;
}
if (data.exists()) {
- try(ObjectInputStream ois = new ObjectInputStream(new FileInputStream(data))) {
+ try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(data.toPath()))) {
synchronized (this) {
Object o = ois.readObject();
CounterBean c = null;
if (o instanceof CounterBean) {
- c = (CounterBean)o;
+ c = (CounterBean) o;
}
ois.close();
return c == null ? 0 : c.getGamesPlayed();
diff --git a/Mage.Server/src/main/java/mage/server/game/GameReplay.java b/Mage.Server/src/main/java/mage/server/game/GameReplay.java
index 29c877ead68..fb0a68c6902 100644
--- a/Mage.Server/src/main/java/mage/server/game/GameReplay.java
+++ b/Mage.Server/src/main/java/mage/server/game/GameReplay.java
@@ -1,25 +1,23 @@
-
-
package mage.server.game;
-import java.io.BufferedInputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInput;
-import java.util.UUID;
-import java.util.zip.GZIPInputStream;
import mage.game.Game;
import mage.game.GameState;
import mage.game.GameStates;
import mage.server.Main;
import mage.util.CopierObjectInputStream;
-import mage.utils.StreamUtils;
import org.apache.log4j.Logger;
-
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInput;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.UUID;
+import java.util.zip.GZIPInputStream;
/**
+ * Replay system, outdated and not used. TODO: delete
*
* @author BetaSteward_at_googlemail.com
*/
@@ -59,33 +57,19 @@ public class GameReplay {
}
private Game loadGame(UUID gameId) {
- InputStream file = null;
- InputStream buffer = null;
- InputStream gzip = null;
- ObjectInput input = null;
- try{
- file = new FileInputStream("saved/" + gameId.toString() + ".game");
- buffer = new BufferedInputStream(file);
- gzip = new GZIPInputStream(buffer);
- input = new CopierObjectInputStream(Main.classLoader, gzip);
+ try (InputStream file = Files.newInputStream(Paths.get("saved/" + gameId.toString() + ".game"));
+ InputStream buffer = new BufferedInputStream(file);
+ InputStream gzip = new GZIPInputStream(buffer);
+ ObjectInput input = new CopierObjectInputStream(Main.classLoader, gzip)) {
Game loadGame = (Game) input.readObject();
GameStates states = (GameStates) input.readObject();
loadGame.loadGameStates(states);
return loadGame;
-
- }
- catch(ClassNotFoundException ex) {
- logger.fatal("Cannot load game. Class not found.", ex);
- }
- catch(IOException ex) {
- logger.fatal("Cannot load game:" + gameId, ex);
- } finally {
- StreamUtils.closeQuietly(file);
- StreamUtils.closeQuietly(buffer);
- StreamUtils.closeQuietly(input);
- StreamUtils.closeQuietly(gzip);
+ } catch (ClassNotFoundException e) {
+ logger.fatal("Cannot load game. Class not found.", e);
+ } catch (IOException e) {
+ logger.fatal("Cannot load game:" + gameId, e);
}
return null;
}
-
}
diff --git a/Mage.Sets/src/mage/sets/SecretLairDrop.java b/Mage.Sets/src/mage/sets/SecretLairDrop.java
index 198ad514308..eda198a888c 100644
--- a/Mage.Sets/src/mage/sets/SecretLairDrop.java
+++ b/Mage.Sets/src/mage/sets/SecretLairDrop.java
@@ -756,7 +756,7 @@ public class SecretLairDrop extends ExpansionSet {
cards.add(new SetCardInfo("Elvish Mystic", 805, Rarity.RARE, mage.cards.e.ElvishMystic.class, NON_FULL_USE_VARIOUS));
cards.add(new SetCardInfo("Snapcaster Mage", 808, Rarity.MYTHIC, mage.cards.s.SnapcasterMage.class));
cards.add(new SetCardInfo("Thrun, the Last Troll", 810, Rarity.MYTHIC, mage.cards.t.ThrunTheLastTroll.class));
- //cards.add(new SetCardInfo("Elesh Norn, Grand Cenobite", 811, Rarity.MYTHIC, mage.cards.e.EleshNornGrandCenobite.class, NON_FULL_USE_VARIOUS)); // BUG: breaks download of card 209
+ cards.add(new SetCardInfo("Elesh Norn, Grand Cenobite", 811, Rarity.MYTHIC, mage.cards.e.EleshNornGrandCenobite.class, NON_FULL_USE_VARIOUS)); // BUG: breaks download of card 209
cards.add(new SetCardInfo("Arcane Signet", 820, Rarity.RARE, mage.cards.a.ArcaneSignet.class, NON_FULL_USE_VARIOUS));
cards.add(new SetCardInfo("Norin the Wary", 827, Rarity.RARE, mage.cards.n.NorinTheWary.class));
cards.add(new SetCardInfo("The Scarab God", 900, Rarity.MYTHIC, mage.cards.t.TheScarabGod.class));
diff --git a/Mage/src/main/java/mage/players/net/UserData.java b/Mage/src/main/java/mage/players/net/UserData.java
index 1b99b4d2c59..551abf674df 100644
--- a/Mage/src/main/java/mage/players/net/UserData.java
+++ b/Mage/src/main/java/mage/players/net/UserData.java
@@ -26,7 +26,7 @@ public class UserData implements Serializable {
protected int autoTargetLevel;
protected boolean useSameSettingsForReplacementEffects;
protected boolean useFirstManaAbility = false;
- private String userIdStr;
+ private String userIdStr; // TODO: delete as un-used or use for hardware id instead?
protected Map> requestedHandPlayersList; // game -> players list
protected String matchHistory;