External/mage
1
0
Fork 1
mirror of https://github.com/magefree/mage.git synced 2025-12-20 02:30:08 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

download: reworked scryfall images support:

Browse source 
- download: fixed unmount zip errors on cancel download in some use cases (closes #12536);
- download: significant download speed improvements (now it depends on user's network speed, not api limitations);
- download: added additional error dialogs on bad use cases;
- scryfall: added cards and bulk data api support;
- scryfall: added bulk data download (updates once per week, contains all scryfall cards and store in images\downloading folder, 2 GB size);
- scryfall: added optimized images download without api usage (use direct images links from bulk data, closes #11576);
- scryfall: improved image source searching for some use cases (miss or wrong images problems, closes #12511);
- scryfall: tokens don't use bulk data;
- scryfall: 75k small images downloads 40 minutes and takes 1 GB and 2100 api calls (most of it from tokens);
- scryfall: how-to disable bulk data, e.g. for api testing: -Dxmage.scryfallEnableBulkData=false
This commit is contained in:
Oleg Agafonov 2024-08-03 19:41:14 +04:00
parent 46f7304692
commit 0a55e37c8c
19 changed files with 884 additions and 216 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Mage.Client/src/main/java/mage/client/MageFrame.java
View file

@ -1355,7 +1355,7 @@ public class MageFrame extends javax.swing.JFrame implements MageClient {
userRequestDialog.showDialog(userRequestMessage); userRequestDialog.showDialog(userRequestMessage);
} }
public void showErrorDialog(String errorType, Exception e) { public void showErrorDialog(String errorType, Throwable e) {
String errorMessage = e.getMessage(); String errorMessage = e.getMessage();
if (errorMessage == null || errorMessage.isEmpty() || errorMessage.equals("Null")) { if (errorMessage == null || errorMessage.isEmpty() || errorMessage.equals("Null")) {
errorMessage = e.getClass().getSimpleName() + " - look at server or client logs for more details"; errorMessage = e.getClass().getSimpleName() + " - look at server or client logs for more details";

4
Mage.Client/src/main/java/mage/client/dialog/DownloadImagesDialog.form
View file

@ -387,7 +387,7 @@
<Property name="icon" type="javax.swing.Icon" editor="org.netbeans.modules.form.editors2.IconEditor"> <Property name="icon" type="javax.swing.Icon" editor="org.netbeans.modules.form.editors2.IconEditor">
<Image iconType="3" name="/buttons/search_24.png"/> <Image iconType="3" name="/buttons/search_24.png"/>
</Property> </Property>
<Property name="toolTipText" type="java.lang.String" value="Fast search your flag"/> <Property name="toolTipText" type="java.lang.String" value="Search set to download"/>
<Property name="alignmentX" type="float" value="1.0"/> <Property name="alignmentX" type="float" value="1.0"/>
<Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor"> <Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[25, 25]"/> <Dimension value="[25, 25]"/>
@ -419,7 +419,7 @@
<SubComponents> <SubComponents>
<Component class="javax.swing.JCheckBox" name="checkboxRedownload"> <Component class="javax.swing.JCheckBox" name="checkboxRedownload">
<Properties> <Properties>
<Property name="text" type="java.lang.String" value="&lt;html&gt;Re-download all images"/> <Property name="text" type="java.lang.String" value="&lt;html&gt;Re-download all selected images"/>
<Property name="verticalAlignment" type="int" value="3"/> <Property name="verticalAlignment" type="int" value="3"/>
</Properties> </Properties>
<Constraints> <Constraints>

5
Mage.Client/src/main/java/mage/client/dialog/DownloadImagesDialog.java
View file

@ -358,7 +358,7 @@ public class DownloadImagesDialog extends MageDialog {
panelModeSelect.add(fillerMode1); panelModeSelect.add(fillerMode1);
buttonSearchSet.setIcon(new javax.swing.ImageIcon(getClass().getResource("/buttons/search_24.png"))); // NOI18N buttonSearchSet.setIcon(new javax.swing.ImageIcon(getClass().getResource("/buttons/search_24.png"))); // NOI18N
buttonSearchSet.setToolTipText("Fast search your flag"); buttonSearchSet.setToolTipText("Search set to download");
buttonSearchSet.setAlignmentX(1.0F); buttonSearchSet.setAlignmentX(1.0F);
buttonSearchSet.setPreferredSize(new java.awt.Dimension(25, 25)); buttonSearchSet.setPreferredSize(new java.awt.Dimension(25, 25));
buttonSearchSet.addActionListener(new java.awt.event.ActionListener() { buttonSearchSet.addActionListener(new java.awt.event.ActionListener() {
@ -378,7 +378,7 @@ public class DownloadImagesDialog extends MageDialog {
panelRedownload.setPreferredSize(new java.awt.Dimension(280, 100)); panelRedownload.setPreferredSize(new java.awt.Dimension(280, 100));
panelRedownload.setLayout(new java.awt.BorderLayout()); panelRedownload.setLayout(new java.awt.BorderLayout());
checkboxRedownload.setText("<html>Re-download all images"); checkboxRedownload.setText("<html>Re-download all selected images");
checkboxRedownload.setVerticalAlignment(javax.swing.SwingConstants.BOTTOM); checkboxRedownload.setVerticalAlignment(javax.swing.SwingConstants.BOTTOM);
panelRedownload.add(checkboxRedownload, java.awt.BorderLayout.CENTER); panelRedownload.add(checkboxRedownload, java.awt.BorderLayout.CENTER);
panelRedownload.add(filler1, java.awt.BorderLayout.PAGE_END); panelRedownload.add(filler1, java.awt.BorderLayout.PAGE_END);
@ -444,6 +444,7 @@ public class DownloadImagesDialog extends MageDialog {
}//GEN-LAST:event_buttonStopActionPerformed }//GEN-LAST:event_buttonStopActionPerformed
private void doClose(int retStatus) { private void doClose(int retStatus) {
returnStatus = retStatus; returnStatus = retStatus;
setVisible(false); setVisible(false);
dispose(); dispose();

18
Mage.Client/src/main/java/mage/client/remote/XmageURLConnection.java
View file

@ -52,6 +52,7 @@ public class XmageURLConnection {
Proxy proxy = null; Proxy proxy = null;
HttpURLConnection connection = null; HttpURLConnection connection = null;
HttpLoggingType loggingType = HttpLoggingType.ERRORS; HttpLoggingType loggingType = HttpLoggingType.ERRORS;
boolean forceGZipEncoding = false;
public XmageURLConnection(String url) { public XmageURLConnection(String url) {
this.url = url; this.url = url;
@ -75,6 +76,10 @@ public class XmageURLConnection {
} }
} }
public void setForceGZipEncoding(boolean enable) {
this.forceGZipEncoding = enable;
}
/** /**
* Connect to server * Connect to server
*/ */
@ -130,7 +135,11 @@ public class XmageURLConnection {
} }
private void initDefaultHeaders() { private void initDefaultHeaders() {
// warning, do not add Accept-Encoding - it processing inside URLConnection for http/https links (trying to use gzip by default) // warning, Accept-Encoding processing inside URLConnection for http/https links (trying to use gzip by default)
// use force encoding for special use cases (example: download big text file as zip file)
if (forceGZipEncoding) {
this.connection.setRequestProperty("Accept-Encoding", "gzip");
}
// user agent due standard notation User-Agent: <product> / <product-version> <comment> // user agent due standard notation User-Agent: <product> / <product-version> <comment>
// warning, dot not add os, language and other details // warning, dot not add os, language and other details
@ -290,13 +299,18 @@ public class XmageURLConnection {
return ""; return "";
} }
public static InputStream downloadBinary(String resourceUrl) {
return downloadBinary(resourceUrl, false);
}
/** /**
* Fast download of binary data * Fast download of binary data
* *
* @return stream on OK 200 response or null on any other errors * @return stream on OK 200 response or null on any other errors
*/ */
public static InputStream downloadBinary(String resourceUrl) { public static InputStream downloadBinary(String resourceUrl, boolean downloadAsGZip) {
XmageURLConnection con = new XmageURLConnection(resourceUrl); XmageURLConnection con = new XmageURLConnection(resourceUrl);
con.setForceGZipEncoding(downloadAsGZip);
con.startConnection(); con.startConnection();
if (con.isConnected()) { if (con.isConnected()) {
try { try {

5
Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java
View file

@ -301,10 +301,9 @@ public final class ManaSymbols {
if (SvgUtils.haveSvgSupport()) { if (SvgUtils.haveSvgSupport()) {
file = getSymbolFileNameAsSVG(symbol); file = getSymbolFileNameAsSVG(symbol);
if (file.exists()) { if (file.exists()) {
try { try(InputStream fileStream = Files.newInputStream(file.toPath())) {
InputStream fileStream = new FileInputStream(file);
image = loadSymbolAsSVG(fileStream, file.getPath(), size, size); image = loadSymbolAsSVG(fileStream, file.getPath(), size, size);
} catch (FileNotFoundException ignore) { } catch (IOException ignore) {
} }
} }
} }

9
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java
View file

@ -4,7 +4,10 @@ import mage.client.util.CardLanguage;
import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.images.CardDownloadData;
import java.util.*; import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/** /**
* @author North, JayDi85 * @author North, JayDi85
@ -70,4 +73,8 @@ public interface CardImageSource {
default boolean isTokenImageProvided(String setCode, String cardName, Integer tokenNumber) { default boolean isTokenImageProvided(String setCode, String cardName, Integer tokenNumber) {
return false; return false;
} }
default void onFinished() {
// cleanup temp resources
}
} }

19
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiBulkData.java Normal file
View file

@ -0,0 +1,19 @@
package org.mage.plugins.card.dl.sources;
import java.util.Date;
/**
* Scryfall API: bulk file links to download
* <p>
* API docs: <a href="https://scryfall.com/docs/api/bulk-data">here</a>
*
* @author JayDi85
*/
public class ScryfallApiBulkData {
public String object;
public String type;
public Date updated_at;
public long size;
public String download_uri;
}

156
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCard.java Normal file
View file

@ -0,0 +1,156 @@
package org.mage.plugins.card.dl.sources;
import mage.cards.decks.CardNameUtil;
import java.util.List;
import java.util.Map;
/**
* Scryfall API: card info
* <p>
* API docs: <a href="https://scryfall.com/docs/api/cards">here</a>
* <p>
* Field values depends on data source, check it on null (as example: bulk data from oracle cards and bulk data from all cards)
*
* @author JayDi85
*/
public class ScryfallApiCard {
public String name;
public String set; // set code
public String collector_number;
public String lang;
public String layout; // card type like adventure, etc, see https://scryfall.com/docs/api/layouts
public List<ScryfallApiCardFace> card_faces;
public String image_status; // missing, placeholder, lowres, or highres_scan
public Map<String, String> image_uris;
// fast access fields, fills on loading
transient public String imageSmall = "";
transient public String imageNormal = "";
transient public String imageLarge = "";
// potentially interesting fields, can be used in other places
//public UUID oracle_id; // TODO: implement card hint with oracle/cr ruling texts (see Rulings bulk data)
//public Integer edhrec_rank; // TODO: use it to rating cards for AI and draft bots
//public Object legalities; // TODO: add verify check for bans list
//public Boolean full_art; // TODO: add verify check for full art usage in sets
//public Object prices; // TODO: add total deck price and table limit by deck's price
//public Date released_at; // the date this card was first released.
//public String watermark; // background watermark image for some cards
public void prepareCompatibleData() {
if (this.image_uris != null) {
this.imageSmall = this.image_uris.getOrDefault("small", "");
this.imageNormal = this.image_uris.getOrDefault("normal", "");
this.imageLarge = this.image_uris.getOrDefault("large", "");
this.image_uris = null;
}
if (this.card_faces != null) {
this.card_faces.forEach(ScryfallApiCardFace::prepareCompatibleData);
}
// workaround for adventure card name fix:
// - scryfall: Ondu Knotmaster // Throw a Line
// - xmage: Ondu Knotmaster
if (this.layout.equals("adventure")) {
this.name = this.card_faces.get(0).name;
}
// workaround for flip card name and image fixes:
// - scryfall: Budoka Pupil // Ichiga, Who Topples Oaks
// - xmage: Budoka Pupil
if (this.layout.equals("flip")) {
if (!this.card_faces.get(0).imageNormal.isEmpty()
|| !this.card_faces.get(1).imageNormal.isEmpty()) {
throw new IllegalArgumentException("Scryfall: unsupported data type, flip parts must not have images data in scryfall "
+ this.set + " - " + this.collector_number + " - " + this.name);
}
// fix name
this.name = this.card_faces.get(0).name;
// fix image (xmage uses diff cards for flips, but it's same image)
// example: https://scryfall.com/card/sok/103/homura-human-ascendant-homuras-essence
this.card_faces.get(0).image_uris = this.image_uris;
this.card_faces.get(0).imageSmall = this.imageSmall;
this.card_faces.get(0).imageNormal = this.imageNormal;
this.card_faces.get(0).imageLarge = this.imageLarge;
this.card_faces.get(1).image_uris = this.image_uris;
this.card_faces.get(1).imageSmall = this.imageSmall;
this.card_faces.get(1).imageNormal = this.imageNormal;
this.card_faces.get(1).imageLarge = this.imageLarge;
}
// workaround for reversed cards:
// - scryfall: Command Tower // Command Tower
// - xmage: Command Tower (second side as diff card and direct link image), example: https://scryfall.com/card/rex/26/command-tower-command-tower
if (this.layout.equals("reversible_card")) {
if (!this.card_faces.get(0).name.equals(this.card_faces.get(1).name)) {
throw new IllegalArgumentException("Scryfall: unsupported data type, reversible_card has diff faces "
+ this.set + " - " + this.collector_number + " - " + this.name);
}
this.name = this.card_faces.get(0).name;
}
// workaround for non ascii names
// - scryfall uses original names like Arna Kennerüd, Skycaptain
// - xmage need ascii only names like Arna Kennerud, Skycaptain
this.name = CardNameUtil.normalizeCardName(this.name);
// workaround for non scii card numbers
// - scryfall uses unicode numbers for reprints like Chandra Nalaar - dd2 - 34★ https://scryfall.com/card/dd2/34%E2%98%85/
// - xmage uses ascii alternative Chandra Nalaar - dd2 - 34*
this.collector_number = transformCardNumberFromScryfallToXmage(this.collector_number);
}
public static String transformCardNumberFromXmageToScryfall(String cardNumber) {
String res = cardNumber;
if (res.endsWith("*")) {
res = res.substring(0, res.length() - 1) + "★";
}
if (res.endsWith("+")) {
res = res.substring(0, res.length() - 1) + "†";
}
if (res.endsWith("Ph")) {
res = res.substring(0, res.length() - 2) + "Φ";
}
return res;
}
public static String transformCardNumberFromScryfallToXmage(String cardNumber) {
String res = cardNumber;
if (res.endsWith("★")) {
res = res.substring(0, res.length() - 1) + "*";
}
if (res.endsWith("†")) {
res = res.substring(0, res.length() - 1) + "+";
}
if (res.endsWith("Φ")) {
res = res.substring(0, res.length() - 1) + "Ph";
}
return res;
}
public String findImage(String imageSize) {
// api possible values:
// - small
// - normal
// - large
// - png
// - art_crop
// - border_crop
switch (imageSize) {
case "small":
return this.imageSmall;
case "normal":
return this.imageNormal;
case "large":
return this.imageLarge;
default:
throw new IllegalArgumentException("Unsupported image size: " + imageSize);
}
}
}

50
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallApiCardFace.java Normal file
View file

@ -0,0 +1,50 @@
package org.mage.plugins.card.dl.sources;
import java.util.Map;
/**
* Scryfall API: card face info (used for double faced cards)
* <p>
* API docs: <a href="https://scryfall.com/docs/api/cards">here</a>
* <p>
*
* @author JayDi85
*/
public class ScryfallApiCardFace {
public String name;
public Map<String, String> image_uris;
// fast access fields, fills on loading
transient public String imageSmall = "";
transient public String imageNormal = "";
transient public String imageLarge = "";
public void prepareCompatibleData() {
if (this.image_uris != null) {
this.imageSmall = this.image_uris.getOrDefault("small", "");
this.imageNormal = this.image_uris.getOrDefault("normal", "");
this.imageLarge = this.image_uris.getOrDefault("large", "");
this.image_uris = null;
}
}
public String findImage(String imageSize) {
// api possible values:
// - small
// - normal
// - large
// - png
// - art_crop
// - border_crop
switch (imageSize) {
case "small":
return this.imageSmall;
case "normal":
return this.imageNormal;
case "large":
return this.imageLarge;
default:
throw new IllegalArgumentException("Unsupported image size: " + imageSize);
}
}
}

493
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java
View file

@ -1,22 +1,36 @@
package org.mage.plugins.card.dl.sources; package org.mage.plugins.card.dl.sources;
import com.google.gson.Gson;
import com.google.gson.JsonArray; import com.google.gson.JsonArray;
import com.google.gson.JsonObject; import com.google.gson.JsonObject;
import com.google.gson.JsonParser; import com.google.gson.JsonParser;
import com.google.gson.stream.JsonReader;
import mage.MageException; import mage.MageException;
import mage.client.remote.XmageURLConnection; import mage.client.remote.XmageURLConnection;
import mage.client.util.CardLanguage; import mage.client.util.CardLanguage;
import mage.util.JsonUtil; import mage.util.JsonUtil;
import net.java.truevfs.access.TFile;
import net.java.truevfs.access.TFileInputStream;
import net.java.truevfs.access.TFileOutputStream;
import net.java.truevfs.access.TVFS;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.dl.DownloadServiceInfo;
import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.images.CardDownloadData;
import java.io.InputStream; import java.io.*;
import java.io.InputStreamReader; import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.GZIPInputStream;
import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
/** /**
* Download: scryfall API support to download images and other data
*
* @author JayDi85 * @author JayDi85
*/ */
public class ScryfallImageSource implements CardImageSource { public class ScryfallImageSource implements CardImageSource {
@ -32,6 +46,21 @@ public class ScryfallImageSource implements CardImageSource {
private static final ReentrantLock waitBeforeRequestLock = new ReentrantLock(); private static final ReentrantLock waitBeforeRequestLock = new ReentrantLock();
private static final int DOWNLOAD_TIMEOUT_MS = 300; private static final int DOWNLOAD_TIMEOUT_MS = 300;
// bulk images optimization, how it works
// - scryfall limit all calls to API endpoints, but allow direct calls to CDN (scryfall.io) without limitation
// - so download and prepare whole scryfall database
// - for each downloading card try to find it in bulk database and use direct link from it
// - only card images supported (without tokens, emblems, etc)
private static final boolean SCRYFALL_BULK_FILES_ENABLED = true; // use bulk data to find direct links instead API calls
private static final long SCRYFALL_BULK_FILES_OUTDATE_IN_MILLIS = 7 * 24 * 3600 * 1000; // after 1 week need to download again
private static final String SCRYFALL_BULK_FILES_DATABASE_SOURCE_API = "https://api.scryfall.com/bulk-data/all-cards"; // 300 MB in zip
private static final boolean SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE = false; // default: false - for faster debug only, ignore write operations
// run app with -Dxmage.scryfallEnableBulkData=false to disable bulk data (e.g. for testing api links generation)
private static final String SCRYFALL_BULK_FILES_PROPERTY = "xmage.scryfallEnableBulkData";
private static final List<ScryfallApiCard> bulkCardsDatabaseAll = new ArrayList<>(); // 100MB memory footprint, cleaning on download stop
private static final Map<String, ScryfallApiCard> bulkCardsDatabaseDefault = new HashMap<>(); // card/set/number
public static ScryfallImageSource getInstance() { public static ScryfallImageSource getInstance() {
return instance; return instance;
} }
@ -117,7 +146,7 @@ public class ScryfallImageSource implements CardImageSource {
// basic cards by api call (redirect to img link) // basic cards by api call (redirect to img link)
// example: https://api.scryfall.com/cards/xln/121/en?format=image // example: https://api.scryfall.com/cards/xln/121/en?format=image
if (baseUrl == null) { if (baseUrl == null) {
String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId()); String cn = ScryfallApiCard.transformCardNumberFromXmageToScryfall(card.getCollectorId());
baseUrl = String.format("https://api.scryfall.com/cards/%s/%s/%s?format=image", baseUrl = String.format("https://api.scryfall.com/cards/%s/%s/%s?format=image",
formatSetName(card.getSet(), isToken), formatSetName(card.getSet(), isToken),
cn, cn,
@ -132,12 +161,12 @@ public class ScryfallImageSource implements CardImageSource {
// include_variations=true added to deal with the cards that scryfall has marked as variations that seem to sometimes fail // include_variations=true added to deal with the cards that scryfall has marked as variations that seem to sometimes fail
// eg https://api.scryfall.com/cards/4ed/134†?format=image fails // eg https://api.scryfall.com/cards/4ed/134†?format=image fails
// eg https://api.scryfall.com/cards/4ed/134†?format=image&include_variations=true succeeds // eg https://api.scryfall.com/cards/4ed/134†?format=image&include_variations=true succeeds
} }
return new CardImageUrls(baseUrl, alternativeUrl); return new CardImageUrls(baseUrl, alternativeUrl);
} }
// TODO: delete face code after bulk data implemented?
private String getFaceImageUrl(CardDownloadData card, boolean isToken) throws Exception { private String getFaceImageUrl(CardDownloadData card, boolean isToken) throws Exception {
final String defaultCode = CardLanguage.ENGLISH.getCode(); final String defaultCode = CardLanguage.ENGLISH.getCode();
final String localizedCode = languageAliases.getOrDefault(this.getCurrentLanguage(), defaultCode); final String localizedCode = languageAliases.getOrDefault(this.getCurrentLanguage(), defaultCode);
@ -147,14 +176,8 @@ public class ScryfallImageSource implements CardImageSource {
String apiUrl = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId()); String apiUrl = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId());
if (apiUrl != null) { if (apiUrl != null) {
if (apiUrl.endsWith("*/")) {
apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "★/";
} else if (apiUrl.endsWith("+/")) {
apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "†/";
} else if (apiUrl.endsWith("Ph/")) {
apiUrl = apiUrl.substring(0, apiUrl.length() - 3) + "Φ/";
}
// BY DIRECT URL // BY DIRECT URL
// direct url already contains scryfall compatible card numbers (with unicode chars)
// direct links via hardcoded API path. Used for cards with non-ASCII collector numbers // direct links via hardcoded API path. Used for cards with non-ASCII collector numbers
if (localizedCode.equals(defaultCode)) { if (localizedCode.equals(defaultCode)) {
// english only, so can use workaround without loc param (scryfall download first available card) // english only, so can use workaround without loc param (scryfall download first available card)
@ -168,7 +191,7 @@ public class ScryfallImageSource implements CardImageSource {
} else { } else {
// BY CARD NUMBER // BY CARD NUMBER
// localized and default // localized and default
String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId()); String cn = ScryfallApiCard.transformCardNumberFromXmageToScryfall(card.getCollectorId());
needUrls.add(String.format("https://api.scryfall.com/cards/%s/%s/%s", needUrls.add(String.format("https://api.scryfall.com/cards/%s/%s/%s",
formatSetName(card.getSet(), isToken), formatSetName(card.getSet(), isToken),
cn, cn,
@ -221,9 +244,23 @@ public class ScryfallImageSource implements CardImageSource {
@Override @Override
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) { public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
// prepare download list example (
preparedUrls.clear(); preparedUrls.clear();
// direct images without api calls
if (!prepareBulkData(downloadServiceInfo, downloadList)) {
return false;
}
analyseBulkData(downloadServiceInfo, downloadList);
// second side images need lookup for
if (!prepareSecondSideImages(downloadServiceInfo, downloadList)) {
return false;
}
return true;
}
private boolean prepareSecondSideImages(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
// prepare stats // prepare stats
int needPrepareCount = 0; int needPrepareCount = 0;
int currentPrepareCount = 0; int currentPrepareCount = 0;
@ -235,11 +272,16 @@ public class ScryfallImageSource implements CardImageSource {
updatePrepareStats(downloadServiceInfo, needPrepareCount, currentPrepareCount); updatePrepareStats(downloadServiceInfo, needPrepareCount, currentPrepareCount);
for (CardDownloadData card : downloadList) { for (CardDownloadData card : downloadList) {
// need cancel // fast cancel
if (downloadServiceInfo.isNeedCancel()) { if (downloadServiceInfo.isNeedCancel()) {
return false; return false;
} }
// already prepare, e.g. on bulk data
if (preparedUrls.containsKey(card)) {
continue;
}
// prepare the back face URL // prepare the back face URL
if (card.isSecondSide()) { if (card.isSecondSide()) {
currentPrepareCount++; currentPrepareCount++;
@ -259,10 +301,410 @@ public class ScryfallImageSource implements CardImageSource {
return true; return true;
} }
private boolean prepareBulkData(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
boolean isEnabled = SCRYFALL_BULK_FILES_ENABLED;
if (System.getProperty(SCRYFALL_BULK_FILES_PROPERTY) != null) {
isEnabled = Boolean.parseBoolean(System.getProperty(SCRYFALL_BULK_FILES_PROPERTY));
}
if (!isEnabled) {
return true;
}
// if up to date
if (isBulkDataPrepared()) {
return true;
}
// NEED TO DOWNLOAD
// clean
TFile bulkTempFile = prepareTempFileForBulkData();
if (bulkTempFile.exists()) {
try {
if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
bulkTempFile.rm();
TVFS.umount();
}
} catch (IOException e) {
return false;
}
}
// find actual download link
String s = XmageURLConnection.downloadText(SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
if (s.isEmpty()) {
logger.error("Can't get bulk info from scryfall api " + SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
return false;
}
ScryfallApiBulkData bulkData = new Gson().fromJson(s, ScryfallApiBulkData.class);
if (bulkData == null
|| bulkData.download_uri == null
|| bulkData.download_uri.isEmpty()
|| bulkData.size == 0) {
logger.error("Unknown bulk info format from scryfall api " + SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
return false;
}
// download
if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
logger.info("Scryfall: downloading additional data files, size " + bulkData.size / (1024 * 1024) + " MB");
String url = bulkData.download_uri;
InputStream inputStream = XmageURLConnection.downloadBinary(url, true);
OutputStream outputStream = null;
try {
try {
if (inputStream == null) {
logger.error("Can't get bulk data from scryfall api " + url);
return false;
}
outputStream = new TFileOutputStream(bulkTempFile);
byte[] buf = new byte[5 * 1024 * 1024]; // 5 MB buffer
int len;
long needDownload = bulkData.size / 7; // text -> zip size multiplier
long doneDownload = 0;
while ((len = inputStream.read(buf)) != -1) {
// fast cancel
if (downloadServiceInfo.isNeedCancel()) {
// stop download and delete current file
inputStream.close();
outputStream.close();
if (bulkTempFile.exists()) {
bulkTempFile.rm();
}
return false;
}
// all fine, can save data part
outputStream.write(buf, 0, len);
doneDownload += len;
updateBulkDownloadStats(downloadServiceInfo, doneDownload, needDownload);
}
outputStream.flush();
// unpack
logger.info("Scryfall: unpacking files...");
Path zippedBulkPath = Paths.get(getBulkTempFileName());
Path textBulkPath = Paths.get(getBulkStaticFileName());
Files.deleteIfExists(textBulkPath);
try (GZIPInputStream in = new GZIPInputStream(Files.newInputStream(zippedBulkPath));
FileOutputStream out = new FileOutputStream(textBulkPath.toString())) {
byte[] buffer = new byte[5 * 1024 * 1024];
long needUnpack = bulkData.size; // zip -> text
long doneUnpack = 0;
while ((len = in.read(buffer)) > 0) {
// fast cancel
if (downloadServiceInfo.isNeedCancel()) {
out.flush();
Files.deleteIfExists(textBulkPath);
return false;
}
out.write(buffer, 0, len);
doneUnpack += len;
updateBulkUnpackingStats(downloadServiceInfo, doneUnpack, needUnpack);
}
}
logger.info("Scryfall: unpacking done");
// all fine
} finally {
if (inputStream != null) {
inputStream.close();
}
if (outputStream != null) {
outputStream.close();
}
}
} catch (Exception e) {
logger.error("Catch unknown error while download bulk data from scryfall api " + url, e);
return false;
}
}
return isBulkDataPrepared();
}
private boolean isBulkDataPrepared() {
// already loaded
if (bulkCardsDatabaseAll.size() > 0) {
return true;
}
// file not exists
Path textBulkPath = Paths.get(getBulkStaticFileName());
if (!Files.exists(textBulkPath)) {
return false;
}
// file outdated
try {
if (System.currentTimeMillis() - Files.getLastModifiedTime(textBulkPath).toMillis() > SCRYFALL_BULK_FILES_OUTDATE_IN_MILLIS) {
logger.info("Scryfall: bulk files outdated, need to download new");
return false;
}
} catch (IOException ignore) {
}
// bulk files are too big, so must read and parse it in stream mode only
// 500k reprints in diff languages
Gson gson = new Gson();
try (TFileInputStream inputStream = new TFileInputStream(textBulkPath.toFile());
InputStreamReader inputReader = new InputStreamReader(inputStream);
JsonReader jsonReader = new JsonReader(inputReader)) {
bulkCardsDatabaseAll.clear();
bulkCardsDatabaseDefault.clear();
jsonReader.beginArray();
while (jsonReader.hasNext()) {
ScryfallApiCard card = gson.fromJson(jsonReader, ScryfallApiCard.class);
// prepare data
// memory optimization: fewer data, from 1145 MB to 470 MB
card.prepareCompatibleData();
// keep only usefully languages
// memory optimization: fewer items, from 470 MB to 96 MB
// make sure final list will have one version
// example: japan only card must be in final list for any selected languages https://scryfall.com/card/war/180★/
String key = String.format("%s/%s/%s", card.name, card.set, card.collector_number);
if (!card.lang.equals(this.currentLanguage.getCode())
&& !card.lang.equals(CardLanguage.ENGLISH.getCode())
&& bulkCardsDatabaseDefault.containsKey(key)) {
continue;
}
// default versions list depends on scryfall bulk data order (en first)
bulkCardsDatabaseDefault.put(key, card);
bulkCardsDatabaseAll.add(card);
}
jsonReader.close();
return bulkCardsDatabaseAll.size() > 0;
} catch (Exception e) {
logger.error("Can't read bulk file (possible reason: broken format)");
try {
// clean up
if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
Files.deleteIfExists(textBulkPath);
}
} catch (IOException ignore) {
}
}
return false;
}
private void analyseBulkData(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
// prepare card indexes
// name/set/number/lang - normal cards
Map<String, String> bulkImagesIndexAll = createBulkImagesIndex(downloadServiceInfo, bulkCardsDatabaseAll, true);
// name/set/number - default cards
Map<String, String> bulkImagesIndexDefault = createBulkImagesIndex(downloadServiceInfo, bulkCardsDatabaseDefault.values(), false);
// find good images
AtomicInteger statsOldPrepared = new AtomicInteger();
AtomicInteger statsDirectLinks = new AtomicInteger();
AtomicInteger statsNewPrepared = new AtomicInteger();
AtomicInteger statsNotFound = new AtomicInteger();
for (CardDownloadData card : downloadList) {
// fast cancel
if (downloadServiceInfo.isNeedCancel()) {
return;
}
// already prepared, e.g. from direct download links or other sources
if (preparedUrls.containsKey(card)) {
statsOldPrepared.incrementAndGet();
continue;
}
// only cards supported
if (card.isToken()) {
continue;
}
// ignore direct links
String directLink = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId());
if (directLink != null) {
statsDirectLinks.incrementAndGet();
continue;
}
String searchingName = card.getName();
String searchingSet = card.getSet().toLowerCase(Locale.ENGLISH);
String searchingNumber = card.getCollectorId();
// current language
String key = String.format("%s/%s/%s/%s",
searchingName,
searchingSet,
searchingNumber,
this.currentLanguage
);
String link = bulkImagesIndexAll.getOrDefault(key, "");
// default en language
if (link.isEmpty()) {
key = String.format("%s/%s/%s/%s",
searchingName,
searchingSet,
searchingNumber,
CardLanguage.ENGLISH.getCode()
);
link = bulkImagesIndexAll.getOrDefault(key, "");
}
// default non en-language
if (link.isEmpty()) {
key = String.format("%s/%s/%s/",
searchingName,
searchingSet,
searchingNumber);
link = bulkImagesIndexDefault.getOrDefault(key, "");
}
if (link.isEmpty()) {
// how-to fix:
// - make sure name notation compatible, see workarounds in ScryfallApiCard.prepareCompatibleData
// - makue sure it's not fake card (xmage can use fake cards instead doubled-images) -- add it to ScryfallImageSupportCards.directDownloadLinks
logger.info("Scryfall: bulk image not found (outdated cards data in xmage?): " + key + " ");
statsNotFound.incrementAndGet();
} else {
preparedUrls.put(card, link);
statsNewPrepared.incrementAndGet();
}
}
logger.info(String.format("Scryfall: bulk optimization result - need cards: %d, already prepared: %d, direct links: %d, new prepared: %d, NOT FOUND: %d",
downloadList.size(),
statsOldPrepared.get(),
statsDirectLinks.get(),
statsNewPrepared.get(),
statsNotFound.get() // all not founded cards must be fixed
));
}
private Map<String, String> createBulkImagesIndex(DownloadServiceInfo downloadServiceInfo, Collection<ScryfallApiCard> sourceList, boolean useLocalization) {
Map<String, String> res = new HashMap<>();
for (ScryfallApiCard card : sourceList) {
// fast cancel
if (downloadServiceInfo.isNeedCancel()) {
return res;
}
// main card
String image = card.findImage(getImageQuality());
if (!image.isEmpty()) {
String mainKey = String.format("%s/%s/%s/%s",
card.name,
card.set,
card.collector_number,
useLocalization ? card.lang : ""
);
if (res.containsKey(mainKey)) {
// how-to fix: rework whole card number logic in xmage and scryfall
throw new IllegalArgumentException("Wrong code usage: scryfall used unique card numbers, but found duplicated: " + mainKey);
}
res.put(mainKey, image);
}
// faces
if (card.card_faces != null) {
// hints:
// 1. Not all faces has images - example: adventure cards
// 2. Some faces contain fake data in second face, search it on scryfall by "set_type:memorabilia"
// example: https://scryfall.com/card/aznr/25/clearwater-pathway-clearwater-pathway
// 3. Some faces contain diff images of the same card, layout = reversible_card
// example: https://scryfall.com/card/rex/26/command-tower-command-tower
Set<String> usedNames = new HashSet<>();
card.card_faces.forEach(face -> {
// workaround to ignore fake data, see above about memorabilia
if (usedNames.contains(face.name)) {
return;
}
usedNames.add(face.name);
String faceImage = face.findImage(getImageQuality());
if (!faceImage.isEmpty()) {
String faceKey = String.format("%s/%s/%s/%s",
face.name,
card.set,
card.collector_number,
useLocalization ? card.lang : ""
);
// workaround for flip cards - ignore first face - it already in the index
// example: https://scryfall.com/card/sok/103/homura-human-ascendant-homuras-essence
if (card.layout.equals("flip")
&& card.name.equals(face.name)
&& res.containsKey(faceKey)) {
return;
}
if (res.containsKey(faceKey)) {
// how-to fix: rework whole card number logic in xmage and scryfall
throw new IllegalArgumentException("Wrong code usage: scryfall used unique card numbers, but found duplicated: " + faceKey);
}
res.put(faceKey, faceImage);
}
});
}
}
return res;
}
private String getBulkTempFileName() {
return getImagesDir() + File.separator + "downloading" + File.separator + "scryfall_bulk_cards.json.gz";
}
private String getBulkStaticFileName() {
return getImagesDir() + File.separator + "downloading" + File.separator + "scryfall_bulk_cards.json";
}
private TFile prepareTempFileForBulkData() {
TFile file = new TFile(getBulkTempFileName());
TFile parent = file.getParentFile();
if (parent != null) {
if (!parent.exists()) {
parent.mkdirs();
}
}
return file;
}
private void updateBulkDownloadStats(DownloadServiceInfo service, long done, long need) {
int doneMb = (int) (done / (1024 * 1024));
int needMb = (int) (need / (1024 * 1024));
synchronized (service.getSync()) {
service.updateProgressMessage(
String.format("Step 1 of 3. Downloading additional data... %d of %d MB", doneMb, needMb),
doneMb,
needMb
);
}
}
private void updateBulkUnpackingStats(DownloadServiceInfo service, long done, long need) {
int doneMb = (int) (done / (1024 * 1024));
int needMb = (int) (need / (1024 * 1024));
synchronized (service.getSync()) {
service.updateProgressMessage(
String.format("Step 2 of 3. Unpacking additional files... %d of %d MB", doneMb, needMb),
doneMb,
needMb
);
}
}
private void updatePrepareStats(DownloadServiceInfo service, int need, int current) { private void updatePrepareStats(DownloadServiceInfo service, int need, int current) {
synchronized (service.getSync()) { synchronized (service.getSync()) {
service.updateProgressMessage( service.updateProgressMessage(
String.format("Preparing download list... %d of %d", current, need), String.format("Step 3 of 3. Preparing download list... %d of %d", current, need),
current, current,
need need
); );
@ -334,7 +776,12 @@ public class ScryfallImageSource implements CardImageSource {
public void doPause(String fullUrl) { public void doPause(String fullUrl) {
// scryfall recommends 300 ms timeout per each request to API to work under a rate limit // scryfall recommends 300 ms timeout per each request to API to work under a rate limit
// possible error: 429 Too Many Requests // possible error: 429 Too Many Requests
// TODO: add diff endpoint supports (api calls with timeout, cdn/direct calls without timeout)
// cdn source can be safe and called without pause
if (fullUrl.contains("scryfall.io")) {
return;
}
waitBeforeRequest(); waitBeforeRequest();
} }
@ -387,4 +834,18 @@ public class ScryfallImageSource implements CardImageSource {
// only direct tokens from set // only direct tokens from set
return ScryfallImageSupportTokens.findTokenLink(setCode, cardName, tokenNumber) != null; return ScryfallImageSupportTokens.findTokenLink(setCode, cardName, tokenNumber) != null;
} }
/**
* Image quality
*/
public String getImageQuality() {
return "large";
}
@Override
public void onFinished() {
// cleanup resources
bulkCardsDatabaseAll.clear();
bulkCardsDatabaseDefault.clear();
}
} }

14
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceNormal.java
View file

@ -11,19 +11,19 @@ import java.util.stream.Collectors;
public class ScryfallImageSourceNormal extends ScryfallImageSource { public class ScryfallImageSourceNormal extends ScryfallImageSource {
private static final ScryfallImageSourceNormal instanceNormal = new ScryfallImageSourceNormal(); private static final ScryfallImageSourceNormal instanceNormal = new ScryfallImageSourceNormal();
public static ScryfallImageSource getInstance() { public static ScryfallImageSource getInstance() {
return instanceNormal; return instanceNormal;
} }
private static String innerModifyUrlString(String oneUrl) { private static String innerModifyUrlString(String oneUrl) {
return oneUrl.replaceFirst("/large/","/normal/").replaceFirst("format=image","format=image&version=normal"); return oneUrl.replaceFirst("/large/", "/normal/").replaceFirst("format=image", "format=image&version=normal");
} }
private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) { private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) {
List<String> downloadUrls = cardUrls.getDownloadList().stream() List<String> downloadUrls = cardUrls.getDownloadList().stream()
.map(ScryfallImageSourceNormal::innerModifyUrlString) .map(ScryfallImageSourceNormal::innerModifyUrlString)
.collect(Collectors.toList()); .collect(Collectors.toList());
return new CardImageUrls(downloadUrls); return new CardImageUrls(downloadUrls);
} }
@ -48,4 +48,8 @@ public class ScryfallImageSourceNormal extends ScryfallImageSource {
return 92f; return 92f;
} }
@Override
public String getImageQuality() {
return "normal";
}
} }

14
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSourceSmall.java
View file

@ -11,19 +11,19 @@ import java.util.stream.Collectors;
public class ScryfallImageSourceSmall extends ScryfallImageSource { public class ScryfallImageSourceSmall extends ScryfallImageSource {
private static final ScryfallImageSourceSmall instanceSmall = new ScryfallImageSourceSmall(); private static final ScryfallImageSourceSmall instanceSmall = new ScryfallImageSourceSmall();
public static ScryfallImageSource getInstance() { public static ScryfallImageSource getInstance() {
return instanceSmall; return instanceSmall;
} }
private static String innerModifyUrlString(String oneUrl) { private static String innerModifyUrlString(String oneUrl) {
return oneUrl.replaceFirst("/large/","/small/").replaceFirst("format=image","format=image&version=small"); return oneUrl.replaceFirst("/large/", "/small/").replaceFirst("format=image", "format=image&version=small");
} }
private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) { private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) {
List<String> downloadUrls = cardUrls.getDownloadList().stream() List<String> downloadUrls = cardUrls.getDownloadList().stream()
.map(ScryfallImageSourceSmall::innerModifyUrlString) .map(ScryfallImageSourceSmall::innerModifyUrlString)
.collect(Collectors.toList()); .collect(Collectors.toList());
return new CardImageUrls(downloadUrls); return new CardImageUrls(downloadUrls);
} }
@ -48,4 +48,8 @@ public class ScryfallImageSourceSmall extends ScryfallImageSource {
return 14f; return 14f;
} }
@Override
public String getImageQuality() {
return "small";
}
} }

13
Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSupportCards.java
View file

@ -681,19 +681,6 @@ public class ScryfallImageSupportCards {
return null; return null;
} }
public static String prepareCardNumber(String xmageCardNumber) {
if (xmageCardNumber.endsWith("*")) {
return xmageCardNumber.substring(0, xmageCardNumber.length() - 1) + "★";
}
if (xmageCardNumber.endsWith("+")) {
return xmageCardNumber.substring(0, xmageCardNumber.length() - 1) + "†";
}
if (xmageCardNumber.endsWith("Ph")) {
return xmageCardNumber.substring(0, xmageCardNumber.length() - 2) + "Φ";
}
return xmageCardNumber;
}
public static String findDirectDownloadLink(String setCode, String cardName, String cardNumber) { public static String findDirectDownloadLink(String setCode, String cardName, String cardNumber) {
String key = findDirectDownloadKey(setCode, cardName, cardNumber); String key = findDirectDownloadKey(setCode, cardName, cardNumber);
return directDownloadLinks.get(key); return directDownloadLinks.get(key);

189
Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java
View file

@ -66,8 +66,8 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
// protect from wrong data save // protect from wrong data save
// there are possible land images with small sizes, so must research content in check // there are possible land images with small sizes, so must research content in check
private static final int MIN_FILE_SIZE_OF_GOOD_IMAGE = 1024 * 6; // broken private static final int MIN_FILE_SIZE_OF_GOOD_IMAGE = 1024 * 6; // smaller files will be mark as broken
private static final int MIN_FILE_SIZE_OF_POSSIBLE_BAD_IMAGE = 1024 * 15; // possible broken (need content check) private static final int MIN_FILE_SIZE_OF_POSSIBLE_BAD_IMAGE = 1024 * 8; // smaller files will be checked for possible broken mark (slow)
private final DownloadImagesDialog uiDialog; private final DownloadImagesDialog uiDialog;
private boolean needCancel; private boolean needCancel;
@ -77,8 +77,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
private List<CardInfo> cardsAll; private List<CardInfo> cardsAll;
private List<CardDownloadData> cardsMissing; private List<CardDownloadData> cardsMissing;
private final List<CardDownloadData> cardsDownloadQueue; private final List<CardDownloadData> cardsDownloadQueue;
private int missingCardsCount = 0;
private int missingTokensCount = 0;
private final List<String> selectedSets = new ArrayList<>(); private final List<String> selectedSets = new ArrayList<>();
private static CardImageSource selectedSource; private static CardImageSource selectedSource;
@ -429,13 +427,13 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
} }
private void updateProgressText(int cardCount, int tokenCount) { private void updateProgressText(int cardCount, int tokenCount) {
missingTokensCount = 0; int missingTokensCount = 0;
for (CardDownloadData card : cardsMissing) { for (CardDownloadData card : cardsMissing) {
if (card.isToken()) { if (card.isToken()) {
missingTokensCount++; missingTokensCount++;
} }
} }
missingCardsCount = cardsMissing.size() - missingTokensCount; int missingCardsCount = cardsMissing.size() - missingTokensCount;
uiDialog.setCurrentInfo("Missing: " + missingCardsCount + " card images / " + missingTokensCount + " token images"); uiDialog.setCurrentInfo("Missing: " + missingCardsCount + " card images / " + missingTokensCount + " token images");
int imageSum = cardCount + tokenCount; int imageSum = cardCount + tokenCount;
@ -446,7 +444,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
} else if (cardIndex == 0) { } else if (cardIndex == 0) {
statusEnd = "image download NOT STARTED. Please start."; statusEnd = "image download NOT STARTED. Please start.";
} else { } else {
statusEnd = String.format("image downloading... Please wait [%.1f Mb left].", mb); statusEnd = String.format("image downloading... Please wait [%.1f MB left].", mb);
} }
updateProgressMessage(String.format("%d of %d (%d cards and %d tokens) %s", updateProgressMessage(String.format("%d of %d (%d cards and %d tokens) %s",
0, imageSum, cardCount, tokenCount, statusEnd 0, imageSum, cardCount, tokenCount, statusEnd
@ -618,7 +616,8 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
} }
}); });
if (badContentChecks.get() > 10000) { if (badContentChecks.get() > 1000) {
// how-to fix: download full small images and run checks -- make sure it finds fewer files to check
logger.warn("Wrong code usage: too many file content checks (" + badContentChecks.get() + ") - try to decrease min file size"); logger.warn("Wrong code usage: too many file content checks (" + badContentChecks.get() + ") - try to decrease min file size");
} }
@ -630,86 +629,93 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
this.cardIndex = 0; this.cardIndex = 0;
this.resetErrorCount(); this.resetErrorCount();
File base = new File(getImagesDir());
if (!base.exists()) {
base.mkdir();
}
int downloadThreadsAmount = Math.max(1, Integer.parseInt((String) uiDialog.getDownloadThreadsCombo().getSelectedItem()));
logger.info("Started download of " + cardsDownloadQueue.size() + " images"
+ " from source: " + selectedSource.getSourceName()
+ ", language: " + selectedSource.getCurrentLanguage().getCode()
+ ", threads: " + downloadThreadsAmount);
updateProgressMessage("Preparing download list...");
if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) {
update(0, cardsDownloadQueue.size());
ExecutorService executor = Executors.newFixedThreadPool(
downloadThreadsAmount,
new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_IMAGES_DOWNLOADER, false)
);
for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) {
try {
CardDownloadData card = cardsDownloadQueue.get(i);
logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')');
CardImageUrls urls;
if (card.isToken()) {
if (!"0".equals(card.getCollectorId())) {
continue;
}
urls = selectedSource.generateTokenUrl(card);
} else {
urls = selectedSource.generateCardUrl(card);
}
if (urls == null) {
String imageRef = selectedSource.getNextHttpImageUrl();
String fileName = selectedSource.getFileForHttpImage(imageRef);
if (imageRef != null && fileName != null) {
imageRef = selectedSource.getSourceName() + imageRef;
try {
card.setToken(selectedSource.isTokenSource());
Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages());
executor.execute(task);
} catch (Exception ex) {
}
} else if (selectedSource.getTotalImages() == -1) {
logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')');
synchronized (sync) {
update(cardIndex + 1, cardsDownloadQueue.size());
}
}
} else {
Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size());
executor.execute(task);
}
} catch (Exception ex) {
logger.error(ex, ex);
}
}
executor.shutdown();
while (!executor.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ignore) {
}
}
}
try { try {
TVFS.umount(); File base = new File(getImagesDir());
} catch (FsSyncException e) { if (!base.exists()) {
logger.fatal("Couldn't unmount zip files", e); base.mkdir();
JOptionPane.showMessageDialog(null, "Couldn't unmount zip files", "Error", JOptionPane.ERROR_MESSAGE); }
int downloadThreadsAmount = Math.max(1, Integer.parseInt((String) uiDialog.getDownloadThreadsCombo().getSelectedItem()));
logger.info("Started download of " + cardsDownloadQueue.size() + " images"
+ " from source: " + selectedSource.getSourceName()
+ ", language: " + selectedSource.getCurrentLanguage().getCode()
+ ", threads: " + downloadThreadsAmount);
updateProgressMessage("Preparing download list...");
if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) {
update(0, cardsDownloadQueue.size());
ExecutorService executor = Executors.newFixedThreadPool(
downloadThreadsAmount,
new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_IMAGES_DOWNLOADER, false)
);
for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) {
try {
CardDownloadData card = cardsDownloadQueue.get(i);
logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')');
CardImageUrls urls;
if (card.isToken()) {
if (!"0".equals(card.getCollectorId())) {
continue;
}
urls = selectedSource.generateTokenUrl(card);
} else {
urls = selectedSource.generateCardUrl(card);
}
if (urls == null) {
String imageRef = selectedSource.getNextHttpImageUrl();
String fileName = selectedSource.getFileForHttpImage(imageRef);
if (imageRef != null && fileName != null) {
imageRef = selectedSource.getSourceName() + imageRef;
try {
card.setToken(selectedSource.isTokenSource());
Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages());
executor.execute(task);
} catch (Exception ex) {
}
} else if (selectedSource.getTotalImages() == -1) {
logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')');
synchronized (sync) {
update(cardIndex + 1, cardsDownloadQueue.size());
}
}
} else {
Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size());
executor.execute(task);
}
} catch (Exception ex) {
logger.error(ex, ex);
}
}
executor.shutdown();
while (!executor.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ignore) {
}
}
}
} catch (Throwable e) {
logger.error("Catch unknown error while downloading: " + e, e);
MageFrame.getInstance().showErrorDialog("Catch unknown error while downloading " + e, e);
} finally { } finally {
// // must close all active archives anyway
try {
TVFS.umount();
} catch (FsSyncException e) {
logger.fatal("Couldn't unmount zip files " + e, e);
MageFrame.getInstance().showErrorDialog("Couldn't unmount zip files " + e, e);
}
} }
// cleanup resources
Arrays.stream(DownloadSources.values()).forEach(resource -> {
resource.source.onFinished();
});
// stop // stop
reloadCardsToDownload(uiDialog.getSetsCombo().getSelectedItem().toString()); reloadCardsToDownload(uiDialog.getSetsCombo().getSelectedItem().toString());
enableDialogButtons(); enableDialogButtons();
@ -882,16 +888,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
// user cancelled // user cancelled
if (DownloadPicturesService.getInstance().isNeedCancel()) { if (DownloadPicturesService.getInstance().isNeedCancel()) {
// stop download, save current state and exit // stop download, save current state and exit
TFile archive = destFile.getTopLevelArchive(); // real archives save will be done on download service finish
///* not need to unmout/close - it's auto action
if (archive != null && archive.exists()) {
logger.info("User canceled download. Closing archive file: " + destFile.toString());
try {
TVFS.umount(archive);
} catch (Exception e) {
logger.error("Can't close archive file: " + e.getMessage(), e);
}
}
try { try {
TFile.rm(fileTempImage); TFile.rm(fileTempImage);
} catch (Exception e) { } catch (Exception e) {
@ -946,11 +943,11 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
if (cardIndex < needDownloadCount) { if (cardIndex < needDownloadCount) {
// downloading // downloading
float mb = ((needDownloadCount - lastCardIndex) * selectedSource.getAverageSizeKb()) / 1024; float mb = ((needDownloadCount - lastCardIndex) * selectedSource.getAverageSizeKb()) / 1024;
updateProgressMessage(String.format("%d of %d image downloading... Please wait [%.1f Mb left].", updateProgressMessage(String.format("%d of %d image downloading... Please wait [%.1f MB left].",
lastCardIndex, needDownloadCount, mb), lastCardIndex, needDownloadCount); lastCardIndex, needDownloadCount, mb), lastCardIndex, needDownloadCount);
} else { } else {
// finished // finished
updateProgressMessage("Image download DONE, refreshing stats... Please wait."); updateProgressMessage("Image download DONE, saving last files and refreshing stats... Please wait.");
List<CardDownloadData> downloadedCards = Collections.synchronizedList(new ArrayList<>()); List<CardDownloadData> downloadedCards = Collections.synchronizedList(new ArrayList<>());
DownloadPicturesService.this.cardsMissing.parallelStream().forEach(cardDownloadData -> { DownloadPicturesService.this.cardsMissing.parallelStream().forEach(cardDownloadData -> {
TFile file = new TFile(CardImageUtils.buildImagePathToCardOrToken(cardDownloadData)); TFile file = new TFile(CardImageUtils.buildImagePathToCardOrToken(cardDownloadData));

36
Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java
View file

@ -1,28 +1,16 @@
package org.mage.plugins.card.utils; package org.mage.plugins.card.utils;
import mage.cards.repository.CardInfo;
import mage.cards.repository.CardRepository;
import mage.cards.repository.TokenRepository; import mage.cards.repository.TokenRepository;
import mage.client.MageFrame;
import mage.client.constants.Constants; import mage.client.constants.Constants;
import mage.client.dialog.PreferencesDialog; import mage.client.dialog.PreferencesDialog;
import mage.client.remote.XmageURLConnection;
import mage.remote.Connection;
import mage.remote.Connection.ProxyType;
import mage.view.CardView; import mage.view.CardView;
import net.java.truevfs.access.TVFS;
import net.java.truevfs.kernel.spec.FsSyncException;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.images.CardDownloadData;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.nio.file.FileVisitResult; import java.nio.file.FileVisitResult;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -31,7 +19,6 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Locale; import java.util.Locale;
import java.util.prefs.Preferences;
public final class CardImageUtils { public final class CardImageUtils {
@ -191,19 +178,16 @@ public final class CardImageUtils {
return getImagesDir() + File.separator + "FACE" + File.separator + fixSetNameForWindows(setCode) + File.separator + prepareCardNameForFile(cardName) + ".jpg"; return getImagesDir() + File.separator + "FACE" + File.separator + fixSetNameForWindows(setCode) + File.separator + prepareCardNameForFile(cardName) + ".jpg";
} }
public static Proxy getProxyFromPreferences() {
Preferences prefs = MageFrame.getPreferences();
Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None"));
if (proxyType != ProxyType.NONE) {
String proxyServer = prefs.get("proxyAddress", "");
int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0"));
return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort));
}
return null;
}
public static void checkAndFixImageFiles() { public static void checkAndFixImageFiles() {
// search broken, temp or outdated files and delete it // search broken, temp or outdated files and delete it
// make sure all archives was closed (e.g. on second call of download dialog)
try {
TVFS.umount();
} catch (FsSyncException e) {
LOGGER.fatal("Couldn't unmount zip files on searching broken images " + e, e);
}
// real images check is slow, so it used on images download only (not here) // real images check is slow, so it used on images download only (not here)
Path rootPath = new File(CardImageUtils.getImagesDir()).toPath(); Path rootPath = new File(CardImageUtils.getImagesDir()).toPath();
if (!Files.exists(rootPath)) { if (!Files.exists(rootPath)) {

19
Mage.Plugins/Mage.Counter.Plugin/src/main/java/org/mage/plugins/counter/CounterPluginImpl.java
View file

@ -9,13 +9,14 @@ import net.xeoh.plugins.base.annotations.meta.Author;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import java.io.*; import java.io.*;
import java.nio.file.Files;
/** /**
* Implementation of {@link CounterPlugin}.<br/> * Implementation of {@link CounterPlugin}.<br/>
* Stores data in data folder. * Stores data in data folder.
* *
* @version 0.1 14.11.2010 Initial Version
* @author nantuko * @author nantuko
* @version 0.1 14.11.2010 Initial Version
*/ */
@PluginImplementation @PluginImplementation
@Author(name = "nantuko") @Author(name = "nantuko")
@ -69,11 +70,11 @@ public class CounterPluginImpl implements CounterPlugin {
if (data.exists()) { if (data.exists()) {
int prev = 0; int prev = 0;
try(ObjectInputStream ois = new ObjectInputStream(new FileInputStream(data))) { try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(data.toPath()))) {
Object o = ois.readObject(); Object o = ois.readObject();
CounterBean c; CounterBean c;
if (o instanceof CounterBean) { if (o instanceof CounterBean) {
c = (CounterBean)o; c = (CounterBean) o;
prev = c.getGamesPlayed(); prev = c.getGamesPlayed();
} }
} catch (EOFException e) { } catch (EOFException e) {
@ -84,10 +85,10 @@ public class CounterPluginImpl implements CounterPlugin {
throw new PluginException(e); throw new PluginException(e);
} }
try(ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(data))) { try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(data))) {
synchronized (this) { synchronized (this) {
CounterBean c = new CounterBean(); CounterBean c = new CounterBean();
c.setGamesPlayed(prev+1); c.setGamesPlayed(prev + 1);
oos.writeObject(c); oos.writeObject(c);
oos.close(); oos.close();
} }
@ -107,12 +108,12 @@ public class CounterPluginImpl implements CounterPlugin {
return 0; return 0;
} }
if (data.exists()) { if (data.exists()) {
try(ObjectInputStream ois = new ObjectInputStream(new FileInputStream(data))) { try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(data.toPath()))) {
synchronized (this) { synchronized (this) {
Object o = ois.readObject(); Object o = ois.readObject();
CounterBean c = null; CounterBean c = null;
if (o instanceof CounterBean) { if (o instanceof CounterBean) {
c = (CounterBean)o; c = (CounterBean) o;
} }
ois.close(); ois.close();
return c == null ? 0 : c.getGamesPlayed(); return c == null ? 0 : c.getGamesPlayed();

50
Mage.Server/src/main/java/mage/server/game/GameReplay.java
View file

@ -1,25 +1,23 @@
package mage.server.game; package mage.server.game;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInput;
import java.util.UUID;
import java.util.zip.GZIPInputStream;
import mage.game.Game; import mage.game.Game;
import mage.game.GameState; import mage.game.GameState;
import mage.game.GameStates; import mage.game.GameStates;
import mage.server.Main; import mage.server.Main;
import mage.util.CopierObjectInputStream; import mage.util.CopierObjectInputStream;
import mage.utils.StreamUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInput;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.UUID;
import java.util.zip.GZIPInputStream;
/** /**
* Replay system, outdated and not used. TODO: delete
* *
* @author BetaSteward_at_googlemail.com * @author BetaSteward_at_googlemail.com
*/ */
@ -59,33 +57,19 @@ public class GameReplay {
} }
private Game loadGame(UUID gameId) { private Game loadGame(UUID gameId) {
InputStream file = null; try (InputStream file = Files.newInputStream(Paths.get("saved/" + gameId.toString() + ".game"));
InputStream buffer = null; InputStream buffer = new BufferedInputStream(file);
InputStream gzip = null; InputStream gzip = new GZIPInputStream(buffer);
ObjectInput input = null; ObjectInput input = new CopierObjectInputStream(Main.classLoader, gzip)) {
try{
file = new FileInputStream("saved/" + gameId.toString() + ".game");
buffer = new BufferedInputStream(file);
gzip = new GZIPInputStream(buffer);
input = new CopierObjectInputStream(Main.classLoader, gzip);
Game loadGame = (Game) input.readObject(); Game loadGame = (Game) input.readObject();
GameStates states = (GameStates) input.readObject(); GameStates states = (GameStates) input.readObject();
loadGame.loadGameStates(states); loadGame.loadGameStates(states);
return loadGame; return loadGame;
} catch (ClassNotFoundException e) {
} logger.fatal("Cannot load game. Class not found.", e);
catch(ClassNotFoundException ex) { } catch (IOException e) {
logger.fatal("Cannot load game. Class not found.", ex); logger.fatal("Cannot load game:" + gameId, e);
}
catch(IOException ex) {
logger.fatal("Cannot load game:" + gameId, ex);
} finally {
StreamUtils.closeQuietly(file);
StreamUtils.closeQuietly(buffer);
StreamUtils.closeQuietly(input);
StreamUtils.closeQuietly(gzip);
} }
return null; return null;
} }
} }

2
Mage.Sets/src/mage/sets/SecretLairDrop.java
View file

@ -756,7 +756,7 @@ public class SecretLairDrop extends ExpansionSet {
cards.add(new SetCardInfo("Elvish Mystic", 805, Rarity.RARE, mage.cards.e.ElvishMystic.class, NON_FULL_USE_VARIOUS)); cards.add(new SetCardInfo("Elvish Mystic", 805, Rarity.RARE, mage.cards.e.ElvishMystic.class, NON_FULL_USE_VARIOUS));
cards.add(new SetCardInfo("Snapcaster Mage", 808, Rarity.MYTHIC, mage.cards.s.SnapcasterMage.class)); cards.add(new SetCardInfo("Snapcaster Mage", 808, Rarity.MYTHIC, mage.cards.s.SnapcasterMage.class));
cards.add(new SetCardInfo("Thrun, the Last Troll", 810, Rarity.MYTHIC, mage.cards.t.ThrunTheLastTroll.class)); cards.add(new SetCardInfo("Thrun, the Last Troll", 810, Rarity.MYTHIC, mage.cards.t.ThrunTheLastTroll.class));
//cards.add(new SetCardInfo("Elesh Norn, Grand Cenobite", 811, Rarity.MYTHIC, mage.cards.e.EleshNornGrandCenobite.class, NON_FULL_USE_VARIOUS)); // BUG: breaks download of card 209 cards.add(new SetCardInfo("Elesh Norn, Grand Cenobite", 811, Rarity.MYTHIC, mage.cards.e.EleshNornGrandCenobite.class, NON_FULL_USE_VARIOUS)); // BUG: breaks download of card 209
cards.add(new SetCardInfo("Arcane Signet", 820, Rarity.RARE, mage.cards.a.ArcaneSignet.class, NON_FULL_USE_VARIOUS)); cards.add(new SetCardInfo("Arcane Signet", 820, Rarity.RARE, mage.cards.a.ArcaneSignet.class, NON_FULL_USE_VARIOUS));
cards.add(new SetCardInfo("Norin the Wary", 827, Rarity.RARE, mage.cards.n.NorinTheWary.class)); cards.add(new SetCardInfo("Norin the Wary", 827, Rarity.RARE, mage.cards.n.NorinTheWary.class));
cards.add(new SetCardInfo("The Scarab God", 900, Rarity.MYTHIC, mage.cards.t.TheScarabGod.class)); cards.add(new SetCardInfo("The Scarab God", 900, Rarity.MYTHIC, mage.cards.t.TheScarabGod.class));

2
Mage/src/main/java/mage/players/net/UserData.java
View file

@ -26,7 +26,7 @@ public class UserData implements Serializable {
protected int autoTargetLevel; protected int autoTargetLevel;
protected boolean useSameSettingsForReplacementEffects; protected boolean useSameSettingsForReplacementEffects;
protected boolean useFirstManaAbility = false; protected boolean useFirstManaAbility = false;
private String userIdStr; private String userIdStr; // TODO: delete as un-used or use for hardware id instead?
protected Map<UUID, Set<UUID>> requestedHandPlayersList; // game -> players list protected Map<UUID, Set<UUID>> requestedHandPlayersList; // game -> players list
protected String matchHistory; protected String matchHistory;