forked from External/mage
download: reworked scryfall images support:
- download: fixed unmount zip errors on cancel download in some use cases (closes #12536); - download: significant download speed improvements (now it depends on user's network speed, not api limitations); - download: added additional error dialogs on bad use cases; - scryfall: added cards and bulk data api support; - scryfall: added bulk data download (updates once per week, contains all scryfall cards and store in images\downloading folder, 2 GB size); - scryfall: added optimized images download without api usage (use direct images links from bulk data, closes #11576); - scryfall: improved image source searching for some use cases (miss or wrong images problems, closes #12511); - scryfall: tokens don't use bulk data; - scryfall: 75k small images downloads 40 minutes and takes 1 GB and 2100 api calls (most of it from tokens); - scryfall: how-to disable bulk data, e.g. for api testing: -Dxmage.scryfallEnableBulkData=false
This commit is contained in:
Oleg Agafonov
parent
46f7304692
commit
0a55e37c8c
19 changed files with 884 additions and 216 deletions
|
|
@ -1355,7 +1355,7 @@ public class MageFrame extends javax.swing.JFrame implements MageClient {
|
|||
userRequestDialog.showDialog(userRequestMessage);
|
||||
}
|
||||
|
||||
public void showErrorDialog(String errorType, Exception e) {
|
||||
public void showErrorDialog(String errorType, Throwable e) {
|
||||
String errorMessage = e.getMessage();
|
||||
if (errorMessage == null || errorMessage.isEmpty() || errorMessage.equals("Null")) {
|
||||
errorMessage = e.getClass().getSimpleName() + " - look at server or client logs for more details";
|
||||
|
|
|
|||
|
|
@ -387,7 +387,7 @@
|
|||
<Property name="icon" type="javax.swing.Icon" editor="org.netbeans.modules.form.editors2.IconEditor">
|
||||
<Image iconType="3" name="/buttons/search_24.png"/>
|
||||
</Property>
|
||||
<Property name="toolTipText" type="java.lang.String" value="Fast search your flag"/>
|
||||
<Property name="toolTipText" type="java.lang.String" value="Search set to download"/>
|
||||
<Property name="alignmentX" type="float" value="1.0"/>
|
||||
<Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
|
||||
<Dimension value="[25, 25]"/>
|
||||
|
|
@ -419,7 +419,7 @@
|
|||
<SubComponents>
|
||||
<Component class="javax.swing.JCheckBox" name="checkboxRedownload">
|
||||
<Properties>
|
||||
<Property name="text" type="java.lang.String" value="<html>Re-download all images"/>
|
||||
<Property name="text" type="java.lang.String" value="<html>Re-download all selected images"/>
|
||||
<Property name="verticalAlignment" type="int" value="3"/>
|
||||
</Properties>
|
||||
<Constraints>
|
||||
|
|
|
|||
|
|
@ -358,7 +358,7 @@ public class DownloadImagesDialog extends MageDialog {
|
|||
panelModeSelect.add(fillerMode1);
|
||||
|
||||
buttonSearchSet.setIcon(new javax.swing.ImageIcon(getClass().getResource("/buttons/search_24.png"))); // NOI18N
|
||||
buttonSearchSet.setToolTipText("Fast search your flag");
|
||||
buttonSearchSet.setToolTipText("Search set to download");
|
||||
buttonSearchSet.setAlignmentX(1.0F);
|
||||
buttonSearchSet.setPreferredSize(new java.awt.Dimension(25, 25));
|
||||
buttonSearchSet.addActionListener(new java.awt.event.ActionListener() {
|
||||
|
|
@ -378,7 +378,7 @@ public class DownloadImagesDialog extends MageDialog {
|
|||
panelRedownload.setPreferredSize(new java.awt.Dimension(280, 100));
|
||||
panelRedownload.setLayout(new java.awt.BorderLayout());
|
||||
|
||||
checkboxRedownload.setText("<html>Re-download all images");
|
||||
checkboxRedownload.setText("<html>Re-download all selected images");
|
||||
checkboxRedownload.setVerticalAlignment(javax.swing.SwingConstants.BOTTOM);
|
||||
panelRedownload.add(checkboxRedownload, java.awt.BorderLayout.CENTER);
|
||||
panelRedownload.add(filler1, java.awt.BorderLayout.PAGE_END);
|
||||
|
|
@ -444,6 +444,7 @@ public class DownloadImagesDialog extends MageDialog {
|
|||
}//GEN-LAST:event_buttonStopActionPerformed
|
||||
|
||||
private void doClose(int retStatus) {
|
||||
|
||||
returnStatus = retStatus;
|
||||
setVisible(false);
|
||||
dispose();
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ public class XmageURLConnection {
|
|||
Proxy proxy = null;
|
||||
HttpURLConnection connection = null;
|
||||
HttpLoggingType loggingType = HttpLoggingType.ERRORS;
|
||||
boolean forceGZipEncoding = false;
|
||||
|
||||
public XmageURLConnection(String url) {
|
||||
this.url = url;
|
||||
|
|
@ -75,6 +76,10 @@ public class XmageURLConnection {
|
|||
}
|
||||
}
|
||||
|
||||
public void setForceGZipEncoding(boolean enable) {
|
||||
this.forceGZipEncoding = enable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to server
|
||||
*/
|
||||
|
|
@ -130,7 +135,11 @@ public class XmageURLConnection {
|
|||
}
|
||||
|
||||
private void initDefaultHeaders() {
|
||||
// warning, do not add Accept-Encoding - it processing inside URLConnection for http/https links (trying to use gzip by default)
|
||||
// warning, Accept-Encoding processing inside URLConnection for http/https links (trying to use gzip by default)
|
||||
// use force encoding for special use cases (example: download big text file as zip file)
|
||||
if (forceGZipEncoding) {
|
||||
this.connection.setRequestProperty("Accept-Encoding", "gzip");
|
||||
}
|
||||
|
||||
// user agent due standard notation User-Agent: <product> / <product-version> <comment>
|
||||
// warning, dot not add os, language and other details
|
||||
|
|
@ -290,13 +299,18 @@ public class XmageURLConnection {
|
|||
return "";
|
||||
}
|
||||
|
||||
public static InputStream downloadBinary(String resourceUrl) {
|
||||
return downloadBinary(resourceUrl, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fast download of binary data
|
||||
*
|
||||
* @return stream on OK 200 response or null on any other errors
|
||||
*/
|
||||
public static InputStream downloadBinary(String resourceUrl) {
|
||||
public static InputStream downloadBinary(String resourceUrl, boolean downloadAsGZip) {
|
||||
XmageURLConnection con = new XmageURLConnection(resourceUrl);
|
||||
con.setForceGZipEncoding(downloadAsGZip);
|
||||
con.startConnection();
|
||||
if (con.isConnected()) {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -301,10 +301,9 @@ public final class ManaSymbols {
|
|||
if (SvgUtils.haveSvgSupport()) {
|
||||
file = getSymbolFileNameAsSVG(symbol);
|
||||
if (file.exists()) {
|
||||
try {
|
||||
InputStream fileStream = new FileInputStream(file);
|
||||
try(InputStream fileStream = Files.newInputStream(file.toPath())) {
|
||||
image = loadSymbolAsSVG(fileStream, file.getPath(), size, size);
|
||||
} catch (FileNotFoundException ignore) {
|
||||
} catch (IOException ignore) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,10 @@ import mage.client.util.CardLanguage;
|
|||
import org.mage.plugins.card.dl.DownloadServiceInfo;
|
||||
import org.mage.plugins.card.images.CardDownloadData;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author North, JayDi85
|
||||
|
|
@ -70,4 +73,8 @@ public interface CardImageSource {
|
|||
default boolean isTokenImageProvided(String setCode, String cardName, Integer tokenNumber) {
|
||||
return false;
|
||||
}
|
||||
|
||||
default void onFinished() {
|
||||
// cleanup temp resources
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
package org.mage.plugins.card.dl.sources;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* Scryfall API: bulk file links to download
|
||||
* <p>
|
||||
* API docs: <a href="https://scryfall.com/docs/api/bulk-data">here</a>
|
||||
*
|
||||
* @author JayDi85
|
||||
*/
|
||||
public class ScryfallApiBulkData {
|
||||
|
||||
public String object;
|
||||
public String type;
|
||||
public Date updated_at;
|
||||
public long size;
|
||||
public String download_uri;
|
||||
}
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
package org.mage.plugins.card.dl.sources;
|
||||
|
||||
import mage.cards.decks.CardNameUtil;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Scryfall API: card info
|
||||
* <p>
|
||||
* API docs: <a href="https://scryfall.com/docs/api/cards">here</a>
|
||||
* <p>
|
||||
* Field values depends on data source, check it on null (as example: bulk data from oracle cards and bulk data from all cards)
|
||||
*
|
||||
* @author JayDi85
|
||||
*/
|
||||
public class ScryfallApiCard {
|
||||
|
||||
public String name;
|
||||
public String set; // set code
|
||||
public String collector_number;
|
||||
public String lang;
|
||||
public String layout; // card type like adventure, etc, see https://scryfall.com/docs/api/layouts
|
||||
public List<ScryfallApiCardFace> card_faces;
|
||||
public String image_status; // missing, placeholder, lowres, or highres_scan
|
||||
public Map<String, String> image_uris;
|
||||
|
||||
// fast access fields, fills on loading
|
||||
transient public String imageSmall = "";
|
||||
transient public String imageNormal = "";
|
||||
transient public String imageLarge = "";
|
||||
|
||||
// potentially interesting fields, can be used in other places
|
||||
//public UUID oracle_id; // TODO: implement card hint with oracle/cr ruling texts (see Rulings bulk data)
|
||||
//public Integer edhrec_rank; // TODO: use it to rating cards for AI and draft bots
|
||||
//public Object legalities; // TODO: add verify check for bans list
|
||||
//public Boolean full_art; // TODO: add verify check for full art usage in sets
|
||||
//public Object prices; // TODO: add total deck price and table limit by deck's price
|
||||
//public Date released_at; // the date this card was first released.
|
||||
//public String watermark; // background watermark image for some cards
|
||||
|
||||
public void prepareCompatibleData() {
|
||||
if (this.image_uris != null) {
|
||||
this.imageSmall = this.image_uris.getOrDefault("small", "");
|
||||
this.imageNormal = this.image_uris.getOrDefault("normal", "");
|
||||
this.imageLarge = this.image_uris.getOrDefault("large", "");
|
||||
this.image_uris = null;
|
||||
}
|
||||
|
||||
if (this.card_faces != null) {
|
||||
this.card_faces.forEach(ScryfallApiCardFace::prepareCompatibleData);
|
||||
}
|
||||
|
||||
// workaround for adventure card name fix:
|
||||
// - scryfall: Ondu Knotmaster // Throw a Line
|
||||
// - xmage: Ondu Knotmaster
|
||||
if (this.layout.equals("adventure")) {
|
||||
this.name = this.card_faces.get(0).name;
|
||||
}
|
||||
|
||||
// workaround for flip card name and image fixes:
|
||||
// - scryfall: Budoka Pupil // Ichiga, Who Topples Oaks
|
||||
// - xmage: Budoka Pupil
|
||||
if (this.layout.equals("flip")) {
|
||||
if (!this.card_faces.get(0).imageNormal.isEmpty()
|
||||
|| !this.card_faces.get(1).imageNormal.isEmpty()) {
|
||||
throw new IllegalArgumentException("Scryfall: unsupported data type, flip parts must not have images data in scryfall "
|
||||
+ this.set + " - " + this.collector_number + " - " + this.name);
|
||||
}
|
||||
|
||||
// fix name
|
||||
this.name = this.card_faces.get(0).name;
|
||||
|
||||
// fix image (xmage uses diff cards for flips, but it's same image)
|
||||
// example: https://scryfall.com/card/sok/103/homura-human-ascendant-homuras-essence
|
||||
this.card_faces.get(0).image_uris = this.image_uris;
|
||||
this.card_faces.get(0).imageSmall = this.imageSmall;
|
||||
this.card_faces.get(0).imageNormal = this.imageNormal;
|
||||
this.card_faces.get(0).imageLarge = this.imageLarge;
|
||||
this.card_faces.get(1).image_uris = this.image_uris;
|
||||
this.card_faces.get(1).imageSmall = this.imageSmall;
|
||||
this.card_faces.get(1).imageNormal = this.imageNormal;
|
||||
this.card_faces.get(1).imageLarge = this.imageLarge;
|
||||
}
|
||||
|
||||
// workaround for reversed cards:
|
||||
// - scryfall: Command Tower // Command Tower
|
||||
// - xmage: Command Tower (second side as diff card and direct link image), example: https://scryfall.com/card/rex/26/command-tower-command-tower
|
||||
if (this.layout.equals("reversible_card")) {
|
||||
if (!this.card_faces.get(0).name.equals(this.card_faces.get(1).name)) {
|
||||
throw new IllegalArgumentException("Scryfall: unsupported data type, reversible_card has diff faces "
|
||||
+ this.set + " - " + this.collector_number + " - " + this.name);
|
||||
}
|
||||
this.name = this.card_faces.get(0).name;
|
||||
}
|
||||
|
||||
// workaround for non ascii names
|
||||
// - scryfall uses original names like Arna Kennerüd, Skycaptain
|
||||
// - xmage need ascii only names like Arna Kennerud, Skycaptain
|
||||
this.name = CardNameUtil.normalizeCardName(this.name);
|
||||
|
||||
// workaround for non scii card numbers
|
||||
// - scryfall uses unicode numbers for reprints like Chandra Nalaar - dd2 - 34★ https://scryfall.com/card/dd2/34%E2%98%85/
|
||||
// - xmage uses ascii alternative Chandra Nalaar - dd2 - 34*
|
||||
this.collector_number = transformCardNumberFromScryfallToXmage(this.collector_number);
|
||||
|
||||
}
|
||||
|
||||
public static String transformCardNumberFromXmageToScryfall(String cardNumber) {
|
||||
String res = cardNumber;
|
||||
if (res.endsWith("*")) {
|
||||
res = res.substring(0, res.length() - 1) + "★";
|
||||
}
|
||||
if (res.endsWith("+")) {
|
||||
res = res.substring(0, res.length() - 1) + "†";
|
||||
}
|
||||
if (res.endsWith("Ph")) {
|
||||
res = res.substring(0, res.length() - 2) + "Φ";
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
public static String transformCardNumberFromScryfallToXmage(String cardNumber) {
|
||||
String res = cardNumber;
|
||||
if (res.endsWith("★")) {
|
||||
res = res.substring(0, res.length() - 1) + "*";
|
||||
}
|
||||
if (res.endsWith("†")) {
|
||||
res = res.substring(0, res.length() - 1) + "+";
|
||||
}
|
||||
if (res.endsWith("Φ")) {
|
||||
res = res.substring(0, res.length() - 1) + "Ph";
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
public String findImage(String imageSize) {
|
||||
// api possible values:
|
||||
// - small
|
||||
// - normal
|
||||
// - large
|
||||
// - png
|
||||
// - art_crop
|
||||
// - border_crop
|
||||
switch (imageSize) {
|
||||
case "small":
|
||||
return this.imageSmall;
|
||||
case "normal":
|
||||
return this.imageNormal;
|
||||
case "large":
|
||||
return this.imageLarge;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported image size: " + imageSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package org.mage.plugins.card.dl.sources;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Scryfall API: card face info (used for double faced cards)
|
||||
* <p>
|
||||
* API docs: <a href="https://scryfall.com/docs/api/cards">here</a>
|
||||
* <p>
|
||||
*
|
||||
* @author JayDi85
|
||||
*/
|
||||
public class ScryfallApiCardFace {
|
||||
public String name;
|
||||
public Map<String, String> image_uris;
|
||||
|
||||
// fast access fields, fills on loading
|
||||
transient public String imageSmall = "";
|
||||
transient public String imageNormal = "";
|
||||
transient public String imageLarge = "";
|
||||
|
||||
public void prepareCompatibleData() {
|
||||
if (this.image_uris != null) {
|
||||
this.imageSmall = this.image_uris.getOrDefault("small", "");
|
||||
this.imageNormal = this.image_uris.getOrDefault("normal", "");
|
||||
this.imageLarge = this.image_uris.getOrDefault("large", "");
|
||||
this.image_uris = null;
|
||||
}
|
||||
}
|
||||
|
||||
public String findImage(String imageSize) {
|
||||
// api possible values:
|
||||
// - small
|
||||
// - normal
|
||||
// - large
|
||||
// - png
|
||||
// - art_crop
|
||||
// - border_crop
|
||||
switch (imageSize) {
|
||||
case "small":
|
||||
return this.imageSmall;
|
||||
case "normal":
|
||||
return this.imageNormal;
|
||||
case "large":
|
||||
return this.imageLarge;
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported image size: " + imageSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,22 +1,36 @@
|
|||
package org.mage.plugins.card.dl.sources;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import com.google.gson.stream.JsonReader;
|
||||
import mage.MageException;
|
||||
import mage.client.remote.XmageURLConnection;
|
||||
import mage.client.util.CardLanguage;
|
||||
import mage.util.JsonUtil;
|
||||
import net.java.truevfs.access.TFile;
|
||||
import net.java.truevfs.access.TFileInputStream;
|
||||
import net.java.truevfs.access.TFileOutputStream;
|
||||
import net.java.truevfs.access.TVFS;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.mage.plugins.card.dl.DownloadServiceInfo;
|
||||
import org.mage.plugins.card.images.CardDownloadData;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir;
|
||||
|
||||
/**
|
||||
* Download: scryfall API support to download images and other data
|
||||
*
|
||||
* @author JayDi85
|
||||
*/
|
||||
public class ScryfallImageSource implements CardImageSource {
|
||||
|
|
@ -32,6 +46,21 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
private static final ReentrantLock waitBeforeRequestLock = new ReentrantLock();
|
||||
private static final int DOWNLOAD_TIMEOUT_MS = 300;
|
||||
|
||||
// bulk images optimization, how it works
|
||||
// - scryfall limit all calls to API endpoints, but allow direct calls to CDN (scryfall.io) without limitation
|
||||
// - so download and prepare whole scryfall database
|
||||
// - for each downloading card try to find it in bulk database and use direct link from it
|
||||
// - only card images supported (without tokens, emblems, etc)
|
||||
private static final boolean SCRYFALL_BULK_FILES_ENABLED = true; // use bulk data to find direct links instead API calls
|
||||
private static final long SCRYFALL_BULK_FILES_OUTDATE_IN_MILLIS = 7 * 24 * 3600 * 1000; // after 1 week need to download again
|
||||
private static final String SCRYFALL_BULK_FILES_DATABASE_SOURCE_API = "https://api.scryfall.com/bulk-data/all-cards"; // 300 MB in zip
|
||||
private static final boolean SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE = false; // default: false - for faster debug only, ignore write operations
|
||||
// run app with -Dxmage.scryfallEnableBulkData=false to disable bulk data (e.g. for testing api links generation)
|
||||
private static final String SCRYFALL_BULK_FILES_PROPERTY = "xmage.scryfallEnableBulkData";
|
||||
|
||||
private static final List<ScryfallApiCard> bulkCardsDatabaseAll = new ArrayList<>(); // 100MB memory footprint, cleaning on download stop
|
||||
private static final Map<String, ScryfallApiCard> bulkCardsDatabaseDefault = new HashMap<>(); // card/set/number
|
||||
|
||||
public static ScryfallImageSource getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
|
@ -117,7 +146,7 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
// basic cards by api call (redirect to img link)
|
||||
// example: https://api.scryfall.com/cards/xln/121/en?format=image
|
||||
if (baseUrl == null) {
|
||||
String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId());
|
||||
String cn = ScryfallApiCard.transformCardNumberFromXmageToScryfall(card.getCollectorId());
|
||||
baseUrl = String.format("https://api.scryfall.com/cards/%s/%s/%s?format=image",
|
||||
formatSetName(card.getSet(), isToken),
|
||||
cn,
|
||||
|
|
@ -132,12 +161,12 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
// include_variations=true added to deal with the cards that scryfall has marked as variations that seem to sometimes fail
|
||||
// eg https://api.scryfall.com/cards/4ed/134†?format=image fails
|
||||
// eg https://api.scryfall.com/cards/4ed/134†?format=image&include_variations=true succeeds
|
||||
|
||||
}
|
||||
|
||||
return new CardImageUrls(baseUrl, alternativeUrl);
|
||||
}
|
||||
|
||||
// TODO: delete face code after bulk data implemented?
|
||||
private String getFaceImageUrl(CardDownloadData card, boolean isToken) throws Exception {
|
||||
final String defaultCode = CardLanguage.ENGLISH.getCode();
|
||||
final String localizedCode = languageAliases.getOrDefault(this.getCurrentLanguage(), defaultCode);
|
||||
|
|
@ -147,14 +176,8 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
|
||||
String apiUrl = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId());
|
||||
if (apiUrl != null) {
|
||||
if (apiUrl.endsWith("*/")) {
|
||||
apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "★/";
|
||||
} else if (apiUrl.endsWith("+/")) {
|
||||
apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "†/";
|
||||
} else if (apiUrl.endsWith("Ph/")) {
|
||||
apiUrl = apiUrl.substring(0, apiUrl.length() - 3) + "Φ/";
|
||||
}
|
||||
// BY DIRECT URL
|
||||
// direct url already contains scryfall compatible card numbers (with unicode chars)
|
||||
// direct links via hardcoded API path. Used for cards with non-ASCII collector numbers
|
||||
if (localizedCode.equals(defaultCode)) {
|
||||
// english only, so can use workaround without loc param (scryfall download first available card)
|
||||
|
|
@ -168,7 +191,7 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
} else {
|
||||
// BY CARD NUMBER
|
||||
// localized and default
|
||||
String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId());
|
||||
String cn = ScryfallApiCard.transformCardNumberFromXmageToScryfall(card.getCollectorId());
|
||||
needUrls.add(String.format("https://api.scryfall.com/cards/%s/%s/%s",
|
||||
formatSetName(card.getSet(), isToken),
|
||||
cn,
|
||||
|
|
@ -221,9 +244,23 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
|
||||
@Override
|
||||
public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
|
||||
// prepare download list example (
|
||||
preparedUrls.clear();
|
||||
|
||||
// direct images without api calls
|
||||
if (!prepareBulkData(downloadServiceInfo, downloadList)) {
|
||||
return false;
|
||||
}
|
||||
analyseBulkData(downloadServiceInfo, downloadList);
|
||||
|
||||
// second side images need lookup for
|
||||
if (!prepareSecondSideImages(downloadServiceInfo, downloadList)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean prepareSecondSideImages(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
|
||||
// prepare stats
|
||||
int needPrepareCount = 0;
|
||||
int currentPrepareCount = 0;
|
||||
|
|
@ -235,11 +272,16 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
updatePrepareStats(downloadServiceInfo, needPrepareCount, currentPrepareCount);
|
||||
|
||||
for (CardDownloadData card : downloadList) {
|
||||
// need cancel
|
||||
// fast cancel
|
||||
if (downloadServiceInfo.isNeedCancel()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// already prepare, e.g. on bulk data
|
||||
if (preparedUrls.containsKey(card)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// prepare the back face URL
|
||||
if (card.isSecondSide()) {
|
||||
currentPrepareCount++;
|
||||
|
|
@ -259,10 +301,410 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
return true;
|
||||
}
|
||||
|
||||
private boolean prepareBulkData(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
|
||||
boolean isEnabled = SCRYFALL_BULK_FILES_ENABLED;
|
||||
if (System.getProperty(SCRYFALL_BULK_FILES_PROPERTY) != null) {
|
||||
isEnabled = Boolean.parseBoolean(System.getProperty(SCRYFALL_BULK_FILES_PROPERTY));
|
||||
}
|
||||
if (!isEnabled) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// if up to date
|
||||
if (isBulkDataPrepared()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// NEED TO DOWNLOAD
|
||||
|
||||
// clean
|
||||
TFile bulkTempFile = prepareTempFileForBulkData();
|
||||
if (bulkTempFile.exists()) {
|
||||
try {
|
||||
if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
|
||||
bulkTempFile.rm();
|
||||
TVFS.umount();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// find actual download link
|
||||
String s = XmageURLConnection.downloadText(SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
|
||||
if (s.isEmpty()) {
|
||||
logger.error("Can't get bulk info from scryfall api " + SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
|
||||
return false;
|
||||
}
|
||||
ScryfallApiBulkData bulkData = new Gson().fromJson(s, ScryfallApiBulkData.class);
|
||||
if (bulkData == null
|
||||
|| bulkData.download_uri == null
|
||||
|| bulkData.download_uri.isEmpty()
|
||||
|| bulkData.size == 0) {
|
||||
logger.error("Unknown bulk info format from scryfall api " + SCRYFALL_BULK_FILES_DATABASE_SOURCE_API);
|
||||
return false;
|
||||
}
|
||||
|
||||
// download
|
||||
if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
|
||||
logger.info("Scryfall: downloading additional data files, size " + bulkData.size / (1024 * 1024) + " MB");
|
||||
String url = bulkData.download_uri;
|
||||
InputStream inputStream = XmageURLConnection.downloadBinary(url, true);
|
||||
OutputStream outputStream = null;
|
||||
try {
|
||||
try {
|
||||
if (inputStream == null) {
|
||||
logger.error("Can't get bulk data from scryfall api " + url);
|
||||
return false;
|
||||
}
|
||||
outputStream = new TFileOutputStream(bulkTempFile);
|
||||
|
||||
byte[] buf = new byte[5 * 1024 * 1024]; // 5 MB buffer
|
||||
int len;
|
||||
long needDownload = bulkData.size / 7; // text -> zip size multiplier
|
||||
long doneDownload = 0;
|
||||
while ((len = inputStream.read(buf)) != -1) {
|
||||
// fast cancel
|
||||
if (downloadServiceInfo.isNeedCancel()) {
|
||||
// stop download and delete current file
|
||||
inputStream.close();
|
||||
outputStream.close();
|
||||
if (bulkTempFile.exists()) {
|
||||
bulkTempFile.rm();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// all fine, can save data part
|
||||
outputStream.write(buf, 0, len);
|
||||
|
||||
doneDownload += len;
|
||||
updateBulkDownloadStats(downloadServiceInfo, doneDownload, needDownload);
|
||||
}
|
||||
outputStream.flush();
|
||||
|
||||
// unpack
|
||||
logger.info("Scryfall: unpacking files...");
|
||||
Path zippedBulkPath = Paths.get(getBulkTempFileName());
|
||||
Path textBulkPath = Paths.get(getBulkStaticFileName());
|
||||
Files.deleteIfExists(textBulkPath);
|
||||
try (GZIPInputStream in = new GZIPInputStream(Files.newInputStream(zippedBulkPath));
|
||||
FileOutputStream out = new FileOutputStream(textBulkPath.toString())) {
|
||||
byte[] buffer = new byte[5 * 1024 * 1024];
|
||||
long needUnpack = bulkData.size; // zip -> text
|
||||
long doneUnpack = 0;
|
||||
while ((len = in.read(buffer)) > 0) {
|
||||
// fast cancel
|
||||
if (downloadServiceInfo.isNeedCancel()) {
|
||||
out.flush();
|
||||
Files.deleteIfExists(textBulkPath);
|
||||
return false;
|
||||
}
|
||||
|
||||
out.write(buffer, 0, len);
|
||||
|
||||
doneUnpack += len;
|
||||
updateBulkUnpackingStats(downloadServiceInfo, doneUnpack, needUnpack);
|
||||
}
|
||||
}
|
||||
logger.info("Scryfall: unpacking done");
|
||||
|
||||
// all fine
|
||||
} finally {
|
||||
if (inputStream != null) {
|
||||
inputStream.close();
|
||||
}
|
||||
if (outputStream != null) {
|
||||
outputStream.close();
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("Catch unknown error while download bulk data from scryfall api " + url, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return isBulkDataPrepared();
|
||||
}
|
||||
|
||||
private boolean isBulkDataPrepared() {
|
||||
|
||||
// already loaded
|
||||
if (bulkCardsDatabaseAll.size() > 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// file not exists
|
||||
Path textBulkPath = Paths.get(getBulkStaticFileName());
|
||||
if (!Files.exists(textBulkPath)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// file outdated
|
||||
try {
|
||||
if (System.currentTimeMillis() - Files.getLastModifiedTime(textBulkPath).toMillis() > SCRYFALL_BULK_FILES_OUTDATE_IN_MILLIS) {
|
||||
logger.info("Scryfall: bulk files outdated, need to download new");
|
||||
return false;
|
||||
}
|
||||
} catch (IOException ignore) {
|
||||
}
|
||||
|
||||
// bulk files are too big, so must read and parse it in stream mode only
|
||||
// 500k reprints in diff languages
|
||||
Gson gson = new Gson();
|
||||
try (TFileInputStream inputStream = new TFileInputStream(textBulkPath.toFile());
|
||||
InputStreamReader inputReader = new InputStreamReader(inputStream);
|
||||
JsonReader jsonReader = new JsonReader(inputReader)) {
|
||||
|
||||
bulkCardsDatabaseAll.clear();
|
||||
bulkCardsDatabaseDefault.clear();
|
||||
|
||||
jsonReader.beginArray();
|
||||
while (jsonReader.hasNext()) {
|
||||
ScryfallApiCard card = gson.fromJson(jsonReader, ScryfallApiCard.class);
|
||||
|
||||
// prepare data
|
||||
// memory optimization: fewer data, from 1145 MB to 470 MB
|
||||
card.prepareCompatibleData();
|
||||
|
||||
// keep only usefully languages
|
||||
// memory optimization: fewer items, from 470 MB to 96 MB
|
||||
// make sure final list will have one version
|
||||
// example: japan only card must be in final list for any selected languages https://scryfall.com/card/war/180★/
|
||||
String key = String.format("%s/%s/%s", card.name, card.set, card.collector_number);
|
||||
if (!card.lang.equals(this.currentLanguage.getCode())
|
||||
&& !card.lang.equals(CardLanguage.ENGLISH.getCode())
|
||||
&& bulkCardsDatabaseDefault.containsKey(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// default versions list depends on scryfall bulk data order (en first)
|
||||
bulkCardsDatabaseDefault.put(key, card);
|
||||
bulkCardsDatabaseAll.add(card);
|
||||
}
|
||||
jsonReader.close();
|
||||
return bulkCardsDatabaseAll.size() > 0;
|
||||
} catch (Exception e) {
|
||||
logger.error("Can't read bulk file (possible reason: broken format)");
|
||||
try {
|
||||
// clean up
|
||||
if (!SCRYFALL_BULK_FILES_DEBUG_READ_ONLY_MODE) {
|
||||
Files.deleteIfExists(textBulkPath);
|
||||
}
|
||||
} catch (IOException ignore) {
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void analyseBulkData(DownloadServiceInfo downloadServiceInfo, List<CardDownloadData> downloadList) {
|
||||
// prepare card indexes
|
||||
// name/set/number/lang - normal cards
|
||||
Map<String, String> bulkImagesIndexAll = createBulkImagesIndex(downloadServiceInfo, bulkCardsDatabaseAll, true);
|
||||
// name/set/number - default cards
|
||||
Map<String, String> bulkImagesIndexDefault = createBulkImagesIndex(downloadServiceInfo, bulkCardsDatabaseDefault.values(), false);
|
||||
|
||||
// find good images
|
||||
AtomicInteger statsOldPrepared = new AtomicInteger();
|
||||
AtomicInteger statsDirectLinks = new AtomicInteger();
|
||||
AtomicInteger statsNewPrepared = new AtomicInteger();
|
||||
AtomicInteger statsNotFound = new AtomicInteger();
|
||||
for (CardDownloadData card : downloadList) {
|
||||
// fast cancel
|
||||
if (downloadServiceInfo.isNeedCancel()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// already prepared, e.g. from direct download links or other sources
|
||||
if (preparedUrls.containsKey(card)) {
|
||||
statsOldPrepared.incrementAndGet();
|
||||
continue;
|
||||
}
|
||||
|
||||
// only cards supported
|
||||
if (card.isToken()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// ignore direct links
|
||||
String directLink = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId());
|
||||
if (directLink != null) {
|
||||
statsDirectLinks.incrementAndGet();
|
||||
continue;
|
||||
}
|
||||
|
||||
String searchingName = card.getName();
|
||||
String searchingSet = card.getSet().toLowerCase(Locale.ENGLISH);
|
||||
String searchingNumber = card.getCollectorId();
|
||||
|
||||
// current language
|
||||
String key = String.format("%s/%s/%s/%s",
|
||||
searchingName,
|
||||
searchingSet,
|
||||
searchingNumber,
|
||||
this.currentLanguage
|
||||
);
|
||||
String link = bulkImagesIndexAll.getOrDefault(key, "");
|
||||
|
||||
// default en language
|
||||
if (link.isEmpty()) {
|
||||
key = String.format("%s/%s/%s/%s",
|
||||
searchingName,
|
||||
searchingSet,
|
||||
searchingNumber,
|
||||
CardLanguage.ENGLISH.getCode()
|
||||
);
|
||||
link = bulkImagesIndexAll.getOrDefault(key, "");
|
||||
}
|
||||
|
||||
// default non en-language
|
||||
if (link.isEmpty()) {
|
||||
key = String.format("%s/%s/%s/",
|
||||
searchingName,
|
||||
searchingSet,
|
||||
searchingNumber);
|
||||
link = bulkImagesIndexDefault.getOrDefault(key, "");
|
||||
}
|
||||
|
||||
if (link.isEmpty()) {
|
||||
// how-to fix:
|
||||
// - make sure name notation compatible, see workarounds in ScryfallApiCard.prepareCompatibleData
|
||||
// - makue sure it's not fake card (xmage can use fake cards instead doubled-images) -- add it to ScryfallImageSupportCards.directDownloadLinks
|
||||
logger.info("Scryfall: bulk image not found (outdated cards data in xmage?): " + key + " ");
|
||||
statsNotFound.incrementAndGet();
|
||||
} else {
|
||||
preparedUrls.put(card, link);
|
||||
statsNewPrepared.incrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(String.format("Scryfall: bulk optimization result - need cards: %d, already prepared: %d, direct links: %d, new prepared: %d, NOT FOUND: %d",
|
||||
downloadList.size(),
|
||||
statsOldPrepared.get(),
|
||||
statsDirectLinks.get(),
|
||||
statsNewPrepared.get(),
|
||||
statsNotFound.get() // all not founded cards must be fixed
|
||||
));
|
||||
}
|
||||
|
||||
private Map<String, String> createBulkImagesIndex(DownloadServiceInfo downloadServiceInfo, Collection<ScryfallApiCard> sourceList, boolean useLocalization) {
|
||||
Map<String, String> res = new HashMap<>();
|
||||
for (ScryfallApiCard card : sourceList) {
|
||||
// fast cancel
|
||||
if (downloadServiceInfo.isNeedCancel()) {
|
||||
return res;
|
||||
}
|
||||
|
||||
// main card
|
||||
String image = card.findImage(getImageQuality());
|
||||
if (!image.isEmpty()) {
|
||||
String mainKey = String.format("%s/%s/%s/%s",
|
||||
card.name,
|
||||
card.set,
|
||||
card.collector_number,
|
||||
useLocalization ? card.lang : ""
|
||||
);
|
||||
if (res.containsKey(mainKey)) {
|
||||
// how-to fix: rework whole card number logic in xmage and scryfall
|
||||
throw new IllegalArgumentException("Wrong code usage: scryfall used unique card numbers, but found duplicated: " + mainKey);
|
||||
}
|
||||
res.put(mainKey, image);
|
||||
}
|
||||
|
||||
// faces
|
||||
if (card.card_faces != null) {
|
||||
// hints:
|
||||
// 1. Not all faces has images - example: adventure cards
|
||||
// 2. Some faces contain fake data in second face, search it on scryfall by "set_type:memorabilia"
|
||||
// example: https://scryfall.com/card/aznr/25/clearwater-pathway-clearwater-pathway
|
||||
// 3. Some faces contain diff images of the same card, layout = reversible_card
|
||||
// example: https://scryfall.com/card/rex/26/command-tower-command-tower
|
||||
|
||||
Set<String> usedNames = new HashSet<>();
|
||||
card.card_faces.forEach(face -> {
|
||||
// workaround to ignore fake data, see above about memorabilia
|
||||
if (usedNames.contains(face.name)) {
|
||||
return;
|
||||
}
|
||||
usedNames.add(face.name);
|
||||
|
||||
String faceImage = face.findImage(getImageQuality());
|
||||
if (!faceImage.isEmpty()) {
|
||||
String faceKey = String.format("%s/%s/%s/%s",
|
||||
face.name,
|
||||
card.set,
|
||||
card.collector_number,
|
||||
useLocalization ? card.lang : ""
|
||||
);
|
||||
|
||||
// workaround for flip cards - ignore first face - it already in the index
|
||||
// example: https://scryfall.com/card/sok/103/homura-human-ascendant-homuras-essence
|
||||
if (card.layout.equals("flip")
|
||||
&& card.name.equals(face.name)
|
||||
&& res.containsKey(faceKey)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (res.containsKey(faceKey)) {
|
||||
// how-to fix: rework whole card number logic in xmage and scryfall
|
||||
throw new IllegalArgumentException("Wrong code usage: scryfall used unique card numbers, but found duplicated: " + faceKey);
|
||||
}
|
||||
res.put(faceKey, faceImage);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
private String getBulkTempFileName() {
|
||||
return getImagesDir() + File.separator + "downloading" + File.separator + "scryfall_bulk_cards.json.gz";
|
||||
}
|
||||
|
||||
private String getBulkStaticFileName() {
|
||||
return getImagesDir() + File.separator + "downloading" + File.separator + "scryfall_bulk_cards.json";
|
||||
}
|
||||
|
||||
private TFile prepareTempFileForBulkData() {
|
||||
TFile file = new TFile(getBulkTempFileName());
|
||||
TFile parent = file.getParentFile();
|
||||
if (parent != null) {
|
||||
if (!parent.exists()) {
|
||||
parent.mkdirs();
|
||||
}
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
private void updateBulkDownloadStats(DownloadServiceInfo service, long done, long need) {
|
||||
int doneMb = (int) (done / (1024 * 1024));
|
||||
int needMb = (int) (need / (1024 * 1024));
|
||||
synchronized (service.getSync()) {
|
||||
service.updateProgressMessage(
|
||||
String.format("Step 1 of 3. Downloading additional data... %d of %d MB", doneMb, needMb),
|
||||
doneMb,
|
||||
needMb
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateBulkUnpackingStats(DownloadServiceInfo service, long done, long need) {
|
||||
int doneMb = (int) (done / (1024 * 1024));
|
||||
int needMb = (int) (need / (1024 * 1024));
|
||||
synchronized (service.getSync()) {
|
||||
service.updateProgressMessage(
|
||||
String.format("Step 2 of 3. Unpacking additional files... %d of %d MB", doneMb, needMb),
|
||||
doneMb,
|
||||
needMb
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void updatePrepareStats(DownloadServiceInfo service, int need, int current) {
|
||||
synchronized (service.getSync()) {
|
||||
service.updateProgressMessage(
|
||||
String.format("Preparing download list... %d of %d", current, need),
|
||||
String.format("Step 3 of 3. Preparing download list... %d of %d", current, need),
|
||||
current,
|
||||
need
|
||||
);
|
||||
|
|
@ -334,7 +776,12 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
public void doPause(String fullUrl) {
|
||||
// scryfall recommends 300 ms timeout per each request to API to work under a rate limit
|
||||
// possible error: 429 Too Many Requests
|
||||
// TODO: add diff endpoint supports (api calls with timeout, cdn/direct calls without timeout)
|
||||
|
||||
// cdn source can be safe and called without pause
|
||||
if (fullUrl.contains("scryfall.io")) {
|
||||
return;
|
||||
}
|
||||
|
||||
waitBeforeRequest();
|
||||
}
|
||||
|
||||
|
|
@ -387,4 +834,18 @@ public class ScryfallImageSource implements CardImageSource {
|
|||
// only direct tokens from set
|
||||
return ScryfallImageSupportTokens.findTokenLink(setCode, cardName, tokenNumber) != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Image quality
|
||||
*/
|
||||
public String getImageQuality() {
|
||||
return "large";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFinished() {
|
||||
// cleanup resources
|
||||
bulkCardsDatabaseAll.clear();
|
||||
bulkCardsDatabaseDefault.clear();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ public class ScryfallImageSourceNormal extends ScryfallImageSource {
|
|||
}
|
||||
|
||||
private static String innerModifyUrlString(String oneUrl) {
|
||||
return oneUrl.replaceFirst("/large/","/normal/").replaceFirst("format=image","format=image&version=normal");
|
||||
return oneUrl.replaceFirst("/large/", "/normal/").replaceFirst("format=image", "format=image&version=normal");
|
||||
}
|
||||
|
||||
private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) {
|
||||
|
|
@ -48,4 +48,8 @@ public class ScryfallImageSourceNormal extends ScryfallImageSource {
|
|||
return 92f;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getImageQuality() {
|
||||
return "normal";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ public class ScryfallImageSourceSmall extends ScryfallImageSource {
|
|||
}
|
||||
|
||||
private static String innerModifyUrlString(String oneUrl) {
|
||||
return oneUrl.replaceFirst("/large/","/small/").replaceFirst("format=image","format=image&version=small");
|
||||
return oneUrl.replaceFirst("/large/", "/small/").replaceFirst("format=image", "format=image&version=small");
|
||||
}
|
||||
|
||||
private static CardImageUrls innerModifyUrl(CardImageUrls cardUrls) {
|
||||
|
|
@ -48,4 +48,8 @@ public class ScryfallImageSourceSmall extends ScryfallImageSource {
|
|||
return 14f;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getImageQuality() {
|
||||
return "small";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -681,19 +681,6 @@ public class ScryfallImageSupportCards {
|
|||
return null;
|
||||
}
|
||||
|
||||
public static String prepareCardNumber(String xmageCardNumber) {
|
||||
if (xmageCardNumber.endsWith("*")) {
|
||||
return xmageCardNumber.substring(0, xmageCardNumber.length() - 1) + "★";
|
||||
}
|
||||
if (xmageCardNumber.endsWith("+")) {
|
||||
return xmageCardNumber.substring(0, xmageCardNumber.length() - 1) + "†";
|
||||
}
|
||||
if (xmageCardNumber.endsWith("Ph")) {
|
||||
return xmageCardNumber.substring(0, xmageCardNumber.length() - 2) + "Φ";
|
||||
}
|
||||
return xmageCardNumber;
|
||||
}
|
||||
|
||||
public static String findDirectDownloadLink(String setCode, String cardName, String cardNumber) {
|
||||
String key = findDirectDownloadKey(setCode, cardName, cardNumber);
|
||||
return directDownloadLinks.get(key);
|
||||
|
|
|
|||
|
|
@ -66,8 +66,8 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
|
||||
// protect from wrong data save
|
||||
// there are possible land images with small sizes, so must research content in check
|
||||
private static final int MIN_FILE_SIZE_OF_GOOD_IMAGE = 1024 * 6; // broken
|
||||
private static final int MIN_FILE_SIZE_OF_POSSIBLE_BAD_IMAGE = 1024 * 15; // possible broken (need content check)
|
||||
private static final int MIN_FILE_SIZE_OF_GOOD_IMAGE = 1024 * 6; // smaller files will be mark as broken
|
||||
private static final int MIN_FILE_SIZE_OF_POSSIBLE_BAD_IMAGE = 1024 * 8; // smaller files will be checked for possible broken mark (slow)
|
||||
|
||||
private final DownloadImagesDialog uiDialog;
|
||||
private boolean needCancel;
|
||||
|
|
@ -77,8 +77,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
private List<CardInfo> cardsAll;
|
||||
private List<CardDownloadData> cardsMissing;
|
||||
private final List<CardDownloadData> cardsDownloadQueue;
|
||||
private int missingCardsCount = 0;
|
||||
private int missingTokensCount = 0;
|
||||
|
||||
private final List<String> selectedSets = new ArrayList<>();
|
||||
private static CardImageSource selectedSource;
|
||||
|
|
@ -429,13 +427,13 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
}
|
||||
|
||||
private void updateProgressText(int cardCount, int tokenCount) {
|
||||
missingTokensCount = 0;
|
||||
int missingTokensCount = 0;
|
||||
for (CardDownloadData card : cardsMissing) {
|
||||
if (card.isToken()) {
|
||||
missingTokensCount++;
|
||||
}
|
||||
}
|
||||
missingCardsCount = cardsMissing.size() - missingTokensCount;
|
||||
int missingCardsCount = cardsMissing.size() - missingTokensCount;
|
||||
|
||||
uiDialog.setCurrentInfo("Missing: " + missingCardsCount + " card images / " + missingTokensCount + " token images");
|
||||
int imageSum = cardCount + tokenCount;
|
||||
|
|
@ -446,7 +444,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
} else if (cardIndex == 0) {
|
||||
statusEnd = "image download NOT STARTED. Please start.";
|
||||
} else {
|
||||
statusEnd = String.format("image downloading... Please wait [%.1f Mb left].", mb);
|
||||
statusEnd = String.format("image downloading... Please wait [%.1f MB left].", mb);
|
||||
}
|
||||
updateProgressMessage(String.format("%d of %d (%d cards and %d tokens) %s",
|
||||
0, imageSum, cardCount, tokenCount, statusEnd
|
||||
|
|
@ -618,7 +616,8 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
}
|
||||
});
|
||||
|
||||
if (badContentChecks.get() > 10000) {
|
||||
if (badContentChecks.get() > 1000) {
|
||||
// how-to fix: download full small images and run checks -- make sure it finds fewer files to check
|
||||
logger.warn("Wrong code usage: too many file content checks (" + badContentChecks.get() + ") - try to decrease min file size");
|
||||
}
|
||||
|
||||
|
|
@ -630,6 +629,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
this.cardIndex = 0;
|
||||
this.resetErrorCount();
|
||||
|
||||
try {
|
||||
File base = new File(getImagesDir());
|
||||
if (!base.exists()) {
|
||||
base.mkdir();
|
||||
|
|
@ -637,7 +637,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
|
||||
int downloadThreadsAmount = Math.max(1, Integer.parseInt((String) uiDialog.getDownloadThreadsCombo().getSelectedItem()));
|
||||
|
||||
|
||||
logger.info("Started download of " + cardsDownloadQueue.size() + " images"
|
||||
+ " from source: " + selectedSource.getSourceName()
|
||||
+ ", language: " + selectedSource.getCurrentLanguage().getCode()
|
||||
|
|
@ -699,16 +698,23 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (Throwable e) {
|
||||
logger.error("Catch unknown error while downloading: " + e, e);
|
||||
MageFrame.getInstance().showErrorDialog("Catch unknown error while downloading " + e, e);
|
||||
} finally {
|
||||
// must close all active archives anyway
|
||||
try {
|
||||
TVFS.umount();
|
||||
} catch (FsSyncException e) {
|
||||
logger.fatal("Couldn't unmount zip files", e);
|
||||
JOptionPane.showMessageDialog(null, "Couldn't unmount zip files", "Error", JOptionPane.ERROR_MESSAGE);
|
||||
} finally {
|
||||
//
|
||||
logger.fatal("Couldn't unmount zip files " + e, e);
|
||||
MageFrame.getInstance().showErrorDialog("Couldn't unmount zip files " + e, e);
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup resources
|
||||
Arrays.stream(DownloadSources.values()).forEach(resource -> {
|
||||
resource.source.onFinished();
|
||||
});
|
||||
|
||||
// stop
|
||||
reloadCardsToDownload(uiDialog.getSetsCombo().getSelectedItem().toString());
|
||||
|
|
@ -882,16 +888,7 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
// user cancelled
|
||||
if (DownloadPicturesService.getInstance().isNeedCancel()) {
|
||||
// stop download, save current state and exit
|
||||
TFile archive = destFile.getTopLevelArchive();
|
||||
///* not need to unmout/close - it's auto action
|
||||
if (archive != null && archive.exists()) {
|
||||
logger.info("User canceled download. Closing archive file: " + destFile.toString());
|
||||
try {
|
||||
TVFS.umount(archive);
|
||||
} catch (Exception e) {
|
||||
logger.error("Can't close archive file: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
// real archives save will be done on download service finish
|
||||
try {
|
||||
TFile.rm(fileTempImage);
|
||||
} catch (Exception e) {
|
||||
|
|
@ -946,11 +943,11 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements
|
|||
if (cardIndex < needDownloadCount) {
|
||||
// downloading
|
||||
float mb = ((needDownloadCount - lastCardIndex) * selectedSource.getAverageSizeKb()) / 1024;
|
||||
updateProgressMessage(String.format("%d of %d image downloading... Please wait [%.1f Mb left].",
|
||||
updateProgressMessage(String.format("%d of %d image downloading... Please wait [%.1f MB left].",
|
||||
lastCardIndex, needDownloadCount, mb), lastCardIndex, needDownloadCount);
|
||||
} else {
|
||||
// finished
|
||||
updateProgressMessage("Image download DONE, refreshing stats... Please wait.");
|
||||
updateProgressMessage("Image download DONE, saving last files and refreshing stats... Please wait.");
|
||||
List<CardDownloadData> downloadedCards = Collections.synchronizedList(new ArrayList<>());
|
||||
DownloadPicturesService.this.cardsMissing.parallelStream().forEach(cardDownloadData -> {
|
||||
TFile file = new TFile(CardImageUtils.buildImagePathToCardOrToken(cardDownloadData));
|
||||
|
|
|
|||
|
|
@ -1,28 +1,16 @@
|
|||
package org.mage.plugins.card.utils;
|
||||
|
||||
import mage.cards.repository.CardInfo;
|
||||
import mage.cards.repository.CardRepository;
|
||||
import mage.cards.repository.TokenRepository;
|
||||
import mage.client.MageFrame;
|
||||
import mage.client.constants.Constants;
|
||||
import mage.client.dialog.PreferencesDialog;
|
||||
import mage.client.remote.XmageURLConnection;
|
||||
import mage.remote.Connection;
|
||||
import mage.remote.Connection.ProxyType;
|
||||
import mage.view.CardView;
|
||||
import net.java.truevfs.access.TVFS;
|
||||
import net.java.truevfs.kernel.spec.FsSyncException;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.mage.plugins.card.images.CardDownloadData;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Proxy;
|
||||
import java.net.URL;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
|
@ -31,7 +19,6 @@ import java.nio.file.attribute.BasicFileAttributes;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Locale;
|
||||
import java.util.prefs.Preferences;
|
||||
|
||||
public final class CardImageUtils {
|
||||
|
||||
|
|
@ -191,19 +178,16 @@ public final class CardImageUtils {
|
|||
return getImagesDir() + File.separator + "FACE" + File.separator + fixSetNameForWindows(setCode) + File.separator + prepareCardNameForFile(cardName) + ".jpg";
|
||||
}
|
||||
|
||||
public static Proxy getProxyFromPreferences() {
|
||||
Preferences prefs = MageFrame.getPreferences();
|
||||
Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None"));
|
||||
if (proxyType != ProxyType.NONE) {
|
||||
String proxyServer = prefs.get("proxyAddress", "");
|
||||
int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0"));
|
||||
return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static void checkAndFixImageFiles() {
|
||||
// search broken, temp or outdated files and delete it
|
||||
|
||||
// make sure all archives was closed (e.g. on second call of download dialog)
|
||||
try {
|
||||
TVFS.umount();
|
||||
} catch (FsSyncException e) {
|
||||
LOGGER.fatal("Couldn't unmount zip files on searching broken images " + e, e);
|
||||
}
|
||||
|
||||
// real images check is slow, so it used on images download only (not here)
|
||||
Path rootPath = new File(CardImageUtils.getImagesDir()).toPath();
|
||||
if (!Files.exists(rootPath)) {
|
||||
|
|
|
|||
|
|
@ -9,13 +9,14 @@ import net.xeoh.plugins.base.annotations.meta.Author;
|
|||
import org.apache.log4j.Logger;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
|
||||
/**
|
||||
* Implementation of {@link CounterPlugin}.<br/>
|
||||
* Stores data in data folder.
|
||||
*
|
||||
* @version 0.1 14.11.2010 Initial Version
|
||||
* @author nantuko
|
||||
* @version 0.1 14.11.2010 Initial Version
|
||||
*/
|
||||
@PluginImplementation
|
||||
@Author(name = "nantuko")
|
||||
|
|
@ -69,11 +70,11 @@ public class CounterPluginImpl implements CounterPlugin {
|
|||
if (data.exists()) {
|
||||
int prev = 0;
|
||||
|
||||
try(ObjectInputStream ois = new ObjectInputStream(new FileInputStream(data))) {
|
||||
try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(data.toPath()))) {
|
||||
Object o = ois.readObject();
|
||||
CounterBean c;
|
||||
if (o instanceof CounterBean) {
|
||||
c = (CounterBean)o;
|
||||
c = (CounterBean) o;
|
||||
prev = c.getGamesPlayed();
|
||||
}
|
||||
} catch (EOFException e) {
|
||||
|
|
@ -84,10 +85,10 @@ public class CounterPluginImpl implements CounterPlugin {
|
|||
throw new PluginException(e);
|
||||
}
|
||||
|
||||
try(ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(data))) {
|
||||
try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(data))) {
|
||||
synchronized (this) {
|
||||
CounterBean c = new CounterBean();
|
||||
c.setGamesPlayed(prev+1);
|
||||
c.setGamesPlayed(prev + 1);
|
||||
oos.writeObject(c);
|
||||
oos.close();
|
||||
}
|
||||
|
|
@ -107,12 +108,12 @@ public class CounterPluginImpl implements CounterPlugin {
|
|||
return 0;
|
||||
}
|
||||
if (data.exists()) {
|
||||
try(ObjectInputStream ois = new ObjectInputStream(new FileInputStream(data))) {
|
||||
try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(data.toPath()))) {
|
||||
synchronized (this) {
|
||||
Object o = ois.readObject();
|
||||
CounterBean c = null;
|
||||
if (o instanceof CounterBean) {
|
||||
c = (CounterBean)o;
|
||||
c = (CounterBean) o;
|
||||
}
|
||||
ois.close();
|
||||
return c == null ? 0 : c.getGamesPlayed();
|
||||
|
|
|
|||
|
|
@ -1,25 +1,23 @@
|
|||
|
||||
|
||||
package mage.server.game;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.ObjectInput;
|
||||
import java.util.UUID;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import mage.game.Game;
|
||||
import mage.game.GameState;
|
||||
import mage.game.GameStates;
|
||||
import mage.server.Main;
|
||||
import mage.util.CopierObjectInputStream;
|
||||
import mage.utils.StreamUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.ObjectInput;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.UUID;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
/**
|
||||
* Replay system, outdated and not used. TODO: delete
|
||||
*
|
||||
* @author BetaSteward_at_googlemail.com
|
||||
*/
|
||||
|
|
@ -59,33 +57,19 @@ public class GameReplay {
|
|||
}
|
||||
|
||||
private Game loadGame(UUID gameId) {
|
||||
InputStream file = null;
|
||||
InputStream buffer = null;
|
||||
InputStream gzip = null;
|
||||
ObjectInput input = null;
|
||||
try{
|
||||
file = new FileInputStream("saved/" + gameId.toString() + ".game");
|
||||
buffer = new BufferedInputStream(file);
|
||||
gzip = new GZIPInputStream(buffer);
|
||||
input = new CopierObjectInputStream(Main.classLoader, gzip);
|
||||
try (InputStream file = Files.newInputStream(Paths.get("saved/" + gameId.toString() + ".game"));
|
||||
InputStream buffer = new BufferedInputStream(file);
|
||||
InputStream gzip = new GZIPInputStream(buffer);
|
||||
ObjectInput input = new CopierObjectInputStream(Main.classLoader, gzip)) {
|
||||
Game loadGame = (Game) input.readObject();
|
||||
GameStates states = (GameStates) input.readObject();
|
||||
loadGame.loadGameStates(states);
|
||||
return loadGame;
|
||||
|
||||
}
|
||||
catch(ClassNotFoundException ex) {
|
||||
logger.fatal("Cannot load game. Class not found.", ex);
|
||||
}
|
||||
catch(IOException ex) {
|
||||
logger.fatal("Cannot load game:" + gameId, ex);
|
||||
} finally {
|
||||
StreamUtils.closeQuietly(file);
|
||||
StreamUtils.closeQuietly(buffer);
|
||||
StreamUtils.closeQuietly(input);
|
||||
StreamUtils.closeQuietly(gzip);
|
||||
} catch (ClassNotFoundException e) {
|
||||
logger.fatal("Cannot load game. Class not found.", e);
|
||||
} catch (IOException e) {
|
||||
logger.fatal("Cannot load game:" + gameId, e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -756,7 +756,7 @@ public class SecretLairDrop extends ExpansionSet {
|
|||
cards.add(new SetCardInfo("Elvish Mystic", 805, Rarity.RARE, mage.cards.e.ElvishMystic.class, NON_FULL_USE_VARIOUS));
|
||||
cards.add(new SetCardInfo("Snapcaster Mage", 808, Rarity.MYTHIC, mage.cards.s.SnapcasterMage.class));
|
||||
cards.add(new SetCardInfo("Thrun, the Last Troll", 810, Rarity.MYTHIC, mage.cards.t.ThrunTheLastTroll.class));
|
||||
//cards.add(new SetCardInfo("Elesh Norn, Grand Cenobite", 811, Rarity.MYTHIC, mage.cards.e.EleshNornGrandCenobite.class, NON_FULL_USE_VARIOUS)); // BUG: breaks download of card 209
|
||||
cards.add(new SetCardInfo("Elesh Norn, Grand Cenobite", 811, Rarity.MYTHIC, mage.cards.e.EleshNornGrandCenobite.class, NON_FULL_USE_VARIOUS)); // BUG: breaks download of card 209
|
||||
cards.add(new SetCardInfo("Arcane Signet", 820, Rarity.RARE, mage.cards.a.ArcaneSignet.class, NON_FULL_USE_VARIOUS));
|
||||
cards.add(new SetCardInfo("Norin the Wary", 827, Rarity.RARE, mage.cards.n.NorinTheWary.class));
|
||||
cards.add(new SetCardInfo("The Scarab God", 900, Rarity.MYTHIC, mage.cards.t.TheScarabGod.class));
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ public class UserData implements Serializable {
|
|||
protected int autoTargetLevel;
|
||||
protected boolean useSameSettingsForReplacementEffects;
|
||||
protected boolean useFirstManaAbility = false;
|
||||
private String userIdStr;
|
||||
private String userIdStr; // TODO: delete as un-used or use for hardware id instead?
|
||||
protected Map<UUID, Set<UUID>> requestedHandPlayersList; // game -> players list
|
||||
|
||||
protected String matchHistory;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue