diff --git a/Mage.Client/src/main/java/mage/client/MageFrame.java b/Mage.Client/src/main/java/mage/client/MageFrame.java index f6be85deefd..a92f5f70253 100644 --- a/Mage.Client/src/main/java/mage/client/MageFrame.java +++ b/Mage.Client/src/main/java/mage/client/MageFrame.java @@ -46,7 +46,7 @@ import mage.remote.Connection; import mage.remote.Connection.ProxyType; import mage.util.DebugUtil; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.utils.MageVersion; import mage.view.GameEndView; import mage.view.UserRequestMessage; @@ -134,7 +134,7 @@ public class MageFrame extends javax.swing.JFrame implements MageClient { private static final MageUI UI = new MageUI(); private static final ScheduledExecutorService PING_SENDER_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_PING_SENDER) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_PING_SENDER) ); private static UpdateMemUsageTask updateMemUsageTask; diff --git a/Mage.Client/src/main/java/mage/client/deckeditor/DeckEditorPanel.java b/Mage.Client/src/main/java/mage/client/deckeditor/DeckEditorPanel.java index ed27da771e3..90b74859bf3 100644 --- a/Mage.Client/src/main/java/mage/client/deckeditor/DeckEditorPanel.java +++ b/Mage.Client/src/main/java/mage/client/deckeditor/DeckEditorPanel.java @@ -26,7 +26,7 @@ import mage.game.GameException; import mage.remote.Session; import mage.util.DeckUtil; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.view.CardView; import mage.view.SimpleCardView; import org.apache.log4j.Logger; @@ -1496,7 +1496,7 @@ public class DeckEditorPanel extends javax.swing.JPanel { private void btnSubmitTimerActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_btnSubmitTimerActionPerformed ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_SUBMIT_TIMER) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_SUBMIT_TIMER) ); timeToSubmit = 60; this.btnSubmitTimer.setEnabled(false); diff --git a/Mage.Client/src/main/java/mage/client/game/FeedbackPanel.java b/Mage.Client/src/main/java/mage/client/game/FeedbackPanel.java index a562a947a0b..82a2f87bf33 100644 --- a/Mage.Client/src/main/java/mage/client/game/FeedbackPanel.java +++ b/Mage.Client/src/main/java/mage/client/game/FeedbackPanel.java @@ -9,7 +9,7 @@ import mage.client.util.gui.ArrowBuilder; import mage.constants.PlayerAction; import mage.constants.TurnPhase; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import org.apache.log4j.Logger; import javax.swing.*; @@ -45,7 +45,7 @@ public class FeedbackPanel extends javax.swing.JPanel { private static final int AUTO_CLOSE_END_DIALOG_TIMEOUT_SECS = 8; private static final ScheduledExecutorService AUTO_CLOSE_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_AUTO_CLOSE_TIMER) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_AUTO_CLOSE_TIMER) ); /** diff --git a/Mage.Client/src/main/java/mage/client/remote/XmageURLConnection.java b/Mage.Client/src/main/java/mage/client/remote/XmageURLConnection.java new file mode 100644 index 00000000000..37e115d1f60 --- /dev/null +++ b/Mage.Client/src/main/java/mage/client/remote/XmageURLConnection.java @@ -0,0 +1,294 @@ +package mage.client.remote; + +import mage.client.dialog.PreferencesDialog; +import mage.remote.Connection; +import mage.utils.MageVersion; +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.URL; +import java.util.Map; + +/** + * Network: proxy class to set up and use network connections like URLConnection + *

+ * It used stream logic for data access + *

+ * For text: + * - download text data by XmageURLConnection.downloadText + *

+ * For binary data (e.g. file download): + * - get stream by XmageURLConnection.downloadBinary + * - save stream to file or process + *

+ * TODO: no needs in POST requests (support only GET), but can be added later for another third party APIs + * + * @author JayDi85 + */ +public class XmageURLConnection { + + private static final MageVersion version = new MageVersion(XmageURLConnection.class); + private static final Logger logger = Logger.getLogger(XmageURLConnection.class); + + private static final int CONNECTION_STARTING_TIMEOUT_MS = 10000; + private static final int CONNECTION_READING_TIMEOUT_MS = 60000; + + final String url; + Proxy proxy = null; + HttpURLConnection connection = null; + HttpLoggingType loggingType = HttpLoggingType.ERRORS; + + public XmageURLConnection(String url) { + this.url = url; + } + + // example: 404 Not Found xxx + enum HttpLoggingType { + NONE, + ERRORS, + ALL + } + + /** + * Add additional headers like non standard user agent, etc + */ + public void setRequestHeaders(Map additionalHeaders) { + makeSureConnectionStarted(); + + for (String key : additionalHeaders.keySet()) { + this.connection.setRequestProperty(key, additionalHeaders.get(key)); + } + } + + /** + * Connect to server + */ + public void startConnection() { + initDefaultProxy(); + + try { + URL url = new URL(this.url); + + // proxy settings + if (this.proxy != null) { + this.connection = (HttpURLConnection) url.openConnection(this.proxy); + } else { + this.connection = (HttpURLConnection) url.openConnection(); + } + + // additional settings + this.connection.setConnectTimeout(CONNECTION_STARTING_TIMEOUT_MS); + this.connection.setReadTimeout(CONNECTION_READING_TIMEOUT_MS); + + initDefaultHeaders(); + } catch (IOException e) { + this.connection = null; + } + } + + public void initDefaultProxy() { + Connection.ProxyType configProxyType = Connection.ProxyType.valueByText(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_TYPE, "None")); + Proxy.Type type; + switch (configProxyType) { + case HTTP: + type = Proxy.Type.HTTP; + break; + case SOCKS: + type = Proxy.Type.SOCKS; + break; + case NONE: + default: + type = Proxy.Type.DIRECT; + break; + } + + this.proxy = Proxy.NO_PROXY; + if (type != Proxy.Type.DIRECT) { + try { + String address = PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_ADDRESS, ""); + int port = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_PORT, "80")); + this.proxy = new Proxy(type, new InetSocketAddress(address, port)); + } catch (Exception e) { + throw new RuntimeException("Network: can't create proxy, check your settings or reset it - " + e, e); + } + } + } + + private void initDefaultHeaders() { + // warning, do not add Accept-Encoding - it processing inside URLConnection for http/https links (trying to use gzip by default) + + // user agent due standard notation User-Agent: / + // warning, dot not add os, language and other details + this.connection.setRequestProperty("User-Agent", String.format("XMage/%s build: %s", + version.toString(false), version.getBuildTime())); + } + + /** + * Connect to server's resource + */ + public void connect() throws IOException { + makeSureConnectionStarted(); + + this.connection.connect(); + printHttpResult(); + } + + public boolean isConnected() { + return this.connection != null; + } + + /** + * Get http status code from a web server (200 for ok, -1 for not connect, 400 and other for errors) + */ + public int getResponseCode() { + makeSureConnectionStarted(); + + try { + return this.connection.getResponseCode(); + } catch (IOException ignore) { + return -1; + } + } + + /** + * Get total file size to download (call it after start connection) + * -1 for unknown size + * 0 for text or small files + *

+ * Warning, result depends on Accept-Encoding, so use it for information only + */ + public int getContentLength() { + makeSureConnectionStarted(); + + return this.connection.getContentLength(); + } + + /** + * Get http status message from a web server like Not Found + */ + public String getResponseMessage() { + makeSureConnectionStarted(); + + try { + return this.connection.getResponseMessage(); + } catch (IOException ignore) { + return ""; + } + } + + /** + * Get returned html from a web server + */ + public String getErrorResponseAsString() { + makeSureConnectionStarted(); + + java.util.Scanner s = new java.util.Scanner(this.connection.getErrorStream()).useDelimiter("\\A"); + return s.hasNext() ? s.next() : ""; + } + + /** + * Get non-text data as stream, e.g. binary file content + */ + public InputStream getGoodResponseAsStream() throws IOException { + makeSureConnectionStarted(); + + return this.connection.getInputStream(); + } + + /** + * Get text data as string, e.g. html document + */ + public String getGoodResponseAsString() { + makeSureConnectionStarted(); + StringBuffer tmp = new StringBuffer(); + BufferedReader in = null; + try { + in = new BufferedReader(new InputStreamReader(this.getGoodResponseAsStream())); + String line; + while ((line = in.readLine()) != null) { + tmp.append(line); + } + } catch (IOException e) { + throw new RuntimeException("Network: can't get text data from " + this.url + " - " + e, e); + } + + return String.valueOf(tmp); + } + + private void makeSureConnectionStarted() { + if (!isConnected()) { + throw new IllegalArgumentException("Wrong code usage: must call startConnection first", new Throwable()); + } + } + + /** + * Fast download of text data + * + * @return downloaded text on OK 200 response or empty on any other errors + */ + public static String downloadText(String resourceUrl) { + XmageURLConnection con = new XmageURLConnection(resourceUrl); + con.startConnection(); + if (con.isConnected()) { + try { + con.connect(); + if (con.getResponseCode() == 200) { + return con.getGoodResponseAsString(); + } + } catch (IOException e) { + logger.error(e, e); + } + } + return ""; + } + + /** + * Fast download of binary data + * + * @return stream on OK 200 response or null on any other errors + */ + public static InputStream downloadBinary(String resourceUrl) { + XmageURLConnection con = new XmageURLConnection(resourceUrl); + con.startConnection(); + if (con.isConnected()) { + try { + con.connect(); + if (con.getResponseCode() == 200) { + return con.getGoodResponseAsStream(); + } + } catch (IOException e) { + logger.error(e, e); + } + } + return null; + } + + private void printHttpResult() { + if (this.connection == null) { + return; + } + + boolean needPrint; + switch (this.loggingType) { + case NONE: + needPrint = false; + break; + case ERRORS: + needPrint = getResponseCode() != HttpURLConnection.HTTP_OK; + break; + case ALL: + default: + needPrint = true; + } + + if (needPrint) { + logger.info(String.format("http request %d %s %s", this.getResponseCode(), this.getResponseMessage(), this.url)); + } + } +} \ No newline at end of file diff --git a/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java b/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java index d23a5e3d7d4..6f0fae9c542 100644 --- a/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java +++ b/Mage.Client/src/main/java/org/mage/card/arcane/ManaSymbols.java @@ -22,13 +22,15 @@ import java.awt.image.BufferedImage; import java.io.*; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.regex.Pattern; import java.util.stream.IntStream; +/** + * Mana symbol resources + */ public final class ManaSymbols { private static final Logger logger = Logger.getLogger(ManaSymbols.class); @@ -120,18 +122,13 @@ public final class ManaSymbols { ImageIO.write(image, "png", newFile); } } catch (Exception e) { - logger.warn("Can't generate png image for symbol:" + symbol); + logger.warn("Symbols: can't generate png image for symbol:" + symbol); } } } // preload set images java.util.List setCodes = ExpansionRepository.instance.getSetCodes(); - if (setCodes == null) { - // the cards db file is probaly not included in the client. It will be created after the first connect to a server. - logger.warn("No db information for sets found. Connect to a server to create database file on client side. Then try to restart the client."); - return; - } for (String set : setCodes) { if (withoutSymbols.contains(set)) { @@ -289,9 +286,9 @@ public final class ManaSymbols { // priority: SVG -> GIF // gif remain for backward compatibility - AtomicIntegerArray iconErrors = new AtomicIntegerArray(2); // 0 - svg, 1 - gif + final List svgFails = new ArrayList<>(); // scryfall + final List otherFails = new ArrayList<>(); // gatherer - AtomicBoolean fileErrors = new AtomicBoolean(false); Map sizedSymbols = new ConcurrentHashMap<>(); IntStream.range(0, symbols.length).parallel().forEach(i -> { String symbol = symbols[i]; @@ -305,50 +302,53 @@ public final class ManaSymbols { try { InputStream fileStream = new FileInputStream(file); image = loadSymbolAsSVG(fileStream, file.getPath(), size, size); - } catch (FileNotFoundException e) { - // it's ok to hide error + } catch (FileNotFoundException ignore) { } } } - - // gif if (image == null) { + synchronized (svgFails) { + svgFails.add(symbol); + } + } - iconErrors.incrementAndGet(0); // svg fail - + // gif (if svg fails) + if (image == null) { file = getSymbolFileNameAsGIF(symbol, size); if (file.exists()) { image = loadSymbolAsGIF(file, size, size); } } + if (image == null) { + synchronized (otherFails) { + otherFails.add(symbol); + } + } // save if (image != null) { sizedSymbols.put(symbol, image); - } else { - iconErrors.incrementAndGet(1); // gif fail - fileErrors.set(true); } }); // total errors String errorInfo = ""; - if (iconErrors.get(0) > 0) { - errorInfo += "SVG miss - " + iconErrors.get(0); + if (!svgFails.isEmpty()) { + errorInfo += String.format("SVG miss - %s and %d others", svgFails.get(0), svgFails.size() - 1); } - if (iconErrors.get(1) > 0) { + if (!otherFails.isEmpty()) { if (!errorInfo.isEmpty()) { errorInfo += ", "; } - errorInfo += "GIF miss - " + iconErrors.get(1); + errorInfo += String.format("GIF miss - %s and %d others", otherFails.get(0), otherFails.size() - 1); } if (!errorInfo.isEmpty()) { - logger.warn("Symbols can't be loaded, make sure you download it by main menu - size " + size + ", " + errorInfo); + logger.warn("Symbols: can't load, make sure you download it by main menu - size " + size + ", " + errorInfo); } manaImages.put(size, sizedSymbols); - return !fileErrors.get(); + return errorInfo.isEmpty(); } private static void renameSymbols(String path) { diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/CardPluginImpl.java b/Mage.Client/src/main/java/org/mage/plugins/card/CardPluginImpl.java index 58f5e26e941..c5fc47f5b51 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/CardPluginImpl.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/CardPluginImpl.java @@ -4,7 +4,6 @@ import mage.cards.MageCard; import mage.cards.MagePermanent; import mage.cards.action.ActionCallback; import mage.client.util.GUISizeHelper; -import mage.client.util.ImageCaches; import mage.interfaces.plugin.CardPlugin; import mage.view.CardView; import mage.view.CounterView; @@ -103,7 +102,7 @@ public class CardPluginImpl implements CardPlugin { * yet, so use old component based rendering for the split cards. */ private CardPanel makeCardPanel(CardView view, UUID gameId, boolean loadImage, ActionCallback callback, - boolean isFoil, Dimension dimension, int renderMode, boolean needFullPermanentRender) { + boolean isFoil, Dimension dimension, int renderMode, boolean needFullPermanentRender) { switch (renderMode) { case 0: return new CardPanelRenderModeMTGO(view, gameId, loadImage, callback, isFoil, dimension, @@ -118,7 +117,7 @@ public class CardPluginImpl implements CardPlugin { @Override public MageCard getMagePermanent(PermanentView permanent, Dimension dimension, UUID gameId, ActionCallback callback, - boolean canBeFoil, boolean loadImage, int renderMode, boolean needFullPermanentRender) { + boolean canBeFoil, boolean loadImage, int renderMode, boolean needFullPermanentRender) { CardPanel cardPanel = makeCardPanel(permanent, gameId, loadImage, callback, false, dimension, renderMode, needFullPermanentRender); cardPanel.setShowCastingCost(true); @@ -127,7 +126,7 @@ public class CardPluginImpl implements CardPlugin { @Override public MageCard getMageCard(CardView cardView, Dimension dimension, UUID gameId, ActionCallback callback, - boolean canBeFoil, boolean loadImage, int renderMode, boolean needFullPermanentRender) { + boolean canBeFoil, boolean loadImage, int renderMode, boolean needFullPermanentRender) { CardPanel cardPanel = makeCardPanel(cardView, gameId, loadImage, callback, false, dimension, renderMode, needFullPermanentRender); cardPanel.setShowCastingCost(true); @@ -191,7 +190,7 @@ public class CardPluginImpl implements CardPlugin { && stackPower == cardPower && stackToughness == cardToughness && stackAbilities.equals(cardAbilities) && stackCounters.equals(cardCounters) && (!perm.isCreature() || firstPanelPerm.getOriginalPermanent().hasSummoningSickness() == perm - .getOriginalPermanent().hasSummoningSickness())) { + .getOriginalPermanent().hasSummoningSickness())) { if (!empty(firstPanelPerm.getOriginalPermanent().getAttachments())) { // Put this land to the left of lands with the same name and attachments. @@ -235,7 +234,7 @@ public class CardPluginImpl implements CardPlugin { @Override public int sortPermanents(Map ui, Map cards, boolean nonPermanentsOwnRow, - boolean topPanel) { + boolean topPanel) { // requires to find out is position have been changed that includes: // adding/removing permanents, type change @@ -279,7 +278,7 @@ public class CardPluginImpl implements CardPlugin { cardHeight = Math.round(cardWidth * CardPanel.ASPECT_RATIO); extraCardSpacingX = Math.round(cardWidth * EXTRA_CARD_SPACING_X); cardSpacingX = cardHeight - cardWidth + extraCardSpacingX; // need space for tap animation (horizontal - // position) + // position) cardSpacingY = Math.round(cardHeight * CARD_SPACING_Y); stackSpacingX = stackVertical ? 0 : Math.round(cardWidth * STACK_SPACING_X); stackSpacingY = Math.round(cardHeight * STACK_SPACING_Y); @@ -363,7 +362,7 @@ public class CardPluginImpl implements CardPlugin { } for (int panelIndex = 0, panelCount = stack.size(); panelIndex < panelCount; panelIndex++) { MagePermanent panelPerm = stack.get(panelIndex); // it's original card panel, but you must change - // top layer + // top layer int stackPosition = panelCount - panelIndex - 1; if (cardsPanel != null) { cardsPanel.setComponentZOrder(panelPerm.getTopPanelRef(), panelIndex); @@ -470,7 +469,7 @@ public class CardPluginImpl implements CardPlugin { } private AttachmentLayoutInfos calculateNeededNumberOfVerticalColumns(int currentCol, Map cards, - MageCard cardWithAttachments) { + MageCard cardWithAttachments) { int maxCol = ++currentCol; int attachments = 0; MagePermanent permWithAttachments = (MagePermanent) cardWithAttachments.getMainPanel(); @@ -653,38 +652,34 @@ public class CardPluginImpl implements CardPlugin { final Downloader downloader = new Downloader(); final DownloadGui downloadGui = new DownloadGui(downloader); - LOGGER.info("Symbols download prepare..."); + LOGGER.info("Download: prepare symbols to download..."); Iterable jobs; + // mana symbols (low quality) jobs = new GathererSymbols(); for (DownloadJob job : jobs) { downloader.add(job); } + // set code symbols (low quality) jobs = new GathererSets(); for (DownloadJob job : jobs) { downloader.add(job); } + // mana symbols (high quality) jobs = new ScryfallSymbolsSource(); for (DownloadJob job : jobs) { downloader.add(job); } - /* - * it = new CardFrames(imagesDir); // TODO: delete frames download (not need - * now) - * for (DownloadJob job : it) { - * g.getDownloader().add(job); - * } - */ - + // additional resources jobs = new DirectLinksForDownload(); for (DownloadJob job : jobs) { downloader.add(job); } - LOGGER.info("Symbols download needs " + downloader.getJobs().size() + " files"); + LOGGER.info("Download: app used " + downloader.getJobs().size() + " symbol files"); // download GUI dialog JDialog dialog = new JDialog((Frame) null, "Download symbols", false); @@ -711,6 +706,7 @@ public class CardPluginImpl implements CardPlugin { return null; } }; + // downloader finisher worker.addPropertyChangeListener(new PropertyChangeListener() { @Override @@ -718,7 +714,7 @@ public class CardPluginImpl implements CardPlugin { if (evt.getPropertyName().equals("state")) { if (evt.getNewValue() == SwingWorker.StateValue.DONE) { // all done, can close dialog and refresh symbols for UI - LOGGER.info("Symbols download finished"); + LOGGER.info("Download: symbols download finished"); dialog.dispose(); ManaSymbols.loadImages(); GUISizeHelper.refreshGUIAndCards(false); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java index a0221c4e100..cbbbdb52680 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java @@ -2,16 +2,15 @@ package org.mage.plugins.card.dl; import org.mage.plugins.card.dl.beans.properties.Property; import org.mage.plugins.card.dl.lm.AbstractLaternaBean; -import org.mage.plugins.card.utils.CardImageUtils; import javax.swing.*; -import java.io.*; -import java.net.Proxy; -import java.net.URL; -import java.net.URLConnection; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; /** - * Downloader job to download one resource + * Download: download job to load one resource, used for symbols * * @author Clemens Koza, JayDi85 */ @@ -22,7 +21,7 @@ public class DownloadJob extends AbstractLaternaBean { } private final String name; - private Source source; + private String url; private final Destination destination; private final boolean forceToDownload; // download image everytime, do not keep old image private final Property state = properties.property("state", State.NEW); @@ -30,13 +29,21 @@ public class DownloadJob extends AbstractLaternaBean { private final Property error = properties.property("error"); private final BoundedRangeModel progress = new DefaultBoundedRangeModel(); - public DownloadJob(String name, Source source, Destination destination, boolean forceToDownload) { + public DownloadJob(String name, String url, Destination destination, boolean forceToDownload) { this.name = name; - this.source = source; + this.url = url; this.destination = destination; this.forceToDownload = forceToDownload; } + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + /** * Sets the job's state. If the state is {@link State#ABORTED}, it instead * sets the error to "ABORTED" @@ -80,7 +87,7 @@ public class DownloadJob extends AbstractLaternaBean { */ public void setError(String message, Exception error) { if (message == null) { - message = "Download of " + name + " from " + source.toString() + " caused error: " + error.toString(); + message = "Download: " + name + " from " + url + " caused error - " + error; } this.state.setValue(State.ABORTED); this.error.setValue(error); @@ -145,14 +152,6 @@ public class DownloadJob extends AbstractLaternaBean { return name; } - public Source getSource() { - return source; - } - - public void setSource(Source source) { - this.source = source; - } - public Destination getDestination() { return destination; } @@ -161,71 +160,6 @@ public class DownloadJob extends AbstractLaternaBean { return forceToDownload; } - public static Source fromURL(final String url) { - return fromURL(CardImageUtils.getProxyFromPreferences(), url); - } - - public static Source fromURL(final URL url) { - return fromURL(CardImageUtils.getProxyFromPreferences(), url); - } - - public static Source fromURL(final Proxy proxy, final String url) { - return new Source() { - private URLConnection c; - - public URLConnection getConnection() throws IOException { - if (c == null) { - c = proxy == null ? new URL(url).openConnection() : new URL(url).openConnection(proxy); - } - return c; - } - - @Override - public InputStream open() throws IOException { - return getConnection().getInputStream(); - } - - @Override - public int length() throws IOException { - return getConnection().getContentLength(); - } - - @Override - public String toString() { - return proxy != null ? proxy.type().toString() + ' ' : url; - } - - }; - } - - public static Source fromURL(final Proxy proxy, final URL url) { - return new Source() { - private URLConnection c; - - public URLConnection getConnection() throws IOException { - if (c == null) { - c = proxy == null ? url.openConnection() : url.openConnection(proxy); - } - return c; - } - - @Override - public InputStream open() throws IOException { - return getConnection().getInputStream(); - } - - @Override - public int length() throws IOException { - return getConnection().getContentLength(); - } - - @Override - public String toString() { - return proxy != null ? proxy.type().toString() + ' ' : String.valueOf(url); - } - }; - } - public static Destination toFile(final String file) { return toFile(new File(file)); } @@ -264,13 +198,6 @@ public class DownloadJob extends AbstractLaternaBean { }; } - public interface Source { - - InputStream open() throws IOException; - - int length() throws IOException; - } - public interface Destination { OutputStream open() throws IOException; diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java index 0e5152b3157..72c816d3908 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadServiceInfo.java @@ -7,8 +7,6 @@ import java.net.Proxy; */ public interface DownloadServiceInfo { - Proxy getProxy(); - boolean isNeedCancel(); void incErrorCount(); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java index 6281a8dc69a..ad44256a076 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java @@ -1,7 +1,8 @@ package org.mage.plugins.card.dl; +import mage.client.remote.XmageURLConnection; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import org.apache.log4j.Logger; import org.jetlang.channels.Channel; import org.jetlang.channels.MemoryChannel; @@ -9,13 +10,15 @@ import org.jetlang.core.Callback; import org.jetlang.fibers.Fiber; import org.jetlang.fibers.PoolFiberFactory; import org.mage.plugins.card.dl.DownloadJob.Destination; -import org.mage.plugins.card.dl.DownloadJob.Source; import org.mage.plugins.card.dl.DownloadJob.State; import org.mage.plugins.card.dl.lm.AbstractLaternaBean; import javax.swing.*; -import java.io.*; -import java.net.ConnectException; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -24,7 +27,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; /** - * Symbols downloader + * Download: symbols download service * * @author Clemens Koza, JayDi85 */ @@ -37,7 +40,7 @@ public class Downloader extends AbstractLaternaBean { private CountDownLatch worksCount = null; private final ExecutorService pool = Executors.newCachedThreadPool( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_SYMBOLS_DOWNLOADER, false) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_SYMBOLS_DOWNLOADER, false) ); private final List fibers = new ArrayList<>(); @@ -105,11 +108,11 @@ public class Downloader extends AbstractLaternaBean { worksCount.await(60, TimeUnit.SECONDS); if (worksCount.getCount() != 0) { - logger.warn("Symbols download too long..."); + logger.warn("Download: symbols downloading too long"); } } } catch (InterruptedException e) { - logger.error("Need to stop symbols download..."); + logger.error("Download: symbols downloading must be stopped"); } } @@ -118,8 +121,8 @@ public class Downloader extends AbstractLaternaBean { } /** - * Performs the download job: Transfers data from {@link Source} to - * {@link Destination} and updates the download job's state to reflect the + * Performs the download job: Transfers data from source to + * destination and updates the download job's state to reflect the * progress. */ private class DownloadCallback implements Callback { @@ -133,7 +136,7 @@ public class Downloader extends AbstractLaternaBean { // take new job job.doPrepareAndStartWork(); if (job.getState() != State.WORKING) { - logger.warn("Can't prepare symbols download job: " + job.getName()); + logger.warn("Download: can't prepare symbols download job: " + job.getName() + ", reason: " + job.getError()); worksCount.countDown(); return; } @@ -146,7 +149,6 @@ public class Downloader extends AbstractLaternaBean { // real work for new job // download and save data try { - Source src = job.getSource(); Destination dst = job.getDestination(); BoundedRangeModel progress = job.getProgress(); @@ -155,65 +157,66 @@ public class Downloader extends AbstractLaternaBean { progress.setMaximum(1); progress.setValue(1); } else { - // downloading + // need to download + + // clean local file if (dst.exists()) { try { dst.delete(); - } catch (IOException ex1) { - logger.warn("While deleting not valid file", ex1); + } catch (IOException e) { + logger.warn("Download: can't delete old file " + e, e); } } - progress.setMaximum(src.length()); - InputStream is = new BufferedInputStream(src.open()); - try { - OutputStream os = new BufferedOutputStream(dst.open()); - try { - byte[] buf = new byte[8 * 1024]; - int total = 0; - for (int len; (len = is.read(buf)) != -1; ) { - if (job.getState() == State.ABORTED) { - throw new IOException("Job was aborted"); + + // download + // start debug here with breakpoint like job.getName().contains("C/B") + XmageURLConnection connection = new XmageURLConnection(job.getUrl()); + connection.startConnection(); + if (connection.isConnected()) { + // start downloading + connection.connect(); + if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) { + // all fine, can continue and save + progress.setMaximum(connection.getContentLength()); + try (InputStream inputStream = connection.getGoodResponseAsStream(); + OutputStream outputStream = new BufferedOutputStream(dst.open())) { + byte[] buf = new byte[8 * 1024]; + int total = 0; + for (int len; (len = inputStream.read(buf)) != -1; ) { + // fast cancel + if (job.getState() == State.ABORTED) { + throw new IOException("job was aborted"); + } + progress.setValue(total += len); + outputStream.write(buf, 0, len); } - progress.setValue(total += len); - os.write(buf, 0, len); - } - } catch (IOException ex) { - try { - dst.delete(); - } catch (IOException ex1) { - logger.warn("While deleting", ex1); - } - throw ex; - } finally { - try { - os.close(); + } catch (IOException e) { + // something bad on downloading + logger.warn("Download: " + job.getName() + " - catch error on downloading network resource " + + job.getUrl() + " - " + e, e); + } finally { + // clean up if (!dst.isValid()) { + logger.warn("Download: " + job.getName() + " - downloaded invalid network resource (not exists?) " + job.getUrl()); dst.delete(); - logger.warn("Resource not found " + job.getName() + " from " + job.getSource().toString()); } - } catch (IOException ex) { - logger.warn("While closing", ex); } + } else { + // something bad with resource on server (example: wrong url) + logger.warn("Download: " + job.getName() + " - can't find network resource " + job.getUrl()); } - } finally { - try { - is.close(); - } catch (IOException ex) { - logger.warn("While closing", ex); - } + } else { + // something bad with network (example: can't connect due bad network) + logger.warn("Download: " + job.getName() + " - can't connect to network resource " + job.getUrl()); } } + + // all done job.setState(State.FINISHED); - } catch (ConnectException ex) { - String message; - if (ex.getMessage() != null) { - message = ex.getMessage(); - } else { - message = "Unknown error"; - } - logger.warn("Error resource download " + job.getName() + " from " + job.getSource().toString() + ": " + message); - } catch (IOException ex) { - job.setError(ex); + } catch (Exception e) { + // TODO: save error in other logger.warn? + job.setError(e); + logger.warn("Download: " + job.getName() + " - unknown error for network resource " + job.getUrl() + " - " + e, e); } finally { worksCount.countDown(); } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java index c353095b632..7de570322db 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/CardImageSource.java @@ -54,13 +54,10 @@ public interface CardImageSource { } /** - * Set additional http headers like user agent, referer, cookies, etc + * Set additional headers like user agent, referer, cookies, etc */ default Map getHttpRequestHeaders(String fullUrl) { - Map headers = new LinkedHashMap<>(); - // TODO: add xmage name and client version here - headers.put("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2"); - return headers; + return new LinkedHashMap<>(); } default List getSupportedSets() { diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/DirectLinksForDownload.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/DirectLinksForDownload.java index 0c3acc8068c..1f6baf0d6b3 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/DirectLinksForDownload.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/DirectLinksForDownload.java @@ -6,12 +6,11 @@ import org.mage.plugins.card.dl.DownloadJob; import java.io.File; import java.util.*; -import static org.mage.plugins.card.dl.DownloadJob.fromURL; import static org.mage.plugins.card.dl.DownloadJob.toFile; import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; /** - * Additional images from a third party sources + * Download: additional images from a third party sources, used for symbols * * @author noxx */ @@ -44,7 +43,7 @@ public class DirectLinksForDownload implements Iterable { for (Map.Entry url : directLinks.entrySet()) { File dst = new File(outDir, url.getKey()); // download images every time (need to update low quality image) - jobs.add(new DownloadJob(url.getKey(), fromURL(url.getValue()), toFile(dst), true)); + jobs.add(new DownloadJob(url.getKey(), url.getValue(), toFile(dst), true)); } return jobs.iterator(); } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSets.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSets.java index cde061108b7..aa9dbf2254d 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSets.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSets.java @@ -6,17 +6,18 @@ import mage.client.constants.Constants; import mage.constants.Rarity; import org.apache.log4j.Logger; import org.mage.plugins.card.dl.DownloadJob; -import static org.mage.plugins.card.dl.DownloadJob.fromURL; -import static org.mage.plugins.card.dl.DownloadJob.toFile; -import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; import java.io.File; import java.util.*; -/** - * WARNING, unsupported images plugin, last updates from 2018 - */ +import static org.mage.plugins.card.dl.DownloadJob.toFile; +import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; +/** + * Download: set code symbols download from wizards web size + *

+ * Warning, it's outdated source with low quality images. TODO: must migrate to scryfall like mana icons + */ public class GathererSets implements Iterable { private class CheckResult { @@ -338,6 +339,6 @@ public class GathererSets implements Iterable { set = codeReplacements.get(set); } String url = "https://gatherer.wizards.com/Handlers/Image.ashx?type=symbol&set=" + set + "&size=small&rarity=" + urlRarity; - return new DownloadJob(set + '-' + rarity, fromURL(url), toFile(dst), false); + return new DownloadJob(set + '-' + rarity, url, toFile(dst), false); } } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSymbols.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSymbols.java index f7c51da91b7..e629b612fd6 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSymbols.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/GathererSymbols.java @@ -8,15 +8,15 @@ import java.io.File; import java.util.Iterator; import static java.lang.String.format; -import static org.mage.plugins.card.dl.DownloadJob.fromURL; import static org.mage.plugins.card.dl.DownloadJob.toFile; import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; /** - * The class GathererSymbols. + * Download: mana symbols download from wizards web size + *

+ * Warning, it's outdated source with low quality images. Use scryfall source as primary. * - * @author Clemens Koza - * @version V0.0 25.08.2010 + * @author Clemens Koza, JayDi85 */ public class GathererSymbols implements Iterable { @@ -125,8 +125,7 @@ public class GathererSymbols implements Iterable { } String url = format(urlFmt, sizes[modSizeIndex], symbol); - - return new DownloadJob(sym, fromURL(url), toFile(dst), false); + return new DownloadJob(sym, url, toFile(dst), false); } } }; diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java index ed63f9b99c0..abcc7f9001f 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java @@ -4,18 +4,16 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import mage.MageException; +import mage.client.remote.XmageURLConnection; import mage.client.util.CardLanguage; import mage.util.JsonUtil; import org.apache.log4j.Logger; import org.mage.plugins.card.dl.DownloadServiceInfo; import org.mage.plugins.card.images.CardDownloadData; -import java.io.FileNotFoundException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.Proxy; -import java.net.URL; -import java.net.URLConnection; import java.util.*; /** @@ -31,7 +29,7 @@ public class ScryfallImageSource implements CardImageSource { private CardLanguage currentLanguage = CardLanguage.ENGLISH; // working language private final Map preparedUrls = new HashMap<>(); private static final int DOWNLOAD_TIMEOUT_MS = 100; - + public static ScryfallImageSource getInstance() { return instance; } @@ -89,15 +87,15 @@ public class ScryfallImageSource implements CardImageSource { baseUrl = link + localizedCode + "?format=image"; // workaround to use cards without english images (some promos or special cards) if (link.endsWith("/")) { - alternativeUrl = link.substring(0,link.length()-1) + "?format=image"; + alternativeUrl = link.substring(0, link.length() - 1) + "?format=image"; } } else { // direct link to image baseUrl = link; // workaround to use localization in direct links - if (link.contains("/?format=image")){ - baseUrl = link.replaceFirst("\\?format=image" , localizedCode + "?format=image"); - alternativeUrl = link.replaceFirst("/\\?format=image" , "?format=image"); + if (link.contains("/?format=image")) { + baseUrl = link.replaceFirst("\\?format=image", localizedCode + "?format=image"); + alternativeUrl = link.replaceFirst("/\\?format=image", "?format=image"); } } } @@ -117,7 +115,7 @@ public class ScryfallImageSource implements CardImageSource { // basic cards by api call (redirect to img link) // example: https://api.scryfall.com/cards/xln/121/en?format=image if (baseUrl == null) { - String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId()) ; + String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId()); baseUrl = String.format("https://api.scryfall.com/cards/%s/%s/%s?format=image", formatSetName(card.getSet(), isToken), cn, @@ -135,10 +133,10 @@ public class ScryfallImageSource implements CardImageSource { } - return new CardImageUrls(baseUrl, alternativeUrl ); + return new CardImageUrls(baseUrl, alternativeUrl); } - private String getFaceImageUrl(Proxy proxy, CardDownloadData card, boolean isToken) throws Exception { + private String getFaceImageUrl(CardDownloadData card, boolean isToken) throws Exception { final String defaultCode = CardLanguage.ENGLISH.getCode(); final String localizedCode = languageAliases.getOrDefault(this.getCurrentLanguage(), defaultCode); @@ -148,11 +146,11 @@ public class ScryfallImageSource implements CardImageSource { String apiUrl = ScryfallImageSupportCards.findDirectDownloadLink(card.getSet(), card.getName(), card.getCollectorId()); if (apiUrl != null) { if (apiUrl.endsWith("*/")) { - apiUrl = apiUrl.substring(0 , apiUrl.length() - 2) + "★/" ; + apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "★/"; } else if (apiUrl.endsWith("+/")) { - apiUrl = apiUrl.substring(0 , apiUrl.length() - 2) + "†/" ; + apiUrl = apiUrl.substring(0, apiUrl.length() - 2) + "†/"; } else if (apiUrl.endsWith("Ph/")) { - apiUrl = apiUrl.substring(0 , apiUrl.length() - 3) + "Φ/" ; + apiUrl = apiUrl.substring(0, apiUrl.length() - 3) + "Φ/"; } // BY DIRECT URL // direct links via hardcoded API path. Used for cards with non-ASCII collector numbers @@ -168,7 +166,7 @@ public class ScryfallImageSource implements CardImageSource { } else { // BY CARD NUMBER // localized and default - String cn = ScryfallImageSupportCards.prepareCardNumber (card.getCollectorId()) ; + String cn = ScryfallImageSupportCards.prepareCardNumber(card.getCollectorId()); needUrls.add(String.format("https://api.scryfall.com/cards/%s/%s/%s", formatSetName(card.getSet(), isToken), cn, @@ -183,18 +181,16 @@ public class ScryfallImageSource implements CardImageSource { InputStream jsonStream = null; String jsonUrl = null; for (String currentUrl : needUrls) { - // connect to Scryfall API + // find first workable api endpoint waitBeforeRequest(); - URL cardUrl = new URL(currentUrl); - URLConnection request = (proxy == null ? cardUrl.openConnection() : cardUrl.openConnection(proxy)); - request.connect(); - try { - jsonStream = (InputStream) request.getContent(); - jsonUrl = currentUrl; + + jsonStream = XmageURLConnection.downloadBinary(currentUrl); + if (jsonStream != null) { // found good url, can stop + jsonUrl = currentUrl; break; - } catch (FileNotFoundException e) { - // localized image doesn't exists, try next url + } else { + // localized image doesn't exist, try next url } } @@ -224,8 +220,6 @@ public class ScryfallImageSource implements CardImageSource { @Override public boolean prepareDownloadList(DownloadServiceInfo downloadServiceInfo, List downloadList) { // prepare download list example ( - Proxy proxy = downloadServiceInfo.getProxy(); - preparedUrls.clear(); // prepare stats @@ -248,7 +242,7 @@ public class ScryfallImageSource implements CardImageSource { if (card.isSecondSide()) { currentPrepareCount++; try { - String url = getFaceImageUrl(proxy, card, card.isToken()); + String url = getFaceImageUrl(card, card.isToken()); preparedUrls.put(card, url); } catch (Exception e) { logger.warn("Failed to prepare image URL (back face) for " + card.getName() + " (" + card.getSet() + ") #" diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java index ef943e8ae61..c72dd1cfaa8 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java @@ -1,7 +1,8 @@ package org.mage.plugins.card.dl.sources; +import mage.client.remote.XmageURLConnection; +import org.jsoup.Jsoup; import org.mage.plugins.card.dl.DownloadJob; -import org.mage.plugins.card.utils.CardImageUtils; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; @@ -65,7 +66,8 @@ public class ScryfallSymbolsSource implements Iterable { try { sourceData = new String(Files.readAllBytes(Paths.get(sourcePath))); } catch (IOException e) { - LOGGER.error("Can't open file to parse data: " + sourcePath + " , reason: " + e.getMessage()); + LOGGER.error("Can't open file to parse svg data: " + sourcePath + ", reason: " + e); + return; } // gen symbols list @@ -129,10 +131,11 @@ public class ScryfallSymbolsSource implements Iterable { // listener for data parse after download complete private class ScryfallDownloadOnFinishedListener implements PropertyChangeListener { + private String downloadedFile; - public ScryfallDownloadOnFinishedListener(String ADestFile) { - this.downloadedFile = ADestFile; + public ScryfallDownloadOnFinishedListener(String downloadedFile) { + this.downloadedFile = downloadedFile; } @Override @@ -152,10 +155,15 @@ public class ScryfallSymbolsSource implements Iterable { } @Override - public void onPreparing() throws Exception { + public void onPreparing() { + // parse help page and find real URL with svg icons on it this.cssUrl = ""; - org.jsoup.nodes.Document doc = CardImageUtils.downloadHtmlDocument(CSS_SOURCE_URL); + // download + String sourceData = XmageURLConnection.downloadText(CSS_SOURCE_URL); + org.jsoup.nodes.Document doc = Jsoup.parse(sourceData); + + // process org.jsoup.select.Elements cssList = doc.select(CSS_SOURCE_SELECTOR); if (cssList.size() == 1) { this.cssUrl = cssList.first().attr("href"); @@ -164,13 +172,13 @@ public class ScryfallSymbolsSource implements Iterable { if (this.cssUrl.isEmpty()) { throw new IllegalStateException("Can't find stylesheet url from scryfall colors page."); } else { - this.setSource(fromURL(this.cssUrl)); + this.setUrl(this.cssUrl); } } public ScryfallSymbolsDownloadJob() { - // download init - super("Scryfall symbols source", fromURL(""), toFile(DOWNLOAD_TEMP_FILE), true); // url setup on preparing stage + // download init (real url will be added on prepare) + super("Scryfall symbols source", "", toFile(DOWNLOAD_TEMP_FILE), true); // url setup on preparing stage String destFile = DOWNLOAD_TEMP_FILE; this.addPropertyChangeListener(STATE_PROP_NAME, new ScryfallDownloadOnFinishedListener(destFile)); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java index bacb41aa449..48034a726ba 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java @@ -4,8 +4,10 @@ import mage.cards.Sets; import mage.cards.repository.CardCriteria; import mage.cards.repository.CardInfo; import mage.cards.repository.CardRepository; +import mage.client.remote.XmageURLConnection; import mage.client.util.CardLanguage; import org.apache.log4j.Logger; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -522,7 +524,8 @@ public enum WizardCardsImageSource implements CardImageSource { while (page < 999) { String searchUrl = "https://gatherer.wizards.com/Pages/Search/Default.aspx?sort=cn+&page=" + page + "&action=advanced&output=spoiler&method=visual&set=+%5B%22" + URLSetName + "%22%5D"; logger.debug("URL: " + searchUrl); - Document doc = CardImageUtils.downloadHtmlDocument(searchUrl); + String sourceData = XmageURLConnection.downloadText(searchUrl); + Document doc = Jsoup.parse(sourceData); Elements cardsImages = doc.select("img[src^=../../Handlers/]"); if (cardsImages.isEmpty()) { break; @@ -565,14 +568,15 @@ public enum WizardCardsImageSource implements CardImageSource { return setLinks; } - private void getLandVariations(LinkedHashMap setLinks, String cardSet, int multiverseId, String cardName) throws IOException, NumberFormatException { + private void getLandVariations(LinkedHashMap setLinks, String cardSet, int multiverseId, String cardName) { CardCriteria criteria = new CardCriteria(); criteria.name(cardName); criteria.setCodes(cardSet); List cards = CardRepository.instance.findCards(criteria); String urlLandDocument = "https://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=" + multiverseId; - Document landDoc = CardImageUtils.downloadHtmlDocument(urlLandDocument); + String sourceData = XmageURLConnection.downloadText(urlLandDocument); + Document landDoc = Jsoup.parse(sourceData); Elements variations = landDoc.select("a.variationlink"); if (!variations.isEmpty()) { if (variations.size() > cards.size()) { @@ -617,9 +621,10 @@ public enum WizardCardsImageSource implements CardImageSource { } } - private Map getlocalizedMultiverseIds(Integer englishMultiverseId) throws IOException { + private Map getlocalizedMultiverseIds(Integer englishMultiverseId) { String cardLanguagesUrl = "https://gatherer.wizards.com/Pages/Card/Languages.aspx?multiverseid=" + englishMultiverseId; - Document cardLanguagesDoc = CardImageUtils.downloadHtmlDocument(cardLanguagesUrl); + String sourceData = XmageURLConnection.downloadText(cardLanguagesUrl); + Document cardLanguagesDoc = Jsoup.parse(sourceData); Elements languageTableRows = cardLanguagesDoc.select("tr.cardItem"); Map localizedIds = new HashMap<>(); if (!languageTableRows.isEmpty()) { diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java b/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java index 492bdce5d4b..3e337f2d88a 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/images/DownloadPicturesService.java @@ -9,12 +9,12 @@ import mage.cards.repository.TokenRepository; import mage.client.MageFrame; import mage.client.dialog.DownloadImagesDialog; import mage.client.dialog.PreferencesDialog; +import mage.client.remote.XmageURLConnection; import mage.client.util.CardLanguage; import mage.client.util.GUISizeHelper; import mage.client.util.sets.ConstructedFormats; -import mage.remote.Connection; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import net.java.truevfs.access.TFile; import net.java.truevfs.access.TFileInputStream; import net.java.truevfs.access.TFileOutputStream; @@ -31,7 +31,8 @@ import java.awt.*; import java.awt.event.ItemEvent; import java.awt.image.BufferedImage; import java.io.*; -import java.net.*; +import java.net.SocketException; +import java.net.UnknownHostException; import java.nio.file.AccessDeniedException; import java.util.List; import java.util.*; @@ -83,7 +84,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements private static CardImageSource selectedSource; private final Object sync = new Object(); - private Proxy proxy = Proxy.NO_PROXY; enum DownloadSources { WIZARDS("1. wizards.com - low quality, cards only", WizardCardsImageSource.instance), @@ -635,97 +635,72 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements base.mkdir(); } - Connection.ProxyType configProxyType = Connection.ProxyType.valueByText(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_TYPE, "None")); - Proxy.Type type = Proxy.Type.DIRECT; - switch (configProxyType) { - case HTTP: - type = Proxy.Type.HTTP; - break; - case SOCKS: - type = Proxy.Type.SOCKS; - break; - case NONE: - default: - proxy = Proxy.NO_PROXY; - break; - } - - if (type != Proxy.Type.DIRECT) { - try { - String address = PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_ADDRESS, ""); - Integer port = Integer.parseInt(PreferencesDialog.getCachedValue(PreferencesDialog.KEY_PROXY_PORT, "80")); - proxy = new Proxy(type, new InetSocketAddress(address, port)); - } catch (Exception ex) { - throw new RuntimeException("Gui_DownloadPicturesService : error 1 - " + ex); - } - } - int downloadThreadsAmount = Math.max(1, Integer.parseInt((String) uiDialog.getDownloadThreadsCombo().getSelectedItem())); - if (proxy != null) { - logger.info("Started download of " + cardsDownloadQueue.size() + " images" - + " from source: " + selectedSource.getSourceName() - + ", language: " + selectedSource.getCurrentLanguage().getCode() - + ", threads: " + downloadThreadsAmount); - updateProgressMessage("Preparing download list..."); - if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) { - update(0, cardsDownloadQueue.size()); - ExecutorService executor = Executors.newFixedThreadPool( - downloadThreadsAmount, - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_IMAGES_DOWNLOADER, false) - ); - for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) { - try { - CardDownloadData card = cardsDownloadQueue.get(i); - logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')'); + logger.info("Started download of " + cardsDownloadQueue.size() + " images" + + " from source: " + selectedSource.getSourceName() + + ", language: " + selectedSource.getCurrentLanguage().getCode() + + ", threads: " + downloadThreadsAmount); + updateProgressMessage("Preparing download list..."); + if (selectedSource.prepareDownloadList(this, cardsDownloadQueue)) { + update(0, cardsDownloadQueue.size()); + ExecutorService executor = Executors.newFixedThreadPool( + downloadThreadsAmount, + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_IMAGES_DOWNLOADER, false) + ); + for (int i = 0; i < cardsDownloadQueue.size() && !this.isNeedCancel(); i++) { + try { + CardDownloadData card = cardsDownloadQueue.get(i); - CardImageUrls urls; - if (card.isToken()) { - if (!"0".equals(card.getCollectorId())) { - continue; - } - urls = selectedSource.generateTokenUrl(card); - } else { - urls = selectedSource.generateCardUrl(card); + logger.debug("Downloading image: " + card.getName() + " (" + card.getSet() + ')'); + + CardImageUrls urls; + if (card.isToken()) { + if (!"0".equals(card.getCollectorId())) { + continue; } - - if (urls == null) { - String imageRef = selectedSource.getNextHttpImageUrl(); - String fileName = selectedSource.getFileForHttpImage(imageRef); - if (imageRef != null && fileName != null) { - imageRef = selectedSource.getSourceName() + imageRef; - try { - card.setToken(selectedSource.isTokenSource()); - Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages()); - executor.execute(task); - } catch (Exception ex) { - } - } else if (selectedSource.getTotalImages() == -1) { - logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')'); - synchronized (sync) { - update(cardIndex + 1, cardsDownloadQueue.size()); - } - } - } else { - Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size()); - executor.execute(task); - } - } catch (Exception ex) { - logger.error(ex, ex); + urls = selectedSource.generateTokenUrl(card); + } else { + urls = selectedSource.generateCardUrl(card); } + + if (urls == null) { + String imageRef = selectedSource.getNextHttpImageUrl(); + String fileName = selectedSource.getFileForHttpImage(imageRef); + if (imageRef != null && fileName != null) { + imageRef = selectedSource.getSourceName() + imageRef; + try { + card.setToken(selectedSource.isTokenSource()); + Runnable task = new DownloadTask(card, imageRef, fileName, selectedSource.getTotalImages()); + executor.execute(task); + } catch (Exception ex) { + } + } else if (selectedSource.getTotalImages() == -1) { + logger.info("Image not available on " + selectedSource.getSourceName() + ": " + card.getName() + " (" + card.getSet() + ')'); + synchronized (sync) { + update(cardIndex + 1, cardsDownloadQueue.size()); + } + } + } else { + Runnable task = new DownloadTask(card, urls, cardsDownloadQueue.size()); + executor.execute(task); + } + } catch (Exception ex) { + logger.error(ex, ex); } + } - executor.shutdown(); - while (!executor.isTerminated()) { - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException ignore) { - } + executor.shutdown(); + while (!executor.isTerminated()) { + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException ignore) { } } } + try { TVFS.umount(); } catch (FsSyncException e) { @@ -743,11 +718,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements GUISizeHelper.refreshGUIAndCards(false); } - static String convertStreamToString(InputStream is) { - java.util.Scanner s = new java.util.Scanner(is).useDelimiter("\\A"); - return s.hasNext() ? s.next() : ""; - } - private final class DownloadTask implements Runnable { private final CardDownloadData card; @@ -834,17 +804,17 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements // can download images from many alternative urls List downloadUrls; + XmageURLConnection connection = null; if (this.urls != null) { downloadUrls = this.urls.getDownloadList(); } else { downloadUrls = new ArrayList<>(); } + // try to find first workable link boolean isDownloadOK = false; - URLConnection httpConn = null; List errorsList = new ArrayList<>(); for (String currentUrl : downloadUrls) { - URL url = new URL(currentUrl); // fast stop on cancel if (DownloadPicturesService.getInstance().isNeedCancel()) { @@ -852,29 +822,27 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements } // timeout before each request - selectedSource.doPause(url.toString()); + selectedSource.doPause(currentUrl); - httpConn = url.openConnection(proxy); - if (httpConn != null) { + connection = new XmageURLConnection(currentUrl); + connection.startConnection(); + if (connection.isConnected()) { - // custom headers like user agent - Map headers = selectedSource.getHttpRequestHeaders(url.toString()); - for (String key : headers.keySet()) { - httpConn.setRequestProperty(key, headers.get(key)); - } + // custom headers (ues + connection.setRequestHeaders(selectedSource.getHttpRequestHeaders(currentUrl)); try { - httpConn.connect(); + connection.connect(); } catch (SocketException e) { incErrorCount(); - errorsList.add("Wrong image URL or java app is not allowed to use network. Check your firewall or proxy settings. Error: " + e.getMessage() + ". Image URL: " + url.toString()); + errorsList.add("Wrong image URL or java app is not allowed to use network. Check your firewall or proxy settings. Error: " + e.getMessage() + ". Image URL: " + currentUrl); break; } catch (UnknownHostException e) { incErrorCount(); - errorsList.add("Unknown site. Check your DNS settings. Error: " + e.getMessage() + ". Image URL: " + url.toString()); + errorsList.add("Unknown site. Check your DNS settings. Error: " + e.getMessage() + ". Image URL: " + currentUrl); break; } - int responseCode = ((HttpURLConnection) httpConn).getResponseCode(); + int responseCode = connection.getResponseCode(); // check result if (responseCode != 200) { @@ -883,13 +851,13 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements card.getSet(), card.getName(), responseCode, - url + currentUrl ); errorsList.add(info); if (logger.isDebugEnabled()) { // Shows the returned html from the request to the web server - logger.debug("Returned HTML ERROR:\n" + convertStreamToString(((HttpURLConnection) httpConn).getErrorStream())); + logger.debug("Returned HTML ERROR:\n" + connection.getErrorResponseAsString()); } // go to next try @@ -902,12 +870,12 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements } } - // can save result - if (isDownloadOK && httpConn != null) { + // if workable link found then save result + if (isDownloadOK && connection.isConnected()) { // save data to temp - try (InputStream in = new BufferedInputStream(httpConn.getInputStream()); - OutputStream tfileout = new TFileOutputStream(fileTempImage); - OutputStream out = new BufferedOutputStream(tfileout)) { + try (InputStream in = new BufferedInputStream(connection.getGoodResponseAsStream()); + OutputStream tempFileStream = new TFileOutputStream(fileTempImage); + OutputStream out = new BufferedOutputStream(tempFileStream)) { byte[] buf = new byte[1024]; int len; while ((len = in.read(buf)) != -1) { @@ -931,6 +899,8 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements } return; } + + // all fine, can save data part out.write(buf, 0, len); } } @@ -947,7 +917,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements } catch (Exception e) { logger.error("Can't delete temp file: " + e.getMessage(), e); } - } } else { // download errors @@ -1013,11 +982,6 @@ public class DownloadPicturesService extends DefaultBoundedRangeModel implements uiDialog.getStartButton().setEnabled(true); } - @Override - public Proxy getProxy() { - return proxy; - } - @Override public Object getSync() { return sync; diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java b/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java index 64cf2bd90b3..e61012c8019 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java @@ -6,6 +6,7 @@ import mage.cards.repository.TokenRepository; import mage.client.MageFrame; import mage.client.constants.Constants; import mage.client.dialog.PreferencesDialog; +import mage.client.remote.XmageURLConnection; import mage.remote.Connection; import mage.remote.Connection.ProxyType; import mage.view.CardView; @@ -201,33 +202,6 @@ public final class CardImageUtils { return null; } - public static Document downloadHtmlDocument(String urlString) throws NumberFormatException, IOException { - Preferences prefs = MageFrame.getPreferences(); - Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None")); - Document doc; - if (proxyType == ProxyType.NONE) { - doc = Jsoup.connect(urlString).timeout(60 * 1000).get(); - } else { - String proxyServer = prefs.get("proxyAddress", ""); - int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0")); - URL url = new URL(urlString); - Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort)); - HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy); - uc.setConnectTimeout(10000); - uc.setReadTimeout(60000); - uc.connect(); - - String line; - StringBuffer tmp = new StringBuffer(); - BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream())); - while ((line = in.readLine()) != null) { - tmp.append(line); - } - doc = Jsoup.parse(String.valueOf(tmp)); - } - return doc; - } - public static void checkAndFixImageFiles() { // search broken, temp or outdated files and delete it // real images check is slow, so it used on images download only (not here) diff --git a/Mage.Client/src/test/java/mage/client/util/DownloaderTest.java b/Mage.Client/src/test/java/mage/client/util/DownloaderTest.java new file mode 100644 index 00000000000..6898daf1ed5 --- /dev/null +++ b/Mage.Client/src/test/java/mage/client/util/DownloaderTest.java @@ -0,0 +1,48 @@ +package mage.client.util; + +import mage.client.remote.XmageURLConnection; +import org.junit.Assert; +import org.junit.Test; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; + +/** + * @author JayDi85 + */ +public class DownloaderTest { + + @Test + public void test_DownloadText_ByHttp() { + String s = XmageURLConnection.downloadText("http://google.com"); + Assert.assertTrue("must have text data", s.contains("")); + } + + @Test + public void test_DownloadText_ByHttps() { + String s = XmageURLConnection.downloadText("https://google.com"); + Assert.assertTrue("must have text data", s.contains("")); + } + + @Test + public void test_DownloadFile_ByHttp() throws IOException { + // use any public image here + InputStream stream = XmageURLConnection.downloadBinary("http://www.google.com/tia/tia.png"); + Assert.assertNotNull(stream); + BufferedImage image = ImageIO.read(stream); + Assert.assertNotNull(stream); + Assert.assertTrue("must have image data", image.getWidth() > 0); + } + + @Test + public void test_DownloadFile_ByHttps() throws IOException { + // use any public image here + InputStream stream = XmageURLConnection.downloadBinary("https://www.google.com/tia/tia.png"); + Assert.assertNotNull(stream); + BufferedImage image = ImageIO.read(stream); + Assert.assertNotNull(stream); + Assert.assertTrue("must have image data", image.getWidth() > 0); + } +} diff --git a/Mage.Common/src/main/java/mage/utils/MageVersion.java b/Mage.Common/src/main/java/mage/utils/MageVersion.java index 24ac2d102f6..a991e6999cf 100644 --- a/Mage.Common/src/main/java/mage/utils/MageVersion.java +++ b/Mage.Common/src/main/java/mage/utils/MageVersion.java @@ -91,4 +91,8 @@ public class MageVersion implements Serializable, Comparable { public boolean isDeveloperBuild() { return this.buildTime.contains(JarVersion.JAR_BUILD_TIME_FROM_CLASSES); } + + public String getBuildTime() { + return this.buildTime; + } } diff --git a/Mage.Server.Console/src/main/java/mage/server/console/ConsoleFrame.java b/Mage.Server.Console/src/main/java/mage/server/console/ConsoleFrame.java index 393b8efba55..338b6761e25 100644 --- a/Mage.Server.Console/src/main/java/mage/server/console/ConsoleFrame.java +++ b/Mage.Server.Console/src/main/java/mage/server/console/ConsoleFrame.java @@ -6,7 +6,7 @@ import mage.remote.Connection; import mage.remote.Session; import mage.remote.SessionImpl; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.utils.MageVersion; import org.apache.log4j.Logger; @@ -32,7 +32,7 @@ public class ConsoleFrame extends javax.swing.JFrame implements MageClient { private static final MageVersion version = new MageVersion(ConsoleFrame.class); private static final ScheduledExecutorService PING_SENDER_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_PING_SENDER) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CLIENT_PING_SENDER) ); /** diff --git a/Mage.Server.Plugins/Mage.Player.AI.MA/src/mage/player/ai/ComputerPlayer6.java b/Mage.Server.Plugins/Mage.Player.AI.MA/src/mage/player/ai/ComputerPlayer6.java index 0498e89f07c..d3fc038f720 100644 --- a/Mage.Server.Plugins/Mage.Player.AI.MA/src/mage/player/ai/ComputerPlayer6.java +++ b/Mage.Server.Plugins/Mage.Player.AI.MA/src/mage/player/ai/ComputerPlayer6.java @@ -32,7 +32,7 @@ import mage.target.TargetCard; import mage.util.CardUtil; import mage.util.RandomUtil; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import org.apache.log4j.Logger; import java.util.*; @@ -61,7 +61,7 @@ public class ComputerPlayer6 extends ComputerPlayer { 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_AI_SIMULATION_MAD) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_AI_SIMULATION_MAD) ); protected int maxDepth; protected int maxNodes; diff --git a/Mage.Server.Plugins/Mage.Player.AIMCTS/src/mage/player/ai/ComputerPlayerMCTS.java b/Mage.Server.Plugins/Mage.Player.AIMCTS/src/mage/player/ai/ComputerPlayerMCTS.java index 1b19a95b4dc..f50503d8d2a 100644 --- a/Mage.Server.Plugins/Mage.Player.AIMCTS/src/mage/player/ai/ComputerPlayerMCTS.java +++ b/Mage.Server.Plugins/Mage.Player.AIMCTS/src/mage/player/ai/ComputerPlayerMCTS.java @@ -13,7 +13,7 @@ import mage.game.combat.CombatGroup; import mage.player.ai.MCTSPlayer.NextAction; import mage.players.Player; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import org.apache.log4j.Logger; import java.util.ArrayList; @@ -173,7 +173,7 @@ public class ComputerPlayerMCTS extends ComputerPlayer { 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_AI_SIMULATION_MCTS) // TODO: add player/game to thread name? + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_AI_SIMULATION_MCTS) // TODO: add player/game to thread name? ); } diff --git a/Mage.Server/src/main/java/mage/server/UserManagerImpl.java b/Mage.Server/src/main/java/mage/server/UserManagerImpl.java index 5ca39c8c2c8..571142a9b8d 100644 --- a/Mage.Server/src/main/java/mage/server/UserManagerImpl.java +++ b/Mage.Server/src/main/java/mage/server/UserManagerImpl.java @@ -6,7 +6,7 @@ import mage.server.record.UserStats; import mage.server.record.UserStatsRepository; import mage.server.util.ServerMessagesUtil; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.view.UserView; import org.apache.log4j.Logger; @@ -34,10 +34,10 @@ public class UserManagerImpl implements UserManager { private static final Logger logger = Logger.getLogger(UserManagerImpl.class); protected final ScheduledExecutorService CONNECTION_EXPIRED_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_CONNECTION_EXPIRED_CHECK) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_CONNECTION_EXPIRED_CHECK) ); protected final ScheduledExecutorService USERS_LIST_REFRESH_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_USERS_LIST_REFRESH) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_USERS_LIST_REFRESH) ); private List userInfoList = new ArrayList<>(); // all users list for main room/chat diff --git a/Mage.Server/src/main/java/mage/server/game/GameController.java b/Mage.Server/src/main/java/mage/server/game/GameController.java index 53f5a733abc..44778e69aa2 100644 --- a/Mage.Server/src/main/java/mage/server/game/GameController.java +++ b/Mage.Server/src/main/java/mage/server/game/GameController.java @@ -26,7 +26,7 @@ import mage.server.User; import mage.server.managers.ManagerFactory; import mage.util.MultiAmountMessage; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.utils.StreamUtils; import mage.utils.timer.PriorityTimer; import mage.view.*; @@ -244,7 +244,7 @@ public class GameController implements GameCallback { // wait all players if (JOIN_WAITING_EXECUTOR == null) { JOIN_WAITING_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_GAME_JOIN_WAITING + " " + game.getId()) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_GAME_JOIN_WAITING + " " + game.getId()) ); } JOIN_WAITING_EXECUTOR.scheduleAtFixedRate(() -> { diff --git a/Mage.Server/src/main/java/mage/server/game/GamesRoomImpl.java b/Mage.Server/src/main/java/mage/server/game/GamesRoomImpl.java index a86b9c716e4..052bbdb0130 100644 --- a/Mage.Server/src/main/java/mage/server/game/GamesRoomImpl.java +++ b/Mage.Server/src/main/java/mage/server/game/GamesRoomImpl.java @@ -12,7 +12,7 @@ import mage.server.RoomImpl; import mage.server.User; import mage.server.managers.ManagerFactory; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.view.MatchView; import mage.view.RoomUsersView; import mage.view.TableView; @@ -40,7 +40,7 @@ public class GamesRoomImpl extends RoomImpl implements GamesRoom, Serializable { private static List lobbyMatches = new ArrayList<>(); private static List lobbyUsers = new ArrayList<>(); private static final ScheduledExecutorService UPDATE_LOBBY_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_LOBBY_REFRESH) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_LOBBY_REFRESH) ); private final ManagerFactory managerFactory; diff --git a/Mage.Server/src/main/java/mage/server/util/ServerMessagesUtil.java b/Mage.Server/src/main/java/mage/server/util/ServerMessagesUtil.java index 317f5841140..9b04583f10b 100644 --- a/Mage.Server/src/main/java/mage/server/util/ServerMessagesUtil.java +++ b/Mage.Server/src/main/java/mage/server/util/ServerMessagesUtil.java @@ -1,7 +1,7 @@ package mage.server.util; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.utils.StreamUtils; import org.apache.log4j.Logger; @@ -47,7 +47,7 @@ public enum ServerMessagesUtil { ServerMessagesUtil() { ScheduledExecutorService NEWS_MESSAGES_EXECUTOR = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_NEWS_REFRESH) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_NEWS_REFRESH) ); NEWS_MESSAGES_EXECUTOR.scheduleAtFixedRate(this::reloadMessages, 5, SERVER_MSG_REFRESH_RATE_SECS, TimeUnit.SECONDS); } diff --git a/Mage.Server/src/main/java/mage/server/util/ThreadExecutorImpl.java b/Mage.Server/src/main/java/mage/server/util/ThreadExecutorImpl.java index ddffccc083c..cd475219829 100644 --- a/Mage.Server/src/main/java/mage/server/util/ThreadExecutorImpl.java +++ b/Mage.Server/src/main/java/mage/server/util/ThreadExecutorImpl.java @@ -3,7 +3,7 @@ package mage.server.util; import mage.server.managers.ConfigSettings; import mage.server.managers.ThreadExecutor; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import org.apache.log4j.Logger; import java.util.concurrent.*; @@ -42,31 +42,31 @@ public class ThreadExecutorImpl implements ThreadExecutor { callExecutor = new CachedThreadPoolWithException(); ((ThreadPoolExecutor) callExecutor).setKeepAliveTime(60, TimeUnit.SECONDS); ((ThreadPoolExecutor) callExecutor).allowCoreThreadTimeOut(true); - ((ThreadPoolExecutor) callExecutor).setThreadFactory(new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_CALL_REQUEST)); + ((ThreadPoolExecutor) callExecutor).setThreadFactory(new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_CALL_REQUEST)); //gameExecutor = Executors.newFixedThreadPool(config.getMaxGameThreads()); gameExecutor = new FixedThreadPoolWithException(config.getMaxGameThreads()); ((ThreadPoolExecutor) gameExecutor).setKeepAliveTime(60, TimeUnit.SECONDS); ((ThreadPoolExecutor) gameExecutor).allowCoreThreadTimeOut(true); - ((ThreadPoolExecutor) gameExecutor).setThreadFactory(new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_GAME)); + ((ThreadPoolExecutor) gameExecutor).setThreadFactory(new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_GAME)); //tourney = Executors.newFixedThreadPool(config.getMaxGameThreads() / GAMES_PER_TOURNEY_RATIO); tourneyExecutor = new FixedThreadPoolWithException(config.getMaxGameThreads() / GAMES_PER_TOURNEY_RATIO); ((ThreadPoolExecutor) tourneyExecutor).setKeepAliveTime(60, TimeUnit.SECONDS); ((ThreadPoolExecutor) tourneyExecutor).allowCoreThreadTimeOut(true); - ((ThreadPoolExecutor) tourneyExecutor).setThreadFactory(new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_TOURNEY)); + ((ThreadPoolExecutor) tourneyExecutor).setThreadFactory(new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_TOURNEY)); timeoutExecutor = Executors.newScheduledThreadPool(4); ((ThreadPoolExecutor) timeoutExecutor).setKeepAliveTime(60, TimeUnit.SECONDS); ((ThreadPoolExecutor) timeoutExecutor).allowCoreThreadTimeOut(true); - ((ThreadPoolExecutor) timeoutExecutor).setThreadFactory(new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_TIMEOUT)); + ((ThreadPoolExecutor) timeoutExecutor).setThreadFactory(new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_TIMEOUT)); timeoutIdleExecutor = Executors.newScheduledThreadPool(4); ((ThreadPoolExecutor) timeoutIdleExecutor).setKeepAliveTime(60, TimeUnit.SECONDS); ((ThreadPoolExecutor) timeoutIdleExecutor).allowCoreThreadTimeOut(true); - ((ThreadPoolExecutor) timeoutIdleExecutor).setThreadFactory(new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_TIMEOUT_IDLE)); + ((ThreadPoolExecutor) timeoutIdleExecutor).setThreadFactory(new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_TIMEOUT_IDLE)); - serverHealthExecutor = Executors.newSingleThreadScheduledExecutor(new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_HEALTH)); + serverHealthExecutor = Executors.newSingleThreadScheduledExecutor(new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_SERVICE_HEALTH)); } static class CachedThreadPoolWithException extends ThreadPoolExecutor { diff --git a/Mage.Tests/src/test/java/org/mage/test/load/LoadTest.java b/Mage.Tests/src/test/java/org/mage/test/load/LoadTest.java index 1f587846341..e2da4f62218 100644 --- a/Mage.Tests/src/test/java/org/mage/test/load/LoadTest.java +++ b/Mage.Tests/src/test/java/org/mage/test/load/LoadTest.java @@ -14,7 +14,7 @@ import mage.remote.Session; import mage.remote.SessionImpl; import mage.util.RandomUtil; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import mage.view.*; import org.apache.log4j.Logger; import org.junit.Assert; @@ -328,9 +328,9 @@ public class LoadTest { ExecutorService executerService; if (isRunParallel) { - executerService = Executors.newFixedThreadPool(gamesAmount, new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_TESTS_AI_VS_AI_GAMES)); + executerService = Executors.newFixedThreadPool(gamesAmount, new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_TESTS_AI_VS_AI_GAMES)); } else { - executerService = Executors.newSingleThreadExecutor(new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_TESTS_AI_VS_AI_GAMES)); + executerService = Executors.newSingleThreadExecutor(new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_TESTS_AI_VS_AI_GAMES)); } // save random seeds for repeated results (in decks generating) diff --git a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java index 3386ce12af4..c8f63b04098 100644 --- a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java +++ b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonCard.java @@ -4,11 +4,16 @@ import mage.constants.Rarity; import java.util.List; +/** + * MTGJSON v5: card class + *

+ * Contains card related data nd only used fields, if you need more for tests then just add it here + *

+ * API docs here + * + * @author JayDi85 + */ public final class MtgJsonCard { - // v5 support - // https://mtgjson.com/data-models/card-atomic/ - // contains only used fields, if you need more for tests then just add it here - public String name; public String asciiName; // mtgjson uses it for some cards like El-Hajjaj public String number; // from sets source only, see https://mtgjson.com/data-models/card-set/ @@ -44,7 +49,6 @@ public final class MtgJsonCard { } /** - * * @return single side name like Ice from Fire // Ice */ public String getNameAsFace() { @@ -53,7 +57,6 @@ public final class MtgJsonCard { } /** - * * @return full card name like Fire // Ice */ public String getNameAsFull() { diff --git a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonMetadata.java b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonMetadata.java index 10c10dfae35..a9b117dabf6 100644 --- a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonMetadata.java +++ b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonMetadata.java @@ -1,9 +1,15 @@ package mage.verify.mtgjson; +/** + * MTGJSON v5: metadata class + *

+ * Contains version info + *

+ * API docs here + * + * @author JayDi85 + */ public final class MtgJsonMetadata { - // MTGJSON metadata - // https://mtgjson.com/file-models/meta/ - public String date; public String version; } diff --git a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java index 5bc45c901b8..62ca7fdb5da 100644 --- a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java +++ b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonService.java @@ -1,10 +1,13 @@ package mage.verify.mtgjson; import com.google.gson.Gson; +import mage.client.remote.XmageURLConnection; +import org.apache.log4j.Logger; -import java.io.*; -import java.net.URL; -import java.net.URLConnection; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.text.Normalizer; @@ -12,8 +15,15 @@ import java.util.*; import java.util.stream.Collectors; import java.util.zip.ZipInputStream; +/** + * MTGJSON v5: basic service to work with mtgjson data + * + * @author JayDi85 + */ public final class MtgJsonService { + private static final Logger logger = Logger.getLogger(MtgJsonService.class); + public static Map mtgJsonToXMageCodes = new HashMap<>(); public static Map xMageToMtgJsonCodes = new HashMap<>(); @@ -36,23 +46,36 @@ public final class MtgJsonService { } private static T readFromZip(String filename, Class clazz) throws IOException { + + // build-in file InputStream stream = MtgJsonService.class.getResourceAsStream(filename); - if (stream == null) { - File file = new File(filename); - if (!file.exists()) { - String url = "https://mtgjson.com/api/v5/" + filename; - System.out.println("Downloading " + url + " to " + file.getAbsolutePath()); - URLConnection connection = new URL(url).openConnection(); - connection.setRequestProperty("user-agent", "xmage"); - InputStream download = connection.getInputStream(); - Files.copy(download, file.toPath(), StandardCopyOption.REPLACE_EXISTING); - System.out.println("Downloading DONE"); - } else { - System.out.println("Found file " + filename + " from " + file.getAbsolutePath()); - } - stream = new FileInputStream(file); + if (stream != null) { + logger.info("mtgjson: use build-in file " + filename); + return readFromZip(stream, clazz); } + // already downloaded file + File file = new File(filename); + if (file.exists()) { + logger.info("mtgjson: use existing file " + filename + " from " + file.getAbsolutePath()); + return readFromZip(Files.newInputStream(file.toPath()), clazz); + } + + // new download + String url = "https://mtgjson.com/api/v5/" + filename; + logger.info("mtgjson: downloading new file " + url); + // mtgjson site require user-agent in headers (otherwise it return 403) + stream = XmageURLConnection.downloadBinary(url); + if (stream != null) { + logger.info("mtgjson: download DONE, saved to " + file.getAbsolutePath()); + Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + return readFromZip(Files.newInputStream(file.toPath()), clazz); + } + + throw new IOException("mtgjson: can't found or download file, check your connection " + filename); + } + + private static T readFromZip(InputStream stream, Class clazz) throws IOException { try (ZipInputStream zipInputStream = new ZipInputStream(stream)) { zipInputStream.getNextEntry(); return new Gson().fromJson(new InputStreamReader(zipInputStream), clazz); @@ -243,5 +266,4 @@ public final class MtgJsonService { } } } - } diff --git a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java index f302ec8e02c..4e9a48aee2a 100644 --- a/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java +++ b/Mage.Verify/src/main/java/mage/verify/mtgjson/MtgJsonSet.java @@ -2,10 +2,17 @@ package mage.verify.mtgjson; import java.util.List; +/** + * MTGJSON v5: set class + *

+ * Contains set info and related cards list + * Only used fields, if you need more for tests then just add it here + *

+ * API docs here + * + * @author JayDi85 + */ public final class MtgJsonSet { - // v5 support - // https://mtgjson.com/data-models/card-atomic/ - // contains only used fields, if you need more for tests then just add it here public List cards; public String code; diff --git a/Mage/src/main/java/mage/game/draft/DraftImpl.java b/Mage/src/main/java/mage/game/draft/DraftImpl.java index f9a08345f53..4f2ff6883db 100644 --- a/Mage/src/main/java/mage/game/draft/DraftImpl.java +++ b/Mage/src/main/java/mage/game/draft/DraftImpl.java @@ -8,7 +8,7 @@ import mage.game.events.TableEvent.EventType; import mage.players.Player; import mage.players.PlayerList; import mage.util.ThreadUtils; -import mage.util.XMageThreadFactory; +import mage.util.XmageThreadFactory; import org.apache.log4j.Logger; import java.util.*; @@ -248,7 +248,7 @@ public abstract class DraftImpl implements Draft { if (this.boosterLoadingExecutor == null) { this.boosterLoadingExecutor = Executors.newSingleThreadScheduledExecutor( - new XMageThreadFactory(ThreadUtils.THREAD_PREFIX_TOURNEY_BOOSTERS_SEND) + new XmageThreadFactory(ThreadUtils.THREAD_PREFIX_TOURNEY_BOOSTERS_SEND) ); } diff --git a/Mage/src/main/java/mage/util/XMageThreadFactory.java b/Mage/src/main/java/mage/util/XmageThreadFactory.java similarity index 86% rename from Mage/src/main/java/mage/util/XMageThreadFactory.java rename to Mage/src/main/java/mage/util/XmageThreadFactory.java index be96e40fc9a..d80ccaa308a 100644 --- a/Mage/src/main/java/mage/util/XMageThreadFactory.java +++ b/Mage/src/main/java/mage/util/XmageThreadFactory.java @@ -8,13 +8,13 @@ import java.util.concurrent.atomic.AtomicInteger; * * @author JayDi85 */ -public class XMageThreadFactory implements ThreadFactory { +public class XmageThreadFactory implements ThreadFactory { private final String prefix; private final AtomicInteger counter = new AtomicInteger(); private final boolean isDaemon; - public XMageThreadFactory(String prefix) { + public XmageThreadFactory(String prefix) { this(prefix, true); } @@ -22,7 +22,7 @@ public class XMageThreadFactory implements ThreadFactory { * @param prefix thread's starting name (can be changed by thread itself later) * @param isDaemon mark thread as daemon on non-writeable tasks (e.g. can be terminated at any time without data loss) */ - public XMageThreadFactory(String prefix, boolean isDaemon) { + public XmageThreadFactory(String prefix, boolean isDaemon) { this.prefix = prefix; this.isDaemon = isDaemon; }