diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadGui.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadGui.java index a8438a706af..e3ff2ca179d 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadGui.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadGui.java @@ -158,6 +158,7 @@ public class DownloadGui extends JPanel { b.addActionListener(e -> { switch(this.job.getState()) { case NEW: + case PREPARING: case WORKING: this.job.setState(State.ABORTED); } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java index 6efcfd3eef4..c39fb6d1633 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/DownloadJob.java @@ -23,16 +23,16 @@ import org.mage.plugins.card.utils.CardImageUtils; * The class DownloadJob. * * @version V0.0 25.08.2010 - * @author Clemens Koza + * @author Clemens Koza, JayDi85 */ public class DownloadJob extends AbstractLaternaBean { public enum State { - NEW, WORKING, FINISHED, ABORTED + NEW, PREPARING, WORKING, FINISHED, ABORTED } private final String name; - private final Source source; + private Source source; private final Destination destination; private final Property state = properties.property("state", State.NEW); private final Property message = properties.property("message"); @@ -98,6 +98,36 @@ public class DownloadJob extends AbstractLaternaBean { this.message.setValue(message); } + /** + * Inner prepare cycle from new to working + */ + public void doPrepareAndStartWork() { + if (this.state.getValue() != State.NEW) { + setError("Can't call prepare at this point."); + return; + } + + this.state.setValue(State.PREPARING); + + try { + onPreparing(); + } catch (Exception e) { + setError("Prepare error: " + e.getMessage(), e); + return; + } + + // change to working state on good prepare call + this.state.setValue(State.WORKING); + } + + + /** + * Prepare code to override in custom download tasks (it's calls before work start) + */ + public void onPreparing() throws Exception { + return; + } + /** * Sets the job's message. * @@ -131,6 +161,10 @@ public class DownloadJob extends AbstractLaternaBean { return source; } + public void setSource(Source source) { + this.source = source; + } + public Destination getDestination() { return destination; } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java index 4b71a8d16c0..497ee62d62b 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/Downloader.java @@ -60,6 +60,7 @@ public class Downloader extends AbstractLaternaBean implements Disposable { for (DownloadJob j : jobs) { switch (j.getState()) { case NEW: + case PREPARING: case WORKING: j.setState(State.ABORTED); } @@ -82,8 +83,11 @@ public class Downloader extends AbstractLaternaBean implements Disposable { } public void add(DownloadJob job) { + if (job.getState() == State.PREPARING) { + throw new IllegalArgumentException("Job already preparing"); + } if (job.getState() == State.WORKING) { - throw new IllegalArgumentException("Job already running"); + throw new IllegalArgumentException("Job already working"); } if (job.getState() == State.FINISHED) { throw new IllegalArgumentException("Job already finished"); @@ -106,13 +110,22 @@ public class Downloader extends AbstractLaternaBean implements Disposable { @Override public void onMessage(DownloadJob job) { - //the job won't be processed by multiple threads + + // start to work + // the job won't be processed by multiple threads synchronized (job) { if (job.getState() != State.NEW) { return; } - job.setState(State.WORKING); + + job.doPrepareAndStartWork(); + + if (job.getState() != State.WORKING) { + return; + } } + + // download and save data try { Source src = job.getSource(); Destination dst = job.getDestination(); diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java index e90d1cf9c5d..483bb27e866 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallSymbolsSource.java @@ -9,8 +9,13 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import mage.MageException; import mage.util.StreamUtils; +import org.jsoup.select.Elements; import org.mage.plugins.card.dl.DownloadJob; +import org.mage.plugins.card.utils.CardImageUtils; + +import javax.swing.text.Document; import static org.mage.card.arcane.ManaSymbols.getSymbolFileNameAsSVG; import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; @@ -18,14 +23,14 @@ import static org.mage.plugins.card.utils.CardImageUtils.getImagesDir; // TODO: add force to download symbols (rewrite exist files) /** - * - * @author jaydi85@gmail.com - * + * @author JayDi85 */ public class ScryfallSymbolsSource implements Iterable { - static final String SOURCE_URL = "https://assets.scryfall.com/assets/scryfall.css"; // search css-file on https://scryfall.com/docs/api/colors + //static final String SOURCE_URL = "https://assets.scryfall.com/assets/scryfall.css"; // old version with direct css-file on https://scryfall.com/docs/api/colors + static final String CSS_SOURCE_URL = "https://scryfall.com/docs/api/colors"; + static final String CSS_SOURCE_SELECTOR = "link[rel=stylesheet]"; // static final String STATE_PROP_NAME = "state"; static final String DOWNLOAD_TEMP_FILE = getImagesDir() + File.separator + "temp" + File.separator + "scryfall-symbols-source.txt"; @@ -55,21 +60,21 @@ public class ScryfallSymbolsSource implements Iterable { return jobs.iterator(); } - private void parseData(String sourcePath){ + private void parseData(String sourcePath) { String sourceData = ""; try { sourceData = new String(Files.readAllBytes(Paths.get(sourcePath))); - }catch (IOException e) { + } catch (IOException e) { LOGGER.error("Can't open file to parse data: " + sourcePath + " , reason: " + e.getMessage()); } // gen symbols list ArrayList allMageSymbols = new ArrayList<>(); - for(int i = 0; i < SYMBOLS_LIST.length; i++){ + for (int i = 0; i < SYMBOLS_LIST.length; i++) { allMageSymbols.add(SYMBOLS_LIST[i]); } - for(Integer i = SYMBOLS_NUMBER_START; i <= SYMBOLS_NUMBER_END; i++){ + for (Integer i = SYMBOLS_NUMBER_START; i <= SYMBOLS_NUMBER_END; i++) { allMageSymbols.add(String.valueOf(SYMBOLS_NUMBER_START + i)); } @@ -88,23 +93,22 @@ public class ScryfallSymbolsSource implements Iterable { // dirs maker File dir = getSymbolFileNameAsSVG("W").getParentFile(); - if(!dir.exists()){ + if (!dir.exists()) { dir.mkdirs(); } // decode and save data (only if not exist) - for(String needCode: allMageSymbols){ + for (String needCode : allMageSymbols) { String searchCode = needCode.replace("/", ""); - if(!foundedData.containsKey(searchCode)) - { + if (!foundedData.containsKey(searchCode)) { LOGGER.warn("Can't found symbol code from scryfall: " + searchCode); continue; } File destFile = getSymbolFileNameAsSVG(searchCode); - if (destFile.exists() && (destFile.length() > 0)){ + if (destFile.exists() && (destFile.length() > 0)) { continue; } FileOutputStream stream = null; @@ -118,7 +122,7 @@ public class ScryfallSymbolsSource implements Iterable { stream.write(fileData); LOGGER.info("New svg symbol downloaded: " + needCode); - } catch (Exception e) { + } catch (Exception e) { LOGGER.error("Can't decode svg icon and save to file: " + destFile.getPath() + ", reason: " + e.getMessage()); } finally { StreamUtils.closeQuietly(stream); @@ -127,23 +131,26 @@ public class ScryfallSymbolsSource implements Iterable { } - private class ScryfallSymbolsDownloadJob extends DownloadJob{ + private class ScryfallSymbolsDownloadJob extends DownloadJob { + + private String cssUrl = ""; // need to find url from colors page https://scryfall.com/docs/api/colors // listener for data parse after download complete - private class ScryDownloadOnFinishedListener implements PropertyChangeListener { + private class ScryfallDownloadOnFinishedListener implements PropertyChangeListener { private String downloadedFile; - public ScryDownloadOnFinishedListener(String ADestFile){ + public ScryfallDownloadOnFinishedListener(String ADestFile) { this.downloadedFile = ADestFile; } @Override public void propertyChange(PropertyChangeEvent evt) { - if (!evt.getPropertyName().equals(STATE_PROP_NAME)){ + + if (!evt.getPropertyName().equals(STATE_PROP_NAME)) { throw new IllegalArgumentException("Unknown download property " + evt.getPropertyName()); } - if (evt.getNewValue() != State.FINISHED){ + if (evt.getNewValue() != State.FINISHED) { return; } @@ -152,16 +159,34 @@ public class ScryfallSymbolsSource implements Iterable { } } + @Override + public void onPreparing() throws Exception { + this.cssUrl = ""; + + org.jsoup.nodes.Document doc = CardImageUtils.downloadHtmlDocument(CSS_SOURCE_URL); + org.jsoup.select.Elements cssList = doc.select(CSS_SOURCE_SELECTOR); + if (cssList.size() == 1) { + this.cssUrl = cssList.first().attr("href").toString(); + } + + if (this.cssUrl.isEmpty()) { + throw new IllegalStateException("Can't find stylesheet url from scryfall colors page."); + } else { + this.setSource(fromURL(this.cssUrl)); + } + } + private String destFile = ""; public ScryfallSymbolsDownloadJob() { - super("Scryfall symbols source", fromURL(SOURCE_URL), toFile(DOWNLOAD_TEMP_FILE)); + // download init + super("Scryfall symbols source", fromURL(""), toFile(DOWNLOAD_TEMP_FILE)); // url setup on preparing stage this.destFile = DOWNLOAD_TEMP_FILE; - this.addPropertyChangeListener(STATE_PROP_NAME, new ScryDownloadOnFinishedListener(this.destFile)); + this.addPropertyChangeListener(STATE_PROP_NAME, new ScryfallDownloadOnFinishedListener(this.destFile)); // clear dest file (always download new data) File file = new File(this.destFile); - if (file.exists()){ + if (file.exists()) { file.delete(); } } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java index dff6ed8304c..e4103359ba9 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/WizardCardsImageSource.java @@ -35,6 +35,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.mage.plugins.card.images.CardDownloadData; +import org.mage.plugins.card.utils.CardImageUtils; /** * @author North @@ -535,7 +536,7 @@ public enum WizardCardsImageSource implements CardImageSource { while (page < 999) { String searchUrl = "http://gatherer.wizards.com/Pages/Search/Default.aspx?sort=cn+&page=" + page + "&action=advanced&output=spoiler&method=visual&set=+%5B%22" + URLSetName + "%22%5D"; logger.debug("URL: " + searchUrl); - Document doc = getDocument(searchUrl); + Document doc = CardImageUtils.downloadHtmlDocument(searchUrl); Elements cardsImages = doc.select("img[src^=../../Handlers/]"); if (cardsImages.isEmpty()) { break; @@ -587,33 +588,6 @@ public enum WizardCardsImageSource implements CardImageSource { return setLinks; } - private Document getDocument(String urlString) throws NumberFormatException, IOException { - Preferences prefs = MageFrame.getPreferences(); - Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None")); - Document doc; - if (proxyType == ProxyType.NONE) { - doc = Jsoup.connect(urlString).timeout(60 * 1000).get(); - } else { - String proxyServer = prefs.get("proxyAddress", ""); - int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0")); - URL url = new URL(urlString); - Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort)); - HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy); - uc.setConnectTimeout(10000); - uc.setReadTimeout(60000); - uc.connect(); - - String line; - StringBuffer tmp = new StringBuffer(); - BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream())); - while ((line = in.readLine()) != null) { - tmp.append(line); - } - doc = Jsoup.parse(String.valueOf(tmp)); - } - return doc; - } - private void getLandVariations(LinkedHashMap setLinks, String cardSet, int multiverseId, String cardName) throws IOException, NumberFormatException { CardCriteria criteria = new CardCriteria(); criteria.nameExact(cardName); @@ -621,7 +595,7 @@ public enum WizardCardsImageSource implements CardImageSource { List cards = CardRepository.instance.findCards(criteria); String urlLandDocument = "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=" + multiverseId; - Document landDoc = getDocument(urlLandDocument); + Document landDoc = CardImageUtils.downloadHtmlDocument(urlLandDocument); Elements variations = landDoc.select("a.variationlink"); if (!variations.isEmpty()) { if (variations.size() > cards.size()) { @@ -668,7 +642,7 @@ public enum WizardCardsImageSource implements CardImageSource { private HashMap getlocalizedMultiverseIds(Integer englishMultiverseId) throws IOException { String cardLanguagesUrl = "http://gatherer.wizards.com/Pages/Card/Languages.aspx?multiverseid=" + englishMultiverseId; - Document cardLanguagesDoc = getDocument(cardLanguagesUrl); + Document cardLanguagesDoc = CardImageUtils.downloadHtmlDocument(cardLanguagesUrl); Elements languageTableRows = cardLanguagesDoc.select("tr.cardItem"); HashMap localizedIds = new HashMap<>(); if (!languageTableRows.isEmpty()) { diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java b/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java index 6b7c74587a8..fb7c42aa090 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/utils/CardImageUtils.java @@ -1,11 +1,17 @@ package org.mage.plugins.card.utils; +import java.io.BufferedReader; import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.Proxy; +import java.net.URL; import java.util.HashMap; import java.util.Locale; import java.util.prefs.Preferences; + import mage.client.MageFrame; import mage.client.constants.Constants; import mage.client.dialog.PreferencesDialog; @@ -13,6 +19,8 @@ import mage.remote.Connection; import mage.remote.Connection.ProxyType; import net.java.truevfs.access.TFile; import org.apache.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; import org.mage.plugins.card.images.CardDownloadData; import org.mage.plugins.card.properties.SettingsManager; @@ -22,7 +30,6 @@ public final class CardImageUtils { private static final Logger log = Logger.getLogger(CardImageUtils.class); /** - * * @param card * @return String if image exists, else null */ @@ -54,7 +61,6 @@ public final class CardImageUtils { } /** - * * @param card * @return String regardless of whether image exists */ @@ -280,4 +286,31 @@ public final class CardImageUtils { } return null; } + + public static Document downloadHtmlDocument(String urlString) throws NumberFormatException, IOException { + Preferences prefs = MageFrame.getPreferences(); + Connection.ProxyType proxyType = Connection.ProxyType.valueByText(prefs.get("proxyType", "None")); + Document doc; + if (proxyType == ProxyType.NONE) { + doc = Jsoup.connect(urlString).timeout(60 * 1000).get(); + } else { + String proxyServer = prefs.get("proxyAddress", ""); + int proxyPort = Integer.parseInt(prefs.get("proxyPort", "0")); + URL url = new URL(urlString); + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyServer, proxyPort)); + HttpURLConnection uc = (HttpURLConnection) url.openConnection(proxy); + uc.setConnectTimeout(10000); + uc.setReadTimeout(60000); + uc.connect(); + + String line; + StringBuffer tmp = new StringBuffer(); + BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream())); + while ((line = in.readLine()) != null) { + tmp.append(line); + } + doc = Jsoup.parse(String.valueOf(tmp)); + } + return doc; + } }