From bfa7c4c9a8f3fbff8482dd336309481f64359dad Mon Sep 17 00:00:00 2001 From: Oleg Agafonov Date: Thu, 1 Aug 2024 11:46:37 +0400 Subject: [PATCH] download: improved images download from scryfall source: - fixed wrong timeouts for api (fixed 429 Too Many Requests); - added any/10 parallel threads support; - current api limit is 1 request per 300 ms; --- .../components/ability/AbilityPicker.java | 2 +- .../main/java/org/mage/card/arcane/Util.java | 2 +- .../card/dl/sources/ScryfallImageSource.java | 21 +++++++++++++------ .../mage/cards/decks/importer/CardLookup.java | 2 +- Mage/src/main/java/mage/util/ThreadUtils.java | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Mage.Client/src/main/java/mage/client/components/ability/AbilityPicker.java b/Mage.Client/src/main/java/mage/client/components/ability/AbilityPicker.java index 81fd1e03b87..66ddd3135d9 100644 --- a/Mage.Client/src/main/java/mage/client/components/ability/AbilityPicker.java +++ b/Mage.Client/src/main/java/mage/client/components/ability/AbilityPicker.java @@ -331,7 +331,7 @@ public class AbilityPicker extends JXPanel implements MouseWheelListener { wait(); } } - } catch (InterruptedException ignored) { + } catch (InterruptedException ignore) { } } diff --git a/Mage.Client/src/main/java/org/mage/card/arcane/Util.java b/Mage.Client/src/main/java/org/mage/card/arcane/Util.java index ee057202355..37b863e4d06 100644 --- a/Mage.Client/src/main/java/org/mage/card/arcane/Util.java +++ b/Mage.Client/src/main/java/org/mage/card/arcane/Util.java @@ -68,7 +68,7 @@ public final class Util { public static void sleep(int millis) { try { TimeUnit.MILLISECONDS.sleep(millis); - } catch (InterruptedException ignored) { + } catch (InterruptedException ignore) { } } diff --git a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java index abcc7f9001f..c562513d2ae 100644 --- a/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java +++ b/Mage.Client/src/main/java/org/mage/plugins/card/dl/sources/ScryfallImageSource.java @@ -13,8 +13,8 @@ import org.mage.plugins.card.images.CardDownloadData; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.Proxy; import java.util.*; +import java.util.concurrent.locks.ReentrantLock; /** * @author JayDi85 @@ -28,7 +28,9 @@ public class ScryfallImageSource implements CardImageSource { private final Map languageAliases; private CardLanguage currentLanguage = CardLanguage.ENGLISH; // working language private final Map preparedUrls = new HashMap<>(); - private static final int DOWNLOAD_TIMEOUT_MS = 100; + + private static final ReentrantLock waitBeforeRequestLock = new ReentrantLock(); + private static final int DOWNLOAD_TIMEOUT_MS = 300; public static ScryfallImageSource getInstance() { return instance; @@ -330,15 +332,22 @@ public class ScryfallImageSource implements CardImageSource { @Override public void doPause(String fullUrl) { + // scryfall recommends 300 ms timeout per each request to API to work under a rate limit + // possible error: 429 Too Many Requests + // TODO: add diff endpoint supports (api calls with timeout, cdn/direct calls without timeout) waitBeforeRequest(); } private void waitBeforeRequest() { - // scryfall recommends 50-100 ms timeout per each request to API to work under a rate limit - // possible error: 429 Too Many Requests try { - Thread.sleep(DOWNLOAD_TIMEOUT_MS); - } catch (InterruptedException ignored) { + // single wait queue for all threads - it's guarantee min timeout before each api call + waitBeforeRequestLock.lock(); + try { + Thread.sleep(DOWNLOAD_TIMEOUT_MS); + } finally { + waitBeforeRequestLock.unlock(); + } + } catch (InterruptedException ignore) { } } diff --git a/Mage/src/main/java/mage/cards/decks/importer/CardLookup.java b/Mage/src/main/java/mage/cards/decks/importer/CardLookup.java index e67c672b9af..f4ae4e9750a 100644 --- a/Mage/src/main/java/mage/cards/decks/importer/CardLookup.java +++ b/Mage/src/main/java/mage/cards/decks/importer/CardLookup.java @@ -52,7 +52,7 @@ public class CardLookup { if (result.isPresent()) { return result; } - } catch (NumberFormatException ignored) { + } catch (NumberFormatException ignore) { /* ignored */ } diff --git a/Mage/src/main/java/mage/util/ThreadUtils.java b/Mage/src/main/java/mage/util/ThreadUtils.java index 42afe91bbb5..86d4f7a68d6 100644 --- a/Mage/src/main/java/mage/util/ThreadUtils.java +++ b/Mage/src/main/java/mage/util/ThreadUtils.java @@ -51,7 +51,7 @@ public final class ThreadUtils { public static void sleep(int millis) { try { Thread.sleep(millis); - } catch (InterruptedException ignored) { + } catch (InterruptedException ignore) { } }