download: improved images download from scryfall source:

- fixed wrong timeouts for api (fixed 429 Too Many Requests);
- added any/10 parallel threads support;
- current api limit is 1 request per 300 ms;
This commit is contained in:
Oleg Agafonov 2024-08-01 11:46:37 +04:00
parent b5527a28db
commit bfa7c4c9a8
5 changed files with 19 additions and 10 deletions

View file

@ -331,7 +331,7 @@ public class AbilityPicker extends JXPanel implements MouseWheelListener {
wait(); wait();
} }
} }
} catch (InterruptedException ignored) { } catch (InterruptedException ignore) {
} }
} }

View file

@ -68,7 +68,7 @@ public final class Util {
public static void sleep(int millis) { public static void sleep(int millis) {
try { try {
TimeUnit.MILLISECONDS.sleep(millis); TimeUnit.MILLISECONDS.sleep(millis);
} catch (InterruptedException ignored) { } catch (InterruptedException ignore) {
} }
} }

View file

@ -13,8 +13,8 @@ import org.mage.plugins.card.images.CardDownloadData;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.net.Proxy;
import java.util.*; import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
/** /**
* @author JayDi85 * @author JayDi85
@ -28,7 +28,9 @@ public class ScryfallImageSource implements CardImageSource {
private final Map<CardLanguage, String> languageAliases; private final Map<CardLanguage, String> languageAliases;
private CardLanguage currentLanguage = CardLanguage.ENGLISH; // working language private CardLanguage currentLanguage = CardLanguage.ENGLISH; // working language
private final Map<CardDownloadData, String> preparedUrls = new HashMap<>(); private final Map<CardDownloadData, String> preparedUrls = new HashMap<>();
private static final int DOWNLOAD_TIMEOUT_MS = 100;
private static final ReentrantLock waitBeforeRequestLock = new ReentrantLock();
private static final int DOWNLOAD_TIMEOUT_MS = 300;
public static ScryfallImageSource getInstance() { public static ScryfallImageSource getInstance() {
return instance; return instance;
@ -330,15 +332,22 @@ public class ScryfallImageSource implements CardImageSource {
@Override @Override
public void doPause(String fullUrl) { public void doPause(String fullUrl) {
// scryfall recommends 300 ms timeout per each request to API to work under a rate limit
// possible error: 429 Too Many Requests
// TODO: add diff endpoint supports (api calls with timeout, cdn/direct calls without timeout)
waitBeforeRequest(); waitBeforeRequest();
} }
private void waitBeforeRequest() { private void waitBeforeRequest() {
// scryfall recommends 50-100 ms timeout per each request to API to work under a rate limit try {
// possible error: 429 Too Many Requests // single wait queue for all threads - it's guarantee min timeout before each api call
waitBeforeRequestLock.lock();
try { try {
Thread.sleep(DOWNLOAD_TIMEOUT_MS); Thread.sleep(DOWNLOAD_TIMEOUT_MS);
} catch (InterruptedException ignored) { } finally {
waitBeforeRequestLock.unlock();
}
} catch (InterruptedException ignore) {
} }
} }

View file

@ -52,7 +52,7 @@ public class CardLookup {
if (result.isPresent()) { if (result.isPresent()) {
return result; return result;
} }
} catch (NumberFormatException ignored) { } catch (NumberFormatException ignore) {
/* ignored */ /* ignored */
} }

View file

@ -51,7 +51,7 @@ public final class ThreadUtils {
public static void sleep(int millis) { public static void sleep(int millis) {
try { try {
Thread.sleep(millis); Thread.sleep(millis);
} catch (InterruptedException ignored) { } catch (InterruptedException ignore) {
} }
} }