download: reworked scryfall images support:

- download: fixed unmount zip errors on cancel download in some use cases (closes #12536);
- download: significant download speed improvements (now it depends on user's network speed, not api limitations);
- download: added additional error dialogs on bad use cases;
- scryfall: added cards and bulk data api support;
- scryfall: added bulk data download (updates once per week, contains all scryfall cards and store in images\downloading folder, 2 GB size);
- scryfall: added optimized images download without api usage (use direct images links from bulk data, closes #11576);
- scryfall: improved image source searching for some use cases (miss or wrong images problems, closes #12511);
- scryfall: tokens don't use bulk data;
- scryfall: 75k small images downloads 40 minutes and takes 1 GB and 2100 api calls (most of it from tokens);
- scryfall: how-to disable bulk data, e.g. for api testing: -Dxmage.scryfallEnableBulkData=false
This commit is contained in:
Oleg Agafonov 2024-08-03 19:41:14 +04:00
parent 46f7304692
commit 0a55e37c8c
19 changed files with 884 additions and 216 deletions

View file

@ -52,6 +52,7 @@ public class XmageURLConnection {
Proxy proxy = null;
HttpURLConnection connection = null;
HttpLoggingType loggingType = HttpLoggingType.ERRORS;
boolean forceGZipEncoding = false;
public XmageURLConnection(String url) {
this.url = url;
@ -75,6 +76,10 @@ public class XmageURLConnection {
}
}
public void setForceGZipEncoding(boolean enable) {
this.forceGZipEncoding = enable;
}
/**
* Connect to server
*/
@ -130,7 +135,11 @@ public class XmageURLConnection {
}
private void initDefaultHeaders() {
// warning, do not add Accept-Encoding - it processing inside URLConnection for http/https links (trying to use gzip by default)
// warning, Accept-Encoding processing inside URLConnection for http/https links (trying to use gzip by default)
// use force encoding for special use cases (example: download big text file as zip file)
if (forceGZipEncoding) {
this.connection.setRequestProperty("Accept-Encoding", "gzip");
}
// user agent due standard notation User-Agent: <product> / <product-version> <comment>
// warning, dot not add os, language and other details
@ -290,13 +299,18 @@ public class XmageURLConnection {
return "";
}
public static InputStream downloadBinary(String resourceUrl) {
return downloadBinary(resourceUrl, false);
}
/**
* Fast download of binary data
*
* @return stream on OK 200 response or null on any other errors
*/
public static InputStream downloadBinary(String resourceUrl) {
public static InputStream downloadBinary(String resourceUrl, boolean downloadAsGZip) {
XmageURLConnection con = new XmageURLConnection(resourceUrl);
con.setForceGZipEncoding(downloadAsGZip);
con.startConnection();
if (con.isConnected()) {
try {