forked from External/mage
download: reworked scryfall images support:
- download: fixed unmount zip errors on cancel download in some use cases (closes #12536); - download: significant download speed improvements (now it depends on user's network speed, not api limitations); - download: added additional error dialogs on bad use cases; - scryfall: added cards and bulk data api support; - scryfall: added bulk data download (updates once per week, contains all scryfall cards and store in images\downloading folder, 2 GB size); - scryfall: added optimized images download without api usage (use direct images links from bulk data, closes #11576); - scryfall: improved image source searching for some use cases (miss or wrong images problems, closes #12511); - scryfall: tokens don't use bulk data; - scryfall: 75k small images downloads 40 minutes and takes 1 GB and 2100 api calls (most of it from tokens); - scryfall: how-to disable bulk data, e.g. for api testing: -Dxmage.scryfallEnableBulkData=false
This commit is contained in:
parent
46f7304692
commit
0a55e37c8c
19 changed files with 884 additions and 216 deletions
|
|
@ -52,6 +52,7 @@ public class XmageURLConnection {
|
|||
Proxy proxy = null;
|
||||
HttpURLConnection connection = null;
|
||||
HttpLoggingType loggingType = HttpLoggingType.ERRORS;
|
||||
boolean forceGZipEncoding = false;
|
||||
|
||||
public XmageURLConnection(String url) {
|
||||
this.url = url;
|
||||
|
|
@ -75,6 +76,10 @@ public class XmageURLConnection {
|
|||
}
|
||||
}
|
||||
|
||||
public void setForceGZipEncoding(boolean enable) {
|
||||
this.forceGZipEncoding = enable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to server
|
||||
*/
|
||||
|
|
@ -130,7 +135,11 @@ public class XmageURLConnection {
|
|||
}
|
||||
|
||||
private void initDefaultHeaders() {
|
||||
// warning, do not add Accept-Encoding - it processing inside URLConnection for http/https links (trying to use gzip by default)
|
||||
// warning, Accept-Encoding processing inside URLConnection for http/https links (trying to use gzip by default)
|
||||
// use force encoding for special use cases (example: download big text file as zip file)
|
||||
if (forceGZipEncoding) {
|
||||
this.connection.setRequestProperty("Accept-Encoding", "gzip");
|
||||
}
|
||||
|
||||
// user agent due standard notation User-Agent: <product> / <product-version> <comment>
|
||||
// warning, dot not add os, language and other details
|
||||
|
|
@ -290,13 +299,18 @@ public class XmageURLConnection {
|
|||
return "";
|
||||
}
|
||||
|
||||
public static InputStream downloadBinary(String resourceUrl) {
|
||||
return downloadBinary(resourceUrl, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fast download of binary data
|
||||
*
|
||||
* @return stream on OK 200 response or null on any other errors
|
||||
*/
|
||||
public static InputStream downloadBinary(String resourceUrl) {
|
||||
public static InputStream downloadBinary(String resourceUrl, boolean downloadAsGZip) {
|
||||
XmageURLConnection con = new XmageURLConnection(resourceUrl);
|
||||
con.setForceGZipEncoding(downloadAsGZip);
|
||||
con.startConnection();
|
||||
if (con.isConnected()) {
|
||||
try {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue