Trying to avoid rate-limits & auto-retry on fail
This commit is contained in:
parent
68c2800aec
commit
51d31d16fa
|
@ -6,7 +6,7 @@ export class Downloader {
|
||||||
pool: UrlPool;
|
pool: UrlPool;
|
||||||
chunks: {[U: string]: (Buffer | null)} = {};
|
chunks: {[U: string]: (Buffer | null)} = {};
|
||||||
|
|
||||||
chunk_size = Math.round(0.1*1024*1024);
|
chunk_size = Math.round(0.3*1024*1024);
|
||||||
|
|
||||||
loading: boolean = false;
|
loading: boolean = false;
|
||||||
destroyed: boolean = false;
|
destroyed: boolean = false;
|
||||||
|
@ -90,14 +90,32 @@ export class Downloader {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let r = await axios.get(url[1], {
|
let controller = new AbortController();
|
||||||
responseType: 'arraybuffer',
|
let tmo = setTimeout(() => {
|
||||||
headers: {
|
console.log(this.pool.id, "| took so long");
|
||||||
Range: `bytes=${from}-${to}`
|
controller.abort();
|
||||||
}
|
}, 10000);
|
||||||
});
|
|
||||||
this.chunks[from.toString()] = r.data;
|
|
||||||
|
|
||||||
|
try {
|
||||||
|
let r = await axios.get(url[1], {
|
||||||
|
responseType: 'arraybuffer',
|
||||||
|
headers: {
|
||||||
|
Range: `bytes=${from}-${to}`
|
||||||
|
},
|
||||||
|
signal: controller.signal
|
||||||
|
});
|
||||||
|
this.chunks[from.toString()] = r.data;
|
||||||
|
} catch(e) {
|
||||||
|
clearTimeout(tmo);
|
||||||
|
console.log(this.pool.id, "| failed, retrying");
|
||||||
|
setTimeout(() => {
|
||||||
|
this.download_part(from, to);
|
||||||
|
this.pool.return(url![0], false);
|
||||||
|
}, 2000);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
clearTimeout(tmo);
|
||||||
this.pool.return(url[0]);
|
this.pool.return(url[0]);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
1
index.ts
1
index.ts
|
@ -1,5 +1,6 @@
|
||||||
import { Webserver } from "./webserver";
|
import { Webserver } from "./webserver";
|
||||||
|
|
||||||
|
export let debug = true;
|
||||||
let webserver = new Webserver;
|
let webserver = new Webserver;
|
||||||
|
|
||||||
process.on('uncaughtException', err => {
|
process.on('uncaughtException', err => {
|
||||||
|
|
3
links.ts
3
links.ts
|
@ -1,5 +1,6 @@
|
||||||
import { exec } from "child_process";
|
import { exec } from "child_process";
|
||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
|
import { debug } from ".";
|
||||||
let tor_axios = require('tor-axios');
|
let tor_axios = require('tor-axios');
|
||||||
|
|
||||||
export class Links {
|
export class Links {
|
||||||
|
@ -57,8 +58,10 @@ export class Links {
|
||||||
let url = this.regex_parse(/<img class="xapca-image" src="([^"]*)" alt="">/gm, captcha_page.data, 1);
|
let url = this.regex_parse(/<img class="xapca-image" src="([^"]*)" alt="">/gm, captcha_page.data, 1);
|
||||||
if(!url) return undefined;
|
if(!url) return undefined;
|
||||||
let captcha_image = new URL(url, "https://localhost").href;
|
let captcha_image = new URL(url, "https://localhost").href;
|
||||||
|
if(debug) console.info(`${id} | captcha image: ${captcha_image}`);
|
||||||
|
|
||||||
let captcha = await this.captcha(captcha_image);
|
let captcha = await this.captcha(captcha_image);
|
||||||
|
if(debug) console.info(`${id} | captcha code: ${captcha}`);
|
||||||
|
|
||||||
let form = new URLSearchParams();
|
let form = new URLSearchParams();
|
||||||
|
|
||||||
|
|
3
tools.ts
Normal file
3
tools.ts
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
export function random_int(from: number, to: number): number {
|
||||||
|
return Math.floor((Math.random() * to+1-from) + from);
|
||||||
|
}
|
|
@ -1,6 +1,7 @@
|
||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
import { Downloader } from "./downloader";
|
import { Downloader } from "./downloader";
|
||||||
import { Links } from "./links";
|
import { Links } from "./links";
|
||||||
|
import { random_int } from "./tools";
|
||||||
|
|
||||||
let links = new Links;
|
let links = new Links;
|
||||||
|
|
||||||
|
@ -72,8 +73,8 @@ export class UrlPool {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return(i: number) {
|
return(i: number, sucess: boolean = true) {
|
||||||
this.used[i] = false;
|
setTimeout(() => this.used[i] = false, sucess ? random_int(1000, 5000) : random_int(5000, 60000));
|
||||||
}
|
}
|
||||||
|
|
||||||
add(url: string) {
|
add(url: string) {
|
||||||
|
|
Reference in a new issue