diff --git a/downloader.ts b/downloader.ts index e1834a5..642f676 100644 --- a/downloader.ts +++ b/downloader.ts @@ -6,7 +6,7 @@ export class Downloader { pool: UrlPool; chunks: {[U: string]: (Buffer | null)} = {}; - chunk_size = Math.round(0.1*1024*1024); + chunk_size = Math.round(0.3*1024*1024); loading: boolean = false; destroyed: boolean = false; @@ -90,14 +90,32 @@ export class Downloader { return; } - let r = await axios.get(url[1], { - responseType: 'arraybuffer', - headers: { - Range: `bytes=${from}-${to}` - } - }); - this.chunks[from.toString()] = r.data; + let controller = new AbortController(); + let tmo = setTimeout(() => { + console.log(this.pool.id, "| took so long"); + controller.abort(); + }, 10000); + try { + let r = await axios.get(url[1], { + responseType: 'arraybuffer', + headers: { + Range: `bytes=${from}-${to}` + }, + signal: controller.signal + }); + this.chunks[from.toString()] = r.data; + } catch(e) { + clearTimeout(tmo); + console.log(this.pool.id, "| failed, retrying"); + setTimeout(() => { + this.download_part(from, to); + this.pool.return(url![0], false); + }, 2000); + return; + } + + clearTimeout(tmo); this.pool.return(url[0]); } diff --git a/index.ts b/index.ts index 0ada8b4..6c0c8d2 100644 --- a/index.ts +++ b/index.ts @@ -1,5 +1,6 @@ import { Webserver } from "./webserver"; +export let debug = true; let webserver = new Webserver; process.on('uncaughtException', err => { diff --git a/links.ts b/links.ts index c7b4e5e..1fd7780 100644 --- a/links.ts +++ b/links.ts @@ -1,5 +1,6 @@ import { exec } from "child_process"; import axios from "axios"; +import { debug } from "."; let tor_axios = require('tor-axios'); export class Links { @@ -57,8 +58,10 @@ export class Links { let url = this.regex_parse(//gm, captcha_page.data, 1); if(!url) return undefined; let captcha_image = new URL(url, "https://localhost").href; + if(debug) console.info(`${id} | captcha image: ${captcha_image}`); let captcha = await this.captcha(captcha_image); + if(debug) console.info(`${id} | captcha code: ${captcha}`); let form = new URLSearchParams(); diff --git a/tools.ts b/tools.ts new file mode 100644 index 0000000..d01bfcb --- /dev/null +++ b/tools.ts @@ -0,0 +1,3 @@ +export function random_int(from: number, to: number): number { + return Math.floor((Math.random() * to+1-from) + from); +} \ No newline at end of file diff --git a/urlpool.ts b/urlpool.ts index 2614ef3..464a92a 100644 --- a/urlpool.ts +++ b/urlpool.ts @@ -1,6 +1,7 @@ import axios from "axios"; import { Downloader } from "./downloader"; import { Links } from "./links"; +import { random_int } from "./tools"; let links = new Links; @@ -72,8 +73,8 @@ export class UrlPool { }); } - return(i: number) { - this.used[i] = false; + return(i: number, sucess: boolean = true) { + setTimeout(() => this.used[i] = false, sucess ? random_int(1000, 5000) : random_int(5000, 60000)); } add(url: string) {