Trying to avoid rate-limits & auto-retry on fail

downloader.ts

@@ -6,7 +6,7 @@ export class Downloader {
 	pool: UrlPool;
 	chunks: {[U: string]: (Buffer | null)} = {};
 
-	chunk_size = Math.round(0.1*1024*1024);
+	chunk_size = Math.round(0.3*1024*1024);
 
 	loading: boolean = false;
 	destroyed: boolean = false;
@@ -90,14 +90,32 @@ export class Downloader {
 			return;
 		}
 
-		let r = await axios.get(url[1], {
-			responseType: 'arraybuffer',
-			headers: {
-				Range: `bytes=${from}-${to}`
-			}
-		});
-		this.chunks[from.toString()] = r.data;
+		let controller = new AbortController();
+		let tmo = setTimeout(() => {
+			console.log(this.pool.id, "| took so long");
+			controller.abort();
+		}, 10000);
 
+		try {
+			let r = await axios.get(url[1], {
+				responseType: 'arraybuffer',
+				headers: {
+					Range: `bytes=${from}-${to}`
+				},
+				signal: controller.signal
+			});
+			this.chunks[from.toString()] = r.data;
+		} catch(e) {
+			clearTimeout(tmo);
+			console.log(this.pool.id, "| failed, retrying");
+			setTimeout(() => {
+				this.download_part(from, to);
+				this.pool.return(url![0], false);
+			}, 2000);
+			return;
+		}
+
+		clearTimeout(tmo);
 		this.pool.return(url[0]);
 
 	}

index.ts

@@ -1,5 +1,6 @@
 import { Webserver } from "./webserver";
 
+export let debug = true;
 let webserver = new Webserver;
 
 process.on('uncaughtException', err => {

links.ts

@@ -1,5 +1,6 @@
 import { exec } from "child_process";
 import axios from "axios";
+import { debug } from ".";
 let tor_axios = require('tor-axios');
 
 export class Links {
@@ -57,8 +58,10 @@ export class Links {
 		let url = this.regex_parse(/<img class="xapca-image" src="([^"]*)" alt="">/gm, captcha_page.data, 1);
 		if(!url) return undefined;
 		let captcha_image = new URL(url, "https://localhost").href;
+		if(debug) console.info(`${id} | captcha image: ${captcha_image}`);
 
 		let captcha = await this.captcha(captcha_image);
+		if(debug) console.info(`${id} | captcha code: ${captcha}`);
 
 		let form = new URLSearchParams();
 

tools.ts

@@ -0,0 +1,3 @@
+export function random_int(from: number, to: number): number {
+	return Math.floor((Math.random() * to+1-from) + from);
+}

urlpool.ts

@@ -1,6 +1,7 @@
 import axios from "axios";
 import { Downloader } from "./downloader";
 import { Links } from "./links";
+import { random_int } from "./tools";
 
 let links = new Links;
 
@@ -72,8 +73,8 @@ export class UrlPool {
 		});
 	}
 
-	return(i: number) {
-		this.used[i] = false;
+	return(i: number, sucess: boolean = true) {
+		setTimeout(() => this.used[i] = false, sucess ? random_int(1000, 5000) : random_int(5000, 60000));
 	}
 
 	add(url: string) {