import axios from "axios"; import { UrlPool } from "./urlpool"; export class Downloader { pool: UrlPool; chunks: {[U: string]: (Buffer | null)} = {}; chunk_size = Math.round(0.3*1024*1024); loading: boolean = false; destroyed: boolean = false; from: number; constructor(pool: UrlPool, from: number) { this.pool = pool; this.from = from; } first_chunk(): Buffer | undefined { let keys = Object.keys(this.chunks); let key = keys[0]; if(!key) return; let first_chunk = this.chunks[key]; if(first_chunk instanceof Buffer) { delete this.chunks[key]; this.from += this.chunk_size; return first_chunk!; } } collect() { let collected: Buffer[] = []; let first_chunk = this.first_chunk(); while(first_chunk) { collected.push(first_chunk); first_chunk = this.first_chunk(); } if(collected.length) return Buffer.concat(collected); return null; } cache() { let chunks: string[] = []; let existing_chunks = Object.keys(this.chunks); for(let i=0; i<15; i++) { chunks.push((this.from+(this.chunk_size*i)).toString()); } chunks.forEach(from => { if(existing_chunks.indexOf(from) == -1) { this.download_part(parseInt(from), parseInt(from)+(this.chunk_size-1)); this.chunks[from] = null; } }); } async more(): Promise { if(this.loading) return false; this.loading = true; if(this.from > this.pool.total_size-1) return null; this.cache(); let promise: Promise = new Promise((resolve) => { let wait_for_result = () => { let result = this.collect(); if(result || this.destroyed) { this.loading = false; resolve(result); clearInterval(interval); if(result) console.log(this.pool.id, "| sending:", Math.round(result.length/1024), "kB"); } }; let interval = setInterval(wait_for_result, 100); wait_for_result(); }); return promise; } async download_part(from: number, to: number) { if(to > this.pool.total_size-1) to = this.pool.total_size-1; if(from > this.pool.total_size-1) return; let url = await this.pool.get(); if(!url) throw "No available URL!"; if(this.destroyed == true) { this.pool.return(url[0]); return; } let controller = new AbortController(); let tmo = setTimeout(() => { console.log(this.pool.id, "| took so long"); controller.abort(); }, 10000); try { let r = await axios.get(url[1], { responseType: 'arraybuffer', headers: { Range: `bytes=${from}-${to}` }, signal: controller.signal }); this.chunks[from.toString()] = r.data; } catch(e) { clearTimeout(tmo); console.log(this.pool.id, "| failed, retrying"); setTimeout(() => { this.download_part(from, to); this.pool.return(url![0], false); }, 2000); return; } clearTimeout(tmo); this.pool.return(url[0]); } destroy() { this.destroyed = true; let index = this.pool.downloaders.indexOf(this); this.pool.downloaders.splice(index, 1); } }