129 lines
2.9 KiB
TypeScript
129 lines
2.9 KiB
TypeScript
import axios from "axios";
|
|
import { UrlPool } from "./urlpool";
|
|
|
|
export class Downloader {
|
|
|
|
pool: UrlPool;
|
|
chunks: {[U: string]: (Buffer | null)} = {};
|
|
|
|
chunk_size = Math.round(0.3*1024*1024);
|
|
|
|
loading: boolean = false;
|
|
destroyed: boolean = false;
|
|
|
|
from: number;
|
|
|
|
constructor(pool: UrlPool, from: number) {
|
|
this.pool = pool;
|
|
this.from = from;
|
|
}
|
|
|
|
first_chunk(): Buffer | undefined {
|
|
let keys = Object.keys(this.chunks);
|
|
let key = keys[0];
|
|
if(!key) return;
|
|
let first_chunk = this.chunks[key];
|
|
if(first_chunk instanceof Buffer) {
|
|
delete this.chunks[key];
|
|
this.from += this.chunk_size;
|
|
return first_chunk!;
|
|
}
|
|
}
|
|
|
|
collect() {
|
|
let collected: Buffer[] = [];
|
|
let first_chunk = this.first_chunk();
|
|
while(first_chunk) {
|
|
collected.push(first_chunk);
|
|
first_chunk = this.first_chunk();
|
|
}
|
|
if(collected.length) return Buffer.concat(collected);
|
|
return null;
|
|
}
|
|
|
|
cache() {
|
|
let chunks: string[] = [];
|
|
let existing_chunks = Object.keys(this.chunks);
|
|
for(let i=0; i<15; i++) {
|
|
chunks.push((this.from+(this.chunk_size*i)).toString());
|
|
}
|
|
chunks.forEach(from => {
|
|
if(existing_chunks.indexOf(from) == -1) {
|
|
this.download_part(parseInt(from), parseInt(from)+(this.chunk_size-1));
|
|
this.chunks[from] = null;
|
|
}
|
|
});
|
|
}
|
|
|
|
async more(): Promise<Buffer | null | false> {
|
|
if(this.loading) return false;
|
|
this.loading = true;
|
|
if(this.from > this.pool.total_size-1) return null;
|
|
|
|
this.cache();
|
|
let promise: Promise<Buffer | null> = new Promise((resolve) => {
|
|
let wait_for_result = () => {
|
|
let result = this.collect();
|
|
if(result || this.destroyed) {
|
|
this.loading = false;
|
|
resolve(result);
|
|
clearInterval(interval);
|
|
if(result) console.log(this.pool.id, "| sending:", Math.round(result.length/1024), "kB");
|
|
}
|
|
};
|
|
let interval = setInterval(wait_for_result, 100);
|
|
wait_for_result();
|
|
});
|
|
return promise;
|
|
}
|
|
|
|
async download_part(from: number, to: number) {
|
|
|
|
if(to > this.pool.total_size-1) to = this.pool.total_size-1;
|
|
if(from > this.pool.total_size-1) return;
|
|
|
|
let url = await this.pool.get();
|
|
if(!url) throw "No available URL!";
|
|
|
|
if(this.destroyed == true) {
|
|
this.pool.return(url[0]);
|
|
return;
|
|
}
|
|
|
|
let controller = new AbortController();
|
|
let tmo = setTimeout(() => {
|
|
console.log(this.pool.id, "| took so long");
|
|
controller.abort();
|
|
}, 10000);
|
|
|
|
try {
|
|
let r = await axios.get(url[1], {
|
|
responseType: 'arraybuffer',
|
|
headers: {
|
|
Range: `bytes=${from}-${to}`
|
|
},
|
|
signal: controller.signal
|
|
});
|
|
this.chunks[from.toString()] = r.data;
|
|
} catch(e) {
|
|
clearTimeout(tmo);
|
|
console.log(this.pool.id, "| failed, retrying");
|
|
setTimeout(() => {
|
|
this.download_part(from, to);
|
|
this.pool.return(url![0], false);
|
|
}, 2000);
|
|
return;
|
|
}
|
|
|
|
clearTimeout(tmo);
|
|
this.pool.return(url[0]);
|
|
|
|
}
|
|
|
|
destroy() {
|
|
this.destroyed = true;
|
|
let index = this.pool.downloaders.indexOf(this);
|
|
this.pool.downloaders.splice(index, 1);
|
|
}
|
|
|
|
} |