Znachor/Uloz.to-rychle
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

New downloader implementation

Browse source
This commit is contained in:
Filip Znachor 2023-02-09 00:01:59 +01:00
parent 6577ea99c6
commit 3a633d9794
1 changed files with 31 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
downloader.ts
View file

@ -6,60 +6,48 @@ export class Downloader {
pool: UrlPool; pool: UrlPool;
chunks: {[U: string]: (Buffer | null)} = {}; chunks: {[U: string]: (Buffer | null)} = {};
chunk_size = Math.round(1*1024*1024);
cache_size = 15; cache_size = 15;
loading: boolean = false; loading: boolean = false;
destroyed: boolean = false; destroyed: boolean = false;
from: number; start_position: number;
sent_position: number;
current_position: number;
constructor(pool: UrlPool, from: number) { constructor(pool: UrlPool, start_position: number) {
this.pool = pool; this.pool = pool;
this.from = from; this.start_position = start_position;
} this.sent_position = start_position;
this.current_position = start_position;
first_chunk(): Buffer | undefined { console.log(pool.id, "| new downloader", start_position, "/", pool.total_size);
let keys = Object.keys(this.chunks);
let key = keys[0];
if(!key) return;
let first_chunk = this.chunks[key];
if(first_chunk instanceof Buffer) {
delete this.chunks[key];
this.from += this.chunk_size;
return first_chunk!;
}
} }
collect() { collect() {
let collected: Buffer[] = []; let collected: Buffer[] = [];
let first_chunk = this.first_chunk(); while(true) {
while(first_chunk) { let id = this.sent_position.toString();
collected.push(first_chunk); let c = this.chunks[id];
first_chunk = this.first_chunk(); if(!c) break;
this.sent_position += c.length;
collected.push(c);
delete this.chunks[id];
} }
if(collected.length) return Buffer.concat(collected); if(collected.length) return Buffer.concat(collected);
return null; return null;
} }
cache() { cache() {
let chunks: string[] = []; if(this.current_position - this.sent_position > this.cache_size*1024*1024) return;
let existing_chunks = Object.keys(this.chunks);
for(let i=0; i<this.cache_size; i++) { for(let i=0; i<this.cache_size; i++) {
chunks.push((this.from+(this.chunk_size*i)).toString()); this.download_next(Math.round(1*1024*1024));
} }
chunks.forEach(from => {
if(existing_chunks.indexOf(from) == -1) {
this.download_part(parseInt(from), parseInt(from)+(this.chunk_size-1));
this.chunks[from] = null;
}
});
} }
async more(): Promise<Buffer | null | false> { async more(): Promise<Buffer | null | false> {
if(this.loading) return false; if(this.loading) return false;
this.loading = true; this.loading = true;
if(this.from > this.pool.total_size-1) return null; if(this.sent_position > this.pool.total_size-1) return null;
this.cache(); this.cache();
let promise: Promise<Buffer | null> = new Promise((resolve) => { let promise: Promise<Buffer | null> = new Promise((resolve) => {
@ -69,15 +57,24 @@ export class Downloader {
this.loading = false; this.loading = false;
resolve(result); resolve(result);
clearInterval(interval); clearInterval(interval);
if(result) console.log(this.pool.id, "| sending:", Math.round(result.length/1024), "kB"); if(result) console.log(this.pool.id, "| sending", Math.round(result.length/1024), "kB");
} }
}; };
let interval = setInterval(wait_for_result, 2000); let interval = setInterval(wait_for_result, 1000);
wait_for_result(); wait_for_result();
}); });
return promise; return promise;
} }
async download_next(bytes: number) {
if(this.destroyed) return;
let to = this.current_position + bytes;
if(to > this.pool.total_size-1) to = this.pool.total_size-1;
let from = this.current_position;
this.current_position = to+1;
this.download_part(from, to);
}
async download_part(from: number, to: number) { async download_part(from: number, to: number) {
if(to > this.pool.total_size-1) to = this.pool.total_size-1; if(to > this.pool.total_size-1) to = this.pool.total_size-1;
@ -86,7 +83,7 @@ export class Downloader {
let url = await this.pool.get(); let url = await this.pool.get();
if(!url) throw "No available URL!"; if(!url) throw "No available URL!";
if(this.destroyed == true) { if(this.destroyed) {
this.pool.return(url[0]); this.pool.return(url[0]);
return; return;
} }
@ -125,6 +122,7 @@ export class Downloader {
this.destroyed = true; this.destroyed = true;
let index = this.pool.downloaders.indexOf(this); let index = this.pool.downloaders.indexOf(this);
this.pool.downloaders.splice(index, 1); this.pool.downloaders.splice(index, 1);
console.log(this.pool.id, "| destroyed downloader", this.start_position, "/", this.pool.total_size);
} }
} }