Znachor/Uloz.to-rychle
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

New downloader implementation

Browse source
This commit is contained in:
Filip Znachor 2023-02-09 00:01:59 +01:00
parent 6577ea99c6
commit 3a633d9794
1 changed files with 31 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
downloader.ts
View file

@ -6,60 +6,48 @@ export class Downloader {
pool: UrlPool;
chunks: {[U: string]: (Buffer | null)} = {};
chunk_size = Math.round(1*1024*1024);
cache_size = 15;
loading: boolean = false;
destroyed: boolean = false;
from: number;
start_position: number;
sent_position: number;
current_position: number;
constructor(pool: UrlPool, from: number) {
constructor(pool: UrlPool, start_position: number) {
this.pool = pool;
this.from = from;
}
first_chunk(): Buffer | undefined {
let keys = Object.keys(this.chunks);
let key = keys[0];
if(!key) return;
let first_chunk = this.chunks[key];
if(first_chunk instanceof Buffer) {
delete this.chunks[key];
this.from += this.chunk_size;
return first_chunk!;
}
this.start_position = start_position;
this.sent_position = start_position;
this.current_position = start_position;
console.log(pool.id, "| new downloader", start_position, "/", pool.total_size);
}
collect() {
let collected: Buffer[] = [];
let first_chunk = this.first_chunk();
while(first_chunk) {
collected.push(first_chunk);
first_chunk = this.first_chunk();
while(true) {
let id = this.sent_position.toString();
let c = this.chunks[id];
if(!c) break;
this.sent_position += c.length;
collected.push(c);
delete this.chunks[id];
}
if(collected.length) return Buffer.concat(collected);
return null;
}
cache() {
let chunks: string[] = [];
let existing_chunks = Object.keys(this.chunks);
if(this.current_position - this.sent_position > this.cache_size*1024*1024) return;
for(let i=0; i<this.cache_size; i++) {
chunks.push((this.from+(this.chunk_size*i)).toString());
this.download_next(Math.round(1*1024*1024));
}
chunks.forEach(from => {
if(existing_chunks.indexOf(from) == -1) {
this.download_part(parseInt(from), parseInt(from)+(this.chunk_size-1));
this.chunks[from] = null;
}
});
}
async more(): Promise<Buffer | null | false> {
if(this.loading) return false;
this.loading = true;
if(this.from > this.pool.total_size-1) return null;
if(this.sent_position > this.pool.total_size-1) return null;
this.cache();
let promise: Promise<Buffer | null> = new Promise((resolve) => {
@ -69,15 +57,24 @@ export class Downloader {
this.loading = false;
resolve(result);
clearInterval(interval);
if(result) console.log(this.pool.id, "| sending:", Math.round(result.length/1024), "kB");
if(result) console.log(this.pool.id, "| sending", Math.round(result.length/1024), "kB");
}
};
let interval = setInterval(wait_for_result, 2000);
let interval = setInterval(wait_for_result, 1000);
wait_for_result();
});
return promise;
}
async download_next(bytes: number) {
if(this.destroyed) return;
let to = this.current_position + bytes;
if(to > this.pool.total_size-1) to = this.pool.total_size-1;
let from = this.current_position;
this.current_position = to+1;
this.download_part(from, to);
}
async download_part(from: number, to: number) {
if(to > this.pool.total_size-1) to = this.pool.total_size-1;
@ -86,7 +83,7 @@ export class Downloader {
let url = await this.pool.get();
if(!url) throw "No available URL!";
if(this.destroyed == true) {
if(this.destroyed) {
this.pool.return(url[0]);
return;
}
@ -125,6 +122,7 @@ export class Downloader {
this.destroyed = true;
let index = this.pool.downloaders.indexOf(this);
this.pool.downloaders.splice(index, 1);
console.log(this.pool.id, "| destroyed downloader", this.start_position, "/", this.pool.total_size);
}
}