From b375245bbef6a6d4d4947eb7250252c2a8c644e3 Mon Sep 17 00:00:00 2001 From: Geoff Lamperd Date: Mon, 29 Jan 2024 14:59:02 +1000 Subject: [PATCH 1/5] adds large file test --- test/osfile.test.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/osfile.test.js b/test/osfile.test.js index 855d644..11d3eb7 100644 --- a/test/osfile.test.js +++ b/test/osfile.test.js @@ -49,6 +49,16 @@ describe("fastfile testing suite for osfile", function () { expect(fd.readString(0)).to.be.rejectedWith("Reading a closing file"); await fs.promises.unlink(fileName); }); + + it("should read a large file", async () => { + const fileName = "//wsl.localhost/Ubuntu/home/geoff/ptau/pot10_0006_bcn_prep.ptau"; + const ff = await fastFile.readExisting(fileName, 1024, 1024); + assert(ff.totalSize>0); + + let buff = new Uint8Array(1024); + await ff.readToBuffer(buff, 0, 1024); + assert.equal(ff.pos, 1024); + }); }); From 5474e7aeadc2c9128a4f03cfb840dc7bb60663cc Mon Sep 17 00:00:00 2001 From: Geoff Lamperd Date: Mon, 29 Jan 2024 17:02:21 +1000 Subject: [PATCH 2/5] wrap promises array in a new array --- build/main.cjs | 16 +++++++++++++--- src/osfile.js | 16 +++++++++++++--- test/osfile.test.js | 7 ++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/build/main.cjs b/build/main.cjs index 34aa4e5..edde983 100644 --- a/build/main.cjs +++ b/build/main.cjs @@ -39,6 +39,7 @@ class FastFile { this.reading = false; this.avBuffs = []; this.history = {}; + this.maxPromises = 1 << 3; } _loadPage(p) { @@ -114,8 +115,8 @@ class FastFile { while ( (self.pendingLoads.length>0) && ( (typeof self.pages[self.pendingLoads[0].page] != "undefined" ) - ||( (freePages>0) - ||(deletablePages.length>0)))) { + ||( (freePages>0) + ||(deletablePages.length>0)))) { const load = self.pendingLoads.shift(); if (typeof self.pages[load.page] != "undefined") { self.pages[load.page].pendingOps ++; @@ -165,7 +166,16 @@ class FastFile { } // if (ops.length>1) console.log(ops.length); - Promise.all(ops).then( () => { + // This attempts to avoid 'too many promises for Promise.all' error + const opsWrapper = []; + for (let from=0; from ops.length) to = ops.length; + const opsSlice = ops.slice(from, to); + opsWrapper.push(Promise.all(opsSlice)); + } + + Promise.all(opsWrapper).then( () => { self.reading = false; if (self.pendingLoads.length>0) setImmediate(self._triggerLoad.bind(self)); self._tryClose(); diff --git a/src/osfile.js b/src/osfile.js index fcd8445..fb73dfa 100644 --- a/src/osfile.js +++ b/src/osfile.js @@ -32,6 +32,7 @@ class FastFile { this.reading = false; this.avBuffs = []; this.history = {}; + this.maxPromises = 1 << 3; } _loadPage(p) { @@ -107,8 +108,8 @@ class FastFile { while ( (self.pendingLoads.length>0) && ( (typeof self.pages[self.pendingLoads[0].page] != "undefined" ) - ||( (freePages>0) - ||(deletablePages.length>0)))) { + ||( (freePages>0) + ||(deletablePages.length>0)))) { const load = self.pendingLoads.shift(); if (typeof self.pages[load.page] != "undefined") { self.pages[load.page].pendingOps ++; @@ -158,7 +159,16 @@ class FastFile { } // if (ops.length>1) console.log(ops.length); - Promise.all(ops).then( () => { + // This attempts to avoid 'too many promises for Promise.all' error + const opsWrapper = []; + for (let from=0; from ops.length) to = ops.length; + const opsSlice = ops.slice(from, to); + opsWrapper.push(Promise.all(opsSlice)); + } + + Promise.all(opsWrapper).then( () => { self.reading = false; if (self.pendingLoads.length>0) setImmediate(self._triggerLoad.bind(self)); self._tryClose(); diff --git a/test/osfile.test.js b/test/osfile.test.js index 11d3eb7..6da8552 100644 --- a/test/osfile.test.js +++ b/test/osfile.test.js @@ -52,12 +52,13 @@ describe("fastfile testing suite for osfile", function () { it("should read a large file", async () => { const fileName = "//wsl.localhost/Ubuntu/home/geoff/ptau/pot10_0006_bcn_prep.ptau"; + const bytesToRead = 1<<16; const ff = await fastFile.readExisting(fileName, 1024, 1024); assert(ff.totalSize>0); - let buff = new Uint8Array(1024); - await ff.readToBuffer(buff, 0, 1024); - assert.equal(ff.pos, 1024); + let buff = new Uint8Array(bytesToRead); + await ff.readToBuffer(buff, 0, bytesToRead); + assert.equal(ff.pos, bytesToRead); }); }); From 8c02d5516967c448057536a310a1c4d706a7bad3 Mon Sep 17 00:00:00 2001 From: Geoff Lamperd Date: Thu, 4 Apr 2024 14:42:07 +1000 Subject: [PATCH 3/5] adjusts max promises array length --- src/osfile.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/osfile.js b/src/osfile.js index fb73dfa..830ae30 100644 --- a/src/osfile.js +++ b/src/osfile.js @@ -32,7 +32,7 @@ class FastFile { this.reading = false; this.avBuffs = []; this.history = {}; - this.maxPromises = 1 << 3; + this.maxPromises = 1 << 10; } _loadPage(p) { @@ -159,11 +159,10 @@ class FastFile { } // if (ops.length>1) console.log(ops.length); - // This attempts to avoid 'too many promises for Promise.all' error + // This avoids 'too many promises for Promise.all' error const opsWrapper = []; for (let from=0; from ops.length) to = ops.length; + let to = Math.min(from+this.maxPromises, ops.length); const opsSlice = ops.slice(from, to); opsWrapper.push(Promise.all(opsSlice)); } From 7ce2cb3ba3ba513422bfec0047e876c6eb12b7af Mon Sep 17 00:00:00 2001 From: Geoff Lamperd Date: Thu, 4 Apr 2024 15:06:42 +1000 Subject: [PATCH 4/5] build --- build/main.cjs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build/main.cjs b/build/main.cjs index edde983..97b93b2 100644 --- a/build/main.cjs +++ b/build/main.cjs @@ -39,7 +39,7 @@ class FastFile { this.reading = false; this.avBuffs = []; this.history = {}; - this.maxPromises = 1 << 3; + this.maxPromises = 1 << 10; } _loadPage(p) { @@ -166,11 +166,10 @@ class FastFile { } // if (ops.length>1) console.log(ops.length); - // This attempts to avoid 'too many promises for Promise.all' error + // This avoids 'too many promises for Promise.all' error const opsWrapper = []; for (let from=0; from ops.length) to = ops.length; + let to = Math.min(from+this.maxPromises, ops.length); const opsSlice = ops.slice(from, to); opsWrapper.push(Promise.all(opsSlice)); } From 2e39908b2ab86ec93bcd3c4e4475bb2e8568ffaa Mon Sep 17 00:00:00 2001 From: Geoff Lamperd Date: Thu, 4 Apr 2024 15:14:15 +1000 Subject: [PATCH 5/5] placeholder path in test --- test/osfile.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/osfile.test.js b/test/osfile.test.js index 6da8552..06a1885 100644 --- a/test/osfile.test.js +++ b/test/osfile.test.js @@ -51,7 +51,7 @@ describe("fastfile testing suite for osfile", function () { }); it("should read a large file", async () => { - const fileName = "//wsl.localhost/Ubuntu/home/geoff/ptau/pot10_0006_bcn_prep.ptau"; + const fileName = "//path/to/large/file.ptau"; const bytesToRead = 1<<16; const ff = await fastFile.readExisting(fileName, 1024, 1024); assert(ff.totalSize>0);