roko's basiliskSo, forgive me but I asked a clanker. I'm no more inclined to write JS than you are, save where money is involved where I am clearly a whore, but
// At the top level const MAX_CONCURRENCY = 8; let running = 0; const queue = []; async function limitedFetchCached(id) { if (running >= MAX_CONCURRENCY) { await new Promise(r => queue.push(r)); } running++; try { return await fetchCached(id); // your existing one } finally { running--; if (queue.length) queue.shift()(); } } // 1. Much cleaner decompression (native stream, minimal copies) async function deflateRawDecode(u8) { const ds = new DecompressionStream('deflate-raw'); const stream = new Response(u8).body.pipeThrough(ds); const reader = stream.getReader(); const chunks = []; let total = 0; while (true) { const {value, done} = await reader.read(); if (done) break; chunks.push(value); total += value.length; } if (chunks.length === 1) return chunks[0]; // common case: zero extra copy const out = new Uint8Array(total); let off = 0; for (const c of chunks) { out.set(c, off); off += c.length; } return out; } // 2. Minimal getBlockContents (zero-copy when possible) async function getBlockContents(fb, meta) { let payload = fb; if (meta.compression === CTDeflate) { payload = await deflateRawDecode(payload); } if (meta.sz > payload.length) { const p = new Uint8Array(meta.sz); // allocate exactly once, rest is zero p.set(payload, 0); return p; } return payload; } // 3. New tree walker — no more intermediate full-node concatenations ever async function streamTreeContents(block, chunksOut) { if (block.length < BlockMetaSz) return; // safety for (let i = 0; i < block.length; i += BlockMetaSz) { const meta = deserializeBlockMeta(block.subarray(i, i + BlockMetaSz)); // subarray = zero-copy view const id = hexFromBytes(meta.hash); const fb = await fetchCached(id); const content = await getBlockContents(fb, meta); if (meta.type === BTParent) { await streamTreeContents(content, chunksOut); // recurse directly on this subtree } else { chunksOut.push(content); // leaf data — just collect } } } // 4. Updated reassemble (returns Blob directly — most efficient for your use case) async function reassemble(topId) { const chunks = []; try { const top = await fetchCached(topId.toLowerCase()); await streamTreeContents(top, chunks); // MIME sniff only needs first few bytes (signatures are tiny) const prefix = chunks.length ? chunks[0].subarray(0, 1024) : new Uint8Array(0); const type = detectMIME(prefix); return new Blob(chunks, { type: type || 'application/octet-stream' }); } catch (err) { // partial still works the same way const e = new Error(err.message); e.partial = new Blob(chunks, { type: 'application/octet-stream' }); throw e; } }
GNU social JP is a social network, courtesy of GNU social JP管理人. It runs on GNU social, version 2.0.2-dev, available under the GNU Affero General Public License.
All GNU social JP content and data are available under the Creative Commons Attribution 3.0 license.