From 475d44195a564caf2860a1a6b9fe202b7833bf79 Mon Sep 17 00:00:00 2001 From: Billy Charlton Date: Fri, 5 Jun 2026 18:16:59 +0200 Subject: [PATCH 1/2] fix: some ZSTD files trigger a bug in the JS Zstd streaming decompressor library I am not going to try and fix the library which is here: https://github.com/tadpole-labs/zstd-codec-lib Instead, I work around it by failing over to decompressing the entire network file at once instead of streaming. fixes #526 --- src/workers/WasmXmlNetworkParser.worker.ts | 244 +++++++++++---------- 1 file changed, 128 insertions(+), 116 deletions(-) diff --git a/src/workers/WasmXmlNetworkParser.worker.ts b/src/workers/WasmXmlNetworkParser.worker.ts index a45c7812..9f7642cb 100644 --- a/src/workers/WasmXmlNetworkParser.worker.ts +++ b/src/workers/WasmXmlNetworkParser.worker.ts @@ -81,7 +81,7 @@ function recenterAtlantis() { } async function processChunk(chunk: any) { - // postMessage({ status: 'Got chunk ' + _countLinks }) + postMessage({ status: 'Reading links: ' + (_countLinks || '') }) // build nodes first =========================================== const nodes = chunk.r?.node as { $id: string; $x: string; $y: string }[] @@ -168,7 +168,6 @@ async function processChunk(chunk: any) { } function finalAssembly() { - // console.log(100, _cleanLinks) // clear some ram first nodeIdOffset = {} // _nodeCoords = null @@ -266,144 +265,157 @@ async function parseXML(props?: { // Use a writablestream, which the docs say creates backpressure automatically: // https://developer.mozilla.org/en-US/docs/Web/API/WritableStream - const streamProcessor = new WritableStream( - { - write(incomingChunk: Uint8Array) { - return new Promise(async (resolve, reject) => { - _numChunks++ - if (_isCancelled) reject() - - // cut off chunk at the last line ending so we never split UTF-8 glyphs - let entireChunk = new Uint8Array(_preBytes.length + incomingChunk.length) - entireChunk.set(_preBytes) - entireChunk.set(incomingChunk, _preBytes.length) - _preBytes = new Uint8Array(0) - - const cutoff = entireChunk.lastIndexOf(10) - if (cutoff > -1) { - _preBytes = entireChunk.slice(cutoff + 1) - entireChunk = entireChunk.subarray(0, cutoff) - } + // const streamProcessor = new WritableStream({}, strategy) + const STREAM_PROCESSOR: any = { + write(incomingChunk: Uint8Array) { + return new Promise(async (resolve, reject) => { + _numChunks++ + if (_isCancelled) reject() + + // cut off chunk at the last line ending so we never split UTF-8 glyphs + let entireChunk = new Uint8Array(_preBytes.length + incomingChunk.length) + entireChunk.set(_preBytes) + entireChunk.set(incomingChunk, _preBytes.length) + _preBytes = new Uint8Array(0) + + const cutoff = entireChunk.lastIndexOf(10) + if (cutoff > -1) { + _preBytes = entireChunk.slice(cutoff + 1) + entireChunk = entireChunk.subarray(0, cutoff) + } - let text = _leftovers + _decoder.decode(entireChunk) - entireChunk = null as any - _leftovers = '' - - if (_numChunks == 1) { - if (!_crs) { - const crsLine = text.indexOf('coordinateReferenceSystem') - if (crsLine > -1) { - const line = text.slice(crsLine) - _crs = line.substring(line.indexOf('>') + 1, line.indexOf(' -1) { + const line = text.slice(crsLine) + _crs = line.substring(line.indexOf('>') + 1, line.indexOf(' -1 ? closeTag : '/>' - if (!closeNode) closeNode = text.indexOf(closeTag) > -1 ? closeTag : '/>' + const endOfNodes = text.indexOf(endOfSection) + let linkText = '' + if (endOfNodes > -1) { + linkText = text.slice(endOfNodes) + text = text.slice(0, endOfNodes) + } - const endOfNodes = text.indexOf(endOfSection) - let linkText = '' - if (endOfNodes > -1) { - linkText = text.slice(endOfNodes) - text = text.slice(0, endOfNodes) + const startNode = text.indexOf(searchElement) + if (startNode == -1) reject('no nodes found') + + const lastNode = text.lastIndexOf(closeNode) + const xmlBody = text.slice(startNode, lastNode + closeNode.length) + _leftovers = text.slice(lastNode + closeNode.length) + + if (xmlBody.length) { + _chunkCounter++ + _xmlStagingArea += xmlBody + if (_chunkCounter > 4 || endOfNodes > -1) { + const fullXml = `${_xmlStagingArea}` + _xmlStagingArea = '' + _chunkCounter = 0 + const json = await JSUtil.parseXML(fullXml, {}) + processChunk(json) } + } - const startNode = text.indexOf(searchElement) - if (startNode == -1) reject('no nodes found') + // flip to link mode if we got the tag + if (endOfNodes > -1) { + searchElement = ' -1 && _leftovers.indexOf('') > -1) { + console.log('---TINY NETWORK') + _xmlStagingArea = _leftovers + .substring(_leftovers.indexOf('', '') + .replace('', '') + } + resolve() + }) + }, - if (xmlBody.length) { - _chunkCounter++ - _xmlStagingArea += xmlBody - if (_chunkCounter > 4 || endOfNodes > -1) { + close() { + if (_xmlStagingArea.length) { + // console.log('CLEANUP: Got some leftovers in the staging area') + promises.push( + new Promise(async (resolve, reject) => { + try { const fullXml = `${_xmlStagingArea}` _xmlStagingArea = '' _chunkCounter = 0 - const json = await JSUtil.parseXML(fullXml, {}) + const json = (await JSUtil.parseXML(fullXml)) as any processChunk(json) + // resolve(json) + } catch (e) { + console.error('' + e) + reject('' + e) } - } - - // flip to link mode if we got the tag - if (endOfNodes > -1) { - searchElement = ' -1 && _leftovers.indexOf('') > -1) { - console.log('---TINY NETWORK') - _xmlStagingArea = _leftovers - .substring(_leftovers.indexOf('', '') - .replace('', '') - } - resolve() - }) - }, - - close() { - // if (_leftovers) _xmlStagingArea += _leftovers - - // console.log(22, { _xmlStagingArea, _leftovers }) - // if (_xmlStagingArea.indexOf('') > -1) - // _xmlStagingArea = _xmlStagingArea.replace('', '') - // _xmlStagingArea = _xmlStagingArea.replace('', '') - - if (_xmlStagingArea.length) { - // console.log('CLEANUP: Got some leftovers in the staging area') - promises.push( - new Promise(async (resolve, reject) => { - try { - const fullXml = `${_xmlStagingArea}` - _xmlStagingArea = '' - _chunkCounter = 0 - const json = (await JSUtil.parseXML(fullXml)) as any - processChunk(json) - // resolve(json) - } catch (e) { - console.error('' + e) - reject('' + e) - } - }) - ) - } - console.log('STREAM FINISHED!') - }, - abort(err) { - console.log('STREAM error:', err) - }, + }) + ) + } + console.log('STREAM FINISHED!') }, - strategy - ) + abort(err: any) { + console.log('STREAM error:', err) + }, + } try { + // build the StreamProcessor for processing + let streamProcessor = new WritableStream(STREAM_PROCESSOR, strategy) // get the readable stream from the server - const readableStream = await fileApi.getFileStream(_filename) - const lowerFilename = _filename.toLocaleLowerCase() + let readableStream: any = await fileApi.getFileStream(_filename) // stream results through the data pipe + const lowerFilename = _filename.toLocaleLowerCase() if (lowerFilename.endsWith('.gz')) { const gunzipper = new DecompressionStream('gzip') await readableStream.pipeThrough(gunzipper).pipeTo(streamProcessor) } else if (lowerFilename.endsWith('.zst')) { - const zunzipper = new ZStd.ZstdDecompressionStream() - await readableStream.pipeThrough(zunzipper).pipeTo(streamProcessor) + try { + const zunzipper = new ZStd.ZstdDecompressionStream() + await readableStream.pipeThrough(zunzipper).pipeTo(streamProcessor) + } catch (e) { + console.log('--- zstd streamer failed: will try to ingest all at once instead...') + readableStream = null + let streamProcessor = new WritableStream(STREAM_PROCESSOR, strategy) + + const buffer = await (await fileApi.getFileBlob(_filename)).arrayBuffer() + const data = await ZStd.decompress(new Uint8Array(buffer)) + // now pipe it all at once + await new ReadableStream({ + start(controller) { + const CHUNK_SIZE = 1024 * 1024 + for (let i = 0; i < data.length; i += CHUNK_SIZE) { + controller.enqueue(data.slice(i, i + CHUNK_SIZE)) + } + // a + controller.close() + }, + }).pipeTo(streamProcessor) + } } else { await readableStream.pipeTo(streamProcessor) } From 7fbb17cd0945718a5abbb85d1b9ded7cbf867110 Mon Sep 17 00:00:00 2001 From: Billy Charlton Date: Fri, 12 Jun 2026 13:56:37 +0200 Subject: [PATCH 2/2] fix(comments): show comments from multiple CSVs in plotly chart plugin --- src/layout-manager/DashBoard.vue | 35 +++++++++++++++++++--------- src/plugins/plotly/PlotlyDiagram.vue | 11 +++++++-- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/layout-manager/DashBoard.vue b/src/layout-manager/DashBoard.vue index 8d6308cb..e0bc68ff 100644 --- a/src/layout-manager/DashBoard.vue +++ b/src/layout-manager/DashBoard.vue @@ -43,7 +43,7 @@ //- info/zoom buttons .header-buttons button.button.is-small.is-white( - v-show="card.info || card.comments" + v-show="card.info || hasCardComments(card)" @click="handleToggleInfoClick(card)" :title="infoToggle[card.id] ? 'Hide Info':'Show Info'" ) @@ -206,16 +206,28 @@ export default defineComponent({ }, methods: { + hasCardComments(card: any): string { + if (!card.comments) return '' + return Array.isArray(card.comments) + ? card.comments.join('') + : Object.values(card.comments).join('') + }, + mdInfo(card: { info: any; comments: any }) { let info = '' if (card.info) { const html = MarkdownRenderer.render(card.info) info += html } + if (card.comments) { - if (info) info += '
\n' - const html = MarkdownRenderer.render(card.comments) - info += html + if (info) info += '\n
\n' + Object.values(card.comments).forEach((comments: any, index) => { + if (!comments) return + if (index > 0) info += '\n
\n' + const html = MarkdownRenderer.render(comments) + info += html + }) } return info }, @@ -662,17 +674,18 @@ export default defineComponent({ return tag }, - handleComments(card: any, comments: string[]) { - console.log('GOT COMMENTS') - Vue.set(card, 'comments', '') - console.log(comments) - if (comments?.length) { - const mdRaw = comments + handleComments(card: any, comments: string[] | { filename: string; comments: string[] }) { + if (!card.comments) card.comments = {} as { [filename: string]: string[] } + let filename = Array.isArray(comments) ? '' : comments.filename + let commentStrings = Array.isArray(comments) ? comments : comments.comments + let mdRaw = '' + if (commentStrings?.length) { + mdRaw = commentStrings .map(c => c.slice(1).trim()) .filter(c => !c.startsWith('EPSG:')) // hide EPSG codez which are special .join('\n') - card.comments = mdRaw } + card.comments[filename] = mdRaw }, async handleCardIsLoaded(card: any) { diff --git a/src/plugins/plotly/PlotlyDiagram.vue b/src/plugins/plotly/PlotlyDiagram.vue index e0a14588..03088fae 100644 --- a/src/plugins/plotly/PlotlyDiagram.vue +++ b/src/plugins/plotly/PlotlyDiagram.vue @@ -889,13 +889,17 @@ const MyComponent = defineComponent({ async loadDataset(name: string, ds: DataSet): Promise { this.loadingText = 'Loading datasets...' + this.$emit('comments', { filename: name, comments: [] }) try { const csvData = await this.myDataManager.getDataset( { dataset: ds.file }, { highPrecision: true, subfolder: this.subfolder } ) - if (csvData.comments?.length) this.$emit('comments', csvData.comments) + + if (csvData.comments?.length) { + this.$emit('comments', { filename: name, comments: csvData.comments }) + } ds.data = csvData.allRows ds.name = name @@ -1145,7 +1149,10 @@ const MyComponent = defineComponent({ const dataColumn = dataTable[column] for (let i = 0; i < n; i++) { - if (hasMatchedFilters[i] && !this.checkFilterValue(fullSpecification, dataColumn.values[i])) { + if ( + hasMatchedFilters[i] && + !this.checkFilterValue(fullSpecification, dataColumn.values[i]) + ) { hasMatchedFilters[i] = false } }