From e9c20c50562187c5d8928067730de1871c6d17d1 Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Tue, 14 Nov 2023 22:39:35 +0100 Subject: [PATCH 1/9] bugfix: Avoid merging multiple polylines into one --- src/geojson.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/geojson.js b/src/geojson.js index be8f28c..afd5195 100644 --- a/src/geojson.js +++ b/src/geojson.js @@ -14,7 +14,7 @@ function justType(gjType, shpType) { return function (gj) { var oftype = gj.features.filter(isType(gjType)); return { - geometries: shpType === 'POLYLINE' ? [oftype.map(justCoords)] : oftype.map(justCoords), + geometries: shpType === "POLYLINE" ? oftype.map((_) => [justCoords(_)]) : oftype.map(justCoords), properties: oftype.map(justProps), type: shpType, }; @@ -22,7 +22,7 @@ function justType(gjType, shpType) { } /** - * + * * @param {Feature} feature The feature to get the coordinates from * @returns {number[] | number[][] | number[][][] | number[][][][]} */ @@ -31,8 +31,8 @@ function justCoords(feature) { } /** - * - * @param {Feature} feature The feature to get the properties from + * + * @param {Feature} feature The feature to get the properties from * @returns {Object.} */ function justProps(feature) { From c5a5aa3b6fe48dca0988febffa0cc3b41feca22f Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Tue, 14 Nov 2023 22:40:25 +0100 Subject: [PATCH 2/9] test: Multi polyline with data zip --- test/comparison_files/zip_polyline_1.zip | Bin 0 -> 1533 bytes test/zip.test.js | 62 +++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 test/comparison_files/zip_polyline_1.zip create mode 100644 test/zip.test.js diff --git a/test/comparison_files/zip_polyline_1.zip b/test/comparison_files/zip_polyline_1.zip new file mode 100644 index 0000000000000000000000000000000000000000..7db60548b361efb5259cf8c0fa792979f9d80b65 GIT binary patch literal 1533 zcmWIWW@h1H0D*?pdErT`R(FdqGBAKJ4^S+?-zUwHtQv*rdcn-V z07N)|iUULfM&nQh5eC}Kph4Iy1_qEqCcH+W*auRI#ax&g1dZEVgc^voDh>fCp|_Ny z;%Es_3WU)@uL4i#Sy3_cj6l8w;y4hE>biG+W%drhPyk^R*QF$-F;{c*fda^p;XjbX z1bKRqLmQJ%#6(~42{ioj4Unmj7-gq zjEqgqjALVU0)jk!U4i=Di&9hb$}^KQlynSYbwd3-Lx5(crUONR0(u69<|d}bMwUjV zhL)BlK!#~-Y=Ac-lL!Ou0uVKwBNvD0I?*#fLZ=zjRnb0V6X>xv$BC?S%B~bkp2&pXJ7yTPr1Wg literal 0 HcmV?d00001 diff --git a/test/zip.test.js b/test/zip.test.js new file mode 100644 index 0000000..1545d5a --- /dev/null +++ b/test/zip.test.js @@ -0,0 +1,62 @@ +var expect = require("expect.js"), + zip = require("../src/zip"), + path = require("path"), + fs = require("fs"); + +describe("zip", function () { + describe("#polyline", function () { + it("1. multi-feature polyline with associated feature data", async function () { + const options = { + outputType: "blob", + compression: "STORE", + }; + + var geojson = { + type: "FeatureCollection", + features: [ + { + type: "Feature", + geometry: { + type: "LineString", + coordinates: [ + [0, 0], + [10, 0], + ], + }, + properties: { + name: "Foo", + }, + }, + { + type: "Feature", + geometry: { + type: "LineString", + coordinates: [ + [0, 10], + [10, 10], + ], + }, + properties: { + name: "Bar", + }, + }, + ], + }; + + const zippedShapefile = await zip(geojson, options); + + const testFile = path.join("test", "comparison_files", "zip_polyline_1_test.zip"); + const sourceFile = path.join("test", "comparison_files", "zip_polyline_1.zip"); + + await new Promise(async function (resolve, reject) { + fs.writeFile(testFile, Buffer.from(await zippedShapefile.arrayBuffer()), () => resolve("saved")); + }); + + const testFileData = fs.readFileSync(testFile).toString(); + const sourceFileData = fs.readFileSync(sourceFile).toString(); + expect(testFileData).to.equal(sourceFileData); + + fs.rmSync(testFile); + }); + }); +}); From 99477c51690019605c976b10fa7669dfea8d60c0 Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Wed, 15 Nov 2023 11:23:30 +0100 Subject: [PATCH 3/9] export geojson for custom zip implementation --- dist/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/index.js b/dist/index.js index ae607de..6136273 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,3 +1,4 @@ module.exports.download = require("../src/download"); module.exports.write = require("../src/write"); module.exports.zip = require("../src/zip"); +module.exports.geojson = require("../src/geojson"); From 52a56911818cc12497cc1eb4ad74a56c5767a330 Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Wed, 15 Nov 2023 11:30:05 +0100 Subject: [PATCH 4/9] geojson types --- dist/index.d.ts | 56 ++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/dist/index.d.ts b/dist/index.d.ts index 8db94eb..aae9fcf 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,19 +1,19 @@ declare module "@mapbox/shp-write" { export type OGCGeometry = - 'NULL' | - 'POINT' | - 'POLYLINE' | - 'POLYGON' | - 'MULTIPOINT' | - 'POINTZ' | - 'POLYLINEZ' | - 'POLYGONZ' | - 'MULTIPOINTZ' | - 'POINTM' | - 'POLYLINEM' | - 'POLYGONM' | - 'MULTIPOINTM' | - 'MULTIPATCH'; + | "NULL" + | "POINT" + | "POLYLINE" + | "POLYGON" + | "MULTIPOINT" + | "POINTZ" + | "POLYLINEZ" + | "POLYGONZ" + | "MULTIPOINTZ" + | "POINTM" + | "POLYLINEM" + | "POLYGONM" + | "MULTIPOINTM" + | "MULTIPATCH"; export interface DownloadOptions { folder?: string; @@ -28,7 +28,7 @@ declare module "@mapbox/shp-write" { }; } - type Compression = 'STORE' | 'DEFLATE'; + type Compression = "STORE" | "DEFLATE"; interface OutputByType { base64: string; string: string; @@ -44,14 +44,25 @@ declare module "@mapbox/shp-write" { type OutputType = keyof OutputByType; export interface ZipOptions { - compression: Compression, - outputType: OutputType + compression: Compression; + outputType: OutputType; } - export function download( - geojson: GeoJSON.FeatureCollection, - options?: DownloadOptions & ZipOptions - ): void; + export function download(geojson: GeoJSON.FeatureCollection, options?: DownloadOptions & ZipOptions): void; + + type PreparedGeojsonForWriting = { + geometries: number[] | number[][] | number[][][] | number[][][][]; + properties: {}; + type: string; + }; + + export var geojson = { + point: (geojson: { features: Feature[] }) => PreparedGeojsonForWriting, + line: (geojson: { features: Feature[] }) => PreparedGeojsonForWriting, + multiline: (geojson: { features: Feature[] }) => PreparedGeojsonForWriting, + polygon: (geojson: { features: Feature[] }) => PreparedGeojsonForWriting, + multipolygon: (geojson: { features: Feature[] }) => PreparedGeojsonForWriting, + }; export function write( data: Array, @@ -70,5 +81,6 @@ declare module "@mapbox/shp-write" { export function zip( geojson: GeoJSON.FeatureCollection, options?: DownloadOptions & ZipOptions, - stream?: boolean): Promise; + stream?: boolean + ): Promise; } From 31d8adbdf205446354c04b2bd871cec135568697 Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Wed, 15 Nov 2023 14:03:09 +0100 Subject: [PATCH 5/9] polylineZ support --- src/extent.js | 40 ++++++++------ src/polyZ.js | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/write.js | 96 ++++++++++++++++----------------- 3 files changed, 216 insertions(+), 65 deletions(-) create mode 100644 src/polyZ.js diff --git a/src/extent.js b/src/extent.js index 64f929f..b815929 100644 --- a/src/extent.js +++ b/src/extent.js @@ -1,24 +1,30 @@ module.exports.enlarge = function enlargeExtent(extent, pt) { - if (pt[0] < extent.xmin) extent.xmin = pt[0]; - if (pt[0] > extent.xmax) extent.xmax = pt[0]; - if (pt[1] < extent.ymin) extent.ymin = pt[1]; - if (pt[1] > extent.ymax) extent.ymax = pt[1]; - return extent; + if (pt[0] < extent.xmin) extent.xmin = pt[0]; + if (pt[0] > extent.xmax) extent.xmax = pt[0]; + if (pt[1] < extent.ymin) extent.ymin = pt[1]; + if (pt[1] > extent.ymax) extent.ymax = pt[1]; + if ((pt[2] || 0) < extent.zmin) extent.zmin = pt[2] || 0; + if ((pt[2] || 0) > extent.zmax) extent.zmax = pt[2] || 0; + return extent; }; module.exports.enlargeExtent = function enlargeExtent(extent, ext) { - if (ext.xmax > extent.xmax) extent.xmax = ext.xmax; - if (ext.xmin < extent.xmin) extent.xmin = ext.xmin; - if (ext.ymax > extent.ymax) extent.ymax = ext.ymax; - if (ext.ymin < extent.ymin) extent.ymin = ext.ymin; - return extent; + if (ext.xmax > extent.xmax) extent.xmax = ext.xmax; + if (ext.xmin < extent.xmin) extent.xmin = ext.xmin; + if (ext.ymax > extent.ymax) extent.ymax = ext.ymax; + if (ext.ymin < extent.ymin) extent.ymin = ext.ymin; + if (ext.zmax && ext.zmax > extent.zmax) extent.zmax = ext.zmax; + if (ext.zmin && ext.zmin < extent.zmin) extent.zmin = ext.zmin; + return extent; }; -module.exports.blank = function() { - return { - xmin: Number.MAX_VALUE, - ymin: Number.MAX_VALUE, - xmax: -Number.MAX_VALUE, - ymax: -Number.MAX_VALUE - }; +module.exports.blank = function () { + return { + xmin: Number.MAX_VALUE, + ymin: Number.MAX_VALUE, + zmin: Number.MAX_VALUE, + xmax: -Number.MAX_VALUE, + ymax: -Number.MAX_VALUE, + zmax: -Number.MAX_VALUE, + }; }; diff --git a/src/polyZ.js b/src/polyZ.js new file mode 100644 index 0000000..fcebb53 --- /dev/null +++ b/src/polyZ.js @@ -0,0 +1,145 @@ +var ext = require("./extent"), + types = require("./types"); + +module.exports.write = function writePoints(geometries, extent, shpView, shxView, TYPE) { + var shpI = 0, + shxI = 0, + shxOffset = 100; + + geometries.forEach(writePolyLine); + + function writePolyLine(coordinates, i) { + var flattened = justCoords(coordinates), + noParts = parts([coordinates], TYPE), + contentLength = flattened.length * 32 + 32 + 48 + (noParts - 1) * 4; + + var featureExtent = flattened.reduce(function (extent, c) { + return ext.enlarge(extent, c); + }, ext.blank()); + + // INDEX + shxView.setInt32(shxI, shxOffset / 2); // offset + shxView.setInt32(shxI + 4, contentLength / 2); // offset length + + shxI += 8; + shxOffset += contentLength + 8; + + shpView.setInt32(shpI, i + 1); // record number + shpView.setInt32(shpI + 4, contentLength / 2); // length + shpView.setInt32(shpI + 8, TYPE, true); // POLYLINEZ=13 + shpView.setFloat64(shpI + 12, featureExtent.xmin, true); // EXTENT + shpView.setFloat64(shpI + 20, featureExtent.ymin, true); + shpView.setFloat64(shpI + 28, featureExtent.xmax, true); + shpView.setFloat64(shpI + 36, featureExtent.ymax, true); + shpView.setInt32(shpI + 44, noParts, true); + shpView.setInt32(shpI + 48, flattened.length, true); // POINTS + shpView.setInt32(shpI + 52, 0, true); // The first part - index zero + + var onlyParts = coordinates.reduce(function (arr, coords) { + if (Array.isArray(coords[0][0])) { + arr = arr.concat(coords); + } else { + arr.push(coords); + } + return arr; + }, []); + for (var p = 1; p < noParts; p++) { + shpView.setInt32( + // set part index + shpI + 52 + p * 4, + onlyParts.reduce(function (a, b, idx) { + return idx < p ? a + b.length : a; + }, 0), + true + ); + } + + shpI += 56 + (noParts - 1) * 4; + + var zMin = Number.MAX_VALUE; + var zMax = -Number.MAX_VALUE; + + shpI += 56 + (noParts - 1) * 4; + + flattened.forEach(function writeLine(coords, i) { + if ((coords[2] || 0) < zMin) zMin = coords[2] || 0; + if ((coords[2] || 0) > zMax) zMax = coords[2] || 0; + + shpView.setFloat64(shpI, coords[0], true); // X + shpView.setFloat64(shpI + 8, coords[1], true); // Y + shpI += 16; + }); + + shpI += contentLength + 8; + + // Write z value range + shpView.setFloat64(shpI, zMin, true); + shpView.setFloat64(shpI + 8, zMax, true); + shpI += 16; + + // Write z values. + flattened.forEach(function (p, i) { + shpView.setFloat64(shpI, p[2] || 0, true); + shpI += 8; + }); + } +}; + +module.exports.shpLength = function (geometries) { + var flattened = justCoords(geometries); + var length = geometries.length * 56 + flattened.length * 16; + return (length += 32 + flattened.length * 16); +}; + +module.exports.shxLength = function (geometries) { + return geometries.length * 8; +}; + +module.exports.extent = function (coordinates) { + return justCoords(coordinates).reduce(function (extent, c) { + return ext.enlarge(extent, c); + }, ext.blank()); +}; + +function parts(geometries, TYPE) { + var no = 1; + if ( + TYPE === types.geometries.POLYGON || + TYPE === types.geometries.POLYLINE || + TYPE === types.geometries.POLYGONZ || + TYPE === types.geometries.POLYLINEZ + ) { + no = geometries.reduce(function (no, coords) { + no += coords.length; + if (Array.isArray(coords[0][0][0])) { + // multi + no += coords.reduce(function (no, rings) { + return no + rings.length - 1; // minus outer + }, 0); + } + return no; + }, 0); + } + return no; +} + +module.exports.parts = parts; + +function totalPoints(geometries) { + var sum = 0; + geometries.forEach(function (g) { + sum += g.length; + }); + return sum; +} + +function justCoords(coords, l) { + if (l === undefined) l = []; + if (typeof coords[0][0] == "object") { + return coords.reduce(function (memo, c) { + return memo.concat(justCoords(c)); + }, l); + } else { + return coords; + } +} diff --git a/src/write.js b/src/write.js index 42d9ff9..1c86891 100644 --- a/src/write.js +++ b/src/write.js @@ -1,63 +1,63 @@ -var types = require('./types'); -var dbf = require('dbf'); -var prj = require('./prj'); -var pointWriter = require('./points'); -var polyWriter = require('./poly'); +var types = require("./types"); +var dbf = require("dbf"); +var prj = require("./prj"); +var pointWriter = require("./points"); +var polyWriter = require("./poly"); +var polyZWriter = require("./polyZ"); var writers = { - 1: pointWriter, - 5: polyWriter, - 3: polyWriter + 1: pointWriter, + 5: polyWriter, + 3: polyWriter, + 13: polyZWriter, }; module.exports = write; // Low-level writing interface function write(rows, geometry_type, geometries, callback) { - - var TYPE = types.geometries[geometry_type]; - var writer = writers[TYPE]; - var parts = writer.parts(geometries, TYPE); - var shpLength = 100 + (parts - geometries.length) * 4 + writer.shpLength(geometries); - var shxLength = 100 + writer.shxLength(geometries); - var shpBuffer = new ArrayBuffer(shpLength); - var shpView = new DataView(shpBuffer); - var shxBuffer = new ArrayBuffer(shxLength); - var shxView = new DataView(shxBuffer); - var extent = writer.extent(geometries); - - writeHeader(shpView, TYPE); - writeHeader(shxView, TYPE); - writeExtent(extent, shpView); - writeExtent(extent, shxView); - - writer.write(geometries, extent, - new DataView(shpBuffer, 100), - new DataView(shxBuffer, 100), - TYPE); - - shpView.setInt32(24, shpLength / 2); - shxView.setInt32(24, (50 + geometries.length * 4)); - - var dbfBuf = dbf.structure(rows); - - callback(null, { - shp: shpView, - shx: shxView, - dbf: dbfBuf, - prj: prj - }); + var TYPE = types.geometries[geometry_type]; + var writer = writers[TYPE]; + var parts = writer.parts(geometries, TYPE); + var shpLength = 100 + (parts - geometries.length) * 4 + writer.shpLength(geometries); + var shxLength = 100 + writer.shxLength(geometries); + var shpBuffer = new ArrayBuffer(shpLength); + var shpView = new DataView(shpBuffer); + var shxBuffer = new ArrayBuffer(shxLength); + var shxView = new DataView(shxBuffer); + var extent = writer.extent(geometries); + + writeHeader(shpView, TYPE); + writeHeader(shxView, TYPE); + writeExtent(extent, shpView); + writeExtent(extent, shxView); + + writer.write(geometries, extent, new DataView(shpBuffer, 100), new DataView(shxBuffer, 100), TYPE); + + shpView.setInt32(24, shpLength / 2); + shxView.setInt32(24, 50 + geometries.length * 4); + + var dbfBuf = dbf.structure(rows); + + callback(null, { + shp: shpView, + shx: shxView, + dbf: dbfBuf, + prj: prj, + }); } function writeHeader(view, TYPE) { - view.setInt32(0, 9994); - view.setInt32(28, 1000, true); - view.setInt32(32, TYPE, true); + view.setInt32(0, 9994); + view.setInt32(28, 1000, true); + view.setInt32(32, TYPE, true); } function writeExtent(extent, view) { - view.setFloat64(36, extent.xmin, true); - view.setFloat64(44, extent.ymin, true); - view.setFloat64(52, extent.xmax, true); - view.setFloat64(60, extent.ymax, true); + view.setFloat64(36, extent.xmin, true); + view.setFloat64(44, extent.ymin, true); + view.setFloat64(52, extent.xmax, true); + view.setFloat64(60, extent.ymax, true); + view.setFloat64(68, extent.zmin, true); + view.setFloat64(76, extent.zmax, true); } From a56bf7302b956bae8adabe7e9b508d2245f7ec09 Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Wed, 15 Nov 2023 15:17:59 +0100 Subject: [PATCH 6/9] bigfix - polylinez shpLength --- src/polyZ.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/polyZ.js b/src/polyZ.js index fcebb53..a6f3287 100644 --- a/src/polyZ.js +++ b/src/polyZ.js @@ -88,7 +88,7 @@ module.exports.write = function writePoints(geometries, extent, shpView, shxView module.exports.shpLength = function (geometries) { var flattened = justCoords(geometries); var length = geometries.length * 56 + flattened.length * 16; - return (length += 32 + flattened.length * 16); + return length + 32 + flattened.length * 16; }; module.exports.shxLength = function (geometries) { From 1046520b9e25abe606b274a58ebde9b904c7104d Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Wed, 15 Nov 2023 15:24:08 +0100 Subject: [PATCH 7/9] bugfix: polyz shpLength --- src/polyZ.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/polyZ.js b/src/polyZ.js index a6f3287..e1c998f 100644 --- a/src/polyZ.js +++ b/src/polyZ.js @@ -85,10 +85,10 @@ module.exports.write = function writePoints(geometries, extent, shpView, shxView } }; -module.exports.shpLength = function (geometries) { +module.exports.shpLength = function (geometries, TYPE) { var flattened = justCoords(geometries); - var length = geometries.length * 56 + flattened.length * 16; - return length + 32 + flattened.length * 16; + var length = geometries.length * 56 + flattened.length * 16 + 32 + flattened.length * 16; + return length; }; module.exports.shxLength = function (geometries) { From d55ff27e95ea07ca67a0a08e76a7760b59861c64 Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Wed, 15 Nov 2023 15:55:01 +0100 Subject: [PATCH 8/9] bugfix: polyZ --- src/polyZ.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/polyZ.js b/src/polyZ.js index e1c998f..f932cd2 100644 --- a/src/polyZ.js +++ b/src/polyZ.js @@ -54,8 +54,6 @@ module.exports.write = function writePoints(geometries, extent, shpView, shxView ); } - shpI += 56 + (noParts - 1) * 4; - var zMin = Number.MAX_VALUE; var zMax = -Number.MAX_VALUE; From 3feba8590d83e759eebe2368261e2566bbb59752 Mon Sep 17 00:00:00 2001 From: Matthew Abrman Date: Wed, 15 Nov 2023 16:09:14 +0100 Subject: [PATCH 9/9] bugfix: polyZ --- src/polyZ.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/polyZ.js b/src/polyZ.js index f932cd2..8288afd 100644 --- a/src/polyZ.js +++ b/src/polyZ.js @@ -11,7 +11,7 @@ module.exports.write = function writePoints(geometries, extent, shpView, shxView function writePolyLine(coordinates, i) { var flattened = justCoords(coordinates), noParts = parts([coordinates], TYPE), - contentLength = flattened.length * 32 + 32 + 48 + (noParts - 1) * 4; + contentLength = flattened.length * 16 + 48 + (noParts - 1) * 4 + flattened.length * 8 + 16; var featureExtent = flattened.reduce(function (extent, c) { return ext.enlarge(extent, c); @@ -68,8 +68,6 @@ module.exports.write = function writePoints(geometries, extent, shpView, shxView shpI += 16; }); - shpI += contentLength + 8; - // Write z value range shpView.setFloat64(shpI, zMin, true); shpView.setFloat64(shpI + 8, zMax, true);