From ef7e2f3a3325fec6c0d0d65a54345ce3d2baa8a9 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Fri, 3 Dec 2021 12:27:35 -0800 Subject: [PATCH 001/113] tracklib2 initial experiments --- tracklib2/.gitignore | 1 + tracklib2/Cargo.lock | 711 ++++++++++++++++++++++++++++++++ tracklib2/Cargo.toml | 18 + tracklib2/src/error.rs | 24 ++ tracklib2/src/lib.rs | 2 + tracklib2/src/write/encoders.rs | 268 ++++++++++++ tracklib2/src/write/header.rs | 69 ++++ tracklib2/src/write/metadata.rs | 290 +++++++++++++ tracklib2/src/write/mod.rs | 3 + 9 files changed, 1386 insertions(+) create mode 100644 tracklib2/.gitignore create mode 100644 tracklib2/Cargo.lock create mode 100644 tracklib2/Cargo.toml create mode 100644 tracklib2/src/error.rs create mode 100644 tracklib2/src/lib.rs create mode 100644 tracklib2/src/write/encoders.rs create mode 100644 tracklib2/src/write/header.rs create mode 100644 tracklib2/src/write/metadata.rs create mode 100644 tracklib2/src/write/mod.rs diff --git a/tracklib2/.gitignore b/tracklib2/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/tracklib2/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tracklib2/Cargo.lock b/tracklib2/Cargo.lock new file mode 100644 index 0000000..11e115e --- /dev/null +++ b/tracklib2/Cargo.lock @@ -0,0 +1,711 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitvec" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "bstr" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "build_const" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "cast" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "bitflags", + "textwrap", + "unicode-width", +] + +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + +[[package]] +name = "criterion" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "csv" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + +[[package]] +name = "half" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "leb128" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" + +[[package]] +name = "libc" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "nom" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88034cfd6b4a0d54dd14f4a507eceee36c0b70e5a02236c4e4df571102be17f0" +dependencies = [ + "bitvec", + "memchr", + "version_check", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "plotters" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d1685fbe7beba33de0330629da9d955ac75bd54f33d7b79f9a895590124f6bb" +dependencies = [ + "js-sys", + "num-traits", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "regex" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", +] + +[[package]] +name = "regex-syntax" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" + +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9802ddde94170d186eeee5005b798d9c159fa970403f1be19976d0cfb939b72" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tap" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36474e732d1affd3a6ed582781b3683df3d0563714c59c39591e8ff707cf078e" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracklib2" +version = "0.1.0" +dependencies = [ + "crc", + "criterion", + "leb128", + "nom", + "thiserror", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml new file mode 100644 index 0000000..f58eb9e --- /dev/null +++ b/tracklib2/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "tracklib2" +version = "0.1.0" +authors = ["Dan Larkin "] +edition = "2018" + +[dependencies] +crc = "1.8" +leb128 = "0.2" +nom = {version = "6.0", default_features = false, features = ["alloc"]} +thiserror = "1.0" + +[dev-dependencies] +criterion = "0.3" + +[[bench]] +name = "parsing" +harness = false diff --git a/tracklib2/src/error.rs b/tracklib2/src/error.rs new file mode 100644 index 0000000..27bd27e --- /dev/null +++ b/tracklib2/src/error.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum TracklibError { + #[error("Numeric Bounds Error")] + BoundsError { + #[from] + source: std::num::TryFromIntError, + }, + + #[error("IO Error")] + IOError { + #[from] + source: std::io::Error, + }, + + #[error("Time Error?!")] + TimeError { + #[from] + source: std::time::SystemTimeError, + }, +} + +pub type Result = std::result::Result; diff --git a/tracklib2/src/lib.rs b/tracklib2/src/lib.rs new file mode 100644 index 0000000..5b6f406 --- /dev/null +++ b/tracklib2/src/lib.rs @@ -0,0 +1,2 @@ +mod write; +mod error; diff --git a/tracklib2/src/write/encoders.rs b/tracklib2/src/write/encoders.rs new file mode 100644 index 0000000..ebc5651 --- /dev/null +++ b/tracklib2/src/write/encoders.rs @@ -0,0 +1,268 @@ +use crate::error::Result; +use std::convert::TryFrom; +use std::io::Write; + +pub(crate) trait Encoder: Default { + type T; + fn encode( + &mut self, + value: Option<&Self::T>, + buf: &mut Vec, + presence: &mut Vec, + ) -> Result<()>; +} + +#[derive(Debug, Default)] +pub(crate) struct I64Encoder { + prev: i64, +} + +impl Encoder for I64Encoder { + type T = i64; + + fn encode( + &mut self, + value: Option<&Self::T>, + buf: &mut Vec, + presence: &mut Vec, + ) -> Result<()> { + presence.push(value.is_some()); + let delta = match value { + Some(v) => { + let value = *v; + let delta = value - self.prev; + self.prev = value; + delta + } + None => 0, + }; + + // Write the signed delta from the previous value + leb128::write::signed(buf, delta)?; + Ok(()) + } +} + +#[derive(Debug, Default)] +pub(crate) struct BoolEncoder; + +impl Encoder for BoolEncoder { + type T = bool; + + fn encode( + &mut self, + value: Option<&Self::T>, + buf: &mut Vec, + presence: &mut Vec, + ) -> Result<()> { + presence.push(value.is_some()); + let v = *(value.unwrap_or(&false)) as u8; + buf.write_all(&v.to_le_bytes())?; + Ok(()) + } +} + +#[derive(Debug, Default)] +pub(crate) struct StringEncoder; + +impl Encoder for StringEncoder { + type T = String; + + fn encode( + &mut self, + value: Option<&Self::T>, + buf: &mut Vec, + presence: &mut Vec, + ) -> Result<()> { + presence.push(value.is_some()); + if let Some(string) = value { + // Write the length of the string + leb128::write::unsigned(buf, u64::try_from(string.len()).unwrap())?; + // Write the string itself + buf.write_all(string.as_bytes())?; + } else { + // Write a zero length and nothing else + buf.write_all(&vec![0])?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_i64_encoder() { + let mut data_buf = vec![]; + let mut presence_buf = vec![]; + let mut encoder = I64Encoder::default(); + + assert!(encoder + .encode(Some(&1), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&2), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&3), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&-100), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&-100), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&100), &mut data_buf, &mut presence_buf) + .is_ok()); + + #[rustfmt::skip] + assert_eq!(data_buf, &[0x01, // +1 from 0 + 0x01, // +1 from 1 + 0x01, // +1 from 2 + 0x99, // -103 from 3 + 0x7F, + 0x00, // None + 0x00, // None + 0x00, // staying at -100 + 0xC8, // +200 from -100 + 0x01]); + + #[rustfmt::skip] + assert_eq!(presence_buf, &[true, + true, + true, + true, + false, + false, + true, + true]); + } + + #[test] + fn test_bool_encoder() { + let mut data_buf = vec![]; + let mut presence_buf = vec![]; + let mut encoder = BoolEncoder; + + assert!(encoder + .encode(Some(&true), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&true), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&false), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&false), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&true), &mut data_buf, &mut presence_buf) + .is_ok()); + + #[rustfmt::skip] + assert_eq!(data_buf, &[0x01, + 0x01, + 0x00, + 0x00, + 0x00, + 0x00, + 0x01]); + + #[rustfmt::skip] + assert_eq!(presence_buf, &[true, + true, + true, + false, + false, + true, + true]); + } + + #[test] + fn test_string_encoder() { + let mut data_buf = vec![]; + let mut presence_buf = vec![]; + let mut encoder = StringEncoder; + + assert!(encoder + .encode(Some(&"A".to_string()), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&"B".to_string()), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&"C".to_string()), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode( + Some(&"Hello, World!".to_string()), + &mut data_buf, + &mut presence_buf + ) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + + #[rustfmt::skip] + assert_eq!(data_buf, &[0x01, // length + b'A', // A + 0x00, // None + 0x01, // length + b'B', // B + 0x00, // None + 0x01, // length + b'C', // C + 0x00, // None + 0x0D, // length + b'H', + b'e', + b'l', + b'l', + b'o', + b',', + b' ', + b'W', + b'o', + b'r', + b'l', + b'd', + b'!', + 0x00]); // None + + #[rustfmt::skip] + assert_eq!(presence_buf, &[true, + false, + true, + false, + true, + false, + true, + false]); + } +} diff --git a/tracklib2/src/write/header.rs b/tracklib2/src/write/header.rs new file mode 100644 index 0000000..0ed851d --- /dev/null +++ b/tracklib2/src/write/header.rs @@ -0,0 +1,69 @@ +use crate::error::Result; +use std::io::Write; + +#[rustfmt::skip] +pub const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii + 0x52, // R + 0x57, // W + 0x54, // T + 0x46, // F + 0x0A, // newline + 0x1A, // ctrl-z + 0x0A]; // newline + +#[rustfmt::skip] +fn write_header(out: &mut W, file_version: u8, creator_version: u8, metadata_table_offset: u16, data_table_offset: u16) -> Result { + let mut buf = Vec::with_capacity(24); + + buf.write_all(&RWTFMAGIC)?; // 8 bytes - Magic Number + buf.write_all(&file_version.to_le_bytes())?; // 1 byte - File Version + buf.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - FV Reserve + buf.write_all(&creator_version.to_le_bytes())?; // 1 byte - Creator Version + buf.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - CV Reserve + buf.write_all(&metadata_table_offset.to_le_bytes())?; // 2 bytes - Offset to Metadata Table + buf.write_all(&data_table_offset.to_le_bytes())?; // 2 bytes - Offset to Data Table + buf.write_all(&[0x00, 0x00])?; // 2 bytes - E Reserve + buf.write_all(&crc::crc16::checksum_usb(&buf).to_le_bytes())?; // 2 bytes - Header CRC + + out.write_all(&buf)?; + Ok(buf.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_write_header() { + let mut buf = vec![]; + let written = write_header(&mut buf, 0x00, 0x00, 0x0A, 0x1A); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x89, // magic number + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x00, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x0A, // metadata table offset + 0x00, + 0x1A, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x86, // header crc + 0xB7]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } +} diff --git a/tracklib2/src/write/metadata.rs b/tracklib2/src/write/metadata.rs new file mode 100644 index 0000000..b1f3aa7 --- /dev/null +++ b/tracklib2/src/write/metadata.rs @@ -0,0 +1,290 @@ +use crate::error::{Result, TracklibError}; +use std::convert::TryFrom; +use std::io::Write; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub enum TrackType { + Trip(u32), + Route(u32), + Segment(u32), +} + +enum MetadataEntry { + TrackType(TrackType), + CreatedAt(SystemTime), +} + +impl MetadataEntry { + fn foo(&self) -> Result<(u8, Vec)> { + match self { + Self::TrackType(track_type) => { + // TrackType metadata is 5 bytes + // 1 byte for the type of track + // 4 bytes for the id of the track + let (type_tag, id): (u8, &u32) = match track_type { + TrackType::Trip(id) => (0x00, id), + TrackType::Route(id) => (0x01, id), + TrackType::Segment(id) => (0x02, id), + }; + let mut buf = Vec::with_capacity(5); + buf.write_all(&type_tag.to_le_bytes())?; + buf.write_all(&id.to_le_bytes())?; + Ok((0x00, buf)) + } + Self::CreatedAt(system_time) => { + // CreatedAt metadata is 8 bytes: seconds since epoch + let buf = system_time + .duration_since(UNIX_EPOCH)? + .as_secs() + .to_le_bytes() + .to_vec(); + Ok((0x01, buf)) + } + } + } +} + +#[rustfmt::skip] +fn write_metadata(out: &mut W, entries: Vec) -> Result { + let entry_count = u8::try_from(entries.len())?; + + let mut buf = Vec::new(); + + buf.write_all(&entry_count.to_le_bytes())?; // 1 byte - entry count + + for entry in entries { + let (entry_type, entry_bytes) = entry.foo()?; + let entry_size = u16::try_from(entry_bytes.len())?; + + buf.write_all(&entry_type.to_le_bytes())?; // 1 byte - entry type + buf.write_all(&entry_size.to_le_bytes())?; // 2 bytes - entry size + buf.write_all(&entry_bytes)?; // ? bytes - entry value + } + + buf.write_all(&crc::crc16::checksum_usb(&buf).to_le_bytes())?; // 2 bytes - crc + + out.write_all(&buf)?; + Ok(buf.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn test_write_empty_metadata() { + let mut buf = vec![]; + let written = write_metadata(&mut buf, vec![]); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x00, // zero metadata entries + 0x40, // crc + 0xBF]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } + + #[test] + fn test_only_track_type_trip() { + let mut buf = vec![]; + let written = write_metadata( + &mut buf, + vec![MetadataEntry::TrackType(TrackType::Trip(400))], + ); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x01, // one metadata entry + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x90, // four byte trip ID = 400 + 0x01, + 0x00, + 0x00, + 0xD1, // crc + 0x5F]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } + + #[test] + fn test_only_track_type_route() { + let mut buf = vec![]; + let written = write_metadata( + &mut buf, + vec![MetadataEntry::TrackType(TrackType::Route(64))], + ); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x01, // one metadata entry + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x01, // track type: route = 0x01 + 0x40, // four byte route ID = 64 + 0x00, + 0x00, + 0x00, + 0x85, // crc + 0x9F]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } + + #[test] + fn test_only_track_type_segment() { + let mut buf = vec![]; + let written = write_metadata( + &mut buf, + vec![MetadataEntry::TrackType(TrackType::Segment(u32::MAX))], + ); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x01, // one metadata entry + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x02, // track type: segment = 0x02 + 0xFF, // four byte segment ID = 4,294,967,295 + 0xFF, + 0xFF, + 0xFF, + 0xD5, // crc + 0xCB]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } + + #[test] + fn test_only_created_at_epoch() { + let mut buf = vec![]; + let written = write_metadata(&mut buf, vec![MetadataEntry::CreatedAt(UNIX_EPOCH)]); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x01, // one metadata entry + 0x01, // entry type: created_at = 0x01 + 0x08, // two byte entry size = 8 + 0x00, + 0x00, // eight byte timestamp: zero seconds elapsed + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0xE3, // crc + 0x28]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } + + #[test] + fn test_only_created_at_future() { + let mut buf = vec![]; + let the_future = UNIX_EPOCH + Duration::from_millis(u64::MAX); + let written = write_metadata(&mut buf, vec![MetadataEntry::CreatedAt(the_future)]); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x01, // one metadata entry + 0x01, // entry type: created_at = 0x01 + 0x08, // two byte entry size = 8 + 0x00, + 0xEF, // eight byte timestamp: lots of seconds elapsed + 0xA7, + 0xC6, + 0x4B, + 0x37, + 0x89, + 0x41, + 0x00, + 0x21, // crc + 0x4C]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } + + #[test] + fn test_both() { + let mut buf = vec![]; + let written = write_metadata( + &mut buf, + vec![ + MetadataEntry::TrackType(TrackType::Trip(20)), + MetadataEntry::CreatedAt(UNIX_EPOCH), + ], + ); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x02, // two metadata entries + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x14, // four byte trip ID = 20 + 0x00, + 0x00, + 0x00, + 0x01, // entry type: created_at = 0x01 + 0x08, // two byte entry size = 8 + 0x00, + 0x00, // eight byte timestamp: zero seconds elapsed + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x23, // crc + 0xD2]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } + + #[test] + fn test_duplicate_types() { + let mut buf = vec![]; + let written = write_metadata( + &mut buf, + vec![ + MetadataEntry::TrackType(TrackType::Trip(20)), + MetadataEntry::TrackType(TrackType::Trip(21)), + MetadataEntry::TrackType(TrackType::Route(22)), + ], + ); + assert!(written.is_ok()); + #[rustfmt::skip] + let expected = &[0x03, // three metadata entries + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x14, // four byte trip ID = 20 + 0x00, + 0x00, + 0x00, + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x15, // four byte trip ID = 21 + 0x00, + 0x00, + 0x00, + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x01, // track type: route = 0x01 + 0x16, // four byte route ID = 22 + 0x00, + 0x00, + 0x00, + 0xDE, // crc + 0x57]; + assert_eq!(buf, expected); + assert_eq!(written.unwrap(), expected.len()); + } +} diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs new file mode 100644 index 0000000..e4c4248 --- /dev/null +++ b/tracklib2/src/write/mod.rs @@ -0,0 +1,3 @@ +mod encoders; +mod header; +mod metadata; From dfc327e529bfbb2aa3df3d1f7ee7b4112d4196f9 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Fri, 3 Dec 2021 12:33:35 -0800 Subject: [PATCH 002/113] tracklib2 TrackWriter --- tracklib2/src/write/mod.rs | 2 + tracklib2/src/write/track.rs | 261 +++++++++++++++++++++++++++++++++++ tracklib2/src/write/types.rs | 6 + 3 files changed, 269 insertions(+) create mode 100644 tracklib2/src/write/track.rs create mode 100644 tracklib2/src/write/types.rs diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs index e4c4248..c9fda7e 100644 --- a/tracklib2/src/write/mod.rs +++ b/tracklib2/src/write/mod.rs @@ -1,3 +1,5 @@ mod encoders; mod header; mod metadata; +mod track; +mod types; diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs new file mode 100644 index 0000000..f6bdb08 --- /dev/null +++ b/tracklib2/src/write/track.rs @@ -0,0 +1,261 @@ +use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; +use super::types::FieldType; +use crate::error::Result; +use std::convert::TryFrom; +use std::io::{self, Write}; + +#[derive(Default, Debug)] +struct BufferImpl { + buf: Vec, + presence: Vec, + encoder: E, +} + +impl BufferImpl { + fn write_data(&self, out: &mut W) -> Result { + let written = io::copy(&mut io::Cursor::new(&self.buf), out)?; + // io::copy (annoyingly) returns u64 so coerce it to a usize to return + Ok(usize::try_from(written).unwrap()) + } +} + +#[derive(Debug)] +enum Buffer { + I64(BufferImpl), + Bool(BufferImpl), + String(BufferImpl), +} + +impl Buffer { + fn new(field_type: &FieldType) -> Self { + match field_type { + &FieldType::I64 => Buffer::I64(BufferImpl::default()), + &FieldType::Bool => Buffer::Bool(BufferImpl::default()), + &FieldType::String => Buffer::String(BufferImpl::default()), + } + } +} + +#[derive(Clone, Debug)] +struct FieldDescription { + name: String, + fieldtype: FieldType, +} + +struct TrackWriter { + rows_written: usize, + fields: Vec, + column_data: Vec, +} + +impl TrackWriter { + // TODO: provide a size_hint param to size buffer Vecs (at least presence) + fn new(mapping: Vec<(String, FieldType)>) -> Self { + let mut fields = Vec::with_capacity(mapping.len()); + let mut column_data = Vec::with_capacity(mapping.len()); + + for (name, fieldtype) in mapping { + column_data.push(Buffer::new(&fieldtype)); + fields.push(FieldDescription { name, fieldtype }); + } + + Self { + rows_written: 0, + fields, + column_data, + } + } + + fn fields(&self) -> &[FieldDescription] { + &self.fields + } + + fn field_description(&self, name: &str) -> Option<&FieldDescription> { + self.fields + .iter() + .enumerate() + .filter_map(|(i, field_desc)| { + if field_desc.name == name { + Some(field_desc) + } else { + None + } + }) + .next() + } + + /// mut borrow of self so only one row can be open at a time + fn open_row_builder(&mut self) -> RowBuilder { + self.rows_written += 1; // TODO: bug when you open a row builder but never write data with it? + RowBuilder::new(&self.fields, &mut self.column_data) + } + + fn write_presence_column(&self, out: &mut W) -> Result { + let mut bytes_written = 0; + let bytes_required = (self.fields.len() + 7) / 8; + + for row_i in 0..self.rows_written { + let mut entry: u64 = 0; + for (field_i, buffer) in self.column_data.iter().enumerate() { + let bit = match buffer { + Buffer::I64(buffer_impl) => { + if let Some(true) = buffer_impl.presence.get(row_i) { + 1 + } else { + 0 + } + } + Buffer::Bool(buffer_impl) => { + if let Some(true) = buffer_impl.presence.get(row_i) { + 1 + } else { + 0 + } + } + Buffer::String(buffer_impl) => { + if let Some(true) = buffer_impl.presence.get(row_i) { + 1 + } else { + 0 + } + } + }; + entry |= bit << field_i; + } + + out.write_all(&entry.to_le_bytes()[..bytes_required])?; + bytes_written += bytes_required; + } + + Ok(bytes_written) + } +} + +struct RowBuilder<'a> { + fields: &'a [FieldDescription], + column_data: &'a mut Vec, + field_index: usize, +} + +impl<'a> RowBuilder<'a> { + fn new(fields: &'a [FieldDescription], column_data: &'a mut Vec) -> Self { + Self { + fields, + column_data, + field_index: 0, + } + } + + /// mut borrow of self so only one column writer can be open at a time + fn next_column_writer(&mut self) -> Option { + let field_index = self.field_index; + self.field_index += 1; + + let maybe_field_desc = self.fields.get(field_index); + let maybe_buffer = self.column_data.get_mut(field_index); + + match (maybe_field_desc, maybe_buffer) { + (Some(field_desc), Some(Buffer::I64(ref mut buffer_impl))) => Some( + ColumnWriter::I64ColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + ), + (Some(field_desc), Some(Buffer::Bool(ref mut buffer_impl))) => Some( + ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + ), + (Some(field_desc), Some(Buffer::String(ref mut buffer_impl))) => Some( + ColumnWriter::StringColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + ), + _ => None, + } + } +} + +enum ColumnWriter<'a> { + I64ColumnWriter(ColumnWriterImpl<'a, I64Encoder>), + BoolColumnWriter(ColumnWriterImpl<'a, BoolEncoder>), + StringColumnWriter(ColumnWriterImpl<'a, StringEncoder>), +} + +// TODO: is this must_use correct? +#[must_use] +struct ColumnWriterImpl<'a, E: Encoder> { + field_description: &'a FieldDescription, + buf: &'a mut BufferImpl, +} + +impl<'a, E: Encoder> ColumnWriterImpl<'a, E> { + fn new(field_description: &'a FieldDescription, buf: &'a mut BufferImpl) -> Self { + Self { + field_description, + buf, + } + } + + fn field_description(&self) -> &FieldDescription { + &self.field_description + } + + /// Takes `self` so that only one value per column can be written + fn write(self, value: Option<&E::T>) -> Result<()> { + let BufferImpl { + ref mut encoder, + ref mut buf, + ref mut presence, + .. + } = self.buf; + encoder.encode(value, buf, presence) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_write_presence_column() { + let mut trackwriter = TrackWriter::new(vec![ + ("a".to_string(), FieldType::I64), + ("b".to_string(), FieldType::Bool), + ("c".to_string(), FieldType::String), + ]); + let mut buf = Vec::new(); + let bytes_written = trackwriter.write_presence_column(&mut buf); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), 0); + assert_eq!(buf.len(), 0); + + let m_vals = vec![Some(&42), Some(&0), None, Some(&-20)]; + let k_vals = vec![Some(&true), None, Some(&false), Some(&false)]; + let j_vals = vec![ + None, + Some("hi".to_string()), + Some("tracklib".to_string()), + Some("!".to_string()), + ]; + + for i in 0..4 { + let mut rowbuilder = trackwriter.open_row_builder(); + while let Some(cw) = rowbuilder.next_column_writer() { + match cw { + ColumnWriter::I64ColumnWriter(cwi) => { + assert!(cwi.write(m_vals[i]).is_ok()); + } + ColumnWriter::BoolColumnWriter(cwi) => { + assert!(cwi.write(k_vals[i]).is_ok()); + } + ColumnWriter::StringColumnWriter(cwi) => { + assert!(cwi.write(j_vals[i].as_ref()).is_ok()); + } + } + } + } + + let bytes_written = trackwriter.write_presence_column(&mut buf); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), 4); + assert_eq!(buf.len(), 4); + assert_eq!(buf[0], 0b00000011); + assert_eq!(buf[1], 0b00000101); + assert_eq!(buf[2], 0b00000110); + assert_eq!(buf[3], 0b00000111); + } +} diff --git a/tracklib2/src/write/types.rs b/tracklib2/src/write/types.rs new file mode 100644 index 0000000..a98a955 --- /dev/null +++ b/tracklib2/src/write/types.rs @@ -0,0 +1,6 @@ +#[derive(Debug, Clone)] +pub(crate) enum FieldType { + I64, + Bool, + String, +} From b3c094dcd6f0af1e1f2b1e7a3f06668e92a4af94 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Fri, 3 Dec 2021 14:19:30 -0800 Subject: [PATCH 003/113] move row & column writers into Section --- tracklib2/src/write/metadata.rs | 4 +- tracklib2/src/write/mod.rs | 1 + tracklib2/src/write/section.rs | 254 +++++++++++++++++++++++++++++++ tracklib2/src/write/track.rs | 261 ++------------------------------ tracklib2/src/write/types.rs | 22 +++ 5 files changed, 295 insertions(+), 247 deletions(-) create mode 100644 tracklib2/src/write/section.rs diff --git a/tracklib2/src/write/metadata.rs b/tracklib2/src/write/metadata.rs index b1f3aa7..ce221f2 100644 --- a/tracklib2/src/write/metadata.rs +++ b/tracklib2/src/write/metadata.rs @@ -9,7 +9,7 @@ pub enum TrackType { Segment(u32), } -enum MetadataEntry { +pub enum MetadataEntry { TrackType(TrackType), CreatedAt(SystemTime), } @@ -45,7 +45,7 @@ impl MetadataEntry { } #[rustfmt::skip] -fn write_metadata(out: &mut W, entries: Vec) -> Result { +pub(crate) fn write_metadata(out: &mut W, entries: Vec) -> Result { let entry_count = u8::try_from(entries.len())?; let mut buf = Vec::new(); diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs index c9fda7e..88bdefa 100644 --- a/tracklib2/src/write/mod.rs +++ b/tracklib2/src/write/mod.rs @@ -1,5 +1,6 @@ mod encoders; mod header; mod metadata; +mod section; mod track; mod types; diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs new file mode 100644 index 0000000..2ee0e87 --- /dev/null +++ b/tracklib2/src/write/section.rs @@ -0,0 +1,254 @@ +use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; +use super::types::{FieldDescription, FieldType}; +use crate::error::Result; +use std::convert::TryFrom; +use std::io::{self, Write}; + +#[derive(Default, Debug)] +struct BufferImpl { + buf: Vec, + presence: Vec, + encoder: E, +} + +impl BufferImpl { + fn write_data(&self, out: &mut W) -> Result { + let written = io::copy(&mut io::Cursor::new(&self.buf), out)?; + // io::copy (annoyingly) returns u64 so coerce it to a usize to return + Ok(usize::try_from(written).unwrap()) + } +} + +#[derive(Debug)] +enum Buffer { + I64(BufferImpl), + Bool(BufferImpl), + String(BufferImpl), +} + +impl Buffer { + fn new(field_type: &FieldType) -> Self { + match field_type { + &FieldType::I64 => Buffer::I64(BufferImpl::default()), + &FieldType::Bool => Buffer::Bool(BufferImpl::default()), + &FieldType::String => Buffer::String(BufferImpl::default()), + } + } +} + +enum SectionType { + TrackPoints, + CoursePoints, +} + +pub struct Section { + section_type: SectionType, + rows_written: usize, + fields: Vec, + column_data: Vec, +} + +impl Section { + // TODO: provide a size_hint param to size buffer Vecs (at least presence) + fn new(section_type: SectionType, mapping: Vec<(String, FieldType)>) -> Self { + let mut fields = Vec::with_capacity(mapping.len()); + let mut column_data = Vec::with_capacity(mapping.len()); + + for (name, fieldtype) in mapping { + column_data.push(Buffer::new(&fieldtype)); + fields.push(FieldDescription::new(name, fieldtype)); + } + + Self { + section_type, + rows_written: 0, + fields, + column_data, + } + } + + fn fields(&self) -> &[FieldDescription] { + &self.fields + } + + /// mut borrow of self so only one row can be open at a time + fn open_row_builder(&mut self) -> RowBuilder { + self.rows_written += 1; // TODO: bug when you open a row builder but never write data with it? + RowBuilder::new(&self.fields, &mut self.column_data) + } + + fn write_presence_column(&self, out: &mut W) -> Result { + let mut bytes_written = 0; + let bytes_required = (self.fields.len() + 7) / 8; + + for row_i in 0..self.rows_written { + let mut entry: u64 = 0; + for (field_i, buffer) in self.column_data.iter().enumerate() { + let bit = match buffer { + Buffer::I64(buffer_impl) => { + if let Some(true) = buffer_impl.presence.get(row_i) { + 1 + } else { + 0 + } + } + Buffer::Bool(buffer_impl) => { + if let Some(true) = buffer_impl.presence.get(row_i) { + 1 + } else { + 0 + } + } + Buffer::String(buffer_impl) => { + if let Some(true) = buffer_impl.presence.get(row_i) { + 1 + } else { + 0 + } + } + }; + entry |= bit << field_i; + } + + out.write_all(&entry.to_le_bytes()[..bytes_required])?; + bytes_written += bytes_required; + } + + Ok(bytes_written) + } +} + +struct RowBuilder<'a> { + fields: &'a [FieldDescription], + column_data: &'a mut Vec, + field_index: usize, +} + +impl<'a> RowBuilder<'a> { + fn new(fields: &'a [FieldDescription], column_data: &'a mut Vec) -> Self { + Self { + fields, + column_data, + field_index: 0, + } + } + + /// mut borrow of self so only one column writer can be open at a time + fn next_column_writer(&mut self) -> Option { + let field_index = self.field_index; + self.field_index += 1; + + let maybe_field_desc = self.fields.get(field_index); + let maybe_buffer = self.column_data.get_mut(field_index); + + match (maybe_field_desc, maybe_buffer) { + (Some(field_desc), Some(Buffer::I64(ref mut buffer_impl))) => Some( + ColumnWriter::I64ColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + ), + (Some(field_desc), Some(Buffer::Bool(ref mut buffer_impl))) => Some( + ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + ), + (Some(field_desc), Some(Buffer::String(ref mut buffer_impl))) => Some( + ColumnWriter::StringColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + ), + _ => None, + } + } +} + +enum ColumnWriter<'a> { + I64ColumnWriter(ColumnWriterImpl<'a, I64Encoder>), + BoolColumnWriter(ColumnWriterImpl<'a, BoolEncoder>), + StringColumnWriter(ColumnWriterImpl<'a, StringEncoder>), +} + +// TODO: is this must_use correct? +#[must_use] +struct ColumnWriterImpl<'a, E: Encoder> { + field_description: &'a FieldDescription, + buf: &'a mut BufferImpl, +} + +impl<'a, E: Encoder> ColumnWriterImpl<'a, E> { + fn new(field_description: &'a FieldDescription, buf: &'a mut BufferImpl) -> Self { + Self { + field_description, + buf, + } + } + + fn field_description(&self) -> &FieldDescription { + &self.field_description + } + + /// Takes `self` so that only one value per column can be written + fn write(self, value: Option<&E::T>) -> Result<()> { + let BufferImpl { + ref mut encoder, + ref mut buf, + ref mut presence, + .. + } = self.buf; + encoder.encode(value, buf, presence) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_write_presence_column() { + let mut section = Section::new( + SectionType::TrackPoints, + vec![ + ("a".to_string(), FieldType::I64), + ("b".to_string(), FieldType::Bool), + ("c".to_string(), FieldType::String), + ], + ); + let mut buf = Vec::new(); + let bytes_written = section.write_presence_column(&mut buf); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), 0); + assert_eq!(buf.len(), 0); + + let m_vals = vec![Some(&42), Some(&0), None, Some(&-20)]; + let k_vals = vec![Some(&true), None, Some(&false), Some(&false)]; + let j_vals = vec![ + None, + Some("hi".to_string()), + Some("tracklib".to_string()), + Some("!".to_string()), + ]; + + for i in 0..4 { + let mut rowbuilder = section.open_row_builder(); + while let Some(cw) = rowbuilder.next_column_writer() { + match cw { + ColumnWriter::I64ColumnWriter(cwi) => { + assert!(cwi.write(m_vals[i]).is_ok()); + } + ColumnWriter::BoolColumnWriter(cwi) => { + assert!(cwi.write(k_vals[i]).is_ok()); + } + ColumnWriter::StringColumnWriter(cwi) => { + assert!(cwi.write(j_vals[i].as_ref()).is_ok()); + } + } + } + } + + let bytes_written = section.write_presence_column(&mut buf); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), 4); + assert_eq!(buf.len(), 4); + assert_eq!(buf[0], 0b00000011); + assert_eq!(buf[1], 0b00000101); + assert_eq!(buf[2], 0b00000110); + assert_eq!(buf[3], 0b00000111); + } + + #[test] + fn test_multibyte_presence_column() {} +} diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index f6bdb08..1d673aa 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -1,261 +1,32 @@ -use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; -use super::types::FieldType; -use crate::error::Result; -use std::convert::TryFrom; -use std::io::{self, Write}; +use super::metadata::{self, MetadataEntry}; +use super::section::Section; +use std::io::Write; -#[derive(Default, Debug)] -struct BufferImpl { - buf: Vec, - presence: Vec, - encoder: E, +struct TrackWriter { + out: W, + metadata_entries: Vec, + sections: Vec
, } -impl BufferImpl { - fn write_data(&self, out: &mut W) -> Result { - let written = io::copy(&mut io::Cursor::new(&self.buf), out)?; - // io::copy (annoyingly) returns u64 so coerce it to a usize to return - Ok(usize::try_from(written).unwrap()) - } -} - -#[derive(Debug)] -enum Buffer { - I64(BufferImpl), - Bool(BufferImpl), - String(BufferImpl), -} - -impl Buffer { - fn new(field_type: &FieldType) -> Self { - match field_type { - &FieldType::I64 => Buffer::I64(BufferImpl::default()), - &FieldType::Bool => Buffer::Bool(BufferImpl::default()), - &FieldType::String => Buffer::String(BufferImpl::default()), - } - } -} - -#[derive(Clone, Debug)] -struct FieldDescription { - name: String, - fieldtype: FieldType, -} - -struct TrackWriter { - rows_written: usize, - fields: Vec, - column_data: Vec, -} - -impl TrackWriter { - // TODO: provide a size_hint param to size buffer Vecs (at least presence) - fn new(mapping: Vec<(String, FieldType)>) -> Self { - let mut fields = Vec::with_capacity(mapping.len()); - let mut column_data = Vec::with_capacity(mapping.len()); - - for (name, fieldtype) in mapping { - column_data.push(Buffer::new(&fieldtype)); - fields.push(FieldDescription { name, fieldtype }); - } - +impl TrackWriter { + fn new(out: W) -> Self { Self { - rows_written: 0, - fields, - column_data, + out, + metadata_entries: Vec::new(), + sections: Vec::new(), } } - fn fields(&self) -> &[FieldDescription] { - &self.fields - } - - fn field_description(&self, name: &str) -> Option<&FieldDescription> { - self.fields - .iter() - .enumerate() - .filter_map(|(i, field_desc)| { - if field_desc.name == name { - Some(field_desc) - } else { - None - } - }) - .next() + fn add_metadata_entry(&mut self, entry: metadata::MetadataEntry) { + self.metadata_entries.push(entry); } - /// mut borrow of self so only one row can be open at a time - fn open_row_builder(&mut self) -> RowBuilder { - self.rows_written += 1; // TODO: bug when you open a row builder but never write data with it? - RowBuilder::new(&self.fields, &mut self.column_data) - } - - fn write_presence_column(&self, out: &mut W) -> Result { - let mut bytes_written = 0; - let bytes_required = (self.fields.len() + 7) / 8; - - for row_i in 0..self.rows_written { - let mut entry: u64 = 0; - for (field_i, buffer) in self.column_data.iter().enumerate() { - let bit = match buffer { - Buffer::I64(buffer_impl) => { - if let Some(true) = buffer_impl.presence.get(row_i) { - 1 - } else { - 0 - } - } - Buffer::Bool(buffer_impl) => { - if let Some(true) = buffer_impl.presence.get(row_i) { - 1 - } else { - 0 - } - } - Buffer::String(buffer_impl) => { - if let Some(true) = buffer_impl.presence.get(row_i) { - 1 - } else { - 0 - } - } - }; - entry |= bit << field_i; - } - - out.write_all(&entry.to_le_bytes()[..bytes_required])?; - bytes_written += bytes_required; - } - - Ok(bytes_written) - } -} - -struct RowBuilder<'a> { - fields: &'a [FieldDescription], - column_data: &'a mut Vec, - field_index: usize, -} - -impl<'a> RowBuilder<'a> { - fn new(fields: &'a [FieldDescription], column_data: &'a mut Vec) -> Self { - Self { - fields, - column_data, - field_index: 0, - } - } - - /// mut borrow of self so only one column writer can be open at a time - fn next_column_writer(&mut self) -> Option { - let field_index = self.field_index; - self.field_index += 1; - - let maybe_field_desc = self.fields.get(field_index); - let maybe_buffer = self.column_data.get_mut(field_index); - - match (maybe_field_desc, maybe_buffer) { - (Some(field_desc), Some(Buffer::I64(ref mut buffer_impl))) => Some( - ColumnWriter::I64ColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), - ), - (Some(field_desc), Some(Buffer::Bool(ref mut buffer_impl))) => Some( - ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), - ), - (Some(field_desc), Some(Buffer::String(ref mut buffer_impl))) => Some( - ColumnWriter::StringColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), - ), - _ => None, - } - } -} - -enum ColumnWriter<'a> { - I64ColumnWriter(ColumnWriterImpl<'a, I64Encoder>), - BoolColumnWriter(ColumnWriterImpl<'a, BoolEncoder>), - StringColumnWriter(ColumnWriterImpl<'a, StringEncoder>), -} - -// TODO: is this must_use correct? -#[must_use] -struct ColumnWriterImpl<'a, E: Encoder> { - field_description: &'a FieldDescription, - buf: &'a mut BufferImpl, -} - -impl<'a, E: Encoder> ColumnWriterImpl<'a, E> { - fn new(field_description: &'a FieldDescription, buf: &'a mut BufferImpl) -> Self { - Self { - field_description, - buf, - } - } - - fn field_description(&self) -> &FieldDescription { - &self.field_description - } - - /// Takes `self` so that only one value per column can be written - fn write(self, value: Option<&E::T>) -> Result<()> { - let BufferImpl { - ref mut encoder, - ref mut buf, - ref mut presence, - .. - } = self.buf; - encoder.encode(value, buf, presence) + fn add_section(&mut self, section: Section) { + self.sections.push(section); } } #[cfg(test)] mod tests { use super::*; - - #[test] - fn test_write_presence_column() { - let mut trackwriter = TrackWriter::new(vec![ - ("a".to_string(), FieldType::I64), - ("b".to_string(), FieldType::Bool), - ("c".to_string(), FieldType::String), - ]); - let mut buf = Vec::new(); - let bytes_written = trackwriter.write_presence_column(&mut buf); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), 0); - assert_eq!(buf.len(), 0); - - let m_vals = vec![Some(&42), Some(&0), None, Some(&-20)]; - let k_vals = vec![Some(&true), None, Some(&false), Some(&false)]; - let j_vals = vec![ - None, - Some("hi".to_string()), - Some("tracklib".to_string()), - Some("!".to_string()), - ]; - - for i in 0..4 { - let mut rowbuilder = trackwriter.open_row_builder(); - while let Some(cw) = rowbuilder.next_column_writer() { - match cw { - ColumnWriter::I64ColumnWriter(cwi) => { - assert!(cwi.write(m_vals[i]).is_ok()); - } - ColumnWriter::BoolColumnWriter(cwi) => { - assert!(cwi.write(k_vals[i]).is_ok()); - } - ColumnWriter::StringColumnWriter(cwi) => { - assert!(cwi.write(j_vals[i].as_ref()).is_ok()); - } - } - } - } - - let bytes_written = trackwriter.write_presence_column(&mut buf); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), 4); - assert_eq!(buf.len(), 4); - assert_eq!(buf[0], 0b00000011); - assert_eq!(buf[1], 0b00000101); - assert_eq!(buf[2], 0b00000110); - assert_eq!(buf[3], 0b00000111); - } } diff --git a/tracklib2/src/write/types.rs b/tracklib2/src/write/types.rs index a98a955..ad3284a 100644 --- a/tracklib2/src/write/types.rs +++ b/tracklib2/src/write/types.rs @@ -4,3 +4,25 @@ pub(crate) enum FieldType { Bool, String, } + +#[derive(Clone, Debug)] +pub(crate) struct FieldDescription { + name: String, + fieldtype: FieldType, +} + +impl FieldDescription { + pub(crate) fn new(name: String, fieldtype: FieldType) -> Self { + Self { name, fieldtype } + } + + /// Get a reference to the field description's name. + pub(crate) fn name(&self) -> &str { + self.name.as_str() + } + + /// Get a reference to the field description's fieldtype. + pub(crate) fn fieldtype(&self) -> &FieldType { + &self.fieldtype + } +} From b9af9ba2822035babed3bff138114200d5eb126b Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Fri, 3 Dec 2021 16:37:42 -0800 Subject: [PATCH 004/113] writing types table --- tracklib2/src/write/section.rs | 104 +++++++++++++++++++++++++++++++-- tracklib2/src/write/types.rs | 12 +++- 2 files changed, 109 insertions(+), 7 deletions(-) diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 2ee0e87..d37d02c 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -17,6 +17,10 @@ impl BufferImpl { // io::copy (annoyingly) returns u64 so coerce it to a usize to return Ok(usize::try_from(written).unwrap()) } + + fn data_size(&self) -> usize { + self.buf.len() + } } #[derive(Debug)] @@ -116,6 +120,32 @@ impl Section { Ok(bytes_written) } + + fn write_types_table(&self, out: &mut W) -> Result { + let mut buf = Vec::new(); + + buf.write_all(&u8::try_from(self.fields.len())?.to_le_bytes())?; // 1 byte - number of entries + + for (i, field) in self.fields.iter().enumerate() { + let data_column_size = self.column_data.get(i).map(|buffer| { + match buffer { + Buffer::I64(buffer_impl) => buffer_impl.data_size(), + Buffer::Bool(buffer_impl) => buffer_impl.data_size(), + Buffer::String(buffer_impl) => buffer_impl.data_size(), + } + }).unwrap_or(0); + + buf.write_all(&field.fieldtype().type_tag().to_le_bytes())?; // 1 byte - field type tag + buf.write_all(&u8::try_from(field.name().len())?.to_le_bytes())?; // 1 byte - field name length + buf.write_all(field.name().as_bytes())?; // ? bytes - the name of this field + leb128::write::unsigned(&mut buf, u64::try_from(data_column_size)?)?; // ? bytes - leb128 column data size + } + + buf.write_all(&crc::crc16::checksum_usb(&buf).to_le_bytes())?; // 2 bytes - crc + + out.write_all(&buf)?; + Ok(buf.len()) + } } struct RowBuilder<'a> { @@ -241,14 +271,76 @@ mod tests { let bytes_written = section.write_presence_column(&mut buf); assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), 4); - assert_eq!(buf.len(), 4); - assert_eq!(buf[0], 0b00000011); - assert_eq!(buf[1], 0b00000101); - assert_eq!(buf[2], 0b00000110); - assert_eq!(buf[3], 0b00000111); + assert_eq!(bytes_written.unwrap(), buf.len()); + #[rustfmt::skip] + assert_eq!(buf, &[0b00000011, + 0b00000101, + 0b00000110, + 0b00000111]); } #[test] fn test_multibyte_presence_column() {} + + #[test] + fn test_types_table() { + let mut section = Section::new( + SectionType::TrackPoints, + vec![ + ("m".to_string(), FieldType::I64), + ("k".to_string(), FieldType::Bool), + ("long name!".to_string(), FieldType::String), + ("i".to_string(), FieldType::I64), + ], + ); + + let mut rowbuilder = section.open_row_builder(); + while let Some(cw) = rowbuilder.next_column_writer() { + match cw { + ColumnWriter::I64ColumnWriter(cwi) => { + assert!(cwi.write(Some(&500)).is_ok()); + } + ColumnWriter::BoolColumnWriter(cwi) => { + assert!(cwi.write(Some(&false)).is_ok()); + } + ColumnWriter::StringColumnWriter(cwi) => { + assert!(cwi.write(Some(&"Hello!".to_string())).is_ok()); + } + } + } + + let mut buf = Vec::new(); + let bytes_written = section.write_types_table(&mut buf); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), buf.len()); + #[rustfmt::skip] + assert_eq!(buf, &[0x04, // entry count = 4 + 0x00, // first entry type: i64 = 0 + 0x01, // name len = 1 + b'm', // name = "m" + 0x02, // data size = 2 + 0x05, // second entry type: bool = 5 + 0x01, // name len = 1 + b'k', // name = "k" + 0x01, // data size = 1 + 0x04, // third entry type: string = 4 + 0x0A, // name len = 10 + b'l', // name = "long name!" + b'o', + b'n', + b'g', + b' ', + b'n', + b'a', + b'm', + b'e', + b'!', + 0x07, // data size = 7 ("Hello!" + leb128 length prefix) + 0x00, // fourth entry type: i64 = 0 + 0x01, // name len = 1 + b'i', // name = "i" + 0x02, // data size = 2 + 0x47, // crc + 0x13]); + } } diff --git a/tracklib2/src/write/types.rs b/tracklib2/src/write/types.rs index ad3284a..bdcb3bc 100644 --- a/tracklib2/src/write/types.rs +++ b/tracklib2/src/write/types.rs @@ -1,8 +1,18 @@ #[derive(Debug, Clone)] pub(crate) enum FieldType { I64, - Bool, String, + Bool, +} + +impl FieldType { + pub(crate) fn type_tag(&self) -> u8 { + match self { + Self::I64 => 0x00, + Self::String => 0x04, + Self::Bool => 0x05, + } + } } #[derive(Clone, Debug)] From f9e8813e9a1174c5e7911857da68c4cc6d2946b0 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Fri, 3 Dec 2021 19:07:41 -0800 Subject: [PATCH 005/113] make a bunch of things pub --- tracklib2/src/lib.rs | 4 ++-- tracklib2/src/write/encoders.rs | 8 ++++---- tracklib2/src/write/header.rs | 4 ++-- tracklib2/src/write/mod.rs | 10 +++++----- tracklib2/src/write/section.rs | 20 ++++++++++---------- tracklib2/src/write/types.rs | 12 +++++------- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/tracklib2/src/lib.rs b/tracklib2/src/lib.rs index 5b6f406..e6d4e40 100644 --- a/tracklib2/src/lib.rs +++ b/tracklib2/src/lib.rs @@ -1,2 +1,2 @@ -mod write; -mod error; +pub mod error; +pub mod write; diff --git a/tracklib2/src/write/encoders.rs b/tracklib2/src/write/encoders.rs index ebc5651..81a10cd 100644 --- a/tracklib2/src/write/encoders.rs +++ b/tracklib2/src/write/encoders.rs @@ -2,7 +2,7 @@ use crate::error::Result; use std::convert::TryFrom; use std::io::Write; -pub(crate) trait Encoder: Default { +pub trait Encoder: Default { type T; fn encode( &mut self, @@ -13,7 +13,7 @@ pub(crate) trait Encoder: Default { } #[derive(Debug, Default)] -pub(crate) struct I64Encoder { +pub struct I64Encoder { prev: i64, } @@ -44,7 +44,7 @@ impl Encoder for I64Encoder { } #[derive(Debug, Default)] -pub(crate) struct BoolEncoder; +pub struct BoolEncoder; impl Encoder for BoolEncoder { type T = bool; @@ -63,7 +63,7 @@ impl Encoder for BoolEncoder { } #[derive(Debug, Default)] -pub(crate) struct StringEncoder; +pub struct StringEncoder; impl Encoder for StringEncoder { type T = String; diff --git a/tracklib2/src/write/header.rs b/tracklib2/src/write/header.rs index 0ed851d..b09b58a 100644 --- a/tracklib2/src/write/header.rs +++ b/tracklib2/src/write/header.rs @@ -12,7 +12,7 @@ pub const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii 0x0A]; // newline #[rustfmt::skip] -fn write_header(out: &mut W, file_version: u8, creator_version: u8, metadata_table_offset: u16, data_table_offset: u16) -> Result { +pub(crate) fn write_header(out: &mut W, file_version: u8, creator_version: u8, metadata_table_offset: u16, data_table_offset: u16) -> Result { let mut buf = Vec::with_capacity(24); buf.write_all(&RWTFMAGIC)?; // 8 bytes - Magic Number @@ -24,7 +24,7 @@ fn write_header(out: &mut W, file_version: u8, creator_version: u8, me buf.write_all(&data_table_offset.to_le_bytes())?; // 2 bytes - Offset to Data Table buf.write_all(&[0x00, 0x00])?; // 2 bytes - E Reserve buf.write_all(&crc::crc16::checksum_usb(&buf).to_le_bytes())?; // 2 bytes - Header CRC - + out.write_all(&buf)?; Ok(buf.len()) } diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs index 88bdefa..8d63115 100644 --- a/tracklib2/src/write/mod.rs +++ b/tracklib2/src/write/mod.rs @@ -1,6 +1,6 @@ -mod encoders; +pub mod encoders; mod header; -mod metadata; -mod section; -mod track; -mod types; +pub mod metadata; +pub mod section; +pub mod track; +pub mod types; diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index d37d02c..c9494f0 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -40,7 +40,7 @@ impl Buffer { } } -enum SectionType { +pub enum SectionType { TrackPoints, CoursePoints, } @@ -54,7 +54,7 @@ pub struct Section { impl Section { // TODO: provide a size_hint param to size buffer Vecs (at least presence) - fn new(section_type: SectionType, mapping: Vec<(String, FieldType)>) -> Self { + pub fn new(section_type: SectionType, mapping: Vec<(String, FieldType)>) -> Self { let mut fields = Vec::with_capacity(mapping.len()); let mut column_data = Vec::with_capacity(mapping.len()); @@ -71,12 +71,12 @@ impl Section { } } - fn fields(&self) -> &[FieldDescription] { + pub fn fields(&self) -> &[FieldDescription] { &self.fields } /// mut borrow of self so only one row can be open at a time - fn open_row_builder(&mut self) -> RowBuilder { + pub fn open_row_builder(&mut self) -> RowBuilder { self.rows_written += 1; // TODO: bug when you open a row builder but never write data with it? RowBuilder::new(&self.fields, &mut self.column_data) } @@ -148,7 +148,7 @@ impl Section { } } -struct RowBuilder<'a> { +pub struct RowBuilder<'a> { fields: &'a [FieldDescription], column_data: &'a mut Vec, field_index: usize, @@ -164,7 +164,7 @@ impl<'a> RowBuilder<'a> { } /// mut borrow of self so only one column writer can be open at a time - fn next_column_writer(&mut self) -> Option { + pub fn next_column_writer(&mut self) -> Option { let field_index = self.field_index; self.field_index += 1; @@ -186,7 +186,7 @@ impl<'a> RowBuilder<'a> { } } -enum ColumnWriter<'a> { +pub enum ColumnWriter<'a> { I64ColumnWriter(ColumnWriterImpl<'a, I64Encoder>), BoolColumnWriter(ColumnWriterImpl<'a, BoolEncoder>), StringColumnWriter(ColumnWriterImpl<'a, StringEncoder>), @@ -194,7 +194,7 @@ enum ColumnWriter<'a> { // TODO: is this must_use correct? #[must_use] -struct ColumnWriterImpl<'a, E: Encoder> { +pub struct ColumnWriterImpl<'a, E: Encoder> { field_description: &'a FieldDescription, buf: &'a mut BufferImpl, } @@ -207,12 +207,12 @@ impl<'a, E: Encoder> ColumnWriterImpl<'a, E> { } } - fn field_description(&self) -> &FieldDescription { + pub fn field_description(&self) -> &FieldDescription { &self.field_description } /// Takes `self` so that only one value per column can be written - fn write(self, value: Option<&E::T>) -> Result<()> { + pub fn write(self, value: Option<&E::T>) -> Result<()> { let BufferImpl { ref mut encoder, ref mut buf, diff --git a/tracklib2/src/write/types.rs b/tracklib2/src/write/types.rs index bdcb3bc..283b1b1 100644 --- a/tracklib2/src/write/types.rs +++ b/tracklib2/src/write/types.rs @@ -1,5 +1,5 @@ #[derive(Debug, Clone)] -pub(crate) enum FieldType { +pub enum FieldType { I64, String, Bool, @@ -16,23 +16,21 @@ impl FieldType { } #[derive(Clone, Debug)] -pub(crate) struct FieldDescription { +pub struct FieldDescription { name: String, fieldtype: FieldType, } impl FieldDescription { - pub(crate) fn new(name: String, fieldtype: FieldType) -> Self { + pub fn new(name: String, fieldtype: FieldType) -> Self { Self { name, fieldtype } } - /// Get a reference to the field description's name. - pub(crate) fn name(&self) -> &str { + pub fn name(&self) -> &str { self.name.as_str() } - /// Get a reference to the field description's fieldtype. - pub(crate) fn fieldtype(&self) -> &FieldType { + pub fn fieldtype(&self) -> &FieldType { &self.fieldtype } } From ea0c21e16fac858bf9b5c5422f0080bfe560e506 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 12:31:43 -0800 Subject: [PATCH 006/113] move RWTFMAGIC to a new consts module --- tracklib2/src/consts.rs | 9 +++++++++ tracklib2/src/lib.rs | 1 + tracklib2/src/write/header.rs | 11 +---------- 3 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 tracklib2/src/consts.rs diff --git a/tracklib2/src/consts.rs b/tracklib2/src/consts.rs new file mode 100644 index 0000000..a1b3181 --- /dev/null +++ b/tracklib2/src/consts.rs @@ -0,0 +1,9 @@ +#[rustfmt::skip] +pub(crate) const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii + 0x52, // R + 0x57, // W + 0x54, // T + 0x46, // F + 0x0A, // newline + 0x1A, // ctrl-z + 0x0A]; // newline diff --git a/tracklib2/src/lib.rs b/tracklib2/src/lib.rs index e6d4e40..32d4f94 100644 --- a/tracklib2/src/lib.rs +++ b/tracklib2/src/lib.rs @@ -1,2 +1,3 @@ pub mod error; +mod consts; pub mod write; diff --git a/tracklib2/src/write/header.rs b/tracklib2/src/write/header.rs index b09b58a..f1bbbc6 100644 --- a/tracklib2/src/write/header.rs +++ b/tracklib2/src/write/header.rs @@ -1,16 +1,7 @@ +use crate::consts::RWTFMAGIC; use crate::error::Result; use std::io::Write; -#[rustfmt::skip] -pub const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii - 0x52, // R - 0x57, // W - 0x54, // T - 0x46, // F - 0x0A, // newline - 0x1A, // ctrl-z - 0x0A]; // newline - #[rustfmt::skip] pub(crate) fn write_header(out: &mut W, file_version: u8, creator_version: u8, metadata_table_offset: u16, data_table_offset: u16) -> Result { let mut buf = Vec::with_capacity(24); From bcfc73836cf8c5e1f0eff5aeeb5ff639872d036b Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 12:54:45 -0800 Subject: [PATCH 007/113] upgrade crc dependency --- tracklib2/Cargo.lock | 28 +++++++++++++++------------- tracklib2/Cargo.toml | 2 +- tracklib2/src/consts.rs | 3 +++ tracklib2/src/write/header.rs | 4 ++-- tracklib2/src/write/metadata.rs | 3 ++- tracklib2/src/write/section.rs | 3 ++- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/tracklib2/Cargo.lock b/tracklib2/Cargo.lock index 11e115e..7a99765 100644 --- a/tracklib2/Cargo.lock +++ b/tracklib2/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "atty" version = "0.2.14" @@ -25,9 +27,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "bitvec" -version = "0.19.4" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81" +checksum = "55f93d0ef3363c364d5976646a38f04cf67cfe1d4c8d160cdea02cab2c116b33" dependencies = [ "funty", "radium", @@ -47,12 +49,6 @@ dependencies = [ "serde", ] -[[package]] -name = "build_const" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" - [[package]] name = "bumpalo" version = "3.4.0" @@ -105,13 +101,19 @@ checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" [[package]] name = "crc" -version = "1.8.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" dependencies = [ - "build_const", + "crc-catalog", ] +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + [[package]] name = "criterion" version = "0.3.3" @@ -526,9 +528,9 @@ dependencies = [ [[package]] name = "tap" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36474e732d1affd3a6ed582781b3683df3d0563714c59c39591e8ff707cf078e" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "textwrap" diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml index f58eb9e..ffa029a 100644 --- a/tracklib2/Cargo.toml +++ b/tracklib2/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Dan Larkin "] edition = "2018" [dependencies] -crc = "1.8" +crc = "2.1" leb128 = "0.2" nom = {version = "6.0", default_features = false, features = ["alloc"]} thiserror = "1.0" diff --git a/tracklib2/src/consts.rs b/tracklib2/src/consts.rs index a1b3181..0b0156f 100644 --- a/tracklib2/src/consts.rs +++ b/tracklib2/src/consts.rs @@ -1,3 +1,6 @@ +pub(crate) const CRC16: crc::Crc = crc::Crc::::new(&crc::CRC_16_USB); +pub(crate) const CRC32: crc::Crc = crc::Crc::::new(&crc::CRC_32_BZIP2); + #[rustfmt::skip] pub(crate) const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii 0x52, // R diff --git a/tracklib2/src/write/header.rs b/tracklib2/src/write/header.rs index f1bbbc6..14b101c 100644 --- a/tracklib2/src/write/header.rs +++ b/tracklib2/src/write/header.rs @@ -1,4 +1,4 @@ -use crate::consts::RWTFMAGIC; +use crate::consts::{CRC16, RWTFMAGIC}; use crate::error::Result; use std::io::Write; @@ -14,7 +14,7 @@ pub(crate) fn write_header(out: &mut W, file_version: u8, creator_vers buf.write_all(&metadata_table_offset.to_le_bytes())?; // 2 bytes - Offset to Metadata Table buf.write_all(&data_table_offset.to_le_bytes())?; // 2 bytes - Offset to Data Table buf.write_all(&[0x00, 0x00])?; // 2 bytes - E Reserve - buf.write_all(&crc::crc16::checksum_usb(&buf).to_le_bytes())?; // 2 bytes - Header CRC + buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - Header CRC out.write_all(&buf)?; Ok(buf.len()) diff --git a/tracklib2/src/write/metadata.rs b/tracklib2/src/write/metadata.rs index ce221f2..865414c 100644 --- a/tracklib2/src/write/metadata.rs +++ b/tracklib2/src/write/metadata.rs @@ -1,3 +1,4 @@ +use crate::consts::CRC16; use crate::error::{Result, TracklibError}; use std::convert::TryFrom; use std::io::Write; @@ -61,7 +62,7 @@ pub(crate) fn write_metadata(out: &mut W, entries: Vec) buf.write_all(&entry_bytes)?; // ? bytes - entry value } - buf.write_all(&crc::crc16::checksum_usb(&buf).to_le_bytes())?; // 2 bytes - crc + buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - crc out.write_all(&buf)?; Ok(buf.len()) diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index c9494f0..7165a03 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -1,5 +1,6 @@ use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; use super::types::{FieldDescription, FieldType}; +use crate::consts::{CRC16, CRC32}; use crate::error::Result; use std::convert::TryFrom; use std::io::{self, Write}; @@ -141,7 +142,7 @@ impl Section { leb128::write::unsigned(&mut buf, u64::try_from(data_column_size)?)?; // ? bytes - leb128 column data size } - buf.write_all(&crc::crc16::checksum_usb(&buf).to_le_bytes())?; // 2 bytes - crc + buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - crc out.write_all(&buf)?; Ok(buf.len()) From 0bd320ee80111a5dbc2b5aa3ced6e14134e206e6 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 13:07:18 -0800 Subject: [PATCH 008/113] create crate::types and put things there --- tracklib2/src/lib.rs | 1 + tracklib2/src/{write => }/types.rs | 23 +++++++++++++---------- tracklib2/src/write/metadata.rs | 30 +++++++----------------------- tracklib2/src/write/mod.rs | 1 - tracklib2/src/write/section.rs | 12 +++++++++++- 5 files changed, 32 insertions(+), 35 deletions(-) rename tracklib2/src/{write => }/types.rs (69%) diff --git a/tracklib2/src/lib.rs b/tracklib2/src/lib.rs index 32d4f94..9e7578c 100644 --- a/tracklib2/src/lib.rs +++ b/tracklib2/src/lib.rs @@ -1,3 +1,4 @@ pub mod error; mod consts; +pub mod types; pub mod write; diff --git a/tracklib2/src/write/types.rs b/tracklib2/src/types.rs similarity index 69% rename from tracklib2/src/write/types.rs rename to tracklib2/src/types.rs index 283b1b1..c75a893 100644 --- a/tracklib2/src/write/types.rs +++ b/tracklib2/src/types.rs @@ -5,16 +5,6 @@ pub enum FieldType { Bool, } -impl FieldType { - pub(crate) fn type_tag(&self) -> u8 { - match self { - Self::I64 => 0x00, - Self::String => 0x04, - Self::Bool => 0x05, - } - } -} - #[derive(Clone, Debug)] pub struct FieldDescription { name: String, @@ -34,3 +24,16 @@ impl FieldDescription { &self.fieldtype } } + +#[derive(Debug, PartialEq)] +pub enum TrackType { + Trip(u32), + Route(u32), + Segment(u32), +} + +#[derive(Debug, PartialEq)] +pub enum MetadataEntry { + TrackType(TrackType), + CreatedAt(u64), +} diff --git a/tracklib2/src/write/metadata.rs b/tracklib2/src/write/metadata.rs index 865414c..9bfec1b 100644 --- a/tracklib2/src/write/metadata.rs +++ b/tracklib2/src/write/metadata.rs @@ -1,22 +1,11 @@ use crate::consts::CRC16; use crate::error::{Result, TracklibError}; +use crate::types::{MetadataEntry, TrackType}; use std::convert::TryFrom; use std::io::Write; -use std::time::{SystemTime, UNIX_EPOCH}; - -pub enum TrackType { - Trip(u32), - Route(u32), - Segment(u32), -} - -pub enum MetadataEntry { - TrackType(TrackType), - CreatedAt(SystemTime), -} impl MetadataEntry { - fn foo(&self) -> Result<(u8, Vec)> { + fn write(&self) -> Result<(u8, Vec)> { match self { Self::TrackType(track_type) => { // TrackType metadata is 5 bytes @@ -32,14 +21,9 @@ impl MetadataEntry { buf.write_all(&id.to_le_bytes())?; Ok((0x00, buf)) } - Self::CreatedAt(system_time) => { + Self::CreatedAt(seconds_since_epoch) => { // CreatedAt metadata is 8 bytes: seconds since epoch - let buf = system_time - .duration_since(UNIX_EPOCH)? - .as_secs() - .to_le_bytes() - .to_vec(); - Ok((0x01, buf)) + Ok((0x01, seconds_since_epoch.to_le_bytes().to_vec())) } } } @@ -54,7 +38,7 @@ pub(crate) fn write_metadata(out: &mut W, entries: Vec) buf.write_all(&entry_count.to_le_bytes())?; // 1 byte - entry count for entry in entries { - let (entry_type, entry_bytes) = entry.foo()?; + let (entry_type, entry_bytes) = entry.write()?; let entry_size = u16::try_from(entry_bytes.len())?; buf.write_all(&entry_type.to_le_bytes())?; // 1 byte - entry type @@ -185,7 +169,7 @@ mod tests { #[test] fn test_only_created_at_future() { let mut buf = vec![]; - let the_future = UNIX_EPOCH + Duration::from_millis(u64::MAX); + let the_future = Duration::from_millis(u64::MAX).as_secs(); let written = write_metadata(&mut buf, vec![MetadataEntry::CreatedAt(the_future)]); assert!(written.is_ok()); #[rustfmt::skip] @@ -214,7 +198,7 @@ mod tests { &mut buf, vec![ MetadataEntry::TrackType(TrackType::Trip(20)), - MetadataEntry::CreatedAt(UNIX_EPOCH), + MetadataEntry::CreatedAt(0), ], ); assert!(written.is_ok()); diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs index 8d63115..6f1e869 100644 --- a/tracklib2/src/write/mod.rs +++ b/tracklib2/src/write/mod.rs @@ -3,4 +3,3 @@ mod header; pub mod metadata; pub mod section; pub mod track; -pub mod types; diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 7165a03..5a25b76 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -1,10 +1,20 @@ use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; -use super::types::{FieldDescription, FieldType}; use crate::consts::{CRC16, CRC32}; use crate::error::Result; +use crate::types::{FieldDescription, FieldType}; use std::convert::TryFrom; use std::io::{self, Write}; +impl FieldType { + pub(crate) fn type_tag(&self) -> u8 { + match self { + Self::I64 => 0x00, + Self::String => 0x04, + Self::Bool => 0x05, + } + } +} + #[derive(Default, Debug)] struct BufferImpl { buf: Vec, From 5c1c3f31edb35ee6b6157f6bf89be75a21cfc7bd Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 13:08:22 -0800 Subject: [PATCH 009/113] write_metadata now takes a slice of MetadataEntry, not a Vec --- tracklib2/src/write/metadata.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tracklib2/src/write/metadata.rs b/tracklib2/src/write/metadata.rs index 9bfec1b..fbce75c 100644 --- a/tracklib2/src/write/metadata.rs +++ b/tracklib2/src/write/metadata.rs @@ -30,7 +30,7 @@ impl MetadataEntry { } #[rustfmt::skip] -pub(crate) fn write_metadata(out: &mut W, entries: Vec) -> Result { +pub(crate) fn write_metadata(out: &mut W, entries: &[MetadataEntry]) -> Result { let entry_count = u8::try_from(entries.len())?; let mut buf = Vec::new(); @@ -60,7 +60,7 @@ mod tests { #[test] fn test_write_empty_metadata() { let mut buf = vec![]; - let written = write_metadata(&mut buf, vec![]); + let written = write_metadata(&mut buf, &[]); assert!(written.is_ok()); #[rustfmt::skip] let expected = &[0x00, // zero metadata entries @@ -73,10 +73,7 @@ mod tests { #[test] fn test_only_track_type_trip() { let mut buf = vec![]; - let written = write_metadata( - &mut buf, - vec![MetadataEntry::TrackType(TrackType::Trip(400))], - ); + let written = write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Trip(400))]); assert!(written.is_ok()); #[rustfmt::skip] let expected = &[0x01, // one metadata entry @@ -97,10 +94,7 @@ mod tests { #[test] fn test_only_track_type_route() { let mut buf = vec![]; - let written = write_metadata( - &mut buf, - vec![MetadataEntry::TrackType(TrackType::Route(64))], - ); + let written = write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Route(64))]); assert!(written.is_ok()); #[rustfmt::skip] let expected = &[0x01, // one metadata entry @@ -123,7 +117,7 @@ mod tests { let mut buf = vec![]; let written = write_metadata( &mut buf, - vec![MetadataEntry::TrackType(TrackType::Segment(u32::MAX))], + &[MetadataEntry::TrackType(TrackType::Segment(u32::MAX))], ); assert!(written.is_ok()); #[rustfmt::skip] @@ -145,7 +139,7 @@ mod tests { #[test] fn test_only_created_at_epoch() { let mut buf = vec![]; - let written = write_metadata(&mut buf, vec![MetadataEntry::CreatedAt(UNIX_EPOCH)]); + let written = write_metadata(&mut buf, &[MetadataEntry::CreatedAt(0)]); assert!(written.is_ok()); #[rustfmt::skip] let expected = &[0x01, // one metadata entry @@ -170,7 +164,7 @@ mod tests { fn test_only_created_at_future() { let mut buf = vec![]; let the_future = Duration::from_millis(u64::MAX).as_secs(); - let written = write_metadata(&mut buf, vec![MetadataEntry::CreatedAt(the_future)]); + let written = write_metadata(&mut buf, &[MetadataEntry::CreatedAt(the_future)]); assert!(written.is_ok()); #[rustfmt::skip] let expected = &[0x01, // one metadata entry @@ -196,7 +190,7 @@ mod tests { let mut buf = vec![]; let written = write_metadata( &mut buf, - vec![ + &[ MetadataEntry::TrackType(TrackType::Trip(20)), MetadataEntry::CreatedAt(0), ], @@ -234,7 +228,7 @@ mod tests { let mut buf = vec![]; let written = write_metadata( &mut buf, - vec![ + &[ MetadataEntry::TrackType(TrackType::Trip(20)), MetadataEntry::TrackType(TrackType::Trip(21)), MetadataEntry::TrackType(TrackType::Route(22)), From 434352630954a5e284d9ef3447f5021860b9578a Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 13:30:57 -0800 Subject: [PATCH 010/113] move header file_version and creator_version to crate::consts --- tracklib2/src/consts.rs | 3 +++ tracklib2/src/write/header.rs | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tracklib2/src/consts.rs b/tracklib2/src/consts.rs index 0b0156f..9710c08 100644 --- a/tracklib2/src/consts.rs +++ b/tracklib2/src/consts.rs @@ -10,3 +10,6 @@ pub(crate) const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii 0x0A, // newline 0x1A, // ctrl-z 0x0A]; // newline + +pub(crate) const RWTF_FILE_VERSION: u8 = 0x01; +pub(crate) const RWTF_CREATOR_VERSION: u8 = 0x00; diff --git a/tracklib2/src/write/header.rs b/tracklib2/src/write/header.rs index 14b101c..ae3ab8e 100644 --- a/tracklib2/src/write/header.rs +++ b/tracklib2/src/write/header.rs @@ -1,15 +1,15 @@ -use crate::consts::{CRC16, RWTFMAGIC}; +use crate::consts::{CRC16, RWTFMAGIC, RWTF_CREATOR_VERSION, RWTF_FILE_VERSION}; use crate::error::Result; use std::io::Write; #[rustfmt::skip] -pub(crate) fn write_header(out: &mut W, file_version: u8, creator_version: u8, metadata_table_offset: u16, data_table_offset: u16) -> Result { +pub(crate) fn write_header(out: &mut W, metadata_table_offset: u16, data_table_offset: u16) -> Result { let mut buf = Vec::with_capacity(24); buf.write_all(&RWTFMAGIC)?; // 8 bytes - Magic Number - buf.write_all(&file_version.to_le_bytes())?; // 1 byte - File Version + buf.write_all(&RWTF_FILE_VERSION.to_le_bytes())?; // 1 byte - File Version buf.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - FV Reserve - buf.write_all(&creator_version.to_le_bytes())?; // 1 byte - Creator Version + buf.write_all(&RWTF_CREATOR_VERSION.to_le_bytes())?; // 1 byte - Creator Version buf.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - CV Reserve buf.write_all(&metadata_table_offset.to_le_bytes())?; // 2 bytes - Offset to Metadata Table buf.write_all(&data_table_offset.to_le_bytes())?; // 2 bytes - Offset to Data Table @@ -27,7 +27,7 @@ mod tests { #[test] fn test_write_header() { let mut buf = vec![]; - let written = write_header(&mut buf, 0x00, 0x00, 0x0A, 0x1A); + let written = write_header(&mut buf, 0x0A, 0x1A); assert!(written.is_ok()); #[rustfmt::skip] let expected = &[0x89, // magic number @@ -38,7 +38,7 @@ mod tests { 0x0A, 0x1A, 0x0A, - 0x00, // file version + 0x01, // file version 0x00, // fv reserve 0x00, 0x00, @@ -53,7 +53,7 @@ mod tests { 0x00, // e reserve 0x00, 0x86, // header crc - 0xB7]; + 0x76]; assert_eq!(buf, expected); assert_eq!(written.unwrap(), expected.len()); } From af0fe5de96e26645544b66ee67610947a7816035 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 13:50:56 -0800 Subject: [PATCH 011/113] don't write a crc after types table It's covered by the crc for the whole data section --- tracklib2/src/write/section.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 5a25b76..0bc56da 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -132,19 +132,22 @@ impl Section { Ok(bytes_written) } + #[rustfmt::skip] fn write_types_table(&self, out: &mut W) -> Result { let mut buf = Vec::new(); buf.write_all(&u8::try_from(self.fields.len())?.to_le_bytes())?; // 1 byte - number of entries for (i, field) in self.fields.iter().enumerate() { - let data_column_size = self.column_data.get(i).map(|buffer| { - match buffer { + let data_column_size = self + .column_data + .get(i) + .map(|buffer| match buffer { Buffer::I64(buffer_impl) => buffer_impl.data_size(), Buffer::Bool(buffer_impl) => buffer_impl.data_size(), Buffer::String(buffer_impl) => buffer_impl.data_size(), - } - }).unwrap_or(0); + }) + .unwrap_or(0); buf.write_all(&field.fieldtype().type_tag().to_le_bytes())?; // 1 byte - field type tag buf.write_all(&u8::try_from(field.name().len())?.to_le_bytes())?; // 1 byte - field name length @@ -152,7 +155,6 @@ impl Section { leb128::write::unsigned(&mut buf, u64::try_from(data_column_size)?)?; // ? bytes - leb128 column data size } - buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - crc out.write_all(&buf)?; Ok(buf.len()) @@ -350,8 +352,6 @@ mod tests { 0x00, // fourth entry type: i64 = 0 0x01, // name len = 1 b'i', // name = "i" - 0x02, // data size = 2 - 0x47, // crc - 0x13]); + 0x02]); // data size = 2 } } From df9104968239ecd65ef71de5f2c115ef9baf0b59 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 14:41:35 -0800 Subject: [PATCH 012/113] Section::write --- tracklib2/Cargo.lock | 6 + tracklib2/Cargo.toml | 1 + tracklib2/src/write/section.rs | 226 ++++++++++++++++++++++++++++++++- 3 files changed, 231 insertions(+), 2 deletions(-) diff --git a/tracklib2/Cargo.lock b/tracklib2/Cargo.lock index 7a99765..39d5cb4 100644 --- a/tracklib2/Cargo.lock +++ b/tracklib2/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "atty" version = "0.2.14" diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml index ffa029a..9400809 100644 --- a/tracklib2/Cargo.toml +++ b/tracklib2/Cargo.toml @@ -11,6 +11,7 @@ nom = {version = "6.0", default_features = false, features = ["alloc"]} thiserror = "1.0" [dev-dependencies] +assert_matches = "1.5" criterion = "0.3" [[bench]] diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 0bc56da..3d8d182 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -6,7 +6,7 @@ use std::convert::TryFrom; use std::io::{self, Write}; impl FieldType { - pub(crate) fn type_tag(&self) -> u8 { + fn type_tag(&self) -> u8 { match self { Self::I64 => 0x00, Self::String => 0x04, @@ -56,6 +56,15 @@ pub enum SectionType { CoursePoints, } +impl SectionType { + fn type_tag(&self) -> u8 { + match self { + Self::TrackPoints => 0x00, + Self::CoursePoints => 0x01, + } + } +} + pub struct Section { section_type: SectionType, rows_written: usize, @@ -155,6 +164,28 @@ impl Section { leb128::write::unsigned(&mut buf, u64::try_from(data_column_size)?)?; // ? bytes - leb128 column data size } + out.write_all(&buf)?; + Ok(buf.len()) + } + + #[rustfmt::skip] + pub(crate) fn write(&self, out: &mut W) -> Result { + let mut buf = Vec::new(); + + buf.write_all(&self.section_type.type_tag().to_le_bytes())?; // 1 byte - section type + buf.write_all(&u32::try_from(self.rows_written)?.to_le_bytes())?; // 4 bytes - number of points in this section + self.write_types_table(&mut buf)?; // ? bytes - types table + self.write_presence_column(&mut buf)?; // ? bytes - presence column + + for buffer in self.column_data.iter() { + match buffer { + Buffer::I64(buffer_impl) => buffer_impl.write_data(&mut buf)?, // \ + Buffer::Bool(buffer_impl) => buffer_impl.write_data(&mut buf)?, // \ + Buffer::String(buffer_impl) => buffer_impl.write_data(&mut buf)?, // > ? bytes - data column + }; + } + + buf.write_all(&CRC32.checksum(&buf).to_le_bytes())?; // 4 bytes - crc out.write_all(&buf)?; Ok(buf.len()) @@ -199,12 +230,14 @@ impl<'a> RowBuilder<'a> { } } +#[derive(Debug)] pub enum ColumnWriter<'a> { I64ColumnWriter(ColumnWriterImpl<'a, I64Encoder>), BoolColumnWriter(ColumnWriterImpl<'a, BoolEncoder>), StringColumnWriter(ColumnWriterImpl<'a, StringEncoder>), } +#[derive(Debug)] // TODO: is this must_use correct? #[must_use] pub struct ColumnWriterImpl<'a, E: Encoder> { @@ -239,6 +272,8 @@ impl<'a, E: Encoder> ColumnWriterImpl<'a, E> { #[cfg(test)] mod tests { use super::*; + use assert_matches::assert_matches; + use std::collections::HashMap; #[test] fn test_write_presence_column() { @@ -293,7 +328,51 @@ mod tests { } #[test] - fn test_multibyte_presence_column() {} + fn test_multibyte_presence_column() { + let mut section = Section::new( + SectionType::TrackPoints, + vec![ + ("1".to_string(), FieldType::Bool), + ("2".to_string(), FieldType::Bool), + ("3".to_string(), FieldType::Bool), + ("4".to_string(), FieldType::Bool), + ("5".to_string(), FieldType::Bool), + ("6".to_string(), FieldType::Bool), + ("7".to_string(), FieldType::Bool), + ("8".to_string(), FieldType::Bool), + ("9".to_string(), FieldType::Bool), + ("10".to_string(), FieldType::Bool), + ("11".to_string(), FieldType::Bool), + ("12".to_string(), FieldType::Bool), + ("13".to_string(), FieldType::Bool), + ("14".to_string(), FieldType::Bool), + ("15".to_string(), FieldType::Bool), + ("16".to_string(), FieldType::Bool), + ("17".to_string(), FieldType::Bool), + ("18".to_string(), FieldType::Bool), + ("19".to_string(), FieldType::Bool), + ("20".to_string(), FieldType::Bool), + ], + ); + + for i in 0..2 { + let mut rowbuilder = section.open_row_builder(); + while let Some(cw) = rowbuilder.next_column_writer() { + match cw { + ColumnWriter::BoolColumnWriter(cwi) => assert!(cwi.write(Some(&true)).is_ok()), + _ => assert!(false, "unexpected column writer type here"), + } + } + } + + let mut buf = Vec::new(); + let bytes_written = section.write_presence_column(&mut buf); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), buf.len()); + #[rustfmt::skip] + assert_eq!(buf, &[0b11111111, 0b11111111, 0b00001111, + 0b11111111, 0b11111111, 0b00001111]); + } #[test] fn test_types_table() { @@ -354,4 +433,147 @@ mod tests { b'i', // name = "i" 0x02]); // data size = 2 } + + #[test] + fn test_writing_a_section() { + enum V { + I64(i64), + Bool(bool), + String(String), + } + + let mut v = Vec::new(); + let mut h = HashMap::new(); + h.insert("a", V::I64(1)); + h.insert("b", V::Bool(false)); + h.insert("c", V::String("Ride".to_string())); + v.push(h); + let mut h = HashMap::new(); + h.insert("a", V::I64(2)); + h.insert("c", V::String("with".to_string())); + v.push(h); + let mut h = HashMap::new(); + h.insert("a", V::I64(4)); + h.insert("b", V::Bool(true)); + h.insert("c", V::String("GPS".to_string())); + v.push(h); + + let mut section = Section::new( + SectionType::TrackPoints, + vec![ + ("a".to_string(), FieldType::I64), + ("b".to_string(), FieldType::Bool), + ("c".to_string(), FieldType::String), + ], + ); + + let mapping = section.fields().to_vec(); + + for entry in v { + let mut rowbuilder = section.open_row_builder(); + + for field_desc in mapping.iter() { + assert_matches!(rowbuilder.next_column_writer(), Some(cw) => { + match cw { + ColumnWriter::I64ColumnWriter(cwi) => { + assert!(cwi.write( + entry + .get(field_desc.name()) + .map(|v| match v { + V::I64(v) => Some(v), + _ => None, + }) + .flatten(), + ).is_ok()); + } + ColumnWriter::BoolColumnWriter(cwi) => { + assert!(cwi.write( + entry + .get(field_desc.name()) + .map(|v| match v { + V::Bool(v) => Some(v), + _ => None, + }) + .flatten(), + ).is_ok()); + } + ColumnWriter::StringColumnWriter(cwi) => { + assert!(cwi.write( + entry + .get(field_desc.name()) + .map(|v| match v { + V::String(v) => Some(v), + _ => None, + }) + .flatten(), + ).is_ok()); + } + } + }); + } + } + + let mut buf = Vec::new(); + let bytes_written = section.write(&mut buf); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), buf.len()); + #[rustfmt::skip] + assert_eq!(buf, &[0x00, // section type = track points + 0x03, // point count + 0x00, + 0x00, + 0x00, + + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x03, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x03, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x0E, // leb128 data size + + // Presence Column + 0b00000111, + 0b00000101, + 0b00000111, + + // Data Column 1 = I64 + 0x01, // 1 + 0x01, // 2 + 0x02, // 4 + + // Data Column 2 = Bool + 0x00, // false + 0x00, // missing + 0x01, // true + + // Data Column 3 = String + 0x04, // length 4 + b'R', + b'i', + b'd', + b'e', + 0x04, // length 4 + b'w', + b'i', + b't', + b'h', + 0x03, // length 3 + b'G', + b'P', + b'S', + + // CRC + 0xAF, + 0xEC, + 0x7D, + 0x70]); + } } From 65b4d9d2b6b5d1348738060a892f21f3c171abe2 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 16:19:23 -0800 Subject: [PATCH 013/113] pub fn write_track --- tracklib2/src/consts.rs | 1 + tracklib2/src/write/track.rs | 420 +++++++++++++++++++++++++++++++++-- 2 files changed, 402 insertions(+), 19 deletions(-) diff --git a/tracklib2/src/consts.rs b/tracklib2/src/consts.rs index 9710c08..61d0c7a 100644 --- a/tracklib2/src/consts.rs +++ b/tracklib2/src/consts.rs @@ -11,5 +11,6 @@ pub(crate) const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii 0x1A, // ctrl-z 0x0A]; // newline +pub(crate) const RWTF_HEADER_SIZE: u16 = 24; pub(crate) const RWTF_FILE_VERSION: u8 = 0x01; pub(crate) const RWTF_CREATOR_VERSION: u8 = 0x00; diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 1d673aa..166fb06 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -1,32 +1,414 @@ -use super::metadata::{self, MetadataEntry}; +use super::metadata::write_metadata; use super::section::Section; -use std::io::Write; +use crate::consts::{CRC16, RWTF_HEADER_SIZE}; +use crate::error::Result; +use crate::types::MetadataEntry; +use std::convert::TryFrom; +use std::io::{self, Write}; -struct TrackWriter { - out: W, - metadata_entries: Vec, - sections: Vec
, -} +#[rustfmt::skip] +fn write_data_table(out: &mut W, section_bufs: &[Vec]) -> Result { + let mut buf = Vec::new(); -impl TrackWriter { - fn new(out: W) -> Self { - Self { - out, - metadata_entries: Vec::new(), - sections: Vec::new(), - } - } + buf.write_all(&u8::try_from(section_bufs.len())?.to_le_bytes())?; // 1 byte - number of sections - fn add_metadata_entry(&mut self, entry: metadata::MetadataEntry) { - self.metadata_entries.push(entry); + for section in section_bufs.iter() { + leb128::write::unsigned(&mut buf, u64::try_from(section.len())?)?; // ? bytes - leb128 section size } - fn add_section(&mut self, section: Section) { - self.sections.push(section); + buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - crc + + out.write_all(&buf)?; + Ok(buf.len()) +} + +pub fn write_track( + out: &mut W, + metadata_entries: &[MetadataEntry], + sections: &[Section], +) -> Result { + let mut bytes_written = 0; + + // write metadata to a buffer so we can measure its size to use in the file header + let mut metadata_buf = Vec::new(); + write_metadata(&mut metadata_buf, metadata_entries)?; + + // write header + bytes_written += super::header::write_header( + out, + RWTF_HEADER_SIZE, + RWTF_HEADER_SIZE + u16::try_from(metadata_buf.len())?, + )?; + + // copy metadata buffer to out + bytes_written += usize::try_from(io::copy(&mut io::Cursor::new(metadata_buf), out)?)?; + + // create bufs for all sections + let section_bufs: Vec> = sections + .iter() + .map(|section| { + let mut buf = Vec::new(); + section.write(&mut buf)?; + Ok(buf) + }) + .collect::>()?; + + // write the data table + bytes_written += write_data_table(out, §ion_bufs)?; + + // now write out all the data sections + for section in section_bufs { + bytes_written += usize::try_from(io::copy(&mut io::Cursor::new(section), out)?)?; } + + Ok(bytes_written) } #[cfg(test)] mod tests { use super::*; + use crate::types::FieldType; + use crate::write::section::{ColumnWriter, SectionType}; + use assert_matches::assert_matches; + use std::collections::HashMap; + + #[test] + fn test_write_empty_data_table() { + let mut buf = Vec::new(); + let bytes_written = write_data_table(&mut buf, &[]); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), buf.len()); + #[rustfmt::skip] + assert_eq!(buf, &[0x00, // zero entries + 0x40, // crc + 0xBF]); + } + + #[test] + fn test_empty_track() { + let mut buf = Vec::new(); + let bytes_written = write_track(&mut buf, &[], &[]); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), buf.len()); + #[rustfmt::skip] + assert_eq!(buf, &[ + // Header + 0x89, // rwtfmagic + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x01, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x18, // metadata table offset + 0x00, + 0x1B, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x84, // header crc + 0xF8, + + // Metadata Table + 0x00, // zero metadata entries + 0x40, // crc + 0xBF, + + // Data Table + 0x00, // zero sections + 0x40, // crc + 0xBF]); + } + + #[test] + fn test_write_a_track() { + let mut section1 = Section::new( + SectionType::CoursePoints, + vec![ + ("m".to_string(), FieldType::I64), + ("k".to_string(), FieldType::Bool), + ("j".to_string(), FieldType::String), + ], + ); + + for _ in 0..5 { + let mut rowbuilder = section1.open_row_builder(); + + while let Some(cw) = rowbuilder.next_column_writer() { + match cw { + ColumnWriter::I64ColumnWriter(cwi) => { + assert!(cwi.write(Some(&42)).is_ok()); + } + ColumnWriter::BoolColumnWriter(cwi) => { + assert!(cwi.write(Some(&true)).is_ok()); + } + ColumnWriter::StringColumnWriter(cwi) => { + assert!(cwi.write(Some(&"hey".to_string())).is_ok()); + } + } + } + } + + enum V { + I64(i64), + Bool(bool), + String(String), + } + + let mut v = Vec::new(); + let mut h = HashMap::new(); + h.insert("a", V::I64(1)); + h.insert("b", V::Bool(false)); + h.insert("c", V::String("Ride".to_string())); + v.push(h); + let mut h = HashMap::new(); + h.insert("a", V::I64(2)); + h.insert("c", V::String("with".to_string())); + v.push(h); + let mut h = HashMap::new(); + h.insert("a", V::I64(4)); + h.insert("b", V::Bool(true)); + h.insert("c", V::String("GPS".to_string())); + v.push(h); + + let mut section2 = Section::new( + SectionType::TrackPoints, + vec![ + ("a".to_string(), FieldType::I64), + ("b".to_string(), FieldType::Bool), + ("c".to_string(), FieldType::String), + ], + ); + + let mapping = section2.fields().to_vec(); + + for entry in v { + let mut rowbuilder = section2.open_row_builder(); + + for field_desc in mapping.iter() { + assert_matches!(rowbuilder.next_column_writer(), Some(cw) => { + match cw { + ColumnWriter::I64ColumnWriter(cwi) => { + assert!(cwi.write( + entry + .get(field_desc.name()) + .map(|v| match v { + V::I64(v) => Some(v), + _ => None, + }) + .flatten(), + ).is_ok()); + } + ColumnWriter::BoolColumnWriter(cwi) => { + assert!(cwi.write( + entry + .get(field_desc.name()) + .map(|v| match v { + V::Bool(v) => Some(v), + _ => None, + }) + .flatten(), + ).is_ok()); + } + ColumnWriter::StringColumnWriter(cwi) => { + assert!(cwi.write( + entry + .get(field_desc.name()) + .map(|v| match v { + V::String(v) => Some(v), + _ => None, + }) + .flatten(), + ).is_ok()); + } + } + }); + } + } + + let mut buf = Vec::new(); + let bytes_written = write_track(&mut buf, &[], &[section1, section2]); + assert!(bytes_written.is_ok()); + assert_eq!(bytes_written.unwrap(), buf.len()); + #[rustfmt::skip] + assert_eq!(buf, &[ + // Header + 0x89, // rwtfmagic + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x01, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x18, // metadata table offset + 0x00, + 0x1B, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x84, // header crc + 0xF8, + + // Metadata Table + 0x00, // zero entries + 0x40, // crc + 0xBF, + + // Data Table + 0x02, // two entries + 0x39, // size of first entry + 0x2D, // size of second entry + 0xFD, // crc + 0xB2, + + + // Section 1 + 0x01, // section type = course points + 0x05, // point count + 0x00, + 0x00, + 0x00, + + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name len + b'm', // name + 0x05, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name len + b'k', // name + 0x05, // leb128 data size + 0x04, // third field type = String + 0x01, // name len + b'j', // name + 0x14, // leb128 data size + + // Presence Column + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + + // Data Column 1 = I64 + 0x2A, // 42 + 0x00, // no change + 0x00, // no change + 0x00, // no change + 0x00, // no change + + // Data Column 2 = Bool + 0x01, // true + 0x01, // true + 0x01, // true + 0x01, // true + 0x01, // true + + // Data Column 3 = String + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + + // CRC + 0x5B, + 0xC3, + 0x0F, + 0x6E, + + + // Section 2 + 0x00, // section type = track points + 0x03, // point count + 0x00, + 0x00, + 0x00, + + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x03, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x03, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x0E, // leb128 data size + + // Presence Column + 0b00000111, + 0b00000101, + 0b00000111, + + // Data Column 1 = I64 + 0x01, // 1 + 0x01, // 2 + 0x02, // 4 + + // Data Column 2 = Bool + 0x00, // false + 0x00, // missing + 0x01, // true + + // Data Column 3 = String + 0x04, // length 4 + b'R', + b'i', + b'd', + b'e', + 0x04, // length 4 + b'w', + b'i', + b't', + b'h', + 0x03, // length 3 + b'G', + b'P', + b'S', + + // CRC + 0xAF, + 0xEC, + 0x7D, + 0x70]); + } } From 99365fbea174f13c1c89bd8533fd13cd9e33931f Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 16:20:15 -0800 Subject: [PATCH 014/113] don't need TimeError anymore --- tracklib2/src/error.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tracklib2/src/error.rs b/tracklib2/src/error.rs index 27bd27e..17964b5 100644 --- a/tracklib2/src/error.rs +++ b/tracklib2/src/error.rs @@ -13,12 +13,6 @@ pub enum TracklibError { #[from] source: std::io::Error, }, - - #[error("Time Error?!")] - TimeError { - #[from] - source: std::time::SystemTimeError, - }, } pub type Result = std::result::Result; From 095263a6c1b2dd7ec2ace0574a8568e7e78234cb Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 16:22:32 -0800 Subject: [PATCH 015/113] read module with CRC code --- tracklib2/src/lib.rs | 1 + tracklib2/src/read/crc.rs | 88 +++++++++++++++++++++++++++++++++++++++ tracklib2/src/read/mod.rs | 1 + 3 files changed, 90 insertions(+) create mode 100644 tracklib2/src/read/crc.rs create mode 100644 tracklib2/src/read/mod.rs diff --git a/tracklib2/src/lib.rs b/tracklib2/src/lib.rs index 9e7578c..748e1de 100644 --- a/tracklib2/src/lib.rs +++ b/tracklib2/src/lib.rs @@ -1,4 +1,5 @@ pub mod error; mod consts; +pub mod read; pub mod types; pub mod write; diff --git a/tracklib2/src/read/crc.rs b/tracklib2/src/read/crc.rs new file mode 100644 index 0000000..df811e0 --- /dev/null +++ b/tracklib2/src/read/crc.rs @@ -0,0 +1,88 @@ +use crate::error::TracklibError; +use nom::{number::complete::le_u16, IResult, Offset}; + +pub(crate) trait CRCImpl: Sized { + fn crc_bytes(bytes: &[u8]) -> Self; + fn read_bytes(input: &[u8]) -> IResult<&[u8], Self, TracklibError>; +} + +impl CRCImpl for u16 { + fn crc_bytes(bytes: &[u8]) -> Self { + crate::consts::CRC16.checksum(bytes) + } + + fn read_bytes(input: &[u8]) -> IResult<&[u8], Self, TracklibError> { + le_u16(input) + } +} + +#[derive(Debug, PartialEq)] +pub(crate) enum CRC { + Valid(T), + Invalid { expected: T, computed: T }, +} + +impl CRC { + fn new(expected: T, computed: T) -> Self { + if expected == computed { + CRC::Valid(expected) + } else { + CRC::Invalid { expected, computed } + } + } +} + +impl CRC { + pub(crate) fn parser<'a>( + start: &'a [u8], + ) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Self, TracklibError> { + move |input: &[u8]| { + let end = start.offset(input); + let computed = CRCImpl::crc_bytes(&start[..end]); + let (input, expected) = CRCImpl::read_bytes(input)?; + Ok((input, Self::new(expected, computed))) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + // #[test] + // fn test_crc_full_thing() { + // let bytes = &[0x00, 0xBF, 0x40,]; + // println!("crc of &[0x00]: {:X?}", u16::crc_bytes(&[0x00]).to_le_bytes()); + // println!("crc of &{:X?}: {:X?}", bytes, u16::crc_bytes(bytes).to_le_bytes()); + // assert_eq!(u16::crc_bytes(bytes), u16::from_le_bytes([0xFF, 0xFF])); + // } + + #[test] + fn test_emtpy_crc() { + let buf = &[0x00, 0x00]; + let result = CRC::::parser(buf)(buf); + assert_matches!(result, Ok((&[], CRC::Valid(crc))) => { + assert_eq!(crc, u16::from_le_bytes([0x00, 0x00])); + }); + } + + #[test] + fn test_simple_crc() { + let buf = &[0x00, 0x40, 0xBF]; + let result = CRC::::parser(buf)(&buf[1..]); + assert_matches!(result, Ok((&[], CRC::Valid(crc))) => { + assert_eq!(crc, u16::from_le_bytes([0x40, 0xBF])); + }); + } + + #[test] + fn test_invalid_crc() { + let buf = &[0x00, 0x12, 0x34]; + let result = CRC::::parser(buf)(&buf[1..]); + assert_matches!(result, Ok((&[], CRC::Invalid {expected, computed})) => { + assert_eq!(expected, u16::from_le_bytes([0x12, 0x34])); + assert_eq!(computed, u16::from_le_bytes([0x40, 0xBF])); + }); + } +} diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs new file mode 100644 index 0000000..86ace2e --- /dev/null +++ b/tracklib2/src/read/mod.rs @@ -0,0 +1 @@ +mod crc; From 40ab2fbde0bff9edfb179e9e9a54e583c328b29c Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 16:23:20 -0800 Subject: [PATCH 016/113] crate::read::header module --- tracklib2/src/read/header.rs | 123 +++++++++++++++++++++++++++++++++++ tracklib2/src/read/mod.rs | 1 + 2 files changed, 124 insertions(+) create mode 100644 tracklib2/src/read/header.rs diff --git a/tracklib2/src/read/header.rs b/tracklib2/src/read/header.rs new file mode 100644 index 0000000..cbd5833 --- /dev/null +++ b/tracklib2/src/read/header.rs @@ -0,0 +1,123 @@ +use super::crc::CRC; +use crate::consts::RWTFMAGIC; +use crate::error::TracklibError; +use nom::{ + bytes::complete::{tag, take}, + number::complete::{le_u16, le_u8}, + IResult, +}; + +#[derive(Debug, PartialEq)] +pub struct Header { + pub(crate) file_version: u8, + pub(crate) creator_version: u8, + pub(crate) metadata_offset: u16, + pub(crate) data_offset: u16, +} + +pub(crate) fn parse_header(input: &[u8]) -> IResult<&[u8], Header, TracklibError> { + let header_start = input; + let (input, _magic) = tag(RWTFMAGIC)(input)?; + let (input, file_version) = le_u8(input)?; + let (input, _fv_reserve) = take(3_usize)(input)?; + let (input, creator_version) = le_u8(input)?; + let (input, _cv_reserve) = take(3_usize)(input)?; + let (input, metadata_offset) = le_u16(input)?; + let (input, data_offset) = le_u16(input)?; + let (input, _e_reserve) = take(2_usize)(input)?; + let (input, checksum) = CRC::::parser(header_start)(input)?; + + match checksum { + CRC::Valid(_) => Ok(( + input, + Header { + file_version, + creator_version, + metadata_offset, + data_offset, + }, + )), + CRC::Invalid { expected, computed } => Err(nom::Err::Error(TracklibError::CRC16Error { + expected, + computed, + })), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + #[test] + fn test_parse_header() { + #[rustfmt::skip] + let buf = &[0x89, // magic number + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x00, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x0A, // metadata table offset + 0x00, + 0x1A, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x86, // header crc + 0xB7]; + assert_matches!(parse_header(buf), Ok((&[], Header{file_version, + creator_version, + metadata_offset, + data_offset})) => { + assert_eq!(file_version, 0x00); + assert_eq!(creator_version, 0x00); + assert_eq!(metadata_offset, 0x0A); + assert_eq!(data_offset, 0x1A); + }); + } + + #[test] + fn test_parse_header_with_invalid_crc() { + #[rustfmt::skip] + let buf = &[0x89, // magic number + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x00, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x0A, // metadata table offset + 0x00, + 0x1A, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x12, // invalid header crc + 0x34]; + assert_matches!(parse_header(buf), Err(nom::Err::Error(TracklibError::CRC16Error{expected, + computed})) => { + assert_eq!(expected, u16::from_le_bytes([0x12, 0x34])); + assert_eq!(computed, u16::from_le_bytes([0x86, 0xB7])); + }); + } +} diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 86ace2e..e2811d1 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1 +1,2 @@ mod crc; +mod header; From ec38b80e3d4cc717e4deea68841752984f3036db Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 16:24:05 -0800 Subject: [PATCH 017/113] crate::read::metadata module --- tracklib2/src/read/metadata.rs | 215 +++++++++++++++++++++++++++++++++ tracklib2/src/read/mod.rs | 1 + 2 files changed, 216 insertions(+) create mode 100644 tracklib2/src/read/metadata.rs diff --git a/tracklib2/src/read/metadata.rs b/tracklib2/src/read/metadata.rs new file mode 100644 index 0000000..68a24ad --- /dev/null +++ b/tracklib2/src/read/metadata.rs @@ -0,0 +1,215 @@ +use super::crc::CRC; +use crate::error::TracklibError; +use crate::types::{MetadataEntry, TrackType}; +use nom::{ + bytes::complete::tag, + multi::length_data, + number::complete::{le_u16, le_u32, le_u64, le_u8}, + IResult, +}; + +fn parse_metadata_entry_track_type( + input: &[u8], +) -> IResult<&[u8], Option, TracklibError> { + let (input, _size) = tag([0x05, 0x00])(input)?; + let (input, type_tag) = le_u8(input)?; + let (input, id) = le_u32(input)?; + + match type_tag { + 0x00 => Ok((input, Some(MetadataEntry::TrackType(TrackType::Trip(id))))), + 0x01 => Ok((input, Some(MetadataEntry::TrackType(TrackType::Route(id))))), + 0x02 => Ok(( + input, + Some(MetadataEntry::TrackType(TrackType::Segment(id))), + )), + _ => Err(nom::Err::Error(TracklibError::ParseError { + error_kind: nom::error::ErrorKind::Tag, + })), + } +} + +fn parse_metadata_entry_created_at( + input: &[u8], +) -> IResult<&[u8], Option, TracklibError> { + let (input, _size) = tag([0x08, 0x00])(input)?; + let (input, seconds_since_epoch) = le_u64(input)?; + + Ok((input, Some(MetadataEntry::CreatedAt(seconds_since_epoch)))) +} + +fn parse_metadata_entry_unknown( + input: &[u8], +) -> IResult<&[u8], Option, TracklibError> { + let (input, _data) = length_data(le_u16)(input)?; + Ok((input, None)) +} + +fn parse_metadata_entry(input: &[u8]) -> IResult<&[u8], Option, TracklibError> { + let (input, type_tag) = le_u8(input)?; + + let (input, maybe_metadata_entry) = match type_tag { + 0x00 => parse_metadata_entry_track_type(input)?, + 0x01 => parse_metadata_entry_created_at(input)?, + _ => parse_metadata_entry_unknown(input)?, + }; + + Ok((input, maybe_metadata_entry)) +} + +pub(crate) fn parse_metadata(input: &[u8]) -> IResult<&[u8], Vec, TracklibError> { + let metadata_start = input; + let (mut input, entry_count) = le_u8(input)?; + + let mut entries = Vec::with_capacity(usize::from(entry_count)); + for _ in 0..entry_count { + let (rest, maybe_entry) = parse_metadata_entry(input)?; + input = rest; + if let Some(entry) = maybe_entry { + entries.push(entry); + } + } + + let (input, checksum) = CRC::::parser(metadata_start)(input)?; + + match checksum { + CRC::Valid(_) => Ok((input, entries)), + CRC::Invalid { expected, computed } => Err(nom::Err::Error(TracklibError::CRC16Error { + expected, + computed, + })), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + #[test] + fn test_empty_metadata() { + #[rustfmt::skip] + let buf = &[0x00, // zero metadata entries + 0x40, // crc + 0xBF]; + assert_matches!(parse_metadata(buf), Ok((&[], entries)) => { + assert_eq!(entries, vec![]) + }); + } + + #[test] + fn test_metadata_both() { + #[rustfmt::skip] + let buf = &[0x02, // two metadata entries + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x14, // four byte trip ID = 20 + 0x00, + 0x00, + 0x00, + 0x01, // entry type: created_at = 0x01 + 0x08, // two byte entry size = 8 + 0x00, + 0x00, // eight byte timestamp: zero seconds elapsed + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x23, // crc + 0xD2]; + assert_matches!(parse_metadata(buf), Ok((&[], entries)) => { + assert_eq!(entries, vec![MetadataEntry::TrackType(TrackType::Trip(20)), + MetadataEntry::CreatedAt(0)]) + }); + } + + #[test] + fn test_unknown_inbetween_known_entries() { + #[rustfmt::skip] + let buf = &[0x03, // two metadata entries + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x14, // four byte trip ID = 20 + 0x00, + 0x00, + 0x00, + 0xEF, // entry type: unknown! + 0x14, // two byte entry size = 20 + 0x00, + 0x00, // 20 byte payload + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x01, // entry type: created_at = 0x01 + 0x08, // two byte entry size = 8 + 0x00, + 0x00, // eight byte timestamp: zero seconds elapsed + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x71, // crc + 0x85]; + assert_matches!(parse_metadata(buf), Ok((&[], entries)) => { + assert_eq!(entries, vec![MetadataEntry::TrackType(TrackType::Trip(20)), + MetadataEntry::CreatedAt(0)]) + }); + } + + #[test] + fn test_invalid_crc() { + #[rustfmt::skip] + let buf = &[0x02, // two metadata entries + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x14, // four byte trip ID = 20 + 0x00, + 0x00, + 0x00, + 0x01, // entry type: created_at = 0x01 + 0x08, // two byte entry size = 8 + 0x00, + 0x00, // eight byte timestamp: zero seconds elapsed + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x12, // crc + 0x34]; + assert_matches!(parse_metadata(buf), Err(nom::Err::Error(TracklibError::CRC16Error{expected, + computed})) => { + assert_eq!(expected, u16::from_le_bytes([0x12, 0x34])); + assert_eq!(computed, u16::from_le_bytes([0x23, 0xD2])); + }); + } +} diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index e2811d1..5481267 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1,2 +1,3 @@ mod crc; mod header; +mod metadata; From 189e70143f3a884a047dde937da675baa8cf99a8 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 6 Dec 2021 17:34:01 -0800 Subject: [PATCH 018/113] parse_types_table --- tracklib2/Cargo.lock | 67 +++++++-------- tracklib2/Cargo.toml | 3 +- tracklib2/src/error.rs | 18 ++++ tracklib2/src/read/header.rs | 3 +- tracklib2/src/read/mod.rs | 1 + tracklib2/src/read/types_table.rs | 135 ++++++++++++++++++++++++++++++ tracklib2/src/types.rs | 7 +- 7 files changed, 191 insertions(+), 43 deletions(-) create mode 100644 tracklib2/src/read/types_table.rs diff --git a/tracklib2/Cargo.lock b/tracklib2/Cargo.lock index 39d5cb4..db404a5 100644 --- a/tracklib2/Cargo.lock +++ b/tracklib2/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + [[package]] name = "assert_matches" version = "1.5.0" @@ -31,18 +37,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" -[[package]] -name = "bitvec" -version = "0.19.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f93d0ef3363c364d5976646a38f04cf67cfe1d4c8d160cdea02cab2c116b33" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "bstr" version = "0.2.14" @@ -230,12 +224,6 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" -[[package]] -name = "funty" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" - [[package]] name = "half" version = "1.6.0" @@ -317,17 +305,34 @@ dependencies = [ "autocfg", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "nom" -version = "6.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88034cfd6b4a0d54dd14f4a507eceee36c0b70e5a02236c4e4df571102be17f0" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" dependencies = [ - "bitvec", "memchr", + "minimal-lexical", "version_check", ] +[[package]] +name = "nom-leb128" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a73b6c3a9ecfff12ad50dedba051ef838d2f478d938bb3e6b3842431028e62" +dependencies = [ + "arrayvec", + "nom", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -383,12 +388,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "radium" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" - [[package]] name = "rayon" version = "1.5.0" @@ -532,12 +531,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "textwrap" version = "0.11.0" @@ -581,10 +574,12 @@ dependencies = [ name = "tracklib2" version = "0.1.0" dependencies = [ + "assert_matches", "crc", "criterion", "leb128", "nom", + "nom-leb128", "thiserror", ] @@ -711,9 +706,3 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "wyz" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml index 9400809..f8054ae 100644 --- a/tracklib2/Cargo.toml +++ b/tracklib2/Cargo.toml @@ -7,7 +7,8 @@ edition = "2018" [dependencies] crc = "2.1" leb128 = "0.2" -nom = {version = "6.0", default_features = false, features = ["alloc"]} +nom = "7.1" +nom-leb128 = "0.2" thiserror = "1.0" [dev-dependencies] diff --git a/tracklib2/src/error.rs b/tracklib2/src/error.rs index 17964b5..98ed39f 100644 --- a/tracklib2/src/error.rs +++ b/tracklib2/src/error.rs @@ -2,6 +2,12 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum TracklibError { + #[error("Parse Error")] + ParseError { error_kind: nom::error::ErrorKind }, + + #[error("CRC Error")] + CRC16Error { expected: u16, computed: u16 }, + #[error("Numeric Bounds Error")] BoundsError { #[from] @@ -16,3 +22,15 @@ pub enum TracklibError { } pub type Result = std::result::Result; + +impl nom::error::ParseError for TracklibError { + fn from_error_kind(input: I, kind: nom::error::ErrorKind) -> Self { + Self::ParseError { error_kind: kind } + } + + fn append(input: I, kind: nom::error::ErrorKind, other: Self) -> Self { + other + } +} + +impl nom::error::ContextError for TracklibError {} diff --git a/tracklib2/src/read/header.rs b/tracklib2/src/read/header.rs index cbd5833..4a8ffe1 100644 --- a/tracklib2/src/read/header.rs +++ b/tracklib2/src/read/header.rs @@ -7,7 +7,8 @@ use nom::{ IResult, }; -#[derive(Debug, PartialEq)] +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] pub struct Header { pub(crate) file_version: u8, pub(crate) creator_version: u8, diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 5481267..58f1e1f 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1,3 +1,4 @@ mod crc; mod header; mod metadata; +mod types_table; diff --git a/tracklib2/src/read/types_table.rs b/tracklib2/src/read/types_table.rs new file mode 100644 index 0000000..872c445 --- /dev/null +++ b/tracklib2/src/read/types_table.rs @@ -0,0 +1,135 @@ +use crate::error::TracklibError; +use crate::types::FieldType; +use nom::{ + multi::{length_count, length_data}, + number::complete::le_u8, + IResult, +}; +use nom_leb128::leb128_u64; +use std::str; + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +struct TypesTableEntry<'a> { + fieldtype: FieldType, + name: &'a str, + size: u64, +} + +fn parse_types_table_entry<'a>( + input: &'a [u8], +) -> IResult<&'a [u8], TypesTableEntry, TracklibError> { + let (input, type_tag) = le_u8(input)?; + let (input, field_name) = length_data(le_u8)(input)?; + let (input, data_size) = leb128_u64(input)?; + + let fieldtype = match type_tag { + 0x00 => FieldType::I64, + 0x04 => FieldType::String, + 0x05 => FieldType::Bool, + _ => { + return Err(nom::Err::Error(TracklibError::ParseError { + error_kind: nom::error::ErrorKind::Tag, + })) + } + }; + + let name = match str::from_utf8(field_name) { + Ok(s) => s, + Err(_) => { + return Err(nom::Err::Error(TracklibError::ParseError { + error_kind: nom::error::ErrorKind::Tag, + })) + } + }; + + Ok(( + input, + TypesTableEntry { + fieldtype, + name, + size: data_size, + }, + )) +} + +fn parse_types_table(input: &[u8]) -> IResult<&[u8], Vec, TracklibError> { + length_count(le_u8, parse_types_table_entry)(input) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + #[test] + fn test_test_parse_types_table() { + #[rustfmt::skip] + let buf = &[0x04, // entry count = 4 + 0x00, // first entry type: i64 = 0 + 0x01, // name len = 1 + b'm', // name = "m" + 0x02, // data size = 2 + 0x05, // second entry type: bool = 5 + 0x01, // name len = 1 + b'k', // name = "k" + 0x01, // data size = 1 + 0x04, // third entry type: string = 4 + 0x0A, // name len = 10 + b'l', // name = "long name!" + b'o', + b'n', + b'g', + b' ', + b'n', + b'a', + b'm', + b'e', + b'!', + 0x07, // data size = 7 ("Hello!" + leb128 length prefix) + 0x00, // fourth entry type: i64 = 0 + 0x01, // name len = 1 + b'i', // name = "i" + 0x02]; // data size = 2 + assert_matches!(parse_types_table(buf), Ok((&[], entries)) => { + assert_eq!(entries, vec![TypesTableEntry{fieldtype: FieldType::I64, + name: "m", + size: 2}, + TypesTableEntry{fieldtype: FieldType::Bool, + name: "k", + size: 1}, + TypesTableEntry{fieldtype: FieldType::String, + name: "long name!", + size: 7}, + TypesTableEntry{fieldtype: FieldType::I64, + name: "i", + size: 2}]); + }); + } + + #[test] + fn test_types_table_invalid_field_tag() { + #[rustfmt::skip] + let buf = &[0x01, // entry count + 0xEF, // first entry type: invalid + 0x01, // name len = 1 + b'm', // name = "m" + 0x02]; // data size = 2 + assert_matches!(parse_types_table(buf), Err(nom::Err::Error(TracklibError::ParseError{error_kind})) => { + assert_eq!(error_kind, nom::error::ErrorKind::Tag); + }); + } + + #[test] + fn test_types_table_invalid_utf8() { + #[rustfmt::skip] + let buf = &[0x01, // entry count + 0x00, // first entry type: I64 + 0x01, // name len = 1 + 0xC0, // name: invalid utf-8 + 0x02]; // data size = 2 + assert_matches!(parse_types_table(buf), Err(nom::Err::Error(TracklibError::ParseError{error_kind})) => { + assert_eq!(error_kind, nom::error::ErrorKind::Tag); + }); + } +} diff --git a/tracklib2/src/types.rs b/tracklib2/src/types.rs index c75a893..5cc1e98 100644 --- a/tracklib2/src/types.rs +++ b/tracklib2/src/types.rs @@ -1,4 +1,5 @@ #[derive(Debug, Clone)] +#[cfg_attr(test, derive(PartialEq))] pub enum FieldType { I64, String, @@ -25,14 +26,16 @@ impl FieldDescription { } } -#[derive(Debug, PartialEq)] +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] pub enum TrackType { Trip(u32), Route(u32), Segment(u32), } -#[derive(Debug, PartialEq)] +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] pub enum MetadataEntry { TrackType(TrackType), CreatedAt(u64), From 9ddaa7a92898349deeefbec109219ad26168ad52 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Sat, 11 Dec 2021 16:45:40 -0800 Subject: [PATCH 019/113] CRCWriter --- tracklib2/src/write/crcwriter.rs | 83 ++++++++++++++++++++++++++++++++ tracklib2/src/write/mod.rs | 1 + 2 files changed, 84 insertions(+) create mode 100644 tracklib2/src/write/crcwriter.rs diff --git a/tracklib2/src/write/crcwriter.rs b/tracklib2/src/write/crcwriter.rs new file mode 100644 index 0000000..4313176 --- /dev/null +++ b/tracklib2/src/write/crcwriter.rs @@ -0,0 +1,83 @@ +use crate::consts::{CRC16, CRC32}; +use std::io::{self, Write}; + +pub(crate) struct CrcWriter<'a, W, B: crc::Width> { + inner: W, + digest: crc::Digest<'a, B>, +} + +/////////// +// CRC16 // +/////////// +impl<'a, W> CrcWriter<'a, W, u16> { + pub(crate) fn new16(writer: W) -> Self { + Self { + inner: writer, + digest: CRC16.digest(), + } + } +} + +impl<'a, W> CrcWriter<'a, W, u16> { + pub(crate) fn into_inner_and_crc(self) -> (W, u16) { + (self.inner, self.digest.finalize()) + } +} + +impl<'a, W: Write> Write for CrcWriter<'a, W, u16> { + fn write(&mut self, buf: &[u8]) -> io::Result { + let amt = self.inner.write(buf)?; + self.digest.update(&buf[..amt]); + Ok(amt) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl<'a, W: Write> CrcWriter<'a, W, u16> { + pub(crate) fn append_crc(self) -> io::Result { + let (mut writer, crc) = self.into_inner_and_crc(); + writer.write_all(&crc.to_le_bytes())?; + Ok(writer) + } +} + +/////////// +// CRC32 // +/////////// +impl<'a, W> CrcWriter<'a, W, u32> { + pub(crate) fn new32(writer: W) -> Self { + Self { + inner: writer, + digest: CRC32.digest(), + } + } +} + +impl<'a, W> CrcWriter<'a, W, u32> { + pub(crate) fn into_inner_and_crc(self) -> (W, u32) { + (self.inner, self.digest.finalize()) + } +} + +impl<'a, W: Write> Write for CrcWriter<'a, W, u32> { + fn write(&mut self, buf: &[u8]) -> io::Result { + let amt = self.inner.write(buf)?; + self.digest.update(&buf[..amt]); + Ok(amt) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl<'a, W: Write> CrcWriter<'a, W, u32> { + pub(crate) fn append_crc(self) -> io::Result { + let (mut writer, crc) = self.into_inner_and_crc(); + writer.write_all(&crc.to_le_bytes())?; + Ok(writer) + } +} diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs index 6f1e869..5a8e38f 100644 --- a/tracklib2/src/write/mod.rs +++ b/tracklib2/src/write/mod.rs @@ -1,3 +1,4 @@ +mod crcwriter; pub mod encoders; mod header; pub mod metadata; From 973866bc08d0b12ac915929201c054a2a93b5d9f Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Sat, 11 Dec 2021 16:46:45 -0800 Subject: [PATCH 020/113] use CRCWriter in header and metadata modules --- tracklib2/src/write/header.rs | 44 +++---- tracklib2/src/write/metadata.rs | 227 ++++++++++++++------------------ 2 files changed, 119 insertions(+), 152 deletions(-) diff --git a/tracklib2/src/write/header.rs b/tracklib2/src/write/header.rs index ae3ab8e..c8fe6a9 100644 --- a/tracklib2/src/write/header.rs +++ b/tracklib2/src/write/header.rs @@ -1,36 +1,35 @@ -use crate::consts::{CRC16, RWTFMAGIC, RWTF_CREATOR_VERSION, RWTF_FILE_VERSION}; +use super::crcwriter::CrcWriter; +use crate::consts::{RWTFMAGIC, RWTF_CREATOR_VERSION, RWTF_FILE_VERSION}; use crate::error::Result; use std::io::Write; #[rustfmt::skip] -pub(crate) fn write_header(out: &mut W, metadata_table_offset: u16, data_table_offset: u16) -> Result { - let mut buf = Vec::with_capacity(24); - - buf.write_all(&RWTFMAGIC)?; // 8 bytes - Magic Number - buf.write_all(&RWTF_FILE_VERSION.to_le_bytes())?; // 1 byte - File Version - buf.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - FV Reserve - buf.write_all(&RWTF_CREATOR_VERSION.to_le_bytes())?; // 1 byte - Creator Version - buf.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - CV Reserve - buf.write_all(&metadata_table_offset.to_le_bytes())?; // 2 bytes - Offset to Metadata Table - buf.write_all(&data_table_offset.to_le_bytes())?; // 2 bytes - Offset to Data Table - buf.write_all(&[0x00, 0x00])?; // 2 bytes - E Reserve - buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - Header CRC - - out.write_all(&buf)?; - Ok(buf.len()) +pub(crate) fn write_header(out: &mut W, metadata_table_offset: u16, data_table_offset: u16) -> Result<()> { + let mut crcwriter = CrcWriter::new16(out); + crcwriter.write_all(&RWTFMAGIC)?; // 8 bytes - Magic Number + crcwriter.write_all(&RWTF_FILE_VERSION.to_le_bytes())?; // 1 byte - File Version + crcwriter.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - FV Reserve + crcwriter.write_all(&RWTF_CREATOR_VERSION.to_le_bytes())?; // 1 byte - Creator Version + crcwriter.write_all(&[0x00, 0x00, 0x00])?; // 3 bytes - CV Reserve + crcwriter.write_all(&metadata_table_offset.to_le_bytes())?; // 2 bytes - Offset to Metadata Table + crcwriter.write_all(&data_table_offset.to_le_bytes())?; // 2 bytes - Offset to Data Table + crcwriter.write_all(&[0x00, 0x00])?; // 2 bytes - E Reserve + crcwriter.append_crc()?; // 2 bytes - Header CRC + Ok(()) } #[cfg(test)] mod tests { use super::*; + use assert_matches::assert_matches; #[test] fn test_write_header() { let mut buf = vec![]; - let written = write_header(&mut buf, 0x0A, 0x1A); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x89, // magic number + assert_matches!(write_header(&mut buf, 0x0A, 0x1A), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x89, // magic number 0x52, 0x57, 0x54, @@ -53,8 +52,7 @@ mod tests { 0x00, // e reserve 0x00, 0x86, // header crc - 0x76]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + 0x76]); + }); } } diff --git a/tracklib2/src/write/metadata.rs b/tracklib2/src/write/metadata.rs index fbce75c..b6fe646 100644 --- a/tracklib2/src/write/metadata.rs +++ b/tracklib2/src/write/metadata.rs @@ -1,82 +1,73 @@ -use crate::consts::CRC16; -use crate::error::{Result, TracklibError}; +use super::crcwriter::CrcWriter; +use crate::error::Result; use crate::types::{MetadataEntry, TrackType}; use std::convert::TryFrom; use std::io::Write; impl MetadataEntry { - fn write(&self) -> Result<(u8, Vec)> { + #[rustfmt::skip] + fn write(&self, out: &mut W) -> Result<()> { match self { Self::TrackType(track_type) => { - // TrackType metadata is 5 bytes - // 1 byte for the type of track - // 4 bytes for the id of the track let (type_tag, id): (u8, &u32) = match track_type { TrackType::Trip(id) => (0x00, id), TrackType::Route(id) => (0x01, id), TrackType::Segment(id) => (0x02, id), }; - let mut buf = Vec::with_capacity(5); - buf.write_all(&type_tag.to_le_bytes())?; - buf.write_all(&id.to_le_bytes())?; - Ok((0x00, buf)) + out.write_all(&[0x00])?; // 1 byte - entry type + out.write_all(&[0x05, 0x00])?; // 2 bytes - entry size + out.write_all(&type_tag.to_le_bytes())?; // 1 byte - TrackType type tag + out.write_all(&id.to_le_bytes())?; // 4 bytes - TrackType id + Ok(()) } Self::CreatedAt(seconds_since_epoch) => { - // CreatedAt metadata is 8 bytes: seconds since epoch - Ok((0x01, seconds_since_epoch.to_le_bytes().to_vec())) + out.write_all(&[0x01])?; // 1 byte - entry type + out.write_all(&[0x08, 0x00])?; // 2 bytes - entry size + out.write_all(&seconds_since_epoch.to_le_bytes())?; // 8 bytes - created_at + Ok(()) } } } } #[rustfmt::skip] -pub(crate) fn write_metadata(out: &mut W, entries: &[MetadataEntry]) -> Result { - let entry_count = u8::try_from(entries.len())?; - - let mut buf = Vec::new(); - - buf.write_all(&entry_count.to_le_bytes())?; // 1 byte - entry count +pub(crate) fn write_metadata(out: &mut W, entries: &[MetadataEntry]) -> Result<()> { + let mut crcwriter = CrcWriter::new16(out); + crcwriter.write_all(&u8::try_from(entries.len())?.to_le_bytes())?; // 1 byte - entry count for entry in entries { - let (entry_type, entry_bytes) = entry.write()?; - let entry_size = u16::try_from(entry_bytes.len())?; - - buf.write_all(&entry_type.to_le_bytes())?; // 1 byte - entry type - buf.write_all(&entry_size.to_le_bytes())?; // 2 bytes - entry size - buf.write_all(&entry_bytes)?; // ? bytes - entry value + entry.write(&mut crcwriter)?; // ? bytes - entry contents } + crcwriter.append_crc()?; // 2 bytes - crc - buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - crc - - out.write_all(&buf)?; - Ok(buf.len()) + Ok(()) } #[cfg(test)] mod tests { use super::*; + use assert_matches::assert_matches; use std::time::Duration; #[test] fn test_write_empty_metadata() { let mut buf = vec![]; - let written = write_metadata(&mut buf, &[]); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x00, // zero metadata entries + assert_matches!(write_metadata(&mut buf, &[]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x00, // zero metadata entries 0x40, // crc - 0xBF]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + 0xBF]); + }); } #[test] fn test_only_track_type_trip() { let mut buf = vec![]; - let written = write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Trip(400))]); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x01, // one metadata entry + assert_matches!(write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Trip(400))]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x01, // one metadata entry 0x00, // entry type: track_type = 0x00 0x05, // two byte entry size = 5 0x00, @@ -86,18 +77,17 @@ mod tests { 0x00, 0x00, 0xD1, // crc - 0x5F]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + 0x5F]); + }); } #[test] fn test_only_track_type_route() { let mut buf = vec![]; - let written = write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Route(64))]); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x01, // one metadata entry + assert_matches!(write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Route(64))]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x01, // one metadata entry 0x00, // entry type: track_type = 0x00 0x05, // two byte entry size = 5 0x00, @@ -107,42 +97,36 @@ mod tests { 0x00, 0x00, 0x85, // crc - 0x9F]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + 0x9F]); + }); } #[test] fn test_only_track_type_segment() { let mut buf = vec![]; - let written = write_metadata( - &mut buf, - &[MetadataEntry::TrackType(TrackType::Segment(u32::MAX))], - ); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x01, // one metadata entry - 0x00, // entry type: track_type = 0x00 - 0x05, // two byte entry size = 5 - 0x00, - 0x02, // track type: segment = 0x02 - 0xFF, // four byte segment ID = 4,294,967,295 - 0xFF, - 0xFF, - 0xFF, - 0xD5, // crc - 0xCB]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + assert_matches!(write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Segment(u32::MAX))]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, &[0x01, // one metadata entry + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x02, // track type: segment = 0x02 + 0xFF, // four byte segment ID = 4,294,967,295 + 0xFF, + 0xFF, + 0xFF, + 0xD5, // crc + 0xCB]); + }); } #[test] fn test_only_created_at_epoch() { let mut buf = vec![]; - let written = write_metadata(&mut buf, &[MetadataEntry::CreatedAt(0)]); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x01, // one metadata entry + assert_matches!(write_metadata(&mut buf, &[MetadataEntry::CreatedAt(0)]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x01, // one metadata entry 0x01, // entry type: created_at = 0x01 0x08, // two byte entry size = 8 0x00, @@ -155,19 +139,18 @@ mod tests { 0x00, 0x00, 0xE3, // crc - 0x28]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + 0x28]); + }); } #[test] fn test_only_created_at_future() { let mut buf = vec![]; let the_future = Duration::from_millis(u64::MAX).as_secs(); - let written = write_metadata(&mut buf, &[MetadataEntry::CreatedAt(the_future)]); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x01, // one metadata entry + assert_matches!(write_metadata(&mut buf, &[MetadataEntry::CreatedAt(the_future)]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x01, // one metadata entry 0x01, // entry type: created_at = 0x01 0x08, // two byte entry size = 8 0x00, @@ -180,63 +163,50 @@ mod tests { 0x41, 0x00, 0x21, // crc - 0x4C]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + 0x4C]); + }); } #[test] fn test_both() { let mut buf = vec![]; - let written = write_metadata( - &mut buf, - &[ - MetadataEntry::TrackType(TrackType::Trip(20)), - MetadataEntry::CreatedAt(0), - ], - ); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x02, // two metadata entries - 0x00, // entry type: track_type = 0x00 - 0x05, // two byte entry size = 5 - 0x00, - 0x00, // track type: trip = 0x00 - 0x14, // four byte trip ID = 20 - 0x00, - 0x00, - 0x00, - 0x01, // entry type: created_at = 0x01 - 0x08, // two byte entry size = 8 - 0x00, - 0x00, // eight byte timestamp: zero seconds elapsed - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x23, // crc - 0xD2]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + assert_matches!(write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Trip(20)), + MetadataEntry::CreatedAt(0)]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, &[0x02, // two metadata entries + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x00, // track type: trip = 0x00 + 0x14, // four byte trip ID = 20 + 0x00, + 0x00, + 0x00, + 0x01, // entry type: created_at = 0x01 + 0x08, // two byte entry size = 8 + 0x00, + 0x00, // eight byte timestamp: zero seconds elapsed + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x23, // crc + 0xD2]); + }); } #[test] fn test_duplicate_types() { let mut buf = vec![]; - let written = write_metadata( - &mut buf, - &[ - MetadataEntry::TrackType(TrackType::Trip(20)), - MetadataEntry::TrackType(TrackType::Trip(21)), - MetadataEntry::TrackType(TrackType::Route(22)), - ], - ); - assert!(written.is_ok()); - #[rustfmt::skip] - let expected = &[0x03, // three metadata entries + assert_matches!(write_metadata(&mut buf, &[MetadataEntry::TrackType(TrackType::Trip(20)), + MetadataEntry::TrackType(TrackType::Trip(21)), + MetadataEntry::TrackType(TrackType::Route(22))]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x03, // three metadata entries 0x00, // entry type: track_type = 0x00 0x05, // two byte entry size = 5 0x00, @@ -262,8 +232,7 @@ mod tests { 0x00, 0x00, 0xDE, // crc - 0x57]; - assert_eq!(buf, expected); - assert_eq!(written.unwrap(), expected.len()); + 0x57]); + }); } } From d6ec5556b30d6c24d435e11bf6c8b3edb8a98aff Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 13 Dec 2021 10:57:51 -0800 Subject: [PATCH 021/113] Data Table now includes all the info needed to parse each section --- tracklib2/src/write/data_table.rs | 120 ++++++++++++++++++++++++++++++ tracklib2/src/write/mod.rs | 1 + 2 files changed, 121 insertions(+) create mode 100644 tracklib2/src/write/data_table.rs diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs new file mode 100644 index 0000000..e149bb3 --- /dev/null +++ b/tracklib2/src/write/data_table.rs @@ -0,0 +1,120 @@ +use super::crcwriter::CrcWriter; +use super::section::Section; +use crate::error::Result; +use std::convert::TryFrom; +use std::io::Write; + +#[rustfmt::skip] +pub(crate) fn write_data_table(out: &mut W, sections: &[Section]) -> Result<()> { + let mut crcwriter = CrcWriter::new16(out); + + crcwriter.write_all(&u8::try_from(sections.len())?.to_le_bytes())?; // 1 byte - number of sections + for section in sections.iter() { + crcwriter.write_all(§ion.type_tag().to_le_bytes())?; // 1 byte - section type + leb128::write::unsigned(&mut crcwriter, u64::try_from(section.rows())?)?; // ? bytes - number of points in this section + leb128::write::unsigned(&mut crcwriter, u64::try_from(section.data_size())?)?; // ? bytes - leb128 section size + section.write_types_table(&mut crcwriter)?; // ? bytes - types table + } + crcwriter.append_crc()?; // 2 bytes - crc + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::FieldType; + use crate::write::section::SectionType; + use assert_matches::assert_matches; + + #[test] + fn test_write_empty_data_table() { + let mut buf = Vec::new(); + assert_matches!(write_data_table(&mut buf, &[]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, &[0x00, // zero entries + 0x40, // crc + 0xBF]); + }); + } + + #[test] + fn test_data_table() { + let section1 = Section::new( + SectionType::TrackPoints, + vec![ + ("a".to_string(), FieldType::I64), + ("b".to_string(), FieldType::Bool), + ("c".to_string(), FieldType::String), + ], + ); + + let section2 = Section::new( + SectionType::CoursePoints, + vec![ + ("Ride".to_string(), FieldType::I64), + ("with".to_string(), FieldType::Bool), + ("GPS".to_string(), FieldType::String), + ], + ); + + let mut buf = Vec::new(); + assert_matches!(write_data_table(&mut buf, &[section1, section2]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x02, // number of sections + + // Section 1 + 0x00, // section type = track points + 0x00, // leb128 section point count + 0x00, // leb128 section data size + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x00, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x00, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x00, // leb128 data size + + + // Section 2 + 0x01, // section type = course points + 0x00, // leb128 section point count + 0x00, // leb128 section data size + + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x04, // name length + b'R', // name + b'i', // name + b'd', // name + b'e', // name + 0x00, // leb128 data size + 0x05, // second field type = Bool + 0x04, // name length + b'w', // name + b'i', // name + b't', // name + b'h', // name + 0x00, // leb128 data size + 0x04, // third field type = String + 0x03, // name length + b'G', // name + b'P', // name + b'S', // name + 0x00, // leb128 data size + + + 0x4E, // crc + 0x88], "{:#04X?}", buf); + }); + } +} diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs index 5a8e38f..35a3019 100644 --- a/tracklib2/src/write/mod.rs +++ b/tracklib2/src/write/mod.rs @@ -1,4 +1,5 @@ mod crcwriter; +mod data_table; pub mod encoders; mod header; pub mod metadata; From 060ee7392adc6d154c43a237cce84107131b267a Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 13 Dec 2021 12:34:14 -0800 Subject: [PATCH 022/113] Each data column now has its own crc --- tracklib2/src/write/section.rs | 300 ++++++++++--------- tracklib2/src/write/track.rs | 511 ++++++++++++++++----------------- 2 files changed, 406 insertions(+), 405 deletions(-) diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 3d8d182..e8c1069 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -1,3 +1,4 @@ +use super::crcwriter::CrcWriter; use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; use crate::consts::{CRC16, CRC32}; use crate::error::Result; @@ -23,10 +24,11 @@ struct BufferImpl { } impl BufferImpl { - fn write_data(&self, out: &mut W) -> Result { - let written = io::copy(&mut io::Cursor::new(&self.buf), out)?; - // io::copy (annoyingly) returns u64 so coerce it to a usize to return - Ok(usize::try_from(written).unwrap()) + fn write_data(&self, out: &mut W) -> Result<()> { + let mut crcwriter = CrcWriter::new32(out); + io::copy(&mut io::Cursor::new(&self.buf), &mut crcwriter)?; + crcwriter.append_crc()?; + Ok(()) } fn data_size(&self) -> usize { @@ -49,6 +51,14 @@ impl Buffer { &FieldType::String => Buffer::String(BufferImpl::default()), } } + + fn len(&self) -> usize { + match self { + Self::I64(buffer_impl) => buffer_impl.buf.len(), + Self::Bool(buffer_impl) => buffer_impl.buf.len(), + Self::String(buffer_impl) => buffer_impl.buf.len(), + } + } } pub enum SectionType { @@ -101,8 +111,23 @@ impl Section { RowBuilder::new(&self.fields, &mut self.column_data) } - fn write_presence_column(&self, out: &mut W) -> Result { - let mut bytes_written = 0; + pub(crate) fn type_tag(&self) -> u8 { + self.section_type.type_tag() + } + + pub(crate) fn rows(&self) -> usize { + self.rows_written + } + + pub(crate) fn data_size(&self) -> usize { + let presence_bytes_required = (self.fields.len() + 7) / 8; + let presence_bytes = presence_bytes_required * self.rows_written; + let data_bytes: usize = self.column_data.iter().map(|buffer| buffer.len()).sum(); + data_bytes + presence_bytes + } + + fn write_presence_column(&self, out: &mut W) -> Result<()> { + let mut crcwriter = CrcWriter::new32(out); let bytes_required = (self.fields.len() + 7) / 8; for row_i in 0..self.rows_written { @@ -134,18 +159,16 @@ impl Section { entry |= bit << field_i; } - out.write_all(&entry.to_le_bytes()[..bytes_required])?; - bytes_written += bytes_required; + crcwriter.write_all(&entry.to_le_bytes()[..bytes_required])?; } + crcwriter.append_crc()?; - Ok(bytes_written) + Ok(()) } #[rustfmt::skip] - fn write_types_table(&self, out: &mut W) -> Result { - let mut buf = Vec::new(); - - buf.write_all(&u8::try_from(self.fields.len())?.to_le_bytes())?; // 1 byte - number of entries + pub(crate) fn write_types_table(&self, out: &mut W) -> Result<()> { + out.write_all(&u8::try_from(self.fields.len())?.to_le_bytes())?; // 1 byte - number of entries for (i, field) in self.fields.iter().enumerate() { let data_column_size = self @@ -158,37 +181,27 @@ impl Section { }) .unwrap_or(0); - buf.write_all(&field.fieldtype().type_tag().to_le_bytes())?; // 1 byte - field type tag - buf.write_all(&u8::try_from(field.name().len())?.to_le_bytes())?; // 1 byte - field name length - buf.write_all(field.name().as_bytes())?; // ? bytes - the name of this field - leb128::write::unsigned(&mut buf, u64::try_from(data_column_size)?)?; // ? bytes - leb128 column data size + out.write_all(&field.fieldtype().type_tag().to_le_bytes())?; // 1 byte - field type tag + out.write_all(&u8::try_from(field.name().len())?.to_le_bytes())?; // 1 byte - field name length + out.write_all(field.name().as_bytes())?; // ? bytes - the name of this field + leb128::write::unsigned(out, u64::try_from(data_column_size)?)?; // ? bytes - leb128 column data size } - out.write_all(&buf)?; - Ok(buf.len()) + Ok(()) } #[rustfmt::skip] - pub(crate) fn write(&self, out: &mut W) -> Result { - let mut buf = Vec::new(); - - buf.write_all(&self.section_type.type_tag().to_le_bytes())?; // 1 byte - section type - buf.write_all(&u32::try_from(self.rows_written)?.to_le_bytes())?; // 4 bytes - number of points in this section - self.write_types_table(&mut buf)?; // ? bytes - types table - self.write_presence_column(&mut buf)?; // ? bytes - presence column - + pub(crate) fn write(&self, out: &mut W) -> Result<()> { + self.write_presence_column(out)?; // ? bytes - presence column with crc for buffer in self.column_data.iter() { match buffer { - Buffer::I64(buffer_impl) => buffer_impl.write_data(&mut buf)?, // \ - Buffer::Bool(buffer_impl) => buffer_impl.write_data(&mut buf)?, // \ - Buffer::String(buffer_impl) => buffer_impl.write_data(&mut buf)?, // > ? bytes - data column + Buffer::I64(buffer_impl) => buffer_impl.write_data(out)?, // \ + Buffer::Bool(buffer_impl) => buffer_impl.write_data(out)?, // \ + Buffer::String(buffer_impl) => buffer_impl.write_data(out)?, // > ? bytes - data column with crc }; } - buf.write_all(&CRC32.checksum(&buf).to_le_bytes())?; // 4 bytes - crc - - out.write_all(&buf)?; - Ok(buf.len()) + Ok(()) } } @@ -286,10 +299,13 @@ mod tests { ], ); let mut buf = Vec::new(); - let bytes_written = section.write_presence_column(&mut buf); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), 0); - assert_eq!(buf.len(), 0); + assert_matches!(section.write_presence_column(&mut buf), Ok(()) => { + assert_eq!(buf, + &[0x00, // crc + 0x00, + 0x00, + 0x00]); + }); let m_vals = vec![Some(&42), Some(&0), None, Some(&-20)]; let k_vals = vec![Some(&true), None, Some(&false), Some(&false)]; @@ -317,14 +333,19 @@ mod tests { } } - let bytes_written = section.write_presence_column(&mut buf); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), buf.len()); - #[rustfmt::skip] - assert_eq!(buf, &[0b00000011, - 0b00000101, - 0b00000110, - 0b00000111]); + let mut buf = Vec::new(); + assert_matches!(section.write_presence_column(&mut buf), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0b00000011, + 0b00000101, + 0b00000110, + 0b00000111, + 0xD2, // crc + 0x61, + 0xA7, + 0xA5]); + }); } #[test] @@ -366,12 +387,16 @@ mod tests { } let mut buf = Vec::new(); - let bytes_written = section.write_presence_column(&mut buf); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), buf.len()); - #[rustfmt::skip] - assert_eq!(buf, &[0b11111111, 0b11111111, 0b00001111, - 0b11111111, 0b11111111, 0b00001111]); + assert_matches!(section.write_presence_column(&mut buf), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0b11111111, 0b11111111, 0b00001111, + 0b11111111, 0b11111111, 0b00001111, + 0x91, // crc + 0xA0, + 0x07, + 0xE3]); + }); } #[test] @@ -402,36 +427,36 @@ mod tests { } let mut buf = Vec::new(); - let bytes_written = section.write_types_table(&mut buf); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), buf.len()); - #[rustfmt::skip] - assert_eq!(buf, &[0x04, // entry count = 4 - 0x00, // first entry type: i64 = 0 - 0x01, // name len = 1 - b'm', // name = "m" - 0x02, // data size = 2 - 0x05, // second entry type: bool = 5 - 0x01, // name len = 1 - b'k', // name = "k" - 0x01, // data size = 1 - 0x04, // third entry type: string = 4 - 0x0A, // name len = 10 - b'l', // name = "long name!" - b'o', - b'n', - b'g', - b' ', - b'n', - b'a', - b'm', - b'e', - b'!', - 0x07, // data size = 7 ("Hello!" + leb128 length prefix) - 0x00, // fourth entry type: i64 = 0 - 0x01, // name len = 1 - b'i', // name = "i" - 0x02]); // data size = 2 + assert_matches!(section.write_types_table(&mut buf), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x04, // entry count = 4 + 0x00, // first entry type: i64 = 0 + 0x01, // name len = 1 + b'm', // name = "m" + 0x02, // data size = 2 + 0x05, // second entry type: bool = 5 + 0x01, // name len = 1 + b'k', // name = "k" + 0x01, // data size = 1 + 0x04, // third entry type: string = 4 + 0x0A, // name len = 10 + b'l', // name = "long name!" + b'o', + b'n', + b'g', + b' ', + b'n', + b'a', + b'm', + b'e', + b'!', + 0x07, // data size = 7 ("Hello!" + leb128 length prefix) + 0x00, // fourth entry type: i64 = 0 + 0x01, // name len = 1 + b'i', // name = "i" + 0x02]); // data size = 2 + }); } #[test] @@ -514,66 +539,55 @@ mod tests { } let mut buf = Vec::new(); - let bytes_written = section.write(&mut buf); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), buf.len()); - #[rustfmt::skip] - assert_eq!(buf, &[0x00, // section type = track points - 0x03, // point count - 0x00, - 0x00, - 0x00, - - // Types Table - 0x03, // field count - 0x00, // first field type = I64 - 0x01, // name length - b'a', // name - 0x03, // leb128 data size - 0x05, // second field type = Bool - 0x01, // name length - b'b', // name - 0x03, // leb128 data size - 0x04, // third field type = String - 0x01, // name length - b'c', // name - 0x0E, // leb128 data size - - // Presence Column - 0b00000111, - 0b00000101, - 0b00000111, - - // Data Column 1 = I64 - 0x01, // 1 - 0x01, // 2 - 0x02, // 4 - - // Data Column 2 = Bool - 0x00, // false - 0x00, // missing - 0x01, // true - - // Data Column 3 = String - 0x04, // length 4 - b'R', - b'i', - b'd', - b'e', - 0x04, // length 4 - b'w', - b'i', - b't', - b'h', - 0x03, // length 3 - b'G', - b'P', - b'S', - - // CRC - 0xAF, - 0xEC, - 0x7D, - 0x70]); + assert_matches!(section.write(&mut buf), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, &[ + // Presence Column + 0b00000111, + 0b00000101, + 0b00000111, + 0x1A, // crc + 0x75, + 0xEA, + 0xC4, + + // Data Column 1 = I64 + 0x01, // 1 + 0x01, // 2 + 0x02, // 4 + 0xCA, // crc + 0xD4, + 0xD8, + 0x92, + + // Data Column 2 = Bool + 0x00, // false + 0x00, // missing + 0x01, // true + 0x48, // crc + 0x9F, + 0x5A, + 0x4C, + + // Data Column 3 = String + 0x04, // length 4 + b'R', + b'i', + b'd', + b'e', + 0x04, // length 4 + b'w', + b'i', + b't', + b'h', + 0x03, // length 3 + b'G', + b'P', + b'S', + 0xA3, // crc + 0x02, + 0xEC, + 0x48]); + }); } } diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 166fb06..5cb589e 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -1,3 +1,4 @@ +use super::data_table::write_data_table; use super::metadata::write_metadata; use super::section::Section; use crate::consts::{CRC16, RWTF_HEADER_SIZE}; @@ -6,127 +7,86 @@ use crate::types::MetadataEntry; use std::convert::TryFrom; use std::io::{self, Write}; -#[rustfmt::skip] -fn write_data_table(out: &mut W, section_bufs: &[Vec]) -> Result { - let mut buf = Vec::new(); - - buf.write_all(&u8::try_from(section_bufs.len())?.to_le_bytes())?; // 1 byte - number of sections - - for section in section_bufs.iter() { - leb128::write::unsigned(&mut buf, u64::try_from(section.len())?)?; // ? bytes - leb128 section size - } - - buf.write_all(&CRC16.checksum(&buf).to_le_bytes())?; // 2 bytes - crc - - out.write_all(&buf)?; - Ok(buf.len()) -} - pub fn write_track( out: &mut W, metadata_entries: &[MetadataEntry], sections: &[Section], -) -> Result { - let mut bytes_written = 0; - +) -> Result<()> { // write metadata to a buffer so we can measure its size to use in the file header let mut metadata_buf = Vec::new(); write_metadata(&mut metadata_buf, metadata_entries)?; // write header - bytes_written += super::header::write_header( + super::header::write_header( out, RWTF_HEADER_SIZE, RWTF_HEADER_SIZE + u16::try_from(metadata_buf.len())?, )?; // copy metadata buffer to out - bytes_written += usize::try_from(io::copy(&mut io::Cursor::new(metadata_buf), out)?)?; - - // create bufs for all sections - let section_bufs: Vec> = sections - .iter() - .map(|section| { - let mut buf = Vec::new(); - section.write(&mut buf)?; - Ok(buf) - }) - .collect::>()?; + io::copy(&mut io::Cursor::new(metadata_buf), out)?; // write the data table - bytes_written += write_data_table(out, §ion_bufs)?; + write_data_table(out, §ions)?; // now write out all the data sections - for section in section_bufs { - bytes_written += usize::try_from(io::copy(&mut io::Cursor::new(section), out)?)?; + for section in sections { + section.write(out)?; } - Ok(bytes_written) + Ok(()) } #[cfg(test)] mod tests { use super::*; - use crate::types::FieldType; + use crate::types::{FieldType, TrackType}; use crate::write::section::{ColumnWriter, SectionType}; use assert_matches::assert_matches; use std::collections::HashMap; - #[test] - fn test_write_empty_data_table() { - let mut buf = Vec::new(); - let bytes_written = write_data_table(&mut buf, &[]); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), buf.len()); - #[rustfmt::skip] - assert_eq!(buf, &[0x00, // zero entries - 0x40, // crc - 0xBF]); - } - #[test] fn test_empty_track() { let mut buf = Vec::new(); - let bytes_written = write_track(&mut buf, &[], &[]); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), buf.len()); - #[rustfmt::skip] - assert_eq!(buf, &[ - // Header - 0x89, // rwtfmagic - 0x52, - 0x57, - 0x54, - 0x46, - 0x0A, - 0x1A, - 0x0A, - 0x01, // file version - 0x00, // fv reserve - 0x00, - 0x00, - 0x00, // creator version - 0x00, // cv reserve - 0x00, - 0x00, - 0x18, // metadata table offset - 0x00, - 0x1B, // data offset - 0x00, - 0x00, // e reserve - 0x00, - 0x84, // header crc - 0xF8, - - // Metadata Table - 0x00, // zero metadata entries - 0x40, // crc - 0xBF, - - // Data Table - 0x00, // zero sections - 0x40, // crc - 0xBF]); + assert_matches!(write_track(&mut buf, &[], &[]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, &[ + // Header + 0x89, // rwtfmagic + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x01, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x18, // metadata table offset + 0x00, + 0x1B, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x84, // header crc + 0xF8, + + // Metadata Table + 0x00, // zero metadata entries + 0x40, // crc + 0xBF, + + // Data Table + 0x00, // zero sections + 0x40, // crc + 0xBF]); + }); } #[test] @@ -236,179 +196,206 @@ mod tests { } let mut buf = Vec::new(); - let bytes_written = write_track(&mut buf, &[], &[section1, section2]); - assert!(bytes_written.is_ok()); - assert_eq!(bytes_written.unwrap(), buf.len()); - #[rustfmt::skip] - assert_eq!(buf, &[ - // Header - 0x89, // rwtfmagic - 0x52, - 0x57, - 0x54, - 0x46, - 0x0A, - 0x1A, - 0x0A, - 0x01, // file version - 0x00, // fv reserve - 0x00, - 0x00, - 0x00, // creator version - 0x00, // cv reserve - 0x00, - 0x00, - 0x18, // metadata table offset - 0x00, - 0x1B, // data offset - 0x00, - 0x00, // e reserve - 0x00, - 0x84, // header crc - 0xF8, - - // Metadata Table - 0x00, // zero entries - 0x40, // crc - 0xBF, - - // Data Table - 0x02, // two entries - 0x39, // size of first entry - 0x2D, // size of second entry - 0xFD, // crc - 0xB2, - - - // Section 1 - 0x01, // section type = course points - 0x05, // point count - 0x00, - 0x00, - 0x00, - - // Types Table - 0x03, // field count - 0x00, // first field type = I64 - 0x01, // name len - b'm', // name - 0x05, // leb128 data size - 0x05, // second field type = Bool - 0x01, // name len - b'k', // name - 0x05, // leb128 data size - 0x04, // third field type = String - 0x01, // name len - b'j', // name - 0x14, // leb128 data size - - // Presence Column - 0b00000111, - 0b00000111, - 0b00000111, - 0b00000111, - 0b00000111, - - // Data Column 1 = I64 - 0x2A, // 42 - 0x00, // no change - 0x00, // no change - 0x00, // no change - 0x00, // no change - - // Data Column 2 = Bool - 0x01, // true - 0x01, // true - 0x01, // true - 0x01, // true - 0x01, // true - - // Data Column 3 = String - 0x03, // length 3 - b'h', - b'e', - b'y', - 0x03, // length 3 - b'h', - b'e', - b'y', - 0x03, // length 3 - b'h', - b'e', - b'y', - 0x03, // length 3 - b'h', - b'e', - b'y', - 0x03, // length 3 - b'h', - b'e', - b'y', - - // CRC - 0x5B, - 0xC3, - 0x0F, - 0x6E, - - - // Section 2 - 0x00, // section type = track points - 0x03, // point count - 0x00, - 0x00, - 0x00, - - // Types Table - 0x03, // field count - 0x00, // first field type = I64 - 0x01, // name length - b'a', // name - 0x03, // leb128 data size - 0x05, // second field type = Bool - 0x01, // name length - b'b', // name - 0x03, // leb128 data size - 0x04, // third field type = String - 0x01, // name length - b'c', // name - 0x0E, // leb128 data size - - // Presence Column - 0b00000111, - 0b00000101, - 0b00000111, - - // Data Column 1 = I64 - 0x01, // 1 - 0x01, // 2 - 0x02, // 4 - - // Data Column 2 = Bool - 0x00, // false - 0x00, // missing - 0x01, // true - - // Data Column 3 = String - 0x04, // length 4 - b'R', - b'i', - b'd', - b'e', - 0x04, // length 4 - b'w', - b'i', - b't', - b'h', - 0x03, // length 3 - b'G', - b'P', - b'S', - - // CRC - 0xAF, - 0xEC, - 0x7D, - 0x70]); + assert_matches!(write_track(&mut buf, + &[MetadataEntry::TrackType(TrackType::Segment(5))], + &[section1, section2]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, &[ + // Header + 0x89, // rwtfmagic + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x01, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x18, // metadata table offset + 0x00, + 0x23, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x89, // header crc + 0x98, + + // Metadata Table + 0x01, // one entry + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x02, // track type: segment = 0x02 + 0x05, // four byte segment ID + 0x00, + 0x00, + 0x00, + 0xD4, // crc + 0x93, + + // Data Table + 0x02, // two sections + + // Data Table Section 1 + 0x01, // type of section = course points + 0x05, // leb128 point count + 0x23, // leb128 data size + + // Types Table for Section 1 + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name len + b'm', // name + 0x05, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name len + b'k', // name + 0x05, // leb128 data size + 0x04, // third field type = String + 0x01, // name len + b'j', // name + 0x14, // leb128 data size + + // Data Table Section 2 + 0x00, // type of section = track points + 0x03, // leb128 point count + 0x17, // leb128 data size + + // Types Table for Section 2 + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x03, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x03, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x0E, // leb128 data size + + // Data Table CRC + 0x6C, + 0xB4, + + // Data Section 1 + + // Presence Column + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + 0xF6, // crc + 0xF8, + 0x0D, + 0x73, + + // Data Column 1 = I64 + 0x2A, // 42 + 0x00, // no change + 0x00, // no change + 0x00, // no change + 0x00, // no change + 0xD0, // crc + 0x8D, + 0x79, + 0x68, + + // Data Column 2 = Bool + 0x01, // true + 0x01, // true + 0x01, // true + 0x01, // true + 0x01, // true + 0xB5, // crc + 0xC9, + 0x8F, + 0xFA, + + // Data Column 3 = String + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x36, // crc + 0x71, + 0x24, + 0x0B, + + // Data Section 2 + + // Presence Column + 0b00000111, + 0b00000101, + 0b00000111, + 0x1A, // crc + 0x75, + 0xEA, + 0xC4, + + // Data Column 1 = I64 + 0x01, // 1 + 0x01, // 2 + 0x02, // 4 + 0xCA, // crc + 0xD4, + 0xD8, + 0x92, + + // Data Column 2 = Bool + 0x00, // false + 0x00, // missing + 0x01, // true + 0x48, // crc + 0x9F, + 0x5A, + 0x4C, + + // Data Column 3 = String + 0x04, // length 4 + b'R', + b'i', + b'd', + b'e', + 0x04, // length 4 + b'w', + b'i', + b't', + b'h', + 0x03, // length 3 + b'G', + b'P', + b'S', + 0xA3, // crc + 0x02, + 0xEC, + 0x48]); + }); } } From 8aba3dc118db8bfe8ff04b86591ae9901ebf8da8 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 13:02:58 -0800 Subject: [PATCH 023/113] move SectionType to crate::types --- tracklib2/src/types.rs | 7 +++++++ tracklib2/src/write/data_table.rs | 5 ++--- tracklib2/src/write/section.rs | 25 ++++++++++--------------- tracklib2/src/write/track.rs | 4 ++-- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tracklib2/src/types.rs b/tracklib2/src/types.rs index 5cc1e98..756b3b8 100644 --- a/tracklib2/src/types.rs +++ b/tracklib2/src/types.rs @@ -40,3 +40,10 @@ pub enum MetadataEntry { TrackType(TrackType), CreatedAt(u64), } + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub enum SectionType { + TrackPoints, + CoursePoints, +} diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index e149bb3..175fc35 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -23,8 +23,7 @@ pub(crate) fn write_data_table(out: &mut W, sections: &[Section]) -> R #[cfg(test)] mod tests { use super::*; - use crate::types::FieldType; - use crate::write::section::SectionType; + use crate::types::{FieldType, SectionType}; use assert_matches::assert_matches; #[test] @@ -114,7 +113,7 @@ mod tests { 0x4E, // crc - 0x88], "{:#04X?}", buf); + 0x88]); }); } } diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index e8c1069..dbf6e75 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -2,7 +2,7 @@ use super::crcwriter::CrcWriter; use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; use crate::consts::{CRC16, CRC32}; use crate::error::Result; -use crate::types::{FieldDescription, FieldType}; +use crate::types::{FieldDescription, FieldType, SectionType}; use std::convert::TryFrom; use std::io::{self, Write}; @@ -16,6 +16,15 @@ impl FieldType { } } +impl SectionType { + fn type_tag(&self) -> u8 { + match self { + Self::TrackPoints => 0x00, + Self::CoursePoints => 0x01, + } + } +} + #[derive(Default, Debug)] struct BufferImpl { buf: Vec, @@ -61,20 +70,6 @@ impl Buffer { } } -pub enum SectionType { - TrackPoints, - CoursePoints, -} - -impl SectionType { - fn type_tag(&self) -> u8 { - match self { - Self::TrackPoints => 0x00, - Self::CoursePoints => 0x01, - } - } -} - pub struct Section { section_type: SectionType, rows_written: usize, diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 5cb589e..7bd87ec 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -40,8 +40,8 @@ pub fn write_track( #[cfg(test)] mod tests { use super::*; - use crate::types::{FieldType, TrackType}; - use crate::write::section::{ColumnWriter, SectionType}; + use crate::types::{FieldType, SectionType, TrackType}; + use crate::write::section::ColumnWriter; use assert_matches::assert_matches; use std::collections::HashMap; From 11a102d42662c8d71e5e1fe24b024169bbd277b3 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 13:04:32 -0800 Subject: [PATCH 024/113] rename these test vecs to match their column names --- tracklib2/src/write/section.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index dbf6e75..ac53ae3 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -302,9 +302,9 @@ mod tests { 0x00]); }); - let m_vals = vec![Some(&42), Some(&0), None, Some(&-20)]; - let k_vals = vec![Some(&true), None, Some(&false), Some(&false)]; - let j_vals = vec![ + let a_vals = vec![Some(&42), Some(&0), None, Some(&-20)]; + let b_vals = vec![Some(&true), None, Some(&false), Some(&false)]; + let c_vals = vec![ None, Some("hi".to_string()), Some("tracklib".to_string()), @@ -316,13 +316,13 @@ mod tests { while let Some(cw) = rowbuilder.next_column_writer() { match cw { ColumnWriter::I64ColumnWriter(cwi) => { - assert!(cwi.write(m_vals[i]).is_ok()); + assert!(cwi.write(a_vals[i]).is_ok()); } ColumnWriter::BoolColumnWriter(cwi) => { - assert!(cwi.write(k_vals[i]).is_ok()); + assert!(cwi.write(b_vals[i]).is_ok()); } ColumnWriter::StringColumnWriter(cwi) => { - assert!(cwi.write(j_vals[i].as_ref()).is_ok()); + assert!(cwi.write(c_vals[i].as_ref()).is_ok()); } } } From b7b23650837c7d012991a78e8e0f7caba1c30abc Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 13:07:20 -0800 Subject: [PATCH 025/113] rewrite presence column writing code This code is capable of writing more than 64 fields, and also the byte order is reversed - making it more clear, IMO. --- tracklib2/src/write/section.rs | 107 +++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 32 deletions(-) diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index ac53ae3..600edbd 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -126,35 +126,26 @@ impl Section { let bytes_required = (self.fields.len() + 7) / 8; for row_i in 0..self.rows_written { - let mut entry: u64 = 0; - for (field_i, buffer) in self.column_data.iter().enumerate() { - let bit = match buffer { - Buffer::I64(buffer_impl) => { - if let Some(true) = buffer_impl.presence.get(row_i) { - 1 - } else { - 0 - } - } - Buffer::Bool(buffer_impl) => { - if let Some(true) = buffer_impl.presence.get(row_i) { - 1 - } else { - 0 - } - } - Buffer::String(buffer_impl) => { - if let Some(true) = buffer_impl.presence.get(row_i) { - 1 - } else { - 0 - } - } + let mut row = vec![0; bytes_required]; + let mut mask: u8 = 1; + let mut bit_index = (self.fields.len() + 7) & !7; // next multiple of 8 + for buffer in self.column_data.iter() { + let is_present = match buffer { + Buffer::I64(buffer_impl) => buffer_impl.presence.get(row_i), + Buffer::Bool(buffer_impl) => buffer_impl.presence.get(row_i), + Buffer::String(buffer_impl) => buffer_impl.presence.get(row_i), }; - entry |= bit << field_i; + + if let Some(true) = is_present { + let byte_index = ((bit_index + 7) / 8) - 1; + row[byte_index] |= mask; + } + mask = mask.rotate_left(1); + + bit_index -= 1; } - crcwriter.write_all(&entry.to_le_bytes()[..bytes_required])?; + crcwriter.write_all(&row)?; } crcwriter.append_crc()?; @@ -385,12 +376,64 @@ mod tests { assert_matches!(section.write_presence_column(&mut buf), Ok(()) => { #[rustfmt::skip] assert_eq!(buf, - &[0b11111111, 0b11111111, 0b00001111, - 0b11111111, 0b11111111, 0b00001111, - 0x91, // crc - 0xA0, - 0x07, - 0xE3]); + &[0b00001111, 0b11111111, 0b11111111, + 0b00001111, 0b11111111, 0b11111111, + 0xDD, // crc + 0xCB, + 0x18, + 0x17]); + }); + } + + #[test] + fn test_write_huge_presence_column() { + let mut section = Section::new( + SectionType::TrackPoints, + (0..80).map(|i| (i.to_string(), FieldType::Bool)).collect(), + ); + + #[rustfmt::skip] + let vals = &[ + Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), // 1 + Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), // 2 + None, None, Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), // 3 + Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), None, None, None, // 4 + Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), // 5 + None, Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), // 6 + Some(&true), Some(&true), Some(&true), Some(&true), None, None, Some(&true), Some(&true), // 7 + None, None, None, None, None, None, None, None, // 8 + Some(&true), Some(&true), Some(&true), Some(&true), None, Some(&true), None, None, // 9 + None, None, None, Some(&true), Some(&true), Some(&true), Some(&true), Some(&true), // 10 + ]; + + let mut rowbuilder = section.open_row_builder(); + let mut i = 0; + while let Some(cw) = rowbuilder.next_column_writer() { + match cw { + ColumnWriter::BoolColumnWriter(cwi) => assert!(cwi.write(vals[i]).is_ok()), + _ => assert!(false, "unexpected column writer type here"), + } + i += 1; + } + + let mut buf = Vec::new(); + assert_matches!(section.write_presence_column(&mut buf), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0b11111000, // 10 + 0b00101111, // 9 + 0b00000000, // 8 + 0b11001111, // 7 + 0b11111110, // 6 + 0b11111111, // 5 + 0b00011111, // 4 + 0b11111100, // 3 + 0b11111111, // 2 + 0b11111111, // 1 + 0x92, // crc + 0x0E, + 0x6F, + 0xC2]); }); } From 1ffe837f13b53f8d0f0ab9b7a313ab6d0da6ed4f Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 13:23:56 -0800 Subject: [PATCH 026/113] crate::read::presence_column module --- tracklib2/src/read/mod.rs | 1 + tracklib2/src/read/presence_column.rs | 178 ++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 tracklib2/src/read/presence_column.rs diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 58f1e1f..bdfc58e 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1,4 +1,5 @@ mod crc; mod header; mod metadata; +mod presence_column; mod types_table; diff --git a/tracklib2/src/read/presence_column.rs b/tracklib2/src/read/presence_column.rs new file mode 100644 index 0000000..290ee41 --- /dev/null +++ b/tracklib2/src/read/presence_column.rs @@ -0,0 +1,178 @@ +use super::crc::CRC; +use crate::error::TracklibError; +use nom::{bytes::complete::take, IResult}; + +#[cfg_attr(test, derive(Debug))] +pub(crate) struct PresenceColumn<'a> { + data: &'a [u8], + fields: usize, + rows: usize, +} + +impl<'a> PresenceColumn<'a> { + fn iter(&self) -> PresenceColumnIter { + PresenceColumnIter { + data: &self.data, + fields: self.fields, + bytes_required: (self.fields + 7) / 8, + } + } +} + +struct PresenceColumnIter<'a> { + data: &'a [u8], + fields: usize, + bytes_required: usize, +} + +impl<'a> Iterator for PresenceColumnIter<'a> { + type Item = PresenceRow<'a>; + + fn next(&mut self) -> Option { + if self.bytes_required <= self.data.len() { + let (row, rest) = self.data.split_at(self.bytes_required); + self.data = rest; + Some(PresenceRow::new(row, self.fields)) + } else { + None + } + } +} + +struct PresenceRow<'a> { + data: &'a [u8], + mask: u8, + bit_index: usize, + fields: usize, +} + +impl<'a> PresenceRow<'a> { + fn new(data: &'a [u8], fields: usize) -> Self { + Self { + data, + mask: 1, + bit_index: (fields + 7) & !7, // next multiple of 8 + fields, + } + } +} + +impl<'a> Iterator for PresenceRow<'a> { + type Item = bool; + + fn next(&mut self) -> Option { + if self.fields > 0 { + let byte_index = ((self.bit_index + 7) / 8) - 1; + let is_field_present = self.data[byte_index] & self.mask > 0; + self.mask = self.mask.rotate_left(1); + self.fields -= 1; + self.bit_index -= 1; + Some(is_field_present) + } else { + None + } + } +} + +pub(crate) fn parse_presence_column<'a>( + fields: usize, + rows: usize, +) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], PresenceColumn<'a>, TracklibError> { + let bytes_required = (fields + 7) / 8; + let size = bytes_required * rows; + move |input: &[u8]| { + let input_start = input; + let (input, data) = take(size)(input)?; + let (input, checksum) = CRC::::parser(input_start)(input)?; + + match checksum { + CRC::Valid(_) => Ok((input, PresenceColumn { data, fields, rows })), + CRC::Invalid { expected, computed } => { + Err(nom::Err::Error(TracklibError::CRC32Error { + expected, + computed, + })) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + #[test] + fn test_read_presence_column() { + #[rustfmt::skip] + let buf = &[0b00000011, + 0b00000101, + 0b00000110, + 0b00000111, + 0xD2, // crc + 0x61, + 0xA7, + 0xA5]; + assert_matches!(parse_presence_column(3, 4)(buf), Ok((&[], presence_column)) => { + let vals: Vec> = presence_column.iter().map(|row| row.collect()).collect(); + #[rustfmt::skip] + assert_eq!(&vals, &[&[true, true, false], + &[true, false, true], + &[false, true, true], + &[true, true, true]]); + + }); + } + + #[test] + fn test_read_multibyte_presence_column() { + #[rustfmt::skip] + let buf = &[0b00001111, 0b11111111, 0b11111111, + 0b00001111, 0b11111111, 0b11111111, + 0xDD, // crc + 0xCB, + 0x18, + 0x17]; + assert_matches!(parse_presence_column(20, 2)(buf), Ok((&[], presence_column)) => { + let vals: Vec> = presence_column.iter().map(|row| row.collect()).collect(); + assert_eq!(&vals, &[std::iter::repeat(true).take(20).collect::>(), + std::iter::repeat(true).take(20).collect::>()]); + + }); + } + + #[test] + fn test_read_huge_presence_column() { + #[rustfmt::skip] + let buf = &[0b11111000, // 10 + 0b00101111, // 9 + 0b00000000, // 8 + 0b11001111, // 7 + 0b11111110, // 6 + 0b11111111, // 5 + 0b00011111, // 4 + 0b11111100, // 3 + 0b11111111, // 2 + 0b11111111, // 1 + 0x92, // crc + 0x0E, + 0x6F, + 0xC2]; + assert_matches!(parse_presence_column(80, 1)(buf), Ok((&[], presence_column)) => { + let vals: Vec> = presence_column.iter().map(|row| row.collect()).collect(); + #[rustfmt::skip] + assert_eq!(&vals, &[&[ + true, true, true, true, true, true, true, true, // 1 + true, true, true, true, true, true, true, true, // 2 + false, false, true, true, true, true, true, true, // 3 + true, true, true, true, true, false, false, false, // 4 + true, true, true, true, true, true, true, true, // 5 + false, true, true, true, true, true, true, true, // 6 + true, true, true, true, false, false, true, true, // 7 + false, false, false, false, false, false, false, false, // 8 + true, true, true, true, false, true, false, false, // 9 + false, false, false, true, true, true, true, true, // 10 + ]]); + }); + } +} From c9a96beabf61ca8dd07be9f3476a793e5ab6f615 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 14:16:22 -0800 Subject: [PATCH 027/113] add crc32 to crate::read::crc --- tracklib2/src/read/crc.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tracklib2/src/read/crc.rs b/tracklib2/src/read/crc.rs index df811e0..4b409a8 100644 --- a/tracklib2/src/read/crc.rs +++ b/tracklib2/src/read/crc.rs @@ -1,5 +1,8 @@ use crate::error::TracklibError; -use nom::{number::complete::le_u16, IResult, Offset}; +use nom::{ + number::complete::{le_u16, le_u32}, + IResult, Offset, +}; pub(crate) trait CRCImpl: Sized { fn crc_bytes(bytes: &[u8]) -> Self; @@ -16,6 +19,16 @@ impl CRCImpl for u16 { } } +impl CRCImpl for u32 { + fn crc_bytes(bytes: &[u8]) -> Self { + crate::consts::CRC32.checksum(bytes) + } + + fn read_bytes(input: &[u8]) -> IResult<&[u8], Self, TracklibError> { + le_u32(input) + } +} + #[derive(Debug, PartialEq)] pub(crate) enum CRC { Valid(T), @@ -50,14 +63,6 @@ mod tests { use super::*; use assert_matches::assert_matches; - // #[test] - // fn test_crc_full_thing() { - // let bytes = &[0x00, 0xBF, 0x40,]; - // println!("crc of &[0x00]: {:X?}", u16::crc_bytes(&[0x00]).to_le_bytes()); - // println!("crc of &{:X?}: {:X?}", bytes, u16::crc_bytes(bytes).to_le_bytes()); - // assert_eq!(u16::crc_bytes(bytes), u16::from_le_bytes([0xFF, 0xFF])); - // } - #[test] fn test_emtpy_crc() { let buf = &[0x00, 0x00]; From 90e98bd4899cac818bb54cf7739cd4227b183a22 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 14:19:08 -0800 Subject: [PATCH 028/113] code cleanup --- tracklib2/src/read/header.rs | 4 ++-- tracklib2/src/read/metadata.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tracklib2/src/read/header.rs b/tracklib2/src/read/header.rs index 4a8ffe1..702e365 100644 --- a/tracklib2/src/read/header.rs +++ b/tracklib2/src/read/header.rs @@ -17,7 +17,7 @@ pub struct Header { } pub(crate) fn parse_header(input: &[u8]) -> IResult<&[u8], Header, TracklibError> { - let header_start = input; + let input_start = input; let (input, _magic) = tag(RWTFMAGIC)(input)?; let (input, file_version) = le_u8(input)?; let (input, _fv_reserve) = take(3_usize)(input)?; @@ -26,7 +26,7 @@ pub(crate) fn parse_header(input: &[u8]) -> IResult<&[u8], Header, TracklibError let (input, metadata_offset) = le_u16(input)?; let (input, data_offset) = le_u16(input)?; let (input, _e_reserve) = take(2_usize)(input)?; - let (input, checksum) = CRC::::parser(header_start)(input)?; + let (input, checksum) = CRC::::parser(input_start)(input)?; match checksum { CRC::Valid(_) => Ok(( diff --git a/tracklib2/src/read/metadata.rs b/tracklib2/src/read/metadata.rs index 68a24ad..132d86e 100644 --- a/tracklib2/src/read/metadata.rs +++ b/tracklib2/src/read/metadata.rs @@ -57,7 +57,7 @@ fn parse_metadata_entry(input: &[u8]) -> IResult<&[u8], Option, T } pub(crate) fn parse_metadata(input: &[u8]) -> IResult<&[u8], Vec, TracklibError> { - let metadata_start = input; + let input_start = input; let (mut input, entry_count) = le_u8(input)?; let mut entries = Vec::with_capacity(usize::from(entry_count)); @@ -69,7 +69,7 @@ pub(crate) fn parse_metadata(input: &[u8]) -> IResult<&[u8], Vec, } } - let (input, checksum) = CRC::::parser(metadata_start)(input)?; + let (input, checksum) = CRC::::parser(input_start)(input)?; match checksum { CRC::Valid(_) => Ok((input, entries)), @@ -92,7 +92,7 @@ mod tests { 0x40, // crc 0xBF]; assert_matches!(parse_metadata(buf), Ok((&[], entries)) => { - assert_eq!(entries, vec![]) + assert_eq!(entries, vec![]); }); } @@ -123,7 +123,7 @@ mod tests { 0xD2]; assert_matches!(parse_metadata(buf), Ok((&[], entries)) => { assert_eq!(entries, vec![MetadataEntry::TrackType(TrackType::Trip(20)), - MetadataEntry::CreatedAt(0)]) + MetadataEntry::CreatedAt(0)]); }); } @@ -177,7 +177,7 @@ mod tests { 0x85]; assert_matches!(parse_metadata(buf), Ok((&[], entries)) => { assert_eq!(entries, vec![MetadataEntry::TrackType(TrackType::Trip(20)), - MetadataEntry::CreatedAt(0)]) + MetadataEntry::CreatedAt(0)]); }); } From 29973313780206873d7e10e4b731cfb729b833ec Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 14:23:34 -0800 Subject: [PATCH 029/113] crate::read::data_table module --- tracklib2/src/read/data_table.rs | 187 +++++++++++++++++++++++++++++++ tracklib2/src/read/mod.rs | 1 + 2 files changed, 188 insertions(+) create mode 100644 tracklib2/src/read/data_table.rs diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs new file mode 100644 index 0000000..b116bea --- /dev/null +++ b/tracklib2/src/read/data_table.rs @@ -0,0 +1,187 @@ +use super::crc::CRC; +use super::types_table::{parse_types_table, TypesTableEntry}; +use crate::error::TracklibError; +use crate::types::SectionType; +use nom::{number::complete::le_u8, IResult}; +use nom_leb128::leb128_u64; +use std::convert::TryFrom; + +#[cfg_attr(test, derive(Debug, PartialEq))] +pub struct DataTableEntry<'a> { + section_type: SectionType, + offset: usize, + size: usize, + rows: usize, + types: Vec>, +} + +impl<'a> DataTableEntry<'a> { + pub fn section_type(&self) -> &SectionType { + &self.section_type + } + + pub fn offset(&self) -> usize { + self.offset + } + + pub fn size(&self) -> usize { + self.size + } + + pub fn rows(&self) -> usize { + self.rows + } + + pub fn types(&self) -> &[TypesTableEntry<'a>] { + self.types.as_slice() + } +} + +fn parse_data_table_entry( + offset: usize, +) -> impl Fn(&[u8]) -> IResult<&[u8], DataTableEntry, TracklibError> { + move |input: &[u8]| { + let (input, type_tag) = le_u8(input)?; + let (input, rows) = leb128_u64(input)?; + let (input, size) = leb128_u64(input)?; + let (input, types) = parse_types_table(input)?; + + let section_type = match type_tag { + 0x00 => SectionType::TrackPoints, + 0x01 => SectionType::CoursePoints, + _ => { + return Err(nom::Err::Error(TracklibError::ParseError { + error_kind: nom::error::ErrorKind::Tag, + })) + } + }; + + Ok(( + input, + DataTableEntry { + section_type, + offset, + size: usize::try_from(size).expect("usize != u64"), + rows: usize::try_from(rows).expect("usize != u64"), + types, + }, + )) + } +} + +pub(crate) fn parse_data_table(input: &[u8]) -> IResult<&[u8], Vec, TracklibError> { + let input_start = input; + let (mut input, entry_count) = le_u8(input)?; + let mut entries = Vec::with_capacity(usize::from(entry_count)); + let mut offset = 0; + for _ in 0..entry_count { + let (rest, entry) = parse_data_table_entry(offset)(input)?; + input = rest; + offset += entry.size; + entries.push(entry); + } + let (input, checksum) = CRC::::parser(input_start)(input)?; + + match checksum { + CRC::Valid(_) => Ok((input, entries)), + CRC::Invalid { expected, computed } => Err(nom::Err::Error(TracklibError::CRC16Error { + expected, + computed, + })), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::FieldType; + use assert_matches::assert_matches; + + #[test] + fn test_parse_data_table() { + #[rustfmt::skip] + let buf = &[0x02, // number of sections + + // Section 1 + 0x00, // section type = track points + 0x00, // leb128 section point count + 0x00, // leb128 section data size + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x00, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x00, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x00, // leb128 data size + + + // Section 2 + 0x01, // section type = course points + 0x00, // leb128 section point count + 0x00, // leb128 section data size + + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x04, // name length + b'R', // name + b'i', // name + b'd', // name + b'e', // name + 0x00, // leb128 data size + 0x05, // second field type = Bool + 0x04, // name length + b'w', // name + b'i', // name + b't', // name + b'h', // name + 0x00, // leb128 data size + 0x04, // third field type = String + 0x03, // name length + b'G', // name + b'P', // name + b'S', // name + 0x00, // leb128 data size + + + 0x4E, // crc + 0x88]; + + assert_matches!(parse_data_table(buf), Ok((&[], entries)) => { + assert_eq!( + entries, + vec![ + DataTableEntry { + section_type: SectionType::TrackPoints, + offset: 0, + size: 0, + rows: 0, + types: vec![ + TypesTableEntry::new_for_tests(FieldType::I64, "a", 0, 0), + TypesTableEntry::new_for_tests(FieldType::Bool, "b", 0, 0), + TypesTableEntry::new_for_tests(FieldType::String, "c", 0, 0) + ] + }, + DataTableEntry { + section_type: SectionType::CoursePoints, + offset: 0, + size: 0, + rows: 0, + types: vec![ + TypesTableEntry::new_for_tests(FieldType::I64, "Ride", 0, 0), + TypesTableEntry::new_for_tests(FieldType::Bool, "with", 0, 0), + TypesTableEntry::new_for_tests(FieldType::String, "GPS", 0, 0) + ] + } + ] + ); + }); + } +} diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index bdfc58e..245dc62 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1,4 +1,5 @@ mod crc; +mod data_table; mod header; mod metadata; mod presence_column; From f3953f1c219667fe64c0dec40ce0b50c7fe5ef72 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 16 Dec 2021 14:25:32 -0800 Subject: [PATCH 030/113] proper conversion from nom::Err to TracklibError --- tracklib2/src/error.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tracklib2/src/error.rs b/tracklib2/src/error.rs index 98ed39f..a2d5582 100644 --- a/tracklib2/src/error.rs +++ b/tracklib2/src/error.rs @@ -5,6 +5,9 @@ pub enum TracklibError { #[error("Parse Error")] ParseError { error_kind: nom::error::ErrorKind }, + #[error("Parse Incomplete")] + ParseIncompleteError { needed: nom::Needed }, + #[error("CRC Error")] CRC16Error { expected: u16, computed: u16 }, @@ -34,3 +37,13 @@ impl nom::error::ParseError for TracklibError { } impl nom::error::ContextError for TracklibError {} + +impl From> for TracklibError { + fn from(error: nom::Err) -> Self { + match error { + nom::Err::Incomplete(needed) => Self::ParseIncompleteError { needed }, + nom::Err::Error(e) => e, + nom::Err::Failure(e) => e, + } + } +} From 394c97cd183f8570650ff3a16decfaede507751b Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 3 Mar 2022 14:18:29 -0800 Subject: [PATCH 031/113] changes to the file format 1) No longer writing 0x00 for missing values. The reader should simply not read any bytes if the presence column indicates no entry. 2) Column data size now includes the trailing CRC32. --- tracklib2/src/lib.rs | 2 +- tracklib2/src/write/data_table.rs | 20 +++++------ tracklib2/src/write/encoders.rs | 55 ++++++++++++++----------------- tracklib2/src/write/section.rs | 30 ++++++++++------- tracklib2/src/write/track.rs | 30 ++++++++--------- 5 files changed, 69 insertions(+), 68 deletions(-) diff --git a/tracklib2/src/lib.rs b/tracklib2/src/lib.rs index 748e1de..6e564bf 100644 --- a/tracklib2/src/lib.rs +++ b/tracklib2/src/lib.rs @@ -1,5 +1,5 @@ -pub mod error; mod consts; +pub mod error; pub mod read; pub mod types; pub mod write; diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index 175fc35..258b106 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -66,27 +66,27 @@ mod tests { // Section 1 0x00, // section type = track points 0x00, // leb128 section point count - 0x00, // leb128 section data size + 0x10, // leb128 section data size // Types Table 0x03, // field count 0x00, // first field type = I64 0x01, // name length b'a', // name - 0x00, // leb128 data size + 0x04, // leb128 data size 0x05, // second field type = Bool 0x01, // name length b'b', // name - 0x00, // leb128 data size + 0x04, // leb128 data size 0x04, // third field type = String 0x01, // name length b'c', // name - 0x00, // leb128 data size + 0x04, // leb128 data size // Section 2 0x01, // section type = course points 0x00, // leb128 section point count - 0x00, // leb128 section data size + 0x10, // leb128 section data size // Types Table 0x03, // field count @@ -96,24 +96,24 @@ mod tests { b'i', // name b'd', // name b'e', // name - 0x00, // leb128 data size + 0x04, // leb128 data size 0x05, // second field type = Bool 0x04, // name length b'w', // name b'i', // name b't', // name b'h', // name - 0x00, // leb128 data size + 0x04, // leb128 data size 0x04, // third field type = String 0x03, // name length b'G', // name b'P', // name b'S', // name - 0x00, // leb128 data size + 0x04, // leb128 data size - 0x4E, // crc - 0x88]); + 0xFB, // crc + 0xF8]); }); } } diff --git a/tracklib2/src/write/encoders.rs b/tracklib2/src/write/encoders.rs index 81a10cd..3d1bed9 100644 --- a/tracklib2/src/write/encoders.rs +++ b/tracklib2/src/write/encoders.rs @@ -27,18 +27,13 @@ impl Encoder for I64Encoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - let delta = match value { - Some(v) => { - let value = *v; - let delta = value - self.prev; - self.prev = value; - delta - } - None => 0, - }; + if let Some(v) = value { + let value = *v; + let delta = value - self.prev; + self.prev = value; + leb128::write::signed(buf, delta)?; + } - // Write the signed delta from the previous value - leb128::write::signed(buf, delta)?; Ok(()) } } @@ -56,8 +51,10 @@ impl Encoder for BoolEncoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - let v = *(value.unwrap_or(&false)) as u8; - buf.write_all(&v.to_le_bytes())?; + if let Some(v) = value { + let v = *(value.unwrap_or(&false)) as u8; + buf.write_all(&v.to_le_bytes())?; + } Ok(()) } } @@ -80,9 +77,6 @@ impl Encoder for StringEncoder { leb128::write::unsigned(buf, u64::try_from(string.len()).unwrap())?; // Write the string itself buf.write_all(string.as_bytes())?; - } else { - // Write a zero length and nothing else - buf.write_all(&vec![0])?; } Ok(()) @@ -130,8 +124,8 @@ mod tests { 0x01, // +1 from 2 0x99, // -103 from 3 0x7F, - 0x00, // None - 0x00, // None + // None + // None 0x00, // staying at -100 0xC8, // +200 from -100 0x01]); @@ -176,13 +170,13 @@ mod tests { .is_ok()); #[rustfmt::skip] - assert_eq!(data_buf, &[0x01, - 0x01, - 0x00, - 0x00, - 0x00, - 0x00, - 0x01]); + assert_eq!(data_buf, &[0x01, // true + 0x01, // true + 0x00, // false + // None + // None + 0x00, // false + 0x01]); // true #[rustfmt::skip] assert_eq!(presence_buf, &[true, @@ -232,13 +226,13 @@ mod tests { #[rustfmt::skip] assert_eq!(data_buf, &[0x01, // length b'A', // A - 0x00, // None + // None 0x01, // length b'B', // B - 0x00, // None + // None 0x01, // length b'C', // C - 0x00, // None + // None 0x0D, // length b'H', b'e', @@ -252,8 +246,9 @@ mod tests { b'r', b'l', b'd', - b'!', - 0x00]); // None + b'!' + // None + ]); #[rustfmt::skip] assert_eq!(presence_buf, &[true, diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 600edbd..c5878c2 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -41,7 +41,8 @@ impl BufferImpl { } fn data_size(&self) -> usize { - self.buf.len() + const CRC_BYTES: usize = 4; + self.buf.len() + CRC_BYTES } } @@ -115,9 +116,14 @@ impl Section { } pub(crate) fn data_size(&self) -> usize { + const CRC_BYTES: usize = 4; let presence_bytes_required = (self.fields.len() + 7) / 8; - let presence_bytes = presence_bytes_required * self.rows_written; - let data_bytes: usize = self.column_data.iter().map(|buffer| buffer.len()).sum(); + let presence_bytes = (presence_bytes_required * self.rows_written) + CRC_BYTES; + let data_bytes: usize = self + .column_data + .iter() + .map(|buffer| buffer.len() + CRC_BYTES) + .sum(); data_bytes + presence_bytes } @@ -472,11 +478,11 @@ mod tests { 0x00, // first entry type: i64 = 0 0x01, // name len = 1 b'm', // name = "m" - 0x02, // data size = 2 + 0x06, // data size = 6 0x05, // second entry type: bool = 5 0x01, // name len = 1 b'k', // name = "k" - 0x01, // data size = 1 + 0x05, // data size = 5 0x04, // third entry type: string = 4 0x0A, // name len = 10 b'l', // name = "long name!" @@ -489,11 +495,11 @@ mod tests { b'm', b'e', b'!', - 0x07, // data size = 7 ("Hello!" + leb128 length prefix) + 0x0B, // data size = 11 0x00, // fourth entry type: i64 = 0 0x01, // name len = 1 b'i', // name = "i" - 0x02]); // data size = 2 + 0x06]); // data size = 6 }); } @@ -600,12 +606,12 @@ mod tests { // Data Column 2 = Bool 0x00, // false - 0x00, // missing + // None 0x01, // true - 0x48, // crc - 0x9F, - 0x5A, - 0x4C, + 0x35, // crc + 0x86, + 0x89, + 0xFB, // Data Column 3 = String 0x04, // length 4 diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 7bd87ec..c6086a2 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -246,46 +246,46 @@ mod tests { // Data Table Section 1 0x01, // type of section = course points 0x05, // leb128 point count - 0x23, // leb128 data size + 0x33, // leb128 data size // Types Table for Section 1 0x03, // field count 0x00, // first field type = I64 0x01, // name len b'm', // name - 0x05, // leb128 data size + 0x09, // leb128 data size 0x05, // second field type = Bool 0x01, // name len b'k', // name - 0x05, // leb128 data size + 0x09, // leb128 data size 0x04, // third field type = String 0x01, // name len b'j', // name - 0x14, // leb128 data size + 0x18, // leb128 data size // Data Table Section 2 0x00, // type of section = track points 0x03, // leb128 point count - 0x17, // leb128 data size + 0x26, // leb128 data size // Types Table for Section 2 0x03, // field count 0x00, // first field type = I64 0x01, // name length b'a', // name - 0x03, // leb128 data size + 0x07, // leb128 data size 0x05, // second field type = Bool 0x01, // name length b'b', // name - 0x03, // leb128 data size + 0x06, // leb128 data size 0x04, // third field type = String 0x01, // name length b'c', // name - 0x0E, // leb128 data size + 0x12, // leb128 data size // Data Table CRC - 0x6C, - 0xB4, + 0x49, + 0xEC, // Data Section 1 @@ -370,12 +370,12 @@ mod tests { // Data Column 2 = Bool 0x00, // false - 0x00, // missing + // None 0x01, // true - 0x48, // crc - 0x9F, - 0x5A, - 0x4C, + 0x35, // crc + 0x86, + 0x89, + 0xFB, // Data Column 3 = String 0x04, // length 4 From b7b08b7cf08ceffdaec0f618732399c98765cc9a Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 7 Mar 2022 11:48:08 -0800 Subject: [PATCH 032/113] crate::read::section_reader and crate::read::decoders modules --- tracklib2/src/error.rs | 3 + tracklib2/src/read/data_table.rs | 8 +- tracklib2/src/read/decoders.rs | 280 +++++++++++++++++++++++++ tracklib2/src/read/mod.rs | 6 +- tracklib2/src/read/presence_column.rs | 184 ++++++++-------- tracklib2/src/read/section_reader.rs | 289 ++++++++++++++++++++++++++ tracklib2/src/read/types_table.rs | 151 +++++++++----- tracklib2/src/types.rs | 9 + 8 files changed, 785 insertions(+), 145 deletions(-) create mode 100644 tracklib2/src/read/decoders.rs create mode 100644 tracklib2/src/read/section_reader.rs diff --git a/tracklib2/src/error.rs b/tracklib2/src/error.rs index a2d5582..da34753 100644 --- a/tracklib2/src/error.rs +++ b/tracklib2/src/error.rs @@ -11,6 +11,9 @@ pub enum TracklibError { #[error("CRC Error")] CRC16Error { expected: u16, computed: u16 }, + #[error("CRC Error")] + CRC32Error { expected: u32, computed: u32 }, + #[error("Numeric Bounds Error")] BoundsError { #[from] diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index b116bea..5f5e46d 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -7,15 +7,15 @@ use nom_leb128::leb128_u64; use std::convert::TryFrom; #[cfg_attr(test, derive(Debug, PartialEq))] -pub struct DataTableEntry<'a> { +pub struct DataTableEntry { section_type: SectionType, offset: usize, size: usize, rows: usize, - types: Vec>, + types: Vec, } -impl<'a> DataTableEntry<'a> { +impl DataTableEntry { pub fn section_type(&self) -> &SectionType { &self.section_type } @@ -32,7 +32,7 @@ impl<'a> DataTableEntry<'a> { self.rows } - pub fn types(&self) -> &[TypesTableEntry<'a>] { + pub fn types(&self) -> &[TypesTableEntry] { self.types.as_slice() } } diff --git a/tracklib2/src/read/decoders.rs b/tracklib2/src/read/decoders.rs new file mode 100644 index 0000000..b81b991 --- /dev/null +++ b/tracklib2/src/read/decoders.rs @@ -0,0 +1,280 @@ +use super::crc::CRC; +use super::presence_column::PresenceColumnView; +use crate::error::Result; +use crate::error::TracklibError; +use nom::{multi::length_data, number::complete::le_u8}; +use nom_leb128::{leb128_i64, leb128_u64}; + +pub(crate) trait Decoder { + type T; + fn decode(&mut self) -> Result>; +} + +fn validate_column(data: &[u8]) -> Result<&[u8]> { + println!("validate_column(data len={})", data.len()); + const CRC_BYTES: usize = 4; + let (column_data, crc_bytes) = data.split_at(data.len() - CRC_BYTES); + let (_, checksum) = CRC::::parser(column_data)(crc_bytes)?; + + match checksum { + CRC::Valid(_) => Ok(column_data), + CRC::Invalid { expected, computed } => { + Err(TracklibError::CRC32Error { expected, computed }) + } + } +} + +#[cfg_attr(test, derive(Debug))] +pub(crate) struct I64Decoder<'a> { + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, + prev: i64, +} + +impl<'a> I64Decoder<'a> { + pub(crate) fn new( + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, + ) -> Result { + let column_data = validate_column(data)?; + + Ok(Self { + data: column_data, + presence_column_view, + prev: 0, + }) + } +} + +impl<'a> Decoder for I64Decoder<'a> { + type T = i64; + + fn decode(&mut self) -> Result> { + if let Some(is_present) = self.presence_column_view.next() { + if is_present { + let (rest, value) = leb128_i64(self.data)?; + let new = self.prev + value; + self.prev = new; + self.data = rest; + Ok(Some(new)) + } else { + Ok(None) + } + } else { + // ran out of rows, this is an error + return Err(TracklibError::ParseIncompleteError { + needed: nom::Needed::Unknown, + }); + } + } +} + +#[cfg_attr(test, derive(Debug))] +pub(crate) struct BoolDecoder<'a> { + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, +} + +impl<'a> BoolDecoder<'a> { + pub(crate) fn new( + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, + ) -> Result { + let column_data = validate_column(data)?; + + Ok(Self { + data: column_data, + presence_column_view, + }) + } +} + +impl<'a> Decoder for BoolDecoder<'a> { + type T = bool; + + fn decode(&mut self) -> Result> { + if let Some(is_present) = self.presence_column_view.next() { + if is_present { + let (rest, value) = le_u8(self.data)?; + self.data = rest; + Ok(Some(value != 0)) + } else { + Ok(None) + } + } else { + // ran out of rows, this is an error + return Err(TracklibError::ParseIncompleteError { + needed: nom::Needed::Unknown, + }); + } + } +} + +#[cfg_attr(test, derive(Debug))] +pub(crate) struct StringDecoder<'a> { + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, +} + +impl<'a> StringDecoder<'a> { + pub(crate) fn new( + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, + ) -> Result { + let column_data = validate_column(data)?; + + Ok(Self { + data: column_data, + presence_column_view, + }) + } +} + +impl<'a> Decoder for StringDecoder<'a> { + type T = String; + + fn decode(&mut self) -> Result> { + if let Some(is_present) = self.presence_column_view.next() { + if is_present { + let (rest, string_bytes) = length_data(leb128_u64)(self.data)?; + self.data = rest; + Ok(Some(String::from_utf8(string_bytes.to_vec()).unwrap())) + } else { + Ok(None) + } + } else { + // ran out of rows, this is an error + return Err(TracklibError::ParseIncompleteError { + needed: nom::Needed::Unknown, + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::read::presence_column::parse_presence_column; + use assert_matches::assert_matches; + + #[test] + fn test_decode_i64() { + #[rustfmt::skip] + let presence_buf = &[0b00000001, + 0b00000001, + 0b00000000, + 0b00000001, + 0b00000001, + 0x32, // crc + 0x65, + 0x57, + 0xFB]; + assert_matches!(parse_presence_column(1, 5)(presence_buf), Ok((&[], presence_column)) => { + let presence_column_view = presence_column.view(0); + #[rustfmt::skip] + let buf = &[0x20, + 0x42, + 0x00, + 0x1, + 0x0C, // crc + 0x01, + 0x49, + 0xA3]; + assert_matches!(I64Decoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(Some(32))); + assert_matches!(decoder.decode(), Ok(Some(-30))); + assert_matches!(decoder.decode(), Ok(None)); + assert_matches!(decoder.decode(), Ok(Some(-30))); + assert_matches!(decoder.decode(), Ok(Some(-29))); + }); + }); + } + + #[test] + fn test_decode_bool() { + #[rustfmt::skip] + let presence_buf = &[0b00000000, + 0b00000001, + 0b00000001, + 0x94, // crc + 0x5E, + 0x43, + 0x9E]; + assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], presence_column)) => { + let presence_column_view = presence_column.view(0); + #[rustfmt::skip] + let buf = &[0x01, + 0x00, + 0x5E, // crc + 0x5A, + 0x51, + 0x2D]; + assert_matches!(BoolDecoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(None)); + assert_matches!(decoder.decode(), Ok(Some(true))); + assert_matches!(decoder.decode(), Ok(Some(false))); + }); + }); + } + + #[test] + fn test_decode_string() { + #[rustfmt::skip] + let presence_buf = &[0b00000000, + 0b00000001, + 0b00000001, + 0x94, // crc + 0x5E, + 0x43, + 0x9E]; + assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], presence_column)) => { + let presence_column_view = presence_column.view(0); + #[rustfmt::skip] + let buf = &[0x01, + b'R', + 0x03, + b'i', + b'd', + b'e', + 0x73, // crc + 0x91, + 0x5A, + 0x74]; + assert_matches!(StringDecoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(None)); + assert_matches!(decoder.decode(), Ok(Some(s)) => { + assert_eq!(s, "R"); + }); + assert_matches!(decoder.decode(), Ok(Some(s)) => { + assert_eq!(s, "ide"); + }); + }); + }); + } + + #[test] + fn test_decode_bad_crc() { + #[rustfmt::skip] + let presence_buf = &[0b00000000, + 0b00000001, + 0b00000001, + 0x94, // crc + 0x5E, + 0x43, + 0x9E]; + assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], presence_column)) => { + let presence_column_view = presence_column.view(0); + #[rustfmt::skip] + let buf = &[0x00, + 0x01, + 0x02, + 0x00, // invalid crc + 0x00, + 0x00, + 0x00]; + assert_matches!(StringDecoder::new(buf, presence_column_view), + Err(crate::error::TracklibError::CRC32Error{expected: 0x00, + computed: 0x9300784D})); + }); + } +} diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 245dc62..d0bc6a9 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1,6 +1,10 @@ +mod columns; mod crc; mod data_table; +mod decoders; mod header; mod metadata; mod presence_column; -mod types_table; +mod section_reader; +pub mod track; +pub mod types_table; diff --git a/tracklib2/src/read/presence_column.rs b/tracklib2/src/read/presence_column.rs index 290ee41..9295db3 100644 --- a/tracklib2/src/read/presence_column.rs +++ b/tracklib2/src/read/presence_column.rs @@ -10,64 +10,55 @@ pub(crate) struct PresenceColumn<'a> { } impl<'a> PresenceColumn<'a> { - fn iter(&self) -> PresenceColumnIter { - PresenceColumnIter { - data: &self.data, - fields: self.fields, - bytes_required: (self.fields + 7) / 8, - } + // TODO: should this return a Result? + /// View into a particular field in the presence column. + pub(crate) fn view(&self, index: usize) -> PresenceColumnView<'a> { + let bit_index = 1 << (index % 8); + let presence_bytes_required = (self.fields + 7) / 8; + let x = (presence_bytes_required * 8 - index + 7) & !7; // next multiple of 8 + let byte_index = (x / 8) - 1; + + PresenceColumnView::new( + &self.data, + bit_index, + byte_index, + presence_bytes_required, + self.rows, + ) } } -struct PresenceColumnIter<'a> { - data: &'a [u8], - fields: usize, - bytes_required: usize, -} - -impl<'a> Iterator for PresenceColumnIter<'a> { - type Item = PresenceRow<'a>; - - fn next(&mut self) -> Option { - if self.bytes_required <= self.data.len() { - let (row, rest) = self.data.split_at(self.bytes_required); - self.data = rest; - Some(PresenceRow::new(row, self.fields)) - } else { - None - } - } -} - -struct PresenceRow<'a> { +#[cfg_attr(test, derive(Debug))] +pub(crate) struct PresenceColumnView<'a> { data: &'a [u8], mask: u8, - bit_index: usize, - fields: usize, + offset: usize, + step: usize, + index: usize, + max: usize, } -impl<'a> PresenceRow<'a> { - fn new(data: &'a [u8], fields: usize) -> Self { +impl<'a> PresenceColumnView<'a> { + fn new(data: &'a [u8], mask: u8, offset: usize, step: usize, max: usize) -> Self { Self { data, - mask: 1, - bit_index: (fields + 7) & !7, // next multiple of 8 - fields, + mask, + offset, + step, + index: 0, + max, } } } -impl<'a> Iterator for PresenceRow<'a> { +impl<'a> Iterator for PresenceColumnView<'a> { type Item = bool; fn next(&mut self) -> Option { - if self.fields > 0 { - let byte_index = ((self.bit_index + 7) / 8) - 1; - let is_field_present = self.data[byte_index] & self.mask > 0; - self.mask = self.mask.rotate_left(1); - self.fields -= 1; - self.bit_index -= 1; - Some(is_field_present) + if self.index < self.max { + let address = self.offset + (self.index * self.step); + self.index += 1; + Some(self.data[address] & self.mask > 0) } else { None } @@ -102,45 +93,6 @@ mod tests { use super::*; use assert_matches::assert_matches; - #[test] - fn test_read_presence_column() { - #[rustfmt::skip] - let buf = &[0b00000011, - 0b00000101, - 0b00000110, - 0b00000111, - 0xD2, // crc - 0x61, - 0xA7, - 0xA5]; - assert_matches!(parse_presence_column(3, 4)(buf), Ok((&[], presence_column)) => { - let vals: Vec> = presence_column.iter().map(|row| row.collect()).collect(); - #[rustfmt::skip] - assert_eq!(&vals, &[&[true, true, false], - &[true, false, true], - &[false, true, true], - &[true, true, true]]); - - }); - } - - #[test] - fn test_read_multibyte_presence_column() { - #[rustfmt::skip] - let buf = &[0b00001111, 0b11111111, 0b11111111, - 0b00001111, 0b11111111, 0b11111111, - 0xDD, // crc - 0xCB, - 0x18, - 0x17]; - assert_matches!(parse_presence_column(20, 2)(buf), Ok((&[], presence_column)) => { - let vals: Vec> = presence_column.iter().map(|row| row.collect()).collect(); - assert_eq!(&vals, &[std::iter::repeat(true).take(20).collect::>(), - std::iter::repeat(true).take(20).collect::>()]); - - }); - } - #[test] fn test_read_huge_presence_column() { #[rustfmt::skip] @@ -159,9 +111,13 @@ mod tests { 0x6F, 0xC2]; assert_matches!(parse_presence_column(80, 1)(buf), Ok((&[], presence_column)) => { - let vals: Vec> = presence_column.iter().map(|row| row.collect()).collect(); + let vals = (0..80) + .map(|i| presence_column.view(i)) + .flat_map(|view| view.collect::>()) + .collect::>(); + #[rustfmt::skip] - assert_eq!(&vals, &[&[ + assert_eq!(&vals, &[ true, true, true, true, true, true, true, true, // 1 true, true, true, true, true, true, true, true, // 2 false, false, true, true, true, true, true, true, // 3 @@ -172,7 +128,67 @@ mod tests { false, false, false, false, false, false, false, false, // 8 true, true, true, true, false, true, false, false, // 9 false, false, false, true, true, true, true, true, // 10 - ]]); + ]); + }); + } + + #[test] + fn test_presence_column_view() { + #[rustfmt::skip] + let buf = &[0b00000011, + 0b00000101, + 0b00000110, + 0b00000111, + 0xD2, // crc + 0x61, + 0xA7, + 0xA5]; + assert_matches!(parse_presence_column(3, 4)(buf), Ok((&[], presence_column)) => { + let view = presence_column.view(0); + assert_eq!(view.collect::>(), &[true, true, false, true]); + + let view = presence_column.view(1); + assert_eq!(view.collect::>(), &[true, false, true, true]); + + let view = presence_column.view(2); + assert_eq!(view.collect::>(), &[false, true, true, true]); + }); + } + + #[test] + #[should_panic] + fn test_presence_column_view_overflowing_column_index() { + #[rustfmt::skip] + let buf = &[0b00000011, + 0b00000101, + 0b00000110, + 0b00000111, + 0xD2, // crc + 0x61, + 0xA7, + 0xA5]; + assert_matches!(parse_presence_column(3, 4)(buf), Ok((&[], presence_column)) => { + presence_column.view(10); + }); + } + + #[test] + fn test_multibyte_presence_column_view() { + #[rustfmt::skip] + let buf = &[0b00000101, 0b11110100, 0b10111110, + 0b00001111, 0b11100111, 0b11111111, + 0xDF, // crc + 0xC7, + 0x91, + 0xF5]; + assert_matches!(parse_presence_column(20, 2)(buf), Ok((&[], presence_column)) => { + assert_eq!(presence_column.view(0).collect::>(), &[false, true]); + assert_eq!(presence_column.view(1).collect::>(), &[true, true]); + + assert_eq!(presence_column.view(9).collect::>(), &[false, true]); + assert_eq!(presence_column.view(10).collect::>(), &[true, true]); + assert_eq!(presence_column.view(11).collect::>(), &[false, false]); + assert_eq!(presence_column.view(12).collect::>(), &[true, false]); }); } } diff --git a/tracklib2/src/read/section_reader.rs b/tracklib2/src/read/section_reader.rs new file mode 100644 index 0000000..92133ad --- /dev/null +++ b/tracklib2/src/read/section_reader.rs @@ -0,0 +1,289 @@ +use super::data_table::DataTableEntry; +use super::decoders::{BoolDecoder, Decoder, I64Decoder, StringDecoder}; +use super::presence_column::parse_presence_column; +use crate::error::Result; +use crate::types::{FieldDescription, FieldType, FieldValue}; + +#[cfg_attr(test, derive(Debug))] +enum ColumnDecoder<'a> { + I64 { + field_description: &'a FieldDescription, + decoder: I64Decoder<'a>, + }, + Bool { + field_description: &'a FieldDescription, + decoder: BoolDecoder<'a>, + }, + String { + field_description: &'a FieldDescription, + decoder: StringDecoder<'a>, + }, +} + +#[cfg_attr(test, derive(Debug))] +pub struct SectionReader<'a> { + data_table_entry: &'a DataTableEntry, + decoders: Vec>, + rows: usize, +} + +impl<'a> SectionReader<'a> { + pub(crate) fn new(input: &'a [u8], data_table_entry: &'a DataTableEntry) -> Result { + let (column_data, presence_column) = + parse_presence_column(data_table_entry.types().len(), data_table_entry.rows())(input)?; + + let decoders = data_table_entry + .types() + .iter() + .enumerate() + .map(|(i, field)| { + let column_data = &column_data[field.offset()..field.offset() + field.size()]; + let presence_column_view = presence_column.view(i); + let field_description = field.field_description(); + let decoder = match field_description.fieldtype() { + FieldType::I64 => ColumnDecoder::I64 { + field_description, + decoder: I64Decoder::new(column_data, presence_column_view)?, + }, + FieldType::Bool => ColumnDecoder::Bool { + field_description, + decoder: BoolDecoder::new(column_data, presence_column_view)?, + }, + FieldType::String => ColumnDecoder::String { + field_description, + decoder: StringDecoder::new(column_data, presence_column_view)?, + }, + }; + Ok(decoder) + }) + .collect::>>()?; + + Ok(Self { + data_table_entry, + decoders, + rows: data_table_entry.rows(), + }) + } + + pub fn data_table_entry(&self) -> &'a DataTableEntry { + self.data_table_entry + } + + pub fn open_column_iter<'r>(&'r mut self) -> Option> { + if self.rows > 0 { + self.rows -= 1; + Some(ColumnIter::new(&mut self.decoders)) + } else { + None + } + } +} + +#[cfg_attr(test, derive(Debug))] +pub struct ColumnIter<'a, 'b> { + decoders: &'a mut Vec>, + index: usize, +} + +impl<'a, 'b> ColumnIter<'a, 'b> { + fn new(decoders: &'a mut Vec>) -> Self { + Self { decoders, index: 0 } + } +} + +impl<'a, 'b> Iterator for ColumnIter<'a, 'b> { + type Item = Result<(&'a FieldDescription, Option)>; + + fn next(&mut self) -> Option { + if let Some(decoder_enum) = self.decoders.get_mut(self.index) { + self.index += 1; + match decoder_enum { + ColumnDecoder::I64 { + ref field_description, + ref mut decoder, + } => Some( + decoder + .decode() + .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::I64(v)))) + .map_err(|e| e), + ), + ColumnDecoder::Bool { + ref field_description, + ref mut decoder, + } => Some( + decoder + .decode() + .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::Bool(v)))) + .map_err(|e| e), + ), + ColumnDecoder::String { + ref field_description, + ref mut decoder, + } => Some( + decoder + .decode() + .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::String(v)))) + .map_err(|e| e), + ), + } + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::read::data_table::parse_data_table; + use assert_matches::assert_matches; + + #[test] + fn test_section_reader() { + #[rustfmt::skip] + let data_table_buf = &[0x01, // number of sections + + // Section 1 + 0x00, // section type = track points + 0x03, // leb128 section point count + 0x26, // leb128 section data size + // Types Table + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x07, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x06, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x12, // leb128 data size + + 0x67, // crc + 0x3C]; + + assert_matches!(parse_data_table(data_table_buf), Ok((&[], data_table_entries)) => { + assert_eq!(data_table_entries.len(), 1); + + #[rustfmt::skip] + let buf = &[ + // Presence Column + 0b00000111, + 0b00000101, + 0b00000111, + 0x1A, // crc + 0x75, + 0xEA, + 0xC4, + + // Data Column 1 = I64 + 0x01, // 1 + 0x01, // 2 + 0x02, // 4 + 0xCA, // crc + 0xD4, + 0xD8, + 0x92, + + // Data Column 2 = Bool + 0x00, // false + // None + 0x01, // true + 0x35, // crc + 0x86, + 0x89, + 0xFB, + + // Data Column 3 = String + 0x04, // length 4 + b'R', + b'i', + b'd', + b'e', + 0x04, // length 4 + b'w', + b'i', + b't', + b'h', + 0x03, // length 3 + b'G', + b'P', + b'S', + 0xA3, // crc + 0x02, + 0xEC, + 0x48]; + + assert_matches!(SectionReader::new(buf, &data_table_entries[0]), Ok(mut section_reader) => { + // Row 1 + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 3); + assert_matches!(&values[0], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); + assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); + assert_eq!(field_value, &Some(FieldValue::I64(1))); + }); + assert_matches!(&values[1], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[1].field_description()); + assert_eq!(*field_description, &FieldDescription::new("b".to_string(), FieldType::Bool)); + assert_eq!(field_value, &Some(FieldValue::Bool(false))); + }); + assert_matches!(&values[2], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[2].field_description()); + assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); + assert_eq!(field_value, &Some(FieldValue::String("Ride".to_string()))); + }); + }); + + // Row 2 + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 3); + assert_matches!(&values[0], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); + assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); + assert_eq!(field_value, &Some(FieldValue::I64(2))); + }); + assert_matches!(&values[1], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[1].field_description()); + assert_eq!(*field_description, &FieldDescription::new("b".to_string(), FieldType::Bool)); + assert_eq!(field_value, &None); + }); + assert_matches!(&values[2], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[2].field_description()); + assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); + assert_eq!(field_value, &Some(FieldValue::String("with".to_string()))); + }); + }); + + // Row 3 + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 3); + assert_matches!(&values[0], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); + assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); + assert_eq!(field_value, &Some(FieldValue::I64(4))); + }); + assert_matches!(&values[1], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[1].field_description()); + assert_eq!(*field_description, &FieldDescription::new("b".to_string(), FieldType::Bool)); + assert_eq!(field_value, &Some(FieldValue::Bool(true))); + }); + assert_matches!(&values[2], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[2].field_description()); + assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); + assert_eq!(field_value, &Some(FieldValue::String("GPS".to_string()))); + }); + }); + + // Trying to get another row will return nothing + assert_matches!(section_reader.open_column_iter(), None); + }); + }); + } +} diff --git a/tracklib2/src/read/types_table.rs b/tracklib2/src/read/types_table.rs index 872c445..7e5650f 100644 --- a/tracklib2/src/read/types_table.rs +++ b/tracklib2/src/read/types_table.rs @@ -1,60 +1,99 @@ use crate::error::TracklibError; -use crate::types::FieldType; -use nom::{ - multi::{length_count, length_data}, - number::complete::le_u8, - IResult, -}; +use crate::types::{FieldDescription, FieldType}; +use nom::{multi::length_data, number::complete::le_u8, IResult}; use nom_leb128::leb128_u64; -use std::str; +use std::convert::TryFrom; #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] -struct TypesTableEntry<'a> { - fieldtype: FieldType, - name: &'a str, - size: u64, +pub struct TypesTableEntry { + field_description: FieldDescription, + size: usize, + offset: usize, +} + +#[cfg(test)] +impl TypesTableEntry { + pub(crate) fn new_for_tests( + fieldtype: FieldType, + name: &str, + size: usize, + offset: usize, + ) -> Self { + Self { + field_description: FieldDescription::new(name.to_string(), fieldtype), + size, + offset, + } + } +} + +impl TypesTableEntry { + pub(crate) fn field_description(&self) -> &FieldDescription { + &self.field_description + } + + pub(crate) fn size(&self) -> usize { + self.size + } + + pub(crate) fn offset(&self) -> usize { + self.offset + } } fn parse_types_table_entry<'a>( - input: &'a [u8], -) -> IResult<&'a [u8], TypesTableEntry, TracklibError> { - let (input, type_tag) = le_u8(input)?; - let (input, field_name) = length_data(le_u8)(input)?; - let (input, data_size) = leb128_u64(input)?; + offset: usize, +) -> impl Fn(&[u8]) -> IResult<&[u8], TypesTableEntry, TracklibError> { + move |input: &[u8]| { + let (input, type_tag) = le_u8(input)?; + let (input, field_name) = length_data(le_u8)(input)?; + let (input, data_size) = leb128_u64(input)?; - let fieldtype = match type_tag { - 0x00 => FieldType::I64, - 0x04 => FieldType::String, - 0x05 => FieldType::Bool, - _ => { - return Err(nom::Err::Error(TracklibError::ParseError { - error_kind: nom::error::ErrorKind::Tag, - })) - } - }; + let fieldtype = match type_tag { + 0x00 => FieldType::I64, + 0x04 => FieldType::String, + 0x05 => FieldType::Bool, + _ => { + return Err(nom::Err::Error(TracklibError::ParseError { + error_kind: nom::error::ErrorKind::Tag, + })) + } + }; - let name = match str::from_utf8(field_name) { - Ok(s) => s, - Err(_) => { - return Err(nom::Err::Error(TracklibError::ParseError { - error_kind: nom::error::ErrorKind::Tag, - })) - } - }; + let name = match String::from_utf8(field_name.to_vec()) { + Ok(s) => s, + Err(_) => { + return Err(nom::Err::Error(TracklibError::ParseError { + error_kind: nom::error::ErrorKind::Tag, + })) + } + }; - Ok(( - input, - TypesTableEntry { - fieldtype, - name, - size: data_size, - }, - )) + Ok(( + input, + TypesTableEntry { + field_description: FieldDescription::new(name, fieldtype), + size: usize::try_from(data_size).expect("usize != u64"), + offset, + }, + )) + } } -fn parse_types_table(input: &[u8]) -> IResult<&[u8], Vec, TracklibError> { - length_count(le_u8, parse_types_table_entry)(input) +pub(crate) fn parse_types_table( + input: &[u8], +) -> IResult<&[u8], Vec, TracklibError> { + let (mut input, entry_count) = le_u8(input)?; + let mut entries = Vec::with_capacity(usize::from(entry_count)); + let mut offset = 0; + for _ in 0..entry_count { + let (rest, entry) = parse_types_table_entry(offset)(input)?; + input = rest; + offset += entry.size; + entries.push(entry); + } + Ok((input, entries)) } #[cfg(test)] @@ -92,18 +131,18 @@ mod tests { b'i', // name = "i" 0x02]; // data size = 2 assert_matches!(parse_types_table(buf), Ok((&[], entries)) => { - assert_eq!(entries, vec![TypesTableEntry{fieldtype: FieldType::I64, - name: "m", - size: 2}, - TypesTableEntry{fieldtype: FieldType::Bool, - name: "k", - size: 1}, - TypesTableEntry{fieldtype: FieldType::String, - name: "long name!", - size: 7}, - TypesTableEntry{fieldtype: FieldType::I64, - name: "i", - size: 2}]); + assert_eq!(entries, vec![TypesTableEntry{field_description: FieldDescription::new("m".to_string(), FieldType::I64), + size: 2, + offset: 0}, + TypesTableEntry{field_description: FieldDescription::new("k".to_string(), FieldType::Bool), + size: 1, + offset: 2}, + TypesTableEntry{field_description: FieldDescription::new("long name!".to_string(), FieldType::String), + size: 7, + offset: 3}, + TypesTableEntry{field_description: FieldDescription::new("i".to_string(), FieldType::I64), + size: 2, + offset: 10}]); }); } diff --git a/tracklib2/src/types.rs b/tracklib2/src/types.rs index 756b3b8..95e5f9c 100644 --- a/tracklib2/src/types.rs +++ b/tracklib2/src/types.rs @@ -7,6 +7,7 @@ pub enum FieldType { } #[derive(Clone, Debug)] +#[cfg_attr(test, derive(PartialEq))] pub struct FieldDescription { name: String, fieldtype: FieldType, @@ -26,6 +27,14 @@ impl FieldDescription { } } +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub enum FieldValue { + I64(i64), + Bool(bool), + String(String), +} + #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum TrackType { From 92e1418964e632c1ade4e1b473cf023150ef2b35 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Wed, 9 Mar 2022 13:13:58 -0800 Subject: [PATCH 033/113] upgrade to 2021 edition --- tracklib2/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml index f8054ae..1eb5e5f 100644 --- a/tracklib2/Cargo.toml +++ b/tracklib2/Cargo.toml @@ -2,7 +2,7 @@ name = "tracklib2" version = "0.1.0" authors = ["Dan Larkin "] -edition = "2018" +edition = "2021" [dependencies] crc = "2.1" From 523c7bd3257177c96265f69e7d932a415d58050d Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Wed, 9 Mar 2022 13:14:17 -0800 Subject: [PATCH 034/113] remove reference to unused crate::read::columns module --- tracklib2/src/read/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index d0bc6a9..89229ba 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1,4 +1,3 @@ -mod columns; mod crc; mod data_table; mod decoders; From 36eb02105c27b7f0fb2fef042662b07b6aa7cdd3 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 10 Mar 2022 15:53:47 -0800 Subject: [PATCH 035/113] rwtfinspect --- rwtfinspect/.gitignore | 1 + rwtfinspect/Cargo.lock | 208 ++++++++++++++++++++ rwtfinspect/Cargo.toml | 10 + rwtfinspect/src/main.rs | 9 + tracklib2/Cargo.lock | 39 ++++ tracklib2/Cargo.toml | 4 + tracklib2/src/read/decoders.rs | 1 - tracklib2/src/read/inspect.rs | 338 +++++++++++++++++++++++++++++++++ tracklib2/src/read/mod.rs | 2 + 9 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 rwtfinspect/.gitignore create mode 100644 rwtfinspect/Cargo.lock create mode 100644 rwtfinspect/Cargo.toml create mode 100644 rwtfinspect/src/main.rs create mode 100644 tracklib2/src/read/inspect.rs diff --git a/rwtfinspect/.gitignore b/rwtfinspect/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/rwtfinspect/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/rwtfinspect/Cargo.lock b/rwtfinspect/Cargo.lock new file mode 100644 index 0000000..0936f32 --- /dev/null +++ b/rwtfinspect/Cargo.lock @@ -0,0 +1,208 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "crc" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "leb128" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", +] + +[[package]] +name = "nom-leb128" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a73b6c3a9ecfff12ad50dedba051ef838d2f478d938bb3e6b3842431028e62" +dependencies = [ + "arrayvec", + "nom", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "proc-macro2" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "rwtfinspect" +version = "0.1.0" +dependencies = [ + "tracklib2", +] + +[[package]] +name = "syn" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "term-table" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e59d7fb313157de2a568be8d81e4d7f9af6e50e697702e8e00190a6566d3b8" +dependencies = [ + "lazy_static", + "regex", + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracklib2" +version = "0.1.0" +dependencies = [ + "crc", + "leb128", + "nom", + "nom-leb128", + "term-table", + "thiserror", +] + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" diff --git a/rwtfinspect/Cargo.toml b/rwtfinspect/Cargo.toml new file mode 100644 index 0000000..25bf34b --- /dev/null +++ b/rwtfinspect/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rwtfinspect" +description = "Inspect RWTF Files" +version = "0.1.0" +authors = ["Dan Larkin "] +license = "Apache-2.0 OR MIT" +edition = "2021" + +[dependencies] +tracklib2 = {path = "../tracklib2", features=["inspect"]} diff --git a/rwtfinspect/src/main.rs b/rwtfinspect/src/main.rs new file mode 100644 index 0000000..9489936 --- /dev/null +++ b/rwtfinspect/src/main.rs @@ -0,0 +1,9 @@ +use tracklib2::read::inspect::inspect; + +fn main() -> Result<(), String> { + let filename = std::env::args().nth(1).ok_or_else(|| format!("usage: rwtfinspect "))?; + let data = std::fs::read(&filename).map_err(|e| format!("Error opening {filename}: {e:?}"))?; + let table = inspect(&data)?; + println!("{table}"); + Ok(()) +} diff --git a/tracklib2/Cargo.lock b/tracklib2/Cargo.lock index db404a5..d728bdf 100644 --- a/tracklib2/Cargo.lock +++ b/tracklib2/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "arrayvec" version = "0.7.2" @@ -352,6 +361,12 @@ dependencies = [ "libc", ] +[[package]] +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + [[package]] name = "oorandom" version = "11.1.3" @@ -419,7 +434,10 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", + "thread_local", ] [[package]] @@ -531,6 +549,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "term-table" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e59d7fb313157de2a568be8d81e4d7f9af6e50e697702e8e00190a6566d3b8" +dependencies = [ + "lazy_static", + "regex", + "unicode-width", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -560,6 +589,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + [[package]] name = "tinytemplate" version = "1.1.0" @@ -580,6 +618,7 @@ dependencies = [ "leb128", "nom", "nom-leb128", + "term-table", "thiserror", ] diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml index 1eb5e5f..e6fd8a2 100644 --- a/tracklib2/Cargo.toml +++ b/tracklib2/Cargo.toml @@ -10,11 +10,15 @@ leb128 = "0.2" nom = "7.1" nom-leb128 = "0.2" thiserror = "1.0" +term-table = {version = "1.3", optional = true} [dev-dependencies] assert_matches = "1.5" criterion = "0.3" +[features] +inspect = ["term-table"] + [[bench]] name = "parsing" harness = false diff --git a/tracklib2/src/read/decoders.rs b/tracklib2/src/read/decoders.rs index b81b991..bc8f6ec 100644 --- a/tracklib2/src/read/decoders.rs +++ b/tracklib2/src/read/decoders.rs @@ -11,7 +11,6 @@ pub(crate) trait Decoder { } fn validate_column(data: &[u8]) -> Result<&[u8]> { - println!("validate_column(data len={})", data.len()); const CRC_BYTES: usize = 4; let (column_data, crc_bytes) = data.split_at(data.len() - CRC_BYTES); let (_, checksum) = CRC::::parser(column_data)(crc_bytes)?; diff --git a/tracklib2/src/read/inspect.rs b/tracklib2/src/read/inspect.rs new file mode 100644 index 0000000..f785872 --- /dev/null +++ b/tracklib2/src/read/inspect.rs @@ -0,0 +1,338 @@ +use crate::types::FieldValue; + +use super::data_table::{parse_data_table, DataTableEntry}; +use super::header::{parse_header, Header}; +use super::metadata::parse_metadata; +use super::section_reader::SectionReader; +use nom::Offset; +use term_table::row::Row; +use term_table::table_cell::{Alignment, TableCell}; +use term_table::Table; + +fn bold(s: &str) -> String { + format!("\x1b[1m{s}\x1b[0m") +} + +fn italic(s: &str) -> String { + format!("\x1b[3m{s}\x1b[0m") +} + +fn strikethrough(s: &str) -> String { + format!("\x1b[9m{s}\x1b[0m") +} + +fn format_header(header: &Header) -> String { + let mut table = Table::new(); + + table.add_row(Row::new(vec![TableCell::new_with_alignment( + bold("Header"), + 2, + Alignment::Center, + )])); + table.add_row(Row::new(vec![ + TableCell::new("File Version"), + TableCell::new_with_alignment(format!("{:#04X}", header.file_version), 1, Alignment::Right), + ])); + table.add_row(Row::new(vec![ + TableCell::new("Creator Version"), + TableCell::new_with_alignment( + format!("{:#04X}", header.creator_version), + 1, + Alignment::Right, + ), + ])); + table.add_row(Row::new(vec![ + TableCell::new("Metadata Offset"), + TableCell::new_with_alignment( + format!("{:#04X}", header.metadata_offset), + 1, + Alignment::Right, + ), + ])); + table.add_row(Row::new(vec![ + TableCell::new("Data Offset"), + TableCell::new_with_alignment(format!("{:#04X}", header.data_offset), 1, Alignment::Right), + ])); + + table.render() +} + +fn try_format_metadata(input: &[u8]) -> String { + let mut table = Table::new(); + + table.add_row(Row::new(vec![TableCell::new_with_alignment( + bold("Metadata"), + 1, + Alignment::Center, + )])); + + match parse_metadata(input) { + Ok((_, metadata_entries)) => { + for entry in metadata_entries { + table.add_row(Row::new(vec![TableCell::new_with_alignment( + format!("{:?}", entry), + 1, + Alignment::Left, + )])); + } + } + Err(e) => { + table.add_row(Row::new(vec![TableCell::new_with_alignment( + format!("{e:?}"), + 1, + Alignment::Left, + )])); + } + } + + table.render() +} + +fn try_format_data_table( + input: &[u8], + data_table_offset: usize, +) -> (Option<(&[u8], Vec)>, String) { + match parse_data_table(&input[data_table_offset..]) { + Ok((data_start, data_table)) => { + let data_start_offset = input.offset(data_start); + let mut out = String::new(); + + for (i, data_table_entry) in data_table.iter().enumerate() { + const CRC_BYTES: usize = 4; + let presence_column_bytes_required = (data_table_entry.types().len() + 7) / 8; + let presence_column_size = + presence_column_bytes_required * data_table_entry.rows() + CRC_BYTES; + + let mut table = Table::new(); + table.add_row(Row::new(vec![TableCell::new_with_alignment( + bold(&format!("Data Section {i}")), + 3, + Alignment::Center, + )])); + table.add_row(Row::new(vec![ + TableCell::new("Section Type"), + TableCell::new_with_alignment( + format!("{:?}", data_table_entry.section_type()), + 2, + Alignment::Right, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new("Rows"), + TableCell::new_with_alignment( + format!("{}", data_table_entry.rows()), + 2, + Alignment::Right, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new("Size"), + TableCell::new_with_alignment( + format!("{:#04X}", data_table_entry.size()), + 2, + Alignment::Right, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new(format!("Offset ({})", italic("relative"))), + TableCell::new_with_alignment( + format!("{:#04X}", data_table_entry.offset()), + 2, + Alignment::Right, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new(format!("Offset ({})", italic("absolute"))), + TableCell::new_with_alignment( + format!("{:#04X}", data_table_entry.offset() + data_start_offset), + 2, + Alignment::Right, + ), + ])); + + for types_table_entry in data_table_entry.types() { + table.add_row(Row::new(vec![ + TableCell::new_with_alignment("Column", 1, Alignment::Left), + TableCell::new_with_alignment( + bold(types_table_entry.field_description().name()), + 2, + Alignment::Center, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new(""), + TableCell::new_with_alignment("Type", 1, Alignment::Left), + TableCell::new_with_alignment( + format!("{:?}", types_table_entry.field_description().fieldtype()), + 1, + Alignment::Right, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new(""), + TableCell::new_with_alignment("Size", 1, Alignment::Left), + TableCell::new_with_alignment( + format!("{:#04X}", types_table_entry.size()), + 1, + Alignment::Right, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new(""), + TableCell::new_with_alignment( + format!("Offset ({})", italic("relative")), + 1, + Alignment::Right, + ), + TableCell::new_with_alignment( + format!("{:#04X}", types_table_entry.offset()), + 1, + Alignment::Right, + ), + ])); + + table.add_row(Row::new(vec![ + TableCell::new(""), + TableCell::new_with_alignment( + format!("Offset ({})", italic("absolute")), + 1, + Alignment::Right, + ), + TableCell::new_with_alignment( + format!( + "{:#04X}", + types_table_entry.offset() + + presence_column_size + + data_table_entry.offset() + + data_start_offset + ), + 1, + Alignment::Right, + ), + ])); + } + + out.push_str(&table.render()); + out.push_str("\n\n"); + } + + (Some((data_start, data_table)), out) + } + Err(e) => { + let mut table = Table::new(); + table.add_row(Row::new(vec![TableCell::new_with_alignment( + bold(&format!("Data Table")), + 1, + Alignment::Center, + )])); + table.add_row(Row::new(vec![TableCell::new_with_alignment( + format!("{e:?}"), + 1, + Alignment::Left, + )])); + + (None, table.render()) + } + } +} + +fn format_val(value: Option) -> String { + if let Some(val) = value { + match val { + FieldValue::I64(v) => format!("{v}"), + FieldValue::Bool(v) => format!("{v}"), + FieldValue::String(v) => v.clone(), + } + } else { + String::from(strikethrough("None")) + } +} + +fn try_format_section(data_start: &[u8], entry_num: usize, entry: &DataTableEntry) -> String { + let mut table = Table::new(); + + let data = &data_start[usize::try_from(entry.offset()).expect("usize != u64")..]; + + match SectionReader::new(data, &entry) { + Ok(mut section_reader) => { + table.add_row(Row::new(vec![TableCell::new_with_alignment( + bold(&format!("Data {entry_num}")), + entry.types().len() + 1, + Alignment::Center, + )])); + + table.add_row(Row::new( + [TableCell::new_with_alignment("#", 1, Alignment::Center)] + .into_iter() + .chain(entry.types().iter().map(|types_table_entry| { + TableCell::new_with_alignment( + types_table_entry.field_description().name(), + 1, + Alignment::Center, + ) + })) + .collect::>(), + )); + + let mut i = 0; + while let Some(columniter) = section_reader.open_column_iter() { + table.add_row(Row::new( + [format!("{i}")] + .into_iter() + .chain(columniter.map(|row_result| { + row_result + .map(|(_, val)| format_val(val)) + .unwrap_or_else(|e| format!("{e:?}")) + })) + .collect::>(), + )); + i += 1; + } + } + Err(e) => { + table.add_row(Row::new(vec![TableCell::new_with_alignment( + format!("{e:?}"), + 1, + Alignment::Left, + )])); + } + } + table.render() +} + +pub fn inspect(input: &[u8]) -> Result { + let mut out = String::new(); + + // Header + let (_, header) = parse_header(input).map_err(|e| format!("Error Parsing Header: {:?}", e))?; + out.push_str(&format_header(&header)); + out.push_str("\n\n"); + + // Metadata + out.push_str(&try_format_metadata( + &input[usize::from(header.metadata_offset)..], + )); + out.push_str("\n\n"); + + // Data Table + let (maybe_data_table, data_table_out) = + try_format_data_table(input, usize::from(header.data_offset)); + out.push_str(&data_table_out); + + // Data + if let Some((data_start, data_table_entries)) = maybe_data_table { + for (i, data_table_entry) in data_table_entries.iter().enumerate() { + out.push_str(&try_format_section(data_start, i, &data_table_entry)); + out.push_str("\n\n"); + } + } + + Ok(out) +} diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 89229ba..3eef940 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -2,6 +2,8 @@ mod crc; mod data_table; mod decoders; mod header; +#[cfg(feature = "inspect")] +pub mod inspect; mod metadata; mod presence_column; mod section_reader; From 0741c0285562814fc507e8b812b6b4000763412f Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 15 Mar 2022 13:08:00 -0700 Subject: [PATCH 036/113] PresenceColumn::view now returns an Option --- tracklib2/src/read/decoders.rs | 138 +++++++++++++------------- tracklib2/src/read/presence_column.rs | 54 +++++----- tracklib2/src/read/section_reader.rs | 9 +- 3 files changed, 106 insertions(+), 95 deletions(-) diff --git a/tracklib2/src/read/decoders.rs b/tracklib2/src/read/decoders.rs index bc8f6ec..4e741c7 100644 --- a/tracklib2/src/read/decoders.rs +++ b/tracklib2/src/read/decoders.rs @@ -168,24 +168,24 @@ mod tests { 0x65, 0x57, 0xFB]; - assert_matches!(parse_presence_column(1, 5)(presence_buf), Ok((&[], presence_column)) => { - let presence_column_view = presence_column.view(0); - #[rustfmt::skip] - let buf = &[0x20, - 0x42, - 0x00, - 0x1, - 0x0C, // crc - 0x01, - 0x49, - 0xA3]; - assert_matches!(I64Decoder::new(buf, presence_column_view), Ok(mut decoder) => { - assert_matches!(decoder.decode(), Ok(Some(32))); - assert_matches!(decoder.decode(), Ok(Some(-30))); - assert_matches!(decoder.decode(), Ok(None)); - assert_matches!(decoder.decode(), Ok(Some(-30))); - assert_matches!(decoder.decode(), Ok(Some(-29))); - }); + let presence_column = + assert_matches!(parse_presence_column(1, 5)(presence_buf), Ok((&[], pc)) => pc); + let presence_column_view = assert_matches!(presence_column.view(0), Some(v) => v); + #[rustfmt::skip] + let buf = &[0x20, + 0x42, + 0x00, + 0x1, + 0x0C, // crc + 0x01, + 0x49, + 0xA3]; + assert_matches!(I64Decoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(Some(32))); + assert_matches!(decoder.decode(), Ok(Some(-30))); + assert_matches!(decoder.decode(), Ok(None)); + assert_matches!(decoder.decode(), Ok(Some(-30))); + assert_matches!(decoder.decode(), Ok(Some(-29))); }); } @@ -199,20 +199,20 @@ mod tests { 0x5E, 0x43, 0x9E]; - assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], presence_column)) => { - let presence_column_view = presence_column.view(0); - #[rustfmt::skip] - let buf = &[0x01, - 0x00, - 0x5E, // crc - 0x5A, - 0x51, - 0x2D]; - assert_matches!(BoolDecoder::new(buf, presence_column_view), Ok(mut decoder) => { - assert_matches!(decoder.decode(), Ok(None)); - assert_matches!(decoder.decode(), Ok(Some(true))); - assert_matches!(decoder.decode(), Ok(Some(false))); - }); + let presence_column = + assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], pc)) => pc); + let presence_column_view = assert_matches!(presence_column.view(0), Some(v) => v); + #[rustfmt::skip] + let buf = &[0x01, + 0x00, + 0x5E, // crc + 0x5A, + 0x51, + 0x2D]; + assert_matches!(BoolDecoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(None)); + assert_matches!(decoder.decode(), Ok(Some(true))); + assert_matches!(decoder.decode(), Ok(Some(false))); }); } @@ -226,27 +226,27 @@ mod tests { 0x5E, 0x43, 0x9E]; - assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], presence_column)) => { - let presence_column_view = presence_column.view(0); - #[rustfmt::skip] - let buf = &[0x01, - b'R', - 0x03, - b'i', - b'd', - b'e', - 0x73, // crc - 0x91, - 0x5A, - 0x74]; - assert_matches!(StringDecoder::new(buf, presence_column_view), Ok(mut decoder) => { - assert_matches!(decoder.decode(), Ok(None)); - assert_matches!(decoder.decode(), Ok(Some(s)) => { - assert_eq!(s, "R"); - }); - assert_matches!(decoder.decode(), Ok(Some(s)) => { - assert_eq!(s, "ide"); - }); + let presence_column = + assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], pc)) => pc); + let presence_column_view = assert_matches!(presence_column.view(0), Some(v) => v); + #[rustfmt::skip] + let buf = &[0x01, + b'R', + 0x03, + b'i', + b'd', + b'e', + 0x73, // crc + 0x91, + 0x5A, + 0x74]; + assert_matches!(StringDecoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(None)); + assert_matches!(decoder.decode(), Ok(Some(s)) => { + assert_eq!(s, "R"); + }); + assert_matches!(decoder.decode(), Ok(Some(s)) => { + assert_eq!(s, "ide"); }); }); } @@ -261,19 +261,23 @@ mod tests { 0x5E, 0x43, 0x9E]; - assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], presence_column)) => { - let presence_column_view = presence_column.view(0); - #[rustfmt::skip] - let buf = &[0x00, - 0x01, - 0x02, - 0x00, // invalid crc - 0x00, - 0x00, - 0x00]; - assert_matches!(StringDecoder::new(buf, presence_column_view), - Err(crate::error::TracklibError::CRC32Error{expected: 0x00, - computed: 0x9300784D})); - }); + let presence_column = + assert_matches!(parse_presence_column(1, 3)(presence_buf), Ok((&[], pc)) => pc); + let presence_column_view = assert_matches!(presence_column.view(0), Some(v) => v); + #[rustfmt::skip] + let buf = &[0x00, + 0x01, + 0x02, + 0x00, // invalid crc + 0x00, + 0x00, + 0x00]; + assert_matches!( + StringDecoder::new(buf, presence_column_view), + Err(crate::error::TracklibError::CRC32Error { + expected: 0x00000000, + computed: 0x9300784D + }) + ); } } diff --git a/tracklib2/src/read/presence_column.rs b/tracklib2/src/read/presence_column.rs index 9295db3..72a9643 100644 --- a/tracklib2/src/read/presence_column.rs +++ b/tracklib2/src/read/presence_column.rs @@ -10,21 +10,24 @@ pub(crate) struct PresenceColumn<'a> { } impl<'a> PresenceColumn<'a> { - // TODO: should this return a Result? /// View into a particular field in the presence column. - pub(crate) fn view(&self, index: usize) -> PresenceColumnView<'a> { - let bit_index = 1 << (index % 8); - let presence_bytes_required = (self.fields + 7) / 8; - let x = (presence_bytes_required * 8 - index + 7) & !7; // next multiple of 8 - let byte_index = (x / 8) - 1; - - PresenceColumnView::new( - &self.data, - bit_index, - byte_index, - presence_bytes_required, - self.rows, - ) + pub(crate) fn view(&self, index: usize) -> Option> { + if index < self.fields { + let bit_index = 1 << (index % 8); + let presence_bytes_required = (self.fields + 7) / 8; + let x = (presence_bytes_required * 8 - index + 7) & !7; // next multiple of 8 + let byte_index = (x / 8) - 1; + + Some(PresenceColumnView::new( + &self.data, + bit_index, + byte_index, + presence_bytes_required, + self.rows, + )) + } else { + None + } } } @@ -112,7 +115,7 @@ mod tests { 0xC2]; assert_matches!(parse_presence_column(80, 1)(buf), Ok((&[], presence_column)) => { let vals = (0..80) - .map(|i| presence_column.view(i)) + .map(|i| presence_column.view(i).unwrap()) .flat_map(|view| view.collect::>()) .collect::>(); @@ -144,19 +147,18 @@ mod tests { 0xA7, 0xA5]; assert_matches!(parse_presence_column(3, 4)(buf), Ok((&[], presence_column)) => { - let view = presence_column.view(0); + let view = presence_column.view(0).unwrap(); assert_eq!(view.collect::>(), &[true, true, false, true]); - let view = presence_column.view(1); + let view = presence_column.view(1).unwrap(); assert_eq!(view.collect::>(), &[true, false, true, true]); - let view = presence_column.view(2); + let view = presence_column.view(2).unwrap(); assert_eq!(view.collect::>(), &[false, true, true, true]); }); } #[test] - #[should_panic] fn test_presence_column_view_overflowing_column_index() { #[rustfmt::skip] let buf = &[0b00000011, @@ -168,7 +170,7 @@ mod tests { 0xA7, 0xA5]; assert_matches!(parse_presence_column(3, 4)(buf), Ok((&[], presence_column)) => { - presence_column.view(10); + assert!(presence_column.view(10).is_none()); }); } @@ -182,13 +184,13 @@ mod tests { 0x91, 0xF5]; assert_matches!(parse_presence_column(20, 2)(buf), Ok((&[], presence_column)) => { - assert_eq!(presence_column.view(0).collect::>(), &[false, true]); - assert_eq!(presence_column.view(1).collect::>(), &[true, true]); + assert_eq!(presence_column.view(0).unwrap().collect::>(), &[false, true]); + assert_eq!(presence_column.view(1).unwrap().collect::>(), &[true, true]); - assert_eq!(presence_column.view(9).collect::>(), &[false, true]); - assert_eq!(presence_column.view(10).collect::>(), &[true, true]); - assert_eq!(presence_column.view(11).collect::>(), &[false, false]); - assert_eq!(presence_column.view(12).collect::>(), &[true, false]); + assert_eq!(presence_column.view(9).unwrap().collect::>(), &[false, true]); + assert_eq!(presence_column.view(10).unwrap().collect::>(), &[true, true]); + assert_eq!(presence_column.view(11).unwrap().collect::>(), &[false, false]); + assert_eq!(presence_column.view(12).unwrap().collect::>(), &[true, false]); }); } } diff --git a/tracklib2/src/read/section_reader.rs b/tracklib2/src/read/section_reader.rs index 92133ad..73ee1f5 100644 --- a/tracklib2/src/read/section_reader.rs +++ b/tracklib2/src/read/section_reader.rs @@ -1,7 +1,7 @@ use super::data_table::DataTableEntry; use super::decoders::{BoolDecoder, Decoder, I64Decoder, StringDecoder}; use super::presence_column::parse_presence_column; -use crate::error::Result; +use crate::error::{Result, TracklibError}; use crate::types::{FieldDescription, FieldType, FieldValue}; #[cfg_attr(test, derive(Debug))] @@ -38,7 +38,12 @@ impl<'a> SectionReader<'a> { .enumerate() .map(|(i, field)| { let column_data = &column_data[field.offset()..field.offset() + field.size()]; - let presence_column_view = presence_column.view(i); + let presence_column_view = + presence_column + .view(i) + .ok_or_else(|| TracklibError::ParseIncompleteError { + needed: nom::Needed::Unknown, + })?; let field_description = field.field_description(); let decoder = match field_description.fieldtype() { FieldType::I64 => ColumnDecoder::I64 { From 7f13a1d613815997d2fc31b1dd11074611b67095 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 17 Mar 2022 12:16:38 -0700 Subject: [PATCH 037/113] F64 data type --- tracklib2/Cargo.lock | 10 +++ tracklib2/Cargo.toml | 1 + tracklib2/src/read/decoders.rs | 112 +++++++++++++++++++++++++++ tracklib2/src/read/inspect.rs | 1 + tracklib2/src/read/section_reader.rs | 80 +++++++++++++++---- tracklib2/src/read/types_table.rs | 5 +- tracklib2/src/types.rs | 2 + tracklib2/src/write/encoders.rs | 98 +++++++++++++++++++++-- tracklib2/src/write/section.rs | 77 ++++++++++++++---- tracklib2/src/write/track.rs | 4 +- 10 files changed, 353 insertions(+), 37 deletions(-) diff --git a/tracklib2/Cargo.lock b/tracklib2/Cargo.lock index d728bdf..09911ad 100644 --- a/tracklib2/Cargo.lock +++ b/tracklib2/Cargo.lock @@ -233,6 +233,15 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "half" version = "1.6.0" @@ -615,6 +624,7 @@ dependencies = [ "assert_matches", "crc", "criterion", + "float-cmp", "leb128", "nom", "nom-leb128", diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml index e6fd8a2..c7809d7 100644 --- a/tracklib2/Cargo.toml +++ b/tracklib2/Cargo.toml @@ -15,6 +15,7 @@ term-table = {version = "1.3", optional = true} [dev-dependencies] assert_matches = "1.5" criterion = "0.3" +float-cmp = "0.9" [features] inspect = ["term-table"] diff --git a/tracklib2/src/read/decoders.rs b/tracklib2/src/read/decoders.rs index 4e741c7..6b32e20 100644 --- a/tracklib2/src/read/decoders.rs +++ b/tracklib2/src/read/decoders.rs @@ -68,6 +68,51 @@ impl<'a> Decoder for I64Decoder<'a> { } } +#[cfg_attr(test, derive(Debug))] +pub(crate) struct F64Decoder<'a> { + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, + prev: i64, +} + +impl<'a> F64Decoder<'a> { + pub(crate) fn new( + data: &'a [u8], + presence_column_view: PresenceColumnView<'a>, + ) -> Result { + let column_data = validate_column(data)?; + + Ok(Self { + data: column_data, + presence_column_view, + prev: 0, + }) + } +} + +impl<'a> Decoder for F64Decoder<'a> { + type T = f64; + + fn decode(&mut self) -> Result> { + if let Some(is_present) = self.presence_column_view.next() { + if is_present { + let (rest, value) = leb128_i64(self.data)?; + let new = self.prev + value; + self.prev = new; + self.data = rest; + Ok(Some((new as f64) / 10e6)) + } else { + Ok(None) + } + } else { + // ran out of rows, this is an error + return Err(TracklibError::ParseIncompleteError { + needed: nom::Needed::Unknown, + }); + } + } +} + #[cfg_attr(test, derive(Debug))] pub(crate) struct BoolDecoder<'a> { data: &'a [u8], @@ -155,6 +200,7 @@ mod tests { use super::*; use crate::read::presence_column::parse_presence_column; use assert_matches::assert_matches; + use float_cmp::assert_approx_eq; #[test] fn test_decode_i64() { @@ -189,6 +235,72 @@ mod tests { }); } + #[test] + fn test_decode_f64() { + #[rustfmt::skip] + let presence_buf = &[0b00000001, + 0b00000001, + 0b00000000, + 0b00000001, + 0b00000001, + 0b00000001, + 0x94, // crc + 0x59, + 0xA0, + 0x40]; + let presence_column = + assert_matches!(parse_presence_column(1, 6)(presence_buf), Ok((&[], pc)) => pc); + let presence_column_view = assert_matches!(presence_column.view(0), Some(v) => v); + #[rustfmt::skip] + let buf = &[0x00, // first storing a 0 + + 0x80, // leb128-encoded difference between prev (0.0) and 1.0 * 10e6 + 0xAD, + 0xE2, + 0x04, + + // None + + 0xC0, // leb128-encoded delta between prev and 2.5 * 10e6 + 0xC3, + 0x93, + 0x07, + + 0xA4, // leb128-encoded delta between prev and 3.00001 * 10e6 + 0x97, + 0xB1, + 0x02, + + 0xDC, // leb128-encoded delta between prev and -100.26 * 10e6 + 0x8B, + 0xCF, + 0x93, + 0x7C, + + 0x52, // crc + 0xD3, + 0xE9, + 0x35]; + assert_matches!(F64Decoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(Some(v)) => { + assert_approx_eq!(f64, v, 0.0); + }); + assert_matches!(decoder.decode(), Ok(Some(v)) => { + assert_approx_eq!(f64, v, 1.0); + }); + assert_matches!(decoder.decode(), Ok(None)); + assert_matches!(decoder.decode(), Ok(Some(v)) => { + assert_approx_eq!(f64, v, 2.5); + }); + assert_matches!(decoder.decode(), Ok(Some(v)) => { + assert_approx_eq!(f64, v, 3.00001); + }); + assert_matches!(decoder.decode(), Ok(Some(v)) => { + assert_approx_eq!(f64, v, -100.26); + }); + }); + } + #[test] fn test_decode_bool() { #[rustfmt::skip] diff --git a/tracklib2/src/read/inspect.rs b/tracklib2/src/read/inspect.rs index f785872..4b24e1a 100644 --- a/tracklib2/src/read/inspect.rs +++ b/tracklib2/src/read/inspect.rs @@ -247,6 +247,7 @@ fn format_val(value: Option) -> String { if let Some(val) = value { match val { FieldValue::I64(v) => format!("{v}"), + FieldValue::F64(v) => format!("{v}"), FieldValue::Bool(v) => format!("{v}"), FieldValue::String(v) => v.clone(), } diff --git a/tracklib2/src/read/section_reader.rs b/tracklib2/src/read/section_reader.rs index 73ee1f5..b55665d 100644 --- a/tracklib2/src/read/section_reader.rs +++ b/tracklib2/src/read/section_reader.rs @@ -1,5 +1,5 @@ use super::data_table::DataTableEntry; -use super::decoders::{BoolDecoder, Decoder, I64Decoder, StringDecoder}; +use super::decoders::*; use super::presence_column::parse_presence_column; use crate::error::{Result, TracklibError}; use crate::types::{FieldDescription, FieldType, FieldValue}; @@ -10,6 +10,10 @@ enum ColumnDecoder<'a> { field_description: &'a FieldDescription, decoder: I64Decoder<'a>, }, + F64 { + field_description: &'a FieldDescription, + decoder: F64Decoder<'a>, + }, Bool { field_description: &'a FieldDescription, decoder: BoolDecoder<'a>, @@ -50,6 +54,10 @@ impl<'a> SectionReader<'a> { field_description, decoder: I64Decoder::new(column_data, presence_column_view)?, }, + FieldType::F64 => ColumnDecoder::F64 { + field_description, + decoder: F64Decoder::new(column_data, presence_column_view)?, + }, FieldType::Bool => ColumnDecoder::Bool { field_description, decoder: BoolDecoder::new(column_data, presence_column_view)?, @@ -112,6 +120,15 @@ impl<'a, 'b> Iterator for ColumnIter<'a, 'b> { .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::I64(v)))) .map_err(|e| e), ), + ColumnDecoder::F64 { + ref field_description, + ref mut decoder, + } => Some( + decoder + .decode() + .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::F64(v)))) + .map_err(|e| e), + ), ColumnDecoder::Bool { ref field_description, ref mut decoder, @@ -153,7 +170,7 @@ mod tests { 0x03, // leb128 section point count 0x26, // leb128 section data size // Types Table - 0x03, // field count + 0x04, // field count 0x00, // first field type = I64 0x01, // name length b'a', // name @@ -166,9 +183,13 @@ mod tests { 0x01, // name length b'c', // name 0x12, // leb128 data size + 0x01, // fourth field type = F64 + 0x01, // name length + b'f', // name + 0x0C, // leb128 data size - 0x67, // crc - 0x3C]; + 0xE9, // crc + 0x0B]; assert_matches!(parse_data_table(data_table_buf), Ok((&[], data_table_entries)) => { assert_eq!(data_table_entries.len(), 1); @@ -177,12 +198,12 @@ mod tests { let buf = &[ // Presence Column 0b00000111, - 0b00000101, - 0b00000111, - 0x1A, // crc - 0x75, - 0xEA, - 0xC4, + 0b00001101, + 0b00001111, + 0xF0, // crc + 0xDB, + 0xAA, + 0x68, // Data Column 1 = I64 0x01, // 1 @@ -220,13 +241,29 @@ mod tests { 0xA3, // crc 0x02, 0xEC, - 0x48]; + 0x48, + + // Data Column 4 = F64 + // None + 0x80, // 1.0 + 0xAD, + 0xE2, + 0x04, + 0xC0, // 2.5 + 0xC3, + 0x93, + 0x07, + 0xCC, // crc + 0xEC, + 0xC5, + 0x15 + ]; assert_matches!(SectionReader::new(buf, &data_table_entries[0]), Ok(mut section_reader) => { // Row 1 assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); - assert_eq!(values.len(), 3); + assert_eq!(values.len(), 4); assert_matches!(&values[0], Ok((field_description, field_value)) => { assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); @@ -242,12 +279,17 @@ mod tests { assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); assert_eq!(field_value, &Some(FieldValue::String("Ride".to_string()))); }); + assert_matches!(&values[3], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[3].field_description()); + assert_eq!(*field_description, &FieldDescription::new("f".to_string(), FieldType::F64)); + assert_eq!(field_value, &None); + }); }); // Row 2 assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); - assert_eq!(values.len(), 3); + assert_eq!(values.len(), 4); assert_matches!(&values[0], Ok((field_description, field_value)) => { assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); @@ -263,12 +305,17 @@ mod tests { assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); assert_eq!(field_value, &Some(FieldValue::String("with".to_string()))); }); + assert_matches!(&values[3], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[3].field_description()); + assert_eq!(*field_description, &FieldDescription::new("f".to_string(), FieldType::F64)); + assert_eq!(field_value, &Some(FieldValue::F64(1.0))); + }); }); // Row 3 assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); - assert_eq!(values.len(), 3); + assert_eq!(values.len(), 4); assert_matches!(&values[0], Ok((field_description, field_value)) => { assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); @@ -284,6 +331,11 @@ mod tests { assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); assert_eq!(field_value, &Some(FieldValue::String("GPS".to_string()))); }); + assert_matches!(&values[3], Ok((field_description, field_value)) => { + assert_eq!(*field_description, data_table_entries[0].types()[3].field_description()); + assert_eq!(*field_description, &FieldDescription::new("f".to_string(), FieldType::F64)); + assert_eq!(field_value, &Some(FieldValue::F64(2.5))); + }); }); // Trying to get another row will return nothing diff --git a/tracklib2/src/read/types_table.rs b/tracklib2/src/read/types_table.rs index 7e5650f..d739f2b 100644 --- a/tracklib2/src/read/types_table.rs +++ b/tracklib2/src/read/types_table.rs @@ -52,6 +52,7 @@ fn parse_types_table_entry<'a>( let fieldtype = match type_tag { 0x00 => FieldType::I64, + 0x01 => FieldType::F64, 0x04 => FieldType::String, 0x05 => FieldType::Bool, _ => { @@ -126,7 +127,7 @@ mod tests { b'e', b'!', 0x07, // data size = 7 ("Hello!" + leb128 length prefix) - 0x00, // fourth entry type: i64 = 0 + 0x01, // fourth entry type: f64 = 0 0x01, // name len = 1 b'i', // name = "i" 0x02]; // data size = 2 @@ -140,7 +141,7 @@ mod tests { TypesTableEntry{field_description: FieldDescription::new("long name!".to_string(), FieldType::String), size: 7, offset: 3}, - TypesTableEntry{field_description: FieldDescription::new("i".to_string(), FieldType::I64), + TypesTableEntry{field_description: FieldDescription::new("i".to_string(), FieldType::F64), size: 2, offset: 10}]); }); diff --git a/tracklib2/src/types.rs b/tracklib2/src/types.rs index 95e5f9c..4155725 100644 --- a/tracklib2/src/types.rs +++ b/tracklib2/src/types.rs @@ -2,6 +2,7 @@ #[cfg_attr(test, derive(PartialEq))] pub enum FieldType { I64, + F64, String, Bool, } @@ -31,6 +32,7 @@ impl FieldDescription { #[cfg_attr(test, derive(PartialEq))] pub enum FieldValue { I64(i64), + F64(f64), Bool(bool), String(String), } diff --git a/tracklib2/src/write/encoders.rs b/tracklib2/src/write/encoders.rs index 3d1bed9..cfcfd0c 100644 --- a/tracklib2/src/write/encoders.rs +++ b/tracklib2/src/write/encoders.rs @@ -27,10 +27,36 @@ impl Encoder for I64Encoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - if let Some(v) = value { - let value = *v; - let delta = value - self.prev; - self.prev = value; + if let Some(val) = value { + let v = *val; + let delta = v - self.prev; + self.prev = v; + leb128::write::signed(buf, delta)?; + } + + Ok(()) + } +} + +#[derive(Debug, Default)] +pub struct F64Encoder { + prev: i64, +} + +impl Encoder for F64Encoder { + type T = f64; + + fn encode( + &mut self, + value: Option<&Self::T>, + buf: &mut Vec, + presence: &mut Vec, + ) -> Result<()> { + presence.push(value.is_some()); + if let Some(val) = value { + let v = (*val * 10e6) as i64; + let delta = v - self.prev; + self.prev = v; leb128::write::signed(buf, delta)?; } @@ -51,8 +77,8 @@ impl Encoder for BoolEncoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - if let Some(v) = value { - let v = *(value.unwrap_or(&false)) as u8; + if let Some(val) = value { + let v = *val as u8; buf.write_all(&v.to_le_bytes())?; } Ok(()) @@ -141,6 +167,66 @@ mod tests { true]); } + #[test] + fn test_f64_encoder() { + let mut data_buf = vec![]; + let mut presence_buf = vec![]; + let mut encoder = F64Encoder::default(); + + assert!(encoder + .encode(Some(&0.0), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&1.0), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(None, &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&2.5), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&3.00001), &mut data_buf, &mut presence_buf) + .is_ok()); + assert!(encoder + .encode(Some(&-100.26), &mut data_buf, &mut presence_buf) + .is_ok()); + + #[rustfmt::skip] + assert_eq!(data_buf, &[0x00, // first storing a 0 + + 0x80, // leb128-encoded difference between prev (0.0) and 1.0 * 10e6 + 0xAD, + 0xE2, + 0x04, + + // None + + 0xC0, // leb128-encoded delta between prev and 2.5 * 10e6 + 0xC3, + 0x93, + 0x07, + + 0xA4, // leb128-encoded delta between prev and 3.00001 * 10e6 + 0x97, + 0xB1, + 0x02, + + 0xDC, // leb128-encoded delta between prev and -100.26 * 10e6 + 0x8B, + 0xCF, + 0x93, + 0x7C]); + + #[rustfmt::skip] + assert_eq!(presence_buf, &[true, + true, + false, + true, + true, + true]); + } + #[test] fn test_bool_encoder() { let mut data_buf = vec![]; diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index c5878c2..ace2f9f 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -1,6 +1,5 @@ use super::crcwriter::CrcWriter; -use super::encoders::{BoolEncoder, Encoder, I64Encoder, StringEncoder}; -use crate::consts::{CRC16, CRC32}; +use super::encoders::*; use crate::error::Result; use crate::types::{FieldDescription, FieldType, SectionType}; use std::convert::TryFrom; @@ -10,6 +9,7 @@ impl FieldType { fn type_tag(&self) -> u8 { match self { Self::I64 => 0x00, + Self::F64 => 0x01, Self::String => 0x04, Self::Bool => 0x05, } @@ -49,6 +49,7 @@ impl BufferImpl { #[derive(Debug)] enum Buffer { I64(BufferImpl), + F64(BufferImpl), Bool(BufferImpl), String(BufferImpl), } @@ -57,6 +58,7 @@ impl Buffer { fn new(field_type: &FieldType) -> Self { match field_type { &FieldType::I64 => Buffer::I64(BufferImpl::default()), + &FieldType::F64 => Buffer::F64(BufferImpl::default()), &FieldType::Bool => Buffer::Bool(BufferImpl::default()), &FieldType::String => Buffer::String(BufferImpl::default()), } @@ -65,6 +67,7 @@ impl Buffer { fn len(&self) -> usize { match self { Self::I64(buffer_impl) => buffer_impl.buf.len(), + Self::F64(buffer_impl) => buffer_impl.buf.len(), Self::Bool(buffer_impl) => buffer_impl.buf.len(), Self::String(buffer_impl) => buffer_impl.buf.len(), } @@ -138,6 +141,7 @@ impl Section { for buffer in self.column_data.iter() { let is_present = match buffer { Buffer::I64(buffer_impl) => buffer_impl.presence.get(row_i), + Buffer::F64(buffer_impl) => buffer_impl.presence.get(row_i), Buffer::Bool(buffer_impl) => buffer_impl.presence.get(row_i), Buffer::String(buffer_impl) => buffer_impl.presence.get(row_i), }; @@ -168,6 +172,7 @@ impl Section { .get(i) .map(|buffer| match buffer { Buffer::I64(buffer_impl) => buffer_impl.data_size(), + Buffer::F64(buffer_impl) => buffer_impl.data_size(), Buffer::Bool(buffer_impl) => buffer_impl.data_size(), Buffer::String(buffer_impl) => buffer_impl.data_size(), }) @@ -187,9 +192,10 @@ impl Section { self.write_presence_column(out)?; // ? bytes - presence column with crc for buffer in self.column_data.iter() { match buffer { - Buffer::I64(buffer_impl) => buffer_impl.write_data(out)?, // \ - Buffer::Bool(buffer_impl) => buffer_impl.write_data(out)?, // \ - Buffer::String(buffer_impl) => buffer_impl.write_data(out)?, // > ? bytes - data column with crc + Buffer::I64(buffer_impl) => buffer_impl.write_data(out)?, // | + Buffer::F64(buffer_impl) => buffer_impl.write_data(out)?, // | + Buffer::Bool(buffer_impl) => buffer_impl.write_data(out)?, // | + Buffer::String(buffer_impl) => buffer_impl.write_data(out)?, // - > ? bytes - data column with crc }; } @@ -224,6 +230,9 @@ impl<'a> RowBuilder<'a> { (Some(field_desc), Some(Buffer::I64(ref mut buffer_impl))) => Some( ColumnWriter::I64ColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), ), + (Some(field_desc), Some(Buffer::F64(ref mut buffer_impl))) => Some( + ColumnWriter::F64ColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + ), (Some(field_desc), Some(Buffer::Bool(ref mut buffer_impl))) => Some( ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), ), @@ -238,6 +247,7 @@ impl<'a> RowBuilder<'a> { #[derive(Debug)] pub enum ColumnWriter<'a> { I64ColumnWriter(ColumnWriterImpl<'a, I64Encoder>), + F64ColumnWriter(ColumnWriterImpl<'a, F64Encoder>), BoolColumnWriter(ColumnWriterImpl<'a, BoolEncoder>), StringColumnWriter(ColumnWriterImpl<'a, StringEncoder>), } @@ -321,6 +331,7 @@ mod tests { ColumnWriter::StringColumnWriter(cwi) => { assert!(cwi.write(c_vals[i].as_ref()).is_ok()); } + ColumnWriter::F64ColumnWriter(_) => {} } } } @@ -452,6 +463,7 @@ mod tests { ("k".to_string(), FieldType::Bool), ("long name!".to_string(), FieldType::String), ("i".to_string(), FieldType::I64), + ("f".to_string(), FieldType::F64), ], ); @@ -467,6 +479,9 @@ mod tests { ColumnWriter::StringColumnWriter(cwi) => { assert!(cwi.write(Some(&"Hello!".to_string())).is_ok()); } + ColumnWriter::F64ColumnWriter(cwi) => { + assert!(cwi.write(Some(&0.0042)).is_ok()); + } } } @@ -474,7 +489,7 @@ mod tests { assert_matches!(section.write_types_table(&mut buf), Ok(()) => { #[rustfmt::skip] assert_eq!(buf, - &[0x04, // entry count = 4 + &[0x05, // entry count = 5 0x00, // first entry type: i64 = 0 0x01, // name len = 1 b'm', // name = "m" @@ -499,7 +514,12 @@ mod tests { 0x00, // fourth entry type: i64 = 0 0x01, // name len = 1 b'i', // name = "i" - 0x06]); // data size = 6 + 0x06, // data size = 6 + 0x01, // fifth entry type: f64 = 1 + 0x01, // name len = 1 + b'f', // name = "f" + 0x07, // data size = 7 + ]); }); } @@ -507,6 +527,7 @@ mod tests { fn test_writing_a_section() { enum V { I64(i64), + F64(f64), Bool(bool), String(String), } @@ -516,6 +537,7 @@ mod tests { h.insert("a", V::I64(1)); h.insert("b", V::Bool(false)); h.insert("c", V::String("Ride".to_string())); + h.insert("d", V::F64(0.0)); v.push(h); let mut h = HashMap::new(); h.insert("a", V::I64(2)); @@ -525,6 +547,7 @@ mod tests { h.insert("a", V::I64(4)); h.insert("b", V::Bool(true)); h.insert("c", V::String("GPS".to_string())); + h.insert("d", V::F64(2112.90125)); v.push(h); let mut section = Section::new( @@ -533,6 +556,7 @@ mod tests { ("a".to_string(), FieldType::I64), ("b".to_string(), FieldType::Bool), ("c".to_string(), FieldType::String), + ("d".to_string(), FieldType::F64), ], ); @@ -577,6 +601,17 @@ mod tests { .flatten(), ).is_ok()); } + ColumnWriter::F64ColumnWriter(cwi) => { + assert!(cwi.write( + entry + .get(field_desc.name()) + .map(|v| match v { + V::F64(v) => Some(v), + _ => None, + }) + .flatten(), + ).is_ok()); + } } }); } @@ -587,13 +622,13 @@ mod tests { #[rustfmt::skip] assert_eq!(buf, &[ // Presence Column - 0b00000111, + 0b00001111, 0b00000101, - 0b00000111, - 0x1A, // crc - 0x75, - 0xEA, - 0xC4, + 0b00001111, + 0x9A, // crc + 0xFC, + 0x27, + 0xEC, // Data Column 1 = I64 0x01, // 1 @@ -631,7 +666,21 @@ mod tests { 0xA3, // crc 0x02, 0xEC, - 0x48]); + 0x48, + + // Data Column 4 = F64 + 0x0, // 0.0 + // None + 0x94, // 2112.90125 + 0xCA, + 0x8C, + 0xDB, + 0xCE, + 0x00, + 0xF0, // crc + 0xA4, + 0x8A, + 0xDD]); }); } } diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index c6086a2..fa71ca4 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -1,7 +1,7 @@ use super::data_table::write_data_table; use super::metadata::write_metadata; use super::section::Section; -use crate::consts::{CRC16, RWTF_HEADER_SIZE}; +use crate::consts::RWTF_HEADER_SIZE; use crate::error::Result; use crate::types::MetadataEntry; use std::convert::TryFrom; @@ -114,6 +114,7 @@ mod tests { ColumnWriter::StringColumnWriter(cwi) => { assert!(cwi.write(Some(&"hey".to_string())).is_ok()); } + ColumnWriter::F64ColumnWriter(_) => {} } } } @@ -190,6 +191,7 @@ mod tests { .flatten(), ).is_ok()); } + ColumnWriter::F64ColumnWriter(_) => {} } }); } From 2b20ea971aa61af31cad25e3ada6bc33f7aa58d3 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 22 Mar 2022 13:14:02 -0700 Subject: [PATCH 038/113] separate the concept of encoders/decoders from bitstreams Encoders (like floats, strings) will take the value we want to store, but the bitstream handles how to actually store it (like i64, bytes). --- tracklib2/src/read/bitstream.rs | 140 +++++++++++++++++++++++++++++++ tracklib2/src/read/decoders.rs | 77 ++++------------- tracklib2/src/read/mod.rs | 1 + tracklib2/src/write/bitstream.rs | 77 +++++++++++++++++ tracklib2/src/write/encoders.rs | 44 +++------- tracklib2/src/write/mod.rs | 1 + 6 files changed, 245 insertions(+), 95 deletions(-) create mode 100644 tracklib2/src/read/bitstream.rs create mode 100644 tracklib2/src/write/bitstream.rs diff --git a/tracklib2/src/read/bitstream.rs b/tracklib2/src/read/bitstream.rs new file mode 100644 index 0000000..b483d50 --- /dev/null +++ b/tracklib2/src/read/bitstream.rs @@ -0,0 +1,140 @@ +use crate::error::{Result, TracklibError}; +use nom::{multi::length_data, number::complete::le_u8}; +use nom_leb128::{leb128_i64, leb128_u64}; + +fn helper<'a>(maybe_is_present: Option, data: &'a [u8]) -> Result> { + match maybe_is_present { + Some(true) => Ok(Some(data)), + Some(false) => Ok(None), + None => Err(TracklibError::ParseIncompleteError { + needed: nom::Needed::Unknown, + }), + } +} + +pub fn read_i64<'a>( + maybe_is_present: Option, + data: &'a [u8], + prev: &mut i64, +) -> Result<(&'a [u8], Option)> { + if let Some(data) = helper(maybe_is_present, data)? { + let (rest, value) = leb128_i64(data)?; + let new = *prev + value; + *prev = new; + Ok((rest, Some(new))) + } else { + Ok((data, None)) + } +} + +pub fn read_bool<'a>( + maybe_is_present: Option, + data: &'a [u8], +) -> Result<(&'a [u8], Option)> { + if let Some(data) = helper(maybe_is_present, data)? { + let (rest, value) = le_u8(data)?; + Ok((rest, Some(value != 0))) + } else { + Ok((data, None)) + } +} + +pub fn read_bytes<'a>( + maybe_is_present: Option, + data: &'a [u8], +) -> Result<(&'a [u8], Option<&'a [u8]>)> { + if let Some(data) = helper(maybe_is_present, data)? { + let (rest, bytes) = length_data(leb128_u64)(data)?; + Ok((rest, Some(bytes))) + } else { + Ok((data, None)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + #[test] + fn test_read_i64() { + #[rustfmt::skip] + let mut buf: &[u8] = &[0x04, + 0x05]; + + let mut prev = 0; + assert_matches!(read_i64(Some(true), buf, &mut prev), Ok((rest, Some(value))) => { + assert_eq!(rest, &buf[1..]); + assert_eq!(value, 4); + buf = rest; + }); + + assert_matches!(read_i64(Some(false), buf, &mut prev), Ok((rest, None)) => { + assert_eq!(rest, buf); + }); + + assert_matches!(read_i64(Some(true), buf, &mut prev), Ok((rest, Some(value))) => { + assert_eq!(rest, &buf[1..]); + assert_eq!(value, 9); // 4 + 5 = 9 + }); + } + + #[test] + fn test_read_bool() { + #[rustfmt::skip] + let mut buf: &[u8] = &[0x00, + 0x01]; + + assert_matches!(read_bool(Some(true), buf), Ok((rest, Some(value))) => { + assert_eq!(rest, &buf[1..]); + assert_eq!(value, false); + buf = rest; + }); + + assert_matches!(read_bool(Some(false), buf), Ok((rest, None)) => { + assert_eq!(rest, buf); + }); + + assert_matches!(read_bool(Some(true), buf), Ok((rest, Some(value))) => { + assert_eq!(rest, &buf[1..]); + assert_eq!(value, true); + }); + } + + #[test] + fn test_read_bytes() { + #[rustfmt::skip] + let mut buf: &[u8] = &[0x02, + b'R', + b'W', + 0x03, + b'G', + b'P', + b'S']; + + assert_matches!(read_bytes(Some(true), buf), Ok((rest, Some(value))) => { + assert_eq!(rest, &buf[3..]); + assert_eq!(value, &[b'R', b'W']); + buf = rest; + }); + + assert_matches!(read_bytes(Some(false), buf), Ok((rest, None)) => { + assert_eq!(rest, buf); + }); + + assert_matches!(read_bytes(Some(true), buf), Ok((rest, Some(value))) => { + assert_eq!(rest, &buf[4..]); + assert_eq!(value, &[b'G', b'P', b'S']); + }); + } + + #[test] + fn test_read_none() { + #[rustfmt::skip] + let buf: &[u8] = &[0x00, + 0x00]; + + let mut prev = 0; + assert_matches!(read_i64(None, buf, &mut prev), Err(_)); + } +} diff --git a/tracklib2/src/read/decoders.rs b/tracklib2/src/read/decoders.rs index 6b32e20..6a7f9c8 100644 --- a/tracklib2/src/read/decoders.rs +++ b/tracklib2/src/read/decoders.rs @@ -1,9 +1,8 @@ +use super::bitstream; use super::crc::CRC; use super::presence_column::PresenceColumnView; use crate::error::Result; use crate::error::TracklibError; -use nom::{multi::length_data, number::complete::le_u8}; -use nom_leb128::{leb128_i64, leb128_u64}; pub(crate) trait Decoder { type T; @@ -49,22 +48,10 @@ impl<'a> Decoder for I64Decoder<'a> { type T = i64; fn decode(&mut self) -> Result> { - if let Some(is_present) = self.presence_column_view.next() { - if is_present { - let (rest, value) = leb128_i64(self.data)?; - let new = self.prev + value; - self.prev = new; - self.data = rest; - Ok(Some(new)) - } else { - Ok(None) - } - } else { - // ran out of rows, this is an error - return Err(TracklibError::ParseIncompleteError { - needed: nom::Needed::Unknown, - }); - } + let (rest, value) = + bitstream::read_i64(self.presence_column_view.next(), self.data, &mut self.prev)?; + self.data = rest; + Ok(value) } } @@ -94,22 +81,10 @@ impl<'a> Decoder for F64Decoder<'a> { type T = f64; fn decode(&mut self) -> Result> { - if let Some(is_present) = self.presence_column_view.next() { - if is_present { - let (rest, value) = leb128_i64(self.data)?; - let new = self.prev + value; - self.prev = new; - self.data = rest; - Ok(Some((new as f64) / 10e6)) - } else { - Ok(None) - } - } else { - // ran out of rows, this is an error - return Err(TracklibError::ParseIncompleteError { - needed: nom::Needed::Unknown, - }); - } + let (rest, value) = + bitstream::read_i64(self.presence_column_view.next(), self.data, &mut self.prev)?; + self.data = rest; + Ok(value.map(|v| (v as f64) / 10e6)) } } @@ -137,20 +112,9 @@ impl<'a> Decoder for BoolDecoder<'a> { type T = bool; fn decode(&mut self) -> Result> { - if let Some(is_present) = self.presence_column_view.next() { - if is_present { - let (rest, value) = le_u8(self.data)?; - self.data = rest; - Ok(Some(value != 0)) - } else { - Ok(None) - } - } else { - // ran out of rows, this is an error - return Err(TracklibError::ParseIncompleteError { - needed: nom::Needed::Unknown, - }); - } + let (rest, value) = bitstream::read_bool(self.presence_column_view.next(), self.data)?; + self.data = rest; + Ok(value) } } @@ -178,20 +142,9 @@ impl<'a> Decoder for StringDecoder<'a> { type T = String; fn decode(&mut self) -> Result> { - if let Some(is_present) = self.presence_column_view.next() { - if is_present { - let (rest, string_bytes) = length_data(leb128_u64)(self.data)?; - self.data = rest; - Ok(Some(String::from_utf8(string_bytes.to_vec()).unwrap())) - } else { - Ok(None) - } - } else { - // ran out of rows, this is an error - return Err(TracklibError::ParseIncompleteError { - needed: nom::Needed::Unknown, - }); - } + let (rest, value) = bitstream::read_bytes(self.presence_column_view.next(), self.data)?; + self.data = rest; + Ok(value.map(|bytes| String::from_utf8_lossy(bytes).into_owned())) } } diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 3eef940..aa58971 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -1,3 +1,4 @@ +pub mod bitstream; mod crc; mod data_table; mod decoders; diff --git a/tracklib2/src/write/bitstream.rs b/tracklib2/src/write/bitstream.rs new file mode 100644 index 0000000..863fd93 --- /dev/null +++ b/tracklib2/src/write/bitstream.rs @@ -0,0 +1,77 @@ +use crate::error::Result; +use std::io::Write; + +pub fn write_i64(value: Option<&i64>, buf: &mut Vec, prev: &mut i64) -> Result<()> { + if let Some(val) = value { + let v = *val; + let delta = v - *prev; + *prev = v; + leb128::write::signed(buf, delta)?; + } + + Ok(()) +} + +pub fn write_bool(value: Option<&bool>, buf: &mut Vec) -> Result<()> { + if let Some(val) = value { + let v = *val as u8; + buf.write_all(&v.to_le_bytes())?; + } + Ok(()) +} + +pub fn write_bytes(value: Option<&[u8]>, buf: &mut Vec) -> Result<()> { + if let Some(val) = value { + // Write len + leb128::write::unsigned(buf, u64::try_from(val.len()).expect("usize != u64"))?; + // Write bytes + buf.write_all(val)?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + + #[test] + fn test_write_i64() { + let mut buf = vec![]; + let mut prev = 0; + assert_matches!(write_i64(Some(&0), &mut buf, &mut prev), Ok(())); + assert_matches!(write_i64(None, &mut buf, &mut prev), Ok(())); + assert_matches!(write_i64(Some(&42), &mut buf, &mut prev), Ok(())); + #[rustfmt::skip] + assert_eq!(buf, &[0x00, + 0x2A]); + } + + #[test] + fn test_write_bool() { + let mut buf = vec![]; + assert_matches!(write_bool(Some(&false), &mut buf), Ok(())); + assert_matches!(write_bool(None, &mut buf), Ok(())); + assert_matches!(write_bool(Some(&true), &mut buf), Ok(())); + #[rustfmt::skip] + assert_eq!(buf, &[0x00, + 0x01]); + } + + #[test] + fn test_write_bytes() { + let mut buf = vec![]; + assert_matches!(write_bytes(Some(&[b'R', b'W']), &mut buf), Ok(())); + assert_matches!(write_bytes(None, &mut buf), Ok(())); + assert_matches!(write_bytes(Some(&[b'G', b'P', b'S']), &mut buf), Ok(())); + #[rustfmt::skip] + assert_eq!(buf, &[0x02, + b'R', + b'W', + 0x03, + b'G', + b'P', + b'S']); + } +} diff --git a/tracklib2/src/write/encoders.rs b/tracklib2/src/write/encoders.rs index cfcfd0c..727d942 100644 --- a/tracklib2/src/write/encoders.rs +++ b/tracklib2/src/write/encoders.rs @@ -1,6 +1,5 @@ +use super::bitstream; use crate::error::Result; -use std::convert::TryFrom; -use std::io::Write; pub trait Encoder: Default { type T; @@ -27,14 +26,7 @@ impl Encoder for I64Encoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - if let Some(val) = value { - let v = *val; - let delta = v - self.prev; - self.prev = v; - leb128::write::signed(buf, delta)?; - } - - Ok(()) + bitstream::write_i64(value, buf, &mut self.prev) } } @@ -53,14 +45,11 @@ impl Encoder for F64Encoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - if let Some(val) = value { - let v = (*val * 10e6) as i64; - let delta = v - self.prev; - self.prev = v; - leb128::write::signed(buf, delta)?; - } - - Ok(()) + bitstream::write_i64( + value.map(|val| (*val * 10e6) as i64).as_ref(), + buf, + &mut self.prev, + ) } } @@ -77,11 +66,7 @@ impl Encoder for BoolEncoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - if let Some(val) = value { - let v = *val as u8; - buf.write_all(&v.to_le_bytes())?; - } - Ok(()) + bitstream::write_bool(value, buf) } } @@ -98,14 +83,7 @@ impl Encoder for StringEncoder { presence: &mut Vec, ) -> Result<()> { presence.push(value.is_some()); - if let Some(string) = value { - // Write the length of the string - leb128::write::unsigned(buf, u64::try_from(string.len()).unwrap())?; - // Write the string itself - buf.write_all(string.as_bytes())?; - } - - Ok(()) + bitstream::write_bytes(value.map(|v| v.as_bytes()), buf) } } @@ -231,7 +209,7 @@ mod tests { fn test_bool_encoder() { let mut data_buf = vec![]; let mut presence_buf = vec![]; - let mut encoder = BoolEncoder; + let mut encoder = BoolEncoder::default(); assert!(encoder .encode(Some(&true), &mut data_buf, &mut presence_buf) @@ -278,7 +256,7 @@ mod tests { fn test_string_encoder() { let mut data_buf = vec![]; let mut presence_buf = vec![]; - let mut encoder = StringEncoder; + let mut encoder = StringEncoder::default(); assert!(encoder .encode(Some(&"A".to_string()), &mut data_buf, &mut presence_buf) diff --git a/tracklib2/src/write/mod.rs b/tracklib2/src/write/mod.rs index 35a3019..f183eff 100644 --- a/tracklib2/src/write/mod.rs +++ b/tracklib2/src/write/mod.rs @@ -1,3 +1,4 @@ +pub mod bitstream; mod crcwriter; mod data_table; pub mod encoders; From 21e061074fbe9267998a1844701f7a6866eebdc7 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 22 Mar 2022 13:19:09 -0700 Subject: [PATCH 039/113] better handling of invalid utf8 --- tracklib2/src/read/data_table.rs | 43 +++++++++++++++++++++++++++++++ tracklib2/src/read/decoders.rs | 29 +++++++++++++++++++++ tracklib2/src/read/types_table.rs | 15 ++++------- tracklib2/src/write/data_table.rs | 36 ++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 10 deletions(-) diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index 5f5e46d..7eb444e 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -106,6 +106,7 @@ mod tests { 0x00, // section type = track points 0x00, // leb128 section point count 0x00, // leb128 section data size + // Types Table 0x03, // field count 0x00, // first field type = I64 @@ -184,4 +185,46 @@ mod tests { ); }); } + + #[test] + fn test_invalid_utf8_fieldname() { + #[rustfmt::skip] + let buf = &[0x01, // number of sections + + // Section 1 + 0x00, // section type = track points + 0x00, // leb128 section point count + 0x00, // leb128 section data size + + // Types Table + 0x01, // field count + 0x00, // first field type = I64 + 0x05, // name length + b'a', // name with invalid utf8 + 0xF0, + 0x90, + 0x80, + b'b', + 0x00, // leb128 data size + + 0xEB, // crc + 0xE9]; + + assert_matches!(parse_data_table(buf), Ok((&[], entries)) => { + assert_eq!( + entries, + vec![ + DataTableEntry { + section_type: SectionType::TrackPoints, + offset: 0, + size: 0, + rows: 0, + types: vec![ + TypesTableEntry::new_for_tests(FieldType::I64, "a�b", 0, 0), + ] + } + ] + ); + }); + } } diff --git a/tracklib2/src/read/decoders.rs b/tracklib2/src/read/decoders.rs index 6a7f9c8..132209a 100644 --- a/tracklib2/src/read/decoders.rs +++ b/tracklib2/src/read/decoders.rs @@ -316,6 +316,35 @@ mod tests { }); } + #[test] + fn test_decode_string_with_invalid_utf8() { + #[rustfmt::skip] + let presence_buf = &[0b00000001, + 0xFC, // crc + 0x5D, + 0x36, + 0xB5]; + let presence_column = + assert_matches!(parse_presence_column(1, 1)(presence_buf), Ok((&[], pc)) => pc); + let presence_column_view = assert_matches!(presence_column.view(0), Some(v) => v); + #[rustfmt::skip] + let buf = &[0x05, + b'a', // name with invalid utf8 + 0xF0, + 0x90, + 0x80, + b'b', + 0xE2, // crc + 0x76, + 0xD0, + 0x42]; + assert_matches!(StringDecoder::new(buf, presence_column_view), Ok(mut decoder) => { + assert_matches!(decoder.decode(), Ok(Some(s)) => { + assert_eq!(s, "a�b"); + }); + }); + } + #[test] fn test_decode_bad_crc() { #[rustfmt::skip] diff --git a/tracklib2/src/read/types_table.rs b/tracklib2/src/read/types_table.rs index d739f2b..4379e03 100644 --- a/tracklib2/src/read/types_table.rs +++ b/tracklib2/src/read/types_table.rs @@ -62,14 +62,7 @@ fn parse_types_table_entry<'a>( } }; - let name = match String::from_utf8(field_name.to_vec()) { - Ok(s) => s, - Err(_) => { - return Err(nom::Err::Error(TracklibError::ParseError { - error_kind: nom::error::ErrorKind::Tag, - })) - } - }; + let name = String::from_utf8_lossy(field_name).into_owned(); Ok(( input, @@ -168,8 +161,10 @@ mod tests { 0x01, // name len = 1 0xC0, // name: invalid utf-8 0x02]; // data size = 2 - assert_matches!(parse_types_table(buf), Err(nom::Err::Error(TracklibError::ParseError{error_kind})) => { - assert_eq!(error_kind, nom::error::ErrorKind::Tag); + assert_matches!(parse_types_table(buf), Ok((&[], entries)) => { + assert_eq!(entries, vec![TypesTableEntry{field_description: FieldDescription::new("�".to_string(), FieldType::I64), + size: 2, + offset: 0}]) }); } } diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index 258b106..53c7d5c 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -116,4 +116,40 @@ mod tests { 0xF8]); }); } + + #[test] + fn test_data_table_with_multibyte_character() { + let section = Section::new( + SectionType::TrackPoints, + vec![("I ♥ NY".to_string(), FieldType::F64)], + ); + + let mut buf = Vec::new(); + assert_matches!(write_data_table(&mut buf, &[section]), Ok(()) => { + #[rustfmt::skip] + assert_eq!(buf, + &[0x01, // number of sections + + // Section 1 + 0x00, // section type = track points + 0x00, // leb128 section point count + 0x08, // leb128 section data size + // Types Table + 0x01, // field count + 0x01, // first field type = F64 + 0x08, // name length + b'I', // name + b' ', + 0xE2, // heart + 0x99, + 0xA5, + b' ', + b'N', + b'Y', + 0x04, // leb128 data size + + 0x7D, // crc + 0x13]); + }); + } } From acec8a58de0c3e3eb06c36c159f721d6e7dd7b42 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 11:31:08 -0700 Subject: [PATCH 040/113] begin renaming Types Table -> Schema --- tracklib2/src/lib.rs | 1 + tracklib2/src/read/data_table.rs | 28 ++--- tracklib2/src/read/inspect.rs | 9 +- tracklib2/src/read/mod.rs | 2 +- tracklib2/src/read/section_reader.rs | 152 +++++++++++++---------- tracklib2/src/read/types_table.rs | 48 +++----- tracklib2/src/schema.rs | 65 ++++++++++ tracklib2/src/types.rs | 32 +---- tracklib2/src/write/data_table.rs | 25 ++-- tracklib2/src/write/section.rs | 174 ++++++++++++--------------- tracklib2/src/write/track.rs | 33 ++--- 11 files changed, 301 insertions(+), 268 deletions(-) create mode 100644 tracklib2/src/schema.rs diff --git a/tracklib2/src/lib.rs b/tracklib2/src/lib.rs index 6e564bf..a2d51b7 100644 --- a/tracklib2/src/lib.rs +++ b/tracklib2/src/lib.rs @@ -1,5 +1,6 @@ mod consts; pub mod error; pub mod read; +pub mod schema; pub mod types; pub mod write; diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index 7eb444e..85a7bfb 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -7,7 +7,7 @@ use nom_leb128::leb128_u64; use std::convert::TryFrom; #[cfg_attr(test, derive(Debug, PartialEq))] -pub struct DataTableEntry { +pub(crate) struct DataTableEntry { section_type: SectionType, offset: usize, size: usize, @@ -16,23 +16,23 @@ pub struct DataTableEntry { } impl DataTableEntry { - pub fn section_type(&self) -> &SectionType { + pub(crate) fn section_type(&self) -> &SectionType { &self.section_type } - pub fn offset(&self) -> usize { + pub(crate) fn offset(&self) -> usize { self.offset } - pub fn size(&self) -> usize { + pub(crate) fn size(&self) -> usize { self.size } - pub fn rows(&self) -> usize { + pub(crate) fn rows(&self) -> usize { self.rows } - pub fn types(&self) -> &[TypesTableEntry] { + pub(crate) fn types(&self) -> &[TypesTableEntry] { self.types.as_slice() } } @@ -94,7 +94,7 @@ pub(crate) fn parse_data_table(input: &[u8]) -> IResult<&[u8], Vec { I64 { - field_description: &'a FieldDescription, + field_definition: &'a FieldDefinition, decoder: I64Decoder<'a>, }, F64 { - field_description: &'a FieldDescription, + field_definition: &'a FieldDefinition, decoder: F64Decoder<'a>, }, Bool { - field_description: &'a FieldDescription, + field_definition: &'a FieldDefinition, decoder: BoolDecoder<'a>, }, String { - field_description: &'a FieldDescription, + field_definition: &'a FieldDefinition, decoder: StringDecoder<'a>, }, } #[cfg_attr(test, derive(Debug))] pub struct SectionReader<'a> { - data_table_entry: &'a DataTableEntry, decoders: Vec>, + section_type: SectionType, + schema: Schema, rows: usize, } @@ -36,34 +38,43 @@ impl<'a> SectionReader<'a> { let (column_data, presence_column) = parse_presence_column(data_table_entry.types().len(), data_table_entry.rows())(input)?; + let schema = Schema::with_fields( + data_table_entry + .types() + .iter() + .map(|types_table_entry| types_table_entry.field_definition().clone()) + .collect(), + ); + let decoders = data_table_entry .types() .iter() .enumerate() - .map(|(i, field)| { - let column_data = &column_data[field.offset()..field.offset() + field.size()]; + .map(|(i, types_table_entry)| { + let column_data = &column_data[types_table_entry.offset() + ..types_table_entry.offset() + types_table_entry.size()]; let presence_column_view = presence_column .view(i) .ok_or_else(|| TracklibError::ParseIncompleteError { needed: nom::Needed::Unknown, })?; - let field_description = field.field_description(); - let decoder = match field_description.fieldtype() { - FieldType::I64 => ColumnDecoder::I64 { - field_description, + let field_definition = types_table_entry.field_definition(); + let decoder = match field_definition.data_type() { + DataType::I64 => ColumnDecoder::I64 { + field_definition, decoder: I64Decoder::new(column_data, presence_column_view)?, }, - FieldType::F64 => ColumnDecoder::F64 { - field_description, + DataType::F64 => ColumnDecoder::F64 { + field_definition, decoder: F64Decoder::new(column_data, presence_column_view)?, }, - FieldType::Bool => ColumnDecoder::Bool { - field_description, + DataType::Bool => ColumnDecoder::Bool { + field_definition, decoder: BoolDecoder::new(column_data, presence_column_view)?, }, - FieldType::String => ColumnDecoder::String { - field_description, + DataType::String => ColumnDecoder::String { + field_definition, decoder: StringDecoder::new(column_data, presence_column_view)?, }, }; @@ -72,14 +83,23 @@ impl<'a> SectionReader<'a> { .collect::>>()?; Ok(Self { - data_table_entry, + schema, + section_type: data_table_entry.section_type().clone(), decoders, rows: data_table_entry.rows(), }) } - pub fn data_table_entry(&self) -> &'a DataTableEntry { - self.data_table_entry + pub fn section_type(&self) -> &SectionType { + &self.section_type + } + + pub fn rows(&self) -> usize { + self.rows + } + + pub fn schema(&self) -> &Schema { + &self.schema } pub fn open_column_iter<'r>(&'r mut self) -> Option> { @@ -105,46 +125,46 @@ impl<'a, 'b> ColumnIter<'a, 'b> { } impl<'a, 'b> Iterator for ColumnIter<'a, 'b> { - type Item = Result<(&'a FieldDescription, Option)>; + type Item = Result<(&'a FieldDefinition, Option)>; fn next(&mut self) -> Option { if let Some(decoder_enum) = self.decoders.get_mut(self.index) { self.index += 1; match decoder_enum { ColumnDecoder::I64 { - ref field_description, + ref field_definition, ref mut decoder, } => Some( decoder .decode() - .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::I64(v)))) + .map(|maybe_v| (*field_definition, maybe_v.map(|v| FieldValue::I64(v)))) .map_err(|e| e), ), ColumnDecoder::F64 { - ref field_description, + ref field_definition, ref mut decoder, } => Some( decoder .decode() - .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::F64(v)))) + .map(|maybe_v| (*field_definition, maybe_v.map(|v| FieldValue::F64(v)))) .map_err(|e| e), ), ColumnDecoder::Bool { - ref field_description, + ref field_definition, ref mut decoder, } => Some( decoder .decode() - .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::Bool(v)))) + .map(|maybe_v| (*field_definition, maybe_v.map(|v| FieldValue::Bool(v)))) .map_err(|e| e), ), ColumnDecoder::String { - ref field_description, + ref field_definition, ref mut decoder, } => Some( decoder .decode() - .map(|maybe_v| (*field_description, maybe_v.map(|v| FieldValue::String(v)))) + .map(|maybe_v| (*field_definition, maybe_v.map(|v| FieldValue::String(v)))) .map_err(|e| e), ), } @@ -264,24 +284,24 @@ mod tests { assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); - assert_matches!(&values[0], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); - assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[0].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("a", DataType::I64)); assert_eq!(field_value, &Some(FieldValue::I64(1))); }); - assert_matches!(&values[1], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[1].field_description()); - assert_eq!(*field_description, &FieldDescription::new("b".to_string(), FieldType::Bool)); + assert_matches!(&values[1], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); assert_eq!(field_value, &Some(FieldValue::Bool(false))); }); - assert_matches!(&values[2], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[2].field_description()); - assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); + assert_matches!(&values[2], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[2].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("c", DataType::String)); assert_eq!(field_value, &Some(FieldValue::String("Ride".to_string()))); }); - assert_matches!(&values[3], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[3].field_description()); - assert_eq!(*field_description, &FieldDescription::new("f".to_string(), FieldType::F64)); + assert_matches!(&values[3], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[3].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); assert_eq!(field_value, &None); }); }); @@ -290,24 +310,24 @@ mod tests { assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); - assert_matches!(&values[0], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); - assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[0].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("a", DataType::I64)); assert_eq!(field_value, &Some(FieldValue::I64(2))); }); - assert_matches!(&values[1], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[1].field_description()); - assert_eq!(*field_description, &FieldDescription::new("b".to_string(), FieldType::Bool)); + assert_matches!(&values[1], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); assert_eq!(field_value, &None); }); - assert_matches!(&values[2], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[2].field_description()); - assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); + assert_matches!(&values[2], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[2].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("c", DataType::String)); assert_eq!(field_value, &Some(FieldValue::String("with".to_string()))); }); - assert_matches!(&values[3], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[3].field_description()); - assert_eq!(*field_description, &FieldDescription::new("f".to_string(), FieldType::F64)); + assert_matches!(&values[3], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[3].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); assert_eq!(field_value, &Some(FieldValue::F64(1.0))); }); }); @@ -316,24 +336,24 @@ mod tests { assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); - assert_matches!(&values[0], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[0].field_description()); - assert_eq!(*field_description, &FieldDescription::new("a".to_string(), FieldType::I64)); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[0].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("a", DataType::I64)); assert_eq!(field_value, &Some(FieldValue::I64(4))); }); - assert_matches!(&values[1], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[1].field_description()); - assert_eq!(*field_description, &FieldDescription::new("b".to_string(), FieldType::Bool)); + assert_matches!(&values[1], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); assert_eq!(field_value, &Some(FieldValue::Bool(true))); }); - assert_matches!(&values[2], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[2].field_description()); - assert_eq!(*field_description, &FieldDescription::new("c".to_string(), FieldType::String)); + assert_matches!(&values[2], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[2].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("c", DataType::String)); assert_eq!(field_value, &Some(FieldValue::String("GPS".to_string()))); }); - assert_matches!(&values[3], Ok((field_description, field_value)) => { - assert_eq!(*field_description, data_table_entries[0].types()[3].field_description()); - assert_eq!(*field_description, &FieldDescription::new("f".to_string(), FieldType::F64)); + assert_matches!(&values[3], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].types()[3].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); assert_eq!(field_value, &Some(FieldValue::F64(2.5))); }); }); diff --git a/tracklib2/src/read/types_table.rs b/tracklib2/src/read/types_table.rs index 4379e03..867e303 100644 --- a/tracklib2/src/read/types_table.rs +++ b/tracklib2/src/read/types_table.rs @@ -1,13 +1,13 @@ use crate::error::TracklibError; -use crate::types::{FieldDescription, FieldType}; +use crate::schema::*; use nom::{multi::length_data, number::complete::le_u8, IResult}; use nom_leb128::leb128_u64; use std::convert::TryFrom; #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] -pub struct TypesTableEntry { - field_description: FieldDescription, +pub(crate) struct TypesTableEntry { + field_definition: FieldDefinition, size: usize, offset: usize, } @@ -15,13 +15,13 @@ pub struct TypesTableEntry { #[cfg(test)] impl TypesTableEntry { pub(crate) fn new_for_tests( - fieldtype: FieldType, name: &str, + data_type: DataType, size: usize, offset: usize, ) -> Self { Self { - field_description: FieldDescription::new(name.to_string(), fieldtype), + field_definition: FieldDefinition::new(name, data_type), size, offset, } @@ -29,8 +29,8 @@ impl TypesTableEntry { } impl TypesTableEntry { - pub(crate) fn field_description(&self) -> &FieldDescription { - &self.field_description + pub(crate) fn field_definition(&self) -> &FieldDefinition { + &self.field_definition } pub(crate) fn size(&self) -> usize { @@ -50,11 +50,11 @@ fn parse_types_table_entry<'a>( let (input, field_name) = length_data(le_u8)(input)?; let (input, data_size) = leb128_u64(input)?; - let fieldtype = match type_tag { - 0x00 => FieldType::I64, - 0x01 => FieldType::F64, - 0x04 => FieldType::String, - 0x05 => FieldType::Bool, + let data_type = match type_tag { + 0x00 => DataType::I64, + 0x01 => DataType::F64, + 0x04 => DataType::String, + 0x05 => DataType::Bool, _ => { return Err(nom::Err::Error(TracklibError::ParseError { error_kind: nom::error::ErrorKind::Tag, @@ -62,12 +62,12 @@ fn parse_types_table_entry<'a>( } }; - let name = String::from_utf8_lossy(field_name).into_owned(); + let name = String::from_utf8_lossy(field_name); Ok(( input, TypesTableEntry { - field_description: FieldDescription::new(name, fieldtype), + field_definition: FieldDefinition::new(name, data_type), size: usize::try_from(data_size).expect("usize != u64"), offset, }, @@ -125,18 +125,10 @@ mod tests { b'i', // name = "i" 0x02]; // data size = 2 assert_matches!(parse_types_table(buf), Ok((&[], entries)) => { - assert_eq!(entries, vec![TypesTableEntry{field_description: FieldDescription::new("m".to_string(), FieldType::I64), - size: 2, - offset: 0}, - TypesTableEntry{field_description: FieldDescription::new("k".to_string(), FieldType::Bool), - size: 1, - offset: 2}, - TypesTableEntry{field_description: FieldDescription::new("long name!".to_string(), FieldType::String), - size: 7, - offset: 3}, - TypesTableEntry{field_description: FieldDescription::new("i".to_string(), FieldType::F64), - size: 2, - offset: 10}]); + assert_eq!(entries, vec![TypesTableEntry::new_for_tests("m", DataType::I64, 2, 0), + TypesTableEntry::new_for_tests("k", DataType::Bool, 1, 2), + TypesTableEntry::new_for_tests("long name!", DataType::String, 7, 3), + TypesTableEntry::new_for_tests("i", DataType::F64, 2, 10)]); }); } @@ -162,9 +154,7 @@ mod tests { 0xC0, // name: invalid utf-8 0x02]; // data size = 2 assert_matches!(parse_types_table(buf), Ok((&[], entries)) => { - assert_eq!(entries, vec![TypesTableEntry{field_description: FieldDescription::new("�".to_string(), FieldType::I64), - size: 2, - offset: 0}]) + assert_eq!(entries, vec![TypesTableEntry::new_for_tests("�", DataType::I64, 2, 0)]) }); } } diff --git a/tracklib2/src/schema.rs b/tracklib2/src/schema.rs new file mode 100644 index 0000000..3a92230 --- /dev/null +++ b/tracklib2/src/schema.rs @@ -0,0 +1,65 @@ +#[derive(Debug)] +pub struct Schema { + fields: Vec, +} + +impl Schema { + pub fn with_fields(fields: Vec) -> Self { + Self { fields } + } + + pub fn fields(&self) -> &[FieldDefinition] { + &self.fields + } +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub struct FieldDefinition { + name: String, + data_type: DataType, +} + +impl FieldDefinition { + pub fn new>(name: S, data_type: DataType) -> Self { + Self { + name: name.into(), + data_type, + } + } + + pub fn name(&self) -> &str { + self.name.as_str() + } + + pub fn data_type(&self) -> &DataType { + &self.data_type + } +} + +#[derive(Debug)] +pub enum BitstreamType { + Bytes, + I64, + Bool, +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub enum DataType { + I64, + Bool, + String, + F64, +} + +impl DataType { + pub fn bitstream_type(&self) -> BitstreamType { + match self { + Self::I64 => BitstreamType::I64, + Self::Bool => BitstreamType::Bool, + Self::String => BitstreamType::Bytes, + Self::F64 => BitstreamType::I64, + } + } +} diff --git a/tracklib2/src/types.rs b/tracklib2/src/types.rs index 4155725..3af3ad7 100644 --- a/tracklib2/src/types.rs +++ b/tracklib2/src/types.rs @@ -1,33 +1,3 @@ -#[derive(Debug, Clone)] -#[cfg_attr(test, derive(PartialEq))] -pub enum FieldType { - I64, - F64, - String, - Bool, -} - -#[derive(Clone, Debug)] -#[cfg_attr(test, derive(PartialEq))] -pub struct FieldDescription { - name: String, - fieldtype: FieldType, -} - -impl FieldDescription { - pub fn new(name: String, fieldtype: FieldType) -> Self { - Self { name, fieldtype } - } - - pub fn name(&self) -> &str { - self.name.as_str() - } - - pub fn fieldtype(&self) -> &FieldType { - &self.fieldtype - } -} - #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum FieldValue { @@ -52,7 +22,7 @@ pub enum MetadataEntry { CreatedAt(u64), } -#[derive(Debug)] +#[derive(Clone, Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum SectionType { TrackPoints, diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index 53c7d5c..b5b4437 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -23,7 +23,8 @@ pub(crate) fn write_data_table(out: &mut W, sections: &[Section]) -> R #[cfg(test)] mod tests { use super::*; - use crate::types::{FieldType, SectionType}; + use crate::schema::*; + use crate::types::SectionType; use assert_matches::assert_matches; #[test] @@ -41,20 +42,20 @@ mod tests { fn test_data_table() { let section1 = Section::new( SectionType::TrackPoints, - vec![ - ("a".to_string(), FieldType::I64), - ("b".to_string(), FieldType::Bool), - ("c".to_string(), FieldType::String), - ], + Schema::with_fields(vec![ + FieldDefinition::new("a", DataType::I64), + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("c", DataType::String), + ]), ); let section2 = Section::new( SectionType::CoursePoints, - vec![ - ("Ride".to_string(), FieldType::I64), - ("with".to_string(), FieldType::Bool), - ("GPS".to_string(), FieldType::String), - ], + Schema::with_fields(vec![ + FieldDefinition::new("Ride", DataType::I64), + FieldDefinition::new("with", DataType::Bool), + FieldDefinition::new("GPS", DataType::String), + ]), ); let mut buf = Vec::new(); @@ -121,7 +122,7 @@ mod tests { fn test_data_table_with_multibyte_character() { let section = Section::new( SectionType::TrackPoints, - vec![("I ♥ NY".to_string(), FieldType::F64)], + Schema::with_fields(vec![FieldDefinition::new("I ♥ NY", DataType::F64)]), ); let mut buf = Vec::new(); diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index ace2f9f..2815341 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -1,11 +1,12 @@ use super::crcwriter::CrcWriter; use super::encoders::*; use crate::error::Result; -use crate::types::{FieldDescription, FieldType, SectionType}; +use crate::schema::*; +use crate::types::SectionType; use std::convert::TryFrom; use std::io::{self, Write}; -impl FieldType { +impl DataType { fn type_tag(&self) -> u8 { match self { Self::I64 => 0x00, @@ -55,12 +56,12 @@ enum Buffer { } impl Buffer { - fn new(field_type: &FieldType) -> Self { - match field_type { - &FieldType::I64 => Buffer::I64(BufferImpl::default()), - &FieldType::F64 => Buffer::F64(BufferImpl::default()), - &FieldType::Bool => Buffer::Bool(BufferImpl::default()), - &FieldType::String => Buffer::String(BufferImpl::default()), + fn new(data_type: &DataType) -> Self { + match data_type { + DataType::I64 => Buffer::I64(BufferImpl::default()), + DataType::Bool => Buffer::Bool(BufferImpl::default()), + DataType::String => Buffer::String(BufferImpl::default()), + DataType::F64 => Buffer::F64(BufferImpl::default()), } } @@ -77,37 +78,35 @@ impl Buffer { pub struct Section { section_type: SectionType, rows_written: usize, - fields: Vec, + schema: Schema, column_data: Vec, } impl Section { // TODO: provide a size_hint param to size buffer Vecs (at least presence) - pub fn new(section_type: SectionType, mapping: Vec<(String, FieldType)>) -> Self { - let mut fields = Vec::with_capacity(mapping.len()); - let mut column_data = Vec::with_capacity(mapping.len()); - - for (name, fieldtype) in mapping { - column_data.push(Buffer::new(&fieldtype)); - fields.push(FieldDescription::new(name, fieldtype)); - } + pub fn new(section_type: SectionType, schema: Schema) -> Self { + let column_data = schema + .fields() + .iter() + .map(|field_def| Buffer::new(field_def.data_type())) + .collect(); Self { section_type, rows_written: 0, - fields, + schema, column_data, } } - pub fn fields(&self) -> &[FieldDescription] { - &self.fields + pub fn schema(&self) -> &Schema { + &self.schema } /// mut borrow of self so only one row can be open at a time pub fn open_row_builder(&mut self) -> RowBuilder { self.rows_written += 1; // TODO: bug when you open a row builder but never write data with it? - RowBuilder::new(&self.fields, &mut self.column_data) + RowBuilder::new(&self.schema, &mut self.column_data) } pub(crate) fn type_tag(&self) -> u8 { @@ -120,7 +119,7 @@ impl Section { pub(crate) fn data_size(&self) -> usize { const CRC_BYTES: usize = 4; - let presence_bytes_required = (self.fields.len() + 7) / 8; + let presence_bytes_required = (self.schema.fields().len() + 7) / 8; let presence_bytes = (presence_bytes_required * self.rows_written) + CRC_BYTES; let data_bytes: usize = self .column_data @@ -132,12 +131,12 @@ impl Section { fn write_presence_column(&self, out: &mut W) -> Result<()> { let mut crcwriter = CrcWriter::new32(out); - let bytes_required = (self.fields.len() + 7) / 8; + let bytes_required = (self.schema.fields().len() + 7) / 8; for row_i in 0..self.rows_written { let mut row = vec![0; bytes_required]; let mut mask: u8 = 1; - let mut bit_index = (self.fields.len() + 7) & !7; // next multiple of 8 + let mut bit_index = (self.schema.fields().len() + 7) & !7; // next multiple of 8 for buffer in self.column_data.iter() { let is_present = match buffer { Buffer::I64(buffer_impl) => buffer_impl.presence.get(row_i), @@ -164,9 +163,9 @@ impl Section { #[rustfmt::skip] pub(crate) fn write_types_table(&self, out: &mut W) -> Result<()> { - out.write_all(&u8::try_from(self.fields.len())?.to_le_bytes())?; // 1 byte - number of entries + out.write_all(&u8::try_from(self.schema.fields().len())?.to_le_bytes())?; // 1 byte - number of entries - for (i, field) in self.fields.iter().enumerate() { + for (i, field_def) in self.schema.fields().iter().enumerate() { let data_column_size = self .column_data .get(i) @@ -178,9 +177,9 @@ impl Section { }) .unwrap_or(0); - out.write_all(&field.fieldtype().type_tag().to_le_bytes())?; // 1 byte - field type tag - out.write_all(&u8::try_from(field.name().len())?.to_le_bytes())?; // 1 byte - field name length - out.write_all(field.name().as_bytes())?; // ? bytes - the name of this field + out.write_all(&field_def.data_type().type_tag().to_le_bytes())?; // 1 byte - field type tag + out.write_all(&u8::try_from(field_def.name().len())?.to_le_bytes())?; // 1 byte - field name length + out.write_all(field_def.name().as_bytes())?; // ? bytes - the name of this field leb128::write::unsigned(out, u64::try_from(data_column_size)?)?; // ? bytes - leb128 column data size } @@ -204,15 +203,15 @@ impl Section { } pub struct RowBuilder<'a> { - fields: &'a [FieldDescription], + schema: &'a Schema, column_data: &'a mut Vec, field_index: usize, } impl<'a> RowBuilder<'a> { - fn new(fields: &'a [FieldDescription], column_data: &'a mut Vec) -> Self { + fn new(schema: &'a Schema, column_data: &'a mut Vec) -> Self { Self { - fields, + schema, column_data, field_index: 0, } @@ -223,21 +222,21 @@ impl<'a> RowBuilder<'a> { let field_index = self.field_index; self.field_index += 1; - let maybe_field_desc = self.fields.get(field_index); + let maybe_field_def = self.schema.fields().get(field_index); let maybe_buffer = self.column_data.get_mut(field_index); - match (maybe_field_desc, maybe_buffer) { - (Some(field_desc), Some(Buffer::I64(ref mut buffer_impl))) => Some( - ColumnWriter::I64ColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + match (maybe_field_def, maybe_buffer) { + (Some(field_def), Some(Buffer::I64(ref mut buffer_impl))) => Some( + ColumnWriter::I64ColumnWriter(ColumnWriterImpl::new(&field_def, buffer_impl)), ), - (Some(field_desc), Some(Buffer::F64(ref mut buffer_impl))) => Some( - ColumnWriter::F64ColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + (Some(field_def), Some(Buffer::F64(ref mut buffer_impl))) => Some( + ColumnWriter::F64ColumnWriter(ColumnWriterImpl::new(&field_def, buffer_impl)), ), - (Some(field_desc), Some(Buffer::Bool(ref mut buffer_impl))) => Some( - ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + (Some(field_def), Some(Buffer::Bool(ref mut buffer_impl))) => Some( + ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(&field_def, buffer_impl)), ), - (Some(field_desc), Some(Buffer::String(ref mut buffer_impl))) => Some( - ColumnWriter::StringColumnWriter(ColumnWriterImpl::new(&field_desc, buffer_impl)), + (Some(field_def), Some(Buffer::String(ref mut buffer_impl))) => Some( + ColumnWriter::StringColumnWriter(ColumnWriterImpl::new(&field_def, buffer_impl)), ), _ => None, } @@ -256,20 +255,20 @@ pub enum ColumnWriter<'a> { // TODO: is this must_use correct? #[must_use] pub struct ColumnWriterImpl<'a, E: Encoder> { - field_description: &'a FieldDescription, + field_definition: &'a FieldDefinition, buf: &'a mut BufferImpl, } impl<'a, E: Encoder> ColumnWriterImpl<'a, E> { - fn new(field_description: &'a FieldDescription, buf: &'a mut BufferImpl) -> Self { + fn new(field_definition: &'a FieldDefinition, buf: &'a mut BufferImpl) -> Self { Self { - field_description, + field_definition, buf, } } - pub fn field_description(&self) -> &FieldDescription { - &self.field_description + pub fn field_definition(&self) -> &FieldDefinition { + &self.field_definition } /// Takes `self` so that only one value per column can be written @@ -294,11 +293,11 @@ mod tests { fn test_write_presence_column() { let mut section = Section::new( SectionType::TrackPoints, - vec![ - ("a".to_string(), FieldType::I64), - ("b".to_string(), FieldType::Bool), - ("c".to_string(), FieldType::String), - ], + Schema::with_fields(vec![ + FieldDefinition::new("a", DataType::I64), + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("c", DataType::String), + ]), ); let mut buf = Vec::new(); assert_matches!(section.write_presence_column(&mut buf), Ok(()) => { @@ -355,28 +354,11 @@ mod tests { fn test_multibyte_presence_column() { let mut section = Section::new( SectionType::TrackPoints, - vec![ - ("1".to_string(), FieldType::Bool), - ("2".to_string(), FieldType::Bool), - ("3".to_string(), FieldType::Bool), - ("4".to_string(), FieldType::Bool), - ("5".to_string(), FieldType::Bool), - ("6".to_string(), FieldType::Bool), - ("7".to_string(), FieldType::Bool), - ("8".to_string(), FieldType::Bool), - ("9".to_string(), FieldType::Bool), - ("10".to_string(), FieldType::Bool), - ("11".to_string(), FieldType::Bool), - ("12".to_string(), FieldType::Bool), - ("13".to_string(), FieldType::Bool), - ("14".to_string(), FieldType::Bool), - ("15".to_string(), FieldType::Bool), - ("16".to_string(), FieldType::Bool), - ("17".to_string(), FieldType::Bool), - ("18".to_string(), FieldType::Bool), - ("19".to_string(), FieldType::Bool), - ("20".to_string(), FieldType::Bool), - ], + Schema::with_fields( + (0..20) + .map(|i| FieldDefinition::new(i.to_string(), DataType::Bool)) + .collect(), + ), ); for i in 0..2 { @@ -406,7 +388,11 @@ mod tests { fn test_write_huge_presence_column() { let mut section = Section::new( SectionType::TrackPoints, - (0..80).map(|i| (i.to_string(), FieldType::Bool)).collect(), + Schema::with_fields( + (0..80) + .map(|i| FieldDefinition::new(i.to_string(), DataType::Bool)) + .collect(), + ), ); #[rustfmt::skip] @@ -458,13 +444,13 @@ mod tests { fn test_types_table() { let mut section = Section::new( SectionType::TrackPoints, - vec![ - ("m".to_string(), FieldType::I64), - ("k".to_string(), FieldType::Bool), - ("long name!".to_string(), FieldType::String), - ("i".to_string(), FieldType::I64), - ("f".to_string(), FieldType::F64), - ], + Schema::with_fields(vec![ + FieldDefinition::new("m", DataType::I64), + FieldDefinition::new("k", DataType::Bool), + FieldDefinition::new("long name!", DataType::String), + FieldDefinition::new("i", DataType::I64), + FieldDefinition::new("f", DataType::F64), + ]), ); let mut rowbuilder = section.open_row_builder(); @@ -552,26 +538,26 @@ mod tests { let mut section = Section::new( SectionType::TrackPoints, - vec![ - ("a".to_string(), FieldType::I64), - ("b".to_string(), FieldType::Bool), - ("c".to_string(), FieldType::String), - ("d".to_string(), FieldType::F64), - ], + Schema::with_fields(vec![ + FieldDefinition::new("a", DataType::I64), + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("c", DataType::String), + FieldDefinition::new("d", DataType::F64), + ]), ); - let mapping = section.fields().to_vec(); + let fields = section.schema().fields().to_vec(); for entry in v { let mut rowbuilder = section.open_row_builder(); - for field_desc in mapping.iter() { + for field_def in fields.iter() { assert_matches!(rowbuilder.next_column_writer(), Some(cw) => { match cw { ColumnWriter::I64ColumnWriter(cwi) => { assert!(cwi.write( entry - .get(field_desc.name()) + .get(field_def.name()) .map(|v| match v { V::I64(v) => Some(v), _ => None, @@ -582,7 +568,7 @@ mod tests { ColumnWriter::BoolColumnWriter(cwi) => { assert!(cwi.write( entry - .get(field_desc.name()) + .get(field_def.name()) .map(|v| match v { V::Bool(v) => Some(v), _ => None, @@ -593,7 +579,7 @@ mod tests { ColumnWriter::StringColumnWriter(cwi) => { assert!(cwi.write( entry - .get(field_desc.name()) + .get(field_def.name()) .map(|v| match v { V::String(v) => Some(v), _ => None, @@ -604,7 +590,7 @@ mod tests { ColumnWriter::F64ColumnWriter(cwi) => { assert!(cwi.write( entry - .get(field_desc.name()) + .get(field_def.name()) .map(|v| match v { V::F64(v) => Some(v), _ => None, diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index fa71ca4..725105c 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -40,7 +40,8 @@ pub fn write_track( #[cfg(test)] mod tests { use super::*; - use crate::types::{FieldType, SectionType, TrackType}; + use crate::schema::*; + use crate::types::{SectionType, TrackType}; use crate::write::section::ColumnWriter; use assert_matches::assert_matches; use std::collections::HashMap; @@ -93,11 +94,11 @@ mod tests { fn test_write_a_track() { let mut section1 = Section::new( SectionType::CoursePoints, - vec![ - ("m".to_string(), FieldType::I64), - ("k".to_string(), FieldType::Bool), - ("j".to_string(), FieldType::String), - ], + Schema::with_fields(vec![ + FieldDefinition::new("m", DataType::I64), + FieldDefinition::new("k", DataType::Bool), + FieldDefinition::new("j", DataType::String), + ]), ); for _ in 0..5 { @@ -143,25 +144,25 @@ mod tests { let mut section2 = Section::new( SectionType::TrackPoints, - vec![ - ("a".to_string(), FieldType::I64), - ("b".to_string(), FieldType::Bool), - ("c".to_string(), FieldType::String), - ], + Schema::with_fields(vec![ + FieldDefinition::new("a", DataType::I64), + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("c", DataType::String), + ]), ); - let mapping = section2.fields().to_vec(); + let fields = section2.schema().fields().to_vec(); for entry in v { let mut rowbuilder = section2.open_row_builder(); - for field_desc in mapping.iter() { + for field_def in fields.iter() { assert_matches!(rowbuilder.next_column_writer(), Some(cw) => { match cw { ColumnWriter::I64ColumnWriter(cwi) => { assert!(cwi.write( entry - .get(field_desc.name()) + .get(field_def.name()) .map(|v| match v { V::I64(v) => Some(v), _ => None, @@ -172,7 +173,7 @@ mod tests { ColumnWriter::BoolColumnWriter(cwi) => { assert!(cwi.write( entry - .get(field_desc.name()) + .get(field_def.name()) .map(|v| match v { V::Bool(v) => Some(v), _ => None, @@ -183,7 +184,7 @@ mod tests { ColumnWriter::StringColumnWriter(cwi) => { assert!(cwi.write( entry - .get(field_desc.name()) + .get(field_def.name()) .map(|v| match v { V::String(v) => Some(v), _ => None, From 62062f0ba2e1c01a62c6189b4b0bbfe1004cc620 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 12:17:58 -0700 Subject: [PATCH 041/113] finish moving Types Table -> Schema in read code --- tracklib2/src/read/data_table.rs | 38 +++++++-------- tracklib2/src/read/inspect.rs | 21 +++++---- tracklib2/src/read/mod.rs | 2 +- .../src/read/{types_table.rs => schema.rs} | 40 ++++++++-------- tracklib2/src/read/section_reader.rs | 46 ++++++++++--------- 5 files changed, 74 insertions(+), 73 deletions(-) rename tracklib2/src/read/{types_table.rs => schema.rs} (75%) diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index 85a7bfb..b178eee 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -1,5 +1,5 @@ use super::crc::CRC; -use super::types_table::{parse_types_table, TypesTableEntry}; +use super::schema::{parse_schema, SchemaEntry}; use crate::error::TracklibError; use crate::types::SectionType; use nom::{number::complete::le_u8, IResult}; @@ -12,7 +12,7 @@ pub(crate) struct DataTableEntry { offset: usize, size: usize, rows: usize, - types: Vec, + schema_entries: Vec, } impl DataTableEntry { @@ -32,8 +32,8 @@ impl DataTableEntry { self.rows } - pub(crate) fn types(&self) -> &[TypesTableEntry] { - self.types.as_slice() + pub(crate) fn schema_entries(&self) -> &[SchemaEntry] { + self.schema_entries.as_slice() } } @@ -44,7 +44,7 @@ fn parse_data_table_entry( let (input, type_tag) = le_u8(input)?; let (input, rows) = leb128_u64(input)?; let (input, size) = leb128_u64(input)?; - let (input, types) = parse_types_table(input)?; + let (input, schema_entries) = parse_schema(input)?; let section_type = match type_tag { 0x00 => SectionType::TrackPoints, @@ -63,7 +63,7 @@ fn parse_data_table_entry( offset, size: usize::try_from(size).expect("usize != u64"), rows: usize::try_from(rows).expect("usize != u64"), - types, + schema_entries, }, )) } @@ -107,7 +107,7 @@ mod tests { 0x00, // leb128 section point count 0x00, // leb128 section data size - // Types Table + // Schema 0x03, // field count 0x00, // first field type = I64 0x01, // name length @@ -128,7 +128,7 @@ mod tests { 0x00, // leb128 section point count 0x00, // leb128 section data size - // Types Table + // Schema 0x03, // field count 0x00, // first field type = I64 0x04, // name length @@ -164,10 +164,10 @@ mod tests { offset: 0, size: 0, rows: 0, - types: vec![ - TypesTableEntry::new_for_tests("a", DataType::I64, 0, 0), - TypesTableEntry::new_for_tests("b", DataType::Bool, 0, 0), - TypesTableEntry::new_for_tests("c", DataType::String, 0, 0) + schema_entries: vec![ + SchemaEntry::new_for_tests("a", DataType::I64, 0, 0), + SchemaEntry::new_for_tests("b", DataType::Bool, 0, 0), + SchemaEntry::new_for_tests("c", DataType::String, 0, 0) ] }, DataTableEntry { @@ -175,10 +175,10 @@ mod tests { offset: 0, size: 0, rows: 0, - types: vec![ - TypesTableEntry::new_for_tests("Ride", DataType::I64, 0, 0), - TypesTableEntry::new_for_tests("with", DataType::Bool, 0, 0), - TypesTableEntry::new_for_tests("GPS", DataType::String, 0, 0) + schema_entries: vec![ + SchemaEntry::new_for_tests("Ride", DataType::I64, 0, 0), + SchemaEntry::new_for_tests("with", DataType::Bool, 0, 0), + SchemaEntry::new_for_tests("GPS", DataType::String, 0, 0) ] } ] @@ -196,7 +196,7 @@ mod tests { 0x00, // leb128 section point count 0x00, // leb128 section data size - // Types Table + // Schema 0x01, // field count 0x00, // first field type = I64 0x05, // name length @@ -219,8 +219,8 @@ mod tests { offset: 0, size: 0, rows: 0, - types: vec![ - TypesTableEntry::new_for_tests("a�b", DataType::I64, 0, 0), + schema_entries: vec![ + SchemaEntry::new_for_tests("a�b", DataType::I64, 0, 0), ] } ] diff --git a/tracklib2/src/read/inspect.rs b/tracklib2/src/read/inspect.rs index 8fd6ab7..a2dcc24 100644 --- a/tracklib2/src/read/inspect.rs +++ b/tracklib2/src/read/inspect.rs @@ -98,7 +98,8 @@ fn try_format_data_table( for (i, data_table_entry) in data_table.iter().enumerate() { const CRC_BYTES: usize = 4; - let presence_column_bytes_required = (data_table_entry.types().len() + 7) / 8; + let presence_column_bytes_required = + (data_table_entry.schema_entries().len() + 7) / 8; let presence_column_size = presence_column_bytes_required * data_table_entry.rows() + CRC_BYTES; @@ -153,11 +154,11 @@ fn try_format_data_table( ), ])); - for types_table_entry in data_table_entry.types() { + for schema_entry in data_table_entry.schema_entries() { table.add_row(Row::new(vec![ TableCell::new_with_alignment("Column", 1, Alignment::Left), TableCell::new_with_alignment( - bold(types_table_entry.field_definition().name()), + bold(schema_entry.field_definition().name()), 2, Alignment::Center, ), @@ -167,7 +168,7 @@ fn try_format_data_table( TableCell::new(""), TableCell::new_with_alignment("Type", 1, Alignment::Left), TableCell::new_with_alignment( - format!("{:?}", types_table_entry.field_definition().data_type()), + format!("{:?}", schema_entry.field_definition().data_type()), 1, Alignment::Right, ), @@ -177,7 +178,7 @@ fn try_format_data_table( TableCell::new(""), TableCell::new_with_alignment("Size", 1, Alignment::Left), TableCell::new_with_alignment( - format!("{:#04X}", types_table_entry.size()), + format!("{:#04X}", schema_entry.size()), 1, Alignment::Right, ), @@ -191,7 +192,7 @@ fn try_format_data_table( Alignment::Right, ), TableCell::new_with_alignment( - format!("{:#04X}", types_table_entry.offset()), + format!("{:#04X}", schema_entry.offset()), 1, Alignment::Right, ), @@ -207,7 +208,7 @@ fn try_format_data_table( TableCell::new_with_alignment( format!( "{:#04X}", - types_table_entry.offset() + schema_entry.offset() + presence_column_size + data_table_entry.offset() + data_start_offset @@ -264,16 +265,16 @@ fn try_format_section(data_start: &[u8], entry_num: usize, entry: &DataTableEntr Ok(mut section_reader) => { table.add_row(Row::new(vec![TableCell::new_with_alignment( bold(&format!("Data {entry_num}")), - entry.types().len() + 1, + entry.schema_entries().len() + 1, Alignment::Center, )])); table.add_row(Row::new( [TableCell::new_with_alignment("#", 1, Alignment::Center)] .into_iter() - .chain(entry.types().iter().map(|types_table_entry| { + .chain(entry.schema_entries().iter().map(|schema_entry| { TableCell::new_with_alignment( - types_table_entry.field_definition().name(), + schema_entry.field_definition().name(), 1, Alignment::Center, ) diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 0b0c7f8..571271a 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -7,6 +7,6 @@ mod header; pub mod inspect; mod metadata; mod presence_column; +mod schema; mod section_reader; pub mod track; -mod types_table; diff --git a/tracklib2/src/read/types_table.rs b/tracklib2/src/read/schema.rs similarity index 75% rename from tracklib2/src/read/types_table.rs rename to tracklib2/src/read/schema.rs index 867e303..ca0ff6f 100644 --- a/tracklib2/src/read/types_table.rs +++ b/tracklib2/src/read/schema.rs @@ -6,14 +6,14 @@ use std::convert::TryFrom; #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] -pub(crate) struct TypesTableEntry { +pub(crate) struct SchemaEntry { field_definition: FieldDefinition, size: usize, offset: usize, } #[cfg(test)] -impl TypesTableEntry { +impl SchemaEntry { pub(crate) fn new_for_tests( name: &str, data_type: DataType, @@ -28,7 +28,7 @@ impl TypesTableEntry { } } -impl TypesTableEntry { +impl SchemaEntry { pub(crate) fn field_definition(&self) -> &FieldDefinition { &self.field_definition } @@ -42,9 +42,9 @@ impl TypesTableEntry { } } -fn parse_types_table_entry<'a>( +fn parse_schema_entry<'a>( offset: usize, -) -> impl Fn(&[u8]) -> IResult<&[u8], TypesTableEntry, TracklibError> { +) -> impl Fn(&[u8]) -> IResult<&[u8], SchemaEntry, TracklibError> { move |input: &[u8]| { let (input, type_tag) = le_u8(input)?; let (input, field_name) = length_data(le_u8)(input)?; @@ -66,7 +66,7 @@ fn parse_types_table_entry<'a>( Ok(( input, - TypesTableEntry { + SchemaEntry { field_definition: FieldDefinition::new(name, data_type), size: usize::try_from(data_size).expect("usize != u64"), offset, @@ -75,14 +75,12 @@ fn parse_types_table_entry<'a>( } } -pub(crate) fn parse_types_table( - input: &[u8], -) -> IResult<&[u8], Vec, TracklibError> { +pub(crate) fn parse_schema(input: &[u8]) -> IResult<&[u8], Vec, TracklibError> { let (mut input, entry_count) = le_u8(input)?; let mut entries = Vec::with_capacity(usize::from(entry_count)); let mut offset = 0; for _ in 0..entry_count { - let (rest, entry) = parse_types_table_entry(offset)(input)?; + let (rest, entry) = parse_schema_entry(offset)(input)?; input = rest; offset += entry.size; entries.push(entry); @@ -96,7 +94,7 @@ mod tests { use assert_matches::assert_matches; #[test] - fn test_test_parse_types_table() { + fn test_test_parse_schema() { #[rustfmt::skip] let buf = &[0x04, // entry count = 4 0x00, // first entry type: i64 = 0 @@ -124,37 +122,37 @@ mod tests { 0x01, // name len = 1 b'i', // name = "i" 0x02]; // data size = 2 - assert_matches!(parse_types_table(buf), Ok((&[], entries)) => { - assert_eq!(entries, vec![TypesTableEntry::new_for_tests("m", DataType::I64, 2, 0), - TypesTableEntry::new_for_tests("k", DataType::Bool, 1, 2), - TypesTableEntry::new_for_tests("long name!", DataType::String, 7, 3), - TypesTableEntry::new_for_tests("i", DataType::F64, 2, 10)]); + assert_matches!(parse_schema(buf), Ok((&[], entries)) => { + assert_eq!(entries, vec![SchemaEntry::new_for_tests("m", DataType::I64, 2, 0), + SchemaEntry::new_for_tests("k", DataType::Bool, 1, 2), + SchemaEntry::new_for_tests("long name!", DataType::String, 7, 3), + SchemaEntry::new_for_tests("i", DataType::F64, 2, 10)]); }); } #[test] - fn test_types_table_invalid_field_tag() { + fn test_schema_invalid_field_tag() { #[rustfmt::skip] let buf = &[0x01, // entry count 0xEF, // first entry type: invalid 0x01, // name len = 1 b'm', // name = "m" 0x02]; // data size = 2 - assert_matches!(parse_types_table(buf), Err(nom::Err::Error(TracklibError::ParseError{error_kind})) => { + assert_matches!(parse_schema(buf), Err(nom::Err::Error(TracklibError::ParseError{error_kind})) => { assert_eq!(error_kind, nom::error::ErrorKind::Tag); }); } #[test] - fn test_types_table_invalid_utf8() { + fn test_schema_invalid_utf8() { #[rustfmt::skip] let buf = &[0x01, // entry count 0x00, // first entry type: I64 0x01, // name len = 1 0xC0, // name: invalid utf-8 0x02]; // data size = 2 - assert_matches!(parse_types_table(buf), Ok((&[], entries)) => { - assert_eq!(entries, vec![TypesTableEntry::new_for_tests("�", DataType::I64, 2, 0)]) + assert_matches!(parse_schema(buf), Ok((&[], entries)) => { + assert_eq!(entries, vec![SchemaEntry::new_for_tests("�", DataType::I64, 2, 0)]) }); } } diff --git a/tracklib2/src/read/section_reader.rs b/tracklib2/src/read/section_reader.rs index 456a333..772eea8 100644 --- a/tracklib2/src/read/section_reader.rs +++ b/tracklib2/src/read/section_reader.rs @@ -35,31 +35,33 @@ pub struct SectionReader<'a> { impl<'a> SectionReader<'a> { pub(crate) fn new(input: &'a [u8], data_table_entry: &'a DataTableEntry) -> Result { - let (column_data, presence_column) = - parse_presence_column(data_table_entry.types().len(), data_table_entry.rows())(input)?; + let (column_data, presence_column) = parse_presence_column( + data_table_entry.schema_entries().len(), + data_table_entry.rows(), + )(input)?; let schema = Schema::with_fields( data_table_entry - .types() + .schema_entries() .iter() - .map(|types_table_entry| types_table_entry.field_definition().clone()) + .map(|schema_entry| schema_entry.field_definition().clone()) .collect(), ); let decoders = data_table_entry - .types() + .schema_entries() .iter() .enumerate() - .map(|(i, types_table_entry)| { - let column_data = &column_data[types_table_entry.offset() - ..types_table_entry.offset() + types_table_entry.size()]; + .map(|(i, schema_entry)| { + let column_data = &column_data + [schema_entry.offset()..schema_entry.offset() + schema_entry.size()]; let presence_column_view = presence_column .view(i) .ok_or_else(|| TracklibError::ParseIncompleteError { needed: nom::Needed::Unknown, })?; - let field_definition = types_table_entry.field_definition(); + let field_definition = schema_entry.field_definition(); let decoder = match field_definition.data_type() { DataType::I64 => ColumnDecoder::I64 { field_definition, @@ -189,7 +191,7 @@ mod tests { 0x00, // section type = track points 0x03, // leb128 section point count 0x26, // leb128 section data size - // Types Table + // Schema 0x04, // field count 0x00, // first field type = I64 0x01, // name length @@ -285,22 +287,22 @@ mod tests { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); assert_matches!(&values[0], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[0].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[0].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("a", DataType::I64)); assert_eq!(field_value, &Some(FieldValue::I64(1))); }); assert_matches!(&values[1], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[1].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); assert_eq!(field_value, &Some(FieldValue::Bool(false))); }); assert_matches!(&values[2], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[2].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[2].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("c", DataType::String)); assert_eq!(field_value, &Some(FieldValue::String("Ride".to_string()))); }); assert_matches!(&values[3], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[3].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[3].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); assert_eq!(field_value, &None); }); @@ -311,22 +313,22 @@ mod tests { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); assert_matches!(&values[0], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[0].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[0].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("a", DataType::I64)); assert_eq!(field_value, &Some(FieldValue::I64(2))); }); assert_matches!(&values[1], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[1].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); assert_eq!(field_value, &None); }); assert_matches!(&values[2], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[2].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[2].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("c", DataType::String)); assert_eq!(field_value, &Some(FieldValue::String("with".to_string()))); }); assert_matches!(&values[3], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[3].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[3].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); assert_eq!(field_value, &Some(FieldValue::F64(1.0))); }); @@ -337,22 +339,22 @@ mod tests { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); assert_matches!(&values[0], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[0].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[0].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("a", DataType::I64)); assert_eq!(field_value, &Some(FieldValue::I64(4))); }); assert_matches!(&values[1], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[1].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); assert_eq!(field_value, &Some(FieldValue::Bool(true))); }); assert_matches!(&values[2], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[2].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[2].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("c", DataType::String)); assert_eq!(field_value, &Some(FieldValue::String("GPS".to_string()))); }); assert_matches!(&values[3], Ok((field_definition, field_value)) => { - assert_eq!(*field_definition, data_table_entries[0].types()[3].field_definition()); + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[3].field_definition()); assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); assert_eq!(field_value, &Some(FieldValue::F64(2.5))); }); From 4c476d7c842460819ef4ee631c84ae71d0140d76 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 12:22:37 -0700 Subject: [PATCH 042/113] finish moving Types Table -> Schema in write code --- tracklib2/src/write/data_table.rs | 8 ++++---- tracklib2/src/write/section.rs | 6 +++--- tracklib2/src/write/track.rs | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index b5b4437..d6afe01 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -13,7 +13,7 @@ pub(crate) fn write_data_table(out: &mut W, sections: &[Section]) -> R crcwriter.write_all(§ion.type_tag().to_le_bytes())?; // 1 byte - section type leb128::write::unsigned(&mut crcwriter, u64::try_from(section.rows())?)?; // ? bytes - number of points in this section leb128::write::unsigned(&mut crcwriter, u64::try_from(section.data_size())?)?; // ? bytes - leb128 section size - section.write_types_table(&mut crcwriter)?; // ? bytes - types table + section.write_schema(&mut crcwriter)?; // ? bytes - schema } crcwriter.append_crc()?; // 2 bytes - crc @@ -68,7 +68,7 @@ mod tests { 0x00, // section type = track points 0x00, // leb128 section point count 0x10, // leb128 section data size - // Types Table + // Schema 0x03, // field count 0x00, // first field type = I64 0x01, // name length @@ -89,7 +89,7 @@ mod tests { 0x00, // leb128 section point count 0x10, // leb128 section data size - // Types Table + // Schema 0x03, // field count 0x00, // first field type = I64 0x04, // name length @@ -135,7 +135,7 @@ mod tests { 0x00, // section type = track points 0x00, // leb128 section point count 0x08, // leb128 section data size - // Types Table + // Schema 0x01, // field count 0x01, // first field type = F64 0x08, // name length diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 2815341..9a2d857 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -162,7 +162,7 @@ impl Section { } #[rustfmt::skip] - pub(crate) fn write_types_table(&self, out: &mut W) -> Result<()> { + pub(crate) fn write_schema(&self, out: &mut W) -> Result<()> { out.write_all(&u8::try_from(self.schema.fields().len())?.to_le_bytes())?; // 1 byte - number of entries for (i, field_def) in self.schema.fields().iter().enumerate() { @@ -441,7 +441,7 @@ mod tests { } #[test] - fn test_types_table() { + fn test_schema() { let mut section = Section::new( SectionType::TrackPoints, Schema::with_fields(vec![ @@ -472,7 +472,7 @@ mod tests { } let mut buf = Vec::new(); - assert_matches!(section.write_types_table(&mut buf), Ok(()) => { + assert_matches!(section.write_schema(&mut buf), Ok(()) => { #[rustfmt::skip] assert_eq!(buf, &[0x05, // entry count = 5 diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 725105c..84a220f 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -251,7 +251,7 @@ mod tests { 0x05, // leb128 point count 0x33, // leb128 data size - // Types Table for Section 1 + // Schema for Section 1 0x03, // field count 0x00, // first field type = I64 0x01, // name len @@ -271,7 +271,7 @@ mod tests { 0x03, // leb128 point count 0x26, // leb128 data size - // Types Table for Section 2 + // Schema for Section 2 0x03, // field count 0x00, // first field type = I64 0x01, // name length From ed32b8c5b079bc149eb9a4742e748f7cc76c6776 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 13:03:26 -0700 Subject: [PATCH 043/113] add a schema version to the file --- tracklib2/src/consts.rs | 1 + tracklib2/src/read/data_table.rs | 11 +++++++---- tracklib2/src/read/schema.rs | 13 +++++++++---- tracklib2/src/read/section_reader.rs | 5 +++-- tracklib2/src/write/data_table.rs | 10 ++++++---- tracklib2/src/write/section.rs | 5 ++++- tracklib2/src/write/track.rs | 6 ++++-- 7 files changed, 34 insertions(+), 17 deletions(-) diff --git a/tracklib2/src/consts.rs b/tracklib2/src/consts.rs index 61d0c7a..0161a8e 100644 --- a/tracklib2/src/consts.rs +++ b/tracklib2/src/consts.rs @@ -14,3 +14,4 @@ pub(crate) const RWTFMAGIC: [u8; 8] = [0x89, // non-ascii pub(crate) const RWTF_HEADER_SIZE: u16 = 24; pub(crate) const RWTF_FILE_VERSION: u8 = 0x01; pub(crate) const RWTF_CREATOR_VERSION: u8 = 0x00; +pub(crate) const SCHEMA_VERSION: u8 = 0x00; diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index b178eee..c11959c 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -108,6 +108,7 @@ mod tests { 0x00, // leb128 section data size // Schema + 0x00, // schema version 0x03, // field count 0x00, // first field type = I64 0x01, // name length @@ -129,6 +130,7 @@ mod tests { 0x00, // leb128 section data size // Schema + 0x00, // schema version 0x03, // field count 0x00, // first field type = I64 0x04, // name length @@ -152,8 +154,8 @@ mod tests { 0x00, // leb128 data size - 0x4E, // crc - 0x88]; + 0x8E, // crc + 0x77]; assert_matches!(parse_data_table(buf), Ok((&[], entries)) => { assert_eq!( @@ -197,6 +199,7 @@ mod tests { 0x00, // leb128 section data size // Schema + 0x00, // schema version 0x01, // field count 0x00, // first field type = I64 0x05, // name length @@ -207,8 +210,8 @@ mod tests { b'b', 0x00, // leb128 data size - 0xEB, // crc - 0xE9]; + 0x41, // crc + 0x43]; assert_matches!(parse_data_table(buf), Ok((&[], entries)) => { assert_eq!( diff --git a/tracklib2/src/read/schema.rs b/tracklib2/src/read/schema.rs index ca0ff6f..d1e1e2b 100644 --- a/tracklib2/src/read/schema.rs +++ b/tracklib2/src/read/schema.rs @@ -1,6 +1,7 @@ +use crate::consts::SCHEMA_VERSION; use crate::error::TracklibError; use crate::schema::*; -use nom::{multi::length_data, number::complete::le_u8, IResult}; +use nom::{bytes::complete::tag, multi::length_data, number::complete::le_u8, IResult}; use nom_leb128::leb128_u64; use std::convert::TryFrom; @@ -76,6 +77,7 @@ fn parse_schema_entry<'a>( } pub(crate) fn parse_schema(input: &[u8]) -> IResult<&[u8], Vec, TracklibError> { + let (input, _schema_version) = tag(&SCHEMA_VERSION.to_le_bytes())(input)?; let (mut input, entry_count) = le_u8(input)?; let mut entries = Vec::with_capacity(usize::from(entry_count)); let mut offset = 0; @@ -96,7 +98,8 @@ mod tests { #[test] fn test_test_parse_schema() { #[rustfmt::skip] - let buf = &[0x04, // entry count = 4 + let buf = &[0x00, // schema version + 0x04, // entry count = 4 0x00, // first entry type: i64 = 0 0x01, // name len = 1 b'm', // name = "m" @@ -133,7 +136,8 @@ mod tests { #[test] fn test_schema_invalid_field_tag() { #[rustfmt::skip] - let buf = &[0x01, // entry count + let buf = &[0x00, // schema version + 0x01, // entry count 0xEF, // first entry type: invalid 0x01, // name len = 1 b'm', // name = "m" @@ -146,7 +150,8 @@ mod tests { #[test] fn test_schema_invalid_utf8() { #[rustfmt::skip] - let buf = &[0x01, // entry count + let buf = &[0x00, // schema version + 0x01, // entry count 0x00, // first entry type: I64 0x01, // name len = 1 0xC0, // name: invalid utf-8 diff --git a/tracklib2/src/read/section_reader.rs b/tracklib2/src/read/section_reader.rs index 772eea8..f2cddd6 100644 --- a/tracklib2/src/read/section_reader.rs +++ b/tracklib2/src/read/section_reader.rs @@ -192,6 +192,7 @@ mod tests { 0x03, // leb128 section point count 0x26, // leb128 section data size // Schema + 0x00, // schema version 0x04, // field count 0x00, // first field type = I64 0x01, // name length @@ -210,8 +211,8 @@ mod tests { b'f', // name 0x0C, // leb128 data size - 0xE9, // crc - 0x0B]; + 0x81, // crc + 0x7D]; assert_matches!(parse_data_table(data_table_buf), Ok((&[], data_table_entries)) => { assert_eq!(data_table_entries.len(), 1); diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index d6afe01..812cae9 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -69,6 +69,7 @@ mod tests { 0x00, // leb128 section point count 0x10, // leb128 section data size // Schema + 0x00, // schema version 0x03, // field count 0x00, // first field type = I64 0x01, // name length @@ -90,6 +91,7 @@ mod tests { 0x10, // leb128 section data size // Schema + 0x00, // schema version 0x03, // field count 0x00, // first field type = I64 0x04, // name length @@ -112,9 +114,8 @@ mod tests { b'S', // name 0x04, // leb128 data size - - 0xFB, // crc - 0xF8]); + 0x20, // crc + 0x5D]); }); } @@ -136,6 +137,7 @@ mod tests { 0x00, // leb128 section point count 0x08, // leb128 section data size // Schema + 0x00, // schema version 0x01, // field count 0x01, // first field type = F64 0x08, // name length @@ -149,7 +151,7 @@ mod tests { b'Y', 0x04, // leb128 data size - 0x7D, // crc + 0x35, // crc 0x13]); }); } diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 9a2d857..e91d442 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -1,5 +1,6 @@ use super::crcwriter::CrcWriter; use super::encoders::*; +use crate::consts::SCHEMA_VERSION; use crate::error::Result; use crate::schema::*; use crate::types::SectionType; @@ -163,6 +164,7 @@ impl Section { #[rustfmt::skip] pub(crate) fn write_schema(&self, out: &mut W) -> Result<()> { + out.write_all(&SCHEMA_VERSION.to_le_bytes())?; // 1 byte - schema version out.write_all(&u8::try_from(self.schema.fields().len())?.to_le_bytes())?; // 1 byte - number of entries for (i, field_def) in self.schema.fields().iter().enumerate() { @@ -475,7 +477,8 @@ mod tests { assert_matches!(section.write_schema(&mut buf), Ok(()) => { #[rustfmt::skip] assert_eq!(buf, - &[0x05, // entry count = 5 + &[0x00, // schema version + 0x05, // entry count = 5 0x00, // first entry type: i64 = 0 0x01, // name len = 1 b'm', // name = "m" diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 84a220f..7123dc2 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -252,6 +252,7 @@ mod tests { 0x33, // leb128 data size // Schema for Section 1 + 0x00, // schema version 0x03, // field count 0x00, // first field type = I64 0x01, // name len @@ -272,6 +273,7 @@ mod tests { 0x26, // leb128 data size // Schema for Section 2 + 0x00, // schema version 0x03, // field count 0x00, // first field type = I64 0x01, // name length @@ -287,8 +289,8 @@ mod tests { 0x12, // leb128 data size // Data Table CRC - 0x49, - 0xEC, + 0xC9, + 0xAF, // Data Section 1 From 9b2ad80cff56fbb3001f01a93a52816b65b04dc5 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 15:10:37 -0700 Subject: [PATCH 044/113] replace SectionType (track/course points) with SectionEncoding We're going to unify the two types of sections (Track Points and Course Points) so we don't need to store that distinction in the file anymore. Meanwhile, we do plan to have encrypted sections, so this will allow for it when the time comes. --- tracklib2/src/read/data_table.rs | 31 ++++++++++++++-------------- tracklib2/src/read/inspect.rs | 4 ++-- tracklib2/src/read/section_reader.rs | 12 +++++------ tracklib2/src/types.rs | 6 +++--- tracklib2/src/write/data_table.rs | 29 +++++++++++++++++--------- tracklib2/src/write/section.rs | 31 ++++++++++------------------ tracklib2/src/write/track.rs | 14 ++++++------- 7 files changed, 63 insertions(+), 64 deletions(-) diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index c11959c..63295ca 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -1,14 +1,14 @@ use super::crc::CRC; use super::schema::{parse_schema, SchemaEntry}; use crate::error::TracklibError; -use crate::types::SectionType; +use crate::types::SectionEncoding; use nom::{number::complete::le_u8, IResult}; use nom_leb128::leb128_u64; use std::convert::TryFrom; #[cfg_attr(test, derive(Debug, PartialEq))] pub(crate) struct DataTableEntry { - section_type: SectionType, + section_encoding: SectionEncoding, offset: usize, size: usize, rows: usize, @@ -16,8 +16,8 @@ pub(crate) struct DataTableEntry { } impl DataTableEntry { - pub(crate) fn section_type(&self) -> &SectionType { - &self.section_type + pub(crate) fn section_encoding(&self) -> &SectionEncoding { + &self.section_encoding } pub(crate) fn offset(&self) -> usize { @@ -46,9 +46,8 @@ fn parse_data_table_entry( let (input, size) = leb128_u64(input)?; let (input, schema_entries) = parse_schema(input)?; - let section_type = match type_tag { - 0x00 => SectionType::TrackPoints, - 0x01 => SectionType::CoursePoints, + let section_encoding = match type_tag { + 0x00 => SectionEncoding::Standard, _ => { return Err(nom::Err::Error(TracklibError::ParseError { error_kind: nom::error::ErrorKind::Tag, @@ -59,7 +58,7 @@ fn parse_data_table_entry( Ok(( input, DataTableEntry { - section_type, + section_encoding, offset, size: usize::try_from(size).expect("usize != u64"), rows: usize::try_from(rows).expect("usize != u64"), @@ -103,7 +102,7 @@ mod tests { let buf = &[0x02, // number of sections // Section 1 - 0x00, // section type = track points + 0x00, // section encoding = standard 0x00, // leb128 section point count 0x00, // leb128 section data size @@ -125,7 +124,7 @@ mod tests { // Section 2 - 0x01, // section type = course points + 0x00, // section encoding = standard 0x00, // leb128 section point count 0x00, // leb128 section data size @@ -154,15 +153,15 @@ mod tests { 0x00, // leb128 data size - 0x8E, // crc - 0x77]; + 0xDA, // crc + 0x8E]; assert_matches!(parse_data_table(buf), Ok((&[], entries)) => { assert_eq!( entries, vec![ DataTableEntry { - section_type: SectionType::TrackPoints, + section_encoding: SectionEncoding::Standard, offset: 0, size: 0, rows: 0, @@ -173,7 +172,7 @@ mod tests { ] }, DataTableEntry { - section_type: SectionType::CoursePoints, + section_encoding: SectionEncoding::Standard, offset: 0, size: 0, rows: 0, @@ -194,7 +193,7 @@ mod tests { let buf = &[0x01, // number of sections // Section 1 - 0x00, // section type = track points + 0x00, // section encoding = standard 0x00, // leb128 section point count 0x00, // leb128 section data size @@ -218,7 +217,7 @@ mod tests { entries, vec![ DataTableEntry { - section_type: SectionType::TrackPoints, + section_encoding: SectionEncoding::Standard, offset: 0, size: 0, rows: 0, diff --git a/tracklib2/src/read/inspect.rs b/tracklib2/src/read/inspect.rs index a2dcc24..eeee997 100644 --- a/tracklib2/src/read/inspect.rs +++ b/tracklib2/src/read/inspect.rs @@ -110,9 +110,9 @@ fn try_format_data_table( Alignment::Center, )])); table.add_row(Row::new(vec![ - TableCell::new("Section Type"), + TableCell::new("Encoding"), TableCell::new_with_alignment( - format!("{:?}", data_table_entry.section_type()), + format!("{:?}", data_table_entry.section_encoding()), 2, Alignment::Right, ), diff --git a/tracklib2/src/read/section_reader.rs b/tracklib2/src/read/section_reader.rs index f2cddd6..8d87b40 100644 --- a/tracklib2/src/read/section_reader.rs +++ b/tracklib2/src/read/section_reader.rs @@ -3,7 +3,7 @@ use super::decoders::*; use super::presence_column::parse_presence_column; use crate::error::{Result, TracklibError}; use crate::schema::*; -use crate::types::{FieldValue, SectionType}; +use crate::types::{FieldValue, SectionEncoding}; #[cfg_attr(test, derive(Debug))] enum ColumnDecoder<'a> { @@ -28,7 +28,7 @@ enum ColumnDecoder<'a> { #[cfg_attr(test, derive(Debug))] pub struct SectionReader<'a> { decoders: Vec>, - section_type: SectionType, + section_encoding: SectionEncoding, schema: Schema, rows: usize, } @@ -86,14 +86,14 @@ impl<'a> SectionReader<'a> { Ok(Self { schema, - section_type: data_table_entry.section_type().clone(), + section_encoding: data_table_entry.section_encoding().clone(), decoders, rows: data_table_entry.rows(), }) } - pub fn section_type(&self) -> &SectionType { - &self.section_type + pub fn section_encoding(&self) -> &SectionEncoding { + &self.section_encoding } pub fn rows(&self) -> usize { @@ -188,7 +188,7 @@ mod tests { let data_table_buf = &[0x01, // number of sections // Section 1 - 0x00, // section type = track points + 0x00, // section encoding = standard 0x03, // leb128 section point count 0x26, // leb128 section data size // Schema diff --git a/tracklib2/src/types.rs b/tracklib2/src/types.rs index 3af3ad7..9fc5554 100644 --- a/tracklib2/src/types.rs +++ b/tracklib2/src/types.rs @@ -24,7 +24,7 @@ pub enum MetadataEntry { #[derive(Clone, Debug)] #[cfg_attr(test, derive(PartialEq))] -pub enum SectionType { - TrackPoints, - CoursePoints, +pub enum SectionEncoding { + Standard, + // Encrypted, } diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index 812cae9..2d4ed63 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -1,16 +1,25 @@ use super::crcwriter::CrcWriter; use super::section::Section; use crate::error::Result; +use crate::types::SectionEncoding; use std::convert::TryFrom; use std::io::Write; +impl SectionEncoding { + fn type_tag(&self) -> u8 { + match self { + Self::Standard => 0x00, + } + } +} + #[rustfmt::skip] pub(crate) fn write_data_table(out: &mut W, sections: &[Section]) -> Result<()> { let mut crcwriter = CrcWriter::new16(out); crcwriter.write_all(&u8::try_from(sections.len())?.to_le_bytes())?; // 1 byte - number of sections for section in sections.iter() { - crcwriter.write_all(§ion.type_tag().to_le_bytes())?; // 1 byte - section type + crcwriter.write_all(§ion.encoding().type_tag().to_le_bytes())?; // 1 byte - section encoding leb128::write::unsigned(&mut crcwriter, u64::try_from(section.rows())?)?; // ? bytes - number of points in this section leb128::write::unsigned(&mut crcwriter, u64::try_from(section.data_size())?)?; // ? bytes - leb128 section size section.write_schema(&mut crcwriter)?; // ? bytes - schema @@ -24,7 +33,7 @@ pub(crate) fn write_data_table(out: &mut W, sections: &[Section]) -> R mod tests { use super::*; use crate::schema::*; - use crate::types::SectionType; + use crate::types::SectionEncoding; use assert_matches::assert_matches; #[test] @@ -41,7 +50,7 @@ mod tests { #[test] fn test_data_table() { let section1 = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields(vec![ FieldDefinition::new("a", DataType::I64), FieldDefinition::new("b", DataType::Bool), @@ -50,7 +59,7 @@ mod tests { ); let section2 = Section::new( - SectionType::CoursePoints, + SectionEncoding::Standard, Schema::with_fields(vec![ FieldDefinition::new("Ride", DataType::I64), FieldDefinition::new("with", DataType::Bool), @@ -65,7 +74,7 @@ mod tests { &[0x02, // number of sections // Section 1 - 0x00, // section type = track points + 0x00, // section encoding = standard 0x00, // leb128 section point count 0x10, // leb128 section data size // Schema @@ -86,7 +95,7 @@ mod tests { // Section 2 - 0x01, // section type = course points + 0x00, // section encoding = standard 0x00, // leb128 section point count 0x10, // leb128 section data size @@ -114,15 +123,15 @@ mod tests { b'S', // name 0x04, // leb128 data size - 0x20, // crc - 0x5D]); + 0x74, // crc + 0xA4]); }); } #[test] fn test_data_table_with_multibyte_character() { let section = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields(vec![FieldDefinition::new("I ♥ NY", DataType::F64)]), ); @@ -133,7 +142,7 @@ mod tests { &[0x01, // number of sections // Section 1 - 0x00, // section type = track points + 0x00, // section encoding = standard 0x00, // leb128 section point count 0x08, // leb128 section data size // Schema diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index e91d442..0c0bbca 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -3,7 +3,7 @@ use super::encoders::*; use crate::consts::SCHEMA_VERSION; use crate::error::Result; use crate::schema::*; -use crate::types::SectionType; +use crate::types::SectionEncoding; use std::convert::TryFrom; use std::io::{self, Write}; @@ -18,15 +18,6 @@ impl DataType { } } -impl SectionType { - fn type_tag(&self) -> u8 { - match self { - Self::TrackPoints => 0x00, - Self::CoursePoints => 0x01, - } - } -} - #[derive(Default, Debug)] struct BufferImpl { buf: Vec, @@ -77,7 +68,7 @@ impl Buffer { } pub struct Section { - section_type: SectionType, + section_encoding: SectionEncoding, rows_written: usize, schema: Schema, column_data: Vec, @@ -85,7 +76,7 @@ pub struct Section { impl Section { // TODO: provide a size_hint param to size buffer Vecs (at least presence) - pub fn new(section_type: SectionType, schema: Schema) -> Self { + pub fn new(section_encoding: SectionEncoding, schema: Schema) -> Self { let column_data = schema .fields() .iter() @@ -93,7 +84,7 @@ impl Section { .collect(); Self { - section_type, + section_encoding, rows_written: 0, schema, column_data, @@ -110,8 +101,8 @@ impl Section { RowBuilder::new(&self.schema, &mut self.column_data) } - pub(crate) fn type_tag(&self) -> u8 { - self.section_type.type_tag() + pub(crate) fn encoding(&self) -> &SectionEncoding { + &self.section_encoding } pub(crate) fn rows(&self) -> usize { @@ -294,7 +285,7 @@ mod tests { #[test] fn test_write_presence_column() { let mut section = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields(vec![ FieldDefinition::new("a", DataType::I64), FieldDefinition::new("b", DataType::Bool), @@ -355,7 +346,7 @@ mod tests { #[test] fn test_multibyte_presence_column() { let mut section = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields( (0..20) .map(|i| FieldDefinition::new(i.to_string(), DataType::Bool)) @@ -389,7 +380,7 @@ mod tests { #[test] fn test_write_huge_presence_column() { let mut section = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields( (0..80) .map(|i| FieldDefinition::new(i.to_string(), DataType::Bool)) @@ -445,7 +436,7 @@ mod tests { #[test] fn test_schema() { let mut section = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields(vec![ FieldDefinition::new("m", DataType::I64), FieldDefinition::new("k", DataType::Bool), @@ -540,7 +531,7 @@ mod tests { v.push(h); let mut section = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields(vec![ FieldDefinition::new("a", DataType::I64), FieldDefinition::new("b", DataType::Bool), diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 7123dc2..524e641 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -41,7 +41,7 @@ pub fn write_track( mod tests { use super::*; use crate::schema::*; - use crate::types::{SectionType, TrackType}; + use crate::types::{SectionEncoding, TrackType}; use crate::write::section::ColumnWriter; use assert_matches::assert_matches; use std::collections::HashMap; @@ -93,7 +93,7 @@ mod tests { #[test] fn test_write_a_track() { let mut section1 = Section::new( - SectionType::CoursePoints, + SectionEncoding::Standard, Schema::with_fields(vec![ FieldDefinition::new("m", DataType::I64), FieldDefinition::new("k", DataType::Bool), @@ -143,7 +143,7 @@ mod tests { v.push(h); let mut section2 = Section::new( - SectionType::TrackPoints, + SectionEncoding::Standard, Schema::with_fields(vec![ FieldDefinition::new("a", DataType::I64), FieldDefinition::new("b", DataType::Bool), @@ -247,7 +247,7 @@ mod tests { 0x02, // two sections // Data Table Section 1 - 0x01, // type of section = course points + 0x00, // section encoding = standard 0x05, // leb128 point count 0x33, // leb128 data size @@ -268,7 +268,7 @@ mod tests { 0x18, // leb128 data size // Data Table Section 2 - 0x00, // type of section = track points + 0x00, // section encoding = standard 0x03, // leb128 point count 0x26, // leb128 data size @@ -289,8 +289,8 @@ mod tests { 0x12, // leb128 data size // Data Table CRC - 0xC9, - 0xAF, + 0xC8, + 0x42, // Data Section 1 From efcb0096d8ca050fb24a0119332df7c38557849f Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 15:35:26 -0700 Subject: [PATCH 045/113] make header fields private, provider getters --- tracklib2/src/read/header.rs | 28 +++++++++++++++++++++++----- tracklib2/src/read/inspect.rs | 12 ++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/tracklib2/src/read/header.rs b/tracklib2/src/read/header.rs index 702e365..01ba0f5 100644 --- a/tracklib2/src/read/header.rs +++ b/tracklib2/src/read/header.rs @@ -9,11 +9,29 @@ use nom::{ #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] -pub struct Header { - pub(crate) file_version: u8, - pub(crate) creator_version: u8, - pub(crate) metadata_offset: u16, - pub(crate) data_offset: u16, +pub(crate) struct Header { + file_version: u8, + creator_version: u8, + metadata_offset: u16, + data_offset: u16, +} + +impl Header { + pub(crate) fn file_version(&self) -> u8 { + self.file_version + } + + pub(crate) fn creator_version(&self) -> u8 { + self.creator_version + } + + pub(crate) fn metadata_offset(&self) -> u16 { + self.metadata_offset + } + + pub(crate) fn data_offset(&self) -> u16 { + self.data_offset + } } pub(crate) fn parse_header(input: &[u8]) -> IResult<&[u8], Header, TracklibError> { diff --git a/tracklib2/src/read/inspect.rs b/tracklib2/src/read/inspect.rs index eeee997..1b49d7d 100644 --- a/tracklib2/src/read/inspect.rs +++ b/tracklib2/src/read/inspect.rs @@ -30,12 +30,12 @@ fn format_header(header: &Header) -> String { )])); table.add_row(Row::new(vec![ TableCell::new("File Version"), - TableCell::new_with_alignment(format!("{:#04X}", header.file_version), 1, Alignment::Right), + TableCell::new_with_alignment(format!("{:#04X}", header.file_version()), 1, Alignment::Right), ])); table.add_row(Row::new(vec![ TableCell::new("Creator Version"), TableCell::new_with_alignment( - format!("{:#04X}", header.creator_version), + format!("{:#04X}", header.creator_version()), 1, Alignment::Right, ), @@ -43,14 +43,14 @@ fn format_header(header: &Header) -> String { table.add_row(Row::new(vec![ TableCell::new("Metadata Offset"), TableCell::new_with_alignment( - format!("{:#04X}", header.metadata_offset), + format!("{:#04X}", header.metadata_offset()), 1, Alignment::Right, ), ])); table.add_row(Row::new(vec![ TableCell::new("Data Offset"), - TableCell::new_with_alignment(format!("{:#04X}", header.data_offset), 1, Alignment::Right), + TableCell::new_with_alignment(format!("{:#04X}", header.data_offset()), 1, Alignment::Right), ])); table.render() @@ -318,13 +318,13 @@ pub fn inspect(input: &[u8]) -> Result { // Metadata out.push_str(&try_format_metadata( - &input[usize::from(header.metadata_offset)..], + &input[usize::from(header.metadata_offset())..], )); out.push_str("\n\n"); // Data Table let (maybe_data_table, data_table_out) = - try_format_data_table(input, usize::from(header.data_offset)); + try_format_data_table(input, usize::from(header.data_offset())); out.push_str(&data_table_out); // Data From 5a9ba8400a63e74925be9339873217dc4b3f6e9e Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 17:00:05 -0700 Subject: [PATCH 046/113] crate::read::track --- tracklib2/src/read/track.rs | 370 ++++++++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 tracklib2/src/read/track.rs diff --git a/tracklib2/src/read/track.rs b/tracklib2/src/read/track.rs new file mode 100644 index 0000000..92a5bb2 --- /dev/null +++ b/tracklib2/src/read/track.rs @@ -0,0 +1,370 @@ +use super::data_table::{parse_data_table, DataTableEntry}; +use super::header::{parse_header, Header}; +use super::metadata::parse_metadata; +use super::section_reader::SectionReader; +use crate::error::Result; +use crate::types::MetadataEntry; + +#[cfg_attr(test, derive(Debug))] +pub struct TrackReader<'a> { + header: Header, + metadata_entries: Vec, + data_table: Vec, + data_start: &'a [u8], +} + +impl<'a> TrackReader<'a> { + pub fn from_bytes(data: &'a [u8]) -> Result { + let (_, header) = parse_header(data)?; + let (_, metadata_entries) = parse_metadata(&data[usize::from(header.metadata_offset())..])?; + let (data_start, data_table) = parse_data_table(&data[usize::from(header.data_offset())..])?; + + Ok(Self { + header, + metadata_entries, + data_table, + data_start, + }) + } + + pub fn file_version(&self) -> u8 { + self.header.file_version() + } + + pub fn creator_version(&self) -> u8 { + self.header.creator_version() + } + + pub fn metadata(&self) -> &[MetadataEntry] { + &self.metadata_entries + } + + pub fn section(&self, index: usize) -> Option> { + let section = self.data_table.get(index)?; + let data = &self.data_start[usize::try_from(section.offset()).expect("usize != u64")..]; + Some(SectionReader::new(data, §ion)) + } + + pub fn sections(&self) -> SectionIter { + SectionIter { + data: &self.data_start, + entries: &self.data_table, + } + } +} + +pub struct SectionIter<'a> { + data: &'a [u8], + entries: &'a [DataTableEntry], +} + +impl<'a> Iterator for SectionIter<'a> { + type Item = Result>; + + fn next(&mut self) -> Option { + if let Some((section, rest)) = self.entries.split_first() { + self.entries = rest; + + let data = &self.data[usize::try_from(section.offset()).expect("usize != u64")..]; + Some(SectionReader::new(data, §ion)) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use crate::types::{TrackType, FieldValue}; + + #[test] + fn test_read_a_track() { + #[rustfmt::skip] + let buf = &[ + // Header + 0x89, // rwtfmagic + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x01, // file version + 0x00, // fv reserve + 0x00, + 0x00, + 0x00, // creator version + 0x00, // cv reserve + 0x00, + 0x00, + 0x18, // metadata table offset + 0x00, + 0x23, // data offset + 0x00, + 0x00, // e reserve + 0x00, + 0x89, // header crc + 0x98, + + // Metadata Table + 0x01, // one entry + 0x00, // entry type: track_type = 0x00 + 0x05, // two byte entry size = 5 + 0x00, + 0x02, // track type: segment = 0x02 + 0x05, // four byte segment ID + 0x00, + 0x00, + 0x00, + 0xD4, // crc + 0x93, + + // Data Table + 0x02, // two sections + + // Data Table Section 1 + 0x00, // section encoding = standard + 0x05, // leb128 point count + 0x33, // leb128 data size + + // Schema for Section 1 + 0x00, // schema version + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name len + b'm', // name + 0x09, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name len + b'k', // name + 0x09, // leb128 data size + 0x04, // third field type = String + 0x01, // name len + b'j', // name + 0x18, // leb128 data size + + // Data Table Section 2 + 0x00, // section encoding = standard + 0x03, // leb128 point count + 0x26, // leb128 data size + + // Schema for Section 2 + 0x00, // schema version + 0x03, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x07, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x06, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x12, // leb128 data size + + // Data Table CRC + 0xC8, + 0x42, + + // Data Section 1 + + // Presence Column + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + 0xF6, // crc + 0xF8, + 0x0D, + 0x73, + + // Data Column 1 = I64 + 0x2A, // 42 + 0x00, // no change + 0x00, // no change + 0x00, // no change + 0x00, // no change + 0xD0, // crc + 0x8D, + 0x79, + 0x68, + + // Data Column 2 = Bool + 0x01, // true + 0x01, // true + 0x01, // true + 0x01, // true + 0x01, // true + 0xB5, // crc + 0xC9, + 0x8F, + 0xFA, + + // Data Column 3 = String + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x03, // length 3 + b'h', + b'e', + b'y', + 0x36, // crc + 0x71, + 0x24, + 0x0B, + + // Data Section 2 + + // Presence Column + 0b00000111, + 0b00000101, + 0b00000111, + 0x1A, // crc + 0x75, + 0xEA, + 0xC4, + + // Data Column 1 = I64 + 0x01, // 1 + 0x01, // 2 + 0x02, // 4 + 0xCA, // crc + 0xD4, + 0xD8, + 0x92, + + // Data Column 2 = Bool + 0x00, // false + // None + 0x01, // true + 0x35, // crc + 0x86, + 0x89, + 0xFB, + + // Data Column 3 = String + 0x04, // length 4 + b'R', + b'i', + b'd', + b'e', + 0x04, // length 4 + b'w', + b'i', + b't', + b'h', + 0x03, // length 3 + b'G', + b'P', + b'S', + 0xA3, // crc + 0x02, + 0xEC, + 0x48]; + + let track = assert_matches!(TrackReader::from_bytes(buf), Ok(track) => track); + + assert_eq!(track.file_version(), 1); + assert_eq!(track.creator_version(), 0); + + assert_eq!(track.metadata().len(), 1); + + assert_matches!(track.metadata()[0], MetadataEntry::TrackType(TrackType::Segment(5))); + + let expected_section_0 = vec![vec![Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string()))], + vec![Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string()))], + vec![Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string()))], + vec![Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string()))], + vec![Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string()))]]; + + let expected_section_1 = vec![vec![Some(FieldValue::I64(1)), + Some(FieldValue::Bool(false)), + Some(FieldValue::String("Ride".to_string()))], + vec![Some(FieldValue::I64(2)), + None, + Some(FieldValue::String("with".to_string()))], + vec![Some(FieldValue::I64(4)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("GPS".to_string()))]]; + + let sections = track + .sections() + .map(|section| { + let mut section = section?; + let mut v = vec![]; + while let Some(columniter) = section.open_column_iter() { + v.push(columniter + .map(|column_result| { + let (_field_def, field_value) = column_result.unwrap(); + field_value + }) + .collect::>()); + } + Ok(v) + }) + .collect::>>(); + + assert_matches!(sections , Ok(sections) => { + assert_eq!(sections.len(), 2); + assert_eq!(sections[0], expected_section_0); + assert_eq!(sections[1], expected_section_1); + }); + + assert_matches!(track.section(0), Some(Ok(mut section)) => { + let mut v = vec![]; + while let Some(columniter) = section.open_column_iter() { + v.push(columniter + .map(|column_result| { + let (_field_def, field_value) = column_result.unwrap(); + field_value + }) + .collect::>()); + } + assert_eq!(v, expected_section_0); + }); + + assert_matches!(track.section(1), Some(Ok(mut section)) => { + let mut v = vec![]; + while let Some(columniter) = section.open_column_iter() { + v.push(columniter + .map(|column_result| { + let (_field_def, field_value) = column_result.unwrap(); + field_value + }) + .collect::>()); + } + assert_eq!(v, expected_section_1); + }); + + assert!(track.section(2).is_none()); + } +} From 38c357ad420f3296b07c30b7fdafb7b7b87f9904 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 17:00:36 -0700 Subject: [PATCH 047/113] roundtrip tests for each column type --- tracklib2/tests/roundtrip.rs | 187 +++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 tracklib2/tests/roundtrip.rs diff --git a/tracklib2/tests/roundtrip.rs b/tracklib2/tests/roundtrip.rs new file mode 100644 index 0000000..8070cb8 --- /dev/null +++ b/tracklib2/tests/roundtrip.rs @@ -0,0 +1,187 @@ +// Only run these tests in a release build because they rely on unchecked math +#[cfg(not(debug_assertions))] +mod tests { + use tracklib2::read::track::TrackReader; + use tracklib2::schema::*; + use tracklib2::types::{FieldValue, SectionEncoding}; + use tracklib2::write::section::{ColumnWriter, Section}; + use tracklib2::write::track::write_track; + + #[test] + fn roundtrip_i64() { + let mut buf = vec![]; + let write_values = &[ + 0, + 20, + -20, + 5000, + -5000, + i64::MIN, + -10, + 0, + i64::MAX, + i64::MIN, + ]; + + // Write + let mut section = Section::new( + SectionEncoding::Standard, + Schema::with_fields(vec![FieldDefinition::new("v", DataType::I64)]), + ); + for v in write_values.iter() { + let mut rowbuilder = section.open_row_builder(); + + while let Some(cw) = rowbuilder.next_column_writer() { + if let ColumnWriter::I64ColumnWriter(cwi) = cw { + assert!(cwi.write(Some(v)).is_ok()); + } + } + } + assert!(write_track(&mut buf, &[], &[section]).is_ok()); + + // Read + let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let mut read_values: Vec = vec![]; + for section in track_reader.sections() { + let mut s = section.unwrap(); + while let Some(columniter) = s.open_column_iter() { + let vals = columniter.collect::>(); + assert_eq!(vals.len(), 1); + let (_field_desc, field_value) = vals[0].as_ref().unwrap(); + if let Some(FieldValue::I64(v)) = field_value { + read_values.push(*v); + } + } + } + + // Compare + assert_eq!(write_values, read_values.as_slice()); + } + + #[test] + fn roundtrip_f64() { + let mut buf = vec![]; + let write_values = &[-200.101, 0.0, 0.1]; + + // Write + let mut section = Section::new( + SectionEncoding::Standard, + Schema::with_fields(vec![FieldDefinition::new("v", DataType::F64)]), + ); + for v in write_values.iter() { + let mut rowbuilder = section.open_row_builder(); + + while let Some(cw) = rowbuilder.next_column_writer() { + if let ColumnWriter::F64ColumnWriter(cwi) = cw { + assert!(cwi.write(Some(v)).is_ok()); + } + } + } + assert!(write_track(&mut buf, &[], &[section]).is_ok()); + + // Read + let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let mut read_values: Vec = vec![]; + for section in track_reader.sections() { + let mut s = section.unwrap(); + while let Some(columniter) = s.open_column_iter() { + let vals = columniter.collect::>(); + assert_eq!(vals.len(), 1); + let (_field_desc, field_value) = vals[0].as_ref().unwrap(); + if let Some(FieldValue::F64(v)) = field_value { + read_values.push(*v); + } + } + } + + // Compare + assert_eq!(write_values, read_values.as_slice()); + } + + #[test] + fn roundtrip_bool() { + let mut buf = vec![]; + let write_values = &[false, true, true, true, false, false, true]; + + // Write + let mut section = Section::new( + SectionEncoding::Standard, + Schema::with_fields(vec![FieldDefinition::new("v", DataType::Bool)]), + ); + for v in write_values.iter() { + let mut rowbuilder = section.open_row_builder(); + + while let Some(cw) = rowbuilder.next_column_writer() { + if let ColumnWriter::BoolColumnWriter(cwi) = cw { + assert!(cwi.write(Some(v)).is_ok()); + } + } + } + assert!(write_track(&mut buf, &[], &[section]).is_ok()); + + // Read + let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let mut read_values: Vec = vec![]; + for section in track_reader.sections() { + let mut s = section.unwrap(); + while let Some(columniter) = s.open_column_iter() { + let vals = columniter.collect::>(); + assert_eq!(vals.len(), 1); + let (_field_desc, field_value) = vals[0].as_ref().unwrap(); + if let Some(FieldValue::Bool(v)) = field_value { + read_values.push(*v); + } + } + } + + // Compare + assert_eq!(write_values, read_values.as_slice()); + } + + #[test] + fn roundtrip_string() { + let mut buf = vec![]; + let write_values = &[ + "".to_string(), + "longer string".to_string(), + r"reallllllllllllllllllllllllllllllllllllllyyyyyyyyyyyyyyyyyyyyyyy + longggggggggggggggggggggggggggggggggggggggggggggggggggggggggg + stringgggggggggggggggggggggggggggggggggg" + .to_string(), + ]; + + // Write + let mut section = Section::new( + SectionEncoding::Standard, + Schema::with_fields(vec![FieldDefinition::new("v", DataType::String)]), + ); + for v in write_values.iter() { + let mut rowbuilder = section.open_row_builder(); + + while let Some(cw) = rowbuilder.next_column_writer() { + if let ColumnWriter::StringColumnWriter(cwi) = cw { + assert!(cwi.write(Some(v)).is_ok()); + } + } + } + assert!(write_track(&mut buf, &[], &[section]).is_ok()); + + // Read + let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let mut read_values: Vec = vec![]; + for section in track_reader.sections() { + let mut s = section.unwrap(); + while let Some(columniter) = s.open_column_iter() { + let vals = columniter.collect::>(); + assert_eq!(vals.len(), 1); + let (_field_desc, field_value) = vals[0].as_ref().unwrap(); + if let Some(FieldValue::String(v)) = field_value { + read_values.push(v.clone()); + } + } + } + + // Compare + assert_eq!(write_values, read_values.as_slice()); + } +} From 9b11755aa32510cd300779f857a68c3585c90685 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 24 Mar 2022 17:04:18 -0700 Subject: [PATCH 048/113] fix some compiler warnings --- tracklib2/src/error.rs | 4 ++-- tracklib2/src/read/data_table.rs | 2 +- tracklib2/src/read/schema.rs | 1 - tracklib2/src/write/data_table.rs | 1 - tracklib2/src/write/metadata.rs | 1 - tracklib2/src/write/section.rs | 3 +-- tracklib2/src/write/track.rs | 1 - 7 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tracklib2/src/error.rs b/tracklib2/src/error.rs index da34753..34b29d8 100644 --- a/tracklib2/src/error.rs +++ b/tracklib2/src/error.rs @@ -30,11 +30,11 @@ pub enum TracklibError { pub type Result = std::result::Result; impl nom::error::ParseError for TracklibError { - fn from_error_kind(input: I, kind: nom::error::ErrorKind) -> Self { + fn from_error_kind(_input: I, kind: nom::error::ErrorKind) -> Self { Self::ParseError { error_kind: kind } } - fn append(input: I, kind: nom::error::ErrorKind, other: Self) -> Self { + fn append(_input: I, _kind: nom::error::ErrorKind, other: Self) -> Self { other } } diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index 63295ca..4b42b86 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -4,7 +4,6 @@ use crate::error::TracklibError; use crate::types::SectionEncoding; use nom::{number::complete::le_u8, IResult}; use nom_leb128::leb128_u64; -use std::convert::TryFrom; #[cfg_attr(test, derive(Debug, PartialEq))] pub(crate) struct DataTableEntry { @@ -24,6 +23,7 @@ impl DataTableEntry { self.offset } + #[cfg(feature = "inspect")] pub(crate) fn size(&self) -> usize { self.size } diff --git a/tracklib2/src/read/schema.rs b/tracklib2/src/read/schema.rs index d1e1e2b..b7096bc 100644 --- a/tracklib2/src/read/schema.rs +++ b/tracklib2/src/read/schema.rs @@ -3,7 +3,6 @@ use crate::error::TracklibError; use crate::schema::*; use nom::{bytes::complete::tag, multi::length_data, number::complete::le_u8, IResult}; use nom_leb128::leb128_u64; -use std::convert::TryFrom; #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] diff --git a/tracklib2/src/write/data_table.rs b/tracklib2/src/write/data_table.rs index 2d4ed63..3941cc5 100644 --- a/tracklib2/src/write/data_table.rs +++ b/tracklib2/src/write/data_table.rs @@ -2,7 +2,6 @@ use super::crcwriter::CrcWriter; use super::section::Section; use crate::error::Result; use crate::types::SectionEncoding; -use std::convert::TryFrom; use std::io::Write; impl SectionEncoding { diff --git a/tracklib2/src/write/metadata.rs b/tracklib2/src/write/metadata.rs index b6fe646..58552ca 100644 --- a/tracklib2/src/write/metadata.rs +++ b/tracklib2/src/write/metadata.rs @@ -1,7 +1,6 @@ use super::crcwriter::CrcWriter; use crate::error::Result; use crate::types::{MetadataEntry, TrackType}; -use std::convert::TryFrom; use std::io::Write; impl MetadataEntry { diff --git a/tracklib2/src/write/section.rs b/tracklib2/src/write/section.rs index 0c0bbca..7ac1ab5 100644 --- a/tracklib2/src/write/section.rs +++ b/tracklib2/src/write/section.rs @@ -4,7 +4,6 @@ use crate::consts::SCHEMA_VERSION; use crate::error::Result; use crate::schema::*; use crate::types::SectionEncoding; -use std::convert::TryFrom; use std::io::{self, Write}; impl DataType { @@ -354,7 +353,7 @@ mod tests { ), ); - for i in 0..2 { + for _ in 0..2 { let mut rowbuilder = section.open_row_builder(); while let Some(cw) = rowbuilder.next_column_writer() { match cw { diff --git a/tracklib2/src/write/track.rs b/tracklib2/src/write/track.rs index 524e641..b472d3a 100644 --- a/tracklib2/src/write/track.rs +++ b/tracklib2/src/write/track.rs @@ -4,7 +4,6 @@ use super::section::Section; use crate::consts::RWTF_HEADER_SIZE; use crate::error::Result; use crate::types::MetadataEntry; -use std::convert::TryFrom; use std::io::{self, Write}; pub fn write_track( From ae50bad5f5783a929429e0224a9242192f571df8 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 29 Mar 2022 11:38:44 -0700 Subject: [PATCH 049/113] predicate pushdown --- tracklib2/Cargo.toml | 1 + tracklib2/src/read/data_table.rs | 4 +- tracklib2/src/read/inspect.rs | 17 +- tracklib2/src/read/mod.rs | 2 +- tracklib2/src/read/schema.rs | 2 +- .../read/{section_reader.rs => section.rs} | 330 +++++++++++++++--- tracklib2/src/read/track.rs | 173 +++++---- tracklib2/src/schema.rs | 7 +- tracklib2/src/types.rs | 2 +- tracklib2/tests/roundtrip.rs | 16 +- 10 files changed, 428 insertions(+), 126 deletions(-) rename tracklib2/src/read/{section_reader.rs => section.rs} (55%) diff --git a/tracklib2/Cargo.toml b/tracklib2/Cargo.toml index c7809d7..5de3eee 100644 --- a/tracklib2/Cargo.toml +++ b/tracklib2/Cargo.toml @@ -18,6 +18,7 @@ criterion = "0.3" float-cmp = "0.9" [features] +default = [] inspect = ["term-table"] [[bench]] diff --git a/tracklib2/src/read/data_table.rs b/tracklib2/src/read/data_table.rs index 4b42b86..8503dde 100644 --- a/tracklib2/src/read/data_table.rs +++ b/tracklib2/src/read/data_table.rs @@ -15,8 +15,8 @@ pub(crate) struct DataTableEntry { } impl DataTableEntry { - pub(crate) fn section_encoding(&self) -> &SectionEncoding { - &self.section_encoding + pub(crate) fn section_encoding(&self) -> SectionEncoding { + self.section_encoding } pub(crate) fn offset(&self) -> usize { diff --git a/tracklib2/src/read/inspect.rs b/tracklib2/src/read/inspect.rs index 1b49d7d..8817c6d 100644 --- a/tracklib2/src/read/inspect.rs +++ b/tracklib2/src/read/inspect.rs @@ -1,7 +1,7 @@ use super::data_table::{parse_data_table, DataTableEntry}; use super::header::{parse_header, Header}; use super::metadata::parse_metadata; -use super::section_reader::SectionReader; +use super::section::Section; use crate::types::FieldValue; use nom::Offset; use term_table::row::Row; @@ -30,7 +30,11 @@ fn format_header(header: &Header) -> String { )])); table.add_row(Row::new(vec![ TableCell::new("File Version"), - TableCell::new_with_alignment(format!("{:#04X}", header.file_version()), 1, Alignment::Right), + TableCell::new_with_alignment( + format!("{:#04X}", header.file_version()), + 1, + Alignment::Right, + ), ])); table.add_row(Row::new(vec![ TableCell::new("Creator Version"), @@ -50,7 +54,11 @@ fn format_header(header: &Header) -> String { ])); table.add_row(Row::new(vec![ TableCell::new("Data Offset"), - TableCell::new_with_alignment(format!("{:#04X}", header.data_offset()), 1, Alignment::Right), + TableCell::new_with_alignment( + format!("{:#04X}", header.data_offset()), + 1, + Alignment::Right, + ), ])); table.render() @@ -260,8 +268,9 @@ fn try_format_section(data_start: &[u8], entry_num: usize, entry: &DataTableEntr let mut table = Table::new(); let data = &data_start[usize::try_from(entry.offset()).expect("usize != u64")..]; + let section = Section::new(data, &entry); - match SectionReader::new(data, &entry) { + match section.reader() { Ok(mut section_reader) => { table.add_row(Row::new(vec![TableCell::new_with_alignment( bold(&format!("Data {entry_num}")), diff --git a/tracklib2/src/read/mod.rs b/tracklib2/src/read/mod.rs index 571271a..fa7ea86 100644 --- a/tracklib2/src/read/mod.rs +++ b/tracklib2/src/read/mod.rs @@ -8,5 +8,5 @@ pub mod inspect; mod metadata; mod presence_column; mod schema; -mod section_reader; +mod section; pub mod track; diff --git a/tracklib2/src/read/schema.rs b/tracklib2/src/read/schema.rs index b7096bc..dd4430d 100644 --- a/tracklib2/src/read/schema.rs +++ b/tracklib2/src/read/schema.rs @@ -4,7 +4,7 @@ use crate::schema::*; use nom::{bytes::complete::tag, multi::length_data, number::complete::le_u8, IResult}; use nom_leb128::leb128_u64; -#[derive(Debug)] +#[derive(Clone, Debug)] #[cfg_attr(test, derive(PartialEq))] pub(crate) struct SchemaEntry { field_definition: FieldDefinition, diff --git a/tracklib2/src/read/section_reader.rs b/tracklib2/src/read/section.rs similarity index 55% rename from tracklib2/src/read/section_reader.rs rename to tracklib2/src/read/section.rs index 8d87b40..1903d35 100644 --- a/tracklib2/src/read/section_reader.rs +++ b/tracklib2/src/read/section.rs @@ -1,10 +1,82 @@ use super::data_table::DataTableEntry; use super::decoders::*; use super::presence_column::parse_presence_column; +use super::schema::SchemaEntry; use crate::error::{Result, TracklibError}; use crate::schema::*; use crate::types::{FieldValue, SectionEncoding}; +#[cfg_attr(test, derive(Debug))] +pub struct Section<'a> { + input: &'a [u8], + data_table_entry: &'a DataTableEntry, +} + +impl<'a> Section<'a> { + pub(crate) fn new(input: &'a [u8], data_table_entry: &'a DataTableEntry) -> Self { + Self { + input, + data_table_entry, + } + } + + pub fn encoding(&self) -> SectionEncoding { + self.data_table_entry.section_encoding() + } + + pub fn schema(&self) -> Schema { + Schema::with_fields( + self.data_table_entry + .schema_entries() + .iter() + .map(|schema_entry| schema_entry.field_definition().clone()) + .collect(), + ) + } + + pub fn rows(&self) -> usize { + self.data_table_entry.rows() + } + + pub fn reader(&self) -> Result { + SectionReader::new( + self.input, + self.data_table_entry + .schema_entries() + .iter() + .enumerate() + .collect(), + self.data_table_entry.schema_entries().len(), + self.data_table_entry.rows(), + ) + } + + pub fn reader_for_schema(&self, schema: &Schema) -> Option> { + let schema_entries = schema + .fields() + .into_iter() + .filter_map(|field| { + self.data_table_entry + .schema_entries() + .iter() + .enumerate() + .find(|(_, schema_entry)| schema_entry.field_definition() == field) + }) + .collect::>(); + + if schema_entries.len() == schema.fields().len() { + Some(SectionReader::new( + self.input, + schema_entries, + self.data_table_entry.schema_entries().len(), + self.data_table_entry.rows(), + )) + } else { + None + } + } +} + #[cfg_attr(test, derive(Debug))] enum ColumnDecoder<'a> { I64 { @@ -28,39 +100,29 @@ enum ColumnDecoder<'a> { #[cfg_attr(test, derive(Debug))] pub struct SectionReader<'a> { decoders: Vec>, - section_encoding: SectionEncoding, - schema: Schema, rows: usize, } impl<'a> SectionReader<'a> { - pub(crate) fn new(input: &'a [u8], data_table_entry: &'a DataTableEntry) -> Result { - let (column_data, presence_column) = parse_presence_column( - data_table_entry.schema_entries().len(), - data_table_entry.rows(), - )(input)?; - - let schema = Schema::with_fields( - data_table_entry - .schema_entries() - .iter() - .map(|schema_entry| schema_entry.field_definition().clone()) - .collect(), - ); + pub(crate) fn new( + input: &'a [u8], + schema_entries: Vec<(usize, &'a SchemaEntry)>, + columns: usize, + rows: usize, + ) -> Result { + let (column_data, presence_column) = parse_presence_column(columns, rows)(input)?; - let decoders = data_table_entry - .schema_entries() - .iter() - .enumerate() - .map(|(i, schema_entry)| { + let decoders = schema_entries + .into_iter() + .map(|(presence_column_index, schema_entry)| { let column_data = &column_data [schema_entry.offset()..schema_entry.offset() + schema_entry.size()]; let presence_column_view = - presence_column - .view(i) - .ok_or_else(|| TracklibError::ParseIncompleteError { + presence_column.view(presence_column_index).ok_or_else(|| { + TracklibError::ParseIncompleteError { needed: nom::Needed::Unknown, - })?; + } + })?; let field_definition = schema_entry.field_definition(); let decoder = match field_definition.data_type() { DataType::I64 => ColumnDecoder::I64 { @@ -84,26 +146,13 @@ impl<'a> SectionReader<'a> { }) .collect::>>()?; - Ok(Self { - schema, - section_encoding: data_table_entry.section_encoding().clone(), - decoders, - rows: data_table_entry.rows(), - }) + Ok(Self { decoders, rows }) } - pub fn section_encoding(&self) -> &SectionEncoding { - &self.section_encoding - } - - pub fn rows(&self) -> usize { + pub fn rows_remaining(&self) -> usize { self.rows } - pub fn schema(&self) -> &Schema { - &self.schema - } - pub fn open_column_iter<'r>(&'r mut self) -> Option> { if self.rows > 0 { self.rows -= 1; @@ -127,7 +176,7 @@ impl<'a, 'b> ColumnIter<'a, 'b> { } impl<'a, 'b> Iterator for ColumnIter<'a, 'b> { - type Item = Result<(&'a FieldDefinition, Option)>; + type Item = Result<(&'b FieldDefinition, Option)>; fn next(&mut self) -> Option { if let Some(decoder_enum) = self.decoders.get_mut(self.index) { @@ -282,8 +331,20 @@ mod tests { 0x15 ]; - assert_matches!(SectionReader::new(buf, &data_table_entries[0]), Ok(mut section_reader) => { + let section = Section::new(buf, &data_table_entries[0]); + + assert_eq!(section.encoding(), SectionEncoding::Standard); + assert_eq!(section.rows(), 3); + assert_eq!(section.schema(), Schema::with_fields(vec![ + FieldDefinition::new("a", DataType::I64), + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("c", DataType::String), + FieldDefinition::new("f", DataType::F64), + ])); + + assert_matches!(section.reader(), Ok(mut section_reader) => { // Row 1 + assert_eq!(section_reader.rows_remaining(), 3); assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); @@ -310,6 +371,7 @@ mod tests { }); // Row 2 + assert_eq!(section_reader.rows_remaining(), 2); assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); @@ -336,6 +398,7 @@ mod tests { }); // Row 3 + assert_eq!(section_reader.rows_remaining(), 1); assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { let values = column_iter.collect::>(); assert_eq!(values.len(), 4); @@ -362,6 +425,191 @@ mod tests { }); // Trying to get another row will return nothing + assert_eq!(section_reader.rows_remaining(), 0); + assert_matches!(section_reader.open_column_iter(), None); + }); + }); + } + + #[test] + fn test_section_reader_for_schema() { + #[rustfmt::skip] + let data_table_buf = &[0x01, // number of sections + + // Section 1 + 0x00, // section encoding = standard + 0x03, // leb128 section point count + 0x26, // leb128 section data size + // Schema + 0x00, // schema version + 0x04, // field count + 0x00, // first field type = I64 + 0x01, // name length + b'a', // name + 0x07, // leb128 data size + 0x05, // second field type = Bool + 0x01, // name length + b'b', // name + 0x06, // leb128 data size + 0x04, // third field type = String + 0x01, // name length + b'c', // name + 0x12, // leb128 data size + 0x01, // fourth field type = F64 + 0x01, // name length + b'f', // name + 0x0C, // leb128 data size + + 0x81, // crc + 0x7D]; + + assert_matches!(parse_data_table(data_table_buf), Ok((&[], data_table_entries)) => { + assert_eq!(data_table_entries.len(), 1); + + #[rustfmt::skip] + let buf = &[ + // Presence Column + 0b00000111, + 0b00001101, + 0b00001111, + 0xF0, // crc + 0xDB, + 0xAA, + 0x68, + + // Data Column 1 = I64 + 0x01, // 1 + 0x01, // 2 + 0x02, // 4 + 0xCA, // crc + 0xD4, + 0xD8, + 0x92, + + // Data Column 2 = Bool + 0x00, // false + // None + 0x01, // true + 0x35, // crc + 0x86, + 0x89, + 0xFB, + + // Data Column 3 = String + 0x04, // length 4 + b'R', + b'i', + b'd', + b'e', + 0x04, // length 4 + b'w', + b'i', + b't', + b'h', + 0x03, // length 3 + b'G', + b'P', + b'S', + 0xA3, // crc + 0x02, + 0xEC, + 0x48, + + // Data Column 4 = F64 + // None + 0x80, // 1.0 + 0xAD, + 0xE2, + 0x04, + 0xC0, // 2.5 + 0xC3, + 0x93, + 0x07, + 0xCC, // crc + 0xEC, + 0xC5, + 0x15 + ]; + + let section = Section::new(buf, &data_table_entries[0]); + + assert_eq!(section.encoding(), SectionEncoding::Standard); + assert_eq!(section.rows(), 3); + assert_eq!(section.schema(), Schema::with_fields(vec![ + FieldDefinition::new("a", DataType::I64), + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("c", DataType::String), + FieldDefinition::new("f", DataType::F64), + ])); + + // Missing field + assert_matches!(section.reader_for_schema(&Schema::with_fields(vec![ + FieldDefinition::new("z", DataType::Bool), + ])), None); + + // Field exists but we're asking for the wrong type + assert_matches!(section.reader_for_schema(&Schema::with_fields(vec![ + FieldDefinition::new("b", DataType::I64), + ])), None); + + // Both of these fields exist + assert_matches!(section.reader_for_schema(&Schema::with_fields(vec![ + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("f", DataType::F64), + ])), Some(Ok(mut section_reader)) => { + // Row 1 + assert_eq!(section_reader.rows_remaining(), 3); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 2); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); + assert_eq!(field_value, &Some(FieldValue::Bool(false))); + }); + assert_matches!(&values[1], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[3].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); + assert_eq!(field_value, &None); + }); + }); + + // Row 2 + assert_eq!(section_reader.rows_remaining(), 2); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 2); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); + assert_eq!(field_value, &None); + }); + assert_matches!(&values[1], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[3].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); + assert_eq!(field_value, &Some(FieldValue::F64(1.0))); + }); + }); + + // Row 3 + assert_eq!(section_reader.rows_remaining(), 1); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 2); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); + assert_eq!(field_value, &Some(FieldValue::Bool(true))); + }); + assert_matches!(&values[1], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[3].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("f", DataType::F64)); + assert_eq!(field_value, &Some(FieldValue::F64(2.5))); + }); + }); + + // Trying to get another row will return nothing + assert_eq!(section_reader.rows_remaining(), 0); assert_matches!(section_reader.open_column_iter(), None); }); }); diff --git a/tracklib2/src/read/track.rs b/tracklib2/src/read/track.rs index 92a5bb2..b34f018 100644 --- a/tracklib2/src/read/track.rs +++ b/tracklib2/src/read/track.rs @@ -1,7 +1,7 @@ use super::data_table::{parse_data_table, DataTableEntry}; use super::header::{parse_header, Header}; use super::metadata::parse_metadata; -use super::section_reader::SectionReader; +use super::section::Section; use crate::error::Result; use crate::types::MetadataEntry; @@ -17,7 +17,8 @@ impl<'a> TrackReader<'a> { pub fn from_bytes(data: &'a [u8]) -> Result { let (_, header) = parse_header(data)?; let (_, metadata_entries) = parse_metadata(&data[usize::from(header.metadata_offset())..])?; - let (data_start, data_table) = parse_data_table(&data[usize::from(header.data_offset())..])?; + let (data_start, data_table) = + parse_data_table(&data[usize::from(header.data_offset())..])?; Ok(Self { header, @@ -39,10 +40,10 @@ impl<'a> TrackReader<'a> { &self.metadata_entries } - pub fn section(&self, index: usize) -> Option> { + pub fn section(&self, index: usize) -> Option
{ let section = self.data_table.get(index)?; let data = &self.data_start[usize::try_from(section.offset()).expect("usize != u64")..]; - Some(SectionReader::new(data, §ion)) + Some(Section::new(data, §ion)) } pub fn sections(&self) -> SectionIter { @@ -59,14 +60,14 @@ pub struct SectionIter<'a> { } impl<'a> Iterator for SectionIter<'a> { - type Item = Result>; + type Item = Section<'a>; fn next(&mut self) -> Option { if let Some((section, rest)) = self.entries.split_first() { self.entries = rest; let data = &self.data[usize::try_from(section.offset()).expect("usize != u64")..]; - Some(SectionReader::new(data, §ion)) + Some(Section::new(data, §ion)) } else { None } @@ -76,8 +77,9 @@ impl<'a> Iterator for SectionIter<'a> { #[cfg(test)] mod tests { use super::*; + use crate::schema::*; + use crate::types::{FieldValue, SectionEncoding, TrackType}; use assert_matches::assert_matches; - use crate::types::{TrackType, FieldValue}; #[test] fn test_read_a_track() { @@ -288,46 +290,71 @@ mod tests { assert_eq!(track.metadata().len(), 1); - assert_matches!(track.metadata()[0], MetadataEntry::TrackType(TrackType::Segment(5))); - - let expected_section_0 = vec![vec![Some(FieldValue::I64(42)), - Some(FieldValue::Bool(true)), - Some(FieldValue::String("hey".to_string()))], - vec![Some(FieldValue::I64(42)), - Some(FieldValue::Bool(true)), - Some(FieldValue::String("hey".to_string()))], - vec![Some(FieldValue::I64(42)), - Some(FieldValue::Bool(true)), - Some(FieldValue::String("hey".to_string()))], - vec![Some(FieldValue::I64(42)), - Some(FieldValue::Bool(true)), - Some(FieldValue::String("hey".to_string()))], - vec![Some(FieldValue::I64(42)), - Some(FieldValue::Bool(true)), - Some(FieldValue::String("hey".to_string()))]]; - - let expected_section_1 = vec![vec![Some(FieldValue::I64(1)), - Some(FieldValue::Bool(false)), - Some(FieldValue::String("Ride".to_string()))], - vec![Some(FieldValue::I64(2)), - None, - Some(FieldValue::String("with".to_string()))], - vec![Some(FieldValue::I64(4)), - Some(FieldValue::Bool(true)), - Some(FieldValue::String("GPS".to_string()))]]; + assert_matches!( + track.metadata()[0], + MetadataEntry::TrackType(TrackType::Segment(5)) + ); + + let expected_section_0 = vec![ + vec![ + Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string())), + ], + vec![ + Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string())), + ], + vec![ + Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string())), + ], + vec![ + Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string())), + ], + vec![ + Some(FieldValue::I64(42)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("hey".to_string())), + ], + ]; + + let expected_section_1 = vec![ + vec![ + Some(FieldValue::I64(1)), + Some(FieldValue::Bool(false)), + Some(FieldValue::String("Ride".to_string())), + ], + vec![ + Some(FieldValue::I64(2)), + None, + Some(FieldValue::String("with".to_string())), + ], + vec![ + Some(FieldValue::I64(4)), + Some(FieldValue::Bool(true)), + Some(FieldValue::String("GPS".to_string())), + ], + ]; let sections = track .sections() .map(|section| { - let mut section = section?; + let mut section_reader = section.reader()?; let mut v = vec![]; - while let Some(columniter) = section.open_column_iter() { - v.push(columniter - .map(|column_result| { - let (_field_def, field_value) = column_result.unwrap(); - field_value - }) - .collect::>()); + while let Some(columniter) = section_reader.open_column_iter() { + v.push( + columniter + .map(|column_result| { + let (_field_def, field_value) = column_result.unwrap(); + field_value + }) + .collect::>(), + ); } Ok(v) }) @@ -339,30 +366,48 @@ mod tests { assert_eq!(sections[1], expected_section_1); }); - assert_matches!(track.section(0), Some(Ok(mut section)) => { - let mut v = vec![]; - while let Some(columniter) = section.open_column_iter() { - v.push(columniter - .map(|column_result| { - let (_field_def, field_value) = column_result.unwrap(); - field_value - }) - .collect::>()); - } - assert_eq!(v, expected_section_0); + assert_matches!(track.section(0), Some(section) => { + assert_eq!(section.encoding(), SectionEncoding::Standard); + assert_eq!(section.rows(), 5); + assert_eq!(section.schema(), Schema::with_fields(vec![ + FieldDefinition::new("m", DataType::I64), + FieldDefinition::new("k", DataType::Bool), + FieldDefinition::new("j", DataType::String), + ])); + assert_matches!(section.reader(), Ok(mut section_reader) => { + let mut v = vec![]; + while let Some(columniter) = section_reader.open_column_iter() { + v.push(columniter + .map(|column_result| { + let (_field_def, field_value) = column_result.unwrap(); + field_value + }) + .collect::>()); + } + assert_eq!(v, expected_section_0); + }); }); - assert_matches!(track.section(1), Some(Ok(mut section)) => { - let mut v = vec![]; - while let Some(columniter) = section.open_column_iter() { - v.push(columniter - .map(|column_result| { - let (_field_def, field_value) = column_result.unwrap(); - field_value - }) - .collect::>()); - } - assert_eq!(v, expected_section_1); + assert_matches!(track.section(1), Some(section) => { + assert_eq!(section.encoding(), SectionEncoding::Standard); + assert_eq!(section.rows(), 3); + assert_eq!(section.schema(), Schema::with_fields(vec![ + FieldDefinition::new("a", DataType::I64), + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("c", DataType::String), + ])); + assert_matches!(section.reader(), Ok(mut section_reader) => { + let mut v = vec![]; + while let Some(columniter) = section_reader.open_column_iter() { + v.push(columniter + .map(|column_result| { + let (_field_def, field_value) = column_result.unwrap(); + field_value + }) + .collect::>()); + } + assert_eq!(v, expected_section_1); + }); }); assert!(track.section(2).is_none()); diff --git a/tracklib2/src/schema.rs b/tracklib2/src/schema.rs index 3a92230..3d00971 100644 --- a/tracklib2/src/schema.rs +++ b/tracklib2/src/schema.rs @@ -1,4 +1,5 @@ #[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] pub struct Schema { fields: Vec, } @@ -13,8 +14,7 @@ impl Schema { } } -#[derive(Clone, Debug)] -#[cfg_attr(test, derive(PartialEq))] +#[derive(Clone, Debug, PartialEq)] pub struct FieldDefinition { name: String, data_type: DataType, @@ -44,8 +44,7 @@ pub enum BitstreamType { Bool, } -#[derive(Clone, Debug)] -#[cfg_attr(test, derive(PartialEq))] +#[derive(Clone, Debug, PartialEq)] pub enum DataType { I64, Bool, diff --git a/tracklib2/src/types.rs b/tracklib2/src/types.rs index 9fc5554..8187d5e 100644 --- a/tracklib2/src/types.rs +++ b/tracklib2/src/types.rs @@ -22,7 +22,7 @@ pub enum MetadataEntry { CreatedAt(u64), } -#[derive(Clone, Debug)] +#[derive(Clone, Copy, Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum SectionEncoding { Standard, diff --git a/tracklib2/tests/roundtrip.rs b/tracklib2/tests/roundtrip.rs index 8070cb8..ba09db0 100644 --- a/tracklib2/tests/roundtrip.rs +++ b/tracklib2/tests/roundtrip.rs @@ -43,8 +43,8 @@ mod tests { let track_reader = TrackReader::from_bytes(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { - let mut s = section.unwrap(); - while let Some(columniter) = s.open_column_iter() { + let mut section_reader = section.reader().unwrap(); + while let Some(columniter) = section_reader.open_column_iter() { let vals = columniter.collect::>(); assert_eq!(vals.len(), 1); let (_field_desc, field_value) = vals[0].as_ref().unwrap(); @@ -83,8 +83,8 @@ mod tests { let track_reader = TrackReader::from_bytes(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { - let mut s = section.unwrap(); - while let Some(columniter) = s.open_column_iter() { + let mut section_reader = section.reader().unwrap(); + while let Some(columniter) = section_reader.open_column_iter() { let vals = columniter.collect::>(); assert_eq!(vals.len(), 1); let (_field_desc, field_value) = vals[0].as_ref().unwrap(); @@ -123,8 +123,8 @@ mod tests { let track_reader = TrackReader::from_bytes(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { - let mut s = section.unwrap(); - while let Some(columniter) = s.open_column_iter() { + let mut section_reader = section.reader().unwrap(); + while let Some(columniter) = section_reader.open_column_iter() { let vals = columniter.collect::>(); assert_eq!(vals.len(), 1); let (_field_desc, field_value) = vals[0].as_ref().unwrap(); @@ -170,8 +170,8 @@ mod tests { let track_reader = TrackReader::from_bytes(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { - let mut s = section.unwrap(); - while let Some(columniter) = s.open_column_iter() { + let mut section_reader = section.reader().unwrap(); + while let Some(columniter) = section_reader.open_column_iter() { let vals = columniter.collect::>(); assert_eq!(vals.len(), 1); let (_field_desc, field_value) = vals[0].as_ref().unwrap(); From 11f1ef73805176efcb8f480922113969c7df9e58 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 31 Mar 2022 11:03:07 -0700 Subject: [PATCH 050/113] rename TrackReader::from_bytes to TrackReader::new --- tracklib2/src/read/track.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracklib2/src/read/track.rs b/tracklib2/src/read/track.rs index b34f018..212e3d8 100644 --- a/tracklib2/src/read/track.rs +++ b/tracklib2/src/read/track.rs @@ -14,7 +14,7 @@ pub struct TrackReader<'a> { } impl<'a> TrackReader<'a> { - pub fn from_bytes(data: &'a [u8]) -> Result { + pub fn new(data: &'a [u8]) -> Result { let (_, header) = parse_header(data)?; let (_, metadata_entries) = parse_metadata(&data[usize::from(header.metadata_offset())..])?; let (data_start, data_table) = @@ -283,7 +283,7 @@ mod tests { 0xEC, 0x48]; - let track = assert_matches!(TrackReader::from_bytes(buf), Ok(track) => track); + let track = assert_matches!(TrackReader::new(buf), Ok(track) => track); assert_eq!(track.file_version(), 1); assert_eq!(track.creator_version(), 0); From 69acea0bf468b344e8c2ccd6e043ce83ce1f4a89 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 31 Mar 2022 12:22:28 -0700 Subject: [PATCH 051/113] start ruby_tracklib2 --- ruby_tracklib2/.gitignore | 11 ++ ruby_tracklib2/Cargo.lock | 249 +++++++++++++++++++++++++ ruby_tracklib2/Cargo.toml | 14 ++ ruby_tracklib2/Gemfile | 2 + ruby_tracklib2/Gemfile.lock | 34 ++++ ruby_tracklib2/Rakefile | 18 ++ ruby_tracklib2/default.nix | 14 ++ ruby_tracklib2/lib/tracklib.rb | 7 + ruby_tracklib2/lib/tracklib/version.rb | 3 + ruby_tracklib2/spec/reader_spec.rb | 208 +++++++++++++++++++++ ruby_tracklib2/spec/spec_helper.rb | 8 + ruby_tracklib2/src/lib.rs | 11 ++ ruby_tracklib2/src/track_reader.rs | 91 +++++++++ ruby_tracklib2/tracklib.gemspec | 28 +++ 14 files changed, 698 insertions(+) create mode 100644 ruby_tracklib2/.gitignore create mode 100644 ruby_tracklib2/Cargo.lock create mode 100644 ruby_tracklib2/Cargo.toml create mode 100644 ruby_tracklib2/Gemfile create mode 100644 ruby_tracklib2/Gemfile.lock create mode 100644 ruby_tracklib2/Rakefile create mode 100644 ruby_tracklib2/default.nix create mode 100644 ruby_tracklib2/lib/tracklib.rb create mode 100644 ruby_tracklib2/lib/tracklib/version.rb create mode 100644 ruby_tracklib2/spec/reader_spec.rb create mode 100644 ruby_tracklib2/spec/spec_helper.rb create mode 100644 ruby_tracklib2/src/lib.rs create mode 100644 ruby_tracklib2/src/track_reader.rs create mode 100644 ruby_tracklib2/tracklib.gemspec diff --git a/ruby_tracklib2/.gitignore b/ruby_tracklib2/.gitignore new file mode 100644 index 0000000..5dd8129 --- /dev/null +++ b/ruby_tracklib2/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ +.nix-gems/ +target/ +*.gem diff --git a/ruby_tracklib2/Cargo.lock b/ruby_tracklib2/Cargo.lock new file mode 100644 index 0000000..585f223 --- /dev/null +++ b/ruby_tracklib2/Cargo.lock @@ -0,0 +1,249 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "crc" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98a04dce437184842841303488f70d0188c5f51437d2a834dc097eafa909a01" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", +] + +[[package]] +name = "nom-leb128" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a73b6c3a9ecfff12ad50dedba051ef838d2f478d938bb3e6b3842431028e62" +dependencies = [ + "arrayvec", + "nom", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ouroboros" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f31a3b678685b150cba82b702dcdc5e155893f63610cf388d30cd988d4ca2bf" +dependencies = [ + "aliasable", + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084fd65d5dd8b3772edccb5ffd1e4b7eba43897ecd0f9401e330e8c542959408" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb37d2df5df740e582f28f8560cf425f52bb267d872fe58358eadb554909f07a" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ruby_tracklib" +version = "0.1.0" +dependencies = [ + "lazy_static", + "ouroboros", + "rutie", + "tracklib2", +] + +[[package]] +name = "rutie" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d81d2a2dec78b6202e8058dea842a39a65135ce8915e66e40d3935f12e52346" +dependencies = [ + "lazy_static", + "libc", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "syn" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracklib2" +version = "0.1.0" +dependencies = [ + "crc", + "leb128", + "nom", + "nom-leb128", + "thiserror", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" diff --git a/ruby_tracklib2/Cargo.toml b/ruby_tracklib2/Cargo.toml new file mode 100644 index 0000000..388184a --- /dev/null +++ b/ruby_tracklib2/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ruby_tracklib" +version = "0.1.0" +edition = "2021" + +[lib] +name = "tracklib" +crate-type = ["cdylib"] + +[dependencies] +lazy_static = "1.4" +ouroboros = "0.15" +rutie = "0.8" +tracklib2 = {path="../tracklib2"} diff --git a/ruby_tracklib2/Gemfile b/ruby_tracklib2/Gemfile new file mode 100644 index 0000000..3be9c3c --- /dev/null +++ b/ruby_tracklib2/Gemfile @@ -0,0 +1,2 @@ +source "https://rubygems.org" +gemspec diff --git a/ruby_tracklib2/Gemfile.lock b/ruby_tracklib2/Gemfile.lock new file mode 100644 index 0000000..ac9c6bb --- /dev/null +++ b/ruby_tracklib2/Gemfile.lock @@ -0,0 +1,34 @@ +PATH + remote: . + specs: + tracklib (0.1.0) + rutie (~> 0.0.4) + +GEM + remote: https://rubygems.org/ + specs: + diff-lcs (1.4.4) + rspec (3.10.0) + rspec-core (~> 3.10.0) + rspec-expectations (~> 3.10.0) + rspec-mocks (~> 3.10.0) + rspec-core (3.10.1) + rspec-support (~> 3.10.0) + rspec-expectations (3.10.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-mocks (3.10.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-support (3.10.3) + rutie (0.0.4) + +PLATFORMS + ruby + +DEPENDENCIES + rspec + tracklib! + +BUNDLED WITH + 2.1.4 diff --git a/ruby_tracklib2/Rakefile b/ruby_tracklib2/Rakefile new file mode 100644 index 0000000..3ceec2c --- /dev/null +++ b/ruby_tracklib2/Rakefile @@ -0,0 +1,18 @@ +require 'rspec/core/rake_task' + +desc 'Build Rust extension' +task :build_lib do + sh 'cargo build --release' +end + +desc 'bundle install' +task :bundle_install do + sh 'bundle install' +end + +RSpec::Core::RakeTask.new(spec: [:bundle_install, :build_lib]) do |t| + t.pattern = "spec/**/*_spec.rb" +end + +task :default => :spec +task :test => :spec diff --git a/ruby_tracklib2/default.nix b/ruby_tracklib2/default.nix new file mode 100644 index 0000000..eb371de --- /dev/null +++ b/ruby_tracklib2/default.nix @@ -0,0 +1,14 @@ +with import {}; +stdenv.mkDerivation rec { + name = "tracklib"; + env = buildEnv { name = name; paths = buildInputs; }; + buildInputs = [ + ruby_2_7 + ]; + shellHook = '' + mkdir -p .nix-gems + export GEM_HOME=$PWD/.nix-gems + export GEM_PATH=$GEM_HOME + export PATH=$GEM_HOME/bin:$PATH + ''; +} diff --git a/ruby_tracklib2/lib/tracklib.rb b/ruby_tracklib2/lib/tracklib.rb new file mode 100644 index 0000000..a6ccb5f --- /dev/null +++ b/ruby_tracklib2/lib/tracklib.rb @@ -0,0 +1,7 @@ +require "tracklib/version" +require 'rutie' + +module Tracklib + class TracklibError < StandardError; end + Rutie.new(:tracklib).init 'Init_Tracklib', __dir__ +end diff --git a/ruby_tracklib2/lib/tracklib/version.rb b/ruby_tracklib2/lib/tracklib/version.rb new file mode 100644 index 0000000..e76b39a --- /dev/null +++ b/ruby_tracklib2/lib/tracklib/version.rb @@ -0,0 +1,3 @@ +module Tracklib + VERSION = "0.1.0" +end diff --git a/ruby_tracklib2/spec/reader_spec.rb b/ruby_tracklib2/spec/reader_spec.rb new file mode 100644 index 0000000..3cf4c32 --- /dev/null +++ b/ruby_tracklib2/spec/reader_spec.rb @@ -0,0 +1,208 @@ +require "spec_helper" + +data = [ + # Header + 0x89, # rwtfmagic + 0x52, + 0x57, + 0x54, + 0x46, + 0x0A, + 0x1A, + 0x0A, + 0x01, # file version + 0x00, # fv reserve + 0x00, + 0x00, + 0x00, # creator version + 0x00, # cv reserve + 0x00, + 0x00, + 0x18, # metadata table offset + 0x00, + 0x23, # data offset + 0x00, + 0x00, # e reserve + 0x00, + 0x89, # header crc + 0x98, + + # Metadata Table + 0x01, # one entry + 0x00, # entry type: track_type = 0x00 + 0x05, # two byte entry size = 5 + 0x00, + 0x02, # track type: segment = 0x02 + 0x05, # four byte segment ID + 0x00, + 0x00, + 0x00, + 0xD4, # crc + 0x93, + + # Data Table + 0x02, # two sections + + # Data Table Section 1 + 0x00, # section encoding = standard + 0x05, # leb128 point count + 0x33, # leb128 data size + + # Schema for Section 1 + 0x00, # schema version + 0x03, # field count + 0x00, # first field type = I64 + 0x01, # name len + 0x6D, # name = m + 0x09, # leb128 data size + 0x05, # second field type = Bool + 0x01, # name len + 0x6B, # name = k + 0x09, # leb128 data size + 0x04, # third field type = String + 0x01, # name len + 0x6A, # name = j + 0x18, # leb128 data size + + # Data Table Section 2 + 0x00, # section encoding = standard + 0x03, # leb128 point count + 0x26, # leb128 data size + + # Schema for Section 2 + 0x00, # schema version + 0x03, # field count + 0x00, # first field type = I64 + 0x01, # name length + 0x61, # name = a + 0x07, # leb128 data size + 0x05, # second field type = Bool + 0x01, # name length + 0x62, # name = b + 0x06, # leb128 data size + 0x04, # third field type = String + 0x01, # name length + 0x63, # name = c + 0x12, # leb128 data size + + # Data Table CRC + 0xC8, + 0x42, + + # Data Section 1 + + # Presence Column + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + 0b00000111, + 0xF6, # crc + 0xF8, + 0x0D, + 0x73, + + # Data Column 1 = I64 + 0x2A, # 42 + 0x00, # no change + 0x00, # no change + 0x00, # no change + 0x00, # no change + 0xD0, # crc + 0x8D, + 0x79, + 0x68, + + # Data Column 2 = Bool + 0x01, # true + 0x01, # true + 0x01, # true + 0x01, # true + 0x01, # true + 0xB5, # crc + 0xC9, + 0x8F, + 0xFA, + + # Data Column 3 = String + 0x03, # length 3 + 0x68, # h + 0x65, # e + 0x79, # y + 0x03, # length 3 + 0x68, # h + 0x65, # e + 0x79, # y + 0x03, # length 3 + 0x68, # h + 0x65, # e + 0x79, # y + 0x03, # length 3 + 0x68, # h + 0x65, # e + 0x79, # y + 0x03, # length 3 + 0x68, # h + 0x65, # e + 0x79, # y + 0x36, # crc + 0x71, + 0x24, + 0x0B, + + # Data Section 2 + + # Presence Column + 0b00000111, + 0b00000101, + 0b00000111, + 0x1A, # crc + 0x75, + 0xEA, + 0xC4, + + # Data Column 1 = I64 + 0x01, # 1 + 0x01, # 2 + 0x02, # 4 + 0xCA, # crc + 0xD4, + 0xD8, + 0x92, + + # Data Column 2 = Bool + 0x00, # false + # None + 0x01, # true + 0x35, # crc + 0x86, + 0x89, + 0xFB, + + # Data Column 3 = String + 0x04, # length 4 + 0x52, # R + 0x69, # i + 0x64, # d + 0x65, # e + 0x04, # length 4 + 0x77, # w + 0x69, # i + 0x74, # t + 0x68, # h + 0x03, # length 3 + 0x47, # G + 0x50, # P + 0x53, # S + 0xA3, # crc + 0x02, + 0xEC, + 0x48 +].pack("c*") + +describe TrackReader do + it "can read metadata" do + track_reader = TrackReader.new(data) + expect(track_reader.metadata()).to eq([[:track_type, :segment, 5]]) + end +end diff --git a/ruby_tracklib2/spec/spec_helper.rb b/ruby_tracklib2/spec/spec_helper.rb new file mode 100644 index 0000000..9c1e978 --- /dev/null +++ b/ruby_tracklib2/spec/spec_helper.rb @@ -0,0 +1,8 @@ +require 'bundler/setup' +Bundler::setup() + +require 'tracklib' + +RSpec::configure do |config| + +end diff --git a/ruby_tracklib2/src/lib.rs b/ruby_tracklib2/src/lib.rs new file mode 100644 index 0000000..62f76ca --- /dev/null +++ b/ruby_tracklib2/src/lib.rs @@ -0,0 +1,11 @@ +mod track_reader; +use rutie::{Class, Object}; + +#[allow(non_snake_case)] +#[no_mangle] +pub extern "C" fn Init_Tracklib() { + Class::new("TrackReader", None).define(|klass| { + klass.def_self("new", track_reader::trackreader_new); + klass.def("metadata", track_reader::trackreader_metadata); + }); +} diff --git a/ruby_tracklib2/src/track_reader.rs b/ruby_tracklib2/src/track_reader.rs new file mode 100644 index 0000000..f3b16f9 --- /dev/null +++ b/ruby_tracklib2/src/track_reader.rs @@ -0,0 +1,91 @@ +use ouroboros::self_referencing; +use rutie::{ + class, methods, wrappable_struct, AnyObject, Array, Class, Integer, Object, RString, Symbol, VM, +}; +use tracklib2; + +#[self_referencing] +pub struct TrackReaderWrapper { + data: Vec, + #[borrows(data)] + #[not_covariant] + track_reader: tracklib2::read::track::TrackReader<'this>, +} + +wrappable_struct!( + TrackReaderWrapper, + TrackReaderWrapperWrapper, + TRACK_READER_WRAPPER +); + +class!(TrackReader); + +methods!( + TrackReader, + rtself, + fn trackreader_new(bytes: RString) -> AnyObject { + let source = bytes.map_err(|e| VM::raise_ex(e)).unwrap(); + let data = source.to_bytes_unchecked().to_vec(); + let wrapper = TrackReaderWrapper::new(data, |d| { + tracklib2::read::track::TrackReader::new(d) + .map_err(|e| VM::raise(Class::from_existing("Exception"), &format!("{}", e))) + .unwrap() + }); + + Class::from_existing("TrackReader").wrap_data(wrapper, &*TRACK_READER_WRAPPER) + }, + fn trackreader_metadata() -> Array { + let metadata_entries = rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| track_reader.metadata()); + + let mut metadata_array = Array::new(); + + for metadata_entry in metadata_entries { + let metadata_entry_array = match metadata_entry { + tracklib2::types::MetadataEntry::TrackType(track_type) => { + let mut metadata_entry_array = Array::new(); + + let (type_name, id) = match track_type { + tracklib2::types::TrackType::Trip(id) => { + (Symbol::new("trip"), Integer::from(*id)) + } + tracklib2::types::TrackType::Route(id) => { + (Symbol::new("route"), Integer::from(*id)) + } + tracklib2::types::TrackType::Segment(id) => { + (Symbol::new("segment"), Integer::from(*id)) + } + }; + + metadata_entry_array.push(Symbol::new("track_type")); + metadata_entry_array.push(type_name); + metadata_entry_array.push(id); + + metadata_entry_array + } + tracklib2::types::MetadataEntry::CreatedAt(created_at) => { + let mut metadata_entry_array = Array::new(); + + metadata_entry_array.push(Symbol::new("created_at")); + + let time_obj = Class::from_existing("Time") + .protect_send("at", &[Integer::from(*created_at).to_any_object()]) + .map_err(|e| VM::raise_ex(e)) + .unwrap() + .protect_send("utc", &[]) + .map_err(|e| VM::raise_ex(e)) + .unwrap(); + + metadata_entry_array.push(time_obj); + + metadata_entry_array + } + }; + + metadata_array.push(metadata_entry_array); + } + + metadata_array + } +); diff --git a/ruby_tracklib2/tracklib.gemspec b/ruby_tracklib2/tracklib.gemspec new file mode 100644 index 0000000..3fd52c4 --- /dev/null +++ b/ruby_tracklib2/tracklib.gemspec @@ -0,0 +1,28 @@ +require_relative 'lib/tracklib/version' + +Gem::Specification.new do |spec| + spec.name = "tracklib" + spec.version = Tracklib::VERSION + spec.authors = ["Dan Larkin"] + spec.email = ["dan@danlarkin.org"] + + spec.summary = "tracklib" + spec.description = "RWGPS tracklib ruby gem" + spec.homepage = "https://ridewithgps.com" + spec.licenses = ["Apache-2.0", "MIT"] + spec.files = ["tracklib.gemspec", + "Rakefile", + "Gemfile", + "lib/tracklib.rb", + "lib/tracklib/version.rb", + "Cargo.toml", + "Cargo.lock"] + spec.files += Dir["src/**/*.rs"] + + spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0") + spec.require_paths = ["lib"] + + spec.add_development_dependency "rspec" + + spec.add_dependency 'rutie', '~> 0.0.4' +end From abd5c17b4d38f8260e79ae76e08ff03a96a971cf Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Thu, 31 Mar 2022 14:19:47 -0700 Subject: [PATCH 052/113] ruby support for file_version and creator_version --- ruby_tracklib2/spec/reader_spec.rb | 6 ++++++ ruby_tracklib2/src/lib.rs | 2 ++ ruby_tracklib2/src/track_reader.rs | 14 ++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/ruby_tracklib2/spec/reader_spec.rb b/ruby_tracklib2/spec/reader_spec.rb index 3cf4c32..c7ff600 100644 --- a/ruby_tracklib2/spec/reader_spec.rb +++ b/ruby_tracklib2/spec/reader_spec.rb @@ -205,4 +205,10 @@ track_reader = TrackReader.new(data) expect(track_reader.metadata()).to eq([[:track_type, :segment, 5]]) end + + it "can read versions" do + track_reader = TrackReader.new(data) + expect(track_reader.file_version()).to eq(1) + expect(track_reader.creator_version()).to eq(0) + end end diff --git a/ruby_tracklib2/src/lib.rs b/ruby_tracklib2/src/lib.rs index 62f76ca..b44e6e8 100644 --- a/ruby_tracklib2/src/lib.rs +++ b/ruby_tracklib2/src/lib.rs @@ -7,5 +7,7 @@ pub extern "C" fn Init_Tracklib() { Class::new("TrackReader", None).define(|klass| { klass.def_self("new", track_reader::trackreader_new); klass.def("metadata", track_reader::trackreader_metadata); + klass.def("file_version", track_reader::trackreader_file_version); + klass.def("creator_version", track_reader::trackreader_creator_version); }); } diff --git a/ruby_tracklib2/src/track_reader.rs b/ruby_tracklib2/src/track_reader.rs index f3b16f9..9cbd9d0 100644 --- a/ruby_tracklib2/src/track_reader.rs +++ b/ruby_tracklib2/src/track_reader.rs @@ -87,5 +87,19 @@ methods!( } metadata_array + }, + fn trackreader_file_version() -> Integer { + Integer::from(u32::from( + rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| track_reader.file_version()), + )) + }, + fn trackreader_creator_version() -> Integer { + Integer::from(u32::from( + rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| track_reader.creator_version()), + )) } ); From 0eb84d5fa78fd95ba58a562463c6b17dc89a078c Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Fri, 1 Apr 2022 13:49:21 -0700 Subject: [PATCH 053/113] ruby support for reading data from sections --- ruby_tracklib2/spec/reader_spec.rb | 28 +++++- ruby_tracklib2/src/lib.rs | 6 ++ ruby_tracklib2/src/track_reader.rs | 142 ++++++++++++++++++++++++++++- tracklib2/src/read/track.rs | 6 +- 4 files changed, 179 insertions(+), 3 deletions(-) diff --git a/ruby_tracklib2/spec/reader_spec.rb b/ruby_tracklib2/spec/reader_spec.rb index c7ff600..31ca328 100644 --- a/ruby_tracklib2/spec/reader_spec.rb +++ b/ruby_tracklib2/spec/reader_spec.rb @@ -112,7 +112,7 @@ 0x8D, 0x79, 0x68, - + # Data Column 2 = Bool 0x01, # true 0x01, # true @@ -211,4 +211,30 @@ expect(track_reader.file_version()).to eq(1) expect(track_reader.creator_version()).to eq(0) end + + it "can iterate through sections" do + track_reader = TrackReader.new(data) + expect(track_reader.section_count()).to eq(2) + + expect(track_reader.section_encoding(0)).to eq(:standard) + expect(track_reader.section_schema(0)).to eq([["m", :i64], ["k", :bool], ["j", :string]]) + expect(track_reader.section_rows(0)).to eq(5) + expect(track_reader.section_data(0)).to eq([{"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}]) + + expect(track_reader.section_encoding(1)).to eq(:standard) + expect(track_reader.section_schema(1)).to eq([["a", :i64], ["b", :bool], ["c", :string]]) + expect(track_reader.section_rows(1)).to eq(3) + expect(track_reader.section_data(1)).to eq([{"a"=>1, "b"=>false, "c"=>"Ride"}, + {"a"=>2, "c"=>"with"}, + {"a"=>4, "b"=>true, "c"=>"GPS"}]) + end + + it "raises an exception for an invalid section index" do + track_reader = TrackReader.new(data) + expect {track_reader.section_encoding(2) }.to raise_error("Section does not exist") + end end diff --git a/ruby_tracklib2/src/lib.rs b/ruby_tracklib2/src/lib.rs index b44e6e8..b7dd05f 100644 --- a/ruby_tracklib2/src/lib.rs +++ b/ruby_tracklib2/src/lib.rs @@ -9,5 +9,11 @@ pub extern "C" fn Init_Tracklib() { klass.def("metadata", track_reader::trackreader_metadata); klass.def("file_version", track_reader::trackreader_file_version); klass.def("creator_version", track_reader::trackreader_creator_version); + + klass.def("section_count", track_reader::trackreader_section_count); + klass.def("section_encoding", track_reader::trackreader_section_encoding); + klass.def("section_schema", track_reader::trackreader_section_schema); + klass.def("section_rows", track_reader::trackreader_section_rows); + klass.def("section_data", track_reader::trackreader_section_data); }); } diff --git a/ruby_tracklib2/src/track_reader.rs b/ruby_tracklib2/src/track_reader.rs index 9cbd9d0..de39b67 100644 --- a/ruby_tracklib2/src/track_reader.rs +++ b/ruby_tracklib2/src/track_reader.rs @@ -1,6 +1,7 @@ use ouroboros::self_referencing; use rutie::{ - class, methods, wrappable_struct, AnyObject, Array, Class, Integer, Object, RString, Symbol, VM, + class, methods, wrappable_struct, AnyObject, Array, Boolean, Class, Float, Hash, Integer, + Object, RString, Symbol, VM, }; use tracklib2; @@ -101,5 +102,144 @@ methods!( .get_data(&*TRACK_READER_WRAPPER) .with_track_reader(|track_reader| track_reader.creator_version()), )) + }, + fn trackreader_section_count() -> Integer { + Integer::from( + u64::try_from( + rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| track_reader.section_count()), + ) + .map_err(|e| VM::raise(Class::from_existing("Exception"), "u64 != usize")) + .unwrap(), + ) + }, + fn trackreader_section_encoding(index: Integer) -> Symbol { + let ruby_index = index.map_err(|e| VM::raise_ex(e)).unwrap(); + let rust_index = usize::try_from(ruby_index.to_u64()) + .map_err(|e| VM::raise(Class::from_existing("Exception"), "u64 != usize")) + .unwrap(); + let encoding = rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| { + track_reader + .section(rust_index) + .map(|section| section.encoding()) + }) + .ok_or_else(|| VM::raise(Class::from_existing("Exception"), "Section does not exist")) + .unwrap(); + + Symbol::new(match encoding { + tracklib2::types::SectionEncoding::Standard => "standard", + }) + }, + fn trackreader_section_schema(index: Integer) -> Array { + let ruby_index = index.map_err(|e| VM::raise_ex(e)).unwrap(); + let rust_index = usize::try_from(ruby_index.to_u64()) + .map_err(|e| VM::raise(Class::from_existing("Exception"), "u64 != usize")) + .unwrap(); + let schema = rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| { + track_reader + .section(rust_index) + .map(|section| section.schema()) + }) + .ok_or_else(|| VM::raise(Class::from_existing("Exception"), "Section does not exist")) + .unwrap(); + + let mut schema_array = Array::new(); + + for field_def in schema.fields() { + let mut field_array = Array::new(); + field_array.push(RString::from(String::from(field_def.name()))); + field_array.push(Symbol::new(match field_def.data_type() { + tracklib2::schema::DataType::I64 => "i64", + tracklib2::schema::DataType::Bool => "bool", + tracklib2::schema::DataType::String => "string", + tracklib2::schema::DataType::F64 => "f64", + })); + schema_array.push(field_array); + } + + schema_array + }, + fn trackreader_section_rows(index: Integer) -> Integer { + let ruby_index = index.map_err(|e| VM::raise_ex(e)).unwrap(); + let rust_index = usize::try_from(ruby_index.to_u64()) + .map_err(|e| VM::raise(Class::from_existing("Exception"), "u64 != usize")) + .unwrap(); + let rows = rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| { + track_reader + .section(rust_index) + .map(|section| section.rows()) + }) + .ok_or_else(|| VM::raise(Class::from_existing("Exception"), "Section does not exist")) + .unwrap(); + + Integer::from( + u64::try_from(rows) + .map_err(|e| VM::raise(Class::from_existing("Exception"), "u64 != usize")) + .unwrap(), + ) + }, + fn trackreader_section_data(index: Integer) -> Array { + let ruby_index = index.map_err(|e| VM::raise_ex(e)).unwrap(); + let rust_index = usize::try_from(ruby_index.to_u64()) + .map_err(|e| VM::raise(Class::from_existing("Exception"), "u64 != usize")) + .unwrap(); + let data = rtself + .get_data(&*TRACK_READER_WRAPPER) + .with_track_reader(|track_reader| { + track_reader.section(rust_index).map(|section| { + let mut section_reader = section + .reader() + .map_err(|e| { + VM::raise(Class::from_existing("Exception"), &format!("{}", e)) + }) + .unwrap(); + + let mut data_array = Array::new(); + while let Some(columniter) = section_reader.open_column_iter() { + let mut row_hash = Hash::new(); + for row in columniter { + let (field_def, maybe_value) = row + .map_err(|e| { + VM::raise(Class::from_existing("Exception"), &format!("{}", e)) + }) + .unwrap(); + + if let Some(value) = maybe_value { + row_hash.store( + RString::from(String::from(field_def.name())), + match value { + tracklib2::types::FieldValue::I64(v) => { + Integer::new(v).to_any_object() + } + tracklib2::types::FieldValue::F64(v) => { + Float::new(v).to_any_object() + } + tracklib2::types::FieldValue::Bool(v) => { + Boolean::new(v).to_any_object() + } + tracklib2::types::FieldValue::String(v) => { + RString::from(String::from(v)).to_any_object() + } + }, + ); + } + } + data_array.push(row_hash); + } + + data_array + }) + }) + .ok_or_else(|| VM::raise(Class::from_existing("Exception"), "Section does not exist")) + .unwrap(); + + data } ); diff --git a/tracklib2/src/read/track.rs b/tracklib2/src/read/track.rs index 212e3d8..6b267cb 100644 --- a/tracklib2/src/read/track.rs +++ b/tracklib2/src/read/track.rs @@ -52,6 +52,10 @@ impl<'a> TrackReader<'a> { entries: &self.data_table, } } + + pub fn section_count(&self) -> usize { + self.data_table.len() + } } pub struct SectionIter<'a> { @@ -287,7 +291,7 @@ mod tests { assert_eq!(track.file_version(), 1); assert_eq!(track.creator_version(), 0); - + assert_eq!(track.section_count(), 2); assert_eq!(track.metadata().len(), 1); assert_matches!( From b3ecc6815027a10ed73fbbad9891338e23cbbac7 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 4 Apr 2022 12:05:25 -0700 Subject: [PATCH 054/113] Section::reader_for_schema now returns Result Previously, it returned Option> and would only give back a Some if all the requested fields were present. Now, in order to (hopefully?) improve usability, it will just skip over any requested fields that don't exist in the schema. --- tracklib2/src/read/section.rs | 115 +++++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 14 deletions(-) diff --git a/tracklib2/src/read/section.rs b/tracklib2/src/read/section.rs index 1903d35..d629f5b 100644 --- a/tracklib2/src/read/section.rs +++ b/tracklib2/src/read/section.rs @@ -51,7 +51,7 @@ impl<'a> Section<'a> { ) } - pub fn reader_for_schema(&self, schema: &Schema) -> Option> { + pub fn reader_for_schema(&self, schema: &Schema) -> Result { let schema_entries = schema .fields() .into_iter() @@ -64,16 +64,12 @@ impl<'a> Section<'a> { }) .collect::>(); - if schema_entries.len() == schema.fields().len() { - Some(SectionReader::new( - self.input, - schema_entries, - self.data_table_entry.schema_entries().len(), - self.data_table_entry.rows(), - )) - } else { - None - } + SectionReader::new( + self.input, + schema_entries, + self.data_table_entry.schema_entries().len(), + self.data_table_entry.rows(), + ) } } @@ -545,18 +541,109 @@ mod tests { // Missing field assert_matches!(section.reader_for_schema(&Schema::with_fields(vec![ FieldDefinition::new("z", DataType::Bool), - ])), None); + ])), Ok(mut section_reader) => { + // Row 1 + assert_eq!(section_reader.rows_remaining(), 3); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + assert_eq!(column_iter.count(), 0); + }); + + // Row 2 + assert_eq!(section_reader.rows_remaining(), 2); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + assert_eq!(column_iter.count(), 0); + }); + + // Row 3 + assert_eq!(section_reader.rows_remaining(), 1); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + assert_eq!(column_iter.count(), 0); + }); + + // Trying to get another row will return nothing + assert_eq!(section_reader.rows_remaining(), 0); + assert_matches!(section_reader.open_column_iter(), None); + }); // Field exists but we're asking for the wrong type assert_matches!(section.reader_for_schema(&Schema::with_fields(vec![ FieldDefinition::new("b", DataType::I64), - ])), None); + ])), Ok(mut section_reader) => { + // Row 1 + assert_eq!(section_reader.rows_remaining(), 3); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + assert_eq!(column_iter.count(), 0); + }); + + // Row 2 + assert_eq!(section_reader.rows_remaining(), 2); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + assert_eq!(column_iter.count(), 0); + }); + + // Row 3 + assert_eq!(section_reader.rows_remaining(), 1); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + assert_eq!(column_iter.count(), 0); + }); + + // Trying to get another row will return nothing + assert_eq!(section_reader.rows_remaining(), 0); + assert_matches!(section_reader.open_column_iter(), None); + }); + + + // Only one of these fields exists + assert_matches!(section.reader_for_schema(&Schema::with_fields(vec![ + FieldDefinition::new("b", DataType::Bool), + FieldDefinition::new("z", DataType::Bool), + ])), Ok(mut section_reader) => { + // Row 1 + assert_eq!(section_reader.rows_remaining(), 3); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 1); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); + assert_eq!(field_value, &Some(FieldValue::Bool(false))); + }); + }); + + // Row 2 + assert_eq!(section_reader.rows_remaining(), 2); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 1); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); + assert_eq!(field_value, &None); + }); + }); + + // Row 3 + assert_eq!(section_reader.rows_remaining(), 1); + assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { + let values = column_iter.collect::>(); + assert_eq!(values.len(), 1); + assert_matches!(&values[0], Ok((field_definition, field_value)) => { + assert_eq!(*field_definition, data_table_entries[0].schema_entries()[1].field_definition()); + assert_eq!(*field_definition, &FieldDefinition::new("b", DataType::Bool)); + assert_eq!(field_value, &Some(FieldValue::Bool(true))); + }); + }); + + // Trying to get another row will return nothing + assert_eq!(section_reader.rows_remaining(), 0); + assert_matches!(section_reader.open_column_iter(), None); + }); // Both of these fields exist assert_matches!(section.reader_for_schema(&Schema::with_fields(vec![ FieldDefinition::new("b", DataType::Bool), FieldDefinition::new("f", DataType::F64), - ])), Some(Ok(mut section_reader)) => { + ])), Ok(mut section_reader) => { // Row 1 assert_eq!(section_reader.rows_remaining(), 3); assert_matches!(section_reader.open_column_iter(), Some(column_iter) => { From e6763d0a51a85143ab337aec9190e2486fa8dacd Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 4 Apr 2022 12:10:02 -0700 Subject: [PATCH 055/113] fix roundtrip tests to use TrackReader::new --- tracklib2/tests/roundtrip.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tracklib2/tests/roundtrip.rs b/tracklib2/tests/roundtrip.rs index ba09db0..f6afdc9 100644 --- a/tracklib2/tests/roundtrip.rs +++ b/tracklib2/tests/roundtrip.rs @@ -40,7 +40,7 @@ mod tests { assert!(write_track(&mut buf, &[], &[section]).is_ok()); // Read - let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let track_reader = TrackReader::new(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { let mut section_reader = section.reader().unwrap(); @@ -80,7 +80,7 @@ mod tests { assert!(write_track(&mut buf, &[], &[section]).is_ok()); // Read - let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let track_reader = TrackReader::new(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { let mut section_reader = section.reader().unwrap(); @@ -120,7 +120,7 @@ mod tests { assert!(write_track(&mut buf, &[], &[section]).is_ok()); // Read - let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let track_reader = TrackReader::new(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { let mut section_reader = section.reader().unwrap(); @@ -167,7 +167,7 @@ mod tests { assert!(write_track(&mut buf, &[], &[section]).is_ok()); // Read - let track_reader = TrackReader::from_bytes(&buf).unwrap(); + let track_reader = TrackReader::new(&buf).unwrap(); let mut read_values: Vec = vec![]; for section in track_reader.sections() { let mut section_reader = section.reader().unwrap(); From e507c2b2e66ee6ed35e6e1e16cf0322e04b68d84 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Mon, 4 Apr 2022 12:11:25 -0700 Subject: [PATCH 056/113] ruby support for predicate pushdown --- ruby_tracklib2/spec/reader_spec.rb | 49 ++++++++++++++++++++++++++++- ruby_tracklib2/spec/spec_helper.rb | 2 +- ruby_tracklib2/src/lib.rs | 1 + ruby_tracklib2/src/schema.rs | 50 ++++++++++++++++++++++++++++++ ruby_tracklib2/src/track_reader.rs | 24 +++++++++----- 5 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 ruby_tracklib2/src/schema.rs diff --git a/ruby_tracklib2/spec/reader_spec.rb b/ruby_tracklib2/spec/reader_spec.rb index 31ca328..04ab999 100644 --- a/ruby_tracklib2/spec/reader_spec.rb +++ b/ruby_tracklib2/spec/reader_spec.rb @@ -235,6 +235,53 @@ it "raises an exception for an invalid section index" do track_reader = TrackReader.new(data) - expect {track_reader.section_encoding(2) }.to raise_error("Section does not exist") + expect { track_reader.section_encoding(2) }.to raise_error("Section does not exist") + end + + it "can select a subset of fields" do + track_reader = TrackReader.new(data) + # field that doesn't exist + expect(track_reader.section_data(0, [["z", :i64]])).to eq([{}, + {}, + {}, + {}, + {}]) + # field that does exist, but it's a different type + expect(track_reader.section_data(0, [["m", :bool]])).to eq([{}, + {}, + {}, + {}, + {}]) + # both of these fields exist + expect(track_reader.section_data(0, [["m", :i64], ["j", :string]])).to eq([{"m"=>42, "j"=>"hey"}, + {"m"=>42, "j"=>"hey"}, + {"m"=>42, "j"=>"hey"}, + {"m"=>42, "j"=>"hey"}, + {"m"=>42, "j"=>"hey"}]) + # one field that exists and one that doesn't + expect(track_reader.section_data(0, [["m", :i64], ["z", :i64]])).to eq([{"m"=>42}, + {"m"=>42}, + {"m"=>42}, + {"m"=>42}, + {"m"=>42}]) + + # Error Conditions: + + # invalid schema type - this is just treated as not passing in a schema + expect(track_reader.section_data(0, "Foo")).to eq([{"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}, + {"m"=>42, "k"=>true, "j"=>"hey"}]) + # invalid schema entry type + expect { track_reader.section_data(0, ["Foo"]) }.to raise_error + # invalid schema entry length + expect { track_reader.section_data(0, [["Foo", :bool, 5]]) }.to raise_error + # invalid schema entry name type + expect { track_reader.section_data(0, [[:Foo, :bool]]) }.to raise_error + # invalid schema entry data type + expect { track_reader.section_data(0, [["Foo", "bool"]]) }.to raise_error + # invalid schema entry data type value + expect { track_reader.section_data(0, [["Foo", :invalid]]) }.to raise_error end end diff --git a/ruby_tracklib2/spec/spec_helper.rb b/ruby_tracklib2/spec/spec_helper.rb index 9c1e978..bed7221 100644 --- a/ruby_tracklib2/spec/spec_helper.rb +++ b/ruby_tracklib2/spec/spec_helper.rb @@ -4,5 +4,5 @@ require 'tracklib' RSpec::configure do |config| - + RSpec::Expectations.configuration.on_potential_false_positives = :nothing end diff --git a/ruby_tracklib2/src/lib.rs b/ruby_tracklib2/src/lib.rs index b7dd05f..17f6570 100644 --- a/ruby_tracklib2/src/lib.rs +++ b/ruby_tracklib2/src/lib.rs @@ -1,3 +1,4 @@ +mod schema; mod track_reader; use rutie::{Class, Object}; diff --git a/ruby_tracklib2/src/schema.rs b/ruby_tracklib2/src/schema.rs new file mode 100644 index 0000000..3ab3496 --- /dev/null +++ b/ruby_tracklib2/src/schema.rs @@ -0,0 +1,50 @@ +use rutie::{Array, Class, Object, RString, Symbol, VM}; + +pub(crate) fn create_schema(ruby_schema: Array) -> tracklib2::schema::Schema { + let fields = ruby_schema + .into_iter() + .map(|ele| { + let ruby_schema_entry = ele + .try_convert_to::() + .map_err(|e| VM::raise_ex(e)) + .unwrap(); + + if ruby_schema_entry.length() == 2 { + let ruby_field_name = ruby_schema_entry + .at(0) + .try_convert_to::() + .map_err(|e| VM::raise_ex(e)) + .unwrap(); + let ruby_data_type = ruby_schema_entry + .at(1) + .try_convert_to::() + .map_err(|e| VM::raise_ex(e)) + .unwrap(); + + let data_type = match ruby_data_type.to_str() { + "i64" => tracklib2::schema::DataType::I64, + "f64" => tracklib2::schema::DataType::F64, + "bool" => tracklib2::schema::DataType::Bool, + "string" => tracklib2::schema::DataType::String, + val @ _ => { + VM::raise( + Class::from_existing("Exception"), + &format!("Schema Data Type '{val}' unknown"), + ); + unreachable!(); + } + }; + + tracklib2::schema::FieldDefinition::new(ruby_field_name.to_string(), data_type) + } else { + VM::raise( + Class::from_existing("Exception"), + "Schema array entries must have length 2", + ); + unreachable!(); + } + }) + .collect::>(); + + tracklib2::schema::Schema::with_fields(fields) +} diff --git a/ruby_tracklib2/src/track_reader.rs b/ruby_tracklib2/src/track_reader.rs index de39b67..5027fc0 100644 --- a/ruby_tracklib2/src/track_reader.rs +++ b/ruby_tracklib2/src/track_reader.rs @@ -185,7 +185,7 @@ methods!( .unwrap(), ) }, - fn trackreader_section_data(index: Integer) -> Array { + fn trackreader_section_data(index: Integer, schema: Array) -> Array { let ruby_index = index.map_err(|e| VM::raise_ex(e)).unwrap(); let rust_index = usize::try_from(ruby_index.to_u64()) .map_err(|e| VM::raise(Class::from_existing("Exception"), "u64 != usize")) @@ -194,12 +194,22 @@ methods!( .get_data(&*TRACK_READER_WRAPPER) .with_track_reader(|track_reader| { track_reader.section(rust_index).map(|section| { - let mut section_reader = section - .reader() - .map_err(|e| { - VM::raise(Class::from_existing("Exception"), &format!("{}", e)) - }) - .unwrap(); + let mut section_reader = if let Ok(ruby_schema) = schema { + let tracklib_schema = crate::schema::create_schema(ruby_schema); + section + .reader_for_schema(&tracklib_schema) + .map_err(|e| { + VM::raise(Class::from_existing("Exception"), &format!("{}", e)) + }) + .unwrap() + } else { + section + .reader() + .map_err(|e| { + VM::raise(Class::from_existing("Exception"), &format!("{}", e)) + }) + .unwrap() + }; let mut data_array = Array::new(); while let Some(columniter) = section_reader.open_column_iter() { From efc5f747b15f4e8c4adae3273b8783c0454c0c61 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 5 Apr 2022 14:22:37 -0700 Subject: [PATCH 057/113] ruby lib: put TrackReader inside the Tracklib module --- ruby_tracklib2/spec/reader_spec.rb | 12 ++++++------ ruby_tracklib2/src/lib.rs | 26 +++++++++++++++----------- ruby_tracklib2/src/track_reader.rs | 6 ++++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/ruby_tracklib2/spec/reader_spec.rb b/ruby_tracklib2/spec/reader_spec.rb index 04ab999..3503afe 100644 --- a/ruby_tracklib2/spec/reader_spec.rb +++ b/ruby_tracklib2/spec/reader_spec.rb @@ -200,20 +200,20 @@ 0x48 ].pack("c*") -describe TrackReader do +describe Tracklib::TrackReader do it "can read metadata" do - track_reader = TrackReader.new(data) + track_reader = Tracklib::TrackReader.new(data) expect(track_reader.metadata()).to eq([[:track_type, :segment, 5]]) end it "can read versions" do - track_reader = TrackReader.new(data) + track_reader = Tracklib::TrackReader.new(data) expect(track_reader.file_version()).to eq(1) expect(track_reader.creator_version()).to eq(0) end it "can iterate through sections" do - track_reader = TrackReader.new(data) + track_reader = Tracklib::TrackReader.new(data) expect(track_reader.section_count()).to eq(2) expect(track_reader.section_encoding(0)).to eq(:standard) @@ -234,12 +234,12 @@ end it "raises an exception for an invalid section index" do - track_reader = TrackReader.new(data) + track_reader = Tracklib::TrackReader.new(data) expect { track_reader.section_encoding(2) }.to raise_error("Section does not exist") end it "can select a subset of fields" do - track_reader = TrackReader.new(data) + track_reader = Tracklib::TrackReader.new(data) # field that doesn't exist expect(track_reader.section_data(0, [["z", :i64]])).to eq([{}, {}, diff --git a/ruby_tracklib2/src/lib.rs b/ruby_tracklib2/src/lib.rs index 17f6570..c0f9d6a 100644 --- a/ruby_tracklib2/src/lib.rs +++ b/ruby_tracklib2/src/lib.rs @@ -1,20 +1,24 @@ mod schema; mod track_reader; -use rutie::{Class, Object}; +use rutie::{Module, Object}; #[allow(non_snake_case)] #[no_mangle] pub extern "C" fn Init_Tracklib() { - Class::new("TrackReader", None).define(|klass| { - klass.def_self("new", track_reader::trackreader_new); - klass.def("metadata", track_reader::trackreader_metadata); - klass.def("file_version", track_reader::trackreader_file_version); - klass.def("creator_version", track_reader::trackreader_creator_version); + Module::from_existing("Tracklib").define(|module| { + module + .define_nested_class("TrackReader", None) + .define(|class| { + class.def_self("new", track_reader::trackreader_new); + class.def("metadata", track_reader::trackreader_metadata); + class.def("file_version", track_reader::trackreader_file_version); + class.def("creator_version", track_reader::trackreader_creator_version); - klass.def("section_count", track_reader::trackreader_section_count); - klass.def("section_encoding", track_reader::trackreader_section_encoding); - klass.def("section_schema", track_reader::trackreader_section_schema); - klass.def("section_rows", track_reader::trackreader_section_rows); - klass.def("section_data", track_reader::trackreader_section_data); + class.def("section_count", track_reader::trackreader_section_count); + class.def("section_encoding", track_reader::trackreader_section_encoding); + class.def("section_schema", track_reader::trackreader_section_schema); + class.def("section_rows", track_reader::trackreader_section_rows); + class.def("section_data", track_reader::trackreader_section_data); + }); }); } diff --git a/ruby_tracklib2/src/track_reader.rs b/ruby_tracklib2/src/track_reader.rs index 5027fc0..b2267c9 100644 --- a/ruby_tracklib2/src/track_reader.rs +++ b/ruby_tracklib2/src/track_reader.rs @@ -1,7 +1,7 @@ use ouroboros::self_referencing; use rutie::{ class, methods, wrappable_struct, AnyObject, Array, Boolean, Class, Float, Hash, Integer, - Object, RString, Symbol, VM, + Module, Object, RString, Symbol, VM, }; use tracklib2; @@ -33,7 +33,9 @@ methods!( .unwrap() }); - Class::from_existing("TrackReader").wrap_data(wrapper, &*TRACK_READER_WRAPPER) + Module::from_existing("Tracklib") + .get_nested_class("TrackReader") + .wrap_data(wrapper, &*TRACK_READER_WRAPPER) }, fn trackreader_metadata() -> Array { let metadata_entries = rtself From 11fb39c4b13aaaca25d13b57eaf60847f8f96e35 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 5 Apr 2022 14:35:11 -0700 Subject: [PATCH 058/113] ruby lib: write metadata --- ruby_tracklib2/spec/writer_spec.rb | 102 +++++++++++++++ ruby_tracklib2/src/lib.rs | 7 ++ ruby_tracklib2/src/write.rs | 193 +++++++++++++++++++++++++++++ 3 files changed, 302 insertions(+) create mode 100644 ruby_tracklib2/spec/writer_spec.rb create mode 100644 ruby_tracklib2/src/write.rs diff --git a/ruby_tracklib2/spec/writer_spec.rb b/ruby_tracklib2/spec/writer_spec.rb new file mode 100644 index 0000000..d8491da --- /dev/null +++ b/ruby_tracklib2/spec/writer_spec.rb @@ -0,0 +1,102 @@ +require "spec_helper" + +describe Tracklib::Section do + it "can write metadata" do + # empty metadata + expect(Tracklib::write_track([], []) + .unpack("C*")[24..]) + .to eq([0x00, # empty metadata table + 0x40, + 0xBF, + + 0x00, # empty data table + 0x40, + 0xBF]) + + # track_type + expect(Tracklib::write_track([[:track_type, :route, 64]], []) + .unpack("C*")[24..]) + .to eq([0x01, # metadata table len = 1 + 0x00, # entry type = track_type + 0x05, # two byte entry size = 5 + 0x00, + 0x01, # track type: route = 0x01 + 0x40, # four byte route ID = 64 + 0x00, + 0x00, + 0x00, + 0x85, # crc + 0x9F, + + 0x00, # empty data table + 0x40, + 0xBF]) + + # created_at + expect(Tracklib::write_track([[:created_at, Time.new(1970, 1, 1, 0, 0, 0, "UTC")]], []) + .unpack("C*")[24..]) + .to eq([0x01, # metadata table len = 1 + 0x01, # entry type = created_at + 0x08, # two byte entry size = 8 + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0xE3, # crc + 0x28, + + 0x00, # empty data table + 0x40, + 0xBF]) + + # both + expect(Tracklib::write_track([[:track_type, :trip, 20], + [:created_at, Time.new(1970, 1, 1, 0, 0, 0, "UTC")]], + []) + .unpack("C*")[24..]) + .to eq([0x02, # two metadata entries + 0x00, # entry type: track_type = 0x00 + 0x05, # two byte entry size = 5 + 0x00, + 0x00, # track type: trip = 0x00 + 0x14, # four byte trip ID = 20 + 0x00, + 0x00, + 0x00, + 0x01, # entry type: created_at = 0x01 + 0x08, # two byte entry size = 8 + 0x00, + 0x00, # eight byte timestamp: zero seconds elapsed + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x23, # crc + 0xD2, + + 0x00, # empty data table + 0x40, + 0xBF]) + + # unknown type + expect { Tracklib::write_track([[:foo, 25]], []) } .to raise_error "Metadata Type 'foo' unknown" + + # invalid length + expect { Tracklib::write_track([[:created_at, Time.now, "foo"]], []) } .to raise_error "Metadata Entries for 'created_at' must have length 2" + + # invalid args + expect { Tracklib::write_track([[]], []) } .to raise_error "Invalid Metadata Entry" + expect { Tracklib::write_track(["Foo"], []) } .to raise_error "Error converting to Array" + expect { Tracklib::write_track([[:created_at, "foo"]], []) } .to raise_error "Error converting to Time" + expect { Tracklib::write_track([[:track_type, :foo, 5]], []) } .to raise_error "Metadata Entry Track Type 'foo' unknown" + expect { Tracklib::write_track([[:track_type, :route, "foo"]], []) } .to raise_error "Error converting to Integer" + end +end diff --git a/ruby_tracklib2/src/lib.rs b/ruby_tracklib2/src/lib.rs index c0f9d6a..16e8c86 100644 --- a/ruby_tracklib2/src/lib.rs +++ b/ruby_tracklib2/src/lib.rs @@ -1,5 +1,6 @@ mod schema; mod track_reader; +mod write; use rutie::{Module, Object}; #[allow(non_snake_case)] @@ -20,5 +21,11 @@ pub extern "C" fn Init_Tracklib() { class.def("section_rows", track_reader::trackreader_section_rows); class.def("section_data", track_reader::trackreader_section_data); }); + + module.define_nested_class("Section", None).define(|class| { + class.def_self("new", write::section_new); + }); + + module.define_module_function("write_track", write::write_track); }); } diff --git a/ruby_tracklib2/src/write.rs b/ruby_tracklib2/src/write.rs new file mode 100644 index 0000000..83c2e4b --- /dev/null +++ b/ruby_tracklib2/src/write.rs @@ -0,0 +1,193 @@ +use rutie::{ + class, methods, module, wrappable_struct, AnyObject, Array, Boolean, Class, Encoding, Float, + Hash, Integer, Module, NilClass, Object, RString, Symbol, VerifiedObject, VM, +}; + +pub struct SectionInner { + inner: tracklib2::write::section::Section, +} + +wrappable_struct!(SectionInner, SectionWrapper, SECTION_WRAPPER); + +class!(Section); + +methods!( + Section, + rtself, + fn section_new(encoding: Symbol, schema: Array, data: Array) -> AnyObject { + let tracklib_schema = + crate::schema::create_schema(schema.map_err(|e| VM::raise_ex(e)).unwrap()); + let tracklib_encoding = match encoding.map_err(|e| VM::raise_ex(e)).unwrap().to_str() { + "standard" => tracklib2::types::SectionEncoding::Standard, + val @ _ => { + VM::raise( + Class::from_existing("Exception"), + &format!("SectionEncoding '{val}' unknown"), + ); + unreachable!(); + } + }; + let mut tracklib_section = + tracklib2::write::section::Section::new(tracklib_encoding, tracklib_schema); + + Module::from_existing("Tracklib") + .get_nested_class("Section") + .wrap_data( + SectionInner { + inner: tracklib_section, + }, + &*SECTION_WRAPPER, + ) + }, +); + +impl VerifiedObject for Section { + fn is_correct_type(object: &T) -> bool { + object.class() == Module::from_existing("Tracklib").get_nested_class("Section") + } + + fn error_message() -> &'static str { + "Error converting to Section" + } +} + +class!(Time); +impl VerifiedObject for Time { + fn is_correct_type(object: &T) -> bool { + object.class() == Class::from_existing("Time") + } + + fn error_message() -> &'static str { + "Error converting to Time" + } +} + +module!(Tracklib); + +methods!( + Tracklib, + rtself, + fn write_track(metadata: Array, sections: Array) -> RString { + let metadata_array = metadata.map_err(|e| VM::raise_ex(e)).unwrap(); + + let metadata_entries = metadata_array + .into_iter() + .map(|metadata_ele| { + let metadata_ele_array = metadata_ele + .try_convert_to::() + .map_err(|e| VM::raise_ex(e)) + .unwrap(); + if metadata_ele_array.length() >= 1 { + let metadata_type = metadata_ele_array + .at(0) + .try_convert_to::() + .map_err(|e| VM::raise_ex(e)) + .unwrap(); + match metadata_type.to_str() { + "track_type" => { + if metadata_ele_array.length() == 3 { + let track_type_symbol = metadata_ele_array + .at(1) + .try_convert_to::() + .map_err(|e| VM::raise_ex(e)) + .unwrap(); + let track_id = metadata_ele_array + .at(2) + .try_convert_to::() + .map_err(|e| VM::raise_ex(e)) + .unwrap() + .to_u32(); + + let track_type = match track_type_symbol.to_str() { + "route" => tracklib2::types::TrackType::Route(track_id), + "trip" => tracklib2::types::TrackType::Trip(track_id), + "segment" => tracklib2::types::TrackType::Segment(track_id), + val @ _ => { + VM::raise( + Class::from_existing("Exception"), + &format!("Metadata Entry Track Type '{val}' unknown"), + ); + unreachable!(); + } + }; + + tracklib2::types::MetadataEntry::TrackType(track_type) + } else { + VM::raise( + Class::from_existing("Exception"), + "Metadata Entries for 'track_type' must have length 3", + ); + unreachable!(); + } + } + "created_at" => { + if metadata_ele_array.length() == 2 { + let created_at_time_obj = metadata_ele_array + .at(1) + .try_convert_to::