Add --compression (-c) option to specify output data compression

lahmatiy · lahmatiy · commit 013ec3ea3c3e · 2025-08-16T18:32:28.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,7 @@
 ## next
 
 - Added `gzip` and `deflate` compression support for input data
+- Added `--compression` (`-c`) option to specify output data compression (`gzip` or `deflate`). When omitted, `gzip` is used by default. For example, `jora -c -e jsonxl` will output gzipped JSONXL (the most compact size combination).
 - Updated jsonxl
     - Fixed an edge case for signed numbers in the range ±[MAX_SAFE_INTEGER/2 … MAX_SAFE_INTEGER].
     - Removed the limitation on total encoded string length exceeding the maximum string length (~500 MB for V8)
diff --git a/fixtures/data-no-format.json b/fixtures/data-no-format.json
@@ -0,0 +1 @@
+{"name":"jora-cli","version":"2.0.0","description":"Command line interface for Jora","keywords":["cli","jora"],"maintainers":[{"name":"Roman Dvornov","email":"rdvornov@gmail.com","github-username":"lahmatiy"}],"license":"MIT","repository":"discoveryjs/jora-cli","bin":{"jora":"./bin/jora"},"type":"module","main":"./src/index.js","scripts":{"test":"mocha --reporter progress","lint":"eslint src/*.js test/*.js","lint-and-test":"npm run lint && npm test","coverage":"c8 --reporter=lcovonly npm test","prepublishOnly":"npm run lint-and-test"},"dependencies":{"@discoveryjs/json-ext":"^0.6.2","ansi-styles":"^6.2.1","clap":"^3.1.1","jora":"1.0.0-beta.13","jora-sandbox":"^1.3.0","open":"^10.1.0","supports-color":"^9.4.0","tempfile":"^5.0.0"},"devDependencies":{"c8":"^10.1.2","eslint":"^8.57.1","mocha":"^10.7.3"},"engines":{"node":">=18.0.0"},"files":["bin","utils","index.js"]}
diff --git a/fixtures/data-no-format.json.gz b/fixtures/data-no-format.json.gz
diff --git a/fixtures/data.json.gz b/fixtures/data.json.gz
diff --git a/fixtures/data.jsonxl.gz b/fixtures/data.jsonxl.gz
diff --git a/src/index.js b/src/index.js
@@ -67,18 +67,38 @@ function normFormat(value) {
     throw new cli.Error(`Bad value "${value}" for ${this.long} option, supported values: ${encodings.join(', ')}`);
 }
 
+function normCompression(value) {
+    if (value === false) {
+        return false;
+    }
+
+    if (value === undefined) {
+        return 'gzip';
+    }
+
+    if (compressions.includes(value)) {
+        return value;
+    }
+
+    throw new cli.Error(`Bad value "${value}" for ${this.long} option, supported values: ${compressions.join(', ')}`);
+}
+
+function normPretty(value) {
+    return value === undefined ? 4 : Number(value) || false;
+}
+
 const encodings = ['json', 'jsonxl'];
+const compressions = ['gzip', 'deflate'];
 const command = cli.command('jora [query]')
     .version('', '', '', outputVersion)
     .option('-q, --query <query>', 'Jora query or path to a query file with extension .jora', normFilepath)
     .option('-i, --input <filename>', 'Input file', normFilepath)
     .option('-o, --output <filename>', 'Output file (outputs to stdout if not set)')
     .option('-e, --encoding <encoding>', 'Output encoding: json (default), jsonxl (snapshot9)', normFormat, 'json')
+    .option('-c, --compression [compression]', 'Output compression: gzip (default when [compression] is omitted), deflate', normCompression, false)
     .option('--dry-run', 'Don\'t output result, only report what it would have done')
     .option('-f, --force', 'Force overwriting output file')
-    .option('-p, --pretty [indent]', 'Pretty print with optionally specified indentation (4 spaces by default)', value =>
-        value === undefined ? 4 : Number(value) || false
-    , false)
+    .option('-p, --pretty [indent]', 'Pretty-prints the output using the specified indentation (defaults to 4 spaces if omitted)', normPretty, false)
     .option('--no-color', 'Suppress color output')
     // .option('-s, --sandbox', 'Output data and query in sandbox')
     .option('--verbose', 'Output debug info about actions')
@@ -114,10 +134,11 @@ const command = cli.command('jora [query]')
         const query = extractQuery(options.query || args[0]);
         const queryFn = prepareQuery(query);
         const resultData = performQuery(queryFn, input.data, undefined);
-        const encoding = options.encoding;
+        const { compression, encoding } = options;
 
         writer.log();
         await writeToDestination(resultData, {
+            compression,
             encoding,
             displayInfo,
             color: options.color && colorsSupported,
@@ -128,16 +149,6 @@ const command = cli.command('jora [query]')
             inputPath: options.input
         }, setStageProgress);
 
-        // if (options.output) {
-        //     pipeline(
-        //         stringifyChunked(result, null, options.pretty),
-        //         fs.createWriteStream(options.outputFile)
-        //     );
-        // } else {
-        //     const serializedResult = JSON.stringify(result ?? null, null, options.pretty);
-        //     console.log(options.color ? colorize(serializedResult) : serializedResult);
-        // }
-
         setStageProgress('done', { time: Date.now() - startTime });
     });
 
diff --git a/src/reporter.js b/src/reporter.js
@@ -97,14 +97,18 @@ export function createDefaultReporter({ log, logTemp }) {
 
             case 'input-encoding': {
                 if (params.compression) {
-                    log(`  Compression: ${params.compression} (auto detected)`);
+                    log(`  Compression: ${params.compression}`);
                 }
 
-                log(`  Encoding: ${params.encoding || '<unknown>'} (auto detected)`);
+                log(`  Encoding: ${params.encoding || '<unknown>'}`);
                 break;
             }
 
             case 'output-encoding':
+                if (params.compression) {
+                    log(`  Compression: ${params.compression}`);
+                }
+
                 log(`  Encoding: ${params.encoding || '<unknown>'}${params.autoEncoding ? ' (auto selected)' : ''}`);
                 break;
 
diff --git a/src/write.js b/src/write.js
@@ -1,5 +1,6 @@
 import fs from 'node:fs';
 import { pipeline } from 'node:stream/promises';
+import { createGzip, createDeflate } from 'node:zlib';
 import { encode } from './tmp/jsonxl-snapshot9.js';
 import { colorize } from './colorize.js';
 import { stringifyChunked, stringifyInfo } from '@discoveryjs/json-ext';
@@ -9,19 +10,24 @@ const now = typeof performace !== 'undefined' && typeof performance.now === 'fun
 const stringBytes = typeof Buffer === 'function' && typeof Buffer.byteLength === 'function'
     ? Buffer.byteLength
     : (str) => str.length; // incorrect but fast fallback
+const compressionTransforms = {
+    gzip: createGzip,
+    deflate: createDeflate
+};
 
 function* createChunkIterator(data, chunkSize = 64 * 1024) {
     for (let offset = 0; offset < data.length; offset += chunkSize) {
-        yield data.subarray(offset, offset + chunkSize);
+        yield Buffer.from(data.subarray(offset, offset + chunkSize));
     }
 }
 
 async function writeIntoStream(stream, data, options, setStageProgress = () => {}) {
-    const { autoEncoding, encoding } = options;
+    const { autoEncoding, encoding, compression } = options;
+    const compressionTransform = compressionTransforms[compression];
     let payload;
     let totalSize;
 
-    setStageProgress('output-encoding', { autoEncoding, encoding });
+    setStageProgress('output-encoding', { autoEncoding, encoding, compression });
 
     switch (encoding) {
         case 'jsonxl': {
@@ -58,6 +64,9 @@ async function writeIntoStream(stream, data, options, setStageProgress = () => {
         const endNewline = encoding !== 'jsonxl';
         const applyColorize = encoding === 'json' && options.color;
         const buffer = [];
+        const pipelineDest = compressionTransform
+            ? [compressionTransform(), stream]
+            : [stream];
 
         await pipeline(async function* () {
             if (isStdStream) {
@@ -89,15 +98,15 @@ async function writeIntoStream(stream, data, options, setStageProgress = () => {
             if (applyColorize) {
                 yield colorize(buffer.join(''));
             }
+        }, ...pipelineDest, { end: !isStdStream });
 
-            if (isStdStream && endNewline) {
-                yield'\n';
+        if (isStdStream) {
+            if (endNewline) {
+                stream.write('\n');
             }
 
-            if (isStdStream) {
-                setStageProgress('finish-stdout', { newline: !endNewline });
-            }
-        }, stream, { end: !isStdStream });
+            setStageProgress('finish-stdout', { newline: !endNewline });
+        }
     } else {
         // dry run
         switch (encoding) {
diff --git a/test/test.js b/test/test.js
@@ -32,7 +32,10 @@ const fixtureDir = path.join(__dirname, '../fixtures');
 const queryFilename = path.join(fixtureDir, 'query.jora');
 const packageJson = fixtureFile('../package.json');
 const fixtureJson = fixtureFile('data.json');
+const fixtureJsonGz = fixtureFile('data.json.gz');
+const fixtureJsonNoFormatGz = fixtureFile('data-no-format.json.gz');
 const fixtureJsonxl = fixtureFile('data.jsonxl');
+const fixtureJsonxlGz = fixtureFile('data.jsonxl.gz');
 const colorFixture = fixtureFile('color-output.json');
 const colorFixtureExpected = fixtureFile('color-output.expected').text.trim();
 const colorFixtureExpectedCompact = fixtureFile('color-output.compact.expected').text.trim();
@@ -237,6 +240,40 @@ describe('jsonxl', () => {
     );
 });
 
+describe('compression', function() {
+    describe('gzip', () => {
+        it('gziped JSON input', () =>
+            run()
+                .input(fixtureJsonGz.raw)
+                .output(fixtureJson.string)
+        );
+
+        it('gziped JSON output', () =>
+            run('-c')
+                .input(fixtureJson.text)
+                .output(fixtureJsonNoFormatGz.raw)
+        );
+
+        it('gziped formatted JSON output', () =>
+            run('-c', '-p')
+                .input(fixtureJson.text)
+                .output(fixtureJsonGz.raw)
+        );
+
+        it('gziped JSONXL input', () =>
+            run()
+                .input(fixtureJsonxlGz.raw)
+                .output(fixtureJsonxl.string)
+        );
+
+        it('gziped JSONXL output', () =>
+            run('-c', '-e', 'jsonxl')
+                .input(fixtureJsonxl.raw)
+                .output(fixtureJsonxlGz.raw)
+        );
+    });
+});
+
 describe('pretty print', function() {
     it('indentation should be 4 spaces by default', () =>
         run('dependencies.keys()', '-p')

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+{"name":"jora-cli","version":"2.0.0","description":"Command line interface for Jora","keywords":["cli","jora"],"maintainers":[{"name":"Roman Dvornov","email":"rdvornov@gmail.com","github-username":"lahmatiy"}],"license":"MIT","repository":"discoveryjs/jora-cli","bin":{"jora":"./bin/jora"},"type":"module","main":"./src/index.js","scripts":{"test":"mocha --reporter progress","lint":"eslint src/.js test/.js","lint-and-test":"npm run lint && npm test","coverage":"c8 --reporter=lcovonly npm test","prepublishOnly":"npm run lint-and-test"},"dependencies":{"@discoveryjs/json-ext":"^0.6.2","ansi-styles":"^6.2.1","clap":"^3.1.1","jora":"1.0.0-beta.13","jora-sandbox":"^1.3.0","open":"^10.1.0","supports-color":"^9.4.0","tempfile":"^5.0.0"},"devDependencies":{"c8":"^10.1.2","eslint":"^8.57.1","mocha":"^10.7.3"},"engines":{"node":">=18.0.0"},"files":["bin","utils","index.js"]}