From 460925e36102cbca6bc9720633ee939aeb278c6d Mon Sep 17 00:00:00 2001 From: James Goodwin <1018296+jgoodw1n@users.noreply.github.com> Date: Fri, 31 Oct 2025 14:41:41 -0400 Subject: [PATCH 1/5] Add :default_missing_path_to_null option to allow missing paths to be retrieved as nulls --- README.md | 26 ++++++++++++- lib/jsonpath.rb | 1 + lib/jsonpath/dig.rb | 4 +- lib/jsonpath/enumerable.rb | 27 ++++++++++++- test/test_jsonpath.rb | 79 +++++++++++++++++++++++++++++++++++++- 5 files changed, 129 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 638a104..4effda3 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ JsonPath.new('$..color').first(json) # => "red" ``` -As well, we can directly create an `Enumerable` at any time using `#[]`. +As well, we can directly create an `Enumerable` at any time using `#[]`. ```ruby enum = JsonPath.new('$..color')[json] @@ -157,7 +157,7 @@ JsonPath.new('$.title', allow_send: true).on(book) ### Other available options By default, JsonPath does not return null values on unexisting paths. -This can be changed using the `:default_path_leaf_to_null` option +This can be changed using the `:default_path_leaf_to_null` option to return nil for leaf nodes: ```ruby JsonPath.new('$..book[*].isbn').on(json) @@ -167,6 +167,28 @@ JsonPath.new('$..book[*].isbn', default_path_leaf_to_null: true).on(json) # => [nil, nil, "0-553-21311-3", "0-395-19395-8"] ``` +Or it can be changed using the `:default_missing_path_to_null` option which will include both +leaf nodes and missing intermediate paths. + +```ruby +data = [ + { "review" => nil }, + { "review" => { "rating" => 5 } }, + { "review" => { "rating" => nil } }, + { "review" => { "comment" => "good" } }, + { "review" => { "rating" => 3 } } +] + +JsonPath.new('$[*].review.rating').on(data) +# => [5, nil, 3] + +JsonPath.new('$[*].review.rating', default_path_leaf_to_null: true).on(data) +# => [5, nil, nil, 3] + +JsonPath.new('$[*].review.rating', default_missing_path_to_null: true).on(data) +# => [nil, 5, nil, nil, 3] +``` + When JsonPath returns a Hash, you can ask to symbolize its keys using the `:symbolize_keys` option diff --git a/lib/jsonpath.rb b/lib/jsonpath.rb index 0388f62..d078fe3 100644 --- a/lib/jsonpath.rb +++ b/lib/jsonpath.rb @@ -16,6 +16,7 @@ class JsonPath DEFAULT_OPTIONS = { :default_path_leaf_to_null => false, + :default_missing_path_to_null => false, :symbolize_keys => false, :use_symbols => false, :allow_send => true, diff --git a/lib/jsonpath/dig.rb b/lib/jsonpath/dig.rb index 7a13004..979e86a 100644 --- a/lib/jsonpath/dig.rb +++ b/lib/jsonpath/dig.rb @@ -42,11 +42,11 @@ def yield_if_diggable(context, k, &blk) nil when Hash k = @options[:use_symbols] ? k.to_sym : k - return yield if context.key?(k) || @options[:default_path_leaf_to_null] + return yield if context.key?(k) || @options[:default_path_leaf_to_null] || @options[:default_missing_path_to_null] else if context.respond_to?(:dig) digged = dig_one(context, k) - yield if !digged.nil? || @options[:default_path_leaf_to_null] + yield if !digged.nil? || @options[:default_path_leaf_to_null] || @options[:default_missing_path_to_null] elsif @options[:allow_send] && context.respond_to?(k.to_s) && !Object.respond_to?(k.to_s) yield end diff --git a/lib/jsonpath/enumerable.rb b/lib/jsonpath/enumerable.rb index 29660ab..7c6ef92 100644 --- a/lib/jsonpath/enumerable.rb +++ b/lib/jsonpath/enumerable.rb @@ -17,6 +17,25 @@ def each(context = @object, key = nil, pos = 0, &blk) @_current_node = node return yield_value(blk, context, key) if pos == @path.size + # If node is nil and we have default_missing_path_to_null option, + # continue processing to potentially yield nil at the end + if node.nil? && @options[:default_missing_path_to_null] && pos < @path.size + case expr = @path[pos] + when '*', '..', '@' + each(nil, nil, pos + 1, &blk) + when '$' + each(nil, nil, pos + 1, &blk) + when /^\[(.*)\]$/ + expr[1, expr.size - 2].split(',').each do |sub_path| + case sub_path[0] + when '\'', '"' + each(nil, nil, pos + 1, &blk) + end + end + end + return + end + case expr = @path[pos] when '*', '..', '@' each(context, key, pos + 1, &blk) @@ -129,9 +148,13 @@ def handle_question_mark(sub_path, node, pos, &blk) def yield_value(blk, context, key) case @mode when nil - blk.call(key ? dig_one(context, key) : context) + if context.nil? && @options[:default_missing_path_to_null] + blk.call(nil) + else + blk.call(key ? dig_one(context, key) : context) + end when :compact - if key && context[key].nil? + if key && context&.dig(key).nil? key.is_a?(Integer) ? context.delete_at(key) : context.delete(key) end when :delete diff --git a/test/test_jsonpath.rb b/test/test_jsonpath.rb index 80fa0bb..633850c 100644 --- a/test/test_jsonpath.rb +++ b/test/test_jsonpath.rb @@ -895,7 +895,7 @@ def test_complex_nested_grouping path = "$..book[?((@['author'] == 'Evelyn Waugh' || @['author'] == 'Herman Melville') && (@['price'] == 33 || @['price'] == 9))]" assert_equal [@object['store']['book'][2]], JsonPath.new(path).on(@object) end - + def test_nested_with_unknown_key path = "$..[?(@.price == 9 || @.price == 33)].title" assert_equal ["Sayings of the Century", "Moby Dick", "Sayings of the Century", "Moby Dick"], JsonPath.new(path).on(@object) @@ -905,7 +905,7 @@ def test_nested_with_unknown_key_filtered_array path = "$..[?(@['price'] == 9 || @['price'] == 33)].title" assert_equal ["Sayings of the Century", "Moby Dick", "Sayings of the Century", "Moby Dick"], JsonPath.new(path).on(@object) end - + def test_runtime_error_frozen_string skip('in ruby version below 2.2.0 this error is not raised') if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.0') || Gem::Version.new(RUBY_VERSION) > Gem::Version::new('2.6') json = ' @@ -1318,4 +1318,79 @@ def test_symbolize_key assert_equal [{"category": "reference"}], JsonPath.new('$..book[0]', symbolize_keys: true).on(data) assert_equal [{"category": "reference"}], JsonPath.new('$..book[0]').on(data, symbolize_keys: true) end + + def test_default_missing_path_to_null_with_book_reviews + data = { 'store' => { + 'book' => [ + { 'category' => 'reference', + 'author' => 'Nigel Rees', + 'title' => 'Sayings of the Century', + 'price' => 9, + 'review' => nil }, + { 'category' => 'fiction', + 'author' => 'Evelyn Waugh', + 'title' => 'Sword of Honour', + 'price' => 13, + 'review' => { 'rating' => 5, 'comment' => 'Excellent!' } }, + { 'category' => 'fiction', + 'author' => 'Herman Melville', + 'title' => 'Moby Dick', + 'price' => 9 } + ] + }} + + current_result = JsonPath.new('$..book[*].review.rating', default_path_leaf_to_null: true).on(data) + assert_equal [5], current_result + + desired_result = JsonPath.new('$..book[*].review.rating', default_path_leaf_to_null: true, default_missing_path_to_null: true).on(data) + assert_equal [nil, 5, nil], desired_result + end + + def test_default_missing_path_to_null_multiple_nesting_levels + data = { 'store' => { + 'book' => [ + { 'title' => 'Book One', + 'metadata' => nil }, # Null at first level + { 'title' => 'Book Two', + 'metadata' => { 'publication' => nil } }, # Null at second level + { 'title' => 'Book Three', + 'metadata' => { + 'publication' => { + 'details' => nil + } + } }, # Null at third level + { 'title' => 'Book Four', + 'metadata' => { + 'publication' => { + 'details' => { + 'isbn' => '978-0123456789' + } + } + } } + ] + }} + + result_without = JsonPath.new('$..book[*].metadata.publication.details.isbn').on(data) + assert_equal ['978-0123456789'], result_without + + result_with = JsonPath.new('$..book[*].metadata.publication.details.isbn', default_missing_path_to_null: true).on(data) + assert_equal [nil, nil, nil, '978-0123456789'], result_with + end + + def test_default_missing_path_to_null_preserves_normal_behavior + result_without = JsonPath.new('$..book[*].author').on(@object) + result_with = JsonPath.new('$..book[*].author', default_missing_path_to_null: true).on(@object) + + expected = ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien', 'Lukyanenko', 'Lukyanenko', 'Lukyanenko'] + assert_equal expected, result_without + assert_equal expected, result_with + end + + def test_default_missing_path_to_null_shows_difference_with_missing_fields + result_without = JsonPath.new('$..book[*].price').on(@object) + result_with = JsonPath.new('$..book[*].price', default_missing_path_to_null: true).on(@object) + + assert_equal [9, 13, 9, 23], result_without + assert_equal [9, 13, 9, 23, nil, nil, nil], result_with + end end From c274bf82b4b568fef201c87d23f79210f82053cc Mon Sep 17 00:00:00 2001 From: James Goodwin <1018296+jgoodw1n@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:41:03 -0400 Subject: [PATCH 2/5] Avoid recursive descent continuation issue --- lib/jsonpath/enumerable.rb | 7 ++++++- test/test_jsonpath.rb | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/jsonpath/enumerable.rb b/lib/jsonpath/enumerable.rb index 7c6ef92..56616a1 100644 --- a/lib/jsonpath/enumerable.rb +++ b/lib/jsonpath/enumerable.rb @@ -20,7 +20,12 @@ def each(context = @object, key = nil, pos = 0, &blk) # If node is nil and we have default_missing_path_to_null option, # continue processing to potentially yield nil at the end if node.nil? && @options[:default_missing_path_to_null] && pos < @path.size - case expr = @path[pos] + # Skip nil continuation during recursive descent + # Check if context already has the target key from current path element + expr = @path[pos] + return if expr =~ /^\['(.*)'\]$/ && context.is_a?(Hash) && context.key?($1) + + case expr when '*', '..', '@' each(nil, nil, pos + 1, &blk) when '$' diff --git a/test/test_jsonpath.rb b/test/test_jsonpath.rb index 633850c..4a48ede 100644 --- a/test/test_jsonpath.rb +++ b/test/test_jsonpath.rb @@ -1393,4 +1393,26 @@ def test_default_missing_path_to_null_shows_difference_with_missing_fields assert_equal [9, 13, 9, 23], result_without assert_equal [9, 13, 9, 23, nil, nil, nil], result_with end + + def test_default_missing_path_to_null_avoids_recursive_descent + data = { 'store' => { + 'book' => [ + { 'title' => 'The Hobbit', + 'category' => 'fantasy' + }, + { 'title' => 'The Great Gatsby', + 'category' => 'classic', + 'metadata' => { 'isbn' => '978-0743273565' } + } + ] + }} + + path = '$.store.book[*].metadata.isbn' + + result_without = JsonPath.new(path).on(data) + assert_equal ['978-0743273565'], result_without + + result_with = JsonPath.new(path, default_missing_path_to_null: true).on(data) + assert_equal [nil, '978-0743273565'], result_with + end end From f12ef44c73f88fae6b4083d230d5a5e9332dcb3e Mon Sep 17 00:00:00 2001 From: James Goodwin <1018296+jgoodw1n@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:20:48 -0500 Subject: [PATCH 3/5] Add default missing path handling to arrays --- lib/jsonpath/enumerable.rb | 11 ++++++++--- test/test_jsonpath.rb | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/jsonpath/enumerable.rb b/lib/jsonpath/enumerable.rb index 56616a1..d0ea52a 100644 --- a/lib/jsonpath/enumerable.rb +++ b/lib/jsonpath/enumerable.rb @@ -20,10 +20,12 @@ def each(context = @object, key = nil, pos = 0, &blk) # If node is nil and we have default_missing_path_to_null option, # continue processing to potentially yield nil at the end if node.nil? && @options[:default_missing_path_to_null] && pos < @path.size - # Skip nil continuation during recursive descent # Check if context already has the target key from current path element + # to avoid duplicating nils in recursive descent expr = @path[pos] - return if expr =~ /^\['(.*)'\]$/ && context.is_a?(Hash) && context.key?($1) + if expr =~ /^\['(.*)'\]$/ && context.is_a?(Hash) && context.key?(::Regexp.last_match(1)) + return + end case expr when '*', '..', '@' @@ -101,7 +103,10 @@ def handle_wildcard(node, expr, _context, _key, pos, &blk) elsif sub_path.count(':') == 0 start_idx = end_idx = process_function_or_literal(array_args[0], 0) next unless start_idx - next if start_idx >= node.size + if start_idx >= node.size + each(nil, nil, pos + 1, &blk) if @options[:default_missing_path_to_null] + next + end else start_idx = process_function_or_literal(array_args[0], 0) next unless start_idx diff --git a/test/test_jsonpath.rb b/test/test_jsonpath.rb index 4a48ede..cdd14dd 100644 --- a/test/test_jsonpath.rb +++ b/test/test_jsonpath.rb @@ -1415,4 +1415,21 @@ def test_default_missing_path_to_null_avoids_recursive_descent result_with = JsonPath.new(path, default_missing_path_to_null: true).on(data) assert_equal [nil, '978-0743273565'], result_with end + + def test_default_missing_path_to_null_with_array_index_access + data = [ + { 'title' => 'Sayings of the Century', + 'reviews' => [{ 'rating' => 5 }] }, + { 'title' => 'Sword of Honour', + 'reviews' => [{ 'rating' => 4 }, { 'rating' => 3 }] }, + { 'title' => 'Moby Dick', + 'reviews' => [{ 'rating' => 5 }] } + ] + + result_without = JsonPath.new('$[*].reviews[1].rating').on(data) + assert_equal [3], result_without + + result_with = JsonPath.new('$[*].reviews[1].rating', default_missing_path_to_null: true).on(data) + assert_equal [nil, 3, nil], result_with + end end From 9a3218266b533490486e7481d7017bcae8df3a20 Mon Sep 17 00:00:00 2001 From: James Goodwin <1018296+jgoodw1n@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:01:22 -0500 Subject: [PATCH 4/5] fix: default_missing_path_to_null and recursive descent seem incompatible, so avoid the combination --- lib/jsonpath.rb | 4 ++++ lib/jsonpath/enumerable.rb | 7 +------ test/test_jsonpath.rb | 41 +++++++++++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/lib/jsonpath.rb b/lib/jsonpath.rb index d078fe3..eba3f30 100644 --- a/lib/jsonpath.rb +++ b/lib/jsonpath.rb @@ -53,6 +53,10 @@ def initialize(path, opts = {}) raise ArgumentError, "character '#{scanner.peek(1)}' not supported in query" end end + + # Disable default_missing_path_to_null option for recursive descent paths + # To avoid duplicating nils during recursive descent for every checked path + @opts[:default_missing_path_to_null] = false if @path.include?('..') end def find_matching_brackets(token, scanner) diff --git a/lib/jsonpath/enumerable.rb b/lib/jsonpath/enumerable.rb index d0ea52a..aae1d62 100644 --- a/lib/jsonpath/enumerable.rb +++ b/lib/jsonpath/enumerable.rb @@ -20,13 +20,8 @@ def each(context = @object, key = nil, pos = 0, &blk) # If node is nil and we have default_missing_path_to_null option, # continue processing to potentially yield nil at the end if node.nil? && @options[:default_missing_path_to_null] && pos < @path.size - # Check if context already has the target key from current path element - # to avoid duplicating nils in recursive descent - expr = @path[pos] - if expr =~ /^\['(.*)'\]$/ && context.is_a?(Hash) && context.key?(::Regexp.last_match(1)) - return - end + expr = @path[pos] case expr when '*', '..', '@' each(nil, nil, pos + 1, &blk) diff --git a/test/test_jsonpath.rb b/test/test_jsonpath.rb index cdd14dd..b370953 100644 --- a/test/test_jsonpath.rb +++ b/test/test_jsonpath.rb @@ -1343,7 +1343,7 @@ def test_default_missing_path_to_null_with_book_reviews assert_equal [5], current_result desired_result = JsonPath.new('$..book[*].review.rating', default_path_leaf_to_null: true, default_missing_path_to_null: true).on(data) - assert_equal [nil, 5, nil], desired_result + assert_equal [5], desired_result end def test_default_missing_path_to_null_multiple_nesting_levels @@ -1374,7 +1374,7 @@ def test_default_missing_path_to_null_multiple_nesting_levels assert_equal ['978-0123456789'], result_without result_with = JsonPath.new('$..book[*].metadata.publication.details.isbn', default_missing_path_to_null: true).on(data) - assert_equal [nil, nil, nil, '978-0123456789'], result_with + assert_equal ['978-0123456789'], result_with end def test_default_missing_path_to_null_preserves_normal_behavior @@ -1391,7 +1391,7 @@ def test_default_missing_path_to_null_shows_difference_with_missing_fields result_with = JsonPath.new('$..book[*].price', default_missing_path_to_null: true).on(@object) assert_equal [9, 13, 9, 23], result_without - assert_equal [9, 13, 9, 23, nil, nil, nil], result_with + assert_equal [9, 13, 9, 23], result_with end def test_default_missing_path_to_null_avoids_recursive_descent @@ -1432,4 +1432,39 @@ def test_default_missing_path_to_null_with_array_index_access result_with = JsonPath.new('$[*].reviews[1].rating', default_missing_path_to_null: true).on(data) assert_equal [nil, 3, nil], result_with end + + def test_default_missing_path_to_null_avoids_key_collision + data = [ + { 'title' => 'Book One', 'author' => nil }, + { 'title' => 'Book Two', 'author' => nil }, + { 'title' => 'Book Three', 'author' => { 'name' => 'Jane Doe', 'title' => 'Dr.' } } + ] + result_without = JsonPath.new('$[*].author.name', default_missing_path_to_null: true).on(data) + assert_equal [nil, nil, 'Jane Doe'], result_without + + result_title = JsonPath.new('$[*].author.title', default_missing_path_to_null: true).on(data) + assert_equal [nil, nil, 'Dr.'], result_title + end + + def test_default_missing_path_to_null_with_recursive_descent + data = { + 'library' => { + 'name' => 'Central Library', + 'books' => [ + { 'title' => 'Book One', 'author' => nil }, + { 'title' => 'Book Two', 'author' => { 'name' => 'Jane Smith' } }, + { 'title' => 'Book Three' } + ], + 'magazines' => [ + { 'title' => 'Mag One', 'author' => { 'name' => 'Bob Jones' } } + ] + } + } + + result_without = JsonPath.new('$..author.name').on(data) + assert_equal ['Jane Smith', 'Bob Jones'], result_without + + result_with = JsonPath.new('$..author.name', default_missing_path_to_null: true).on(data) + assert_equal ['Jane Smith', 'Bob Jones'], result_with + end end From 4bc4fde88754cc4892242e57f7f22690bd3fcdd5 Mon Sep 17 00:00:00 2001 From: James Goodwin <1018296+jgoodw1n@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:20:18 -0500 Subject: [PATCH 5/5] fix: test case cleanup --- test/test_jsonpath.rb | 44 ++++++++++++++----------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/test/test_jsonpath.rb b/test/test_jsonpath.rb index b370953..a776a58 100644 --- a/test/test_jsonpath.rb +++ b/test/test_jsonpath.rb @@ -1319,7 +1319,7 @@ def test_symbolize_key assert_equal [{"category": "reference"}], JsonPath.new('$..book[0]').on(data, symbolize_keys: true) end - def test_default_missing_path_to_null_with_book_reviews + def test_recursive_descent_with_default_options data = { 'store' => { 'book' => [ { 'category' => 'reference', @@ -1339,11 +1339,14 @@ def test_default_missing_path_to_null_with_book_reviews ] }} - current_result = JsonPath.new('$..book[*].review.rating', default_path_leaf_to_null: true).on(data) - assert_equal [5], current_result + leaf_path_result = JsonPath.new('$..book[*].review.rating', default_path_leaf_to_null: true).on(data) + assert_equal [5], leaf_path_result - desired_result = JsonPath.new('$..book[*].review.rating', default_path_leaf_to_null: true, default_missing_path_to_null: true).on(data) - assert_equal [5], desired_result + missing_path_result = JsonPath.new('$..book[*].review.rating', default_missing_path_to_null: true).on(data) + assert_equal [5], missing_path_result + + normal_result = JsonPath.new('$..book[*].review.rating').on(data) + assert_equal [5], normal_result end def test_default_missing_path_to_null_multiple_nesting_levels @@ -1370,31 +1373,14 @@ def test_default_missing_path_to_null_multiple_nesting_levels ] }} - result_without = JsonPath.new('$..book[*].metadata.publication.details.isbn').on(data) + result_without = JsonPath.new('$.store.book[*].metadata.publication.details.isbn').on(data) assert_equal ['978-0123456789'], result_without - result_with = JsonPath.new('$..book[*].metadata.publication.details.isbn', default_missing_path_to_null: true).on(data) - assert_equal ['978-0123456789'], result_with - end - - def test_default_missing_path_to_null_preserves_normal_behavior - result_without = JsonPath.new('$..book[*].author').on(@object) - result_with = JsonPath.new('$..book[*].author', default_missing_path_to_null: true).on(@object) - - expected = ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien', 'Lukyanenko', 'Lukyanenko', 'Lukyanenko'] - assert_equal expected, result_without - assert_equal expected, result_with + result_with = JsonPath.new('$.store.book[*].metadata.publication.details.isbn', default_missing_path_to_null: true).on(data) + assert_equal [nil, nil, nil, '978-0123456789'], result_with end - def test_default_missing_path_to_null_shows_difference_with_missing_fields - result_without = JsonPath.new('$..book[*].price').on(@object) - result_with = JsonPath.new('$..book[*].price', default_missing_path_to_null: true).on(@object) - - assert_equal [9, 13, 9, 23], result_without - assert_equal [9, 13, 9, 23], result_with - end - - def test_default_missing_path_to_null_avoids_recursive_descent + def test_default_missing_path_to_null_finds_missing_keys data = { 'store' => { 'book' => [ { 'title' => 'The Hobbit', @@ -1407,12 +1393,10 @@ def test_default_missing_path_to_null_avoids_recursive_descent ] }} - path = '$.store.book[*].metadata.isbn' - - result_without = JsonPath.new(path).on(data) + result_without = JsonPath.new('$.store.book[*].metadata.isbn').on(data) assert_equal ['978-0743273565'], result_without - result_with = JsonPath.new(path, default_missing_path_to_null: true).on(data) + result_with = JsonPath.new('$.store.book[*].metadata.isbn', default_missing_path_to_null: true).on(data) assert_equal [nil, '978-0743273565'], result_with end