From b25cbdce669fc8002478f157e9d2d31f77587e3e Mon Sep 17 00:00:00 2001 From: djklim87 Date: Thu, 27 Nov 2025 14:35:07 +0100 Subject: [PATCH 1/6] Fix: Struct parsing issues leading to Wrong JSON parsing response if id is bigint --- src/Network/Struct.php | 2 +- .../Network/StructBigIntSerializationTest.php | 201 ++++++++++++++++++ 2 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 test/BuddyCore/Network/StructBigIntSerializationTest.php diff --git a/src/Network/Struct.php b/src/Network/Struct.php index 80cc376..83d921b 100644 --- a/src/Network/Struct.php +++ b/src/Network/Struct.php @@ -236,7 +236,7 @@ private static function getReplacePattern(string $path): string { } } - return $pattern . '("?)([^"{}[\],]+)\1/'; + return $pattern . '("?)(\-?\d+)\1/'; } /** diff --git a/test/BuddyCore/Network/StructBigIntSerializationTest.php b/test/BuddyCore/Network/StructBigIntSerializationTest.php new file mode 100644 index 0000000..db3f1d7 --- /dev/null +++ b/test/BuddyCore/Network/StructBigIntSerializationTest.php @@ -0,0 +1,201 @@ + [ + ['id' => ['type' => 'long long']], + ['s' => ['type' => 'string']], + ['v' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 5047479470261279290, // This is a bigint + 's' => '0000000000', // This should remain a quoted string + 'v' => '0.44721356,0.89442712', + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ]; + + // Create struct with bigint field marked (as Client.php does) + $struct = Struct::fromData($inputData, ['data.0.id']); + + // Get the JSON output + $json = $struct->toJson(); + + // Parse the JSON to verify it's valid + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Generated JSON should be valid and parseable'); + + // Verify the bigint field is correctly handled (no quotes) + $this->assertStringContainsString( + '"id":5047479470261279290', $json, + 'Bigint field should be serialized without quotes' + ); + + // Verify string fields maintain their quotes (this was the bug) + $this->assertStringContainsString( + '"s":"0000000000"', $json, + 'String fields should maintain their quotes' + ); + $this->assertStringContainsString( + '"v":"0.44721356,0.89442712"', $json, + 'String fields should maintain their quotes' + ); + + // Verify the decoded values are correct + $this->assertEquals(5047479470261279290, $decoded[0]['data'][0]['id']); + $this->assertEquals('0000000000', $decoded[0]['data'][0]['s']); + $this->assertEquals('0.44721356,0.89442712', $decoded[0]['data'][0]['v']); + } + + /** + * Test various edge cases where string values might be confused with bigints + * + * @return void + */ + public function testStringFieldsWithNumericContentAreNotAffected(): void { + $testCases = [ + ['field' => '0000000000', 'description' => 'zero-padded string'], + ['field' => '1234567890123456789', 'description' => 'long numeric string'], + ['field' => '00123', 'description' => 'zero-padded numeric string'], + ['field' => '1.23456789', 'description' => 'decimal string'], + ['field' => '+1234567890', 'description' => 'string with plus sign'], + ['field' => '-9876543210', 'description' => 'string with minus sign'], + ['field' => '1e10', 'description' => 'scientific notation string'], + ]; + + foreach ($testCases as $testCase) { + $data = [ + 'bigint_field' => 9223372036854775807, // Real bigint + 'string_field' => $testCase['field'], // String that might look numeric + 'normal_field' => 'test', + ]; + + $struct = Struct::fromData($data, ['bigint_field']); + $json = $struct->toJson(); + + // Verify JSON is valid + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, "Generated JSON should be valid for {$testCase['description']}"); + + // Verify bigint is unquoted + $this->assertStringContainsString('"bigint_field":9223372036854775807', $json); + + // Verify string field maintains quotes + $expectedStringJson = '"string_field":"' . $testCase['field'] . '"'; + $this->assertStringContainsString( + $expectedStringJson, $json, + "String field should maintain quotes for {$testCase['description']}" + ); + + // Verify decoded values + $this->assertEquals(9223372036854775807, $decoded['bigint_field']); + $this->assertEquals($testCase['field'], $decoded['string_field']); + } + } + + /** + * Test nested structures with mixed bigint and string fields + * + * @return void + */ + public function testNestedStructuresWithMixedTypes(): void { + $data = [ + 'level1' => [ + 'bigint' => 5047479470261279290, + 'string' => '0000000000', + 'level2' => [ + 'bigint' => 9223372036854775807, + 'string' => '1234567890123456789', + ], + ], + ]; + + $struct = Struct::fromData($data, ['level1.bigint', 'level1.level2.bigint']); + $json = $struct->toJson(); + + // Verify JSON is valid + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Nested structure JSON should be valid'); + + // Verify bigints are unquoted + $this->assertStringContainsString('"bigint":5047479470261279290', $json); + $this->assertStringContainsString('"bigint":9223372036854775807', $json); + + // Verify strings maintain quotes + $this->assertStringContainsString('"string":"0000000000"', $json); + $this->assertStringContainsString('"string":"1234567890123456789"', $json); + + // Verify structure integrity + $this->assertEquals('0000000000', $decoded['level1']['string']); + $this->assertEquals('1234567890123456789', $decoded['level1']['level2']['string']); + } + + /** + * Test the actual Client.php scenario that was failing + * + * @return void + */ + public function testClientMetaResponseScenario(): void { + // Simulate the exact data structure from Client.php:191 + $array = [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['s' => ['type' => 'string']], + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 5047479470261279290, 's' => '0000000000', 'v' => '0.44721356,0.89442712'], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ]; + + // This is the exact line that was causing the issue + $response = Struct::fromData($array, ['data.0.id'])->toJson(); + + // Verify the response is valid JSON + $this->assertNotNull(json_decode($response), 'Response should be valid JSON'); + + // Verify the specific issue is fixed + $this->assertStringNotContainsString( + ',"s":0000000000,', $response, + 'String field should not lose its quotes' + ); + $this->assertStringContainsString( + ',"s":"0000000000",', $response, + 'String field should maintain its quotes' + ); + } +} From bab887ade7d1b5eee1ea6cc84c363c1b99587cb6 Mon Sep 17 00:00:00 2001 From: djklim87 Date: Thu, 27 Nov 2025 15:05:24 +0100 Subject: [PATCH 2/6] Fix: Struct parsing issues leading to Wrong JSON parsing response if id is bigint --- .../Network/StructBigIntSerializationTest.php | 524 ++++++++++++++---- 1 file changed, 426 insertions(+), 98 deletions(-) diff --git a/test/BuddyCore/Network/StructBigIntSerializationTest.php b/test/BuddyCore/Network/StructBigIntSerializationTest.php index db3f1d7..308cbfe 100644 --- a/test/BuddyCore/Network/StructBigIntSerializationTest.php +++ b/test/BuddyCore/Network/StructBigIntSerializationTest.php @@ -14,157 +14,479 @@ final class StructBigIntSerializationTest extends TestCase { /** - * Test that reproduces the specific JSON serialization bug where string values - * lose their quotes when bigint fields are processed. + * Data provider with real ManticoreSearch response data types + * Based on actual ManticoreSearch responses showing how different field types are returned * - * Bug: String "0000000000" becomes 0000000000 (invalid JSON) - * Root cause: Overly broad regex pattern in getReplacePattern() method - * - * @return void + * @return array,array}> */ - public function testBigIntSerializationDoesNotAffectStringFields(): void { - // This is the exact scenario that was failing - $inputData = [ - [ - 'columns' => [ - ['id' => ['type' => 'long long']], - ['s' => ['type' => 'string']], - ['v' => ['type' => 'string']], + public static function realManticoreResponseProvider(): array { + return [ + 'complete_data_type_coverage' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], // bigint (should be unquoted) + ['title' => ['type' => 'string']], // string (should stay quoted) + ['price' => ['type' => 'float']], // float (should stay unquoted) + ['count' => ['type' => 'long']], // integer (should stay unquoted) + ['is_active' => ['type' => 'long']], // boolean-as-int (should stay unquoted) + ['tags' => ['type' => 'string']], // MVA as string (should stay quoted) + ['meta' => ['type' => 'string']], // JSON as string (should stay quoted) + ], + 'data' => [ + [ + 'id' => 5623974933752184833, // bigint (should be unquoted) + 'title' => '000123', // numeric-looking string (should stay quoted) + 'price' => 19.990000, // float (should stay unquoted) + 'count' => 100, // int (should stay unquoted) + 'is_active' => 1, // bool-as-int (should stay unquoted) + 'tags' => '1,2,3', // MVA string (should stay quoted) + 'meta' => '{"category":"electronics","rating":4.500000}', // JSON (should stay quoted) + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', ], - 'data' => [ - [ - 'id' => 5047479470261279290, // This is a bigint - 's' => '0000000000', // This should remain a quoted string - 'v' => '0.44721356,0.89442712', + ['data.0.id'], // Only id is bigint + [ + 'bigint_unquoted' => '"id":5623974933752184833', + 'string_quoted' => '"title":"000123"', + 'float_unquoted' => '"price":19.99', + 'int_unquoted' => '"count":100', + 'bool_unquoted' => '"is_active":1', + 'mva_quoted' => '"tags":"1,2,3"', + 'json_quoted' => '"meta":"{\"category\":\"electronics\"', + ], + ], + 'edge_case_numeric_strings' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['code' => ['type' => 'string']], + ['tags' => ['type' => 'string']], + ['meta' => ['type' => 'string']], ], + 'data' => [ + [ + 'id' => 9223372036854775807, // bigint (should be unquoted) + 'code' => '0000000000', // original bug scenario (should stay quoted) + 'tags' => '2808348671,2808348672', // MVA with large numbers (should stay quoted) + 'meta' => '{"numbers":[1,2.500000,true,false,null],"text":"000456"}', // complex JSON + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + [ + 'bigint_unquoted' => '"id":9223372036854775807', + 'code_quoted' => '"code":"0000000000"', + 'tags_quoted' => '"tags":"2808348671,2808348672"', + 'meta_quoted' => '"meta":"{\"numbers\":[1,2.500000,true,false,null]', ], - 'total' => 1, - 'error' => '', - 'warning' => '', ], ]; + } - // Create struct with bigint field marked (as Client.php does) - $struct = Struct::fromData($inputData, ['data.0.id']); + /** + * Data provider for edge cases with numeric-looking strings + * These are critical because the regex could potentially affect them + * + * @return array + */ + public static function edgeCaseDataProvider(): array { + return [ + 'string_that_looks_like_bigint' => [ + '9223372036854775807', + 'string that looks exactly like a bigint', + ], + 'original_bug_scenario' => [ + '0000000000', + 'original bug scenario - zero-padded string', + ], + 'negative_number_string' => [ + '-9223372036854775808', + 'negative number as string', + ], + 'mva_with_large_numbers' => [ + '2808348671,2808348672', + 'MVA with large integers as comma-separated string', + ], + 'json_with_mixed_numbers' => [ + '{"numbers":[1,2.500000,true,false,null],"text":"000456"}', + 'JSON string containing various number formats', + ], + 'high_precision_decimal' => [ + '123.456789012345', + 'high precision decimal as string', + ], + 'string_with_spaces_and_numbers' => [ + ' 123 456 ', + 'string with spaces and numbers', + ], + 'hex_like_string' => [ + '0x123456789', + 'hex-like string', + ], + 'binary_like_string' => [ + '0b101010', + 'binary-like string', + ], + ]; + } - // Get the JSON output + /** + * Data provider for mixed data type scenarios + * Simulates real database responses with various field types + * + * @return array,array,array}> + */ + public static function mixedDataTypeScenarios(): array { + return [ + 'database_row_simulation' => [ + [ + 'id' => 9223372036854775807, // bigint + 'user_id' => 12345, // regular int + 'balance' => 1234.56, // float + 'is_premium' => 1, // boolean as int + 'permissions' => '1,5,10,15', // MVA as string + 'settings' => '{"theme":"dark","lang":"en"}', // JSON as string + 'code' => '00001', // numeric string + 'phone' => '+1-555-0123', // string with numbers + ], + ['id'], // only id is bigint + ['permissions', 'settings', 'code', 'phone'], // fields that must stay quoted + ], + 'meta_response_with_bigints' => [ + [ + 'id' => 5623974933752184833, + 'data' => '000000', + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['id'], + ['data'], // the data field should stay quoted + ], + ]; + } + + /** + * Test real ManticoreSearch data types to ensure comprehensive coverage + * + * @dataProvider realManticoreResponseProvider + * @param array> $inputData + * @param array $bigintFields + * @param array $expectedPatterns + * @return void + */ + public function testRealManticoreDataTypes(array $inputData, array $bigintFields, array $expectedPatterns): void { + $struct = Struct::fromData($inputData, $bigintFields); $json = $struct->toJson(); - // Parse the JSON to verify it's valid + // Verify JSON is valid + /** @var array>|null $decoded */ $decoded = json_decode($json, true); $this->assertNotNull($decoded, 'Generated JSON should be valid and parseable'); + $this->assertIsArray($decoded, 'Decoded JSON should be an array'); - // Verify the bigint field is correctly handled (no quotes) - $this->assertStringContainsString( - '"id":5047479470261279290', $json, - 'Bigint field should be serialized without quotes' - ); + // Verify all expected patterns are present + foreach ($expectedPatterns as $description => $pattern) { + $this->assertStringContainsString( + $pattern, + $json, + "Pattern '{$description}' should be present in JSON output" + ); + } - // Verify string fields maintain their quotes (this was the bug) - $this->assertStringContainsString( - '"s":"0000000000"', $json, - 'String fields should maintain their quotes' - ); - $this->assertStringContainsString( - '"v":"0.44721356,0.89442712"', $json, - 'String fields should maintain their quotes' - ); + // Verify bigint fields are unquoted while others maintain their correct format + foreach ($bigintFields as $fieldPath) { + $pathParts = explode('.', $fieldPath); + $currentData = $inputData; + + // Validate path traversal using assertions + foreach ($pathParts as $part) { + $this->assertIsArray( + $currentData, + "Path traversal failed at '{$part}' for field path '{$fieldPath}'" + ); + $this->assertArrayHasKey( + $part, + $currentData, + "Path component '{$part}' not found in field path '{$fieldPath}'" + ); + $currentData = $currentData[$part]; + } + + // At this point, $currentData should be the final scalar value + $this->assertIsScalar( + $currentData, + "Bigint field '{$fieldPath}' should be scalar, got " . gettype($currentData) + ); - // Verify the decoded values are correct - $this->assertEquals(5047479470261279290, $decoded[0]['data'][0]['id']); - $this->assertEquals('0000000000', $decoded[0]['data'][0]['s']); - $this->assertEquals('0.44721356,0.89442712', $decoded[0]['data'][0]['v']); + $bigintValue = (string)$currentData; + $this->assertStringContainsString( + '":' . $bigintValue, + $json, + "Bigint field '{$fieldPath}' should be unquoted" + ); + } } /** - * Test various edge cases where string values might be confused with bigints + * Test edge cases where strings contain numeric content + * These are critical because the regex could potentially affect them * + * @dataProvider edgeCaseDataProvider + * @param string $stringField + * @param string $description * @return void */ - public function testStringFieldsWithNumericContentAreNotAffected(): void { - $testCases = [ - ['field' => '0000000000', 'description' => 'zero-padded string'], - ['field' => '1234567890123456789', 'description' => 'long numeric string'], - ['field' => '00123', 'description' => 'zero-padded numeric string'], - ['field' => '1.23456789', 'description' => 'decimal string'], - ['field' => '+1234567890', 'description' => 'string with plus sign'], - ['field' => '-9876543210', 'description' => 'string with minus sign'], - ['field' => '1e10', 'description' => 'scientific notation string'], + public function testNumericStringEdgeCases(string $stringField, string $description): void { + $data = [ + 'bigint_field' => 9223372036854775807, // Real bigint + 'string_field' => $stringField, // String that might look numeric + 'normal_field' => 'test', ]; - foreach ($testCases as $testCase) { - $data = [ - 'bigint_field' => 9223372036854775807, // Real bigint - 'string_field' => $testCase['field'], // String that might look numeric - 'normal_field' => 'test', - ]; + $struct = Struct::fromData($data, ['bigint_field']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, "Generated JSON should be valid for {$description}"); + $this->assertIsArray($decoded, "Decoded JSON should be an array for {$description}"); - $struct = Struct::fromData($data, ['bigint_field']); - $json = $struct->toJson(); + // Verify bigint is unquoted + $this->assertStringContainsString('"bigint_field":9223372036854775807', $json); - // Verify JSON is valid - $decoded = json_decode($json, true); - $this->assertNotNull($decoded, "Generated JSON should be valid for {$testCase['description']}"); + // Verify string field maintains quotes (this is the critical test) + $expectedStringJson = '"string_field":"' . addslashes($stringField) . '"'; + $this->assertStringContainsString( + $expectedStringJson, + $json, + "String field should maintain quotes for {$description}" + ); - // Verify bigint is unquoted - $this->assertStringContainsString('"bigint_field":9223372036854775807', $json); + // Verify decoded values with proper type checking + $this->assertEquals(9223372036854775807, $decoded['bigint_field']); + $this->assertEquals($stringField, $decoded['string_field']); + } - // Verify string field maintains quotes - $expectedStringJson = '"string_field":"' . $testCase['field'] . '"'; + /** + * Test mixed data type scenarios simulating real database responses + * + * @dataProvider mixedDataTypeScenarios + * @param array $data + * @param array $bigintFields + * @param array $criticalQuotes + * @return void + */ + public function testMixedDataTypeScenarios(array $data, array $bigintFields, array $criticalQuotes): void { + $struct = Struct::fromData($data, $bigintFields); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Mixed data type JSON should be valid'); + $this->assertIsArray($decoded, 'Decoded JSON should be an array'); + + // Verify bigint fields are unquoted + foreach ($bigintFields as $field) { + if (!isset($data[$field])) { + continue; + } + + $value = is_scalar($data[$field]) ? (string)$data[$field] : ''; $this->assertStringContainsString( - $expectedStringJson, $json, - "String field should maintain quotes for {$testCase['description']}" + '"' . $field . '":' . $value, + $json, + "Bigint field '{$field}' should be unquoted" ); + } + + // Verify critical string fields maintain quotes + foreach ($criticalQuotes as $field) { + if (!isset($data[$field])) { + continue; + } - // Verify decoded values - $this->assertEquals(9223372036854775807, $decoded['bigint_field']); - $this->assertEquals($testCase['field'], $decoded['string_field']); + $value = is_scalar($data[$field]) ? (string)$data[$field] : ''; + $expectedPattern = '"' . $field . '":"' . addslashes($value) . '"'; + $this->assertStringContainsString( + $expectedPattern, + $json, + "Critical string field '{$field}' should maintain quotes" + ); } } /** - * Test nested structures with mixed bigint and string fields + * Test float precision preservation + * Ensures that float values are not affected by the bigint regex * * @return void */ - public function testNestedStructuresWithMixedTypes(): void { + public function testFloatPrecisionPreservation(): void { $data = [ - 'level1' => [ - 'bigint' => 5047479470261279290, - 'string' => '0000000000', - 'level2' => [ - 'bigint' => 9223372036854775807, - 'string' => '1234567890123456789', - ], - ], + 'id' => 9223372036854775807, + 'price' => 123.456789012345, + 'rate' => 0.000001, + 'negative_float' => -456.789, + ]; + + $struct = Struct::fromData($data, ['id']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Float precision JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); + + // Verify floats remain unquoted and preserve precision + $this->assertStringContainsString('"price":123.456789012345', $json); + $this->assertStringContainsString('"rate":1.0e-6', $json); + $this->assertStringContainsString('"negative_float":-456.789', $json); + + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals(123.456789012345, $decoded['price']); + $this->assertEquals(0.000001, $decoded['rate']); + $this->assertEquals(-456.789, $decoded['negative_float']); + } + + /** + * Test MVA (Multi-Value Attribute) string preservation + * MVA fields are returned as comma-separated strings and must stay quoted + * + * @return void + */ + public function testMVAStringPreservation(): void { + $data = [ + 'id' => 9223372036854775807, + 'tags' => '1,2,3,100,200', + 'permissions' => '2808348671,2808348672,2808348673', + 'empty_mva' => '', + ]; + + $struct = Struct::fromData($data, ['id']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'MVA string JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); + + // Verify MVA strings maintain quotes + $this->assertStringContainsString('"tags":"1,2,3,100,200"', $json); + $this->assertStringContainsString('"permissions":"2808348671,2808348672,2808348673"', $json); + $this->assertStringContainsString('"empty_mva":""', $json); + + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals('1,2,3,100,200', $decoded['tags']); + $this->assertEquals('2808348671,2808348672,2808348673', $decoded['permissions']); + $this->assertEquals('', $decoded['empty_mva']); + } + + /** + * Test JSON string field preservation + * JSON fields are stored as strings and must maintain their quotes + * + * @return void + */ + public function testJSONStringPreservation(): void { + $data = [ + 'id' => 9223372036854775807, + 'settings' => '{"theme":"dark","lang":"en","version":1}', + 'metadata' => '{"numbers":[1,2.5,true,false,null],"text":"000456"}', + 'empty_json' => '{}', ]; - $struct = Struct::fromData($data, ['level1.bigint', 'level1.level2.bigint']); + $struct = Struct::fromData($data, ['id']); $json = $struct->toJson(); // Verify JSON is valid + /** @var array|null $decoded */ $decoded = json_decode($json, true); - $this->assertNotNull($decoded, 'Nested structure JSON should be valid'); + $this->assertNotNull($decoded, 'JSON string JSON should be valid'); - // Verify bigints are unquoted - $this->assertStringContainsString('"bigint":5047479470261279290', $json); - $this->assertStringContainsString('"bigint":9223372036854775807', $json); + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); - // Verify strings maintain quotes - $this->assertStringContainsString('"string":"0000000000"', $json); - $this->assertStringContainsString('"string":"1234567890123456789"', $json); + // Verify JSON strings maintain quotes + $this->assertStringContainsString( + '"settings":"{\\"theme\\":\\"dark\\",\\"lang\\":\\"en\\",\\"version\\":1}"', + $json + ); + $this->assertStringContainsString( + '"metadata":"{\\"numbers\\":[1,2.5,true,false,null],\\"text\\":\\"000456\\"}"', + $json + ); + $this->assertStringContainsString('"empty_json":"{}"', $json); - // Verify structure integrity - $this->assertEquals('0000000000', $decoded['level1']['string']); - $this->assertEquals('1234567890123456789', $decoded['level1']['level2']['string']); + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals('{"theme":"dark","lang":"en","version":1}', $decoded['settings']); + $this->assertEquals('{"numbers":[1,2.5,true,false,null],"text":"000456"}', $decoded['metadata']); + $this->assertEquals('{}', $decoded['empty_json']); } /** - * Test the actual Client.php scenario that was failing + * Test negative number handling + * Ensures negative bigints and regular numbers work correctly * * @return void */ - public function testClientMetaResponseScenario(): void { + public function testNegativeNumberHandling(): void { + $data = [ + 'negative_bigint' => -9223372036854775808, + 'negative_int' => -42, + 'negative_float' => -123.456, + 'string_negative' => '-999', + ]; + + $struct = Struct::fromData($data, ['negative_bigint']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Negative number JSON should be valid'); + + // Verify negative bigint is unquoted + $this->assertStringContainsString('"negative_bigint":-9.223372036854776e+18', $json); + + // Verify other negative numbers remain unquoted + $this->assertStringContainsString('"negative_int":-42', $json); + $this->assertStringContainsString('"negative_float":-123.456', $json); + + // Verify negative string maintains quotes + $this->assertStringContainsString('"string_negative":"-999"', $json); + + // Verify decoded values are correct + $this->assertEquals(-9223372036854775808, $decoded['negative_bigint']); + $this->assertEquals(-42, $decoded['negative_int']); + $this->assertEquals(-123.456, $decoded['negative_float']); + $this->assertEquals('-999', $decoded['string_negative']); + } + + /** + * Test the actual Client.php:191 scenario that was failing + * This simulates the exact bug scenario with SHOW META responses + * + * @return void + */ + public function testClientMetaResponseIntegration(): void { // Simulate the exact data structure from Client.php:191 $array = [ [ @@ -186,7 +508,7 @@ public function testClientMetaResponseScenario(): void { $response = Struct::fromData($array, ['data.0.id'])->toJson(); // Verify the response is valid JSON - $this->assertNotNull(json_decode($response), 'Response should be valid JSON'); + $this->assertNotNull(json_decode($response), 'Client response should be valid JSON'); // Verify the specific issue is fixed $this->assertStringNotContainsString( @@ -197,5 +519,11 @@ public function testClientMetaResponseScenario(): void { ',"s":"0000000000",', $response, 'String field should maintain its quotes' ); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":5047479470261279290', $response); + + // Verify other string field maintains quotes + $this->assertStringContainsString('"v":"0.44721356,0.89442712"', $response); } } From d51e7fb5a1ada8f4573485aa54b0c03b52a8b3e0 Mon Sep 17 00:00:00 2001 From: djklim87 Date: Thu, 27 Nov 2025 15:05:24 +0100 Subject: [PATCH 3/6] Fix: Struct parsing issues leading to Wrong JSON parsing response if id is bigint --- src/Network/Struct.php | 122 +++- .../Network/StructBigIntSerializationTest.php | 524 +++++++++++++--- .../Network/StructSingleResponseTest.php | 572 ++++++++++++++++++ 3 files changed, 1115 insertions(+), 103 deletions(-) create mode 100644 test/BuddyCore/Network/StructSingleResponseTest.php diff --git a/src/Network/Struct.php b/src/Network/Struct.php index 83d921b..899ee1a 100644 --- a/src/Network/Struct.php +++ b/src/Network/Struct.php @@ -139,7 +139,13 @@ public static function fromJson(string $json): self { /** @var array */ $result = (array)simdjson_decode($json, true, static::JSON_DEPTH); $bigIntFields = []; - if (static::hasBigInt($json)) { + + // PRIMARY: Extract bigint fields from column metadata if present + $hasColumns = isset($result['columns']) && is_array($result['columns']); + if ($hasColumns) { + static::addBigIntFieldsFromColumns($result, $bigIntFields); + } elseif (static::hasBigInt($json)) { + // FALLBACK: Only run heuristic if columns metadata is missing // We need here to keep original json decode cuzit has bigIntFields /** @var array */ $modified = json_decode($json, true, static::JSON_DEPTH, static::JSON_FLAGS | JSON_BIGINT_AS_STRING); @@ -213,7 +219,7 @@ static function (array $matches): string { }, $serialized ); - if (!isset($json)) { + if ($json === false) { throw new Exception('Cannot encode data to JSON'); } return $json; @@ -241,6 +247,8 @@ private static function getReplacePattern(string $path): string { /** * Traverse the data and track all fields that are big integers + * Skips fields that were already identified via column metadata + * * @param mixed $data * @param mixed $originalData * @param array $bigIntFields @@ -264,14 +272,90 @@ private static function traverseAndTrack( } $originalValue = $originalData[$key]; - if (is_string($value) && is_numeric($originalValue) && strlen($value) > 9) { - $bigIntFields[] = $currentPath; - } elseif (is_array($value) && is_array($originalValue)) { + if (is_array($value) && is_array($originalValue)) { static::traverseAndTrack($value, $originalValue, $bigIntFields, $currentPath); + continue; } + + static::trackBigIntIfNeeded($value, $originalValue, $currentPath, $bigIntFields); } } + /** + * Check if a numeric string exceeds PHP_INT_MAX/MIN boundaries + * Uses optimized string length with conditional trimming for best performance + * Handles leading zeros and sign correctly + * + * @param string $value Numeric string to check + * @return bool True if value exceeds 64-bit integer range + */ + private static function isBigIntBoundary(string $value): bool { + $firstChar = $value[0]; + + // OPTIMIZATION: Only ltrim if value has leading sign or zeros + // This avoids expensive function call for ~95% of clean numeric values + $absValue = ($firstChar === '-' || $firstChar === '0') + ? ltrim($value, '-0') + : $value; + + $magnitude = strlen($absValue); + + // PHP_INT_MAX has 19 digits, so we can quickly filter most values + // > 19 digits: definitely a bigint + if ($magnitude > 19) { + return true; + } + + // <= 18 digits: definitely not a bigint + if ($magnitude < 19) { + return false; + } + + // Exactly 19 digits: need string comparison against PHP_INT_MAX/MIN boundaries + // Note: We must use different boundaries for positive vs negative + // because PHP_INT_MAX and |PHP_INT_MIN| differ by 1 + if ($firstChar === '-') { + // Negative: 19-digit negative is bigint if absolute value > |PHP_INT_MIN| + // |PHP_INT_MIN| = 9223372036854775808 + return $absValue > '9223372036854775808'; + } + + // Positive: 19-digit positive is bigint if > PHP_INT_MAX + return $absValue > (string)PHP_INT_MAX; + } + + /** + * Check if field is a big integer and add it to tracking list + * + * @param mixed $value + * @param mixed $originalValue + * @param string $currentPath + * @param array &$bigIntFields + * @return void + */ + private static function trackBigIntIfNeeded( + mixed $value, + mixed $originalValue, + string $currentPath, + array &$bigIntFields + ): void { + if (!is_string($value) || !is_numeric($originalValue)) { + return; + } + + // Use precise boundary check instead of arbitrary strlen + if (!static::isBigIntBoundary($value)) { + return; + } + + // Skip if already identified via column metadata + if (in_array($currentPath, $bigIntFields, true)) { + return; + } + + $bigIntFields[] = $currentPath; + } + /** * Check if the JSON string contains any big integers * @param string $json @@ -281,6 +365,34 @@ private static function hasBigInt(string $json): bool { return !!preg_match('/(? $response Response with 'columns' metadata + * @param array &$bigIntFields Fields array to populate with bigint paths + * @return void + */ + private static function addBigIntFieldsFromColumns(array $response, array &$bigIntFields): void { + if (!isset($response['columns']) || !is_array($response['columns'])) { + return; + } + + foreach ($response['columns'] as $columnIndex => $columnDef) { + if (!is_array($columnDef)) { + continue; + } + + foreach ($columnDef as $fieldName => $fieldInfo) { + if (!is_array($fieldInfo) || !(($fieldInfo['type'] ?? null) === 'long long')) { + continue; + } + + $bigIntFields[] = "data.{$columnIndex}.{$fieldName}"; + } + } + } + /** * Count elements of an object * @link https://php.net/manual/en/countable.count.php diff --git a/test/BuddyCore/Network/StructBigIntSerializationTest.php b/test/BuddyCore/Network/StructBigIntSerializationTest.php index db3f1d7..308cbfe 100644 --- a/test/BuddyCore/Network/StructBigIntSerializationTest.php +++ b/test/BuddyCore/Network/StructBigIntSerializationTest.php @@ -14,157 +14,479 @@ final class StructBigIntSerializationTest extends TestCase { /** - * Test that reproduces the specific JSON serialization bug where string values - * lose their quotes when bigint fields are processed. + * Data provider with real ManticoreSearch response data types + * Based on actual ManticoreSearch responses showing how different field types are returned * - * Bug: String "0000000000" becomes 0000000000 (invalid JSON) - * Root cause: Overly broad regex pattern in getReplacePattern() method - * - * @return void + * @return array,array}> */ - public function testBigIntSerializationDoesNotAffectStringFields(): void { - // This is the exact scenario that was failing - $inputData = [ - [ - 'columns' => [ - ['id' => ['type' => 'long long']], - ['s' => ['type' => 'string']], - ['v' => ['type' => 'string']], + public static function realManticoreResponseProvider(): array { + return [ + 'complete_data_type_coverage' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], // bigint (should be unquoted) + ['title' => ['type' => 'string']], // string (should stay quoted) + ['price' => ['type' => 'float']], // float (should stay unquoted) + ['count' => ['type' => 'long']], // integer (should stay unquoted) + ['is_active' => ['type' => 'long']], // boolean-as-int (should stay unquoted) + ['tags' => ['type' => 'string']], // MVA as string (should stay quoted) + ['meta' => ['type' => 'string']], // JSON as string (should stay quoted) + ], + 'data' => [ + [ + 'id' => 5623974933752184833, // bigint (should be unquoted) + 'title' => '000123', // numeric-looking string (should stay quoted) + 'price' => 19.990000, // float (should stay unquoted) + 'count' => 100, // int (should stay unquoted) + 'is_active' => 1, // bool-as-int (should stay unquoted) + 'tags' => '1,2,3', // MVA string (should stay quoted) + 'meta' => '{"category":"electronics","rating":4.500000}', // JSON (should stay quoted) + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', ], - 'data' => [ - [ - 'id' => 5047479470261279290, // This is a bigint - 's' => '0000000000', // This should remain a quoted string - 'v' => '0.44721356,0.89442712', + ['data.0.id'], // Only id is bigint + [ + 'bigint_unquoted' => '"id":5623974933752184833', + 'string_quoted' => '"title":"000123"', + 'float_unquoted' => '"price":19.99', + 'int_unquoted' => '"count":100', + 'bool_unquoted' => '"is_active":1', + 'mva_quoted' => '"tags":"1,2,3"', + 'json_quoted' => '"meta":"{\"category\":\"electronics\"', + ], + ], + 'edge_case_numeric_strings' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['code' => ['type' => 'string']], + ['tags' => ['type' => 'string']], + ['meta' => ['type' => 'string']], ], + 'data' => [ + [ + 'id' => 9223372036854775807, // bigint (should be unquoted) + 'code' => '0000000000', // original bug scenario (should stay quoted) + 'tags' => '2808348671,2808348672', // MVA with large numbers (should stay quoted) + 'meta' => '{"numbers":[1,2.500000,true,false,null],"text":"000456"}', // complex JSON + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + [ + 'bigint_unquoted' => '"id":9223372036854775807', + 'code_quoted' => '"code":"0000000000"', + 'tags_quoted' => '"tags":"2808348671,2808348672"', + 'meta_quoted' => '"meta":"{\"numbers\":[1,2.500000,true,false,null]', ], - 'total' => 1, - 'error' => '', - 'warning' => '', ], ]; + } - // Create struct with bigint field marked (as Client.php does) - $struct = Struct::fromData($inputData, ['data.0.id']); + /** + * Data provider for edge cases with numeric-looking strings + * These are critical because the regex could potentially affect them + * + * @return array + */ + public static function edgeCaseDataProvider(): array { + return [ + 'string_that_looks_like_bigint' => [ + '9223372036854775807', + 'string that looks exactly like a bigint', + ], + 'original_bug_scenario' => [ + '0000000000', + 'original bug scenario - zero-padded string', + ], + 'negative_number_string' => [ + '-9223372036854775808', + 'negative number as string', + ], + 'mva_with_large_numbers' => [ + '2808348671,2808348672', + 'MVA with large integers as comma-separated string', + ], + 'json_with_mixed_numbers' => [ + '{"numbers":[1,2.500000,true,false,null],"text":"000456"}', + 'JSON string containing various number formats', + ], + 'high_precision_decimal' => [ + '123.456789012345', + 'high precision decimal as string', + ], + 'string_with_spaces_and_numbers' => [ + ' 123 456 ', + 'string with spaces and numbers', + ], + 'hex_like_string' => [ + '0x123456789', + 'hex-like string', + ], + 'binary_like_string' => [ + '0b101010', + 'binary-like string', + ], + ]; + } - // Get the JSON output + /** + * Data provider for mixed data type scenarios + * Simulates real database responses with various field types + * + * @return array,array,array}> + */ + public static function mixedDataTypeScenarios(): array { + return [ + 'database_row_simulation' => [ + [ + 'id' => 9223372036854775807, // bigint + 'user_id' => 12345, // regular int + 'balance' => 1234.56, // float + 'is_premium' => 1, // boolean as int + 'permissions' => '1,5,10,15', // MVA as string + 'settings' => '{"theme":"dark","lang":"en"}', // JSON as string + 'code' => '00001', // numeric string + 'phone' => '+1-555-0123', // string with numbers + ], + ['id'], // only id is bigint + ['permissions', 'settings', 'code', 'phone'], // fields that must stay quoted + ], + 'meta_response_with_bigints' => [ + [ + 'id' => 5623974933752184833, + 'data' => '000000', + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['id'], + ['data'], // the data field should stay quoted + ], + ]; + } + + /** + * Test real ManticoreSearch data types to ensure comprehensive coverage + * + * @dataProvider realManticoreResponseProvider + * @param array> $inputData + * @param array $bigintFields + * @param array $expectedPatterns + * @return void + */ + public function testRealManticoreDataTypes(array $inputData, array $bigintFields, array $expectedPatterns): void { + $struct = Struct::fromData($inputData, $bigintFields); $json = $struct->toJson(); - // Parse the JSON to verify it's valid + // Verify JSON is valid + /** @var array>|null $decoded */ $decoded = json_decode($json, true); $this->assertNotNull($decoded, 'Generated JSON should be valid and parseable'); + $this->assertIsArray($decoded, 'Decoded JSON should be an array'); - // Verify the bigint field is correctly handled (no quotes) - $this->assertStringContainsString( - '"id":5047479470261279290', $json, - 'Bigint field should be serialized without quotes' - ); + // Verify all expected patterns are present + foreach ($expectedPatterns as $description => $pattern) { + $this->assertStringContainsString( + $pattern, + $json, + "Pattern '{$description}' should be present in JSON output" + ); + } - // Verify string fields maintain their quotes (this was the bug) - $this->assertStringContainsString( - '"s":"0000000000"', $json, - 'String fields should maintain their quotes' - ); - $this->assertStringContainsString( - '"v":"0.44721356,0.89442712"', $json, - 'String fields should maintain their quotes' - ); + // Verify bigint fields are unquoted while others maintain their correct format + foreach ($bigintFields as $fieldPath) { + $pathParts = explode('.', $fieldPath); + $currentData = $inputData; + + // Validate path traversal using assertions + foreach ($pathParts as $part) { + $this->assertIsArray( + $currentData, + "Path traversal failed at '{$part}' for field path '{$fieldPath}'" + ); + $this->assertArrayHasKey( + $part, + $currentData, + "Path component '{$part}' not found in field path '{$fieldPath}'" + ); + $currentData = $currentData[$part]; + } + + // At this point, $currentData should be the final scalar value + $this->assertIsScalar( + $currentData, + "Bigint field '{$fieldPath}' should be scalar, got " . gettype($currentData) + ); - // Verify the decoded values are correct - $this->assertEquals(5047479470261279290, $decoded[0]['data'][0]['id']); - $this->assertEquals('0000000000', $decoded[0]['data'][0]['s']); - $this->assertEquals('0.44721356,0.89442712', $decoded[0]['data'][0]['v']); + $bigintValue = (string)$currentData; + $this->assertStringContainsString( + '":' . $bigintValue, + $json, + "Bigint field '{$fieldPath}' should be unquoted" + ); + } } /** - * Test various edge cases where string values might be confused with bigints + * Test edge cases where strings contain numeric content + * These are critical because the regex could potentially affect them * + * @dataProvider edgeCaseDataProvider + * @param string $stringField + * @param string $description * @return void */ - public function testStringFieldsWithNumericContentAreNotAffected(): void { - $testCases = [ - ['field' => '0000000000', 'description' => 'zero-padded string'], - ['field' => '1234567890123456789', 'description' => 'long numeric string'], - ['field' => '00123', 'description' => 'zero-padded numeric string'], - ['field' => '1.23456789', 'description' => 'decimal string'], - ['field' => '+1234567890', 'description' => 'string with plus sign'], - ['field' => '-9876543210', 'description' => 'string with minus sign'], - ['field' => '1e10', 'description' => 'scientific notation string'], + public function testNumericStringEdgeCases(string $stringField, string $description): void { + $data = [ + 'bigint_field' => 9223372036854775807, // Real bigint + 'string_field' => $stringField, // String that might look numeric + 'normal_field' => 'test', ]; - foreach ($testCases as $testCase) { - $data = [ - 'bigint_field' => 9223372036854775807, // Real bigint - 'string_field' => $testCase['field'], // String that might look numeric - 'normal_field' => 'test', - ]; + $struct = Struct::fromData($data, ['bigint_field']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, "Generated JSON should be valid for {$description}"); + $this->assertIsArray($decoded, "Decoded JSON should be an array for {$description}"); - $struct = Struct::fromData($data, ['bigint_field']); - $json = $struct->toJson(); + // Verify bigint is unquoted + $this->assertStringContainsString('"bigint_field":9223372036854775807', $json); - // Verify JSON is valid - $decoded = json_decode($json, true); - $this->assertNotNull($decoded, "Generated JSON should be valid for {$testCase['description']}"); + // Verify string field maintains quotes (this is the critical test) + $expectedStringJson = '"string_field":"' . addslashes($stringField) . '"'; + $this->assertStringContainsString( + $expectedStringJson, + $json, + "String field should maintain quotes for {$description}" + ); - // Verify bigint is unquoted - $this->assertStringContainsString('"bigint_field":9223372036854775807', $json); + // Verify decoded values with proper type checking + $this->assertEquals(9223372036854775807, $decoded['bigint_field']); + $this->assertEquals($stringField, $decoded['string_field']); + } - // Verify string field maintains quotes - $expectedStringJson = '"string_field":"' . $testCase['field'] . '"'; + /** + * Test mixed data type scenarios simulating real database responses + * + * @dataProvider mixedDataTypeScenarios + * @param array $data + * @param array $bigintFields + * @param array $criticalQuotes + * @return void + */ + public function testMixedDataTypeScenarios(array $data, array $bigintFields, array $criticalQuotes): void { + $struct = Struct::fromData($data, $bigintFields); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Mixed data type JSON should be valid'); + $this->assertIsArray($decoded, 'Decoded JSON should be an array'); + + // Verify bigint fields are unquoted + foreach ($bigintFields as $field) { + if (!isset($data[$field])) { + continue; + } + + $value = is_scalar($data[$field]) ? (string)$data[$field] : ''; $this->assertStringContainsString( - $expectedStringJson, $json, - "String field should maintain quotes for {$testCase['description']}" + '"' . $field . '":' . $value, + $json, + "Bigint field '{$field}' should be unquoted" ); + } + + // Verify critical string fields maintain quotes + foreach ($criticalQuotes as $field) { + if (!isset($data[$field])) { + continue; + } - // Verify decoded values - $this->assertEquals(9223372036854775807, $decoded['bigint_field']); - $this->assertEquals($testCase['field'], $decoded['string_field']); + $value = is_scalar($data[$field]) ? (string)$data[$field] : ''; + $expectedPattern = '"' . $field . '":"' . addslashes($value) . '"'; + $this->assertStringContainsString( + $expectedPattern, + $json, + "Critical string field '{$field}' should maintain quotes" + ); } } /** - * Test nested structures with mixed bigint and string fields + * Test float precision preservation + * Ensures that float values are not affected by the bigint regex * * @return void */ - public function testNestedStructuresWithMixedTypes(): void { + public function testFloatPrecisionPreservation(): void { $data = [ - 'level1' => [ - 'bigint' => 5047479470261279290, - 'string' => '0000000000', - 'level2' => [ - 'bigint' => 9223372036854775807, - 'string' => '1234567890123456789', - ], - ], + 'id' => 9223372036854775807, + 'price' => 123.456789012345, + 'rate' => 0.000001, + 'negative_float' => -456.789, + ]; + + $struct = Struct::fromData($data, ['id']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Float precision JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); + + // Verify floats remain unquoted and preserve precision + $this->assertStringContainsString('"price":123.456789012345', $json); + $this->assertStringContainsString('"rate":1.0e-6', $json); + $this->assertStringContainsString('"negative_float":-456.789', $json); + + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals(123.456789012345, $decoded['price']); + $this->assertEquals(0.000001, $decoded['rate']); + $this->assertEquals(-456.789, $decoded['negative_float']); + } + + /** + * Test MVA (Multi-Value Attribute) string preservation + * MVA fields are returned as comma-separated strings and must stay quoted + * + * @return void + */ + public function testMVAStringPreservation(): void { + $data = [ + 'id' => 9223372036854775807, + 'tags' => '1,2,3,100,200', + 'permissions' => '2808348671,2808348672,2808348673', + 'empty_mva' => '', + ]; + + $struct = Struct::fromData($data, ['id']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'MVA string JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); + + // Verify MVA strings maintain quotes + $this->assertStringContainsString('"tags":"1,2,3,100,200"', $json); + $this->assertStringContainsString('"permissions":"2808348671,2808348672,2808348673"', $json); + $this->assertStringContainsString('"empty_mva":""', $json); + + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals('1,2,3,100,200', $decoded['tags']); + $this->assertEquals('2808348671,2808348672,2808348673', $decoded['permissions']); + $this->assertEquals('', $decoded['empty_mva']); + } + + /** + * Test JSON string field preservation + * JSON fields are stored as strings and must maintain their quotes + * + * @return void + */ + public function testJSONStringPreservation(): void { + $data = [ + 'id' => 9223372036854775807, + 'settings' => '{"theme":"dark","lang":"en","version":1}', + 'metadata' => '{"numbers":[1,2.5,true,false,null],"text":"000456"}', + 'empty_json' => '{}', ]; - $struct = Struct::fromData($data, ['level1.bigint', 'level1.level2.bigint']); + $struct = Struct::fromData($data, ['id']); $json = $struct->toJson(); // Verify JSON is valid + /** @var array|null $decoded */ $decoded = json_decode($json, true); - $this->assertNotNull($decoded, 'Nested structure JSON should be valid'); + $this->assertNotNull($decoded, 'JSON string JSON should be valid'); - // Verify bigints are unquoted - $this->assertStringContainsString('"bigint":5047479470261279290', $json); - $this->assertStringContainsString('"bigint":9223372036854775807', $json); + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); - // Verify strings maintain quotes - $this->assertStringContainsString('"string":"0000000000"', $json); - $this->assertStringContainsString('"string":"1234567890123456789"', $json); + // Verify JSON strings maintain quotes + $this->assertStringContainsString( + '"settings":"{\\"theme\\":\\"dark\\",\\"lang\\":\\"en\\",\\"version\\":1}"', + $json + ); + $this->assertStringContainsString( + '"metadata":"{\\"numbers\\":[1,2.5,true,false,null],\\"text\\":\\"000456\\"}"', + $json + ); + $this->assertStringContainsString('"empty_json":"{}"', $json); - // Verify structure integrity - $this->assertEquals('0000000000', $decoded['level1']['string']); - $this->assertEquals('1234567890123456789', $decoded['level1']['level2']['string']); + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals('{"theme":"dark","lang":"en","version":1}', $decoded['settings']); + $this->assertEquals('{"numbers":[1,2.5,true,false,null],"text":"000456"}', $decoded['metadata']); + $this->assertEquals('{}', $decoded['empty_json']); } /** - * Test the actual Client.php scenario that was failing + * Test negative number handling + * Ensures negative bigints and regular numbers work correctly * * @return void */ - public function testClientMetaResponseScenario(): void { + public function testNegativeNumberHandling(): void { + $data = [ + 'negative_bigint' => -9223372036854775808, + 'negative_int' => -42, + 'negative_float' => -123.456, + 'string_negative' => '-999', + ]; + + $struct = Struct::fromData($data, ['negative_bigint']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Negative number JSON should be valid'); + + // Verify negative bigint is unquoted + $this->assertStringContainsString('"negative_bigint":-9.223372036854776e+18', $json); + + // Verify other negative numbers remain unquoted + $this->assertStringContainsString('"negative_int":-42', $json); + $this->assertStringContainsString('"negative_float":-123.456', $json); + + // Verify negative string maintains quotes + $this->assertStringContainsString('"string_negative":"-999"', $json); + + // Verify decoded values are correct + $this->assertEquals(-9223372036854775808, $decoded['negative_bigint']); + $this->assertEquals(-42, $decoded['negative_int']); + $this->assertEquals(-123.456, $decoded['negative_float']); + $this->assertEquals('-999', $decoded['string_negative']); + } + + /** + * Test the actual Client.php:191 scenario that was failing + * This simulates the exact bug scenario with SHOW META responses + * + * @return void + */ + public function testClientMetaResponseIntegration(): void { // Simulate the exact data structure from Client.php:191 $array = [ [ @@ -186,7 +508,7 @@ public function testClientMetaResponseScenario(): void { $response = Struct::fromData($array, ['data.0.id'])->toJson(); // Verify the response is valid JSON - $this->assertNotNull(json_decode($response), 'Response should be valid JSON'); + $this->assertNotNull(json_decode($response), 'Client response should be valid JSON'); // Verify the specific issue is fixed $this->assertStringNotContainsString( @@ -197,5 +519,11 @@ public function testClientMetaResponseScenario(): void { ',"s":"0000000000",', $response, 'String field should maintain its quotes' ); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":5047479470261279290', $response); + + // Verify other string field maintains quotes + $this->assertStringContainsString('"v":"0.44721356,0.89442712"', $response); } } diff --git a/test/BuddyCore/Network/StructSingleResponseTest.php b/test/BuddyCore/Network/StructSingleResponseTest.php new file mode 100644 index 0000000..19f4b87 --- /dev/null +++ b/test/BuddyCore/Network/StructSingleResponseTest.php @@ -0,0 +1,572 @@ + [ + ['id' => ['type' => 'long long']], + ['s' => ['type' => 'string']], + ['v' => ['type' => 'string'], + ], + 'data' => [ + [ + 'id' => 5047479470261279290, // bigint (should be unquoted) + 's' => '0000000000', // string (should stay quoted) + 'v' => '0.44721356,0.89442712', + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ], + ]; + + // This simulates the sizeof($array) == 1 scenario that was bypassed in Client.php + $response = Struct::fromData($singleResponseArray, ['data.0.id'])->toJson(); + + // Verify JSON is valid (this was failing in production) + $this->assertNotNull(json_decode($response), 'Single response JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":5047479470261279290', $response); + + // Verify strings maintain quotes (critical test - this was the bug) + $this->assertStringContainsString('"s":"0000000000"', $response); + $this->assertStringContainsString('"v":"0.44721356,0.89442712"', $response); + + // Verify no invalid unquoted strings (the exact production bug) + $this->assertStringNotContainsString(',"s":0000000000,', $response); + } + + /** + * Data provider for KNN-specific scenarios + * + * @return array,array,array}> + */ + public static function knnResponseProvider(): array { + return [ + 'knn_with_vector_field' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 9223372036854775807, 'v' => '0.1,0.2,0.3'], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"id":9223372036854775807', '"v":"0.1,0.2,0.3"'], + ], + 'knn_with_multiple_bigints' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['user_id' => ['type' => 'long long']], + ], + 'data' => [ + ['id' => 1234567890123456789, 'user_id' => 9876543210987654321], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id', 'data.0.user_id'], + ['"id":1234567890123456789', '"user_id":9.876543210987655e+18'], + ], + 'knn_with_numeric_strings' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['code' => ['type' => 'string']], + ['tags' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 5623974933752184833, + 'code' => '000123', + 'tags' => '999,1000,1001', + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"id":5623974933752184833', '"code":"000123"', '"tags":"999,1000,1001"'], + ], + ]; + } + + /** + * @dataProvider knnResponseProvider + * @param array $data + * @param array $bigintFields + * @param array $expectedPatterns + * @return void + */ + public function testKNNSpecificScenarios(array $data, array $bigintFields, array $expectedPatterns): void { + $response = Struct::fromData([$data], $bigintFields)->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'KNN response JSON should be valid'); + + // Verify all expected patterns are present + foreach ($expectedPatterns as $pattern) { + $this->assertStringContainsString($pattern, $response); + } + } + + /** + * Test the actual Client.php scenario that was failing + * This simulates the exact bug scenario with SHOW META responses + * + * @return void + */ + public function testClientMetaResponseIntegration(): void { + // Simulate the exact data structure from Client.php:191 + $array = [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['s' => ['type' => 'string']], + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 5047479470261279290, 's' => '0000000000', 'v' => '0.44721356,0.89442712'], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ]; + + // This is the exact line that was causing the issue + $response = Struct::fromData($array, ['data.0.id'])->toJson(); + + // Verify the response is valid JSON + $this->assertNotNull(json_decode($response), 'Client response should be valid JSON'); + + // Verify the specific issue is fixed + $this->assertStringNotContainsString( + ',"s":0000000000,', $response, + 'String field should not lose its quotes' + ); + $this->assertStringContainsString( + ',"s":"0000000000",', $response, + 'String field should maintain its quotes' + ); + } + + /** + * Test that BigInt field detection now works correctly using column type information + * This verifies the root cause fix for the production bug + * + * @return void + */ + public function testBigIntFieldDetectionFromColumns(): void { + // Simulate the exact production response structure + $jsonResponse = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], // This should be detected as bigint + ['s' => ['type' => 'string']], // This should NOT be detected as bigint + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 5047479470261279290, 's' => '0000000000', 'v' => '0.44721356,0.89442712'], + + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ] + ); + + $this->assertNotNull($jsonResponse, 'JSON encoding should work'); + + // Create struct using fromJson (which should now use column-based detection) + $struct = Struct::fromJson($jsonResponse); + + // Verify BigInt fields are correctly identified + $bigIntFields = $struct->getBigIntFields(); + + // Should contain the bigint field, not the string field + echo json_encode($bigIntFields, JSON_PRETTY_PRINT); + $this->assertContains('data.0.id', $bigIntFields, 'BigInt field should be correctly identified'); + $this->assertNotContains('data.0.s', $bigIntFields, 'String field should not be misidentified as BigInt'); + + // Verify JSON serialization works correctly + $json = $struct->toJson(); + $this->assertNotNull(json_decode($json), 'Serialized JSON should be valid'); + + // Verify bigint is unquoted and string is quoted + $this->assertStringContainsString('"id":5047479470261279290', $json); + $this->assertStringContainsString('"s":"0000000000"', $json); + $this->assertStringNotContainsString('"s":0000000000', $json); + } + + /** + * Test edge cases for single response processing + * + * @return void + */ + public function testSingleResponseEdgeCases(): void { + $edgeCases = [ + 'empty_data' => [ + 'data' => [], + 'bigint_fields' => [], + 'should_be_valid' => true, + ], + 'null_values' => [ + 'data' => [ + 'id' => null, + 'name' => 'test', + ], + 'bigint_fields' => [], + 'should_be_valid' => true, + ], + 'mixed_types' => [ + 'data' => [ + 'id' => 123, + 'active' => true, + 'score' => 95.5, + 'tags' => 'tag1,tag2', + ], + 'bigint_fields' => ['id'], + 'should_be_valid' => true, + ], + ]; + + foreach ($edgeCases as $caseName => $case) { + $testData = [$case['data']]; + $response = Struct::fromData($testData, $case['bigint_fields'])->toJson(); + + if (!$case['should_be_valid']) { + continue; + } + + $this->assertNotNull( + json_decode($response), + "Edge case '{$caseName}' should produce valid JSON" + ); + } + } + + /** + * Test PHP_INT_MAX boundary (9223372036854775807) + * Within PHP integer limits, stays as integer in PHP + * + * @return void + */ + public function testPHPIntMaxBoundary(): void { + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['description' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 9223372036854775807, + 'description' => 'test_description', + ], + ], + ] + ); + + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'PHP_INT_MAX should produce valid JSON'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $response); + + // Verify string representation stays quoted + $this->assertStringContainsString('"description":"test_description"', $response); + + // Verify bigint field was detected from columns + $this->assertContains('data.0.id', $struct->getBigIntFields()); + } + + /** + * Test PHP_INT_MAX + 1 (9223372036854775808) + * Exceeds PHP_INT_MAX, becomes float in PHP, json_encode uses scientific notation + * This tests that column metadata correctly identifies it as bigint + * + * @return void + */ + public function testPHPIntMaxPlusOne(): void { + // PHP_INT_MAX + 1 becomes float in JSON decode + // json_encode will output: 9.2233720368547758e+18 + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['description' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 9223372036854775808, // Beyond PHP_INT_MAX + 'description' => '9223372036854775808', + ], + ], + ] + ); + + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid (this would fail without column metadata!) + $this->assertNotNull( + json_decode($response), + 'PHP_INT_MAX + 1 should be handled correctly via column metadata' + ); + + // Verify bigint field is detected from columns + $this->assertContains('data.0.id', $struct->getBigIntFields()); + + // Verify string field is NOT detected as bigint + $this->assertNotContains('data.0.description', $struct->getBigIntFields()); + } + + /** + * Test PHP_INT_MIN boundary (-9223372036854775808) + * Negative boundary value, stays as integer in PHP + * Note: -9223372036854775808 becomes scientific notation in some JSON encodes + * + * @return void + */ + public function testPHPIntMinBoundary(): void { + $jsonInput = json_encode( + [ + 'columns' => [ + ['balance' => ['type' => 'long long']], + ['label' => ['type' => 'string']], + ], + 'data' => [ + [ + 'balance' => -9223372036854775807, // Use -1 to avoid scientific notation edge case + 'label' => 'test_label', + ], + ], + ] + ); + + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'PHP_INT_MIN should produce valid JSON'); + + // Verify negative bigint is unquoted + $this->assertStringContainsString('"balance":-9223372036854775807', $response); + + // Verify string stays quoted + $this->assertStringContainsString('"label":"test_label"', $response); + + // Verify bigint field was detected from columns + $this->assertContains('data.0.balance', $struct->getBigIntFields()); + } + + /** + * Test PHP_INT_MIN - 1 (-9223372036854775809) + * Exceeds PHP_INT_MIN, becomes float in PHP, json_encode uses scientific notation + * This tests that column metadata correctly identifies it as bigint + * + * @return void + */ + public function testPHPIntMinMinusOne(): void { + // PHP_INT_MIN - 1 becomes float in JSON decode + // json_encode will output: -9.2233720368547758e+18 + $jsonInput = json_encode( + [ + 'columns' => [ + ['balance' => ['type' => 'long long']], + ['label' => ['type' => 'string']], + ], + 'data' => [ + [ + 'balance' => -9223372036854775809, // Beyond PHP_INT_MIN + 'label' => '-9223372036854775809', + ], + ], + ] + ); + + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid (this would fail without column metadata!) + $this->assertNotNull( + json_decode($response), + 'PHP_INT_MIN - 1 should be handled correctly via column metadata' + ); + + // Verify bigint field is detected from columns + $this->assertContains('data.0.balance', $struct->getBigIntFields()); + + // Verify string field is NOT detected as bigint + $this->assertNotContains('data.0.label', $struct->getBigIntFields()); + } + + /** + * Test maximum unsigned 64-bit value (2^64 - 1) + * 18446744073709551615 - the largest possible 64-bit unsigned integer + * + * @return void + */ + public function testMax64BitUnsigned(): void { + // Max uint64 = 18446744073709551615 + $maxUint64 = 18446744073709551615; + + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['code' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => $maxUint64, // Max uint64 + 'code' => '18446744073709551615', + ], + ], + ] + ); + + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'Max uint64 should produce valid JSON'); + + // Verify bigint field is detected from columns + $this->assertContains('data.0.id', $struct->getBigIntFields()); + + // Verify string field stays quoted + $this->assertStringContainsString('"code":"18446744073709551615"', $response); + } + + /** + * Critical test: Numeric string at boundary values should NOT be marked as bigint + * even if they look like large numbers, because they're not in 'long long' columns + * This proves the hybrid approach prevents false positives + * + * @return void + */ + public function testNumericStringNotMisidentifiedAsBigint(): void { + // This tests that even very large numeric strings are kept quoted + // when they're not marked as 'long long' in columns + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['huge_string' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 123, + 'huge_string' => '18446744073709551615', // Looks like bigint but is string! + ], + ], + ] + ); + + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'Should handle large numeric strings'); + + // Verify bigint detection is correct + $this->assertContains('data.0.id', $struct->getBigIntFields()); + $this->assertNotContains('data.0.huge_string', $struct->getBigIntFields()); + + // Verify string stays quoted (this is the critical check) + $this->assertStringContainsString('"huge_string":"18446744073709551615"', $response); + $this->assertStringNotContainsString('"huge_string":18446744073709551615', $response); + } + + /** + * Test isBigIntBoundary() helper using reflection for precise boundary detection + * This tests the mathematical correctness of the new heuristic fallback + * + * @return void + */ + public function testIsBigIntBoundaryDetection(): void { + // Use reflection to access the private method + $method = new ReflectionMethod(Struct::class, 'isBigIntBoundary'); + $method->setAccessible(true); + + // Test cases: [value, expected_result] + $testCases = [ + // Within PHP_INT_MAX (9223372036854775807) + ['1', false], + ['123', false], + ['9223372036854775806', false], // PHP_INT_MAX - 1 + ['9223372036854775807', false], // PHP_INT_MAX (at boundary) + + // Beyond PHP_INT_MAX + ['9223372036854775808', true], // PHP_INT_MAX + 1 + ['18446744073709551615', true], // Max uint64 + + // Large numbers with many digits + ['12345678901234567890', true], // 20 digits + ['123456789012345678901', true], // 21 digits + + // Within PHP_INT_MIN (-9223372036854775808) + ['-1', false], + ['-123', false], + ['-9223372036854775807', false], // PHP_INT_MIN + 1 + ['-9223372036854775808', false], // PHP_INT_MIN (at boundary) + + // Beyond PHP_INT_MIN + ['-9223372036854775809', true], // PHP_INT_MIN - 1 + ['-18446744073709551615', true], // Negative max uint64 + + // Zero-padded numbers (common source of false positives) + ['0000000000', false], // Padded zero + ['00123', false], // Padded small number + ['0000009223372036854775807', false], // Padded PHP_INT_MAX + + // Negative zero-padded + ['-0000000001', false], // Padded negative + ]; + + foreach ($testCases as [$value, $expected]) { + $result = $method->invoke(null, $value); + $this->assertSame( + $expected, + $result, + "isBigIntBoundary('{$value}') should return " . ($expected ? 'true' : 'false') + ); + } + } +} From d65a422840783abae25a553b3bd5ab7cf4110af3 Mon Sep 17 00:00:00 2001 From: djklim87 Date: Mon, 1 Dec 2025 18:51:43 +0100 Subject: [PATCH 4/6] Fix: Struct parsing issues leading to Wrong JSON parsing response if id is bigint --- src/Network/Struct.php | 24 +++- .../Network/StructSingleResponseTest.php | 123 ++++++++++++++++-- 2 files changed, 133 insertions(+), 14 deletions(-) diff --git a/src/Network/Struct.php b/src/Network/Struct.php index 899ee1a..9fecf50 100644 --- a/src/Network/Struct.php +++ b/src/Network/Struct.php @@ -83,7 +83,14 @@ public function offsetExists(mixed $name): bool { */ public function offsetUnset(mixed $name): void { unset($this->data[$name]); - $this->keys = array_keys($this->data); + $this->keys = array_values( + array_filter( + $this->keys, + static function (mixed $key) use ($name): bool { + return $key !== $name; + } + ) + ); } /** @@ -219,7 +226,7 @@ static function (array $matches): string { }, $serialized ); - if ($json === false) { + if (!is_string($json)) { throw new Exception('Cannot encode data to JSON'); } return $json; @@ -242,7 +249,7 @@ private static function getReplacePattern(string $path): string { } } - return $pattern . '("?)(\-?\d+)\1/'; + return $pattern . '("?)([^"{}[\],]+)\1/'; } /** @@ -265,6 +272,8 @@ private static function traverseAndTrack( return; } + /** @var array */ + $bigIntFieldsLookup = array_flip($bigIntFields); foreach ($data as $key => &$value) { $currentPath = $path ? "$path.$key" : "$key"; if (!isset($originalData[$key])) { @@ -274,10 +283,11 @@ private static function traverseAndTrack( $originalValue = $originalData[$key]; if (is_array($value) && is_array($originalValue)) { static::traverseAndTrack($value, $originalValue, $bigIntFields, $currentPath); + $bigIntFieldsLookup = array_flip($bigIntFields); continue; } - static::trackBigIntIfNeeded($value, $originalValue, $currentPath, $bigIntFields); + static::trackBigIntIfNeeded($value, $originalValue, $currentPath, $bigIntFields, $bigIntFieldsLookup); } } @@ -331,13 +341,15 @@ private static function isBigIntBoundary(string $value): bool { * @param mixed $originalValue * @param string $currentPath * @param array &$bigIntFields + * @param array $bigIntFieldsLookup Fast O(1) lookup set * @return void */ private static function trackBigIntIfNeeded( mixed $value, mixed $originalValue, string $currentPath, - array &$bigIntFields + array &$bigIntFields, + array $bigIntFieldsLookup = [] ): void { if (!is_string($value) || !is_numeric($originalValue)) { return; @@ -349,7 +361,7 @@ private static function trackBigIntIfNeeded( } // Skip if already identified via column metadata - if (in_array($currentPath, $bigIntFields, true)) { + if (isset($bigIntFieldsLookup[$currentPath])) { return; } diff --git a/test/BuddyCore/Network/StructSingleResponseTest.php b/test/BuddyCore/Network/StructSingleResponseTest.php index 19f4b87..8027a8b 100644 --- a/test/BuddyCore/Network/StructSingleResponseTest.php +++ b/test/BuddyCore/Network/StructSingleResponseTest.php @@ -62,7 +62,7 @@ public function testSingleResponseBigIntSerialization(): void { /** * Data provider for KNN-specific scenarios * - * @return array,array,array}> + * @return array,array,array}> */ public static function knnResponseProvider(): array { return [ @@ -119,6 +119,104 @@ public static function knnResponseProvider(): array { ['data.0.id'], ['"id":5623974933752184833', '"code":"000123"', '"tags":"999,1000,1001"'], ], + 'with_quoted_scientific_notation' => [ + [ + 'columns' => [ + ['user_id' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'user_id' => 9876543210987654321, // Becomes float in PHP, then quoted in JSON + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.user_id'], + ['"user_id":9.876543210987655e+18'], // Should be unquoted scientific notation + ], + 'with_unquoted_scientific_notation' => [ + [ + 'columns' => [ + ['balance' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'balance' => 9223372036854775808, // Beyond PHP_INT_MAX, becomes float + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.balance'], + [ + // Regex pattern to match scientific notation with varying precision + ['/balance":9\.22337203685477[0-9]e\+18/', 'Scientific notation should be unquoted'], + ], + ], + 'with_float_field' => [ + [ + 'columns' => [ + ['price' => ['type' => 'float']], + ['id' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'price' => 1234567890.123, + 'id' => 9223372036854775807, + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"price":1234567890.123', '"id":9223372036854775807'], + ], + 'with_mva_array_field' => [ + [ + 'columns' => [ + ['tags' => ['type' => 'uint', 'multi' => true]], // MVA - multi-value attribute + ['id' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'tags' => [1, 2, 3, 9223372036854775807], + 'id' => 5623974933752184833, + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"tags":[1,2,3,9223372036854775807]', '"id":5623974933752184833'], + ], + 'with_mixed_types' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['name' => ['type' => 'string']], + ['price' => ['type' => 'float']], + ['tags' => ['type' => 'string']], // JSON-serialized array + ], + 'data' => [ + [ + 'id' => 9223372036854775807, + 'name' => 'product_name', + 'price' => 99.99, + 'tags' => '[1,2,3]', // JSON array as string + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"id":9223372036854775807', '"name":"product_name"', '"price":99.99', '"tags":"[1,2,3]"'], + ], ]; } @@ -126,7 +224,7 @@ public static function knnResponseProvider(): array { * @dataProvider knnResponseProvider * @param array $data * @param array $bigintFields - * @param array $expectedPatterns + * @param array $expectedPatterns * @return void */ public function testKNNSpecificScenarios(array $data, array $bigintFields, array $expectedPatterns): void { @@ -137,7 +235,13 @@ public function testKNNSpecificScenarios(array $data, array $bigintFields, array // Verify all expected patterns are present foreach ($expectedPatterns as $pattern) { - $this->assertStringContainsString($pattern, $response); + if (is_array($pattern)) { + // For regex patterns (used for floating-point precision variations) + $this->assertMatchesRegularExpression($pattern[0], $response, $pattern[1]); + } else { + // For exact string matches + $this->assertStringContainsString($pattern, $response); + } } } @@ -207,7 +311,7 @@ public function testBigIntFieldDetectionFromColumns(): void { ] ); - $this->assertNotNull($jsonResponse, 'JSON encoding should work'); + $this->assertIsString($jsonResponse, 'JSON encoding should work'); // Create struct using fromJson (which should now use column-based detection) $struct = Struct::fromJson($jsonResponse); @@ -266,10 +370,7 @@ public function testSingleResponseEdgeCases(): void { $testData = [$case['data']]; $response = Struct::fromData($testData, $case['bigint_fields'])->toJson(); - if (!$case['should_be_valid']) { - continue; - } - + // All test cases are expected to produce valid JSON $this->assertNotNull( json_decode($response), "Edge case '{$caseName}' should produce valid JSON" @@ -299,6 +400,7 @@ public function testPHPIntMaxBoundary(): void { ] ); + $this->assertIsString($jsonInput); $struct = Struct::fromJson($jsonInput); $response = $struct->toJson(); @@ -340,6 +442,7 @@ public function testPHPIntMaxPlusOne(): void { ] ); + $this->assertIsString($jsonInput); $struct = Struct::fromJson($jsonInput); $response = $struct->toJson(); @@ -379,6 +482,7 @@ public function testPHPIntMinBoundary(): void { ] ); + $this->assertIsString($jsonInput); $struct = Struct::fromJson($jsonInput); $response = $struct->toJson(); @@ -420,6 +524,7 @@ public function testPHPIntMinMinusOne(): void { ] ); + $this->assertIsString($jsonInput); $struct = Struct::fromJson($jsonInput); $response = $struct->toJson(); @@ -461,6 +566,7 @@ public function testMax64BitUnsigned(): void { ] ); + $this->assertIsString($jsonInput); $struct = Struct::fromJson($jsonInput); $response = $struct->toJson(); @@ -499,6 +605,7 @@ public function testNumericStringNotMisidentifiedAsBigint(): void { ] ); + $this->assertIsString($jsonInput); $struct = Struct::fromJson($jsonInput); $response = $struct->toJson(); From 635614e676996449205a1a3adb760ade1f4f54c0 Mon Sep 17 00:00:00 2001 From: djklim87 Date: Thu, 18 Dec 2025 15:53:51 +0100 Subject: [PATCH 5/6] review fixes --- src/Network/Struct.php | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/Network/Struct.php b/src/Network/Struct.php index 9fecf50..2db7d0f 100644 --- a/src/Network/Struct.php +++ b/src/Network/Struct.php @@ -83,14 +83,7 @@ public function offsetExists(mixed $name): bool { */ public function offsetUnset(mixed $name): void { unset($this->data[$name]); - $this->keys = array_values( - array_filter( - $this->keys, - static function (mixed $key) use ($name): bool { - return $key !== $name; - } - ) - ); + $this->keys = array_keys($this->data); } /** @@ -156,7 +149,11 @@ public static function fromJson(string $json): self { // We need here to keep original json decode cuzit has bigIntFields /** @var array */ $modified = json_decode($json, true, static::JSON_DEPTH, static::JSON_FLAGS | JSON_BIGINT_AS_STRING); - static::traverseAndTrack($modified, $result, $bigIntFields); + + /** @var array $bigIntFieldsLookup */ + $bigIntFieldsLookup = $bigIntFields ? array_fill_keys($bigIntFields, 1) : []; + static::traverseAndTrack($modified, $result, $bigIntFields, $bigIntFieldsLookup); + $result = $modified; } @@ -226,7 +223,7 @@ static function (array $matches): string { }, $serialized ); - if (!is_string($json)) { + if (!isset($json)) { throw new Exception('Cannot encode data to JSON'); } return $json; @@ -259,6 +256,7 @@ private static function getReplacePattern(string $path): string { * @param mixed $data * @param mixed $originalData * @param array $bigIntFields + * @param array $bigIntFieldsLookup * @param string $path * @return void */ @@ -266,14 +264,13 @@ private static function traverseAndTrack( mixed &$data, mixed $originalData, array &$bigIntFields, + array &$bigIntFieldsLookup, string $path = '' ): void { if (!is_array($data) || !is_array($originalData)) { return; } - /** @var array */ - $bigIntFieldsLookup = array_flip($bigIntFields); foreach ($data as $key => &$value) { $currentPath = $path ? "$path.$key" : "$key"; if (!isset($originalData[$key])) { @@ -282,8 +279,7 @@ private static function traverseAndTrack( $originalValue = $originalData[$key]; if (is_array($value) && is_array($originalValue)) { - static::traverseAndTrack($value, $originalValue, $bigIntFields, $currentPath); - $bigIntFieldsLookup = array_flip($bigIntFields); + static::traverseAndTrack($value, $originalValue, $bigIntFields, $bigIntFieldsLookup, $currentPath); continue; } @@ -341,7 +337,7 @@ private static function isBigIntBoundary(string $value): bool { * @param mixed $originalValue * @param string $currentPath * @param array &$bigIntFields - * @param array $bigIntFieldsLookup Fast O(1) lookup set + * @param array &$bigIntFieldsLookup Fast O(1) lookup set * @return void */ private static function trackBigIntIfNeeded( @@ -349,7 +345,7 @@ private static function trackBigIntIfNeeded( mixed $originalValue, string $currentPath, array &$bigIntFields, - array $bigIntFieldsLookup = [] + array &$bigIntFieldsLookup ): void { if (!is_string($value) || !is_numeric($originalValue)) { return; @@ -360,11 +356,12 @@ private static function trackBigIntIfNeeded( return; } - // Skip if already identified via column metadata + // Skip if already identified via column metadata or earlier traversal if (isset($bigIntFieldsLookup[$currentPath])) { return; } + $bigIntFieldsLookup[$currentPath] = 1; $bigIntFields[] = $currentPath; } From c67426987c5b38add58f98044ebbdc49aa1eb9e5 Mon Sep 17 00:00:00 2001 From: djklim87 Date: Thu, 18 Dec 2025 16:44:18 +0100 Subject: [PATCH 6/6] review fixes --- src/Network/Struct.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Network/Struct.php b/src/Network/Struct.php index 2db7d0f..234e5b4 100644 --- a/src/Network/Struct.php +++ b/src/Network/Struct.php @@ -151,7 +151,7 @@ public static function fromJson(string $json): self { $modified = json_decode($json, true, static::JSON_DEPTH, static::JSON_FLAGS | JSON_BIGINT_AS_STRING); /** @var array $bigIntFieldsLookup */ - $bigIntFieldsLookup = $bigIntFields ? array_fill_keys($bigIntFields, 1) : []; + $bigIntFieldsLookup = []; static::traverseAndTrack($modified, $result, $bigIntFields, $bigIntFieldsLookup); $result = $modified;