Skip to content

Add JSON aggregate support for HasMany references #1286

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
woodholly wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

Add JSON aggregate support for HasMany references #1286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"oracle",
"postgresql"
],
"version": "dev-develop",
"version": "6.0.x-dev",
Copy link
Member

@mvorisek mvorisek Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

revert

"authors": [
{
"name": "Romans Malinovskis",
Expand Down
9 changes: 9 additions & 0 deletions src/Model/Join.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,15 @@ public function __construct(string $foreignTable)
}
}

/**
* Used internally for JSON aggregate field resolution with dot notation
* We need this to prevent using slow ReflectionClass.
*/
public function getForeignTable(): string
{
return $this->foreignTable;
}

/**
* @internal should be not used outside atk4/data, for Migrator only
*/
Expand Down
20 changes: 20 additions & 0 deletions src/Persistence/Sql/Mssql/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

use Atk4\Data\Exception;
use Atk4\Data\Field;
use Atk4\Data\Persistence\Sql\Expression as BaseExpression;
use Atk4\Data\Persistence\Sql\Expressionable;
use Atk4\Data\Persistence\Sql\Query as BaseQuery;
use Atk4\Data\Persistence\Sql\RawExpression;
Expand Down Expand Up @@ -263,6 +264,25 @@ public function jsonTable(Expressionable $json, array $columns, string $rowsPath
return $query;
}

/**
* @param array<string, Expressionable> $keyValuePairs
*/
public function fxJsonObject(array $keyValuePairs): BaseExpression
{
$parts = [];
foreach ($keyValuePairs as $key => $value) {
$parts[] = new RawExpression($this->escapeStringLiteral($key));
$parts[] = $value;
}

return $this->expr('json_object(' . implode(', ', array_fill(0, count($parts), '[]')) . ')', $parts);
}

public function jsonArrayAgg(Expressionable $expr): BaseExpression
{
return $this->expr('json_array([])', [$expr]);
}

#[\Override]
public function exists()
{
Expand Down
27 changes: 27 additions & 0 deletions src/Persistence/Sql/Mysql/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,33 @@ public function fxJsonArray(array $values)
]);
}

/**
* @param array<string, Expressionable> $keyValuePairs
*/
public function fxJsonObject(array $keyValuePairs): BaseExpression
{
if (!Connection::isServerMariaDb($this->connection) && version_compare($this->connection->getServerVersion(), '5.7.8') < 0) {
throw new \Exception('JSON_OBJECT requires MySQL 5.7.8+');
}

$parts = [];
foreach ($keyValuePairs as $key => $value) {
$parts[] = new RawExpression($this->escapeStringLiteral($key));
$parts[] = $value;
}

return $this->expr('json_object(' . implode(', ', array_fill(0, count($parts), '[]')) . ')', $parts);
}

public function jsonArrayAgg(Expressionable $expr): BaseExpression
{
if (!Connection::isServerMariaDb($this->connection) && version_compare($this->connection->getServerVersion(), '5.7.8') < 0) {
throw new \Exception('JSON_ARRAYAGG requires MySQL 5.7.8+');
}

return $this->expr('json_arrayagg([])', [$expr]);
}

/**
* @return ($forJsonValue is true ? array{string, string, string|null} : string)
*/
Expand Down
19 changes: 19 additions & 0 deletions src/Persistence/Sql/Oracle/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,25 @@ public function exists()
);
}

/**
* @param array<string, Expressionable> $keyValuePairs
*/
public function fxJsonObject(array $keyValuePairs): BaseExpression
{
$parts = [];
foreach ($keyValuePairs as $key => $value) {
$parts[] = new RawExpression($this->escapeStringLiteral($key));
$parts[] = $value;
}

return $this->expr('json_object(' . implode(', ', array_fill(0, count($parts), '[]')) . ')', $parts);
}

public function jsonArrayAgg(Expressionable $expr): BaseExpression
{
return $this->expr('json_arrayagg([])', [$expr]);
}

#[\Override]
protected function _execute(?object $connection, bool $fromExecuteStatement)
{
Expand Down
19 changes: 19 additions & 0 deletions src/Persistence/Sql/Postgresql/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,4 +227,23 @@ public function jsonTable(Expressionable $json, array $columns, string $rowsPath

return $query;
}

/**
* @param array<string, Expressionable> $keyValuePairs
*/
public function fxJsonObject(array $keyValuePairs): BaseExpression
{
$parts = [];
foreach ($keyValuePairs as $key => $value) {
$parts[] = new RawExpression($this->escapeStringLiteral($key));
$parts[] = $value;
}

return $this->expr('json_build_object(' . implode(', ', array_fill(0, count($parts), '[]')) . ')', $parts);
}

public function jsonArrayAgg(Expressionable $expr): BaseExpression
{
return $this->expr('json_agg([])', [$expr]);
}
}
3 changes: 3 additions & 0 deletions src/Persistence/Sql/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

/**
* Perform query operation on SQL server (such as select, insert, delete, etc).
*
* @method Expression fxJsonObject(array<string, Expressionable> $keyValuePairs)
* @method Expression jsonArrayAgg(Expressionable $expr)
Copy link
Member

@mvorisek mvorisek Nov 21, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This functionality has already #1268 PR but it is buggy on MySQL & MariaDB.

In MariaDB it is a confirmed bug.

In MySQL I did not investigate it in depth yet. Maybe it depends on configuration option.

The PR should be merged first and we should not integrate it before MySQL/MariaDB is either fixed or we can always throw an error if and only if the bug is applicable.

I see you are interested in HasMany aggregate = json and you have my support for it 👍.

To better understand your needs, what are your usecases for this PR?

Copy link
Contributor Author

@woodholly woodholly Nov 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see you are interested in HasMany aggregate = json and you have my support for it 👍.

Well, good low-code design should have nice, short methods! :)
By the way, this approach uses a bit of "magic" - the dot notation. What's your take on that? Personally, I'm not usually a fan of magic in libraries, but I'll make an exception for JSON aggregation. Most of the time, that JSON is used right after a join, so the convenience is worth it.

To better understand your needs, what are your usecases for this PR?

Example: movie database https://raw.githubusercontent.com/geldata/imdbench/master/docs/schema.png

Task: Get data for a movie page

  1. Fetch movie data by ID
  2. Include full cast list and full directors list
  3. Do this ASAP = in a single SQL query

This is difficult or impossible to achieve without JSON aggregation.

With this PR, I can accomplish this:

class Movie extends \App\Core\Model {
      protected function init(): void {
         ...
         $this->hasMany('directors', ['model' => function ($persistence) {
              $model = new Directors($persistence);
              $model->join('person');
              return $model;
          }, 'theirField' => 'movie_id'])
              ->addField('directors_json', [
                  'aggregate' => 'json',
                  'fields' => [
                      'person.id',
                      'full_name' => fn($q, $r) => $q->expr('CONCAT([], CASE WHEN [] != "" THEN CONCAT(" ", []) ELSE "" END, " ", [])', [$r('person.first_name  '), $r('person.middle_name'), $r('person.middle_name'), $r('person.last_name')]),
                      'person.image',
                  ],
                  'type' => 'json',
              ]);
         $this->hasMany('cast', ['model' => function ($persistence) {
              $model = new Cast($persistence);
              $model->join('person');
              return $model;
          }, 'theirField' => 'movie_id'])
              ->addField('cast_json', [
                  'aggregate' => 'json',
                  'fields' => [
                      'person.id',
                      'full_name' => fn($q, $r) => $q->expr('CONCAT([], CASE WHEN [] != "" THEN CONCAT(" ", []) ELSE "" END, " ", [])', [$r('person.first_name  '), $r('person.middle_name'), $r('person.middle_name'), $r('person.last_name')]),
                      'person.image',
                  ],
                  'type' => 'json',
              ]);
...
}

$movie = new Movie($persistence);
$movie = $movie->load($id);

return json_encode([
      'id' => $movie->get('id'),
    ...
      'directors' => $movie->get('directors_json') ?? [],
      'cast' => $movie->get('cast_json') ?? [],
]);

This example clearly shows why we need both JSON aggregation and dot notation working together.

*/
abstract class Query extends Expression
{
Expand Down
20 changes: 20 additions & 0 deletions src/Persistence/Sql/Sqlite/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace Atk4\Data\Persistence\Sql\Sqlite;

use Atk4\Data\Persistence\Sql\ExecuteException;
use Atk4\Data\Persistence\Sql\Expression as BaseExpression;
use Atk4\Data\Persistence\Sql\Expressionable;
use Atk4\Data\Persistence\Sql\Query as BaseQuery;
use Atk4\Data\Persistence\Sql\RawExpression;
Expand Down Expand Up @@ -172,6 +173,25 @@ public function fxJsonArray(array $values)
]);
}

/**
* @param array<string, Expressionable> $keyValuePairs
*/
public function fxJsonObject(array $keyValuePairs): BaseExpression
{
$parts = [];
foreach ($keyValuePairs as $key => $value) {
$parts[] = new RawExpression($this->escapeStringLiteral($key));
$parts[] = $value;
}

return $this->expr('json_object(' . implode(', ', array_fill(0, count($parts), '[]')) . ')', $parts);
}

public function jsonArrayAgg(Expressionable $expr): BaseExpression
{
return $this->expr('json_group_array([])', [$expr]);
}

#[\Override]
public function fxJsonValue(Expressionable $json, string $path, string $type, ?Expressionable $jsonRootType = null)
{
Expand Down
86 changes: 86 additions & 0 deletions src/Reference/HasMany.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
use Atk4\Data\Exception;
use Atk4\Data\Field;
use Atk4\Data\Model;
use Atk4\Data\Model\Join;
use Atk4\Data\Persistence\Sql\Expressionable;
use Atk4\Data\Reference;

class HasMany extends Reference
Expand Down Expand Up @@ -155,6 +157,90 @@ public function addField(string $fieldName, array $defaults = []): Field
$fx = function () use ($defaults, $field) {
return $this->refLink()->action('fx0', [$defaults['aggregate'], $field]);
};
} elseif ($defaults['aggregate'] === 'json') {
Copy link
Member

@mvorisek mvorisek Dec 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

json-array and json-object should be probably supported, as the first is much more storage effective and ordered (even on MySQL)

if (!isset($defaults['fields'])) {
throw new Exception('JSON aggregate requires "fields" parameter with array of field names');
}

$jsonFields = $defaults['fields'];
unset($defaults['fields']); // Remove from field defaults

$fx = function () use ($jsonFields) {
$theirModel = $this->refLink();
$query = $theirModel->action('select', [[]]);

// Helper to resolve field reference with dot notation support
$resolveField = static function ($fieldRef) use ($theirModel, $query) {
if (is_string($fieldRef) && str_contains($fieldRef, '.')) {
// Dot notation: 'join.field' - resolve from join
[$joinName, $fieldName] = explode('.', $fieldRef, 2);

// Find the join by iterating through model's joins
$join = null;
foreach ($theirModel->elements as $element) {
if ($element instanceof Join) {
if ($element->getForeignTable() === $joinName) {
$join = $element;

break;
}
}
}

if (!$join) {
throw (new Exception('Join not found for field reference'))
->addMoreInfo('field', $fieldRef)
->addMoreInfo('join', $joinName);
}

// Get field from join's foreign table
// The field is qualified as joinAlias.fieldName in the query
$joinAlias = $join->foreignAlias ?? ('_' . $joinName);

return $query->expr($joinAlias . '.' . $fieldName);
}

// Regular field reference
return $theirModel->getField($fieldRef);
};

$jsonPairs = [];
foreach ($jsonFields as $key => $fieldSpec) {
// Determine output key name
if (is_int($key)) {
// No custom name - auto-generate from field spec
if (is_string($fieldSpec) && str_contains($fieldSpec, '.')) {
// Strip join prefix: 'person.first_name' -> 'first_name'
$jsonKey = explode('.', $fieldSpec, 2)[1];
} else {
$jsonKey = is_string($fieldSpec) ? $fieldSpec : $key;
}
} else {
// Custom name provided
$jsonKey = $key;
}

if (is_string($fieldSpec)) {
$jsonPairs[$jsonKey] = $resolveField($fieldSpec);
} elseif ($fieldSpec instanceof Expressionable) {
$jsonPairs[$jsonKey] = $fieldSpec;
} elseif (is_callable($fieldSpec)) {
$jsonPairs[$jsonKey] = $fieldSpec($query, $resolveField);
} elseif (is_array($fieldSpec) && isset($fieldSpec['expr'])) {
if (is_callable($fieldSpec['expr'])) {
$jsonPairs[$jsonKey] = $fieldSpec['expr']($query, $resolveField);
} else {
$resolvedArgs = array_map($resolveField, $fieldSpec['args'] ?? []);
$jsonPairs[$jsonKey] = $query->expr($fieldSpec['expr'], $resolvedArgs);
}
}
}

$jsonObj = $query->fxJsonObject($jsonPairs);
Copy link
Member

@mvorisek mvorisek Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is legit usecase for a new method.

See #1267 for non-associative PR.

I would like to have this reviewed and merged separately.

Please reuse as much as code from #1267 as possible.

Like the non-object version, the object (associative) version should support MySQL 5.6, it should be possible to construct JSON string reliably without native JSON support.

$jsonAgg = $query->jsonArrayAgg($jsonObj);

return $theirModel->action('field', [$jsonAgg]);
};
} else {
$fx = function () use ($defaults, $field) {
$args = [$defaults['aggregate'], $field];
Expand Down
Loading
Loading