Skip to content

New format schema #1405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
otrok7 wants to merge 14 commits into
base: main
Choose a base branch
from
Draft

New format schema #1405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
3f71de5
migrate up
otrok7 Nov 27, 2025
764ada2
New Format Schema
otrok7 Nov 28, 2025
996cd39
Corrections for GetSearchResults interface
otrok7 Nov 28, 2025
d6da94d
Format Create and Delete
otrok7 Nov 29, 2025
5374c10
Set RouteKeyName on Format view
otrok7 Nov 30, 2025
5a4fa3e
Remove View and Get Test Running
otrok7 Dec 1, 2025
4f46bff
Fix for NAWS Dump
otrok7 Dec 1, 2025
e5129e6
small fixes
otrok7 Dec 1, 2025
741a7eb
Correct structural problem from tests
otrok7 Dec 2, 2025
c4911a8
Fixes to Partial Updates
otrok7 Dec 2, 2025
cde3e3e
Ignore nonsense in format strings
otrok7 Dec 2, 2025
a8dc46e
NAWS Export adjusted for new format schema
otrok7 Dec 2, 2025
5d6da83
Lint
otrok7 Dec 2, 2025
8ddd778
Remove Unused Functions
otrok7 Dec 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/app/Console/Commands/ImportRootServers.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
use App\Interfaces\RootServerRepositoryInterface;
use App\Interfaces\ServiceBodyRepositoryInterface;
use App\Models\Change;
use App\Models\Format;
use App\Models\FormatMain;
use App\Models\FormatTranslation;
use App\Models\Meeting;
use App\Models\MeetingData;
use App\Models\MeetingLongData;
Expand Down Expand Up @@ -79,7 +80,7 @@ private function deleteNonAggregatorData(): void
MeetingData::query()->whereIn('meetingid_bigint', $meetingIds)->whereNot('meetingid_bigint', 0)->delete();
MeetingLongData::query()->whereIn('meetingid_bigint', $meetingIds)->whereNot('meetingid_bigint', 0)->delete();
ServiceBody::query()->whereNull('root_server_id')->delete();
Format::query()->whereNull('root_server_id')->delete();
FormatMain::query()->whereNull('root_server_id')->delete();
Change::query()->delete();
}

Expand Down Expand Up @@ -264,7 +265,8 @@ private function analyzeTables(): void
$prefix . (new MeetingData)->getTable(),
$prefix . (new MeetingLongData)->getTable(),
$prefix . (new ServiceBody)->getTable(),
$prefix . (new Format)->getTable(),
$prefix . (new FormatMain)->getTable(),
$prefix . (new FormatTranslation)->getTable(),
$prefix . (new RootServer)->getTable(),
$prefix . (new RootServerStatistics)->getTable(),
];
Expand Down
9 changes: 6 additions & 3 deletions src/app/Console/Commands/PrimeDatabaseFromLegacyTomato.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@

use App\Interfaces\RootServerRepositoryInterface;
use App\Models\Change;
use App\Models\Format;
use App\Models\FormatMain;
use App\Models\FormatTranslation;
use App\Models\Meeting;
use App\Models\MeetingData;
use App\Models\MeetingLongData;
Expand Down Expand Up @@ -55,7 +56,8 @@ private function deleteAllData(): void
MeetingData::query()->whereNot('meetingid_bigint', 0)->delete();
MeetingLongData::query()->whereNot('meetingid_bigint', 0)->delete();
ServiceBody::query()->truncate();
Format::query()->truncate();
FormatMain::query()->truncate();
FormatTranslation::query()->truncate();
Change::query()->truncate();
}

Expand Down Expand Up @@ -176,7 +178,8 @@ private function analyzeTables(): void
$prefix . (new MeetingData)->getTable(),
$prefix . (new MeetingLongData)->getTable(),
$prefix . (new ServiceBody)->getTable(),
$prefix . (new Format)->getTable(),
$prefix . (new FormatMain)->getTable(),
$prefix . (new FormatTranslation())->getTable(),
];
foreach ($tableNames as $tableName) {
DB::statement("ANALYZE TABLE $tableName;");
Expand Down
31 changes: 17 additions & 14 deletions src/app/Http/Controllers/Admin/FormatController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
use App\Interfaces\FormatRepositoryInterface;
use App\Models\Format;
use App\Models\FormatType;
use App\Models\MeetingFormats;
use App\Rules\FormatTranslationKey;
use App\Rules\FormatTranslations;
use Illuminate\Http\Request;
Expand Down Expand Up @@ -56,11 +57,11 @@ public function partialUpdate(Request $request, Format $format)
collect(['worldId', 'type', 'translations'])
->mapWithKeys(function ($fieldName, $_) use ($request, $format) {
if ($fieldName == 'worldId') {
return [$fieldName => $request->has($fieldName) ? $request->input($fieldName) : $format->worldid_mixed];
return [$fieldName => $request->has($fieldName) ? $request->input($fieldName) : $format->main->worldid_mixed];
} elseif ($fieldName == 'type') {
return [$fieldName => $request->has($fieldName) ? $request->input($fieldName) : (!is_null($format->format_type_enum) ? FormatType::getApiEnumFromKey($format->format_type_enum): null)];
return [$fieldName => $request->has($fieldName) ? $request->input($fieldName) : (!is_null($format->main->format_type_enum) ? FormatType::getApiEnumFromKey($format->main->format_type_enum): null)];
} else {
return [$fieldName => $request->has($fieldName) ? $request->input($fieldName) : $format->translations->map(function ($translation) {
return [$fieldName => $request->has($fieldName) ? $request->input($fieldName) : $format->main->translations->map(function ($translation) {
return [
'key' => $translation->key_string,
'name' => $translation->name_string,
Expand Down Expand Up @@ -88,7 +89,7 @@ public function destroy(Request $request, Format $format)
$this->formatRepository->getHybridFormat()->shared_id_bigint,
])]]);

if ($format->meetings()->exists()) {
if (MeetingFormats::query()->where('format_id', $format->shared_id_bigint)->first()) {
return new JsonResponse([
'message' => 'You cannot delete a format while meetings are using it.'
], 409);
Expand Down Expand Up @@ -124,15 +125,17 @@ function ($attribute, $value, $fail) {

private function buildValuesArray(Collection $validated)
{
return collect($validated['translations'])->map(function ($translation) use ($validated) {
return [
'format_type_enum' => isset($validated['type']) ? FormatType::getKeyFromApiEnum($validated['type']) : null,
'worldid_mixed' => $validated['worldId'] ?? null,
'lang_enum' => $translation['language'],
'key_string' => $translation['key'],
'name_string' => $translation['name'],
'description_string' => $translation['description'],
];
})->toArray();
return collect([
'format_type_enum' => isset($validated['type']) ? FormatType::getKeyFromApiEnum($validated['type']) : null,
'worldid_mixed' => $validated['worldId'] ?? null,
'translations' => array_map(function ($translation) {
return [
'lang_enum' => $translation['language'],
'key_string' => $translation['key'],
'name_string' => $translation['name'],
'description_string' => $translation['description'],
];
}, $validated['translations']),
])->toArray();
}
}
16 changes: 10 additions & 6 deletions src/app/Http/Controllers/Admin/MeetingController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,12 @@ public function partialUpdate(Request $request, Meeting $meeting)
$request->merge(
collect(Meeting::$mainFields)
->merge($stockDataFields)
->merge(['formats'])
->mapWithKeys(function ($fieldName) use ($request, $meeting, $meetingData) {
if ($fieldName == 'service_body_bigint') {
return ['serviceBodyId' => $request->has('serviceBodyId') ? $request->input('serviceBodyId') : $meeting->service_body_bigint];
} elseif ($fieldName == 'formats') {
return ['formatIds' => $request->has('formatIds') ? $request->input('formatIds') : (empty($meeting->formats) ? collect([]) : collect(explode(',', $meeting->formats))->map(fn ($id) => intval($id))->reject(fn ($id) => $id == $this->getVirtualFormatId() || $id == $this->getHybridFormatId() || $id == $this->getTemporarilyClosedFormatId())->toArray())];
return ['formatIds' => $request->has('formatIds') ? $request->input('formatIds') : (empty($meeting->formatIds) ? collect([]) : $meeting->formatIds->pluck('format_id')->reject(fn ($id) => $id == $this->getVirtualFormatId() || $id == $this->getHybridFormatId() || $id == $this->getTemporarilyClosedFormatId())->toArray())];
} elseif ($fieldName == 'venue_type') {
return ['venueType' => $request->has('venueType') ? $request->input('venueType') : $meeting->venue_type];
} elseif ($fieldName == 'weekday_tinyint') {
Expand Down Expand Up @@ -176,7 +177,7 @@ private function validateInputs(Request $request, $skipVenueTypeLocationValidati
array_merge([
'serviceBodyId' => 'required|int|exists:comdef_service_bodies,id_bigint',
'formatIds' => 'present|array',
'formatIds.*' => ['int', 'exists:comdef_formats,shared_id_bigint', Rule::notIn([$this->getVirtualFormatId(), $this->getTemporarilyClosedFormatId(), $this->getHybridFormatId()])],
'formatIds.*' => ['int', 'exists:comdef_formats_translations,shared_id_bigint', Rule::notIn([$this->getVirtualFormatId(), $this->getTemporarilyClosedFormatId(), $this->getHybridFormatId()])],
'venueType' => ['required', Rule::in(Meeting::VALID_VENUE_TYPES)],
'temporarilyVirtual' => 'sometimes|boolean',
'day' => 'required|int|between:0,6',
Expand Down Expand Up @@ -239,9 +240,12 @@ private function getTemporarilyClosedFormatId(): int
return $id;
}

private function buildFormatsString(Collection $validated): string
private function buildFormatsArray(Collection $validated): array
{
$formatIds = $validated['formatIds'];
$formatIds = [];
if (!empty($validated['formatIds'])) {
$formatIds = $validated['formatIds'];
}
$temporarilyVirtual = boolval($validated['temporarilyVirtual'] ?? false);
$venueType = $validated['venueType'];
if ($venueType == Meeting::VENUE_TYPE_VIRTUAL) {
Expand All @@ -252,14 +256,14 @@ private function buildFormatsString(Collection $validated): string
} elseif ($venueType == Meeting::VENUE_TYPE_HYBRID) {
array_push($formatIds, $this->getHybridFormatId());
}
return collect($formatIds)->sort()->unique()->join(',');
return $formatIds;
}

private function buildValuesArray(Collection $validated): array
{
$values = [
'service_body_bigint' => $validated['serviceBodyId'],
'formats' => $this->buildFormatsString($validated),
'formats' => $this->buildFormatsArray($validated),
'venue_type' => $validated['venueType'],
'weekday_tinyint' => $validated['day'],
'time_zone' => $validated['timeZone'] ?? null,
Expand Down
6 changes: 2 additions & 4 deletions src/app/Http/Controllers/Query/SwitcherController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,8 +355,6 @@ private function getSearchResults(Request $request, ?string $dataFormat = null):
pageNum: $pageNum,
);

// This code to calculate the formats fields is really inefficient, but necessary because
// we don't have foreign keys between the meetings and formats tables.
$langEnum = $request->input('lang_enum', config('app.locale'));
$formats = $this->formatRepository->search(
rootServersInclude: $rootServersInclude,
Expand Down Expand Up @@ -695,7 +693,7 @@ private function getNawsDump($request): StreamedResponse
$meetings = $meetings->concat($deletedMeetings);
$allFormats = $this->formatRepository->search(langEnums: [legacy_config('language')], showAll: true)
->reject(fn ($fmt) => is_null($fmt->key_string) || empty(trim($fmt->key_string)));
$formatIdToWorldId = $allFormats->mapWithKeys(fn ($fmt, $_) => [$fmt->shared_id_bigint => $fmt->worldid_mixed]);
$formatIdToWorldId = $allFormats->mapWithKeys(fn ($fmt, $_) => [$fmt->shared_id_bigint => $fmt->main->worldid_mixed]);
$formatIdToKeyString = $allFormats->mapWithKeys(fn ($fmt, $_) => [$fmt->shared_id_bigint => $fmt->key_string]);
$formatIdToNameString = $allFormats->mapWithKeys(fn ($fmt, $_) => [$fmt->shared_id_bigint => $fmt->name_string]);
// $lastChanged is a dictionary whose keys are meeting IDs and whose values are the last time that meeting was changed
Expand All @@ -719,7 +717,7 @@ private function getNawsDump($request): StreamedResponse
->merge($meeting->longdata->mapWithKeys(fn($data, $_) => [$data->key => $data->data_blob])->toBase());
}

$allMeetingFormatIds = collect(explode(',', $meeting->formats ?? ''));
$allMeetingFormatIds = $meeting->formatIds->pluck('format_id');
// list of format world ids
$allNawsMeetingFormats = $allMeetingFormatIds
->map(fn ($id) => $formatIdToWorldId->get(intval($id)))
Expand Down
9 changes: 5 additions & 4 deletions src/app/Http/Resources/Admin/FormatResource.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@ class FormatResource extends JsonResource

public function toArray($request)
{
$main = (get_class($this->resource) != "App\Models\FormatMain") ? $this->resource->main : $this->resource;
return [
'id' => $this->shared_id_bigint,
'worldId' => $this->worldid_mixed,
'type' => FormatType::getApiEnumFromKey($this->format_type_enum),
'translations' => $this->translations->map(function ($translation) {
'id' => $main->shared_id_bigint,
'worldId' => $main->worldid_mixed,
'type' => FormatType::getApiEnumFromKey($main->format_type_enum),
'translations' => $main->translations->map(function ($translation) {
return [
'key' => $translation->key_string ?? '',
'name' => $translation->name_string ?? '',
Expand Down
6 changes: 1 addition & 5 deletions src/app/Http/Resources/Admin/MeetingResource.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,7 @@ public function toArray($request)
->toBase()
);

$formatIds = empty($this->formats) ? collect([]) : collect(explode(',', $this->formats))
->map(fn ($id) => intval($id))
->reject(fn ($id) => !self::$formatsById->has($id))
->sort();

$formatIds = $this->formatIds->pluck('format_id');
return array_merge(
[
'id' => $this->id_bigint,
Expand Down
4 changes: 2 additions & 2 deletions src/app/Http/Resources/Query/FormatResource.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ public function toArray($request)
'description_string' => $this->description_string ?? '',
'lang' => $this->lang_enum,
'id' => (string)$this->shared_id_bigint,
'world_id' => $this->worldid_mixed ?? '',
'format_type_enum' => $this->format_type_enum ?? '',
'world_id' => $this->main->worldid_mixed ?? '',
'format_type_enum' => $this->main->format_type_enum ?? '',
'root_server_uri' => $isAggregatorModeEnabled && $this->root_server_id ? $this->rootServer->url : $request->getSchemeAndHttpHost() . $request->getBaseUrl(),
'root_server_id' => $this->when($isAggregatorModeEnabled, $this->root_server_id ?? '')
];
Expand Down
11 changes: 6 additions & 5 deletions src/app/Interfaces/FormatRepositoryInterface.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

namespace App\Interfaces;

use App\Models\Format;
use App\Models\FormatTranslation;
use App\Models\FormatMain;
use App\Repositories\Import\FormatImportResult;
use Illuminate\Support\Collection;

Expand All @@ -20,10 +21,10 @@ public function search(
bool $eagerRootServers = false
): Collection;
public function getAsTranslations(array $formatIds = null): Collection;
public function getVirtualFormat(): Format;
public function getHybridFormat(): Format;
public function getTemporarilyClosedFormat(): Format;
public function create(array $sharedFormatsValues): Format;
public function getVirtualFormat(): FormatTranslation;
public function getHybridFormat(): FormatTranslation;
public function getTemporarilyClosedFormat(): FormatTranslation;
public function create(array $sharedFormatsValues): FormatMain;
public function update(int $sharedId, array $sharedFormatsValues): bool;
public function delete(int $sharedId): bool;
public function import(int $rootServerId, Collection $externalObjects): FormatImportResult;
Expand Down
48 changes: 1 addition & 47 deletions src/app/Models/Format.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,52 +2,6 @@

namespace App\Models;

use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;

class Format extends Model
class Format extends FormatTranslation
{
protected $table = 'comdef_formats';
public $timestamps = false;
protected $fillable = [
'root_server_id',
'source_id',
'shared_id_bigint',
'key_string',
'worldid_mixed',
'lang_enum',
'name_string',
'description_string',
'format_type_enum',
];

public function getRouteKeyName()
{
return 'shared_id_bigint';
}

public function rootServer()
{
return $this->belongsTo(RootServer::class, 'root_server_id');
}

public function translations()
{
return $this
->hasMany(self::class, 'shared_id_bigint', 'shared_id_bigint')
->orderBy('lang_enum');
}

public function meetings()
{
// TODO once we fix the database schema, these will be proper fks, and we can simply do a $this->hasMany
$formatId = $this->attributes['shared_id_bigint'];
return Meeting::query()->where(function (Builder $query) use ($formatId) {
$query
->orWhere('formats', "$formatId")
->orWhere('formats', 'LIKE', "$formatId,%")
->orWhere('formats', 'LIKE', "%,$formatId,%")
->orWhere('formats', 'LIKE', "%,$formatId");
});
}
}
38 changes: 38 additions & 0 deletions src/app/Models/FormatMain.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
<?php

namespace App\Models;

use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;

class FormatMain extends Model
{
protected $table = 'comdef_formats_main';
protected $primaryKey = 'shared_id_bigint';
public $incrementing = true;
public $timestamps = false;
protected $fillable = [
'root_server_id',
'source_id',
'worldid_mixed',
'format_type_enum',
];

public function getRouteKeyName()
{
return 'shared_id_bigint';
}

public function rootServer()
{
return $this->belongsTo(RootServer::class, 'root_server_id');
}
public function translations()
{
return $this->hasMany(FormatTranslation::class, 'shared_id_bigint', 'shared_id_bigint');
}
public function meetings()
{
return $this->belongsToMany(Meeting::class, 'comdef_meeting_formats', 'format_id', 'meeting_id');
}
}
34 changes: 34 additions & 0 deletions src/app/Models/FormatTranslation.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
<?php

namespace App\Models;

use Illuminate\Database\Eloquent\Model;

class FormatTranslation extends Model
{
protected $table = 'comdef_formats_translations';
public $timestamps = false;
protected $fillable = [
'root_server_id',
'source_id',
'shared_id_bigint',
'key_string',
'lang_enum',
'name_string',
'description_string',
];

public function getRouteKeyName()
{
return 'shared_id_bigint';
}

public function rootServer()
{
return $this->belongsTo(RootServer::class, 'root_server_id');
}
public function main()
{
return $this->belongsTo(FormatMain::class, 'shared_id_bigint', 'shared_id_bigint');
}
}
Loading
Loading