From 57cd05876113a5e71926dc4d4ea3dbc21cc1ccab Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 14 Oct 2025 11:23:29 +0200 Subject: [PATCH 1/2] docs: add Docker instructions to README.md Part of a set of README.md updates for all our major websites. Also tidies up test rebuild and database startup on slower systems. Test-bot: skip --- README.md | 153 +++++++++++++++++------------------- build.sh | 13 ++- resources/init-container.sh | 8 +- tests/TestDBBuild.inc.php | 8 +- tests/_prepend_rebuild.php | 3 + 5 files changed, 101 insertions(+), 84 deletions(-) create mode 100644 tests/_prepend_rebuild.php diff --git a/README.md b/README.md index 858deab..5f4a3f4 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,111 @@ # api.keyman.com -The following is outdated and will be replaced with Docker/Apache +This is the source for the website https://api.keyman.com/, which hosts the +database backend for Keyman websites. This site runs on Apache in a Docker +container, and the database itself runs on SQL Server for Linux in a separate +container. -## Configuration +## Other Keyman websites -Currently, this site runs only on a Windows host with IIS and Microsoft SQL Server. +* **[api.keyman.com]** - database backend for Keyman websites +* **[help.keyman.com]** - documentation home for Keyman +* **[keyman.com]** - Keyman home +* **[keymanweb.com]** - KeymanWeb online keyboard +* **[s.keyman.com]** - static Javascript, font, and related resources +* **[website-local-proxy]** - run all Keyman sites on localhost on the same port -## Prerequisites +## How to run api.keyman.com locally -* Windows -* Chocolatey -* PHP 7.4 -* MS SQL Server 2016 or later including FullText Search +When run locally, this site can be accessed at http://localhost:8058 or +http://api.keyman.com.localhost:8058. -* `configure.ps1` automatically installs chocolatey, PHP, Composer, SQL Server and PHP-PDO driver - for SQL Server. This script is not particularly sophisticated, so for manual config, copy and - paste elements from the script. +**Recommended:** Use [website-local-proxy] to run multiple keyman.com sites +all from the same port (default port 80). -## Setup +**Recommended:** Use [shared-sites] to control startup and shutdown of all +keyman.com sites together. -1. Install the dependencies: +### Prerequisites -``` -composer install -``` - -2. Configure your local environment by copying tools/db/localenv.php.in to tools/db/localenv.php - and completing the details therein. - -3. Build the backend database from live data: +The host machine needs the following apps installed: +* [Git] +* Bash 5.x (on Windows, you can use Git Bash that comes with [Git]) +* [Docker Desktop] -``` -composer build -``` +
+ Configuration of Docker on Windows -## Tests + On Windows machines, you can setup Docker in two different ways, either of + which should work: + * [Enable Hyper-V on Windows 11](https://techcommunity.microsoft.com/t5/educator-developer-blog/step-by-step-enabling-hyper-v-for-use-on-windows-11/ba-p/3745905) + * [WSL2](https://ubuntu.com/tutorials/install-ubuntu-on-wsl2-on-windows-10#1-overview) -Test suites run with mock data from the tests/data folder. If this data is refreshed, fixtures -will probably need to be updated accordingly as the data in them will have become stale. +
-To run tests: +### Actions -``` -composer test -``` +#### Build the Docker image -To force a rebuild of the test database (e.g. if schema changes): +The first time you want to start up the site, or if there have been Docker +configuration changes, you will need to rebuild the Docker images. Start a bash +shell, and from this folder, run: -``` -TEST_REBUILD=1 composer test +```sh +./build.sh build ``` -## Configuring a new Azure Database +#### Start the Docker container -1. Create an Azure SQL Server -2. Create an Azure SQL Database, e.g. called 'keymanapi' -3. Run the following script on the master database, replacing password as necessary: +To start up the website, in bash, run: +```sh +./build.sh start --debug ``` --- logins for staging -CREATE LOGIN [k0] WITH PASSWORD=N'password' -GO - -CREATE LOGIN [k1] WITH PASSWORD=N'password' -GO --- logins for production -CREATE LOGIN [production_k0] WITH PASSWORD=N'password' -GO +Once the container starts, you can access the api.keyman.com site at +http://localhost:8058 or http://api.keyman.com.localhost:8058 -CREATE LOGIN [production_k1] WITH PASSWORD=N'password' -GO -``` +#### Stop the Docker container -4. Run the following script on the keymanapi database: +In bash, run: +```sh +./build.sh stop ``` --- Schemas, users and roles for staging -CREATE SCHEMA [k0] -GO - -CREATE SCHEMA [k1] -GO -CREATE USER [k0] FOR LOGIN [k0] WITH DEFAULT_SCHEMA=[k0] -GO +#### Remove the Docker container and image -CREATE USER [k1] FOR LOGIN [k1] WITH DEFAULT_SCHEMA=[k1] -GO +In bash, run: -ALTER ROLE db_owner ADD MEMBER k0 -GO +```sh +./build.sh clean +``` -ALTER ROLE db_owner ADD MEMBER k1 -GO +#### Running tests --- Schemas, users and roles for production -CREATE SCHEMA [production_k0] -GO +Test suites run with mock data from the tests/data folder. To check APIs, broken +links and .php file conformance, when the site is running, in bash, run: -CREATE SCHEMA [production_k1] -GO +```sh +./build.sh test +``` -CREATE USER [production_k0] FOR LOGIN [production_k0] WITH DEFAULT_SCHEMA=[production_k0] -GO +To force a rebuild of the test database from the mock data (for example if +schema changes and this is not automatically detected): -CREATE USER [production_k1] FOR LOGIN [production_k1] WITH DEFAULT_SCHEMA=[production_k1] -GO +```sh +./build.sh test --rebuild-test-fixtures +``` -ALTER ROLE db_owner ADD MEMBER production_k0 -GO +[Git]: https://git-scm.com/downloads +[Docker Desktop]: https://docs.docker.com/get-docker/ +[api.keyman.com]: https://github.com/keymanapp/api.keyman.com +[help.keyman.com]: https://github.com/keymanapp/help.keyman.com +[keyman.com]: https://github.com/keymanapp/keyman.com +[keymanweb.com]: https://github.com/keymanapp/keymanweb.com +[s.keyman.com]: https://github.com/keymanapp/s.keyman.com +[website-local-proxy]: https://github.com/keymanapp/website-local-proxy +[shared-sites]: https://github.com/keymanapp/shared-sites +[enable Hyper-V]: https://techcommunity.microsoft.com/t5/educator-developer-blog/step-by-step-enabling-hyper-v-for-use-on-windows-11/ba-p/3745905 +[enable WSL2]: https://ubuntu.com/tutorials/install-ubuntu-on-wsl2-on-windows-10#1-overview -ALTER ROLE db_owner ADD MEMBER production_k1 -GO -``` diff --git a/build.sh b/build.sh index 87e3a8a..466d6f7 100755 --- a/build.sh +++ b/build.sh @@ -28,6 +28,7 @@ builder_describe \ "start" \ "stop" \ "test" \ + "--rebuild-test-fixtures Rebuild the test fixtures from live data" \ ":db Build the database" \ ":app Build the site" @@ -37,6 +38,10 @@ function test_docker_container() { echo "TIER_TEST" > tier.txt # Note: ci.yml replicates these + if builder_has_option --rebuild-test-fixtures; then + touch rebuild-test-fixtures.txt + fi + # Run unit tests docker exec $API_KEYMAN_CONTAINER_DESC sh -c "vendor/bin/phpunit --testdox" @@ -92,13 +97,17 @@ function start_docker_container_db() { # Setup database builder_echo "Setting up DB container" - docker run -m 2048m --rm -d -p $PORT:1433 \ + docker run --rm -d -p $PORT:1433 \ -e "ACCEPT_EULA=Y" \ -e "MSSQL_AGENT_ENABLED=true" \ -e "MSSQL_SA_PASSWORD=yourStrong(\!)Password" \ --name $CONTAINER_DESC \ $CONTAINER_NAME + builder_echo "Sleeping for 30 seconds to give database time to spin up" + builder_echo "(DB may crash if connected to, too early, on some systems)" + sleep 30s + builder_echo green "SQL Server Listening on localhost:$PORT" } @@ -137,6 +146,8 @@ function start_docker_container_app() { ADD_HOST="--add-host host.docker.internal:host-gateway" fi + builder_echo "Checking network settings" + db_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${API_KEYMAN_DB_IMAGE_NAME}) builder_echo "Spooling up site container" diff --git a/resources/init-container.sh b/resources/init-container.sh index 03b79fd..2f98400 100755 --- a/resources/init-container.sh +++ b/resources/init-container.sh @@ -1,10 +1,16 @@ #!/usr/bin/env bash -echo "---- Sleep 15 Before Generating DB ----" +openssl version +uname -a + +echo "---- Sleep 15 for SQL Server to start before generating DB ----" sleep 15; # If we know we are immediately going to run tests, there's no need to build # the database and then rebuild it again as a test database! if [[ ! -f /var/www/html/tier.txt ]] || [[ $(getActiveSchema(); try { diff --git a/tests/_prepend_rebuild.php b/tests/_prepend_rebuild.php new file mode 100644 index 0000000..0fdc6b3 --- /dev/null +++ b/tests/_prepend_rebuild.php @@ -0,0 +1,3 @@ + \ No newline at end of file From ab3833692b520a4e82d95ab122ea8e5dd5fed983 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 14 Oct 2025 15:37:46 +0200 Subject: [PATCH 2/2] chore: use a broader query for stats and just return everything Stats was returning a malformed result because of assumptions about the shape of the result. Instead, return all the data and leave interpretation up to user. This is okay because it's an internal-use-only script at present. Test-bot: skip --- script/statistics/annual-statistics.inc.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/statistics/annual-statistics.inc.php b/script/statistics/annual-statistics.inc.php index 40f2ced..8d4a4ed 100644 --- a/script/statistics/annual-statistics.inc.php +++ b/script/statistics/annual-statistics.inc.php @@ -21,8 +21,8 @@ function execute($mssql, $startDate, $endDate) { $stmt->bindParam(":prmEndDate", $endDate); $stmt->execute(); - $data = $stmt->fetchAll()[0]; - $data = array_filter($data, "Keyman\\Site\\com\\keyman\\api\\filter_columns_by_name", ARRAY_FILTER_USE_KEY ); + $data = $stmt->fetchAll(); + // $data = array_filter($data, "Keyman\\Site\\com\\keyman\\api\\filter_columns_by_name", ARRAY_FILTER_USE_KEY ); return $data; } }