diff --git a/README.md b/README.md index 858deab..5f4a3f4 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,111 @@ # api.keyman.com -The following is outdated and will be replaced with Docker/Apache +This is the source for the website https://api.keyman.com/, which hosts the +database backend for Keyman websites. This site runs on Apache in a Docker +container, and the database itself runs on SQL Server for Linux in a separate +container. -## Configuration +## Other Keyman websites -Currently, this site runs only on a Windows host with IIS and Microsoft SQL Server. +* **[api.keyman.com]** - database backend for Keyman websites +* **[help.keyman.com]** - documentation home for Keyman +* **[keyman.com]** - Keyman home +* **[keymanweb.com]** - KeymanWeb online keyboard +* **[s.keyman.com]** - static Javascript, font, and related resources +* **[website-local-proxy]** - run all Keyman sites on localhost on the same port -## Prerequisites +## How to run api.keyman.com locally -* Windows -* Chocolatey -* PHP 7.4 -* MS SQL Server 2016 or later including FullText Search +When run locally, this site can be accessed at http://localhost:8058 or +http://api.keyman.com.localhost:8058. -* `configure.ps1` automatically installs chocolatey, PHP, Composer, SQL Server and PHP-PDO driver - for SQL Server. This script is not particularly sophisticated, so for manual config, copy and - paste elements from the script. +**Recommended:** Use [website-local-proxy] to run multiple keyman.com sites +all from the same port (default port 80). -## Setup +**Recommended:** Use [shared-sites] to control startup and shutdown of all +keyman.com sites together. -1. Install the dependencies: +### Prerequisites -``` -composer install -``` - -2. Configure your local environment by copying tools/db/localenv.php.in to tools/db/localenv.php - and completing the details therein. - -3. Build the backend database from live data: +The host machine needs the following apps installed: +* [Git] +* Bash 5.x (on Windows, you can use Git Bash that comes with [Git]) +* [Docker Desktop] -``` -composer build -``` +
+ Configuration of Docker on Windows -## Tests + On Windows machines, you can setup Docker in two different ways, either of + which should work: + * [Enable Hyper-V on Windows 11](https://techcommunity.microsoft.com/t5/educator-developer-blog/step-by-step-enabling-hyper-v-for-use-on-windows-11/ba-p/3745905) + * [WSL2](https://ubuntu.com/tutorials/install-ubuntu-on-wsl2-on-windows-10#1-overview) -Test suites run with mock data from the tests/data folder. If this data is refreshed, fixtures -will probably need to be updated accordingly as the data in them will have become stale. +
-To run tests: +### Actions -``` -composer test -``` +#### Build the Docker image -To force a rebuild of the test database (e.g. if schema changes): +The first time you want to start up the site, or if there have been Docker +configuration changes, you will need to rebuild the Docker images. Start a bash +shell, and from this folder, run: -``` -TEST_REBUILD=1 composer test +```sh +./build.sh build ``` -## Configuring a new Azure Database +#### Start the Docker container -1. Create an Azure SQL Server -2. Create an Azure SQL Database, e.g. called 'keymanapi' -3. Run the following script on the master database, replacing password as necessary: +To start up the website, in bash, run: +```sh +./build.sh start --debug ``` --- logins for staging -CREATE LOGIN [k0] WITH PASSWORD=N'password' -GO - -CREATE LOGIN [k1] WITH PASSWORD=N'password' -GO --- logins for production -CREATE LOGIN [production_k0] WITH PASSWORD=N'password' -GO +Once the container starts, you can access the api.keyman.com site at +http://localhost:8058 or http://api.keyman.com.localhost:8058 -CREATE LOGIN [production_k1] WITH PASSWORD=N'password' -GO -``` +#### Stop the Docker container -4. Run the following script on the keymanapi database: +In bash, run: +```sh +./build.sh stop ``` --- Schemas, users and roles for staging -CREATE SCHEMA [k0] -GO - -CREATE SCHEMA [k1] -GO -CREATE USER [k0] FOR LOGIN [k0] WITH DEFAULT_SCHEMA=[k0] -GO +#### Remove the Docker container and image -CREATE USER [k1] FOR LOGIN [k1] WITH DEFAULT_SCHEMA=[k1] -GO +In bash, run: -ALTER ROLE db_owner ADD MEMBER k0 -GO +```sh +./build.sh clean +``` -ALTER ROLE db_owner ADD MEMBER k1 -GO +#### Running tests --- Schemas, users and roles for production -CREATE SCHEMA [production_k0] -GO +Test suites run with mock data from the tests/data folder. To check APIs, broken +links and .php file conformance, when the site is running, in bash, run: -CREATE SCHEMA [production_k1] -GO +```sh +./build.sh test +``` -CREATE USER [production_k0] FOR LOGIN [production_k0] WITH DEFAULT_SCHEMA=[production_k0] -GO +To force a rebuild of the test database from the mock data (for example if +schema changes and this is not automatically detected): -CREATE USER [production_k1] FOR LOGIN [production_k1] WITH DEFAULT_SCHEMA=[production_k1] -GO +```sh +./build.sh test --rebuild-test-fixtures +``` -ALTER ROLE db_owner ADD MEMBER production_k0 -GO +[Git]: https://git-scm.com/downloads +[Docker Desktop]: https://docs.docker.com/get-docker/ +[api.keyman.com]: https://github.com/keymanapp/api.keyman.com +[help.keyman.com]: https://github.com/keymanapp/help.keyman.com +[keyman.com]: https://github.com/keymanapp/keyman.com +[keymanweb.com]: https://github.com/keymanapp/keymanweb.com +[s.keyman.com]: https://github.com/keymanapp/s.keyman.com +[website-local-proxy]: https://github.com/keymanapp/website-local-proxy +[shared-sites]: https://github.com/keymanapp/shared-sites +[enable Hyper-V]: https://techcommunity.microsoft.com/t5/educator-developer-blog/step-by-step-enabling-hyper-v-for-use-on-windows-11/ba-p/3745905 +[enable WSL2]: https://ubuntu.com/tutorials/install-ubuntu-on-wsl2-on-windows-10#1-overview -ALTER ROLE db_owner ADD MEMBER production_k1 -GO -``` diff --git a/build.sh b/build.sh index 87e3a8a..b31ef75 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ ## START STANDARD SITE BUILD SCRIPT INCLUDE readonly THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" readonly BOOTSTRAP="$(dirname "$THIS_SCRIPT")/resources/bootstrap.inc.sh" -readonly BOOTSTRAP_VERSION=v1.0.6 +readonly BOOTSTRAP_VERSION=v1.0.7 [ -f "$BOOTSTRAP" ] && source "$BOOTSTRAP" || source <(curl -fs https://raw.githubusercontent.com/keymanapp/shared-sites/$BOOTSTRAP_VERSION/bootstrap.inc.sh) ## END STANDARD SITE BUILD SCRIPT INCLUDE @@ -28,6 +28,7 @@ builder_describe \ "start" \ "stop" \ "test" \ + "--rebuild-test-fixtures Rebuild the test fixtures from live data" \ ":db Build the database" \ ":app Build the site" @@ -37,6 +38,10 @@ function test_docker_container() { echo "TIER_TEST" > tier.txt # Note: ci.yml replicates these + if builder_has_option --rebuild-test-fixtures; then + touch rebuild-test-fixtures.txt + fi + # Run unit tests docker exec $API_KEYMAN_CONTAINER_DESC sh -c "vendor/bin/phpunit --testdox" @@ -92,13 +97,17 @@ function start_docker_container_db() { # Setup database builder_echo "Setting up DB container" - docker run -m 2048m --rm -d -p $PORT:1433 \ + docker run --rm -d -p $PORT:1433 \ -e "ACCEPT_EULA=Y" \ -e "MSSQL_AGENT_ENABLED=true" \ -e "MSSQL_SA_PASSWORD=yourStrong(\!)Password" \ --name $CONTAINER_DESC \ $CONTAINER_NAME + builder_echo "Sleeping for 30 seconds to give database time to spin up" + builder_echo "(DB may crash if connected to, too early, on some systems)" + sleep 30s + builder_echo green "SQL Server Listening on localhost:$PORT" } @@ -137,6 +146,8 @@ function start_docker_container_app() { ADD_HOST="--add-host host.docker.internal:host-gateway" fi + builder_echo "Checking network settings" + db_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${API_KEYMAN_DB_IMAGE_NAME}) builder_echo "Spooling up site container" diff --git a/resources/init-container.sh b/resources/init-container.sh index 03b79fd..2f98400 100755 --- a/resources/init-container.sh +++ b/resources/init-container.sh @@ -1,10 +1,16 @@ #!/usr/bin/env bash -echo "---- Sleep 15 Before Generating DB ----" +openssl version +uname -a + +echo "---- Sleep 15 for SQL Server to start before generating DB ----" sleep 15; # If we know we are immediately going to run tests, there's no need to build # the database and then rebuild it again as a test database! if [[ ! -f /var/www/html/tier.txt ]] || [[ $(bindParam(":prmEndDate", $endDate); $stmt->execute(); - $data = $stmt->fetchAll()[0]; - $data = array_filter($data, "Keyman\\Site\\com\\keyman\\api\\filter_columns_by_name", ARRAY_FILTER_USE_KEY ); + $data = $stmt->fetchAll(); + //$data = array_filter($data, "Keyman\\Site\\com\\keyman\\api\\filter_columns_by_name", ARRAY_FILTER_USE_KEY ); return $data; } } diff --git a/script/statistics/annual.php b/script/statistics/annual.php index fe617bd..44469f2 100644 --- a/script/statistics/annual.php +++ b/script/statistics/annual.php @@ -29,4 +29,9 @@ $stats = new \Keyman\Site\com\keyman\api\AnnualStatistics(); $data = $stats->execute($mssql, $startDate, $endDate); - json_print($data); + $rows = []; + foreach($data as $row) { + $rows[$row[0]] = ["Value" => $row[2], "Comment" => $row[1]]; + } + + json_print($rows); diff --git a/static-data/aag.tab b/static-data/aag.tab new file mode 100644 index 0000000..64d8ea3 --- /dev/null +++ b/static-data/aag.tab @@ -0,0 +1 @@ +# This file should be populated with data from https://github.com/keymanapp/all-access-goals diff --git a/tests/TestDBBuild.inc.php b/tests/TestDBBuild.inc.php index 8449822..5d573bc 100644 --- a/tests/TestDBBuild.inc.php +++ b/tests/TestDBBuild.inc.php @@ -46,9 +46,11 @@ static function Build() // Connect to database. TODO: refactor with DBConnect $dci = new \DatabaseConnectionInfo(); - $env = getenv(); - $force = !empty($env['TEST_REBUILD']); - unset($env['TEST_REBUILD']); // wow, but means this only gets run once :) + $force = file_exists(__DIR__ . '/../rebuild-test-fixtures.txt'); + if($force) { + \build_log("Rebuilding test fixtures\n"); + unlink(__DIR__ . '/../rebuild-test-fixtures.txt'); // means this only gets run once + } $schema = $dci->getActiveSchema(); try { diff --git a/tests/_prepend_rebuild.php b/tests/_prepend_rebuild.php new file mode 100644 index 0000000..0fdc6b3 --- /dev/null +++ b/tests/_prepend_rebuild.php @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/tools/db/build/aag.sql b/tools/db/build/aag.sql new file mode 100644 index 0000000..2243ff9 --- /dev/null +++ b/tools/db/build/aag.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS t_aag_language; + +CREATE TABLE t_aag_language ( + language_id nvarchar(3) not null primary key +); + diff --git a/tools/db/build/build_aag_data_script.inc.php b/tools/db/build/build_aag_data_script.inc.php new file mode 100644 index 0000000..a1bf0b8 --- /dev/null +++ b/tools/db/build/build_aag_data_script.inc.php @@ -0,0 +1,209 @@ +script_path = $data_root; + $this->force = $do_force; + + if(!is_dir($this->script_path)) { + mkdir($this->script_path, 0777, true) || fail("Unable to create folder {$this->script_path}"); + } + + reportTime(); + + if(!$this->cache_aag_languages(__DIR__ . '/../../../static-data/aag.tab', 'aag.sql', 't_aag_language')) { + fail("Failed to build data from aag.tab"); + } + + return true; + } + + /* + * Takes the AAG languages tab and turns it into a script to import it. + */ + + function cache_aag_languages($tabfilename, $sqlfilename, $table) { + return $this->create_tab_delimited_data_script($tabfilename, $sqlfilename, $table); + } + + /* + * Downloads an ISO639-3 file and builds a script to import it. + */ + + function cache_iso639_3_file($url, $tabfilename, $sqlfilename, $table, $columns) { + return $this->cache_tab_delimited_data($url, $tabfilename, $sqlfilename, $table, $columns); + } + + + private $languages = array(); + private $scripts = array(); + private $regions = array(); + + /** + * Build a SQL script to insert language-subtag-registry data into the database + */ + function build_sql_data_script_subtags() { + $cache_file = $this->script_path . "language-subtag-registry"; + if(!cache($this->DBDataSources->uriLanguageSubtagRegistry, $cache_file, 60 * 60 * 24 * 7, $this->force)) { + return false; + } + + if(($file = file($cache_file, FILE_IGNORE_NEW_LINES)) === FALSE) { + return false; + } + if(($file = $this->unwrap($file)) === FALSE) { + return false; + } + if(!$this->process_subtag_file($file)) { + return false; + } + + return + $this->generate_language_inserts() . + $this->generate_language_index_inserts() . + $this->generate_script_inserts() . + $this->generate_region_inserts(); + } + + /** + * language-subtag-registry wraps long lines with a two-space prefix on + * subsequent lines. So easiest to unwrap those lines before processing. + */ + function unwrap($array) { + $p = ''; + for($i = sizeof($array)-1; $i >= 0; $i--) { + if(substr($array[$i], 0, 2) == ' ') { + $p = substr($array[$i], 2, 1024) . ' ' . trim($p); + $array[$i] = 'WRAP:'.$array[$i]; + } elseif(!empty($p)) { + $array[$i] .= ' ' . trim($p); + $p = ''; + } + } + return $array; + } + + /** + * Loads the entries we are interested in from the language-subtag-registry + * into arrays for processing. + */ + function process_subtag_file($file) { + $row = array(); + foreach($file as $line) { + $line = trim($line); + if($line == '%%') { + if(!empty($row)) $this->process_entry($row); + $row = array(); + continue; + } + if($line == '') continue; + + $v = explode(':', $line); + $id = $v[0]; $v = trim($v[1]); + if(array_key_exists($id, $row)) { + $this->to_array($row, $id); + array_push($row[$id], $v); + } else { + $row[$id] = $v; + } + } + + return true; + } + + /** + * Processes a single entry, as delimited by %% in the language-subtag-registry, + * and adds it to the appropriate array. At this time, we are only interested in + * the subtag and the description(s) for the given entry. + */ + function process_entry($row) { + if(!isset($row['Type'])) return; + $this->to_array($row, 'Description'); + if($row['Description'][0] == 'Private use') { + // no 'scope' set for script, region private use descriptive subtags + // We don't want the "private use" subtags as they are a range rather than + // a single subtag + return; + } + if(isset($row['Scope']) && $row['Scope'] == 'private-use') return; + + // We'll work with all subtags as lower case for search etc + if(!isset($row['Subtag'])) return; + $subtag = strtolower($row['Subtag']); + + switch($row['Type']) { + case 'language': + $this->languages[$subtag] = $row['Description']; + break; + case 'script': + $this->scripts[$subtag] = $row['Description']; + break; + case 'region': + $this->regions[$subtag] = $row['Description']; + break; + } + } + + /** + * Generate an SQL script to insert entries in to the t_language table + */ + function generate_language_inserts() { + $result = "" ; + + $comma=''; + foreach($this->languages as $lang => $detail) { + $result .= "INSERT t_language (language_id) VALUES({$this->sqlv(null,$lang)})\n"; + } + return $result; + } + + /** + * Generate an SQL script to insert entries in to the t_language_index table + */ + function generate_language_index_inserts() { + $result = ""; + foreach($this->languages as $lang => $detail) { + foreach($detail as $name) { + $result .= "INSERT t_language_index (language_id, name) VALUES ({$this->sqlv(null,$lang)},{$this->sqlv(null,$name)})\n"; + } + } + return $result; + } + + /** + * Generate an SQL script to insert entries in to the t_script table + */ + function generate_script_inserts() { + $result = ""; + + foreach($this->scripts as $script => $detail) { + $result .= "INSERT t_script (script_id, name) VALUES ({$this->sqlv(null,$script)},{$this->sqlv(null,$detail[0])})\n"; + } + return $result; + } + + /** + * Generate an SQL script to insert entries in to the t_region table + */ + function generate_region_inserts() { + $result = ""; + + foreach($this->regions as $region => $detail) { + $result .= "INSERT t_region (region_id, name) VALUES ({$this->sqlv(null,$region)},{$this->sqlv(null,$detail[0])})\n"; + } + return $result; + } + + /** + * Helper function to convert a value into an array if it isn't already + */ + function to_array(&$row, $id) { + if(!is_array($row[$id])) $row[$id] = array($row[$id]); + } + + } +?> \ No newline at end of file