fix(common/models): reconnects unit tests for worker-internal submodules 🎡

common/models/templates/package.json

@@ -33,7 +33,7 @@
   },
   "scripts": {
     "build": "tsc",
-    "pretest": "npm run build",
+    "pretest": "tsc -b tsconfig.bundled.json",
     "test": "mocha -r test/helpers.js"
   },
   "bugs": {
@@ -47,7 +47,7 @@
     "@types/node": "^14.0.4",
     "chai": "^4.3.4",
     "mocha": "^8.4.0",
-    "typescript": "^3.8.3"
+    "typescript": "^4.5.4"
   },
   "dependencies": {
     "@keymanapp/models-wordbreakers": "*"

common/models/templates/test/helpers.js

@@ -7,8 +7,10 @@ var _ = global;
 // TODO: then mocha invocation is as follows:
 // TODO:     mocha -r @keymanapp/models-test-helpers test/
 
-// KMW string must be included, so do it here:
-require('@keymanapp/web-utils');
+// Ensure that we can successfully load the module & apply kmwLength, as it's
+// needed for some of the unit tests.
+require('../build/index.bundled.js');
+
 assert.ok('💩'.kmwLength);
 
 /**
@@ -21,7 +23,7 @@ _.jsonFixture = function (name) {
 /**
  * Returns the Context of an empty buffer; no text, at both the start and
  * end of the buffer.
- * 
+ *
  * @returns {Context}
  */
 _.emptyContext = function emptyContext() {

common/models/templates/test/test-common.js

@@ -3,7 +3,7 @@
  */
 
 var assert = require('chai').assert;
-var models = require('../').models;
+var models = require('../build/index.bundled.js').models;
 
 describe('Common utility functions', function() {
   // TODO:  unit tests for other common utility functions
@@ -97,7 +97,7 @@ describe('Common utility functions', function() {
       let apple = {
         insert: 'apple',
         deleteLeft: 0,
-        deleteRight: 2 
+        deleteRight: 2
       };
 
       let banana = {
@@ -143,7 +143,7 @@ describe('Common utility functions', function() {
       let apple = {
         insert: 'apple',
         deleteLeft: 0,
-        deleteRight: 2 
+        deleteRight: 2
       };
 
       let banana = {
@@ -167,7 +167,7 @@ describe('Common utility functions', function() {
       let apple = {
         insert: 'apple',
         deleteLeft: 0,
-        deleteRight: 2 
+        deleteRight: 2
       };
 
       let banana = {

common/models/templates/test/test-priority-queue.js

@@ -3,12 +3,12 @@
  */
 
 var assert = require('chai').assert;
-var PriorityQueue = require('../').models.PriorityQueue;
+var PriorityQueue = require('../build/index.bundled.js').models.PriorityQueue;
 
 describe('Priority queue', function() {
   it('can act as a min-heap', function () {
     let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6];
-    
+
     let queue = new PriorityQueue((a, b) => a - b);
     input.forEach((input) => queue.enqueue(input));
 
@@ -41,7 +41,7 @@ describe('Priority queue', function() {
 
   it('can act as a max-heap', function () {
     let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6];
-    
+
     let queue = new PriorityQueue((a, b) => b - a);
     input.forEach((input) => queue.enqueue(input));
 

common/models/templates/test/test-quote-behavior.js

@@ -3,7 +3,7 @@
  */
 
 var assert = require('chai').assert;
-var QuoteBehavior = require('../').models.QuoteBehavior;
+var QuoteBehavior = require('../build/index.bundled.js').models.QuoteBehavior;
 
 describe('Quote behaviors', function() {
   describe('Script directionality', function() {
@@ -144,15 +144,15 @@ describe('Quote behaviors', function() {
         quotesForKeepSuggestion: { open: `“`, close: `”`},
         insertAfterWord: " "
       }
-      
+
       assert.throws(function() {
         QuoteBehavior.apply(QuoteBehavior.default, "hello", englishPunctuation, QuoteBehavior.default);
       });
-  
+
       assert.throws(function() {
         QuoteBehavior.apply(QuoteBehavior.useQuotes, "hello", englishPunctuation, QuoteBehavior.default);
       });
-  
+
       assert.throws(function() {
         QuoteBehavior.apply(QuoteBehavior.noQuotes, "hello", englishPunctuation, QuoteBehavior.default);
       });

common/models/templates/test/test-tokenization.js

@@ -3,7 +3,7 @@
  */
 
 var assert = require('chai').assert;
-var models = require('../').models;
+var models = require('../build/index.bundled.js').models;
 var wordBreakers = require('@keymanapp/models-wordbreakers').wordBreakers;
 
 describe('Tokenization functions', function() {
@@ -132,15 +132,15 @@ describe('Tokenization functions', function() {
         left: '', startOfBuffer: true,
         right: '', endOfBuffer: true
       };
-  
+
       let tokenization = models.tokenize(wordBreakers.default, context);
 
       let expectedResult = {
         left: [],
         right: [],
         caretSplitsToken: false
       };
-  
+
       assert.deepEqual(tokenization, expectedResult);
     });
 
@@ -153,7 +153,7 @@ describe('Tokenization functions', function() {
         right: [],
         caretSplitsToken: false
       };
-  
+
       assert.deepEqual(tokenization, expectedResult);
     });
 
@@ -163,15 +163,15 @@ describe('Tokenization functions', function() {
         left: ' ', startOfBuffer: true,
         right: '', endOfBuffer: true
       };
-  
+
       let tokenization = models.tokenize(wordBreakers.default, context);
 
       let expectedResult = {
         left: [''],
         right: [],
         caretSplitsToken: false
       };
-  
+
       assert.deepEqual(tokenization, expectedResult);
     });
 
@@ -233,7 +233,7 @@ describe('Tokenization functions', function() {
         case 'ស្រុកខ្មែរ':
           return [srok, shiftSpan(khmer, srok.length)]; // array of the two.
         case 'កខ្មែរ':
-          // I'd admittedly be at least somewhat surprised if a real wordbreaker got this 
+          // I'd admittedly be at least somewhat surprised if a real wordbreaker got this
           // and similar situations perfectly right... but at least it gives us what
           // we need for a test.
           return [k, shiftSpan(khmer, k.length)];

common/models/templates/test/test-trie-model.js

@@ -3,7 +3,7 @@
  */
 
 var assert = require('chai').assert;
-var TrieModel = require('../').models.TrieModel;
+var TrieModel = require('../build/index.bundled.js').models.TrieModel;
 
 describe('LMLayerWorker trie model for word lists', function() {
   describe('instantiation', function () {
@@ -20,7 +20,7 @@ describe('LMLayerWorker trie model for word lists', function() {
           }
         }
       })
-      
+
       assert.equal(model.punctuation.insertAfterWord, spaceMark);
       assert.equal(model.punctuation.quotesForKeepSuggestion.open, openQuote);
       assert.equal(model.punctuation.quotesForKeepSuggestion.close, closeQuote);
@@ -176,7 +176,7 @@ describe('LMLayerWorker trie model for word lists', function() {
       );
     });
   });
-  
+
   describe('The default key function', function () {
     it('uses the default key function', function () {
       var model = new TrieModel(jsonFixture('tries/accented'));

common/models/templates/test/test-trie-traversal.js

@@ -3,7 +3,7 @@
  */
 
 var assert = require('chai').assert;
-var TrieModel = require('../').models.TrieModel;
+var TrieModel = require('../build/index.bundled.js').models.TrieModel;
 
 // Useful for tests related to strings with supplementary pairs.
 var smpForUnicode = function(code){
@@ -52,7 +52,7 @@ describe('Trie traversal abstractions', function() {
         assert.isEmpty(child.traversal().entries);
 
         for(tChild of traversalInner1.children()) {
-          if(tChild.char == 'h') { 
+          if(tChild.char == 'h') {
             hSuccess = true;
             let traversalInner2 = tChild.traversal();
             assert.isDefined(traversalInner2);
@@ -64,7 +64,7 @@ describe('Trie traversal abstractions', function() {
                 eSuccess = true;
                 let traversalInner3 = hChild.traversal();
                 assert.isDefined(traversalInner3);
-                
+
                 assert.isDefined(traversalInner3.entries);
                 assert.equal(traversalInner3.entries[0], "the");
 
@@ -104,7 +104,7 @@ describe('Trie traversal abstractions', function() {
         assert.isEmpty(child.traversal().entries);
 
         for(tChild of traversalInner1.children()) {
-          if(tChild.char == 'r') { 
+          if(tChild.char == 'r') {
             let traversalInner2 = tChild.traversal();
             assert.isDefined(traversalInner2);
             assert.isArray(tChild.traversal().entries);
@@ -185,7 +185,7 @@ describe('Trie traversal abstractions', function() {
         assert.isEmpty(child.traversal().entries);
 
         for(aChild of traversalInner1.children()) {
-          if(aChild.char == smpP) { 
+          if(aChild.char == smpP) {
             pSuccess = true;
             let traversalInner2 = aChild.traversal();
             assert.isDefined(traversalInner2);

common/models/templates/tsconfig.bundled.json

@@ -0,0 +1,21 @@
+{
+  // This variant of the tsconfig.json exists to create a 'leaf', 'bundled'
+  // version of the models/templates build product.  The same reference
+  // cannot be prepended twice in a composite tsc build, posing problems
+  // for certain down-line builds if the two tsconfigs are not differentiated.
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outFile": "build/index.bundled.js",
+  },
+  "references": [
+    { "path": "../../web/keyman-version", "prepend": true},
+    { "path": "../../web/utils", "prepend": true },
+    { "path": "../types" }
+  ],
+  "include": [
+	"src/**/*.ts"
+  ],
+  "exclude": [
+	"test"
+  ]
+}

common/models/wordbreakers/package.json

@@ -14,7 +14,7 @@
   ],
   "homepage": "https://github.com/keymanapp/keyman",
   "license": "MIT",
-  "main": "index.js",
+  "main": "build/index.js",
   "directories": {
     "lib": "lib",
     "test": "test"

common/models/wordbreakers/src/default/index.ts

@@ -3,8 +3,8 @@
 namespace wordBreakers {
   /**
    * Word breaker based on Unicode Standard Annex #29, Section 4.1:
-   * Default Word Boundary Specification.   
-   * 
+   * Default Word Boundary Specification.
+   *
    * @see http://unicode.org/reports/tr29/#Word_Boundaries
    * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0
    */
@@ -349,6 +349,12 @@ namespace wordBreakers {
 
   /**
    * Binary search for the word break property of a given CODE POINT.
+   *
+   * The auto-generated data.ts master array defines a **character range**
+   * lookup table.  If a character's codepoint is equal to or greater than
+   * the I.Start value for an entry and exclusively less than the next entry,
+   * it falls in the first entry's range bucket and is classified accordingly
+   * by this method.
    */
   function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty {
     // All items that are not found in the array are assigned the 'Other' property.

common/models/wordbreakers/test/test-ascii-word-breaker.js

@@ -1,5 +1,5 @@
 var assert = require('chai').assert;
-var breakASCIIWords = require('..').wordBreakers['ascii'];
+var breakASCIIWords = require('../build').wordBreakers['ascii'];
 
 describe('The ASCII word breaker', function () {
   it('should break simple English sentences', function () {

common/models/wordbreakers/test/test-default-word-breaker.js

@@ -3,9 +3,9 @@
  */
 
 const assert = require('chai').assert;
-const breakWords = require('..').wordBreakers['default'];
+const breakWords = require('../build').wordBreakers['default'];
 
-const SHY = '\u00AD';
+const SHY = '\u00AD'; // Other, Format.  The "Soft HYphen" - usually invisible unless needed for word-wrapping.
 
 describe('The default word breaker', function () {
   it('should break multilingual text', function () {
@@ -20,4 +20,16 @@ describe('The default word breaker', function () {
       "let's", 'eat', 'phở', '!', '🥣'
     ]);
   });
+
+  it('handles heavily-punctuated English text', function() {
+    // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1.
+    let breaks = breakWords(
+      `The quick ("brown") fox can't jump 32.3 feet, right?`
+    );
+    let words = breaks.map(span => span.text);
+    assert.deepEqual(words, [
+      'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't",
+      'jump', '32.3', 'feet', ',', 'right', '?'
+    ]);
+  });
 });

common/models/wordbreakers/test/test-placeholder-word-breaker.js

@@ -1,5 +1,5 @@
 const assert = require('chai').assert;
-const breakWords = require('..').wordBreakers['placeholder'];
+const breakWords = require('../build').wordBreakers['placeholder'];
 
 
 describe('The placeholder word breaker', function () {

common/predictive-text/unit_tests/test.sh

@@ -33,6 +33,18 @@ test-headless ( ) {
     _FLAGS="$_FLAGS --reporter mocha-teamcity-reporter"
   fi
 
+  pushd "$KEYMAN_ROOT/common/models/wordbreakers"
+  npm run test || fail "models/wordbreakers tests failed"
+  popd
+
+  pushd "$KEYMAN_ROOT/common/models/templates"
+  npm run test || fail "models/templates tests failed"
+  popd
+
+  pushd "$KEYMAN_ROOT/common/models/types"
+  npm run test || fail "models/types tests failed"
+  popd
+
   npm run mocha -- --recursive $_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js
 }
 

common/web/utils/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@keymanapp/web-utils",
   "description": "Common utility functions used throughout other Keyman packages",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "./build/index.js",
+  "types": "./build/index.d.ts",
   "scripts": {
     "build": "gosh ./build.sh",
     "tsc": "tsc"