thien · thien · Jun 29, 2024 · Jun 29, 2024 · Jun 29, 2024 · Jun 29, 2024
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -0,0 +1,24 @@
+name: Run Lightweight Tests
+
+on: [push, pull_request]
+
+jobs:
+  lightweight-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false 
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Rye 
+      uses: eifinger/setup-rye@v4
+      with:
+        version: 'latest'
+    - name: Install Project
+      run: rye sync 
+    - name: Download misc. Dependencies
+      run: rye run python -m nltk.downloader cmudict averaged_perceptron_tagger
+    - name: Run Lightweight Tests
+      run: |
+        export PYTHONPATH=$PYTHONPATH:$(pwd)
+        rye run tests-light
diff --git a/.gitignore b/.gitignore
@@ -24,4 +24,22 @@ datasets/machine_translation/global_voices/training/*
 *.pyc
 *.atok
 base/models
+base/results
 hansard.36
+.DS_Store
+specialk/results
+specialk/telegram.json
+cache
+*.ruff_cache
+.vscode
+.venv
+.arrow
+tensorboard_logs
+/models/
+.envrc
+tb_logs
+*.parquet
+*.arrow
+*.ts
+*.ckpt
+specialk/notebooks/lightning_logs/*
diff --git a/.python-version b/.python-version
@@ -0,0 +1 @@
+3.9.7
diff --git a/assets/tokenizer/de_small_word_moses b/assets/tokenizer/de_small_word_moses
diff --git a/assets/tokenizer/en_small_word_moses b/assets/tokenizer/en_small_word_moses
diff --git a/assets/tokenizer/fr_en_bpe b/assets/tokenizer/fr_en_bpe
diff --git a/assets/tokenizer/fr_en_word_moses b/assets/tokenizer/fr_en_word_moses
diff --git a/assets/tokenizer/sentencepiece/enfr.model b/assets/tokenizer/sentencepiece/enfr.model