From 7dcecf8836e518495ab463ee44d942863b0f398c Mon Sep 17 00:00:00 2001 From: jon Date: Tue, 4 Jul 2023 15:17:40 +0100 Subject: [PATCH 01/11] fix args passing to sweep --- elk/training/sweep.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index e4aca5a0..b13136f4 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -171,7 +171,13 @@ def execute(self): num_gpus=run.num_gpus, min_gpu_mem=run.min_gpu_mem, skip_supervised=run.supervised == "none", + prompt_indices=run.prompt_indices, + concatenated_layer_offset=run.concatenated_layer_offset, + datasets=run.datasets, + debug=run.debug, + disable_cache=run.disable_cache, ) + eval.execute(highlight_color="green") if self.visualize: From a697d24680bbea4fdbe6945610a18e79b76847f1 Mon Sep 17 00:00:00 2001 From: jon Date: Tue, 4 Jul 2023 14:23:03 +0000 Subject: [PATCH 02/11] remove datasets --- elk/training/sweep.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index b13136f4..27306431 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -162,6 +162,7 @@ def execute(self): continue assert run.out_dir is not None + # TODO we should fix this so that this isn't needed eval = Eval( data=replace( run.data, model=model, datasets=(eval_dataset,) @@ -173,7 +174,9 @@ def execute(self): skip_supervised=run.supervised == "none", prompt_indices=run.prompt_indices, concatenated_layer_offset=run.concatenated_layer_offset, - datasets=run.datasets, + # datasets=run.datasets, + # this isn't needed because it's + # immediately overwritten debug=run.debug, disable_cache=run.disable_cache, ) From aa1dc8861e935dffaf047846d4abc831e3769b86 Mon Sep 17 00:00:00 2001 From: jon Date: Mon, 10 Jul 2023 13:39:44 +0100 Subject: [PATCH 03/11] change variable name --- elk/evaluation/evaluate.py | 2 +- elk/run.py | 8 ++++---- elk/training/ccs_reporter.py | 4 ++-- elk/training/classifier.py | 2 +- elk/training/sweep.py | 12 ++++++++---- tests/test_classifier.py | 2 +- tests/test_smoke_elicit.py | 4 ++-- tests/test_smoke_eval.py | 6 +++--- 8 files changed, 22 insertions(+), 18 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index d6054e33..ffbe2d97 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -23,7 +23,7 @@ def __post_init__(self): # Set our output directory before super().execute() does if not self.out_dir: root = elk_reporter_dir() / self.source - self.out_dir = root / "transfer" / "+".join(self.data.datasets) + self.out_dir = root / "transfer" / "+".join(self.extract.datasets) def execute(self, highlight_color: Color = "cyan"): return super().execute(highlight_color, split_type="val") diff --git a/elk/run.py b/elk/run.py index fb8903cc..5ab020d7 100644 --- a/elk/run.py +++ b/elk/run.py @@ -33,7 +33,7 @@ @dataclass class Run(ABC, Serializable): - data: Extract + extract: Extract out_dir: Path | None = None """Directory to save results to. If None, a directory will be created automatically.""" @@ -67,14 +67,14 @@ def execute( min_gpu_mem=self.min_gpu_mem, split_type=split_type, ) - for cfg in self.data.explode() + for cfg in self.extract.explode() ] if self.out_dir is None: # Save in a memorably-named directory inside of # ELK_REPORTER_DIR// - ds_name = "+".join(self.data.datasets) - root = elk_reporter_dir() / self.data.model / ds_name + ds_name = "+".join(self.extract.datasets) + root = elk_reporter_dir() / self.extract.model / ds_name self.out_dir = memorably_named_dir(root) diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py index cd161dd9..a5d7d628 100644 --- a/elk/training/ccs_reporter.py +++ b/elk/training/ccs_reporter.py @@ -145,7 +145,7 @@ def reset_parameters(self): theta = torch.randn(1, probe.in_features + 1, device=probe.weight.device) theta /= theta.norm() probe.weight.data = theta[:, :-1] - probe.bias.data = theta[:, -1] + probe.bias.extract = theta[:, -1] elif self.config.init == "default": for layer in self.probe: @@ -219,7 +219,7 @@ def fit(self, hiddens: Tensor) -> float: if self.config.init == "pca": diffs = torch.flatten(x_pos - x_neg, 0, 1) _, __, V = torch.pca_lowrank(diffs, q=i + 1) - self.probe[0].weight.data = V[:, -1, None].T + self.probe[0].weight.extract = V[:, -1, None].T if self.config.optimizer == "lbfgs": loss = self.train_loop_lbfgs(x_neg, x_pos) diff --git a/elk/training/classifier.py b/elk/training/classifier.py index 148da939..4e7adc97 100644 --- a/elk/training/classifier.py +++ b/elk/training/classifier.py @@ -51,7 +51,7 @@ def __init__( self.linear = torch.nn.Linear( input_dim, num_classes if num_classes > 2 else 1, device=device, dtype=dtype ) - self.linear.bias.data.zero_() + self.linear.bias.extract.zero_() self.linear.weight.data.zero_() def forward(self, x: Tensor) -> Tensor: diff --git a/elk/training/sweep.py b/elk/training/sweep.py index 27306431..bebebfbb 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -53,7 +53,7 @@ class Sweep: # A bit of a hack to add all the command line arguments from Elicit run_template: Elicit = Elicit( - data=Extract( + extract=Extract( model="", datasets=("",), ) @@ -132,7 +132,9 @@ def execute(self): out_dir = sweep_dir / model / dataset_str data = replace( - self.run_template.data, model=model, datasets=train_datasets + self.run_template.extract, + model=model, + datasets=train_datasets, ) run = replace(self.run_template, data=data, out_dir=out_dir) if var_weight is not None and neg_cov_weight is not None: @@ -164,8 +166,10 @@ def execute(self): assert run.out_dir is not None # TODO we should fix this so that this isn't needed eval = Eval( - data=replace( - run.data, model=model, datasets=(eval_dataset,) + extract=replace( + run.extract, + model=model, + datasets=(eval_dataset,), ), source=run.out_dir, out_dir=run.out_dir / "transfer" / eval_dataset, diff --git a/tests/test_classifier.py b/tests/test_classifier.py index bdc9023d..10b86a19 100644 --- a/tests/test_classifier.py +++ b/tests/test_classifier.py @@ -28,7 +28,7 @@ def test_classifier_roughly_same_sklearn(): ) # check that the weights are roughly the same sklearn_coef = torch.from_numpy(model.coef_) - torch_coef = classifier.linear.weight.data + torch_coef = classifier.linear.weight.extract torch.testing.assert_close(sklearn_coef, torch_coef, atol=1e-2, rtol=1e-2) # check that on a new sample, the predictions are roughly the same diff --git a/tests/test_smoke_elicit.py b/tests/test_smoke_elicit.py index bac0f398..4bfb430d 100644 --- a/tests/test_smoke_elicit.py +++ b/tests/test_smoke_elicit.py @@ -10,7 +10,7 @@ def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path): model_path, min_mem = "sshleifer/tiny-gpt2", 10 * 1024**2 dataset_name = "imdb" elicit = Elicit( - data=Extract( + extract=Extract( model=model_path, datasets=(dataset_name,), max_examples=(10, 10), @@ -41,7 +41,7 @@ def test_smoke_elicit_run_tiny_gpt2_eigen(tmp_path: Path): model_path, min_mem = "sshleifer/tiny-gpt2", 10 * 1024**2 dataset_name = "imdb" elicit = Elicit( - data=Extract( + extract=Extract( model=model_path, datasets=(dataset_name,), max_examples=(10, 10), diff --git a/tests/test_smoke_eval.py b/tests/test_smoke_eval.py index 4efd7112..0fe70cd4 100644 --- a/tests/test_smoke_eval.py +++ b/tests/test_smoke_eval.py @@ -26,7 +26,7 @@ def setup_elicit( Returns the elicit run configuration. """ elicit = Elicit( - data=Extract( + extract=Extract( model=model_path, datasets=(dataset_name,), max_examples=(10, 10), @@ -54,7 +54,7 @@ def eval_run(elicit: Elicit, transfer_datasets: tuple[str, ...] = ()) -> float: Returns a reference time (in seconds) for file modification checking. """ tmp_path = elicit.out_dir - extract = elicit.data + extract = elicit.extract assert tmp_path is not None # record elicit modification time as reference. @@ -64,7 +64,7 @@ def eval_run(elicit: Elicit, transfer_datasets: tuple[str, ...] = ()) -> float: # update datasets to a different dataset extract.datasets = transfer_datasets - eval = Eval(data=extract, source=tmp_path) + eval = Eval(extract=extract, source=tmp_path) eval.execute() return start_time_sec From 0acd41c457c3d5fa8e31b93949ef645a55a0d455 Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Jul 2023 14:27:26 +0100 Subject: [PATCH 04/11] move to make_eval --- elk/training/sweep.py | 20 +------------------- elk/training/train.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index bebebfbb..ec6e6b66 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -165,26 +165,8 @@ def execute(self): assert run.out_dir is not None # TODO we should fix this so that this isn't needed - eval = Eval( - extract=replace( - run.extract, - model=model, - datasets=(eval_dataset,), - ), - source=run.out_dir, - out_dir=run.out_dir / "transfer" / eval_dataset, - num_gpus=run.num_gpus, - min_gpu_mem=run.min_gpu_mem, - skip_supervised=run.supervised == "none", - prompt_indices=run.prompt_indices, - concatenated_layer_offset=run.concatenated_layer_offset, - # datasets=run.datasets, - # this isn't needed because it's - # immediately overwritten - debug=run.debug, - disable_cache=run.disable_cache, - ) + eval = run.make_eval(model, eval_dataset) eval.execute(highlight_color="green") if self.visualize: diff --git a/elk/training/train.py b/elk/training/train.py index 8392f2d9..cf6f6d75 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -1,7 +1,7 @@ """Main training loop.""" from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Literal @@ -11,6 +11,7 @@ from simple_parsing import subgroups from simple_parsing.helpers.serialization import save +from ..evaluation import Eval from ..metrics import evaluate_preds, to_one_hot from ..run import Run from ..training.supervised import train_supervised @@ -48,6 +49,27 @@ def create_models_dir(self, out_dir: Path): return reporter_dir, lr_dir + def make_eval(self, model, eval_dataset): + return Eval( + extract=replace( + self.extract, + model=model, + datasets=(eval_dataset,), + ), + source=self.out_dir, + out_dir=self.out_dir / "transfer" / eval_dataset, + num_gpus=self.num_gpus, + min_gpu_mem=self.min_gpu_mem, + skip_supervised=self.supervised == "none", + prompt_indices=self.prompt_indices, + concatenated_layer_offset=self.concatenated_layer_offset, + # datasets=run.datasets isn't needed because it's + # immediately overwritten + debug=self.debug, + disable_cache=self.disable_cache, + ) + + def apply_to_layer( self, layer: int, From 025acb1d5bed83c51ccc1ffb6fea0fb45e418f8a Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Jul 2023 14:35:43 +0100 Subject: [PATCH 05/11] Revert "change variable name" This reverts commit f2cdbb104c383093a5d827edb75b507c18b15b89. --- elk/evaluation/evaluate.py | 2 +- elk/run.py | 8 ++++---- elk/training/ccs_reporter.py | 4 ++-- elk/training/classifier.py | 2 +- elk/training/sweep.py | 6 ++---- tests/test_classifier.py | 2 +- tests/test_smoke_elicit.py | 4 ++-- tests/test_smoke_eval.py | 6 +++--- 8 files changed, 16 insertions(+), 18 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index ffbe2d97..d6054e33 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -23,7 +23,7 @@ def __post_init__(self): # Set our output directory before super().execute() does if not self.out_dir: root = elk_reporter_dir() / self.source - self.out_dir = root / "transfer" / "+".join(self.extract.datasets) + self.out_dir = root / "transfer" / "+".join(self.data.datasets) def execute(self, highlight_color: Color = "cyan"): return super().execute(highlight_color, split_type="val") diff --git a/elk/run.py b/elk/run.py index 5ab020d7..fb8903cc 100644 --- a/elk/run.py +++ b/elk/run.py @@ -33,7 +33,7 @@ @dataclass class Run(ABC, Serializable): - extract: Extract + data: Extract out_dir: Path | None = None """Directory to save results to. If None, a directory will be created automatically.""" @@ -67,14 +67,14 @@ def execute( min_gpu_mem=self.min_gpu_mem, split_type=split_type, ) - for cfg in self.extract.explode() + for cfg in self.data.explode() ] if self.out_dir is None: # Save in a memorably-named directory inside of # ELK_REPORTER_DIR// - ds_name = "+".join(self.extract.datasets) - root = elk_reporter_dir() / self.extract.model / ds_name + ds_name = "+".join(self.data.datasets) + root = elk_reporter_dir() / self.data.model / ds_name self.out_dir = memorably_named_dir(root) diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py index a5d7d628..cd161dd9 100644 --- a/elk/training/ccs_reporter.py +++ b/elk/training/ccs_reporter.py @@ -145,7 +145,7 @@ def reset_parameters(self): theta = torch.randn(1, probe.in_features + 1, device=probe.weight.device) theta /= theta.norm() probe.weight.data = theta[:, :-1] - probe.bias.extract = theta[:, -1] + probe.bias.data = theta[:, -1] elif self.config.init == "default": for layer in self.probe: @@ -219,7 +219,7 @@ def fit(self, hiddens: Tensor) -> float: if self.config.init == "pca": diffs = torch.flatten(x_pos - x_neg, 0, 1) _, __, V = torch.pca_lowrank(diffs, q=i + 1) - self.probe[0].weight.extract = V[:, -1, None].T + self.probe[0].weight.data = V[:, -1, None].T if self.config.optimizer == "lbfgs": loss = self.train_loop_lbfgs(x_neg, x_pos) diff --git a/elk/training/classifier.py b/elk/training/classifier.py index 4e7adc97..148da939 100644 --- a/elk/training/classifier.py +++ b/elk/training/classifier.py @@ -51,7 +51,7 @@ def __init__( self.linear = torch.nn.Linear( input_dim, num_classes if num_classes > 2 else 1, device=device, dtype=dtype ) - self.linear.bias.extract.zero_() + self.linear.bias.data.zero_() self.linear.weight.data.zero_() def forward(self, x: Tensor) -> Tensor: diff --git a/elk/training/sweep.py b/elk/training/sweep.py index ec6e6b66..9a239345 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -53,7 +53,7 @@ class Sweep: # A bit of a hack to add all the command line arguments from Elicit run_template: Elicit = Elicit( - extract=Extract( + data=Extract( model="", datasets=("",), ) @@ -132,9 +132,7 @@ def execute(self): out_dir = sweep_dir / model / dataset_str data = replace( - self.run_template.extract, - model=model, - datasets=train_datasets, + self.run_template.data, model=model, datasets=train_datasets ) run = replace(self.run_template, data=data, out_dir=out_dir) if var_weight is not None and neg_cov_weight is not None: diff --git a/tests/test_classifier.py b/tests/test_classifier.py index 10b86a19..bdc9023d 100644 --- a/tests/test_classifier.py +++ b/tests/test_classifier.py @@ -28,7 +28,7 @@ def test_classifier_roughly_same_sklearn(): ) # check that the weights are roughly the same sklearn_coef = torch.from_numpy(model.coef_) - torch_coef = classifier.linear.weight.extract + torch_coef = classifier.linear.weight.data torch.testing.assert_close(sklearn_coef, torch_coef, atol=1e-2, rtol=1e-2) # check that on a new sample, the predictions are roughly the same diff --git a/tests/test_smoke_elicit.py b/tests/test_smoke_elicit.py index 4bfb430d..bac0f398 100644 --- a/tests/test_smoke_elicit.py +++ b/tests/test_smoke_elicit.py @@ -10,7 +10,7 @@ def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path): model_path, min_mem = "sshleifer/tiny-gpt2", 10 * 1024**2 dataset_name = "imdb" elicit = Elicit( - extract=Extract( + data=Extract( model=model_path, datasets=(dataset_name,), max_examples=(10, 10), @@ -41,7 +41,7 @@ def test_smoke_elicit_run_tiny_gpt2_eigen(tmp_path: Path): model_path, min_mem = "sshleifer/tiny-gpt2", 10 * 1024**2 dataset_name = "imdb" elicit = Elicit( - extract=Extract( + data=Extract( model=model_path, datasets=(dataset_name,), max_examples=(10, 10), diff --git a/tests/test_smoke_eval.py b/tests/test_smoke_eval.py index 0fe70cd4..4efd7112 100644 --- a/tests/test_smoke_eval.py +++ b/tests/test_smoke_eval.py @@ -26,7 +26,7 @@ def setup_elicit( Returns the elicit run configuration. """ elicit = Elicit( - extract=Extract( + data=Extract( model=model_path, datasets=(dataset_name,), max_examples=(10, 10), @@ -54,7 +54,7 @@ def eval_run(elicit: Elicit, transfer_datasets: tuple[str, ...] = ()) -> float: Returns a reference time (in seconds) for file modification checking. """ tmp_path = elicit.out_dir - extract = elicit.extract + extract = elicit.data assert tmp_path is not None # record elicit modification time as reference. @@ -64,7 +64,7 @@ def eval_run(elicit: Elicit, transfer_datasets: tuple[str, ...] = ()) -> float: # update datasets to a different dataset extract.datasets = transfer_datasets - eval = Eval(extract=extract, source=tmp_path) + eval = Eval(data=extract, source=tmp_path) eval.execute() return start_time_sec From 80507fc182dd817a1d9dbf8ebf750fdf642dd494 Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Jul 2023 14:43:10 +0100 Subject: [PATCH 06/11] fix remnant of renaming --- elk/training/sweep.py | 1 - elk/training/train.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index 9a239345..ff6cc8e8 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -5,7 +5,6 @@ from datasets import get_dataset_config_info from transformers import AutoConfig -from ..evaluation import Eval from ..extraction import Extract from ..files import memorably_named_dir, sweeps_dir from ..plotting.visualize import visualize_sweep diff --git a/elk/training/train.py b/elk/training/train.py index cf6f6d75..2e6c2433 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -51,8 +51,8 @@ def create_models_dir(self, out_dir: Path): def make_eval(self, model, eval_dataset): return Eval( - extract=replace( - self.extract, + data=replace( + self.data, model=model, datasets=(eval_dataset,), ), From 06cb70ee02a2ef10060139fc8c8cf0e44f28626c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:43:48 +0000 Subject: [PATCH 07/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/training/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elk/training/train.py b/elk/training/train.py index 2e6c2433..8929ce7c 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -69,7 +69,6 @@ def make_eval(self, model, eval_dataset): disable_cache=self.disable_cache, ) - def apply_to_layer( self, layer: int, From ea56c631793484d9e544d611098ccd1e0f2221bf Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Jul 2023 14:44:13 +0100 Subject: [PATCH 08/11] rename to elicit --- elk/training/sweep.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index ff6cc8e8..fb6beba1 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -133,19 +133,19 @@ def execute(self): data = replace( self.run_template.data, model=model, datasets=train_datasets ) - run = replace(self.run_template, data=data, out_dir=out_dir) + elicit = replace(self.run_template, data=data, out_dir=out_dir) if var_weight is not None and neg_cov_weight is not None: - assert isinstance(run.net, EigenFitterConfig) - run.net.var_weight = var_weight - run.net.neg_cov_weight = neg_cov_weight + assert isinstance(elicit.net, EigenFitterConfig) + elicit.net.var_weight = var_weight + elicit.net.neg_cov_weight = neg_cov_weight # Add hyperparameter values to output directory if needed - assert run.out_dir is not None - run.out_dir /= f"var_weight={var_weight:.2f}" - run.out_dir /= f"neg_cov_weight={neg_cov_weight:.2f}" + assert elicit.out_dir is not None + elicit.out_dir /= f"var_weight={var_weight:.2f}" + elicit.out_dir /= f"neg_cov_weight={neg_cov_weight:.2f}" try: - run.execute() + elicit.execute() except torch.linalg.LinAlgError as e: print(colorize(f"LinAlgError: {e}", "red")) continue @@ -160,10 +160,10 @@ def execute(self): if eval_dataset in train_datasets: continue - assert run.out_dir is not None + assert elicit.out_dir is not None # TODO we should fix this so that this isn't needed - eval = run.make_eval(model, eval_dataset) + eval = elicit.make_eval(model, eval_dataset) eval.execute(highlight_color="green") if self.visualize: From 9988cc5cc9704ffc453519c9d7b239f010cd5f01 Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Jul 2023 14:52:19 +0100 Subject: [PATCH 09/11] remove unnecessary comments --- elk/training/sweep.py | 1 - elk/training/train.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index fb6beba1..d57c4600 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -161,7 +161,6 @@ def execute(self): continue assert elicit.out_dir is not None - # TODO we should fix this so that this isn't needed eval = elicit.make_eval(model, eval_dataset) eval.execute(highlight_color="green") diff --git a/elk/training/train.py b/elk/training/train.py index 8929ce7c..8099e25a 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -63,8 +63,7 @@ def make_eval(self, model, eval_dataset): skip_supervised=self.supervised == "none", prompt_indices=self.prompt_indices, concatenated_layer_offset=self.concatenated_layer_offset, - # datasets=run.datasets isn't needed because it's - # immediately overwritten + # datasets isn't needed because it's immediately overwritten debug=self.debug, disable_cache=self.disable_cache, ) From f7b96b00ac0ecc4f3ea00a39093d9cd6a732bc76 Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Jul 2023 15:17:15 +0100 Subject: [PATCH 10/11] fix pyright --- elk/training/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elk/training/train.py b/elk/training/train.py index 8099e25a..82316506 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -50,6 +50,7 @@ def create_models_dir(self, out_dir: Path): return reporter_dir, lr_dir def make_eval(self, model, eval_dataset): + assert self.out_dir is not None return Eval( data=replace( self.data, From 35c53f5e1c2b6373e7b5b9e2dfce587350f52bab Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Jul 2023 15:33:11 +0100 Subject: [PATCH 11/11] remove check --- elk/training/sweep.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index d57c4600..bbd761c9 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -160,8 +160,6 @@ def execute(self): if eval_dataset in train_datasets: continue - assert elicit.out_dir is not None - eval = elicit.make_eval(model, eval_dataset) eval.execute(highlight_color="green")