From 934cd54e9123fcaa6de36c42957e0229ea10e600 Mon Sep 17 00:00:00 2001 From: Nora Belrose Date: Wed, 26 Apr 2023 10:19:33 +0000 Subject: [PATCH 01/57] Log ensembled metrics --- elk/evaluation/evaluate.py | 39 +++++++++++++++++++------------ elk/metrics/eval.py | 36 +++++++++++++++++++++-------- elk/run.py | 2 +- elk/training/train.py | 47 +++++++++++++++++++++++--------------- 4 files changed, 82 insertions(+), 42 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 57ee6d03..2b9e8509 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -45,21 +45,32 @@ def apply_to_layer( for ds_name, (val_h, val_gt, _) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} - val_result = evaluate_preds(val_gt, reporter(val_h)) - row_bufs["eval"].append({**meta, **val_result.to_dict()}) + val_credences = reporter(val_h) + for mode in ("none", "partial", "full"): + row_bufs["eval"].append( + { + **meta, + "ensembling": mode, + **evaluate_preds(val_gt, val_credences, mode).to_dict(), + } + ) - lr_dir = experiment_dir / "lr_models" - if not self.skip_supervised and lr_dir.exists(): - with open(lr_dir / f"layer_{layer}.pt", "rb") as f: - lr_models = torch.load(f, map_location=device) - if not isinstance(lr_models, list): # backward compatibility - lr_models = [lr_models] + lr_dir = experiment_dir / "lr_models" + if not self.skip_supervised and lr_dir.exists(): + with open(lr_dir / f"layer_{layer}.pt", "rb") as f: + lr_models = torch.load(f, map_location=device) + if not isinstance(lr_models, list): # backward compatibility + lr_models = [lr_models] - for i, model in enumerate(lr_models): - model.eval() - lr_result = evaluate_preds(val_gt, model(val_h)) - row_bufs["lr_eval"].append( - {"inlp_iter": i, **meta, **lr_result.to_dict()} - ) + for i, model in enumerate(lr_models): + model.eval() + row_bufs["lr_eval"].append( + { + "ensembling": mode, + "inlp_iter": i, + **meta, + **evaluate_preds(val_gt, model(val_h), mode).to_dict(), + } + ) return {k: pd.DataFrame(v) for k, v in row_bufs.items()} diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index dcc5ce35..653beae5 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -1,4 +1,5 @@ from dataclasses import asdict, dataclass +from typing import Literal import torch from einops import repeat @@ -37,16 +38,20 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: else {} ) auroc_dict = {f"{prefix}auroc_{k}": v for k, v in asdict(self.roc_auc).items()} - return {**acc_dict, **cal_acc_dict, **cal_dict, **auroc_dict} + return {**auroc_dict, **cal_acc_dict, **acc_dict, **cal_dict} -def evaluate_preds(y_true: Tensor, y_logits: Tensor) -> EvalResult: +def evaluate_preds( + y_true: Tensor, + y_logits: Tensor, + ensembling: Literal["none", "partial", "full"] = "none", +) -> EvalResult: """ Evaluate the performance of a classification model. Args: y_true: Ground truth tensor of shape (N,). - y_pred: Predicted class tensor of shape (N, variants, n_classes). + y_logits: Predicted class tensor of shape (N, variants, n_classes). Returns: dict: A dictionary containing the accuracy, AUROC, and ECE. @@ -54,16 +59,29 @@ def evaluate_preds(y_true: Tensor, y_logits: Tensor) -> EvalResult: (n, v, c) = y_logits.shape assert y_true.shape == (n,) - # Clustered bootstrap confidence intervals for AUROC - y_true = repeat(y_true, "n -> n v", v=v) - auroc = roc_auc_ci(to_one_hot(y_true, c).long().flatten(1), y_logits.flatten(1)) - acc = accuracy_ci(y_true, y_logits.argmax(dim=-1)) - + if ensembling == "full": + y_logits = y_logits.mean(dim=1) + else: + y_true = repeat(y_true, "n -> n v", v=v) + + y_pred = y_logits.argmax(dim=-1) + if ensembling == "none": + auroc = roc_auc_ci(to_one_hot(y_true, c).long().flatten(1), y_logits.flatten(1)) + elif ensembling in ("partial", "full"): + # Pool together the negative and positive class logits + if c == 2: + auroc = roc_auc_ci(y_true, y_logits[..., 1] - y_logits[..., 0]) + else: + auroc = roc_auc_ci(to_one_hot(y_true, c).long(), y_logits) + else: + raise ValueError(f"Unknown mode: {ensembling}") + + acc = accuracy_ci(y_true, y_pred) cal_acc = None cal_err = None if c == 2: - pos_probs = y_logits.softmax(-1)[..., 1] + pos_probs = torch.sigmoid(y_logits[..., 1] - y_logits[..., 0]) # Calibrated accuracy cal_thresh = pos_probs.float().quantile(y_true.float().mean()) diff --git a/elk/run.py b/elk/run.py index 838c228f..d7fa549e 100644 --- a/elk/run.py +++ b/elk/run.py @@ -173,7 +173,7 @@ def apply_to_layers( finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): - df = pd.concat(dfs).sort_values(by="layer") + df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: save_debug_log(self.datasets, self.out_dir) diff --git a/elk/training/train.py b/elk/training/train.py index ddecc06f..ad5a799a 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -124,24 +124,35 @@ def apply_to_layer( for ds_name, (val_h, val_gt, val_lm_preds) in val_dict.items(): meta = {"dataset": ds_name, "layer": layer} - val_result = evaluate_preds(val_gt, reporter(val_h)) - row_bufs["eval"].append( - { - **meta, - "pseudo_auroc": pseudo_auroc, - "train_loss": train_loss, - **val_result.to_dict(), - } - ) - - if val_lm_preds is not None: - lm_result = evaluate_preds(val_gt, val_lm_preds) - row_bufs["lm_eval"].append({**meta, **lm_result.to_dict()}) - - for i, model in enumerate(lr_models): - lr_result = evaluate_preds(val_gt, model(val_h)) - row_bufs["lr_eval"].append( - {"inlp_iter": i, **meta, **lr_result.to_dict()} + val_credences = reporter(val_h) + for mode in ("none", "partial", "full"): + row_bufs["eval"].append( + { + **meta, + "ensembling": mode, + **evaluate_preds(val_gt, val_credences, mode).to_dict(), + "pseudo_auroc": pseudo_auroc, + "train_loss": train_loss, + } ) + if val_lm_preds is not None: + row_bufs["lm_eval"].append( + { + **meta, + "ensembling": mode, + **evaluate_preds(val_gt, val_lm_preds, mode).to_dict(), + } + ) + + for i, model in enumerate(lr_models): + row_bufs["lr_eval"].append( + { + **meta, + "ensembling": mode, + "inlp_iter": i, + **evaluate_preds(val_gt, model(val_h), mode).to_dict(), + } + ) + return {k: pd.DataFrame(v) for k, v in row_bufs.items()} From dff69bf7184c4ea2d53043ed0009c7ebaf658f52 Mon Sep 17 00:00:00 2001 From: Nora Belrose Date: Wed, 26 Apr 2023 10:42:04 +0000 Subject: [PATCH 02/57] Fixing pyright version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6575e57a..1a787fba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dev = [ "hypothesis", "pre-commit", "pytest", - "pyright", + "pyright==1.1.304", "scikit-learn", ] From a493b857469cd55564d642b6cbc91dc9580c700b Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sun, 30 Apr 2023 00:40:22 +0200 Subject: [PATCH 03/57] experiment with layer ensembling --- elk/evaluation/evaluate.py | 7 ++++-- elk/example.py | 49 ++++++++++++++++++++++++++++++++++++++ elk/run.py | 13 +++++++--- 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 elk/example.py diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 2b9e8509..653e8def 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -30,7 +30,7 @@ def __post_init__(self): def apply_to_layer( self, layer: int, devices: list[str], world_size: int - ) -> dict[str, pd.DataFrame]: + ) -> tuple[dict[str, pd.DataFrame], list[dict]]: """Evaluate a single reporter on a single layer.""" device = self.get_device(devices, world_size) val_output = self.prepare_data(device, layer, "val") @@ -42,10 +42,13 @@ def apply_to_layer( reporter.eval() row_bufs = defaultdict(list) + + vals = [] for ds_name, (val_h, val_gt, _) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) + vals.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) for mode in ("none", "partial", "full"): row_bufs["eval"].append( { @@ -73,4 +76,4 @@ def apply_to_layer( } ) - return {k: pd.DataFrame(v) for k, v in row_bufs.items()} + return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, vals) diff --git a/elk/example.py b/elk/example.py new file mode 100644 index 00000000..aff5ecde --- /dev/null +++ b/elk/example.py @@ -0,0 +1,49 @@ +import pickle +from pathlib import Path +import numpy as np + +from torch import Tensor +import torch +from elk.metrics.eval import to_one_hot + +from elk.metrics.roc_auc import roc_auc_ci + +root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') + +# load pickle file +with open(root / 'vals.pkl', 'rb') as f: + vals_buffers = pickle.load(f) + +y_logits_means = [] +y_trues_list = [] +for vals in vals_buffers: + y_logits = vals[0]["val_credences"] + y_trues = vals[0]["val_gt"] + (n, v, c) = y_logits.shape + assert y_trues.shape == (n,) + + y_logits = y_logits.mean(dim=1) + + y_logits_means.append(y_logits) + y_trues_list.append(y_trues) + + if c == 2: + auroc = roc_auc_ci(y_trues, y_logits[..., 1] - y_logits[..., 0]) + else: + auroc = roc_auc_ci(to_one_hot(y_trues, c).long(), y_logits) + + print("layer", vals[0]["layer"], "auroc", auroc) + + +y_trues = y_trues_list[22:-1] +y_logits = y_logits_means[22:-1] + +layer_mean = torch.mean(torch.stack(y_logits), dim=2) + +breakpoint() + +i = 0 +for y_logits, y_true in zip(y_logits_means, y_trues): + auroc = roc_auc_ci(y_true, layer_mean[..., 1] - layer_mean[..., 0]) + print("auroc", auroc) + i = i + 1 diff --git a/elk/run.py b/elk/run.py index d7fa549e..b9cff284 100644 --- a/elk/run.py +++ b/elk/run.py @@ -1,4 +1,5 @@ import os +import pickle import random from abc import ABC, abstractmethod from collections import defaultdict @@ -165,10 +166,11 @@ def apply_to_layers( with ctx.Pool(num_devices) as pool: mapper = pool.imap_unordered if num_devices > 1 else map df_buffers = defaultdict(list) - + vals_buffers = [] try: - for df_dict in tqdm(mapper(func, layers), total=len(layers)): - for k, v in df_dict.items(): + for df_dict, vals in tqdm(mapper(func, layers), total=len(layers)): + vals_buffers.append(vals) + for k, v in df_dict.items(): # type: ignore df_buffers[k].append(v) finally: # Make sure the CSVs are written even if we crash or get interrupted @@ -177,3 +179,8 @@ def apply_to_layers( df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: save_debug_log(self.datasets, self.out_dir) + # save vals_buffers as pickle + with open(self.out_dir / f"vals.pkl", "wb") as f: + pickle.dump(vals_buffers, f) + + print("Saved vals to ", self.out_dir / f"vals.pkl") \ No newline at end of file From af5def6cc53427734aaaf44dd3a9b04d225b1818 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sun, 30 Apr 2023 17:40:21 +0200 Subject: [PATCH 04/57] add draft example for ensembling datasets --- elk/example.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/elk/example.py b/elk/example.py index aff5ecde..7feb4b0a 100644 --- a/elk/example.py +++ b/elk/example.py @@ -8,15 +8,28 @@ from elk.metrics.roc_auc import roc_auc_ci +# imdb root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') +# boolq +# root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') +# elk eval "microsoft/ +# deberta-large-mnli/i +# mdb/quizzical-allen" +# microsoft/deberta-l +# arge-mnli imdb --num +# _gpus 1 + # load pickle file with open(root / 'vals.pkl', 'rb') as f: vals_buffers = pickle.load(f) y_logits_means = [] y_trues_list = [] +k_prompts_aurocs = [] for vals in vals_buffers: + print("vals.shape", len(vals)) + y_logits = vals[0]["val_credences"] y_trues = vals[0]["val_gt"] (n, v, c) = y_logits.shape @@ -32,18 +45,25 @@ else: auroc = roc_auc_ci(to_one_hot(y_trues, c).long(), y_logits) + k_prompts_aurocs.append(auroc) + print("layer", vals[0]["layer"], "auroc", auroc) +def get_best_aurocs_indices(aurocs, max=5): + sorted_indices = sorted(range(len(aurocs)), key=lambda i: aurocs[i].estimate) + # the best aurocs are at the end of the list + return sorted_indices[-max:] + +best_aurocs_indices = get_best_aurocs_indices(k_prompts_aurocs) +print("best_aurocs_indices", best_aurocs_indices) + +y_trues = [y_trues_list[i] for i in best_aurocs_indices] +y_logits = [y_logits_means[i] for i in best_aurocs_indices] -y_trues = y_trues_list[22:-1] -y_logits = y_logits_means[22:-1] +y_logits_layers = torch.stack(y_logits) +y_layer_logits_means = torch.mean(y_logits_layers, dim=0) -layer_mean = torch.mean(torch.stack(y_logits), dim=2) +auroc = roc_auc_ci(y_trues[2], y_layer_logits_means[..., 1] - y_layer_logits_means[..., 0]) +print(auroc) -breakpoint() -i = 0 -for y_logits, y_true in zip(y_logits_means, y_trues): - auroc = roc_auc_ci(y_true, layer_mean[..., 1] - layer_mean[..., 0]) - print("auroc", auroc) - i = i + 1 From 04a2a82c2ad570fc9e4d0907449971636e84a0d5 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sun, 30 Apr 2023 16:09:25 +0000 Subject: [PATCH 05/57] add comment --- elk/example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elk/example.py b/elk/example.py index 7feb4b0a..c784cefd 100644 --- a/elk/example.py +++ b/elk/example.py @@ -49,6 +49,7 @@ print("layer", vals[0]["layer"], "auroc", auroc) +# only use to find pattern in data def get_best_aurocs_indices(aurocs, max=5): sorted_indices = sorted(range(len(aurocs)), key=lambda i: aurocs[i].estimate) # the best aurocs are at the end of the list From cda7de7765c5a6e849e78447afa89557e9d48a27 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sun, 30 Apr 2023 18:59:03 +0200 Subject: [PATCH 06/57] add eval in comments --- elk/example.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/elk/example.py b/elk/example.py index 7feb4b0a..6666820a 100644 --- a/elk/example.py +++ b/elk/example.py @@ -13,12 +13,7 @@ # boolq # root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') -# elk eval "microsoft/ -# deberta-large-mnli/i -# mdb/quizzical-allen" -# microsoft/deberta-l -# arge-mnli imdb --num -# _gpus 1 +# elk eval "microsoft/deberta-large-mnli/imdb/quizzical-allen" microsoft/deberta-large-mnli imdb --num_gpus 1 # load pickle file with open(root / 'vals.pkl', 'rb') as f: From 0ceaa3a4abb617e00699b087c040183f235662ce Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Mon, 1 May 2023 14:44:24 +0200 Subject: [PATCH 07/57] add different root --- elk/example.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/elk/example.py b/elk/example.py index c784cefd..454ca3a3 100644 --- a/elk/example.py +++ b/elk/example.py @@ -9,7 +9,9 @@ from elk.metrics.roc_auc import roc_auc_ci # imdb -root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') +# root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') +# root = Path('/home/wombat_share/laurito/elk_reporters/microsoft/deberta-v2-xxlarge-mnli/super_glue:boolq/nervous-mclean/transfer_eval') +root = Path('/home/wombat_share/laurito/elk_reporters/huggyllama/llama-13b/imdb/silly-hoover/transfer_eval') # boolq # root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') @@ -50,7 +52,7 @@ print("layer", vals[0]["layer"], "auroc", auroc) # only use to find pattern in data -def get_best_aurocs_indices(aurocs, max=5): +def get_best_aurocs_indices(aurocs, max=3): sorted_indices = sorted(range(len(aurocs)), key=lambda i: aurocs[i].estimate) # the best aurocs are at the end of the list return sorted_indices[-max:] @@ -68,3 +70,4 @@ def get_best_aurocs_indices(aurocs, max=5): print(auroc) +# elk eval "microsoft/deberta-large-mnli/super_glue:boolq/nervous-mclean" microsoft/deberta-large-mnli "super_glue:boolq" --num_gpus 1 \ No newline at end of file From 0bd274f2a3a13d97f6a2e2a29a4da77863b1279c Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sat, 27 May 2023 23:57:56 +0200 Subject: [PATCH 08/57] add empty list of vals --- elk/training/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elk/training/train.py b/elk/training/train.py index e0aee29b..e4aee466 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -53,7 +53,7 @@ def apply_to_layer( layer: int, devices: list[str], world_size: int, - ) -> dict[str, pd.DataFrame]: + ) -> tuple[dict[str, pd.DataFrame], list[dict]]: """Train a single reporter on a single layer.""" self.make_reproducible(seed=self.net.seed + layer) @@ -192,4 +192,4 @@ def apply_to_layer( } ) - return {k: pd.DataFrame(v) for k, v in row_bufs.items()} + return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, []) From 994af9bed201fe6a7803d6a80f255a7ec377db18 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sat, 17 Jun 2023 18:55:38 +0200 Subject: [PATCH 09/57] add first version of layer ensembling to eval --- elk/example.py | 68 --------------------------------------------- elk/metrics/eval.py | 43 ++++++++++++++++++++++++++++ elk/run.py | 14 +++++++--- 3 files changed, 53 insertions(+), 72 deletions(-) delete mode 100644 elk/example.py diff --git a/elk/example.py b/elk/example.py deleted file mode 100644 index 68e15e0a..00000000 --- a/elk/example.py +++ /dev/null @@ -1,68 +0,0 @@ -import pickle -from pathlib import Path -import numpy as np - -from torch import Tensor -import torch -from elk.metrics.eval import to_one_hot - -from elk.metrics.roc_auc import roc_auc_ci - -# imdb -# root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') -# root = Path('/home/wombat_share/laurito/elk_reporters/microsoft/deberta-v2-xxlarge-mnli/super_glue:boolq/nervous-mclean/transfer_eval') -root = Path('/home/wombat_share/laurito/elk_reporters/huggyllama/llama-13b/imdb/silly-hoover/transfer_eval') - -# boolq -# root = Path('/home/laurito/elk-reporters/microsoft/deberta-large-mnli/imdb/quizzical-allen/transfer_eval') -# elk eval "microsoft/deberta-large-mnli/imdb/quizzical-allen" microsoft/deberta-large-mnli imdb --num_gpus 1 - -# load pickle file -with open(root / 'vals.pkl', 'rb') as f: - vals_buffers = pickle.load(f) - -y_logits_means = [] -y_trues_list = [] -k_prompts_aurocs = [] -for vals in vals_buffers: - print("vals.shape", len(vals)) - - y_logits = vals[0]["val_credences"] - y_trues = vals[0]["val_gt"] - (n, v, c) = y_logits.shape - assert y_trues.shape == (n,) - - y_logits = y_logits.mean(dim=1) - - y_logits_means.append(y_logits) - y_trues_list.append(y_trues) - - if c == 2: - auroc = roc_auc_ci(y_trues, y_logits[..., 1] - y_logits[..., 0]) - else: - auroc = roc_auc_ci(to_one_hot(y_trues, c).long(), y_logits) - - k_prompts_aurocs.append(auroc) - - print("layer", vals[0]["layer"], "auroc", auroc) - -# only use to find pattern in data -def get_best_aurocs_indices(aurocs, max=3): - sorted_indices = sorted(range(len(aurocs)), key=lambda i: aurocs[i].estimate) - # the best aurocs are at the end of the list - return sorted_indices[-max:] - -best_aurocs_indices = get_best_aurocs_indices(k_prompts_aurocs) -print("best_aurocs_indices", best_aurocs_indices) - -y_trues = [y_trues_list[i] for i in best_aurocs_indices] -y_logits = [y_logits_means[i] for i in best_aurocs_indices] - -y_logits_layers = torch.stack(y_logits) -y_layer_logits_means = torch.mean(y_logits_layers, dim=0) - -auroc = roc_auc_ci(y_trues[2], y_layer_logits_means[..., 1] - y_layer_logits_means[..., 0]) -print(auroc) - - -# elk eval "microsoft/deberta-large-mnli/super_glue:boolq/nervous-mclean" microsoft/deberta-large-mnli "super_glue:boolq" --num_gpus 1 \ No newline at end of file diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 653beae5..432369f0 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -93,6 +93,49 @@ def evaluate_preds( return EvalResult(acc, cal_acc, cal_err, auroc) +def layer_ensembling(vals_buffers) -> EvalResult: + y_logits_means = [] + y_trues_list = [] + k_prompts_aurocs = [] + for vals in vals_buffers: + print("vals.shape", len(vals)) + + y_logits = vals[0]["val_credences"] + y_trues = vals[0]["val_gt"] + (n, _, c) = y_logits.shape + assert y_trues.shape == (n,) + + y_logits = y_logits.mean(dim=1) + + y_logits_means.append(y_logits) + y_trues_list.append(y_trues) + + if c == 2: + auroc = roc_auc_ci(y_trues, y_logits[..., 1] - y_logits[..., 0]) + else: + auroc = roc_auc_ci(to_one_hot(y_trues, c).long(), y_logits) + + k_prompts_aurocs.append(auroc) + + print("layer", vals[0]["layer"], "auroc", auroc) + + # only use to find pattern in data + def get_best_aurocs_indices(aurocs, max=3): + sorted_indices = sorted(range(len(aurocs)), key=lambda i: aurocs[i].estimate) + # the best aurocs are at the end of the list + return sorted_indices[-max:] + + best_aurocs_indices = get_best_aurocs_indices(k_prompts_aurocs) + print("best_aurocs_indices", best_aurocs_indices) + + y_trues = [y_trues_list[i] for i in best_aurocs_indices] + y_logits = [y_logits_means[i] for i in best_aurocs_indices] + + y_logits_layers = torch.stack(y_logits) + y_layer_logits_means = torch.mean(y_logits_layers, dim=0) + + auroc = roc_auc_ci(y_trues[2], y_layer_logits_means[..., 1] - y_layer_logits_means[..., 0]) + return EvalResult(None, None, None, auroc) def to_one_hot(labels: Tensor, n_classes: int) -> Tensor: """ diff --git a/elk/run.py b/elk/run.py index c0ba52a9..e10ea265 100644 --- a/elk/run.py +++ b/elk/run.py @@ -18,6 +18,8 @@ from torch import Tensor from tqdm import tqdm +from elk.metrics.eval import layer_ensembling + from .debug_logging import save_debug_log from .extraction import Extract, extract from .extraction.dataset_name import DatasetDictWithName @@ -183,6 +185,7 @@ def apply_to_layers( try: for df_dict, vals in tqdm(mapper(func, layers), total=len(layers)): vals_buffers.append(vals) + print("vals", vals) for k, v in df_dict.items(): # type: ignore df_buffers[k].append(v) finally: @@ -192,8 +195,11 @@ def apply_to_layers( df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: save_debug_log(self.datasets, self.out_dir) - # save vals_buffers as pickle - with open(self.out_dir / f"vals.pkl", "wb") as f: - pickle.dump(vals_buffers, f) + breakpoint() + print("hi") + # layer_ensembling_results = layer_ensembling(vals_buffers) + - print("Saved vals to ", self.out_dir / f"vals.pkl") \ No newline at end of file + # for name, dfs in df_buffers.items(): + # df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) + # df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) \ No newline at end of file From 6ca191624debf08e0f18715d0c72e500b2a24751 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Mon, 19 Jun 2023 13:03:51 +0200 Subject: [PATCH 10/57] add vals to train --- elk/training/train.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elk/training/train.py b/elk/training/train.py index e4aee466..d863ba74 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -137,12 +137,14 @@ def apply_to_layer( lr_models = [] row_bufs = defaultdict(list) + vals = [] for ds_name in val_dict: val_h, val_gt, val_lm_preds = val_dict[ds_name] train_h, train_gt, train_lm_preds = train_dict[ds_name] meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) + vals.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) train_credences = reporter(train_h) for mode in ("none", "partial", "full"): row_bufs["eval"].append( @@ -192,4 +194,4 @@ def apply_to_layer( } ) - return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, []) + return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, vals) From b0d0f83a53e17abff2f0e598e0dd8e99c1bc4216 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Mon, 19 Jun 2023 15:16:38 +0200 Subject: [PATCH 11/57] refactoring & cleanup of eval and layer ensembling --- elk/metrics/eval.py | 143 +++++++++++++++++++++++--------------------- elk/run.py | 24 ++++---- 2 files changed, 84 insertions(+), 83 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 432369f0..c1503892 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -3,6 +3,7 @@ import torch from einops import repeat +import pandas as pd from torch import Tensor from .accuracy import AccuracyResult, accuracy_ci @@ -41,6 +42,37 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: return {**auroc_dict, **cal_acc_dict, **acc_dict, **cal_dict} +def calc_auroc(y_logits, y_true, ensembling, num_classes): + if ensembling == "none": + auroc = roc_auc_ci(to_one_hot(y_true, num_classes).long().flatten(1), y_logits.flatten(1)) + elif ensembling in ("partial", "full"): + # Pool together the negative and positive class logits + if num_classes == 2: + auroc = roc_auc_ci(y_true, y_logits[..., 1] - y_logits[..., 0]) + else: + auroc = roc_auc_ci(to_one_hot(y_true, num_classes).long(), y_logits) + else: + raise ValueError(f"Unknown mode: {ensembling}") + + return auroc + + +def calc_calibrated_accuracies(y_true, pos_probs): + cal_thresh = pos_probs.float().quantile(y_true.float().mean()) + cal_preds = pos_probs.gt(cal_thresh).to(torch.int) + cal_acc = accuracy_ci(y_true, cal_preds) + return cal_acc + +def calc_calibrated_errors(y_true, pos_probs): + cal = CalibrationError().update(y_true.flatten(), pos_probs.flatten()) + cal_err = cal.compute() + return cal_err + +def calc_accuracies(y_logits, y_true): + y_pred = y_logits.argmax(dim=-1) + return accuracy_ci(y_true, y_pred) + + def evaluate_preds( y_true: Tensor, y_logits: Tensor, @@ -50,92 +82,65 @@ def evaluate_preds( Evaluate the performance of a classification model. Args: - y_true: Ground truth tensor of shape (N,). - y_logits: Predicted class tensor of shape (N, variants, n_classes). + y_true: Ground truth tensor of shape (n,). + y_logits: Predicted class tensor of shape (n, num_variants, num_classes). Returns: dict: A dictionary containing the accuracy, AUROC, and ECE. """ - (n, v, c) = y_logits.shape + (n, num_variants, num_classes) = y_logits.shape assert y_true.shape == (n,) - + if ensembling == "full": y_logits = y_logits.mean(dim=1) else: - y_true = repeat(y_true, "n -> n v", v=v) - - y_pred = y_logits.argmax(dim=-1) - if ensembling == "none": - auroc = roc_auc_ci(to_one_hot(y_true, c).long().flatten(1), y_logits.flatten(1)) - elif ensembling in ("partial", "full"): - # Pool together the negative and positive class logits - if c == 2: - auroc = roc_auc_ci(y_true, y_logits[..., 1] - y_logits[..., 0]) - else: - auroc = roc_auc_ci(to_one_hot(y_true, c).long(), y_logits) - else: - raise ValueError(f"Unknown mode: {ensembling}") - - acc = accuracy_ci(y_true, y_pred) - cal_acc = None - cal_err = None - - if c == 2: - pos_probs = torch.sigmoid(y_logits[..., 1] - y_logits[..., 0]) - - # Calibrated accuracy - cal_thresh = pos_probs.float().quantile(y_true.float().mean()) - cal_preds = pos_probs.gt(cal_thresh).to(torch.int) - cal_acc = accuracy_ci(y_true, cal_preds) - - cal = CalibrationError().update(y_true.flatten(), pos_probs.flatten()) - cal_err = cal.compute() + y_true = repeat(y_true, "n -> n v", v=num_variants) + + return calc_eval_results(y_true, y_logits, ensembling, num_classes) + +def calc_eval_results(y_true, y_logits, ensembling, num_classes): + acc = calc_accuracies(y_logits=y_logits, y_true=y_true) + + pos_probs = torch.sigmoid(y_logits[..., 1] - y_logits[..., 0]) + cal_acc = calc_calibrated_accuracies(y_true=y_true, pos_probs=pos_probs) if num_classes == 2 else None + cal_err = calc_calibrated_errors(y_true=y_true, pos_probs=pos_probs) if num_classes == 2 else None + + auroc = calc_auroc(y_logits=y_logits, + y_true=y_true, + ensembling=ensembling, + num_classes=num_classes) return EvalResult(acc, cal_acc, cal_err, auroc) -def layer_ensembling(vals_buffers) -> EvalResult: +def layer_ensembling(layer_outputs) -> EvalResult: y_logits_means = [] - y_trues_list = [] - k_prompts_aurocs = [] - for vals in vals_buffers: - print("vals.shape", len(vals)) - - y_logits = vals[0]["val_credences"] - y_trues = vals[0]["val_gt"] - (n, _, c) = y_logits.shape - assert y_trues.shape == (n,) - - y_logits = y_logits.mean(dim=1) + y_trues = [] + + for layer_output in layer_outputs: + y_logits = layer_output[0]["val_credences"] - y_logits_means.append(y_logits) - y_trues_list.append(y_trues) - - if c == 2: - auroc = roc_auc_ci(y_trues, y_logits[..., 1] - y_logits[..., 0]) - else: - auroc = roc_auc_ci(to_one_hot(y_trues, c).long(), y_logits) - - k_prompts_aurocs.append(auroc) - - print("layer", vals[0]["layer"], "auroc", auroc) - - # only use to find pattern in data - def get_best_aurocs_indices(aurocs, max=3): - sorted_indices = sorted(range(len(aurocs)), key=lambda i: aurocs[i].estimate) - # the best aurocs are at the end of the list - return sorted_indices[-max:] - - best_aurocs_indices = get_best_aurocs_indices(k_prompts_aurocs) - print("best_aurocs_indices", best_aurocs_indices) - - y_trues = [y_trues_list[i] for i in best_aurocs_indices] - y_logits = [y_logits_means[i] for i in best_aurocs_indices] + # full ensembling + y_logits_means.append(y_logits.mean(dim=1)) + + y_true = layer_output[0]["val_gt"] + y_trues.append(y_true) + + num_classes = layer_outputs[0][0]["val_credences"].shape[2] + + # get logits and ground_truth from middle to last layer + middle_index = len(y_trues) // 2 + y_trues = y_trues[middle_index:] + y_logits = y_logits_means[middle_index:] y_logits_layers = torch.stack(y_logits) + + # layer ensembling of the stacked logits y_layer_logits_means = torch.mean(y_logits_layers, dim=0) - auroc = roc_auc_ci(y_trues[2], y_layer_logits_means[..., 1] - y_layer_logits_means[..., 0]) - return EvalResult(None, None, None, auroc) + return calc_eval_results(y_true=y_trues[2], + y_logits=y_layer_logits_means, + ensembling="full", + num_classes=num_classes) def to_one_hot(labels: Tensor, n_classes: int) -> Tensor: """ diff --git a/elk/run.py b/elk/run.py index e10ea265..32a2943a 100644 --- a/elk/run.py +++ b/elk/run.py @@ -3,7 +3,7 @@ import random from abc import ABC, abstractmethod from collections import defaultdict -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import partial from pathlib import Path from typing import Callable, Literal @@ -18,7 +18,7 @@ from torch import Tensor from tqdm import tqdm -from elk.metrics.eval import layer_ensembling +from elk.metrics.eval import EvalResult, layer_ensembling from .debug_logging import save_debug_log from .extraction import Extract, extract @@ -181,25 +181,21 @@ def apply_to_layers( with ctx.Pool(num_devices) as pool: mapper = pool.imap_unordered if num_devices > 1 else map df_buffers = defaultdict(list) - vals_buffers = [] + layer_outputs = [] try: - for df_dict, vals in tqdm(mapper(func, layers), total=len(layers)): - vals_buffers.append(vals) - print("vals", vals) + for df_dict, layer_output in tqdm(mapper(func, layers), total=len(layers)): + layer_outputs.append(layer_output) for k, v in df_dict.items(): # type: ignore df_buffers[k].append(v) finally: + # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: save_debug_log(self.datasets, self.out_dir) - breakpoint() - print("hi") - # layer_ensembling_results = layer_ensembling(vals_buffers) - - - # for name, dfs in df_buffers.items(): - # df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) - # df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) \ No newline at end of file + + layer_ensembling_results: EvalResult = layer_ensembling(layer_outputs) + df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) + df.round(4).to_csv(self.out_dir / f"layer_ensembling_results.csv", index=False) \ No newline at end of file From 241a03a0fdfea11a395dfb3efd1ec038da7096b4 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Mon, 19 Jun 2023 21:42:58 +0200 Subject: [PATCH 12/57] add annotations --- elk/metrics/eval.py | 60 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index c1503892..f36e03b6 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -57,18 +57,50 @@ def calc_auroc(y_logits, y_true, ensembling, num_classes): return auroc -def calc_calibrated_accuracies(y_true, pos_probs): +def calc_calibrated_accuracies(y_true, pos_probs) -> AccuracyResult: + """ + Calculate the calibrated accuracies + + Args: + y_true: Ground truth tensor of shape (n,). + pos_probs: Predicted class tensor of shape (n, num_variants, num_classes). + + Returns: + AccuracyResult: A dictionary containing the accuracy and confidence interval. + """ + cal_thresh = pos_probs.float().quantile(y_true.float().mean()) cal_preds = pos_probs.gt(cal_thresh).to(torch.int) cal_acc = accuracy_ci(y_true, cal_preds) return cal_acc -def calc_calibrated_errors(y_true, pos_probs): +def calc_calibrated_errors(y_true, pos_probs) -> CalibrationEstimate: + """ + Calculate the expected calibration error. + + Args: + y_true: Ground truth tensor of shape (n,). + y_logits: Predicted class tensor of shape (n, num_variants, num_classes). + + Returns: + CalibrationEstimate: + """ + cal = CalibrationError().update(y_true.flatten(), pos_probs.flatten()) cal_err = cal.compute() return cal_err -def calc_accuracies(y_logits, y_true): +def calc_accuracies(y_logits, y_true) -> AccuracyResult: + """ + Calculate the accuracy + + Args: + y_true: Ground truth tensor of shape (n,). + y_logits: Predicted class tensor of shape (n, num_variants, num_classes). + + Returns: + AccuracyResult: A dictionary containing the accuracy and confidence interval. + """ y_pred = y_logits.argmax(dim=-1) return accuracy_ci(y_true, y_pred) @@ -98,7 +130,18 @@ def evaluate_preds( return calc_eval_results(y_true, y_logits, ensembling, num_classes) -def calc_eval_results(y_true, y_logits, ensembling, num_classes): +def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: + """ + Calculate the evaluation results + + Args: + y_true: Ground truth tensor of shape (n,). + y_logits: Predicted class tensor of shape (n, num_variants, num_classes). + ensembling: The ensembling mode. + + Returns: + EvalResult: The result of evaluating a classifier containing the accuracy, calibrated accuracies, calibrated errors, and AUROC. + """ acc = calc_accuracies(y_logits=y_logits, y_true=y_true) pos_probs = torch.sigmoid(y_logits[..., 1] - y_logits[..., 0]) @@ -113,6 +156,15 @@ def calc_eval_results(y_true, y_logits, ensembling, num_classes): return EvalResult(acc, cal_acc, cal_err, auroc) def layer_ensembling(layer_outputs) -> EvalResult: + """ + Return EvalResult after ensembling the probe output of the middle to last layers + + Args: + layer_outputs: A list of dictionaries containing the ground truth and predicted class tensor of shape (n, num_variants, num_classes). + + Returns: + EvalResult: The result of evaluating a classifier containing the accuracy, calibrated accuracies, calibrated errors, and AUROC. + """ y_logits_means = [] y_trues = [] From e8d042a912e3aa6c7555cab96358142407d766cf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 19:46:10 +0000 Subject: [PATCH 13/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/evaluation/evaluate.py | 2 +- elk/metrics/eval.py | 77 +++++++++++++++++++++++--------------- elk/run.py | 14 ++++--- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index f156d09e..e315f307 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -44,7 +44,7 @@ def apply_to_layer( reporter.eval() row_bufs = defaultdict(list) - + vals = [] for ds_name, (val_h, val_gt, _) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index f36e03b6..b28feeff 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -3,7 +3,6 @@ import torch from einops import repeat -import pandas as pd from torch import Tensor from .accuracy import AccuracyResult, accuracy_ci @@ -44,7 +43,9 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: def calc_auroc(y_logits, y_true, ensembling, num_classes): if ensembling == "none": - auroc = roc_auc_ci(to_one_hot(y_true, num_classes).long().flatten(1), y_logits.flatten(1)) + auroc = roc_auc_ci( + to_one_hot(y_true, num_classes).long().flatten(1), y_logits.flatten(1) + ) elif ensembling in ("partial", "full"): # Pool together the negative and positive class logits if num_classes == 2: @@ -53,43 +54,45 @@ def calc_auroc(y_logits, y_true, ensembling, num_classes): auroc = roc_auc_ci(to_one_hot(y_true, num_classes).long(), y_logits) else: raise ValueError(f"Unknown mode: {ensembling}") - + return auroc def calc_calibrated_accuracies(y_true, pos_probs) -> AccuracyResult: """ Calculate the calibrated accuracies - + Args: y_true: Ground truth tensor of shape (n,). pos_probs: Predicted class tensor of shape (n, num_variants, num_classes). - + Returns: AccuracyResult: A dictionary containing the accuracy and confidence interval. """ - + cal_thresh = pos_probs.float().quantile(y_true.float().mean()) cal_preds = pos_probs.gt(cal_thresh).to(torch.int) cal_acc = accuracy_ci(y_true, cal_preds) return cal_acc + def calc_calibrated_errors(y_true, pos_probs) -> CalibrationEstimate: """ Calculate the expected calibration error. - + Args: y_true: Ground truth tensor of shape (n,). y_logits: Predicted class tensor of shape (n, num_variants, num_classes). - + Returns: - CalibrationEstimate: + CalibrationEstimate: """ - + cal = CalibrationError().update(y_true.flatten(), pos_probs.flatten()) cal_err = cal.compute() return cal_err + def calc_accuracies(y_logits, y_true) -> AccuracyResult: """ Calculate the accuracy @@ -122,7 +125,7 @@ def evaluate_preds( """ (n, num_variants, num_classes) = y_logits.shape assert y_true.shape == (n,) - + if ensembling == "full": y_logits = y_logits.mean(dim=1) else: @@ -130,6 +133,7 @@ def evaluate_preds( return calc_eval_results(y_true, y_logits, ensembling, num_classes) + def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: """ Calculate the evaluation results @@ -138,47 +142,55 @@ def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: y_true: Ground truth tensor of shape (n,). y_logits: Predicted class tensor of shape (n, num_variants, num_classes). ensembling: The ensembling mode. - + Returns: EvalResult: The result of evaluating a classifier containing the accuracy, calibrated accuracies, calibrated errors, and AUROC. """ acc = calc_accuracies(y_logits=y_logits, y_true=y_true) - + pos_probs = torch.sigmoid(y_logits[..., 1] - y_logits[..., 0]) - cal_acc = calc_calibrated_accuracies(y_true=y_true, pos_probs=pos_probs) if num_classes == 2 else None - cal_err = calc_calibrated_errors(y_true=y_true, pos_probs=pos_probs) if num_classes == 2 else None - - auroc = calc_auroc(y_logits=y_logits, - y_true=y_true, - ensembling=ensembling, - num_classes=num_classes) + cal_acc = ( + calc_calibrated_accuracies(y_true=y_true, pos_probs=pos_probs) + if num_classes == 2 + else None + ) + cal_err = ( + calc_calibrated_errors(y_true=y_true, pos_probs=pos_probs) + if num_classes == 2 + else None + ) + + auroc = calc_auroc( + y_logits=y_logits, y_true=y_true, ensembling=ensembling, num_classes=num_classes + ) return EvalResult(acc, cal_acc, cal_err, auroc) + def layer_ensembling(layer_outputs) -> EvalResult: """ Return EvalResult after ensembling the probe output of the middle to last layers - + Args: layer_outputs: A list of dictionaries containing the ground truth and predicted class tensor of shape (n, num_variants, num_classes). - + Returns: EvalResult: The result of evaluating a classifier containing the accuracy, calibrated accuracies, calibrated errors, and AUROC. """ y_logits_means = [] y_trues = [] - + for layer_output in layer_outputs: y_logits = layer_output[0]["val_credences"] - + # full ensembling y_logits_means.append(y_logits.mean(dim=1)) - + y_true = layer_output[0]["val_gt"] y_trues.append(y_true) - + num_classes = layer_outputs[0][0]["val_credences"].shape[2] - + # get logits and ground_truth from middle to last layer middle_index = len(y_trues) // 2 y_trues = y_trues[middle_index:] @@ -189,10 +201,13 @@ def layer_ensembling(layer_outputs) -> EvalResult: # layer ensembling of the stacked logits y_layer_logits_means = torch.mean(y_logits_layers, dim=0) - return calc_eval_results(y_true=y_trues[2], - y_logits=y_layer_logits_means, - ensembling="full", - num_classes=num_classes) + return calc_eval_results( + y_true=y_trues[2], + y_logits=y_layer_logits_means, + ensembling="full", + num_classes=num_classes, + ) + def to_one_hot(labels: Tensor, n_classes: int) -> Tensor: """ diff --git a/elk/run.py b/elk/run.py index 32a2943a..969b808e 100644 --- a/elk/run.py +++ b/elk/run.py @@ -1,9 +1,8 @@ import os -import pickle import random from abc import ABC, abstractmethod from collections import defaultdict -from dataclasses import asdict, dataclass +from dataclasses import dataclass from functools import partial from pathlib import Path from typing import Callable, Literal @@ -183,12 +182,13 @@ def apply_to_layers( df_buffers = defaultdict(list) layer_outputs = [] try: - for df_dict, layer_output in tqdm(mapper(func, layers), total=len(layers)): + for df_dict, layer_output in tqdm( + mapper(func, layers), total=len(layers) + ): layer_outputs.append(layer_output) - for k, v in df_dict.items(): # type: ignore + for k, v in df_dict.items(): # type: ignore df_buffers[k].append(v) finally: - # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) @@ -198,4 +198,6 @@ def apply_to_layers( layer_ensembling_results: EvalResult = layer_ensembling(layer_outputs) df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) - df.round(4).to_csv(self.out_dir / f"layer_ensembling_results.csv", index=False) \ No newline at end of file + df.round(4).to_csv( + self.out_dir / "layer_ensembling_results.csv", index=False + ) From a4ace25821fa03f15c58f2b31159adf735975c79 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Mon, 19 Jun 2023 22:12:03 +0200 Subject: [PATCH 14/57] rename vals to layer_outputs --- elk/evaluation/evaluate.py | 6 +++--- elk/training/train.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index f156d09e..7cd8533d 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -45,12 +45,12 @@ def apply_to_layer( row_bufs = defaultdict(list) - vals = [] + layer_outputs = [] for ds_name, (val_h, val_gt, _) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - vals.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) + layer_outputs.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) for mode in ("none", "partial", "full"): row_bufs["eval"].append( { @@ -78,4 +78,4 @@ def apply_to_layer( } ) - return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, vals) + return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_outputs) diff --git a/elk/training/train.py b/elk/training/train.py index d863ba74..2275422e 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -137,14 +137,14 @@ def apply_to_layer( lr_models = [] row_bufs = defaultdict(list) - vals = [] + layer_outputs = [] for ds_name in val_dict: val_h, val_gt, val_lm_preds = val_dict[ds_name] train_h, train_gt, train_lm_preds = train_dict[ds_name] meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - vals.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) + layer_outputs.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) train_credences = reporter(train_h) for mode in ("none", "partial", "full"): row_bufs["eval"].append( @@ -194,4 +194,4 @@ def apply_to_layer( } ) - return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, vals) + return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_outputs) From 2156ad8328c1b283fe6459fe28e6433d0003e655 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 20:13:09 +0000 Subject: [PATCH 15/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/evaluation/evaluate.py | 6 ++++-- elk/training/train.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 7cd8533d..667eb588 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -44,13 +44,15 @@ def apply_to_layer( reporter.eval() row_bufs = defaultdict(list) - + layer_outputs = [] for ds_name, (val_h, val_gt, _) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - layer_outputs.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) + layer_outputs.append( + {**meta, "val_gt": val_gt, "val_credences": val_credences} + ) for mode in ("none", "partial", "full"): row_bufs["eval"].append( { diff --git a/elk/training/train.py b/elk/training/train.py index 2275422e..d0d1e11b 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -144,7 +144,9 @@ def apply_to_layer( meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - layer_outputs.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) + layer_outputs.append( + {**meta, "val_gt": val_gt, "val_credences": val_credences} + ) train_credences = reporter(train_h) for mode in ("none", "partial", "full"): row_bufs["eval"].append( From e391da657cc7fa4a3a869d01ad10c356de342054 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Mon, 19 Jun 2023 22:19:35 +0200 Subject: [PATCH 16/57] fir formatting --- elk/evaluation/evaluate.py | 7 +++++-- elk/metrics/eval.py | 9 ++++++--- elk/training/train.py | 4 +++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 7cd8533d..58f84fc0 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -44,13 +44,16 @@ def apply_to_layer( reporter.eval() row_bufs = defaultdict(list) - + layer_outputs = [] for ds_name, (val_h, val_gt, _) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - layer_outputs.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) + layer_outputs.append( + {**meta, "val_gt": val_gt, "val_credences": val_credences} + ) + for mode in ("none", "partial", "full"): row_bufs["eval"].append( { diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index b28feeff..1bee1c23 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -144,7 +144,8 @@ def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: ensembling: The ensembling mode. Returns: - EvalResult: The result of evaluating a classifier containing the accuracy, calibrated accuracies, calibrated errors, and AUROC. + EvalResult: The result of evaluating a classifier containing the accuracy, + calibrated accuracies, calibrated errors, and AUROC. """ acc = calc_accuracies(y_logits=y_logits, y_true=y_true) @@ -172,10 +173,12 @@ def layer_ensembling(layer_outputs) -> EvalResult: Return EvalResult after ensembling the probe output of the middle to last layers Args: - layer_outputs: A list of dictionaries containing the ground truth and predicted class tensor of shape (n, num_variants, num_classes). + layer_outputs: A list of dictionaries containing the ground truth and + predicted class tensor of shape (n, num_variants, num_classes). Returns: - EvalResult: The result of evaluating a classifier containing the accuracy, calibrated accuracies, calibrated errors, and AUROC. + EvalResult: The result of evaluating a classifier containing the accuracy, + calibrated accuracies, calibrated errors, and AUROC. """ y_logits_means = [] y_trues = [] diff --git a/elk/training/train.py b/elk/training/train.py index 2275422e..d0d1e11b 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -144,7 +144,9 @@ def apply_to_layer( meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - layer_outputs.append({**meta, "val_gt": val_gt, "val_credences": val_credences}) + layer_outputs.append( + {**meta, "val_gt": val_gt, "val_credences": val_credences} + ) train_credences = reporter(train_h) for mode in ("none", "partial", "full"): row_bufs["eval"].append( From 528367d7efb9b42f35443263d70b1c1949e16fb2 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Wed, 21 Jun 2023 12:30:31 +0000 Subject: [PATCH 17/57] make layer ensembling work on multiple gpus --- elk/metrics/eval.py | 4 ++-- elk/training/train.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 1bee1c23..80869dff 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -184,12 +184,12 @@ def layer_ensembling(layer_outputs) -> EvalResult: y_trues = [] for layer_output in layer_outputs: - y_logits = layer_output[0]["val_credences"] + y_logits = layer_output[0]["val_credences"].cpu() # full ensembling y_logits_means.append(y_logits.mean(dim=1)) - y_true = layer_output[0]["val_gt"] + y_true = layer_output[0]["val_gt"].cpu() y_trues.append(y_true) num_classes = layer_outputs[0][0]["val_credences"].shape[2] diff --git a/elk/training/train.py b/elk/training/train.py index d0d1e11b..b906fd54 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -144,9 +144,11 @@ def apply_to_layer( meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) + layer_outputs.append( - {**meta, "val_gt": val_gt, "val_credences": val_credences} + {**meta, "val_gt": val_gt.detach(), "val_credences": val_credences.detach()} ) + train_credences = reporter(train_h) for mode in ("none", "partial", "full"): row_bufs["eval"].append( From d4df5175c36277023e7a823fd9745a9b3aec7d15 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 12:30:51 +0000 Subject: [PATCH 18/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/training/train.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elk/training/train.py b/elk/training/train.py index b906fd54..c785d5a6 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -146,7 +146,11 @@ def apply_to_layer( val_credences = reporter(val_h) layer_outputs.append( - {**meta, "val_gt": val_gt.detach(), "val_credences": val_credences.detach()} + { + **meta, + "val_gt": val_gt.detach(), + "val_credences": val_credences.detach(), + } ) train_credences = reporter(train_h) From 2661ea13b15bfa5ea988a87a5e0736827617ec6f Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Wed, 21 Jun 2023 12:42:02 +0000 Subject: [PATCH 19/57] make sure we use the same device --- elk/metrics/eval.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 80869dff..2916ba0c 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -183,13 +183,14 @@ def layer_ensembling(layer_outputs) -> EvalResult: y_logits_means = [] y_trues = [] + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') for layer_output in layer_outputs: - y_logits = layer_output[0]["val_credences"].cpu() + y_logits = layer_output[0]["val_credences"].to(device) # full ensembling y_logits_means.append(y_logits.mean(dim=1)) - y_true = layer_output[0]["val_gt"].cpu() + y_true = layer_output[0]["val_gt"].to(device) y_trues.append(y_true) num_classes = layer_outputs[0][0]["val_credences"].shape[2] From 21cccb7c7f2e8238f96700bd5dc3895120b6e1ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 12:42:15 +0000 Subject: [PATCH 20/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/metrics/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 2916ba0c..d0827dec 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -183,7 +183,7 @@ def layer_ensembling(layer_outputs) -> EvalResult: y_logits_means = [] y_trues = [] - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") for layer_output in layer_outputs: y_logits = layer_output[0]["val_credences"].to(device) From 2495c3a9cd06064e0cb3c23173282404a79da6b1 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Wed, 21 Jun 2023 16:10:19 +0000 Subject: [PATCH 21/57] calc layer ensembling for all prompt ensembling modes --- elk/metrics/eval.py | 6 +++--- elk/run.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 2916ba0c..9e6ff91d 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -168,7 +168,7 @@ def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: return EvalResult(acc, cal_acc, cal_err, auroc) -def layer_ensembling(layer_outputs) -> EvalResult: +def layer_ensembling(layer_outputs: list, ensembling: str) -> EvalResult: """ Return EvalResult after ensembling the probe output of the middle to last layers @@ -183,7 +183,7 @@ def layer_ensembling(layer_outputs) -> EvalResult: y_logits_means = [] y_trues = [] - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") for layer_output in layer_outputs: y_logits = layer_output[0]["val_credences"].to(device) @@ -208,7 +208,7 @@ def layer_ensembling(layer_outputs) -> EvalResult: return calc_eval_results( y_true=y_trues[2], y_logits=y_layer_logits_means, - ensembling="full", + ensembling=ensembling, num_classes=num_classes, ) diff --git a/elk/run.py b/elk/run.py index 969b808e..93493c6f 100644 --- a/elk/run.py +++ b/elk/run.py @@ -17,7 +17,7 @@ from torch import Tensor from tqdm import tqdm -from elk.metrics.eval import EvalResult, layer_ensembling +from elk.metrics.eval import layer_ensembling from .debug_logging import save_debug_log from .extraction import Extract, extract @@ -196,8 +196,19 @@ def apply_to_layers( if self.debug: save_debug_log(self.datasets, self.out_dir) - layer_ensembling_results: EvalResult = layer_ensembling(layer_outputs) - df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) - df.round(4).to_csv( + dfs = [] + for ensemble in [ + "full", + "partial", + "none", + ]: # TODO: Replace ensemble strings with enums everywhere + layer_ensembling_results = layer_ensembling(layer_outputs, ensemble) + df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) + df = df.round(4) + df["ensemble"] = ensemble + dfs.append(df) + + df_conc = pd.concat(dfs) + df_conc.to_csv( self.out_dir / "layer_ensembling_results.csv", index=False ) From 908308b1da691ee2de8affc18940501ed4676112 Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 23 Jun 2023 15:10:14 +0100 Subject: [PATCH 22/57] implement ensembling enum --- elk/evaluation/evaluate.py | 13 ++++++++----- elk/metrics/eval.py | 14 ++++++++------ elk/plotting/visualize.py | 30 +++++++++++++++++------------- elk/run.py | 14 +++++++------- elk/training/train.py | 3 ++- elk/utils/types.py | 11 +++++++++++ 6 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 elk/utils/types.py diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 667eb588..da5ea0c1 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -11,6 +11,7 @@ from ..run import Run from ..training import Reporter from ..utils import Color +from ..utils.types import Ensembling @dataclass(kw_only=True) @@ -53,12 +54,12 @@ def apply_to_layer( layer_outputs.append( {**meta, "val_gt": val_gt, "val_credences": val_credences} ) - for mode in ("none", "partial", "full"): + for ensembling in Ensembling.all(): row_bufs["eval"].append( { **meta, - "ensembling": mode, - **evaluate_preds(val_gt, val_credences, mode).to_dict(), + "ensembling": ensembling.value, + **evaluate_preds(val_gt, val_credences, ensembling).to_dict(), } ) @@ -73,10 +74,12 @@ def apply_to_layer( model.eval() row_bufs["lr_eval"].append( { - "ensembling": mode, + "ensembling": ensembling.value, "inlp_iter": i, **meta, - **evaluate_preds(val_gt, model(val_h), mode).to_dict(), + **evaluate_preds( + val_gt, model(val_h), ensembling + ).to_dict(), } ) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 9e6ff91d..79093c5e 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -1,10 +1,10 @@ from dataclasses import asdict, dataclass -from typing import Literal import torch from einops import repeat from torch import Tensor +from ..utils.types import Ensembling from .accuracy import AccuracyResult, accuracy_ci from .calibration import CalibrationError, CalibrationEstimate from .roc_auc import RocAucResult, roc_auc_ci @@ -42,11 +42,11 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: def calc_auroc(y_logits, y_true, ensembling, num_classes): - if ensembling == "none": + if ensembling == Ensembling.NONE: auroc = roc_auc_ci( to_one_hot(y_true, num_classes).long().flatten(1), y_logits.flatten(1) ) - elif ensembling in ("partial", "full"): + elif ensembling in (Ensembling.PARTIAL, Ensembling.FULL): # Pool together the negative and positive class logits if num_classes == 2: auroc = roc_auc_ci(y_true, y_logits[..., 1] - y_logits[..., 0]) @@ -111,7 +111,7 @@ def calc_accuracies(y_logits, y_true) -> AccuracyResult: def evaluate_preds( y_true: Tensor, y_logits: Tensor, - ensembling: Literal["none", "partial", "full"] = "none", + ensembling: Ensembling = Ensembling.NONE, ) -> EvalResult: """ Evaluate the performance of a classification model. @@ -119,6 +119,7 @@ def evaluate_preds( Args: y_true: Ground truth tensor of shape (n,). y_logits: Predicted class tensor of shape (n, num_variants, num_classes). + ensembling: The ensembling mode. Returns: dict: A dictionary containing the accuracy, AUROC, and ECE. @@ -126,7 +127,7 @@ def evaluate_preds( (n, num_variants, num_classes) = y_logits.shape assert y_true.shape == (n,) - if ensembling == "full": + if ensembling == Ensembling.FULL: y_logits = y_logits.mean(dim=1) else: y_true = repeat(y_true, "n -> n v", v=num_variants) @@ -168,13 +169,14 @@ def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: return EvalResult(acc, cal_acc, cal_err, auroc) -def layer_ensembling(layer_outputs: list, ensembling: str) -> EvalResult: +def layer_ensembling(layer_outputs: list, ensembling: Ensembling) -> EvalResult: """ Return EvalResult after ensembling the probe output of the middle to last layers Args: layer_outputs: A list of dictionaries containing the ground truth and predicted class tensor of shape (n, num_variants, num_classes). + ensembling: The ensembling mode. Returns: EvalResult: The result of evaluating a classifier containing the accuracy, diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py index a82fd7d7..7771d9ce 100644 --- a/elk/plotting/visualize.py +++ b/elk/plotting/visualize.py @@ -9,6 +9,8 @@ from rich.console import Console from rich.table import Table +from elk.utils.types import Ensembling + @dataclass class SweepByDsMultiplot: @@ -19,16 +21,16 @@ class SweepByDsMultiplot: def render( self, sweep: "SweepVisualization", - with_transfer=False, - ensembles=["full", "partial", "none"], - write=False, + with_transfer: bool = False, + ensemblings: Ensembling = Ensembling.all(), + write: bool = False, ) -> go.Figure: """Render the multiplot visualization. Args: sweep: The SweepVisualization instance containing the data. with_transfer: Flag indicating whether to include transfer eval data. - ensembles: Filter for which ensembing options to include. + ensemblings: Filter for which ensembing options to include. write: Flag indicating whether to write the visualization to disk. Returns: @@ -49,10 +51,10 @@ def render( x_title="Layer", y_title="AUROC", ) - color_map = dict(zip(ensembles, qualitative.Plotly)) + color_map = dict(zip(ensemblings, qualitative.Plotly)) - for ensemble in ensembles: - ensemble_data: pd.DataFrame = df[df["ensembling"] == ensemble] + for ensembling in ensemblings: + ensemble_data: pd.DataFrame = df[df["ensembling"] == ensembling.value] if with_transfer: # TODO write tests ensemble_data = ensemble_data.groupby( ["eval_dataset", "layer", "ensembling"], as_index=False @@ -77,11 +79,11 @@ def render( x=dataset_data["layer"], y=dataset_data["auroc_estimate"], mode="lines", - name=ensemble, + name=ensembling.value, showlegend=False if dataset_name != unique_datasets[0] else True, - line=dict(color=color_map[ensemble]), + line=dict(color=color_map[ensembling]), ), row=row, col=col, @@ -115,7 +117,7 @@ class TransferEvalHeatmap: layer: int score_type: str = "auroc_estimate" - ensembling: str = "full" + ensembling: Ensembling = Ensembling.FULL def render(self, df: pd.DataFrame) -> go.Figure: """Render the heatmap visualization. @@ -245,7 +247,7 @@ def render_and_save( sweep: "SweepVisualization", dataset_names: list[str] | None = None, score_type="auroc_estimate", - ensembling="full", + ensembling=Ensembling.FULL, ) -> None: """Render and save the visualization for the model. @@ -262,7 +264,9 @@ def render_and_save( model_path.mkdir(parents=True, exist_ok=True) if self.is_transfer: for layer in range(layer_min, layer_max + 1): - filtered = df[(df["layer"] == layer) & (df["ensembling"] == ensembling)] + filtered = df[ + (df["layer"] == layer) & (df["ensembling"] == ensembling.value) + ] fig = TransferEvalHeatmap( layer, score_type=score_type, ensembling=ensembling ).render(filtered) @@ -382,7 +386,7 @@ def render_table( Returns: The generated score table as a pandas DataFrame. """ - df = self.df[self.df["ensembling"] == "partial"] + df = self.df[self.df["ensembling"] == Ensembling.PARTIAL.value] # For each model, we use the layer whose mean AUROC is the highest best_layers, model_dfs = [], [] diff --git a/elk/run.py b/elk/run.py index 93493c6f..42734420 100644 --- a/elk/run.py +++ b/elk/run.py @@ -31,6 +31,7 @@ select_split, select_usable_devices, ) +from .utils.types import Ensembling @dataclass @@ -197,15 +198,14 @@ def apply_to_layers( save_debug_log(self.datasets, self.out_dir) dfs = [] - for ensemble in [ - "full", - "partial", - "none", - ]: # TODO: Replace ensemble strings with enums everywhere - layer_ensembling_results = layer_ensembling(layer_outputs, ensemble) + + for ensembling in Ensembling.all(): + layer_ensembling_results = layer_ensembling( + layer_outputs, ensembling + ) df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) df = df.round(4) - df["ensemble"] = ensemble + df["ensemble"] = ensembling dfs.append(df) df_conc = pd.concat(dfs) diff --git a/elk/training/train.py b/elk/training/train.py index c785d5a6..1f82a6f9 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -14,6 +14,7 @@ from ..metrics import evaluate_preds, to_one_hot from ..run import Run from ..training.supervised import train_supervised +from ..utils.types import Ensembling from ..utils.typing import assert_type from .ccs_reporter import CcsReporter, CcsReporterConfig from .eigen_reporter import EigenReporter, EigenReporterConfig @@ -154,7 +155,7 @@ def apply_to_layer( ) train_credences = reporter(train_h) - for mode in ("none", "partial", "full"): + for mode in Ensembling.all(): row_bufs["eval"].append( { **meta, diff --git a/elk/utils/types.py b/elk/utils/types.py new file mode 100644 index 00000000..3ea75a9c --- /dev/null +++ b/elk/utils/types.py @@ -0,0 +1,11 @@ +from enum import Enum + + +class Ensembling(Enum): + FULL = "full" + PARTIAL = "partial" + NONE = "none" + + @staticmethod + def all() -> list["Ensembling"]: + return list(Ensembling) From fc980d7e8433dd368eab0792c6b9a1b1872524e8 Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 23 Jun 2023 15:30:43 +0100 Subject: [PATCH 23/57] Fix ensembling value writing error --- elk/run.py | 1 + elk/training/train.py | 30 +++++++++++++++++++----------- tests/test_smoke_elicit.py | 2 ++ tests/test_smoke_eval.py | 1 + 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/elk/run.py b/elk/run.py index 42734420..a01aaeb2 100644 --- a/elk/run.py +++ b/elk/run.py @@ -192,6 +192,7 @@ def apply_to_layers( finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): + print(dfs) df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: diff --git a/elk/training/train.py b/elk/training/train.py index 1f82a6f9..d670a192 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -155,12 +155,12 @@ def apply_to_layer( ) train_credences = reporter(train_h) - for mode in Ensembling.all(): + for ensembling in Ensembling.all(): row_bufs["eval"].append( { **meta, - "ensembling": mode, - **evaluate_preds(val_gt, val_credences, mode).to_dict(), + "ensembling": ensembling.value, + **evaluate_preds(val_gt, val_credences, ensembling).to_dict(), "pseudo_auroc": pseudo_auroc, "train_loss": train_loss, } @@ -169,8 +169,10 @@ def apply_to_layer( row_bufs["train_eval"].append( { **meta, - "ensembling": mode, - **evaluate_preds(train_gt, train_credences, mode).to_dict(), + "ensembling": ensembling.value, + **evaluate_preds( + train_gt, train_credences, ensembling + ).to_dict(), "train_loss": train_loss, } ) @@ -179,8 +181,10 @@ def apply_to_layer( row_bufs["lm_eval"].append( { **meta, - "ensembling": mode, - **evaluate_preds(val_gt, val_lm_preds, mode).to_dict(), + "ensembling": ensembling.value, + **evaluate_preds( + val_gt, val_lm_preds, ensembling + ).to_dict(), } ) @@ -188,8 +192,10 @@ def apply_to_layer( row_bufs["train_lm_eval"].append( { **meta, - "ensembling": mode, - **evaluate_preds(train_gt, train_lm_preds, mode).to_dict(), + "ensembling": ensembling.value, + **evaluate_preds( + train_gt, train_lm_preds, ensembling + ).to_dict(), } ) @@ -197,9 +203,11 @@ def apply_to_layer( row_bufs["lr_eval"].append( { **meta, - "ensembling": mode, + "ensembling": ensembling.value, "inlp_iter": i, - **evaluate_preds(val_gt, model(val_h), mode).to_dict(), + **evaluate_preds( + val_gt, model(val_h), ensembling + ).to_dict(), } ) diff --git a/tests/test_smoke_elicit.py b/tests/test_smoke_elicit.py index 7cf0e8c9..369b86a6 100644 --- a/tests/test_smoke_elicit.py +++ b/tests/test_smoke_elicit.py @@ -31,6 +31,7 @@ def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path): "lr_models", "reporters", "eval.csv", + "layer_ensembling_results.csv", ] for file in expected_files: assert file in created_file_names @@ -62,6 +63,7 @@ def test_smoke_elicit_run_tiny_gpt2_eigen(tmp_path: Path): "lr_models", "reporters", "eval.csv", + "layer_ensembling_results.csv", ] for file in expected_files: assert file in created_file_names diff --git a/tests/test_smoke_eval.py b/tests/test_smoke_eval.py index d58db6cd..2ff6819d 100644 --- a/tests/test_smoke_eval.py +++ b/tests/test_smoke_eval.py @@ -11,6 +11,7 @@ "cfg.yaml", "fingerprints.yaml", "eval.csv", + "layer_ensembling_results.csv", ] From 5aa30a9645bc9d0dabdaa24479e54fdc2cefdcac Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 23 Jun 2023 15:32:11 +0100 Subject: [PATCH 24/57] accidentally a print --- elk/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elk/run.py b/elk/run.py index a01aaeb2..42734420 100644 --- a/elk/run.py +++ b/elk/run.py @@ -192,7 +192,6 @@ def apply_to_layers( finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): - print(dfs) df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: From d5b8584dec3bbf7884e279dc718a8b8be702857c Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 23 Jun 2023 16:06:12 +0100 Subject: [PATCH 25/57] slightly refactor layer stuff and fix tests --- elk/metrics/eval.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 79093c5e..516ae741 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -131,7 +131,6 @@ def evaluate_preds( y_logits = y_logits.mean(dim=1) else: y_true = repeat(y_true, "n -> n v", v=num_variants) - return calc_eval_results(y_true, y_logits, ensembling, num_classes) @@ -182,34 +181,24 @@ def layer_ensembling(layer_outputs: list, ensembling: Ensembling) -> EvalResult: EvalResult: The result of evaluating a classifier containing the accuracy, calibrated accuracies, calibrated errors, and AUROC. """ + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") y_logits_means = [] - y_trues = [] + y_true = layer_outputs[0][0]["val_gt"].to(device) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") for layer_output in layer_outputs: y_logits = layer_output[0]["val_credences"].to(device) - - # full ensembling - y_logits_means.append(y_logits.mean(dim=1)) - - y_true = layer_output[0]["val_gt"].to(device) - y_trues.append(y_true) + y_logits_means.append(y_logits.mean(dim=1)) # full ensembling num_classes = layer_outputs[0][0]["val_credences"].shape[2] - # get logits and ground_truth from middle to last layer - middle_index = len(y_trues) // 2 - y_trues = y_trues[middle_index:] - y_logits = y_logits_means[middle_index:] - - y_logits_layers = torch.stack(y_logits) - + middle_index = len(layer_outputs) // 2 + y_logits_stacked = torch.stack(y_logits_means[middle_index:]) # layer ensembling of the stacked logits - y_layer_logits_means = torch.mean(y_logits_layers, dim=0) + y_logits_stacked_mean = torch.mean(y_logits_stacked, dim=0) return calc_eval_results( - y_true=y_trues[2], - y_logits=y_layer_logits_means, + y_true=y_true, + y_logits=y_logits_stacked_mean, ensembling=ensembling, num_classes=num_classes, ) From 638081435e778fd3fa316c1a5de357b2bf6e466d Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 23 Jun 2023 16:17:07 +0100 Subject: [PATCH 26/57] try fixing type hints --- elk/plotting/visualize.py | 3 ++- elk/utils/types.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py index 7771d9ce..4550212d 100644 --- a/elk/plotting/visualize.py +++ b/elk/plotting/visualize.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from pathlib import Path +from typing import Iterable import pandas as pd import plotly.express as px @@ -22,7 +23,7 @@ def render( self, sweep: "SweepVisualization", with_transfer: bool = False, - ensemblings: Ensembling = Ensembling.all(), + ensemblings: Iterable[Ensembling] = Ensembling.all(), write: bool = False, ) -> go.Figure: """Render the multiplot visualization. diff --git a/elk/utils/types.py b/elk/utils/types.py index 3ea75a9c..add1a4d1 100644 --- a/elk/utils/types.py +++ b/elk/utils/types.py @@ -7,5 +7,5 @@ class Ensembling(Enum): NONE = "none" @staticmethod - def all() -> list["Ensembling"]: - return list(Ensembling) + def all() -> tuple["Ensembling"]: + return tuple(Ensembling) From 98d19b762ddc0244ec500b235dea0f439e9d9c7b Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 23 Jun 2023 16:40:36 +0100 Subject: [PATCH 27/57] tidy up output --- elk/run.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/elk/run.py b/elk/run.py index 42734420..92d1e800 100644 --- a/elk/run.py +++ b/elk/run.py @@ -205,10 +205,15 @@ def apply_to_layers( ) df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) df = df.round(4) - df["ensemble"] = ensembling + df["ensembling"] = ensembling.value dfs.append(df) - df_conc = pd.concat(dfs) - df_conc.to_csv( + df_concat = pd.concat(dfs) + # Rearrange the columns so that ensembling is in front + columns = ["ensemble"] + [ + col for col in df_concat.columns if col != "ensembling" + ] + df_concat = df_concat[columns] + df_concat.to_csv( self.out_dir / "layer_ensembling_results.csv", index=False ) From e6914e16640681c2e10ec345df42664f6a499609 Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 23 Jun 2023 16:42:01 +0100 Subject: [PATCH 28/57] accidentally a char --- elk/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elk/run.py b/elk/run.py index 92d1e800..3f487a1f 100644 --- a/elk/run.py +++ b/elk/run.py @@ -210,7 +210,7 @@ def apply_to_layers( df_concat = pd.concat(dfs) # Rearrange the columns so that ensembling is in front - columns = ["ensemble"] + [ + columns = ["ensembling"] + [ col for col in df_concat.columns if col != "ensembling" ] df_concat = df_concat[columns] From 29b1cb8a9414730bba7b977b46af8c84935badf4 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sat, 24 Jun 2023 14:47:00 +0000 Subject: [PATCH 29/57] rename to PromptEnsembling --- elk/evaluation/evaluate.py | 4 ++-- elk/metrics/eval.py | 12 ++++++------ elk/plotting/visualize.py | 10 +++++----- elk/run.py | 4 ++-- elk/training/train.py | 4 ++-- elk/utils/types.py | 6 +++--- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index da5ea0c1..9f6aa1fb 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -11,7 +11,7 @@ from ..run import Run from ..training import Reporter from ..utils import Color -from ..utils.types import Ensembling +from ..utils.types import PromptEnsembling @dataclass(kw_only=True) @@ -54,7 +54,7 @@ def apply_to_layer( layer_outputs.append( {**meta, "val_gt": val_gt, "val_credences": val_credences} ) - for ensembling in Ensembling.all(): + for ensembling in PromptEnsembling.all(): row_bufs["eval"].append( { **meta, diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 516ae741..b8c6a899 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -4,7 +4,7 @@ from einops import repeat from torch import Tensor -from ..utils.types import Ensembling +from ..utils.types import PromptEnsembling from .accuracy import AccuracyResult, accuracy_ci from .calibration import CalibrationError, CalibrationEstimate from .roc_auc import RocAucResult, roc_auc_ci @@ -42,11 +42,11 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: def calc_auroc(y_logits, y_true, ensembling, num_classes): - if ensembling == Ensembling.NONE: + if ensembling == PromptEnsembling.NONE: auroc = roc_auc_ci( to_one_hot(y_true, num_classes).long().flatten(1), y_logits.flatten(1) ) - elif ensembling in (Ensembling.PARTIAL, Ensembling.FULL): + elif ensembling in (PromptEnsembling.PARTIAL, PromptEnsembling.FULL): # Pool together the negative and positive class logits if num_classes == 2: auroc = roc_auc_ci(y_true, y_logits[..., 1] - y_logits[..., 0]) @@ -111,7 +111,7 @@ def calc_accuracies(y_logits, y_true) -> AccuracyResult: def evaluate_preds( y_true: Tensor, y_logits: Tensor, - ensembling: Ensembling = Ensembling.NONE, + ensembling: PromptEnsembling = PromptEnsembling.NONE, ) -> EvalResult: """ Evaluate the performance of a classification model. @@ -127,7 +127,7 @@ def evaluate_preds( (n, num_variants, num_classes) = y_logits.shape assert y_true.shape == (n,) - if ensembling == Ensembling.FULL: + if ensembling == PromptEnsembling.FULL: y_logits = y_logits.mean(dim=1) else: y_true = repeat(y_true, "n -> n v", v=num_variants) @@ -168,7 +168,7 @@ def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: return EvalResult(acc, cal_acc, cal_err, auroc) -def layer_ensembling(layer_outputs: list, ensembling: Ensembling) -> EvalResult: +def layer_ensembling(layer_outputs: list, ensembling: PromptEnsembling) -> EvalResult: """ Return EvalResult after ensembling the probe output of the middle to last layers diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py index 4550212d..1eeb223c 100644 --- a/elk/plotting/visualize.py +++ b/elk/plotting/visualize.py @@ -10,7 +10,7 @@ from rich.console import Console from rich.table import Table -from elk.utils.types import Ensembling +from elk.utils.types import PromptEnsembling @dataclass @@ -23,7 +23,7 @@ def render( self, sweep: "SweepVisualization", with_transfer: bool = False, - ensemblings: Iterable[Ensembling] = Ensembling.all(), + ensemblings: Iterable[PromptEnsembling] = PromptEnsembling.all(), write: bool = False, ) -> go.Figure: """Render the multiplot visualization. @@ -118,7 +118,7 @@ class TransferEvalHeatmap: layer: int score_type: str = "auroc_estimate" - ensembling: Ensembling = Ensembling.FULL + ensembling: PromptEnsembling = PromptEnsembling.FULL def render(self, df: pd.DataFrame) -> go.Figure: """Render the heatmap visualization. @@ -248,7 +248,7 @@ def render_and_save( sweep: "SweepVisualization", dataset_names: list[str] | None = None, score_type="auroc_estimate", - ensembling=Ensembling.FULL, + ensembling=PromptEnsembling.FULL, ) -> None: """Render and save the visualization for the model. @@ -387,7 +387,7 @@ def render_table( Returns: The generated score table as a pandas DataFrame. """ - df = self.df[self.df["ensembling"] == Ensembling.PARTIAL.value] + df = self.df[self.df["ensembling"] == PromptEnsembling.PARTIAL.value] # For each model, we use the layer whose mean AUROC is the highest best_layers, model_dfs = [], [] diff --git a/elk/run.py b/elk/run.py index 3f487a1f..e369eae5 100644 --- a/elk/run.py +++ b/elk/run.py @@ -31,7 +31,7 @@ select_split, select_usable_devices, ) -from .utils.types import Ensembling +from .utils.types import PromptEnsembling @dataclass @@ -199,7 +199,7 @@ def apply_to_layers( dfs = [] - for ensembling in Ensembling.all(): + for ensembling in PromptEnsembling.all(): layer_ensembling_results = layer_ensembling( layer_outputs, ensembling ) diff --git a/elk/training/train.py b/elk/training/train.py index d670a192..acacba4c 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -14,7 +14,7 @@ from ..metrics import evaluate_preds, to_one_hot from ..run import Run from ..training.supervised import train_supervised -from ..utils.types import Ensembling +from ..utils.types import PromptEnsembling from ..utils.typing import assert_type from .ccs_reporter import CcsReporter, CcsReporterConfig from .eigen_reporter import EigenReporter, EigenReporterConfig @@ -155,7 +155,7 @@ def apply_to_layer( ) train_credences = reporter(train_h) - for ensembling in Ensembling.all(): + for ensembling in PromptEnsembling.all(): row_bufs["eval"].append( { **meta, diff --git a/elk/utils/types.py b/elk/utils/types.py index add1a4d1..eadeb81a 100644 --- a/elk/utils/types.py +++ b/elk/utils/types.py @@ -1,11 +1,11 @@ from enum import Enum -class Ensembling(Enum): +class PromptEnsembling(Enum): FULL = "full" PARTIAL = "partial" NONE = "none" @staticmethod - def all() -> tuple["Ensembling"]: - return tuple(Ensembling) + def all() -> tuple["PromptEnsembling"]: + return tuple(PromptEnsembling) From bed615a4adf4aeed433e9330287ac73dabacb8e0 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sun, 9 Jul 2023 20:36:40 +0000 Subject: [PATCH 30/57] add annotations and types --- elk/metrics/eval.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index b8c6a899..cc4c54d8 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -41,7 +41,19 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: return {**auroc_dict, **cal_acc_dict, **acc_dict, **cal_dict} -def calc_auroc(y_logits, y_true, ensembling, num_classes): +def calc_auroc(y_logits: Tensor, y_true: Tensor, ensembling: PromptEnsembling, num_classes: int) -> RocAucResult: + """ + Calculate the AUROC + + Args: + y_true: Ground truth tensor of shape (n,). + y_logits: Predicted class tensor of shape (n, num_variants, num_classes). + ensembling: The ensembling mode. + num_classes: The number of classes. + + Returns: + RocAucResult: A dictionary containing the AUROC and confidence interval. + """ if ensembling == PromptEnsembling.NONE: auroc = roc_auc_ci( to_one_hot(y_true, num_classes).long().flatten(1), y_logits.flatten(1) @@ -134,7 +146,7 @@ def evaluate_preds( return calc_eval_results(y_true, y_logits, ensembling, num_classes) -def calc_eval_results(y_true, y_logits, ensembling, num_classes) -> EvalResult: +def calc_eval_results(y_true: Tensor, y_logits: Tensor, ensembling: PromptEnsembling, num_classes: int) -> EvalResult: """ Calculate the evaluation results From bf49e99295c118bebb9e80622d3f3896a91ddf0c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 9 Jul 2023 20:36:54 +0000 Subject: [PATCH 31/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/metrics/eval.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index cc4c54d8..7b654551 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -41,7 +41,9 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: return {**auroc_dict, **cal_acc_dict, **acc_dict, **cal_dict} -def calc_auroc(y_logits: Tensor, y_true: Tensor, ensembling: PromptEnsembling, num_classes: int) -> RocAucResult: +def calc_auroc( + y_logits: Tensor, y_true: Tensor, ensembling: PromptEnsembling, num_classes: int +) -> RocAucResult: """ Calculate the AUROC @@ -146,7 +148,9 @@ def evaluate_preds( return calc_eval_results(y_true, y_logits, ensembling, num_classes) -def calc_eval_results(y_true: Tensor, y_logits: Tensor, ensembling: PromptEnsembling, num_classes: int) -> EvalResult: +def calc_eval_results( + y_true: Tensor, y_logits: Tensor, ensembling: PromptEnsembling, num_classes: int +) -> EvalResult: """ Calculate the evaluation results From 1f5d8be1e52ba4bfb6d9e6a2f4c17bc8b2735d0d Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sun, 9 Jul 2023 20:44:29 +0000 Subject: [PATCH 32/57] clearer naming: prompt_ensembling --- elk/metrics/eval.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index cc4c54d8..66382a68 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -41,31 +41,31 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: return {**auroc_dict, **cal_acc_dict, **acc_dict, **cal_dict} -def calc_auroc(y_logits: Tensor, y_true: Tensor, ensembling: PromptEnsembling, num_classes: int) -> RocAucResult: +def calc_auroc(y_logits: Tensor, y_true: Tensor, prompt_ensembling: PromptEnsembling, num_classes: int) -> RocAucResult: """ Calculate the AUROC Args: y_true: Ground truth tensor of shape (n,). y_logits: Predicted class tensor of shape (n, num_variants, num_classes). - ensembling: The ensembling mode. + prompt_ensembling: The prompt_ensembling mode. num_classes: The number of classes. Returns: RocAucResult: A dictionary containing the AUROC and confidence interval. """ - if ensembling == PromptEnsembling.NONE: + if prompt_ensembling == PromptEnsembling.NONE: auroc = roc_auc_ci( to_one_hot(y_true, num_classes).long().flatten(1), y_logits.flatten(1) ) - elif ensembling in (PromptEnsembling.PARTIAL, PromptEnsembling.FULL): + elif prompt_ensembling in (PromptEnsembling.PARTIAL, PromptEnsembling.FULL): # Pool together the negative and positive class logits if num_classes == 2: auroc = roc_auc_ci(y_true, y_logits[..., 1] - y_logits[..., 0]) else: auroc = roc_auc_ci(to_one_hot(y_true, num_classes).long(), y_logits) else: - raise ValueError(f"Unknown mode: {ensembling}") + raise ValueError(f"Unknown mode: {prompt_ensembling}") return auroc @@ -123,7 +123,7 @@ def calc_accuracies(y_logits, y_true) -> AccuracyResult: def evaluate_preds( y_true: Tensor, y_logits: Tensor, - ensembling: PromptEnsembling = PromptEnsembling.NONE, + prompt_ensembling: PromptEnsembling = PromptEnsembling.NONE, ) -> EvalResult: """ Evaluate the performance of a classification model. @@ -131,7 +131,7 @@ def evaluate_preds( Args: y_true: Ground truth tensor of shape (n,). y_logits: Predicted class tensor of shape (n, num_variants, num_classes). - ensembling: The ensembling mode. + prompt_ensembling: The prompt_ensembling mode. Returns: dict: A dictionary containing the accuracy, AUROC, and ECE. @@ -139,21 +139,21 @@ def evaluate_preds( (n, num_variants, num_classes) = y_logits.shape assert y_true.shape == (n,) - if ensembling == PromptEnsembling.FULL: + if prompt_ensembling == PromptEnsembling.FULL: y_logits = y_logits.mean(dim=1) else: y_true = repeat(y_true, "n -> n v", v=num_variants) - return calc_eval_results(y_true, y_logits, ensembling, num_classes) + return calc_eval_results(y_true, y_logits, prompt_ensembling, num_classes) -def calc_eval_results(y_true: Tensor, y_logits: Tensor, ensembling: PromptEnsembling, num_classes: int) -> EvalResult: +def calc_eval_results(y_true: Tensor, y_logits: Tensor, prompt_ensembling: PromptEnsembling, num_classes: int) -> EvalResult: """ Calculate the evaluation results Args: y_true: Ground truth tensor of shape (n,). y_logits: Predicted class tensor of shape (n, num_variants, num_classes). - ensembling: The ensembling mode. + prompt_ensembling: The prompt_ensembling mode. Returns: EvalResult: The result of evaluating a classifier containing the accuracy, @@ -174,20 +174,20 @@ def calc_eval_results(y_true: Tensor, y_logits: Tensor, ensembling: PromptEnsemb ) auroc = calc_auroc( - y_logits=y_logits, y_true=y_true, ensembling=ensembling, num_classes=num_classes + y_logits=y_logits, y_true=y_true, prompt_ensembling=prompt_ensembling, num_classes=num_classes ) return EvalResult(acc, cal_acc, cal_err, auroc) -def layer_ensembling(layer_outputs: list, ensembling: PromptEnsembling) -> EvalResult: +def layer_ensembling(layer_outputs: list, prompt_ensembling: PromptEnsembling) -> EvalResult: """ - Return EvalResult after ensembling the probe output of the middle to last layers + Return EvalResult after prompt_ensembling the probe output of the middle to last layers Args: layer_outputs: A list of dictionaries containing the ground truth and predicted class tensor of shape (n, num_variants, num_classes). - ensembling: The ensembling mode. + prompt_ensembling: The prompt_ensembling mode. Returns: EvalResult: The result of evaluating a classifier containing the accuracy, @@ -199,19 +199,19 @@ def layer_ensembling(layer_outputs: list, ensembling: PromptEnsembling) -> EvalR for layer_output in layer_outputs: y_logits = layer_output[0]["val_credences"].to(device) - y_logits_means.append(y_logits.mean(dim=1)) # full ensembling + y_logits_means.append(y_logits.mean(dim=1)) # full prompt_ensembling num_classes = layer_outputs[0][0]["val_credences"].shape[2] # get logits and ground_truth from middle to last layer middle_index = len(layer_outputs) // 2 y_logits_stacked = torch.stack(y_logits_means[middle_index:]) - # layer ensembling of the stacked logits + # layer prompt_ensembling of the stacked logits y_logits_stacked_mean = torch.mean(y_logits_stacked, dim=0) return calc_eval_results( y_true=y_true, y_logits=y_logits_stacked_mean, - ensembling=ensembling, + prompt_ensembling=prompt_ensembling, num_classes=num_classes, ) From ec377165d4e8893158648e6192e2f351b09ddb67 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:54:44 +0000 Subject: [PATCH 33/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/metrics/eval.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 8a344872..53ad729e 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -178,13 +178,18 @@ def calc_eval_results( ) auroc = calc_auroc( - y_logits=y_logits, y_true=y_true, prompt_ensembling=prompt_ensembling, num_classes=num_classes + y_logits=y_logits, + y_true=y_true, + prompt_ensembling=prompt_ensembling, + num_classes=num_classes, ) return EvalResult(acc, cal_acc, cal_err, auroc) -def layer_ensembling(layer_outputs: list, prompt_ensembling: PromptEnsembling) -> EvalResult: +def layer_ensembling( + layer_outputs: list, prompt_ensembling: PromptEnsembling +) -> EvalResult: """ Return EvalResult after prompt_ensembling the probe output of the middle to last layers From 193662491ed4f407623fd744b62ca9596f048ae2 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Wed, 12 Jul 2023 13:57:56 +0000 Subject: [PATCH 34/57] better name for ensembling --- elk/evaluation/evaluate.py | 10 +++++----- elk/metrics/eval.py | 4 ++-- elk/training/train.py | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 9f6aa1fb..12aa974b 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -54,12 +54,12 @@ def apply_to_layer( layer_outputs.append( {**meta, "val_gt": val_gt, "val_credences": val_credences} ) - for ensembling in PromptEnsembling.all(): + for prompt_ensembling in PromptEnsembling.all(): row_bufs["eval"].append( { **meta, - "ensembling": ensembling.value, - **evaluate_preds(val_gt, val_credences, ensembling).to_dict(), + "prompt_ensembling": prompt_ensembling.value, + **evaluate_preds(val_gt, val_credences, prompt_ensembling).to_dict(), } ) @@ -74,11 +74,11 @@ def apply_to_layer( model.eval() row_bufs["lr_eval"].append( { - "ensembling": ensembling.value, + "prompt_ensembling": prompt_ensembling.value, "inlp_iter": i, **meta, **evaluate_preds( - val_gt, model(val_h), ensembling + val_gt, model(val_h), prompt_ensembling ).to_dict(), } ) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 8a344872..090f2feb 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -42,7 +42,7 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: def calc_auroc( - y_logits: Tensor, y_true: Tensor, ensembling: PromptEnsembling, num_classes: int + y_logits: Tensor, y_true: Tensor, prompt_ensembling: PromptEnsembling, num_classes: int ) -> RocAucResult: """ Calculate the AUROC @@ -149,7 +149,7 @@ def evaluate_preds( def calc_eval_results( - y_true: Tensor, y_logits: Tensor, ensembling: PromptEnsembling, num_classes: int + y_true: Tensor, y_logits: Tensor, prompt_ensembling: PromptEnsembling, num_classes: int ) -> EvalResult: """ Calculate the evaluation results diff --git a/elk/training/train.py b/elk/training/train.py index 587412f8..e995c9a7 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -146,12 +146,12 @@ def apply_to_layer( ) train_credences = reporter(train_h) - for ensembling in PromptEnsembling.all(): + for prompt_ensembling in PromptEnsembling.all(): row_bufs["eval"].append( { **meta, - "ensembling": ensembling.value, - **evaluate_preds(val_gt, val_credences, ensembling).to_dict(), + "prompt_ensembling": prompt_ensembling.value, + **evaluate_preds(val_gt, val_credences, prompt_ensembling).to_dict(), "pseudo_auroc": pseudo_auroc, "train_loss": train_loss, } @@ -160,9 +160,9 @@ def apply_to_layer( row_bufs["train_eval"].append( { **meta, - "ensembling": ensembling.value, + "prompt_ensembling": prompt_ensembling.value, **evaluate_preds( - train_gt, train_credences, ensembling + train_gt, train_credences, prompt_ensembling ).to_dict(), "train_loss": train_loss, } @@ -172,9 +172,9 @@ def apply_to_layer( row_bufs["lm_eval"].append( { **meta, - "ensembling": ensembling.value, + "prompt_ensembling": prompt_ensembling.value, **evaluate_preds( - val_gt, val_lm_preds, ensembling + val_gt, val_lm_preds, prompt_ensembling ).to_dict(), } ) @@ -183,9 +183,9 @@ def apply_to_layer( row_bufs["train_lm_eval"].append( { **meta, - "ensembling": ensembling.value, + "prompt_ensembling": prompt_ensembling.value, **evaluate_preds( - train_gt, train_lm_preds, ensembling + train_gt, train_lm_preds, prompt_ensembling ).to_dict(), } ) @@ -194,10 +194,10 @@ def apply_to_layer( row_bufs["lr_eval"].append( { **meta, - "ensembling": ensembling.value, + "prompt_ensembling": prompt_ensembling.value, "inlp_iter": i, **evaluate_preds( - val_gt, model(val_h), ensembling + val_gt, model(val_h), prompt_ensembling ).to_dict(), } ) From 484788e808f8357a83984e7e1799ea301c99f94d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:58:11 +0000 Subject: [PATCH 35/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/evaluation/evaluate.py | 4 +++- elk/metrics/eval.py | 10 ++++++++-- elk/training/train.py | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 12aa974b..543d358d 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -59,7 +59,9 @@ def apply_to_layer( { **meta, "prompt_ensembling": prompt_ensembling.value, - **evaluate_preds(val_gt, val_credences, prompt_ensembling).to_dict(), + **evaluate_preds( + val_gt, val_credences, prompt_ensembling + ).to_dict(), } ) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index a2160f3b..55548a3e 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -42,7 +42,10 @@ def to_dict(self, prefix: str = "") -> dict[str, float]: def calc_auroc( - y_logits: Tensor, y_true: Tensor, prompt_ensembling: PromptEnsembling, num_classes: int + y_logits: Tensor, + y_true: Tensor, + prompt_ensembling: PromptEnsembling, + num_classes: int, ) -> RocAucResult: """ Calculate the AUROC @@ -149,7 +152,10 @@ def evaluate_preds( def calc_eval_results( - y_true: Tensor, y_logits: Tensor, prompt_ensembling: PromptEnsembling, num_classes: int + y_true: Tensor, + y_logits: Tensor, + prompt_ensembling: PromptEnsembling, + num_classes: int, ) -> EvalResult: """ Calculate the evaluation results diff --git a/elk/training/train.py b/elk/training/train.py index e995c9a7..a75609b9 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -151,7 +151,9 @@ def apply_to_layer( { **meta, "prompt_ensembling": prompt_ensembling.value, - **evaluate_preds(val_gt, val_credences, prompt_ensembling).to_dict(), + **evaluate_preds( + val_gt, val_credences, prompt_ensembling + ).to_dict(), "pseudo_auroc": pseudo_auroc, "train_loss": train_loss, } From b6de9576966bbf19cf54d2b4117ab239c6938115 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Thu, 13 Jul 2023 19:19:00 +0000 Subject: [PATCH 36/57] remove pseudo auroc --- elk/training/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elk/training/train.py b/elk/training/train.py index ad4d4fd9..fa613886 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -156,7 +156,6 @@ def apply_to_layer( **evaluate_preds( val_gt, val_credences, prompt_ensembling ).to_dict(), - "pseudo_auroc": pseudo_auroc, "train_loss": train_loss, } ) From cf32b0c870ee190349fb3535d107b778c5759feb Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Thu, 13 Jul 2023 19:24:04 +0000 Subject: [PATCH 37/57] rename to prompt_ensembling --- elk/plotting/visualize.py | 24 ++++++++++++------------ elk/run.py | 14 +++++++------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py index 1eeb223c..dc68ba1d 100644 --- a/elk/plotting/visualize.py +++ b/elk/plotting/visualize.py @@ -54,11 +54,11 @@ def render( ) color_map = dict(zip(ensemblings, qualitative.Plotly)) - for ensembling in ensemblings: - ensemble_data: pd.DataFrame = df[df["ensembling"] == ensembling.value] + for prompt_ensembling in ensemblings: + ensemble_data: pd.DataFrame = df[df["prompt_ensembling"] == prompt_ensembling.value] if with_transfer: # TODO write tests ensemble_data = ensemble_data.groupby( - ["eval_dataset", "layer", "ensembling"], as_index=False + ["eval_dataset", "layer", "prompt_ensembling"], as_index=False ).agg({"auroc_estimate": "mean"}) else: ensemble_data = ensemble_data[ @@ -80,11 +80,11 @@ def render( x=dataset_data["layer"], y=dataset_data["auroc_estimate"], mode="lines", - name=ensembling.value, + name=prompt_ensembling.value, showlegend=False if dataset_name != unique_datasets[0] else True, - line=dict(color=color_map[ensembling]), + line=dict(color=color_map[prompt_ensembling]), ), row=row, col=col, @@ -96,7 +96,7 @@ def render( fig.update_layout( legend=dict( - title="Ensembling", + title="prompt_ensembling", ), title=f"AUROC Trend: {self.model_name}", ) @@ -118,7 +118,7 @@ class TransferEvalHeatmap: layer: int score_type: str = "auroc_estimate" - ensembling: PromptEnsembling = PromptEnsembling.FULL + prompt_ensembling: PromptEnsembling = PromptEnsembling.FULL def render(self, df: pd.DataFrame) -> go.Figure: """Render the heatmap visualization. @@ -248,7 +248,7 @@ def render_and_save( sweep: "SweepVisualization", dataset_names: list[str] | None = None, score_type="auroc_estimate", - ensembling=PromptEnsembling.FULL, + prompt_ensembling=PromptEnsembling.FULL, ) -> None: """Render and save the visualization for the model. @@ -256,7 +256,7 @@ def render_and_save( sweep: The SweepVisualization instance. dataset_names: List of dataset names to include in the visualization. score_type: The type of score to display. - ensembling: The ensembling option to consider. + prompt_ensembling: The prompt_ensembling option to consider. """ df = self.df model_name = self.model_name @@ -266,10 +266,10 @@ def render_and_save( if self.is_transfer: for layer in range(layer_min, layer_max + 1): filtered = df[ - (df["layer"] == layer) & (df["ensembling"] == ensembling.value) + (df["layer"] == layer) & (df["prompt_ensembling"] == prompt_ensembling.value) ] fig = TransferEvalHeatmap( - layer, score_type=score_type, ensembling=ensembling + layer, score_type=score_type, prompt_ensembling=prompt_ensembling ).render(filtered) fig.write_image(file=model_path / f"{layer}.png") fig = TransferEvalTrend(dataset_names).render(df) @@ -387,7 +387,7 @@ def render_table( Returns: The generated score table as a pandas DataFrame. """ - df = self.df[self.df["ensembling"] == PromptEnsembling.PARTIAL.value] + df = self.df[self.df["prompt_ensembling"] == PromptEnsembling.PARTIAL.value] # For each model, we use the layer whose mean AUROC is the highest best_layers, model_dfs = [], [] diff --git a/elk/run.py b/elk/run.py index e369eae5..594bca4c 100644 --- a/elk/run.py +++ b/elk/run.py @@ -192,26 +192,26 @@ def apply_to_layers( finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): - df = pd.concat(dfs).sort_values(by=["layer", "ensembling"]) + df = pd.concat(dfs).sort_values(by=["layer", "prompt_ensembling"]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: save_debug_log(self.datasets, self.out_dir) dfs = [] - for ensembling in PromptEnsembling.all(): + for prompt_ensembling in PromptEnsembling.all(): layer_ensembling_results = layer_ensembling( - layer_outputs, ensembling + layer_outputs, prompt_ensembling ) df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) df = df.round(4) - df["ensembling"] = ensembling.value + df["prompt_ensembling"] = prompt_ensembling.value dfs.append(df) df_concat = pd.concat(dfs) - # Rearrange the columns so that ensembling is in front - columns = ["ensembling"] + [ - col for col in df_concat.columns if col != "ensembling" + # Rearrange the columns so that prompt_ensembling is in front + columns = ["prompt_ensembling"] + [ + col for col in df_concat.columns if col != "prompt_ensembling" ] df_concat = df_concat[columns] df_concat.to_csv( From 769676a850885a2db317931a5bb0075ea0775ad4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 13 Jul 2023 19:24:19 +0000 Subject: [PATCH 38/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/plotting/visualize.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py index dc68ba1d..3c2b9cdf 100644 --- a/elk/plotting/visualize.py +++ b/elk/plotting/visualize.py @@ -55,7 +55,9 @@ def render( color_map = dict(zip(ensemblings, qualitative.Plotly)) for prompt_ensembling in ensemblings: - ensemble_data: pd.DataFrame = df[df["prompt_ensembling"] == prompt_ensembling.value] + ensemble_data: pd.DataFrame = df[ + df["prompt_ensembling"] == prompt_ensembling.value + ] if with_transfer: # TODO write tests ensemble_data = ensemble_data.groupby( ["eval_dataset", "layer", "prompt_ensembling"], as_index=False @@ -266,7 +268,8 @@ def render_and_save( if self.is_transfer: for layer in range(layer_min, layer_max + 1): filtered = df[ - (df["layer"] == layer) & (df["prompt_ensembling"] == prompt_ensembling.value) + (df["layer"] == layer) + & (df["prompt_ensembling"] == prompt_ensembling.value) ] fig = TransferEvalHeatmap( layer, score_type=score_type, prompt_ensembling=prompt_ensembling From e6c9d4c0961c25e0e7583d7e6e4a76413330ef14 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Thu, 13 Jul 2023 19:35:07 +0000 Subject: [PATCH 39/57] precomit fixes --- elk/metrics/eval.py | 3 ++- elk/plotting/visualize.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 55548a3e..6c6ee3c8 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -197,7 +197,8 @@ def layer_ensembling( layer_outputs: list, prompt_ensembling: PromptEnsembling ) -> EvalResult: """ - Return EvalResult after prompt_ensembling the probe output of the middle to last layers + Return EvalResult after prompt_ensembling + the probe output of the middle to last layers Args: layer_outputs: A list of dictionaries containing the ground truth and diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py index dc68ba1d..3c2b9cdf 100644 --- a/elk/plotting/visualize.py +++ b/elk/plotting/visualize.py @@ -55,7 +55,9 @@ def render( color_map = dict(zip(ensemblings, qualitative.Plotly)) for prompt_ensembling in ensemblings: - ensemble_data: pd.DataFrame = df[df["prompt_ensembling"] == prompt_ensembling.value] + ensemble_data: pd.DataFrame = df[ + df["prompt_ensembling"] == prompt_ensembling.value + ] if with_transfer: # TODO write tests ensemble_data = ensemble_data.groupby( ["eval_dataset", "layer", "prompt_ensembling"], as_index=False @@ -266,7 +268,8 @@ def render_and_save( if self.is_transfer: for layer in range(layer_min, layer_max + 1): filtered = df[ - (df["layer"] == layer) & (df["prompt_ensembling"] == prompt_ensembling.value) + (df["layer"] == layer) + & (df["prompt_ensembling"] == prompt_ensembling.value) ] fig = TransferEvalHeatmap( layer, score_type=score_type, prompt_ensembling=prompt_ensembling From 602815216380de2b465ed7c130c121edd271e432 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Tue, 18 Jul 2023 15:36:26 +0000 Subject: [PATCH 40/57] fix num_classes --- elk/metrics/eval.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 6c6ee3c8..7363f7c3 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -141,15 +141,23 @@ def evaluate_preds( Returns: dict: A dictionary containing the accuracy, AUROC, and ECE. """ + y_logits, y_true, num_classes = prepare(y_logits, y_true, prompt_ensembling) + return calc_eval_results(y_true, y_logits, prompt_ensembling, num_classes) + + +def prepare(y_logits: Tensor, y_true: Tensor, prompt_ensembling: PromptEnsembling): + """ + Prepare the logits and ground truth for evaluation + """ (n, num_variants, num_classes) = y_logits.shape - assert y_true.shape == (n,) + assert y_true.shape == (n,), f"y_true.shape: {y_true.shape} is not equal to n: {n}" if prompt_ensembling == PromptEnsembling.FULL: y_logits = y_logits.mean(dim=1) else: y_true = repeat(y_true, "n -> n v", v=num_variants) - return calc_eval_results(y_true, y_logits, prompt_ensembling, num_classes) + return y_logits, y_true, num_classes def calc_eval_results( y_true: Tensor, @@ -210,17 +218,17 @@ def layer_ensembling( calibrated accuracies, calibrated errors, and AUROC. """ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - y_logits_means = [] + y_logits_collection = [] y_true = layer_outputs[0][0]["val_gt"].to(device) for layer_output in layer_outputs: y_logits = layer_output[0]["val_credences"].to(device) - y_logits_means.append(y_logits.mean(dim=1)) # full prompt_ensembling + y_logits, y_true, num_classes = prepare(y_logits, y_true, prompt_ensembling) + y_logits_collection.append(y_logits) - num_classes = layer_outputs[0][0]["val_credences"].shape[2] # get logits and ground_truth from middle to last layer middle_index = len(layer_outputs) // 2 - y_logits_stacked = torch.stack(y_logits_means[middle_index:]) + y_logits_stacked = torch.stack(y_logits_collection[middle_index:]) # layer prompt_ensembling of the stacked logits y_logits_stacked_mean = torch.mean(y_logits_stacked, dim=0) From 6d7d99a90fcdb4f3898ef14681bb25af1e394972 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:39:21 +0000 Subject: [PATCH 41/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/metrics/eval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 7363f7c3..2159686b 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -159,6 +159,7 @@ def prepare(y_logits: Tensor, y_true: Tensor, prompt_ensembling: PromptEnsemblin return y_logits, y_true, num_classes + def calc_eval_results( y_true: Tensor, y_logits: Tensor, From 4148857b89b965c64cc16ea8f7926fbdeaa545de Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Tue, 18 Jul 2023 17:38:02 +0000 Subject: [PATCH 42/57] fix bug where y_true has a dimension of two --- elk/metrics/eval.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 7363f7c3..61aac802 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -219,11 +219,14 @@ def layer_ensembling( """ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") y_logits_collection = [] - y_true = layer_outputs[0][0]["val_gt"].to(device) for layer_output in layer_outputs: + # all y_trues are identical, so just get the first + y_true = layer_outputs[0][0]["val_gt"].to(device) y_logits = layer_output[0]["val_credences"].to(device) - y_logits, y_true, num_classes = prepare(y_logits, y_true, prompt_ensembling) + y_logits, y_true, num_classes = prepare(y_logits=y_logits, + y_true=y_true, + prompt_ensembling=prompt_ensembling) y_logits_collection.append(y_logits) # get logits and ground_truth from middle to last layer From 964f03ddd2294c04c3f45e722ecdc98452d28ffd Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Tue, 18 Jul 2023 17:38:17 +0000 Subject: [PATCH 43/57] cleanup --- elk/evaluation/evaluate.py | 6 +++--- elk/run.py | 3 ++- elk/training/train.py | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 7325d23f..ce6d7a73 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -44,12 +44,12 @@ def apply_to_layer( row_bufs = defaultdict(list) - layer_outputs = [] + layer_output = [] for ds_name, (val_h, val_gt, _) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - layer_outputs.append( + layer_output.append( {**meta, "val_gt": val_gt, "val_credences": val_credences} ) for prompt_ensembling in PromptEnsembling.all(): @@ -83,4 +83,4 @@ def apply_to_layer( } ) - return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_outputs) + return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_output) diff --git a/elk/run.py b/elk/run.py index 594bca4c..7d1117b1 100644 --- a/elk/run.py +++ b/elk/run.py @@ -201,7 +201,8 @@ def apply_to_layers( for prompt_ensembling in PromptEnsembling.all(): layer_ensembling_results = layer_ensembling( - layer_outputs, prompt_ensembling + layer_outputs=layer_outputs, + prompt_ensembling=prompt_ensembling ) df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) df = df.round(4) diff --git a/elk/training/train.py b/elk/training/train.py index fa613886..5963212d 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -131,7 +131,7 @@ def apply_to_layer( lr_models = [] row_bufs = defaultdict(list) - layer_outputs = [] + layer_output = [] for ds_name in val_dict: val_h, val_gt, val_lm_preds = val_dict[ds_name] train_h, train_gt, train_lm_preds = train_dict[ds_name] @@ -139,7 +139,7 @@ def apply_to_layer( val_credences = reporter(val_h) - layer_outputs.append( + layer_output.append( { **meta, "val_gt": val_gt.detach(), @@ -205,4 +205,4 @@ def apply_to_layer( } ) - return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_outputs) + return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_output) From 06dad69455e950cd4c629b5ec4ff648bf95a892f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:39:46 +0000 Subject: [PATCH 44/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/metrics/eval.py | 6 +++--- elk/run.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index b3f9463a..5e7a232b 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -225,9 +225,9 @@ def layer_ensembling( # all y_trues are identical, so just get the first y_true = layer_outputs[0][0]["val_gt"].to(device) y_logits = layer_output[0]["val_credences"].to(device) - y_logits, y_true, num_classes = prepare(y_logits=y_logits, - y_true=y_true, - prompt_ensembling=prompt_ensembling) + y_logits, y_true, num_classes = prepare( + y_logits=y_logits, y_true=y_true, prompt_ensembling=prompt_ensembling + ) y_logits_collection.append(y_logits) # get logits and ground_truth from middle to last layer diff --git a/elk/run.py b/elk/run.py index 7d1117b1..74f3da69 100644 --- a/elk/run.py +++ b/elk/run.py @@ -201,8 +201,7 @@ def apply_to_layers( for prompt_ensembling in PromptEnsembling.all(): layer_ensembling_results = layer_ensembling( - layer_outputs=layer_outputs, - prompt_ensembling=prompt_ensembling + layer_outputs=layer_outputs, prompt_ensembling=prompt_ensembling ) df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) df = df.round(4) From 0d2545b45cd07469000f9c5d7c8b94748b64e6ed Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Tue, 18 Jul 2023 17:52:53 +0000 Subject: [PATCH 45/57] add y_true_initial --- elk/metrics/eval.py | 12 ++++++++---- elk/run.py | 3 +-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index b3f9463a..f4cdc392 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -221,13 +221,17 @@ def layer_ensembling( device = torch.device("cuda" if torch.cuda.is_available() else "cpu") y_logits_collection = [] + num_classes = None + y_true = None + for layer_output in layer_outputs: # all y_trues are identical, so just get the first - y_true = layer_outputs[0][0]["val_gt"].to(device) y_logits = layer_output[0]["val_credences"].to(device) - y_logits, y_true, num_classes = prepare(y_logits=y_logits, - y_true=y_true, - prompt_ensembling=prompt_ensembling) + y_logits, y_true, num_classes = prepare( + y_logits=y_logits, + y_true=layer_outputs[0][0]["val_gt"].to(device), + prompt_ensembling=prompt_ensembling, + ) y_logits_collection.append(y_logits) # get logits and ground_truth from middle to last layer diff --git a/elk/run.py b/elk/run.py index 7d1117b1..74f3da69 100644 --- a/elk/run.py +++ b/elk/run.py @@ -201,8 +201,7 @@ def apply_to_layers( for prompt_ensembling in PromptEnsembling.all(): layer_ensembling_results = layer_ensembling( - layer_outputs=layer_outputs, - prompt_ensembling=prompt_ensembling + layer_outputs=layer_outputs, prompt_ensembling=prompt_ensembling ) df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) df = df.round(4) From 5952b4b5ac1b4cb751c8cab46881b572ba2e36d5 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Tue, 18 Jul 2023 18:50:33 +0000 Subject: [PATCH 46/57] fix test error --- elk/metrics/eval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index f4cdc392..0a7cd0b3 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -221,8 +221,8 @@ def layer_ensembling( device = torch.device("cuda" if torch.cuda.is_available() else "cpu") y_logits_collection = [] - num_classes = None - y_true = None + num_classes = 2 + y_true = layer_outputs[0][0]["val_gt"].to(device) for layer_output in layer_outputs: # all y_trues are identical, so just get the first From 049cd6331d8c7f8954f1ed5b954def13e653258b Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sat, 22 Jul 2023 21:49:44 +0100 Subject: [PATCH 47/57] replace mode with prompt_ensembling.value --- elk/evaluation/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 3a6d63fe..ed3b9c12 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -67,8 +67,8 @@ def apply_to_layer( row_bufs["lm_eval"].append( { **meta, - "ensembling": mode, - **evaluate_preds(val_gt, val_lm_preds, mode).to_dict(), + "ensembling": prompt_ensembling.value, + **evaluate_preds(val_gt, val_lm_preds, prompt_ensembling.value).to_dict(), } ) From c8236ddcc2a7fe5b844b5ada0eb8a80740cbd303 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 22 Jul 2023 20:49:51 +0000 Subject: [PATCH 48/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/evaluation/evaluate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index ed3b9c12..581ebe0f 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -68,7 +68,9 @@ def apply_to_layer( { **meta, "ensembling": prompt_ensembling.value, - **evaluate_preds(val_gt, val_lm_preds, prompt_ensembling.value).to_dict(), + **evaluate_preds( + val_gt, val_lm_preds, prompt_ensembling.value + ).to_dict(), } ) From 56d1796eaa9a078993f588e92ad47ceeaf59bd21 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Sun, 23 Jul 2023 19:35:04 +0000 Subject: [PATCH 49/57] remove .value for lm_eval --- elk/evaluation/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 581ebe0f..5d98f15e 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -69,7 +69,7 @@ def apply_to_layer( **meta, "ensembling": prompt_ensembling.value, **evaluate_preds( - val_gt, val_lm_preds, prompt_ensembling.value + val_gt, val_lm_preds, prompt_ensembling ).to_dict(), } ) From 4d9c7811acbd58c50e14c02b039c8065a4c2156a Mon Sep 17 00:00:00 2001 From: jon Date: Thu, 27 Jul 2023 11:46:26 +0100 Subject: [PATCH 50/57] add LayerApplied --- elk/evaluation/evaluate.py | 11 ++++++----- elk/run.py | 11 ++++++++++- elk/training/train.py | 8 +++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 5d98f15e..fd792b29 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -8,7 +8,7 @@ from ..files import elk_reporter_dir from ..metrics import evaluate_preds -from ..run import Run +from ..run import LayerApplied, Run from ..utils import Color from ..utils.types import PromptEnsembling @@ -32,7 +32,7 @@ def execute(self, highlight_color: Color = "cyan"): @torch.inference_mode() def apply_to_layer( self, layer: int, devices: list[str], world_size: int - ) -> tuple[dict[str, pd.DataFrame], list[dict]]: + ) -> LayerApplied: """Evaluate a single reporter on a single layer.""" device = self.get_device(devices, world_size) val_output = self.prepare_data(device, layer, "val") @@ -44,7 +44,7 @@ def apply_to_layer( row_bufs = defaultdict(list) - layer_output = [] + layer_output: list[dict] = [] for ds_name, (val_h, val_gt, val_lm_preds) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} @@ -93,5 +93,6 @@ def apply_to_layer( ).to_dict(), } ) - - return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_output) + return LayerApplied( + layer_output, {k: pd.DataFrame(v) for k, v in row_bufs.items()} + ) diff --git a/elk/run.py b/elk/run.py index 74f3da69..00ddb084 100644 --- a/elk/run.py +++ b/elk/run.py @@ -34,6 +34,15 @@ from .utils.types import PromptEnsembling +@dataclass(frozen=True) +class LayerApplied: + layer_output: list[dict] + """The output of the reporter on the layer, should contain credences and ground + truth labels.""" + df_dict: dict[str, pd.DataFrame] + """The evaluation results for the layer.""" + + @dataclass class Run(ABC, Serializable): data: Extract @@ -109,7 +118,7 @@ def execute( @abstractmethod def apply_to_layer( self, layer: int, devices: list[str], world_size: int - ) -> dict[str, pd.DataFrame]: + ) -> LayerApplied: """Train or eval a reporter on a single layer.""" def make_reproducible(self, seed: int): diff --git a/elk/training/train.py b/elk/training/train.py index 6b68c694..b57eef81 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -12,7 +12,7 @@ from simple_parsing.helpers.serialization import save from ..metrics import evaluate_preds, to_one_hot -from ..run import Run +from ..run import LayerApplied, Run from ..training.supervised import train_supervised from ..utils.types import PromptEnsembling from ..utils.typing import assert_type @@ -54,7 +54,7 @@ def apply_to_layer( layer: int, devices: list[str], world_size: int, - ) -> tuple[dict[str, pd.DataFrame], list[dict]]: + ) -> LayerApplied: """Train a single reporter on a single layer.""" self.make_reproducible(seed=self.net.seed + layer) @@ -205,4 +205,6 @@ def apply_to_layer( } ) - return ({k: pd.DataFrame(v) for k, v in row_bufs.items()}, layer_output) + return LayerApplied( + layer_output, {k: pd.DataFrame(v) for k, v in row_bufs.items()} + ) From 8961e95fc911d5c71808e55ab7b78943daca217b Mon Sep 17 00:00:00 2001 From: jon Date: Thu, 27 Jul 2023 13:30:48 +0100 Subject: [PATCH 51/57] fix run.py part --- elk/run.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/elk/run.py b/elk/run.py index 00ddb084..53d24ea6 100644 --- a/elk/run.py +++ b/elk/run.py @@ -110,7 +110,7 @@ def execute( devices = select_usable_devices(self.num_gpus, min_memory=self.min_gpu_mem) num_devices = len(devices) - func: Callable[[int], dict[str, pd.DataFrame]] = partial( + func: Callable[[int], LayerApplied] = partial( self.apply_to_layer, devices=devices, world_size=num_devices ) self.apply_to_layers(func=func, num_devices=num_devices) @@ -167,7 +167,7 @@ def concatenate(self, layers): def apply_to_layers( self, - func: Callable[[int], dict[str, pd.DataFrame]], + func: Callable[[int], LayerApplied], num_devices: int, ): """Apply a function to each layer of the datasets in parallel @@ -192,11 +192,9 @@ def apply_to_layers( df_buffers = defaultdict(list) layer_outputs = [] try: - for df_dict, layer_output in tqdm( - mapper(func, layers), total=len(layers) - ): - layer_outputs.append(layer_output) - for k, v in df_dict.items(): # type: ignore + for res in tqdm(mapper(func, layers), total=len(layers)): + layer_outputs.append(res.layer_output) + for k, v in res.df_dict.items(): # type: ignore df_buffers[k].append(v) finally: # Make sure the CSVs are written even if we crash or get interrupted From bd06cd30ca7c1e99573dbaa324d82dca63fdbdfe Mon Sep 17 00:00:00 2001 From: jon Date: Thu, 27 Jul 2023 20:43:18 +0100 Subject: [PATCH 52/57] multidataset layer ensembling --- elk/evaluation/evaluate.py | 12 ++++----- elk/metrics/eval.py | 47 ++++++++++++++++++++--------------- elk/run.py | 50 +++++++++++++++++++++++--------------- elk/training/train.py | 19 +++++++++++---- 4 files changed, 77 insertions(+), 51 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index fd792b29..32ba25dd 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -8,7 +8,7 @@ from ..files import elk_reporter_dir from ..metrics import evaluate_preds -from ..run import LayerApplied, Run +from ..run import LayerApplied, LayerOutput, Run from ..utils import Color from ..utils.types import PromptEnsembling @@ -44,14 +44,14 @@ def apply_to_layer( row_bufs = defaultdict(list) - layer_output: list[dict] = [] + layer_outputs: list[LayerOutput] = [] + for ds_name, (val_h, val_gt, val_lm_preds) in val_output.items(): meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - layer_output.append( - {**meta, "val_gt": val_gt, "val_credences": val_credences} - ) + + layer_outputs.append(LayerOutput(val_gt, val_credences, meta)) for prompt_ensembling in PromptEnsembling.all(): row_bufs["eval"].append( { @@ -94,5 +94,5 @@ def apply_to_layer( } ) return LayerApplied( - layer_output, {k: pd.DataFrame(v) for k, v in row_bufs.items()} + layer_outputs, {k: pd.DataFrame(v) for k, v in row_bufs.items()} ) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 0a7cd0b3..b3b9fe60 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -10,6 +10,13 @@ from .roc_auc import RocAucResult, roc_auc_ci +@dataclass +class LayerOutput: + val_gt: Tensor + val_credences: Tensor + meta: dict + + @dataclass(frozen=True) class EvalResult: """The result of evaluating a classifier.""" @@ -202,15 +209,30 @@ def calc_eval_results( return EvalResult(acc, cal_acc, cal_err, auroc) +def to_one_hot(labels: Tensor, n_classes: int) -> Tensor: + """ + Convert a tensor of class labels to a one-hot representation. + + Args: + labels (Tensor): A tensor of class labels of shape (N,). + n_classes (int): The total number of unique classes. + + Returns: + Tensor: A one-hot representation tensor of shape (N, n_classes). + """ + one_hot_labels = labels.new_zeros(*labels.shape, n_classes) + return one_hot_labels.scatter_(-1, labels.unsqueeze(-1).long(), 1) + + def layer_ensembling( - layer_outputs: list, prompt_ensembling: PromptEnsembling + layer_outputs: list[LayerOutput], prompt_ensembling: PromptEnsembling ) -> EvalResult: """ Return EvalResult after prompt_ensembling the probe output of the middle to last layers Args: - layer_outputs: A list of dictionaries containing the ground truth and + layer_outputs: A list of LayerOutput containing the ground truth and predicted class tensor of shape (n, num_variants, num_classes). prompt_ensembling: The prompt_ensembling mode. @@ -222,14 +244,14 @@ def layer_ensembling( y_logits_collection = [] num_classes = 2 - y_true = layer_outputs[0][0]["val_gt"].to(device) + y_true = layer_outputs[0].val_gt.to(device) for layer_output in layer_outputs: # all y_trues are identical, so just get the first - y_logits = layer_output[0]["val_credences"].to(device) + y_logits = layer_output.val_credences.to(device) y_logits, y_true, num_classes = prepare( y_logits=y_logits, - y_true=layer_outputs[0][0]["val_gt"].to(device), + y_true=layer_outputs[0].val_gt.to(device), prompt_ensembling=prompt_ensembling, ) y_logits_collection.append(y_logits) @@ -246,18 +268,3 @@ def layer_ensembling( prompt_ensembling=prompt_ensembling, num_classes=num_classes, ) - - -def to_one_hot(labels: Tensor, n_classes: int) -> Tensor: - """ - Convert a tensor of class labels to a one-hot representation. - - Args: - labels (Tensor): A tensor of class labels of shape (N,). - n_classes (int): The total number of unique classes. - - Returns: - Tensor: A one-hot representation tensor of shape (N, n_classes). - """ - one_hot_labels = labels.new_zeros(*labels.shape, n_classes) - return one_hot_labels.scatter_(-1, labels.unsqueeze(-1).long(), 1) diff --git a/elk/run.py b/elk/run.py index 53d24ea6..0b1fccf0 100644 --- a/elk/run.py +++ b/elk/run.py @@ -17,12 +17,11 @@ from torch import Tensor from tqdm import tqdm -from elk.metrics.eval import layer_ensembling - from .debug_logging import save_debug_log from .extraction import Extract, extract from .extraction.dataset_name import DatasetDictWithName from .files import elk_reporter_dir, memorably_named_dir +from .metrics.eval import LayerOutput, layer_ensembling from .utils import ( Color, assert_type, @@ -36,7 +35,7 @@ @dataclass(frozen=True) class LayerApplied: - layer_output: list[dict] + layer_outputs: list[LayerOutput] """The output of the reporter on the layer, should contain credences and ground truth labels.""" df_dict: dict[str, pd.DataFrame] @@ -190,37 +189,48 @@ def apply_to_layers( with ctx.Pool(num_devices) as pool: mapper = pool.imap_unordered if num_devices > 1 else map df_buffers = defaultdict(list) - layer_outputs = [] + layer_outputs: list[LayerOutput] = [] try: for res in tqdm(mapper(func, layers), total=len(layers)): - layer_outputs.append(res.layer_output) + layer_outputs.extend(res.layer_outputs) for k, v in res.df_dict.items(): # type: ignore df_buffers[k].append(v) finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): - df = pd.concat(dfs).sort_values(by=["layer", "prompt_ensembling"]) + PROMPT_ENSEMBLING = "prompt_ensembling" + df = pd.concat(dfs).sort_values(by=["layer", PROMPT_ENSEMBLING]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: save_debug_log(self.datasets, self.out_dir) dfs = [] - - for prompt_ensembling in PromptEnsembling.all(): - layer_ensembling_results = layer_ensembling( - layer_outputs=layer_outputs, prompt_ensembling=prompt_ensembling - ) - df = pd.DataFrame(layer_ensembling_results.to_dict(), index=[0]) - df = df.round(4) - df["prompt_ensembling"] = prompt_ensembling.value - dfs.append(df) + # groupby layer_outputs by their dataset name + grouped_layer_outputs = {} + # Group the LayerOutput objects by dataset name + for layer_output in layer_outputs: + dataset_name = layer_output.meta["dataset"] + if dataset_name in grouped_layer_outputs: + grouped_layer_outputs[dataset_name].append(layer_output) + else: + grouped_layer_outputs[dataset_name] = [layer_output] + + for dataset_name, layer_outputs in grouped_layer_outputs.items(): + for prompt_ensembling in PromptEnsembling.all(): + res = layer_ensembling( + layer_outputs=layer_outputs, + prompt_ensembling=prompt_ensembling, + ) + df = pd.DataFrame(res.to_dict(), index=[0]) + df = df.round(4) + df[PROMPT_ENSEMBLING] = prompt_ensembling.value + df["dataset"] = dataset_name + dfs.append(df) df_concat = pd.concat(dfs) # Rearrange the columns so that prompt_ensembling is in front - columns = ["prompt_ensembling"] + [ - col for col in df_concat.columns if col != "prompt_ensembling" + columns = [PROMPT_ENSEMBLING] + [ + col for col in df_concat.columns if col != PROMPT_ENSEMBLING ] df_concat = df_concat[columns] - df_concat.to_csv( - self.out_dir / "layer_ensembling_results.csv", index=False - ) + df_concat.to_csv(self.out_dir / "layer_ensembling.csv", index=False) diff --git a/elk/training/train.py b/elk/training/train.py index b57eef81..78dad491 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -12,6 +12,7 @@ from simple_parsing.helpers.serialization import save from ..metrics import evaluate_preds, to_one_hot +from ..metrics.eval import LayerOutput from ..run import LayerApplied, Run from ..training.supervised import train_supervised from ..utils.types import PromptEnsembling @@ -139,12 +140,20 @@ def apply_to_layer( val_credences = reporter(val_h) + # layer_output.append( + # { + # **meta, + # "val_gt": val_gt.detach(), + # "val_credences": val_credences.detach(), + # } + # ) + # Using the class layer_output.append( - { - **meta, - "val_gt": val_gt.detach(), - "val_credences": val_credences.detach(), - } + LayerOutput( + val_gt=val_gt.detach(), + val_credences=val_credences.detach(), + meta=meta, + ) ) train_credences = reporter(train_h) From f8882c6c982791737d53147beeecc2c0ebce8c52 Mon Sep 17 00:00:00 2001 From: jon Date: Thu, 27 Jul 2023 21:08:34 +0100 Subject: [PATCH 53/57] little refactoring sorting remove comment --- elk/run.py | 67 +++++++++++++++++++++++-------------------- elk/training/train.py | 9 ------ 2 files changed, 36 insertions(+), 40 deletions(-) diff --git a/elk/run.py b/elk/run.py index 0b1fccf0..86c85428 100644 --- a/elk/run.py +++ b/elk/run.py @@ -32,6 +32,8 @@ ) from .utils.types import PromptEnsembling +PROMPT_ENSEMBLING = "prompt_ensembling" + @dataclass(frozen=True) class LayerApplied: @@ -42,6 +44,36 @@ class LayerApplied: """The evaluation results for the layer.""" +def calculate_layer_outputs(layer_outputs, out_path): + grouped_layer_outputs = {} + for layer_output in layer_outputs: + dataset_name = layer_output.meta["dataset"] + if dataset_name in grouped_layer_outputs: + grouped_layer_outputs[dataset_name].append(layer_output) + else: + grouped_layer_outputs[dataset_name] = [layer_output] + + dfs = [] + for dataset_name, layer_outputs in grouped_layer_outputs.items(): + for prompt_ensembling in PromptEnsembling.all(): + res = layer_ensembling( + layer_outputs=layer_outputs, + prompt_ensembling=prompt_ensembling, + ) + df = pd.DataFrame( + { + "dataset": dataset_name, + PROMPT_ENSEMBLING: prompt_ensembling.value, + **res.to_dict(), + }, + index=[0], + ).round(4) + dfs.append(df) + + df_concat = pd.concat(dfs) + df_concat.to_csv(out_path, index=False) + + @dataclass class Run(ABC, Serializable): data: Extract @@ -198,39 +230,12 @@ def apply_to_layers( finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): - PROMPT_ENSEMBLING = "prompt_ensembling" df = pd.concat(dfs).sort_values(by=["layer", PROMPT_ENSEMBLING]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: save_debug_log(self.datasets, self.out_dir) - dfs = [] - # groupby layer_outputs by their dataset name - grouped_layer_outputs = {} - # Group the LayerOutput objects by dataset name - for layer_output in layer_outputs: - dataset_name = layer_output.meta["dataset"] - if dataset_name in grouped_layer_outputs: - grouped_layer_outputs[dataset_name].append(layer_output) - else: - grouped_layer_outputs[dataset_name] = [layer_output] - - for dataset_name, layer_outputs in grouped_layer_outputs.items(): - for prompt_ensembling in PromptEnsembling.all(): - res = layer_ensembling( - layer_outputs=layer_outputs, - prompt_ensembling=prompt_ensembling, - ) - df = pd.DataFrame(res.to_dict(), index=[0]) - df = df.round(4) - df[PROMPT_ENSEMBLING] = prompt_ensembling.value - df["dataset"] = dataset_name - dfs.append(df) - - df_concat = pd.concat(dfs) - # Rearrange the columns so that prompt_ensembling is in front - columns = [PROMPT_ENSEMBLING] + [ - col for col in df_concat.columns if col != PROMPT_ENSEMBLING - ] - df_concat = df_concat[columns] - df_concat.to_csv(self.out_dir / "layer_ensembling.csv", index=False) + calculate_layer_outputs( + layer_outputs=layer_outputs, + out_path=self.out_dir / "layer_ensembling.csv", + ) diff --git a/elk/training/train.py b/elk/training/train.py index 78dad491..34be7e78 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -139,15 +139,6 @@ def apply_to_layer( meta = {"dataset": ds_name, "layer": layer} val_credences = reporter(val_h) - - # layer_output.append( - # { - # **meta, - # "val_gt": val_gt.detach(), - # "val_credences": val_credences.detach(), - # } - # ) - # Using the class layer_output.append( LayerOutput( val_gt=val_gt.detach(), From 23183bcdb034654fa8c1ddce25af6b5c18831b69 Mon Sep 17 00:00:00 2001 From: jon Date: Thu, 27 Jul 2023 21:17:38 +0100 Subject: [PATCH 54/57] fix tests --- elk/evaluation/evaluate.py | 8 +++++--- elk/run.py | 1 + tests/test_smoke_elicit.py | 4 ++-- tests/test_smoke_eval.py | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py index 32ba25dd..a74dca26 100644 --- a/elk/evaluation/evaluate.py +++ b/elk/evaluation/evaluate.py @@ -12,6 +12,8 @@ from ..utils import Color from ..utils.types import PromptEnsembling +PROMPT_ENSEMBLING = "prompt_ensembling" + @dataclass(kw_only=True) class Eval(Run): @@ -56,7 +58,7 @@ def apply_to_layer( row_bufs["eval"].append( { **meta, - "prompt_ensembling": prompt_ensembling.value, + PROMPT_ENSEMBLING: prompt_ensembling.value, **evaluate_preds( val_gt, val_credences, prompt_ensembling ).to_dict(), @@ -67,7 +69,7 @@ def apply_to_layer( row_bufs["lm_eval"].append( { **meta, - "ensembling": prompt_ensembling.value, + PROMPT_ENSEMBLING: prompt_ensembling.value, **evaluate_preds( val_gt, val_lm_preds, prompt_ensembling ).to_dict(), @@ -85,7 +87,7 @@ def apply_to_layer( model.eval() row_bufs["lr_eval"].append( { - "prompt_ensembling": prompt_ensembling.value, + PROMPT_ENSEMBLING: prompt_ensembling.value, "inlp_iter": i, **meta, **evaluate_preds( diff --git a/elk/run.py b/elk/run.py index 86c85428..f96b72ad 100644 --- a/elk/run.py +++ b/elk/run.py @@ -230,6 +230,7 @@ def apply_to_layers( finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): + print(dfs[0].columns) df = pd.concat(dfs).sort_values(by=["layer", PROMPT_ENSEMBLING]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: diff --git a/tests/test_smoke_elicit.py b/tests/test_smoke_elicit.py index f9db114c..4ae8b22b 100644 --- a/tests/test_smoke_elicit.py +++ b/tests/test_smoke_elicit.py @@ -31,7 +31,7 @@ def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path): "lr_models", "reporters", "eval.csv", - "layer_ensembling_results.csv", + "layer_ensembling.csv", ] for file in expected_files: assert file in created_file_names @@ -63,7 +63,7 @@ def test_smoke_elicit_run_tiny_gpt2_eigen(tmp_path: Path): "lr_models", "reporters", "eval.csv", - "layer_ensembling_results.csv", + "layer_ensembling.csv", ] for file in expected_files: assert file in created_file_names diff --git a/tests/test_smoke_eval.py b/tests/test_smoke_eval.py index 45324faa..7f0bad7e 100644 --- a/tests/test_smoke_eval.py +++ b/tests/test_smoke_eval.py @@ -11,7 +11,7 @@ "cfg.yaml", "fingerprints.yaml", "eval.csv", - "layer_ensembling_results.csv", + "layer_ensembling.csv", ] From d091f9d6695299bf797fd2c80385cc4956917593 Mon Sep 17 00:00:00 2001 From: Walter Laurito Date: Mon, 31 Jul 2023 19:02:40 +0000 Subject: [PATCH 55/57] add annotation + cleanup --- elk/run.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/elk/run.py b/elk/run.py index f96b72ad..25e8fb71 100644 --- a/elk/run.py +++ b/elk/run.py @@ -44,7 +44,15 @@ class LayerApplied: """The evaluation results for the layer.""" -def calculate_layer_outputs(layer_outputs, out_path): +def calculate_layer_outputs(layer_outputs: list[LayerOutput], out_path: Path): + """ + Calculate the layer ensembling results for each dataset + and prompt ensembling and save them to a CSV file. + + Args: + layer_outputs: The layer outputs to calculate the results for. + out_path: The path to save the results to. + """ grouped_layer_outputs = {} for layer_output in layer_outputs: dataset_name = layer_output.meta["dataset"] @@ -230,7 +238,6 @@ def apply_to_layers( finally: # Make sure the CSVs are written even if we crash or get interrupted for name, dfs in df_buffers.items(): - print(dfs[0].columns) df = pd.concat(dfs).sort_values(by=["layer", PROMPT_ENSEMBLING]) df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False) if self.debug: From 776c1863be0ed4ede51942843ee031cafd648346 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 19:04:37 +0000 Subject: [PATCH 56/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- elk/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elk/run.py b/elk/run.py index 25e8fb71..55c44954 100644 --- a/elk/run.py +++ b/elk/run.py @@ -46,9 +46,9 @@ class LayerApplied: def calculate_layer_outputs(layer_outputs: list[LayerOutput], out_path: Path): """ - Calculate the layer ensembling results for each dataset + Calculate the layer ensembling results for each dataset and prompt ensembling and save them to a CSV file. - + Args: layer_outputs: The layer outputs to calculate the results for. out_path: The path to save the results to. From 64e762ab5b72a93f223dfa1d576372e3ad2c93f6 Mon Sep 17 00:00:00 2001 From: jon Date: Fri, 13 Oct 2023 18:44:42 +0200 Subject: [PATCH 57/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci fix merge --- elk/metrics/accuracy.py | 5 ++++- elk/metrics/eval.py | 18 +++++++++++------- pyproject.toml | 4 ++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/elk/metrics/accuracy.py b/elk/metrics/accuracy.py index 33b94632..2f9685f8 100644 --- a/elk/metrics/accuracy.py +++ b/elk/metrics/accuracy.py @@ -14,11 +14,14 @@ class AccuracyResult: """Lower bound of the confidence interval.""" upper: float """Upper bound of the confidence interval.""" + cal_thresh: float | None + """The threshold used to compute the calibrated accuracy.""" def accuracy_ci( y_true: Tensor, y_pred: Tensor, + cal_thresh: float | None = None, *, num_samples: int = 1000, level: float = 0.95, @@ -79,4 +82,4 @@ def accuracy_ci( # Compute the point estimate. Call flatten to ensure that we get a single number # computed across cluster boundaries even if the inputs were clustered. estimate = y_true.flatten().eq(y_pred.flatten()).float().mean().item() - return AccuracyResult(estimate, lower, upper) + return AccuracyResult(estimate, lower, upper, cal_thresh) diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py index 26d3c512..c77dd09e 100644 --- a/elk/metrics/eval.py +++ b/elk/metrics/eval.py @@ -33,7 +33,7 @@ class EvalResult: cal_thresh: float | None """The threshold used to compute the calibrated accuracy.""" - def to_dict(self, prefix: str = "") -> dict[str, float]: + def to_dict(self, prefix: str = "") -> dict[str, float | None]: """Convert the result to a dictionary.""" acc_dict = {f"{prefix}acc_{k}": v for k, v in asdict(self.accuracy).items()} cal_acc_dict = ( @@ -89,6 +89,7 @@ def calc_auroc( return auroc + def calc_calibrated_accuracies(y_true, pos_probs) -> AccuracyResult: """ Calculate the calibrated accuracies @@ -101,11 +102,12 @@ def calc_calibrated_accuracies(y_true, pos_probs) -> AccuracyResult: AccuracyResult: A dictionary containing the accuracy and confidence interval. """ - cal_thresh = pos_probs.float().quantile(y_true.float().mean()) + cal_thresh = pos_probs.float().quantile(y_true.float().mean()).item() cal_preds = pos_probs.gt(cal_thresh).to(torch.int) - cal_acc = accuracy_ci(y_true, cal_preds) + cal_acc = accuracy_ci(y_true, cal_preds, cal_thresh) return cal_acc + def calc_calibrated_errors(y_true, pos_probs) -> CalibrationEstimate: """ Calculate the expected calibration error. @@ -122,6 +124,7 @@ def calc_calibrated_errors(y_true, pos_probs) -> CalibrationEstimate: cal_err = cal.compute() return cal_err + def calc_accuracies(y_logits, y_true) -> AccuracyResult: """ Calculate the accuracy @@ -193,10 +196,11 @@ def calc_eval_results( acc = calc_accuracies(y_logits=y_logits, y_true=y_true) pos_probs = torch.sigmoid(y_logits[..., 1] - y_logits[..., 0]) - cal_acc = ( + cal_acc, cal_thresh = ( calc_calibrated_accuracies(y_true=y_true, pos_probs=pos_probs) if num_classes == 2 - else None + else None, + None, ) cal_err = ( calc_calibrated_errors(y_true=y_true, pos_probs=pos_probs) @@ -207,11 +211,11 @@ def calc_eval_results( auroc = calc_auroc( y_logits=y_logits, y_true=y_true, - prompt_ensembling=prompt_ensembling, + ensembling=prompt_ensembling, num_classes=num_classes, ) - return EvalResult(acc, cal_acc, cal_err, auroc) + return EvalResult(acc, cal_acc, cal_err, auroc, cal_thresh) def to_one_hot(labels: Tensor, n_classes: int) -> Tensor: diff --git a/pyproject.toml b/pyproject.toml index f3f16504..b0a078cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ # We upstreamed bugfixes for Literal types in 0.1.1 "simple-parsing>=0.1.1", # Version 1.11 introduced Fully Sharded Data Parallel, which we plan to use soon - "torch>=1.11.0", + "torch==2.0", # Doesn't really matter but versions < 4.0 are very very old (pre-2016) "tqdm>=4.0.0", # 4.0 introduced the breaking change of using return_dict=True by default @@ -37,7 +37,7 @@ dependencies = [ # For visualization of results "plotly==5.14.1", "kaleido==0.2.1", - "rich==13.3.5" + "rich" ] version = "0.1.1"