diff --git a/LanguageModeling/BERT/config.py b/LanguageModeling/BERT/config.py index df45fef..79ec270 100644 --- a/LanguageModeling/BERT/config.py +++ b/LanguageModeling/BERT/config.py @@ -58,6 +58,10 @@ def get_parser(parser=None): help='use use fp16 or not') parser.add_argument('--use_xla', type=str2bool, nargs='?', const=True, help='Whether to use use xla') + parser.add_argument("--num_accumulation_steps", type=int, default=1, + help='Number of accumulation steps before gradient update, Global batch size = num_accumulation_steps * train_batch_size') + parser.add_argument("--optimizer_type", type=str, default="adam", + help="Optimizer used for training - LAMB or ADAM") # log and resore/save parser.add_argument("--loss_print_every_n_iter", type=int, default=10, required=False, diff --git a/LanguageModeling/BERT/run_pretraining.py b/LanguageModeling/BERT/run_pretraining.py index 0b04de6..3155317 100755 --- a/LanguageModeling/BERT/run_pretraining.py +++ b/LanguageModeling/BERT/run_pretraining.py @@ -29,9 +29,26 @@ parser.add_argument("--data_part_num", type=int, default=32, help="data part number in dataset") parser.add_argument("--iter_num", type=int, default=1144000, help="total iterations to run") parser.add_argument("--batch_size_per_device", type=int, default=64) +parser.add_argument("--debug", type=int, default=0) +parser.add_argument("--data_load_random", type=int, default=1) +parser.add_argument("--model_load", type=str, default=None) + + args = parser.parse_args() configs.print_args(args) + +if args.debug == 1: + flow.config.enable_debug_mode(True) + print('Enable Debug !!!!!!!') + +if args.data_load_random == 1: + random_tmp=True + print('Enable random loading of data !!!!!!!') +else: + random_tmp=False + print('Disable random loading of data !!!!!!!') + batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device @@ -39,8 +56,8 @@ def BertDecoder(data_dir, batch_size, data_part_num, seq_length, max_predictions ofrecord = flow.data.ofrecord_reader(data_dir, batch_size=batch_size, data_part_num=data_part_num, - random_shuffle = True, - shuffle_after_epoch=True) + random_shuffle = random_tmp, + shuffle_after_epoch=random_tmp) blob_confs = {} def _blob_conf(name, shape, dtype=flow.int32): blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) @@ -104,8 +121,13 @@ def main(): snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + + if args.model_load != None: + flow.load_variables(flow.checkpoint.get(args.model_load)) + + print('num_accumulation_steps:', args.num_accumulation_steps) metric = Metric(desc='train', print_steps=args.loss_print_every_n_iter, - batch_size=batch_size, keys=['total_loss', 'mlm_loss', 'nsp_loss']) + batch_size=batch_size * args.num_accumulation_steps, keys=['total_loss', 'mlm_loss', 'nsp_loss']) for step in range(args.iter_num): PretrainJob().async_get(metric.metric_cb(step)) #PretrainJob().async_get(metric.metric_cb(step, epoch=3)) diff --git a/LanguageModeling/BERT/run_pretraining_adam.sh b/LanguageModeling/BERT/run_pretraining_adam.sh new file mode 100644 index 0000000..a2bc89e --- /dev/null +++ b/LanguageModeling/BERT/run_pretraining_adam.sh @@ -0,0 +1,46 @@ +BENCH_ROOT_DIR=/path/to/OneFlow-Benchmark/LanguageModeling/BERT +OUTPUT_DIR=/DATA/disk1/of_output + +DATA_DIR=/DATA/disk1/bert/wiki_seq_len_128 + + +BZ=48 +ITER_NUM=1000000 +max_seq_length=128 +max_predictions_per_seq=20 + +of_log_dir=$OUTPUT_DIR/bert_master/of +rm -rf ${of_log_dir} +mkdir -p ${of_log_dir} +rm -rf core.* + +export PYTHONUNBUFFERED=1 +export ONEFLOW_DEBUG_MODE=True +export GLOG_v=3 +export CUDA_VISIBLE_DEVICES=6 +python3 $BENCH_ROOT_DIR/run_pretraining.py \ + --gpu_num_per_node=1 \ + --num_nodes=1 \ + --learning_rate=1.25e-5 \ + --warmup_proportion=0.01 \ + --weight_decay_rate=0.01 \ + --batch_size_per_device=${BZ} \ + --iter_num=${ITER_NUM} \ + --loss_print_every_n_iter=1 \ + --seq_length=128 \ + --use_fp16 \ + --max_predictions_per_seq=20 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --num_accumulation_steps=1 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ + --data_part_num=64 \ + --data_dir=$DATA_DIR \ + --log_dir=${of_log_dir} \ + --model_save_every_n_iter=50000 \ + --model_save_dir=${of_log_dir} diff --git a/LanguageModeling/BERT/run_pretraining_lamb.sh b/LanguageModeling/BERT/run_pretraining_lamb.sh new file mode 100644 index 0000000..a8ffd8b --- /dev/null +++ b/LanguageModeling/BERT/run_pretraining_lamb.sh @@ -0,0 +1,47 @@ +BENCH_ROOT_DIR=/path/to/OneFlow-Benchmark/LanguageModeling/BERT +OUTPUT_DIR=/DATA/disk1/of_output + +DATA_DIR=/DATA/disk1/bert/wiki_seq_len_128 + + +BZ=16 +ITER_NUM=1000000 +max_seq_length=128 +max_predictions_per_seq=20 + +of_log_dir=$OUTPUT_DIR/bert_master/of +rm -rf ${of_log_dir} +mkdir -p ${of_log_dir} +rm -rf core.* + +export PYTHONUNBUFFERED=1 +export ONEFLOW_DEBUG_MODE=True +export GLOG_v=3 + +python3 $BENCH_ROOT_DIR/run_pretraining.py \ + --gpu_num_per_node=8 \ + --num_nodes=1 \ + --learning_rate=1e-4 \ + --warmup_proportion=0.01 \ + --weight_decay_rate=0.01 \ + --batch_size_per_device=${BZ} \ + --iter_num=${ITER_NUM} \ + --loss_print_every_n_iter=1 \ + --seq_length=128 \ + --use_fp16 \ + --optimizer_type="lamb" \ + --max_predictions_per_seq=20 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --num_accumulation_steps=512 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ + --data_part_num=64 \ + --data_dir=$DATA_DIR \ + --log_dir=${of_log_dir} \ + --model_save_every_n_iter=50000 \ + --model_save_dir=${of_log_dir} diff --git a/LanguageModeling/BERT/run_squad.sh b/LanguageModeling/BERT/run_squad.sh new file mode 100644 index 0000000..20ade70 --- /dev/null +++ b/LanguageModeling/BERT/run_squad.sh @@ -0,0 +1,74 @@ +BENCH_ROOT_DIR=/home/oyy/workspace/OneFlow-Benchmark/LanguageModeling/BERT +# pretrained model dir +PRETRAINED_MODEL=/DATA/disk1/of_output/uncased_L-12_H-768_A-12_oneflow + +# squad ofrecord dataset dir +DATA_ROOT=/DATA/disk1/of_output/bert/of_squad + +# `vocab.txt` dir +REF_ROOT_DIR=/DATA/disk1/of_output/uncased_L-12_H-768_A-12 + +# `evaluate-v*.py` and `dev-v*.json` dir +SQUAD_TOOL_DIR=/DATA/disk1/of_output/bert/of_squad +db_version=${1:-"v2.0"} +if [ $db_version = "v1.1" ]; then + train_example_num=88614 + eval_example_num=10833 + version_2_with_negative="False" +elif [ $db_version = "v2.0" ]; then + train_example_num=131944 + eval_example_num=12232 + version_2_with_negative="True" +else + echo "db_version must be 'v1.1' or 'v2.0'" + exit +fi + +train_data_dir=$DATA_ROOT/train-$db_version +eval_data_dir=$DATA_ROOT/dev-$db_version +LOGFILE=./bert_fp_training.log +export PYTHONUNBUFFERED=1 +export ONEFLOW_DEBUG_MODE=True +export CUDA_VISIBLE_DEVICES=7 +# finetune and eval SQuAD, +# `predictions.json` will be saved to folder `./squad_output` +python3 $BENCH_ROOT_DIR/run_squad.py \ + --model=SQuAD \ + --do_train=True \ + --do_eval=True \ + --gpu_num_per_node=1 \ + --learning_rate=3e-5 \ + --batch_size_per_device=16 \ + --eval_batch_size_per_device=16 \ + --num_epoch=3 \ + --use_fp16 \ + --version_2_with_negative=$version_2_with_negative \ + --loss_print_every_n_iter=20 \ + --do_lower_case=True \ + --seq_length=384 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --log_dir=./log \ + --model_load_dir=${PRETRAINED_MODEL} \ + --save_last_snapshot=True \ + --model_save_dir=./squad_snapshots \ + --vocab_file=$REF_ROOT_DIR/vocab.txt \ + --predict_file=$SQUAD_TOOL_DIR/dev-${db_version}.json \ + --output_dir=./squad_output 2>&1 | tee ${LOGFILE} + + +# evaluate predictions.json to get metrics +python3 $SQUAD_TOOL_DIR/evaluate-${db_version}.py \ + $SQUAD_TOOL_DIR/dev-${db_version}.json \ + ./squad_output/predictions.json + diff --git a/LanguageModeling/BERT/tools/README.md b/LanguageModeling/BERT/tools/README.md new file mode 100644 index 0000000..3a9baa7 --- /dev/null +++ b/LanguageModeling/BERT/tools/README.md @@ -0,0 +1,12 @@ +## Oneflow BERT automatic test tools +Automatically execute bert modle with different parameters and analyze the results +### Dependent environment +- matplotlib + ``` + pip install matplotlib + ``` + +### Features +- `analysis.py`,Analyze the log file to get the total loss mlm loss nsp loos through GPU memory +- `result_analysis.py`,Analyze the running results of the two versions and output reports +- `stitching_pic.py` Multiple pictures are spliced ​​together diff --git a/LanguageModeling/BERT/tools/analysis.py b/LanguageModeling/BERT/tools/analysis.py new file mode 100644 index 0000000..c25b342 --- /dev/null +++ b/LanguageModeling/BERT/tools/analysis.py @@ -0,0 +1,63 @@ +import argparse +import re +import json + +from ctypes import * + + + +def collect_loss(log_file, gpu_num): + print("loss : ",log_file) + + f = open(log_file,"r") + lines = f.readlines() + total_loss = [] + mlm_loss = [] + nsp_loss = [] + throughput =[] + memory=[] + + pattern = re.compile(r"step:\s*(\d+)\s*,\s*total_loss:\s*(\d+\.?\d+)\s*,\s*mlm_loss:\s*(\d+\.?\d+)\s*,\s*nsp_loss:\s*(\d+\.?\d+)\s*,\s*throughput:\s*(\d+\.?\d+)\s*") + for line in lines: + if(line.split(':')[0] == 'step'): + + match = pattern.match(line) + if match: + total_loss.append(match.group(2)) + mlm_loss.append(match.group(3)) + nsp_loss.append(match.group(4)) + throughput.append(match.group(5)) + if(line.split(' [MiB]\\n')[0] == 'b\'memory.used'): + str_tmp = line.split(' [MiB]\\n')[1] + + for i in range(gpu_num): + memory.append(str_tmp.split(' MiB\\n')[i]) + + return total_loss, mlm_loss, nsp_loss,throughput, memory + + +def main(): + parser = argparse.ArgumentParser(description="collect GPU device memory usage") + parser.add_argument("--log_file", type=str, default=None) + parser.add_argument("--mem_file", type=str, default=None) + parser.add_argument("--out_file", type=str, default=None) + parser.add_argument("--gpu_num", type=int, default=1) + + args = parser.parse_args() + + total_loss, mlm_loss, nsp_loss,throughput, memory = collect_loss(args.log_file, args.gpu_num) + + out={} + out['total_loss'] = total_loss + out['mlm_loss'] = mlm_loss + out['nsp_loss'] = nsp_loss + out['throughput'] = throughput + out['memory'] = memory + + string = json.dumps(out) + with open(args.out_file,'w')as f: + f.write(string) + + +if __name__ == "__main__": + main() diff --git a/LanguageModeling/BERT/tools/gpu_memory_usage.py b/LanguageModeling/BERT/tools/gpu_memory_usage.py new file mode 100644 index 0000000..6f3daa8 --- /dev/null +++ b/LanguageModeling/BERT/tools/gpu_memory_usage.py @@ -0,0 +1,63 @@ +import time +import argparse +import pynvml + + +class Device(object): + class Status: + INIT = "INIT" + DETECTING = "DETECTING" + STOP = "STOP" + + start_detecting_mem_threshold = 32 * 1024 * 1024 + + def __init__(self, handle): + self.handle = handle + self.status = self.Status.INIT + self.max_mem_usage = 0 + + def update(self): + info = pynvml.nvmlDeviceGetMemoryInfo(self.handle) + if self.status == self.Status.INIT: + if info.used > self.start_detecting_mem_threshold: + self.status = self.Status.DETECTING + elif self.status == self.Status.DETECTING: + if info.used < self.start_detecting_mem_threshold: + self.status = self.Status.STOP + return False + else: + self.max_mem_usage = max(self.max_mem_usage, info.used) + elif self.status == self.Status.STOP: + raise ValueError("detecting is stop") + else: + raise ValueError("invalid status") + + return True + + +def main(): + parser = argparse.ArgumentParser(description="collect GPU device memory usage") + parser.add_argument("-g", type=int, default=1, help="number of gpu devices") + parser.add_argument("-n", type=float, default=1, help="metrics rate") + args = parser.parse_args() + + pynvml.nvmlInit() + n_gpus = args.g + devices = [Device(pynvml.nvmlDeviceGetHandleByIndex(i)) for i in range(n_gpus)] + + running = True + while running: + time.sleep(args.n) + running = False + for device in devices: + running |= device.update() + + pynvml.nvmlShutdown() + for i, device in enumerate(devices): + max_mem_usage_mbytes = device.max_mem_usage / 1024 / 1024 + print(f"{max_mem_usage_mbytes:.2f}") + + +if __name__ == "__main__": + main() + diff --git a/LanguageModeling/BERT/tools/result_analysis.py b/LanguageModeling/BERT/tools/result_analysis.py new file mode 100644 index 0000000..fe8b988 --- /dev/null +++ b/LanguageModeling/BERT/tools/result_analysis.py @@ -0,0 +1,134 @@ +import json +import matplotlib.pyplot as plt +import argparse +import numpy as np + +def read_file(file1, file2): + + with open(file1,'r') as load_f: + dict1 = json.load(load_f) + + with open(file2,'r') as load_f: + dict2 = json.load(load_f) + + + return dict1, dict2 + +def analysis_loss(y3, f32): + + # Calculate the correlation coefficient + + # if f32 == 1: + # iter = len(y3) + + # else: + # len = y3.shape[0] + # min = y3.min() + # max = y3.max() + # mean = np.mean(y3) + # var = np.var(y3) + + # random = np.random.randn(len)/10000.0 + # print(random) + # print(y3) + # lst=[y3,random] + # res=np.corrcoef(lst) + # print('----->', res) + + len = y3.shape[0] + + if f32 == 1: + iter = np.count_nonzero(y3==0) + tmp = iter/len + print('count zeor = ', iter) + if iter/len > 0.6: + print('Test passed') + return 1 + else: + print('Test failed') + return 0 + else: + + mean = np.mean(y3) + var = np.var(y3) + print('F16---->', abs(mean), var) + + if abs(mean) < 0.001 and var < 0.00001: + print('Test passed') + return 1 + else: + print('Test failed') + return 0 + +def drawing_loss(dict1, dict2, f32, image): + + + m1 = dict1["memory"] + m2 = dict2["memory"] + + print(m1) + print(m2) + table_len = len(m1) + row_labels = ['old','new'] + table_vals = [m1,m2] + + + y1 = dict1["total_loss"] + y2 = dict2["total_loss"] + y1=list(map(float,y1)) + y2=list(map(float,y2)) + x = np.arange(1, len(y1)+1) + y1 = np.array(y1) + y2 = np.array(y2) + y3 = np.subtract(y1,y2) + + result = analysis_loss(y3, f32) + + print(x) + print(y1) + print(y2) + print(y3) + fig = plt.figure(figsize=(24,8), dpi=150) + plt.figure(1) + ax = fig.add_subplot() + + + ax1 = plt.subplot(121) + plt.xlabel('iterations') + plt.plot(x,y1,color='red',label='Diachronic version') + plt.plot(x,y2,color='blue',label='Current version') + plt.title('Loss comparison') + + plt.legend(loc='best') + + ax2 = plt.subplot(122) + plt.xlabel('iterations') + plt.plot(x,y3,color='red') + plt.title('Loss difference') + plt.table(cellText=table_vals, rowLabels=row_labels, colWidths=[0.05]*table_len, loc='best') + + plt.suptitle(image.split('/')[1].split('.')[0],fontsize=20,x=0.5,y=0.98) + + if result == 1: + plt.text(0.9, 1,'PASS', fontsize=50, color='blue', transform=ax.transAxes) + else: + plt.text(0.9, 1,'FAILED',fontsize=50,color='red',transform=ax.transAxes) + plt.savefig(image) + + +def main(): + print('test') + parser = argparse.ArgumentParser(description="Compare and analyze training results and output icons") + parser.add_argument("--cmp1_file", type=str, default=None) + parser.add_argument("--cmp2_file", type=str, default=None) + parser.add_argument("--out_file", type=str, default=None) + parser.add_argument("--f32", type=int, default=0) + args = parser.parse_args() + + dict1, dict2 = read_file(args.cmp1_file, args.cmp2_file) + + drawing_loss(dict1, dict2, args.f32, args.out_file) + + +if __name__ == "__main__": + main() diff --git a/LanguageModeling/BERT/tools/stitching_pic.py b/LanguageModeling/BERT/tools/stitching_pic.py new file mode 100644 index 0000000..c39f8f9 --- /dev/null +++ b/LanguageModeling/BERT/tools/stitching_pic.py @@ -0,0 +1,40 @@ +import matplotlib.pyplot as plt +import argparse +import os +from PIL import Image + +def stiiching_pic(dir, out_file): + + hight = 1200 + width = 3600 + file_lsit = os.listdir(dir) + target = Image.new('RGBA', (width, hight*len(file_lsit))) + left = 0 + right = hight + for file in file_lsit: + + tmp = dir+'/'+file + print(tmp) + image = Image.open(tmp) + # print(image) + # print(target) + target.paste(image, (0, left, width, right)) + left += hight + right += hight + target.save(out_file) + + +def main(): + print('test') + parser = argparse.ArgumentParser(description="Stitching pictures") + parser.add_argument("--dir", type=str, default=None) + parser.add_argument("--out_file", type=str, default=None) + args = parser.parse_args() + + stiiching_pic(args.dir, args.out_file) + + +if __name__ == "__main__": + main() + + \ No newline at end of file diff --git a/LanguageModeling/BERT/train_perbert.sh b/LanguageModeling/BERT/train_perbert.sh new file mode 100644 index 0000000..d5a80a5 --- /dev/null +++ b/LanguageModeling/BERT/train_perbert.sh @@ -0,0 +1,129 @@ +USE_FP16=${1:-0} +#DEBUG +DEBUG_MODE=${2:-0} +# +BATCH_SIZE=${3:-64} +#accumulation +ACCUMULATION_STEMPS=${4-1} +# +OPTIMIZER=${5-adam} +#GPU +GPUS_PER_NODE=${6:-8} +# +ITER_NUM=${7:-100} +# +PRINT_ITER=${8:-1} +# +NODE_RANK=${9:-0} + +LOG_FOLDER=./log/ + +########################################################################################################## +# FP +########################################################################################################## +echo ${USE_FP16} + + +if [ "$USE_FP16" = 1 ];then + FP_CMD=--use_fp16 + FP_NAME=f16 + echo "USE_FP16" +else + FP_CMD= + FP_NAME=f32 + echo "USE_FP32" +fi +########################################################################################################## +# DEBUG_NAME +########################################################################################################## +# if [ DEBUG_MODE==1 ];then +# DEBUG_NAME=debug +# else +# DEBUG_NAME= +# fi + +########################################################################################################## +# Create folder +########################################################################################################## +#bert_f32_pretraining_8gpu_64bs_100iter_lamb_debug +mkdir -p $LOG_FOLDER + +# OUTFILE=bert_pretraining_${FP_NAME}_${GPUS_PER_NODE}gpu_${BATCH_SIZE}bs_${ITER_NUM}iter_${OPTIMIZER}\ +# _${DEBUG_NAME} +# mkdir -p $OUTFILE + +LOGFILE=$LOG_FOLDER/bert_pretraining_${FP_NAME}_${GPUS_PER_NODE}gpu_${BATCH_SIZE}bs_${ITER_NUM}iter_${OPTIMIZER}\ +_${DEBUG_NAME}.log + +MODEL_DIR=./snapshots/ +# DATA_DIR=/DATA/disk1/bert/wiki_seq_len_128/ +DATA_DIR=/data/bert_dataset + +MEM_FILE=$LOG_FOLDER/memory.log + +echo LOGFILE=$LOGFILE +echo DATA_DIR=$DATA_DIR + +#${NNODES}n${GPUS_PER_NODE}g_dp${D_P}_mp${M_P}_pp${P_P}_mbs${MICRO_BATCH_SIZE}_gbs${GLOABAL_BATCH_SIZE}_pretrain_${NODE_RANK}.log +rm -rf ${MODEL_DIR}/* +rm -rf ${LOG_FOLDER}/* + +NVPROF=baseline-report_${NODE_RANK} +#-o ${NVPROF} + + +# -g $GPUS_PER_NODE \ +# -n 0.5 \ + + +#nsys profile --stats=true -o ${NVPROF} \ +python3 run_pretraining.py \ + --gpu_num_per_node=${GPUS_PER_NODE} \ + --num_nodes=1 \ + --learning_rate=1e-4 \ + --warmup_proportion=0.01 \ + --weight_decay_rate=0.01 \ + --batch_size_per_device=${BATCH_SIZE} \ + --iter_num=${ITER_NUM} \ + --loss_print_every_n_iter=${PRINT_ITER} \ + --seq_length=128 \ + --max_predictions_per_seq=20 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --num_accumulation_steps=${ACCUMULATION_STEMPS} \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0 \ + --hidden_dropout_prob=0 \ + --hidden_size_per_head=64 \ + --data_part_num=64 \ + --data_dir=$DATA_DIR \ + --log_dir=${LOG_FOLDER} \ + --model_save_every_n_iter=10000 \ + --save_last_snapshot=True \ + --model_save_dir=./snapshots \ + --debug=${DEBUG_MODE} \ + --data_load_random=0 \ + --model_load=/opt/initial_model \ + ${FP_CMD} \ + --optimizer_type=${OPTIMIZER} \ + 2>&1 | tee ${LOGFILE} + +echo "Writting log to ${LOGFILE}" + +SQLITE=$LOG_FOLDER/bert_pretraining_${GPUS_PER_NODE}gpu_${BATCH_SIZE}bs_${ITER_NUM}iter.sqlite +QDREP=$LOG_FOLDER/bert_pretraining_${GPUS_PER_NODE}gpu_${BATCH_SIZE}bs_${ITER_NUM}iter.qdrep + +# mv $NVPROF.sqlite $SQLITE +# mv $NVPROF.qdrep $QDREP + +json_file=${LOG_FOLDER}out.json +python tools/analysis.py \ + --log_file=$LOGFILE \ + --mem_file=$MEM_FILE \ + --out_file=$json_file \ + --gpu_num=$GPUS_PER_NODE + +# --use_fp16 \ + diff --git a/LanguageModeling/BERT/train_perbert_list.sh b/LanguageModeling/BERT/train_perbert_list.sh new file mode 100644 index 0000000..4417b4a --- /dev/null +++ b/LanguageModeling/BERT/train_perbert_list.sh @@ -0,0 +1,281 @@ +NUM=${2-1} + +file_op() +{ + mkdir -p $1 + mv -f log_f* $1 + + # tar -zcvf $1.tar.gz $1 + # rm -rf $1 +} +################################################################################# +rm -rf out +rm -rf pic +mkdir pic + + +# ############################################################################### +# # f32 adam +# ############################################################################### + +# for ((i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 0 0 64 1 adam +# cp -rf log/ log_f32_${i} +# done + +# file_op out/bert_f32_pretraining_8gpu_64bs_100iter + +# # mkdir bert_f32_pretraining_8gpu_64bs_100iter +# # mv -f log_f32_* bert_f32_pretraining_8gpu_64bs_100iter + +# # tar -zcvf bert_f32_pretraining_8gpu_64bs_100iter.tar.gz \ +# # bert_f32_pretraining_8gpu_64bs_100iter +# # rm -rf bert_f32_pretraining_8gpu_64bs_100iter +# ############################################################################### +# # f32 lamb +# ############################################################################### + +# for (( i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 0 0 64 1 lamb +# cp -rf log/ log_f32_${i} +# done + +# file_op out/bert_f32_pretraining_8gpu_64bs_100iter_lamb + +# # ############################################################################### +# # # f16 adam +# # ############################################################################### + +# for (( i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 1 0 64 1 adam +# cp -rf log/ log_f16_${i} +# done + +# file_op out/bert_f16_pretraining_8gpu_64bs_100iter + +# ############################################################################### +# # f16 lamb +# ############################################################################### + +# for (( i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 1 0 64 1 lamb +# cp -rf log/ log_f16_${i} +# done + +# file_op out/bert_f16_pretraining_8gpu_64bs_100iter_lamb + +# ############################################################################### +# # f32 accumulation adam +# ############################################################################### + +# for (( i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 0 0 32 2 adam +# cp -rf log/ log_f32_${i} +# done + +# file_op out/bert_f32_pretraining_8gpu_64bs_100iter_accumulation + +# ############################################################################### +# # f32 accumulation lamb +# ############################################################################### + +# for (( i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 0 0 32 2 lamb +# cp -rf log/ log_f32_${i} +# done + +# file_op out/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_lamb + +# ############################################################################### +# # f16 accumulation adam +# ############################################################################### + +# for (( i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 1 0 32 2 adam +# cp -rf log/ log_f16_${i} +# done + +# file_op out/bert_f16_pretraining_8gpu_64bs_100iter_accumulation + +# ############################################################################### +# # f16 accumulation lamb +# ############################################################################### + +# for (( i = 1; i <= ${NUM}; i++ )) +# do +# echo $i +# sh train_perbert.sh 1 0 32 2 lamb +# cp -rf log/ log_f16_${i} +# done + +# file_op out/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_lamb + + +############################################################################### +# f32 adam debug +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 0 1 64 1 adam + cp -rf log/ log_f32_${i} +done + +file_op out/bert_f32_pretraining_8gpu_64bs_100iter_debug + +python tools/result_analysis.py --f32=1 \ + --cmp1_file=old/bert_f32_pretraining_8gpu_64bs_100iter_debug/log_f32_1/out.json \ + --cmp2_file=out/bert_f32_pretraining_8gpu_64bs_100iter_debug/log_f32_1/out.json \ + --out=pic/bert_f32_pretraining_8gpu_64bs_100iter_debug.png +############################################################################### +# f32 lamb debug +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 0 1 64 1 lamb + cp -rf log/ log_f32_${i} +done + +file_op out/bert_f32_pretraining_8gpu_64bs_100iter_lamb_debug + +python tools/result_analysis.py --f32=1 \ + --cmp1_file=old/bert_f32_pretraining_8gpu_64bs_100iter_lamb_debug/log_f32_1/out.json \ + --cmp2_file=out/bert_f32_pretraining_8gpu_64bs_100iter_lamb_debug/log_f32_1/out.json \ + --out=pic/bert_f32_pretraining_8gpu_64bs_100iter_lamb_debug.png +############################################################################### +# f16 adam debug +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 1 1 64 1 adam + cp -rf log/ log_f16_${i} +done +file_op out/bert_f16_pretraining_8gpu_64bs_100iter_debug + +python tools/result_analysis.py --f32=0 \ + --cmp1_file=old/bert_f16_pretraining_8gpu_64bs_100iter_debug/log_f16_1/out.json \ + --cmp2_file=out/bert_f16_pretraining_8gpu_64bs_100iter_debug/log_f16_1/out.json \ + --out=pic/bert_f16_pretraining_8gpu_64bs_100iter_debug.png +############################################################################### +# f16 lamb debug +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 1 1 64 1 lamb + cp -rf log/ log_f16_${i} +done + +file_op out/bert_f16_pretraining_8gpu_64bs_100iter_lamb_debug + +python tools/result_analysis.py --f32=0 \ + --cmp1_file=old/bert_f16_pretraining_8gpu_64bs_100iter_lamb_debug/log_f16_1/out.json \ + --cmp2_file=out/bert_f16_pretraining_8gpu_64bs_100iter_lamb_debug/log_f16_1/out.json \ + --out=pic/bert_f16_pretraining_8gpu_64bs_100iter_lamb_debug.png +############################################################################### +# f32 accumulation adam debug +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 0 1 32 2 adam + cp -rf log/ log_f32_${i} + +done + +file_op out/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_debug + +python tools/result_analysis.py --f32=1 \ + --cmp1_file=old/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_debug/log_f32_1/out.json \ + --cmp2_file=out/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_debug/log_f32_1/out.json \ + --out=pic/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_debug.png + +############################################################################### +# f32 accumulation lamb debug +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 0 1 32 2 lamb + cp -rf log/ log_f32_${i} +done + +file_op out/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug + +python tools/result_analysis.py --f32=1 \ + --cmp1_file=old/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug/log_f32_1/out.json \ + --cmp2_file=out/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug/log_f32_1/out.json \ + --out=pic/bert_f32_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug.png + +############################################################################### +# f16 accumulation adam debug +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 1 1 32 2 adam + cp -rf log/ log_f16_${i} +done + +file_op out/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_debug + +python tools/result_analysis.py --f32=0 \ + --cmp1_file=old/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_debug/log_f16_1/out.json \ + --cmp2_file=out/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_debug/log_f16_1/out.json \ + --out=pic/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_debug.png +############################################################################### +# f16 accumulation lamb +############################################################################### + +for (( i = 1; i <= ${NUM}; i++ )) +do + echo $i + sh train_perbert.sh 1 1 32 2 lamb + cp -rf log/ log_f16_${i} +done + +file_op out/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug + +python tools/result_analysis.py --f32=0 \ + --cmp1_file=old/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug/log_f16_1/out.json \ + --cmp2_file=out/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug/log_f16_1/out.json \ + --out=pic/bert_f16_pretraining_8gpu_64bs_100iter_accumulation_lamb_debug.png +# ############################################################################## +# tar +# ############################################################################## + +tar -zcvf out.tar.gz out + +python tools/stitching_pic.py --dir=pic --out_file=./pic/all.png +# rm -rf out +############################################################################### +# upload +############################################################################### + + + + + diff --git a/LanguageModeling/BERT/util.py b/LanguageModeling/BERT/util.py index 6b04d55..bb525c5 100755 --- a/LanguageModeling/BERT/util.py +++ b/LanguageModeling/BERT/util.py @@ -21,6 +21,7 @@ import pandas as pd from datetime import datetime import oneflow as flow +import subprocess def InitNodes(args): @@ -118,6 +119,9 @@ def metric_cb(self, step=0, **kwargs): def callback(outputs): if step == 0: self._clear() + if step == 1: + print(subprocess.check_output("nvidia-smi --query-gpu=memory.used --format=csv ", shell=True)) + for key in self.keys: self.metric_dict[key] += outputs[key].sum() self.metric_dict['n_' + key] += outputs[key].size @@ -133,7 +137,7 @@ def callback(outputs): for key in self.keys: value = self.metric_dict[key] / self.metric_dict['n_' + key] self.update_and_save(key, value, step, **kwargs) - print(', '.join(('{}: {}' if type(v) is int else '{}: {:.3f}').format(k, v) \ + print(', '.join(('{}: {}' if type(v) is int else '{}: {}').format(k, v) \ for k, v in self.metric_dict.items()), time.time()) self._clear() @@ -147,7 +151,14 @@ def CreateOptimizer(args): loss_scale_policy = None if args.use_fp16: loss_scale_policy = flow.optimizer.loss_scale.dynamic_loss_scale(increment_period=2000); - return flow.optimizer.AdamW(lr_scheduler, epsilon=1e-6, weight_decay=args.weight_decay_rate, + + if args.optimizer_type == "lamb": + return flow.optimizer.LAMB(lr_scheduler, beta1=0.9, beta2=0.999, epsilon=1e-6, weight_decay=args.weight_decay_rate, + weight_decay_excludes=["bias", "LayerNorm", "layer_norm"], + grad_clipping=flow.optimizer.grad_clipping.by_global_norm(1.0), + loss_scale_policy=loss_scale_policy) + else: + return flow.optimizer.AdamW(lr_scheduler, epsilon=1e-6, weight_decay=args.weight_decay_rate, weight_decay_excludes=["bias", "LayerNorm", "layer_norm"], grad_clipping=flow.optimizer.grad_clipping.by_global_norm(1.0), loss_scale_policy=loss_scale_policy) @@ -155,6 +166,7 @@ def CreateOptimizer(args): def GetFunctionConfig(args): config = flow.function_config() config.enable_auto_mixed_precision(args.use_fp16) + config.train.num_gradient_accumulation_steps(args.num_accumulation_steps) if args.use_xla: config.use_xla_jit(True) config.enable_fuse_add_to_output(True)