{"adam_epsilon": 1e-08, "alphabet": "gene_prot", "append_eos": true, "asl_gamma_neg": 4.0, "asl_gamma_pos": 1.0, "best_metric_type": "f1", "beta1": 0.9, "beta2": 0.99, "buffer_size": 4096, "cache_dir": null, "classifier_activate_func": "gelu", "classifier_size": 128, "codes_file": null, "config_path": "../config/lucatriple2/lucatriple2_config.json", "cross_atten": false, "dataset_name": "DeepAbBindv2_nucl", "dataset_type": "protein_protein_gene", "delete_old": false, "dev_data_dir": "../dataset/DeepAbBindv2_nucl/protein_protein_gene/binary_class/dev/", "device": "cuda", "do_eval": true, "do_lower_case": false, "do_metrics": true, "do_predict": true, "do_train": true, "dropout_prob": 0.1, "early_stop_begin_epoch": null, "early_stop_num_epoch": null, "emb_activate_func": "gelu", "embedding_complete": true, "embedding_complete_seg_overlap": true, "embedding_fixed_len_a_time": 3072, "embedding_input_size": 2560, "embedding_input_size_a": 2560, "embedding_input_size_b": 2560, "embedding_input_size_c": 2560, "eval_all_checkpoints": false, "evaluate_during_training": true, "evaluate_steps": 1000000, "evaluate_strategy": "step", "fc_activate_func": "gelu", "focal_loss_alpha": 0.7, "focal_loss_gamma": 2.0, "focal_loss_reduce": false, "fp16": false, "fp16_embedding": false, "fp16_opt_level": "O1", "fusion_type": "concat", "gradient_accumulation_steps": 1, "hidden_size": 1024, "ignore_index": -100, "input_mode": "triple", "input_type": "matrix", "intermediate_size": 4096, "label_filepath": "../dataset/DeepAbBindv2_nucl/protein_protein_gene/binary_class/label.txt", "label_size": 2, "label_type": "DeepAbBindv2", "learning_rate": 0.0002, "llm_dir": "..", "llm_dirpath": "../llm/models/lucaone_virus/v1.0/token_level,span_level,seq_level/lucaone_virus/20240815023346/checkpoint-step3800000", "llm_step": "3800000", "llm_task_level": "token_level,span_level,seq_level", "llm_time_str": "20240815023346", "llm_type": "lucaone_virus", "llm_version": "v1.0", "local_rank": -1, "log_dir": "../logs/DeepAbBindv2_nucl/protein_protein_gene/binary_class/lucatriple2/matrix/20250127174006", "logging_steps": 1000, "loss_reduction": "mean", "loss_type": "bce", "lr_decay_rate": 0.9, "lr_update_strategy": "step", "matrix_add_special_token": true, "matrix_dirpath": "/mnt2/sanyuan.hy/matrices/DeepAbBindv2/lucaone_virus/v1.0/20240815023346/3800000", "matrix_embedding_exists": true, "matrix_encoder": false, "matrix_encoder_act": false, "matrix_fc_size": "512", "matrix_max_length": 3827, "matrix_max_length_a": null, "matrix_max_length_b": null, "matrix_max_length_c": null, "matrix_pooling_type": "value_attention", "max_grad_norm": 1.0, "max_sentence_length": null, "max_sentences": null, "max_steps": -1, "model_dirpath": null, "model_type": "lucatriple2", "n_gpu": 1, "no_cuda": false, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "non_ignore": false, "not_append_eos": false, "not_matrix_encoder_shared": false, "not_prepend_bos": false, "not_save_emb_to_disk": false, "not_seq_encoder_shared": false, "num_attention_heads": 0, "num_hidden_layers": 0, "num_train_epochs": 50, "output_dir": "../models/DeepAbBindv2_nucl/protein_protein_gene/binary_class/lucatriple2/matrix/20250127174006", "output_mode": "binary_class", "overwrite_cache": false, "overwrite_output_dir": true, "per_gpu_eval_batch_size": 16, "per_gpu_train_batch_size": 16, "pos_weight": 2.0, "position_embedding_type": "absolute", "prepend_bos": true, "save_all": true, "save_steps": 1000000, "seed": 1221, "self_atten": false, "seq_fc_size": "null", "seq_max_length": 3827, "seq_max_length_a": null, "seq_max_length_b": null, "seq_max_length_c": null, "seq_pooling_type": "value_attention", "seq_subword": false, "seq_vocab_path": "gene_prot", "sigmoid": true, "task_level_type": "seq_level", "task_type": "binary_class", "tb_log_dir": "../tb-logs/DeepAbBindv2_nucl/protein_protein_gene/binary_class/lucatriple2/matrix/20250127174006", "test_data_dir": "../dataset/DeepAbBindv2_nucl/protein_protein_gene/binary_class/test/", "time_str": "20250127174008", "train_data_dir": "../dataset/DeepAbBindv2_nucl/protein_protein_gene/binary_class/train/", "trunc_type": "right", "vector_dirpath": "/mnt2/sanyuan.hy/vectors/DeepAbBindv2/lucaone_virus/v1.0/20240815023346/3800000", "vector_fc_size": "null", "vocab_size": 39, "warmup_steps": 1000, "weight": null, "weight_decay": 0.01, "worker_num": 1} ################################################## n_gpu: 1 ################################################## Inputs: Input Name List: protein_protein_gene,embedding_matrix_a,embedding_matrix_b,embedding_matrix_c ################################################## Encoder Config: {'llm_type': 'lucaone_virus', 'llm_version': 'v1.0', 'llm_step': '3800000', 'llm_dirpath': '../llm/models/lucaone_virus/v1.0/token_level,span_level,seq_level/lucaone_virus/20240815023346/checkpoint-step3800000', 'input_type': 'matrix', 'trunc_type': 'right', 'seq_max_length': 3827, 'atom_seq_max_length': None, 'vector_dirpath': '/mnt2/sanyuan.hy/vectors/DeepAbBindv2/lucaone_virus/v1.0/20240815023346/3800000', 'matrix_dirpath': '/mnt2/sanyuan.hy/matrices/DeepAbBindv2/lucaone_virus/v1.0/20240815023346/3800000', 'local_rank': -1, 'max_sentence_length': None, 'max_sentences': None, 'matrix_add_special_token': True, 'embedding_complete': True, 'embedding_complete_seg_overlap': True, 'embedding_fixed_len_a_time': 3072, 'matrix_embedding_exists': True, 'save_emb_to_disk': True, 'fp16_embedding': False} ################################################## Model Config: LucaConfig { "alphabet": "gene_prot", "attention_probs_dropout_prob": 0.1, "classifier_activate_func": "gelu", "classifier_dropout_prob": 0.1, "classifier_size": 128, "cls_token_id": 2, "cross_atten": false, "directionality": "bidi", "emb_activate_func": "gelu", "embedding_input_size": 2560, "embedding_input_size_a": 2560, "embedding_input_size_b": 2560, "embedding_input_size_c": 2560, "fc_activate_func": "gelu", "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 1024, "ignore_index": -100, "initializer_range": 0.02, "intermediate_size": 4096, "kernel_size": 7, "layer_norm_eps": 1e-12, "layer_norm_type": "post", "loss_reduction": "mean", "matrix_fc_size": [ 512 ], "matrix_max_length": 3827, "matrix_max_length_a": 3827, "matrix_max_length_b": 3827, "matrix_max_length_c": 3827, "matrix_pooling_type": "value_attention", "max_position_embeddings": 3829, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "num_attention_heads": 4, "num_hidden_layers": 4, "pad_token_id": 0, "pos_weight": 2.0, "position_embedding_type": "absolute", "self_atten": false, "sep_token_id": 3, "seq_fc_size": null, "seq_max_length": 3827, "seq_max_length_a": 3827, "seq_max_length_b": 3827, "seq_max_length_c": 3827, "seq_pooling_type": "value_attention", "token_dropout": null, "transformers_version": "4.41.2", "type_vocab_size": 2, "use_luca_layer_norm_v2": true, "vector_fc_size": null, "vocab_size": 39 } ################################################## Mode Architecture: LucaTriple2( (matrix_pooler_a): GlobalMaskValueAttentionPooling1D (2560 -> 2560) (matrix_pooler_b): GlobalMaskValueAttentionPooling1D (2560 -> 2560) (matrix_pooler_c): GlobalMaskValueAttentionPooling1D (2560 -> 2560) (linear_a): ModuleList( (0): ModuleList( (0): Linear(in_features=2560, out_features=512, bias=True) (1): GELU(approximate='none') ) ) (linear_b): ModuleList( (0): ModuleList( (0): Linear(in_features=2560, out_features=512, bias=True) (1): GELU(approximate='none') ) ) (linear_c): ModuleList( (0): ModuleList( (0): Linear(in_features=2560, out_features=512, bias=True) (1): GELU(approximate='none') ) ) (dropout): Dropout(p=0.1, inplace=False) (hidden_layer): Linear(in_features=1536, out_features=128, bias=True) (hidden_act): GELU(approximate='none') (classifier): Linear(in_features=128, out_features=1, bias=True) (output): Sigmoid() (loss_fct): MaskedBCEWithLogitsLoss( (criterion): BCEWithLogitsLoss() ) ) ################################################## Model parameters: 63112961 ################################################## {"total_num": "60.190000M", "total_size": "240.760000MB", "param_sum": "60.190000M", "param_size": "240.760000MB", "buffer_sum": "0.000000M", "buffer_size": "0.000000MB", "trainable_num": "60.189210M", "trainable_size": "240.756840MB"} ################################################## Train dataset len: 33572, batch size: 16, batch num: 2099 Train dataset t_total: 104950, max_steps: -1 ***** Running training ***** Train Dataset Num examples = 33572 Train Dataset Num Epochs = 50 Logging Steps = 1000 Saving Steps = 1000000 Evaluating Strategy = step Evaluating Steps = 1000000 steps Train Dataset Instantaneous batch size per GPU = 16 Train Dataset Total train batch size (w. parallel, distributed & accumulation) = 16 Train Dataset Gradient Accumulation steps = 1 Train Dataset Total optimization steps = 104950 ################################################## Training, Epoch: 0001, Batch: 001000, Sample Num: 16000, Cur Loss: 0.66712379, Cur Avg Loss: 0.84390492, Log Avg loss: 0.84390492, Global Avg Loss: 0.84390492, Time: 0.4069 Steps: 1000, Updated lr: 0.000200 Training, Epoch: 0001, Batch: 002000, Sample Num: 32000, Cur Loss: 0.80111939, Cur Avg Loss: 0.76078585, Log Avg loss: 0.67766678, Global Avg Loss: 0.76078585, Time: 0.4070 Steps: 2000, Updated lr: 0.000198 ***** Running evaluation checkpoint-2099 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-2099 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 853.160238, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.708823, "eval_total_loss": 634.396448, "eval_acc": 0.796144, "eval_prec": 0.851434, "eval_recall": 0.717479, "eval_f1": 0.778738, "eval_roc_auc": 0.889184, "eval_pr_auc": 0.883877, "eval_confusion_matrix": {"tn": 6261, "fp": 896, "fn": 2022, "tp": 5135}, "eval_mcc2": 0.599756, "eval_mcc": 0.599756, "eval_sn": 0.717479, "eval_sp": 0.874808, "update_flag": true, "test_avg_loss": 1.265015, "test_total_loss": 15.180184, "test_acc": 0.672222, "test_prec": 0.536585, "test_recall": 0.354839, "test_f1": 0.427184, "test_roc_auc": 0.595407, "test_pr_auc": 0.414041, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 40, "tp": 22}, "test_mcc2": 0.219604, "test_mcc": 0.219604, "test_sn": 0.354839, "test_sp": 0.838983, "lr": 0.0001978855218855219, "cur_epoch_step": 2099, "train_global_avg_loss": 0.7557987075027481, "train_cur_epoch_loss": 1586.4214870482683, "train_cur_epoch_avg_loss": 0.7557987075027481, "train_cur_epoch_time": 853.1602380275726, "train_cur_epoch_avg_time": 0.4064603325524405, "epoch": 1, "step": 2099} ################################################## Training, Epoch: 0002, Batch: 000901, Sample Num: 14416, Cur Loss: 0.74020481, Cur Avg Loss: 0.55032771, Log Avg loss: 0.56069506, Global Avg Loss: 0.69408892, Time: 0.4068 Steps: 3000, Updated lr: 0.000196 Training, Epoch: 0002, Batch: 001901, Sample Num: 30416, Cur Loss: 0.37821037, Cur Avg Loss: 0.52301931, Log Avg loss: 0.49841445, Global Avg Loss: 0.64517030, Time: 0.4069 Steps: 4000, Updated lr: 0.000194 ***** Running evaluation checkpoint-4198 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-4198 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 853.503384, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.484384, "eval_total_loss": 433.523504, "eval_acc": 0.853919, "eval_prec": 0.834434, "eval_recall": 0.883052, "eval_f1": 0.858054, "eval_roc_auc": 0.933213, "eval_pr_auc": 0.929135, "eval_confusion_matrix": {"tn": 5903, "fp": 1254, "fn": 837, "tp": 6320}, "eval_mcc2": 0.709043, "eval_mcc": 0.709043, "eval_sn": 0.883052, "eval_sp": 0.824787, "update_flag": true, "test_avg_loss": 1.604553, "test_total_loss": 19.254632, "test_acc": 0.672222, "test_prec": 0.523077, "test_recall": 0.548387, "test_f1": 0.535433, "test_roc_auc": 0.646118, "test_pr_auc": 0.53865, "test_confusion_matrix": {"tn": 87, "fp": 31, "fn": 28, "tp": 34}, "test_mcc2": 0.282621, "test_mcc": 0.282621, "test_sn": 0.548387, "test_sp": 0.737288, "lr": 0.00019384704184704187, "cur_epoch_step": 2099, "train_global_avg_loss": 0.6362117170584719, "train_cur_epoch_loss": 1084.3953011631966, "train_cur_epoch_avg_loss": 0.5166247266141956, "train_cur_epoch_time": 853.5033843517303, "train_cur_epoch_avg_time": 0.4066238134119725, "epoch": 2, "step": 4198} ################################################## Training, Epoch: 0003, Batch: 000802, Sample Num: 12832, Cur Loss: 0.52707946, Cur Avg Loss: 0.36239434, Log Avg loss: 0.38077584, Global Avg Loss: 0.59229141, Time: 0.4083 Steps: 5000, Updated lr: 0.000192 Training, Epoch: 0003, Batch: 001802, Sample Num: 28832, Cur Loss: 0.49425507, Cur Avg Loss: 0.36696739, Log Avg loss: 0.37063498, Global Avg Loss: 0.55534867, Time: 0.4059 Steps: 6000, Updated lr: 0.000190 ***** Running evaluation checkpoint-6297 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-6297 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.816572, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.440959, "eval_total_loss": 394.658703, "eval_acc": 0.872572, "eval_prec": 0.836382, "eval_recall": 0.926366, "eval_f1": 0.879077, "eval_roc_auc": 0.94916, "eval_pr_auc": 0.945935, "eval_confusion_matrix": {"tn": 5860, "fp": 1297, "fn": 527, "tp": 6630}, "eval_mcc2": 0.749495, "eval_mcc": 0.749495, "eval_sn": 0.926366, "eval_sp": 0.818779, "update_flag": true, "test_avg_loss": 1.99729, "test_total_loss": 23.967485, "test_acc": 0.677778, "test_prec": 0.533333, "test_recall": 0.516129, "test_f1": 0.52459, "test_roc_auc": 0.632723, "test_pr_auc": 0.518421, "test_confusion_matrix": {"tn": 90, "fp": 28, "fn": 30, "tp": 32}, "test_mcc2": 0.281078, "test_mcc": 0.281078, "test_sn": 0.516129, "test_sp": 0.762712, "lr": 0.00018980856180856183, "cur_epoch_step": 2099, "train_global_avg_loss": 0.5453845213726577, "train_cur_epoch_loss": 763.4695428721607, "train_cur_epoch_avg_loss": 0.3637301300010294, "train_cur_epoch_time": 852.8165724277496, "train_cur_epoch_avg_time": 0.40629660430097647, "epoch": 3, "step": 6297} ################################################## Training, Epoch: 0004, Batch: 000703, Sample Num: 11248, Cur Loss: 0.70954841, Cur Avg Loss: 0.25989072, Log Avg loss: 0.28489748, Global Avg Loss: 0.51671279, Time: 0.4062 Steps: 7000, Updated lr: 0.000188 Training, Epoch: 0004, Batch: 001703, Sample Num: 27248, Cur Loss: 0.05368594, Cur Avg Loss: 0.26992052, Log Avg loss: 0.27697147, Global Avg Loss: 0.48674512, Time: 0.4071 Steps: 8000, Updated lr: 0.000187 ***** Running evaluation checkpoint-8396 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-8396 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.136118, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.405231, "eval_total_loss": 362.681847, "eval_acc": 0.864678, "eval_prec": 0.809682, "eval_recall": 0.953472, "eval_f1": 0.875714, "eval_roc_auc": 0.95643, "eval_pr_auc": 0.955617, "eval_confusion_matrix": {"tn": 5553, "fp": 1604, "fn": 333, "tp": 6824}, "eval_mcc2": 0.741136, "eval_mcc": 0.741136, "eval_sn": 0.953472, "eval_sp": 0.775884, "update_flag": false, "test_avg_loss": 1.787311, "test_total_loss": 21.447729, "test_acc": 0.661111, "test_prec": 0.506494, "test_recall": 0.629032, "test_f1": 0.561151, "test_roc_auc": 0.660334, "test_pr_auc": 0.571624, "test_confusion_matrix": {"tn": 80, "fp": 38, "fn": 23, "tp": 39}, "test_mcc2": 0.294855, "test_mcc": 0.294855, "test_sn": 0.629032, "test_sp": 0.677966, "lr": 0.00018577008177008176, "cur_epoch_step": 2099, "train_global_avg_loss": 0.4764244303350963, "train_cur_epoch_loss": 565.7731860098429, "train_cur_epoch_avg_loss": 0.26954415722241204, "train_cur_epoch_time": 852.1361179351807, "train_cur_epoch_avg_time": 0.40597242398055294, "epoch": 4, "step": 8396} ################################################## Training, Epoch: 0005, Batch: 000604, Sample Num: 9664, Cur Loss: 0.48242223, Cur Avg Loss: 0.19053232, Log Avg loss: 0.22118006, Global Avg Loss: 0.45723789, Time: 0.4057 Steps: 9000, Updated lr: 0.000185 Training, Epoch: 0005, Batch: 001604, Sample Num: 25664, Cur Loss: 0.12855747, Cur Avg Loss: 0.20161445, Log Avg loss: 0.20830805, Global Avg Loss: 0.43234491, Time: 0.4048 Steps: 10000, Updated lr: 0.000183 ***** Running evaluation checkpoint-10495 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-10495 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 851.565019, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.47053, "eval_total_loss": 421.123945, "eval_acc": 0.864259, "eval_prec": 0.807067, "eval_recall": 0.957384, "eval_f1": 0.875823, "eval_roc_auc": 0.957854, "eval_pr_auc": 0.957324, "eval_confusion_matrix": {"tn": 5519, "fp": 1638, "fn": 305, "tp": 6852}, "eval_mcc2": 0.741492, "eval_mcc": 0.741492, "eval_sn": 0.957384, "eval_sp": 0.771133, "update_flag": false, "test_avg_loss": 2.251132, "test_total_loss": 27.01358, "test_acc": 0.655556, "test_prec": 0.5, "test_recall": 0.596774, "test_f1": 0.544118, "test_roc_auc": 0.662794, "test_pr_auc": 0.584624, "test_confusion_matrix": {"tn": 81, "fp": 37, "fn": 25, "tp": 37}, "test_mcc2": 0.273517, "test_mcc": 0.273517, "test_sn": 0.596774, "test_sp": 0.686441, "lr": 0.00018173160173160172, "cur_epoch_step": 2099, "train_global_avg_loss": 0.42168773719922226, "train_cur_epoch_loss": 425.5532848123694, "train_cur_epoch_avg_loss": 0.20274096465572625, "train_cur_epoch_time": 851.5650193691254, "train_cur_epoch_avg_time": 0.40570034271992633, "epoch": 5, "step": 10495} ################################################## Training, Epoch: 0006, Batch: 000505, Sample Num: 8080, Cur Loss: 0.00559127, Cur Avg Loss: 0.13389506, Log Avg loss: 0.16978072, Global Avg Loss: 0.40847544, Time: 0.4060 Steps: 11000, Updated lr: 0.000181 Training, Epoch: 0006, Batch: 001505, Sample Num: 24080, Cur Loss: 0.23791222, Cur Avg Loss: 0.14928157, Log Avg loss: 0.15705175, Global Avg Loss: 0.38752346, Time: 0.4072 Steps: 12000, Updated lr: 0.000179 ***** Running evaluation checkpoint-12594 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-12594 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.491146, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.497371, "eval_total_loss": 445.14691, "eval_acc": 0.898421, "eval_prec": 0.874754, "eval_recall": 0.929999, "eval_f1": 0.901531, "eval_roc_auc": 0.961267, "eval_pr_auc": 0.960697, "eval_confusion_matrix": {"tn": 6204, "fp": 953, "fn": 501, "tp": 6656}, "eval_mcc2": 0.798436, "eval_mcc": 0.798436, "eval_sn": 0.929999, "eval_sp": 0.866844, "update_flag": true, "test_avg_loss": 3.170734, "test_total_loss": 38.048805, "test_acc": 0.677778, "test_prec": 0.533333, "test_recall": 0.516129, "test_f1": 0.52459, "test_roc_auc": 0.649125, "test_pr_auc": 0.580956, "test_confusion_matrix": {"tn": 90, "fp": 28, "fn": 30, "tp": 32}, "test_mcc2": 0.281078, "test_mcc": 0.281078, "test_sn": 0.516129, "test_sp": 0.762712, "lr": 0.0001776931216931217, "cur_epoch_step": 2099, "train_global_avg_loss": 0.3777104868749296, "train_cur_epoch_loss": 331.2730697970255, "train_cur_epoch_avg_loss": 0.1578242352534662, "train_cur_epoch_time": 852.4911460876465, "train_cur_epoch_avg_time": 0.40614156554914077, "epoch": 6, "step": 12594} ################################################## Training, Epoch: 0007, Batch: 000406, Sample Num: 6496, Cur Loss: 0.00040239, Cur Avg Loss: 0.11939596, Log Avg loss: 0.15507907, Global Avg Loss: 0.36964313, Time: 0.4061 Steps: 13000, Updated lr: 0.000177 Training, Epoch: 0007, Batch: 001406, Sample Num: 22496, Cur Loss: 0.06986828, Cur Avg Loss: 0.12537950, Log Avg loss: 0.12780882, Global Avg Loss: 0.35236925, Time: 0.4066 Steps: 14000, Updated lr: 0.000175 ***** Running evaluation checkpoint-14693 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-14693 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.403138, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.739341, "eval_total_loss": 661.710551, "eval_acc": 0.903381, "eval_prec": 0.904909, "eval_recall": 0.901495, "eval_f1": 0.903199, "eval_roc_auc": 0.959933, "eval_pr_auc": 0.958828, "eval_confusion_matrix": {"tn": 6479, "fp": 678, "fn": 705, "tp": 6452}, "eval_mcc2": 0.806768, "eval_mcc": 0.806768, "eval_sn": 0.901495, "eval_sp": 0.905268, "update_flag": true, "test_avg_loss": 4.51968, "test_total_loss": 54.236161, "test_acc": 0.722222, "test_prec": 0.630435, "test_recall": 0.467742, "test_f1": 0.537037, "test_roc_auc": 0.650765, "test_pr_auc": 0.589125, "test_confusion_matrix": {"tn": 101, "fp": 17, "fn": 33, "tp": 29}, "test_mcc2": 0.352625, "test_mcc": 0.352625, "test_sn": 0.467742, "test_sp": 0.855932, "lr": 0.00017365464165464167, "cur_epoch_step": 2099, "train_global_avg_loss": 0.3423269749046473, "train_cur_epoch_loss": 272.92437057111965, "train_cur_epoch_avg_loss": 0.13002590308295361, "train_cur_epoch_time": 852.4031381607056, "train_cur_epoch_avg_time": 0.4060996370465486, "epoch": 7, "step": 14693} ################################################## Training, Epoch: 0008, Batch: 000307, Sample Num: 4912, Cur Loss: 0.10814866, Cur Avg Loss: 0.09414636, Log Avg loss: 0.12554372, Global Avg Loss: 0.33724754, Time: 0.4057 Steps: 15000, Updated lr: 0.000173 Training, Epoch: 0008, Batch: 001307, Sample Num: 20912, Cur Loss: 0.02534328, Cur Avg Loss: 0.10272441, Log Avg loss: 0.10535787, Global Avg Loss: 0.32275444, Time: 0.4061 Steps: 16000, Updated lr: 0.000171 ***** Running evaluation checkpoint-16792 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-16792 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.036945, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.689903, "eval_total_loss": 617.462912, "eval_acc": 0.896255, "eval_prec": 0.87101, "eval_recall": 0.930278, "eval_f1": 0.899669, "eval_roc_auc": 0.961258, "eval_pr_auc": 0.962691, "eval_confusion_matrix": {"tn": 6171, "fp": 986, "fn": 499, "tp": 6658}, "eval_mcc2": 0.794352, "eval_mcc": 0.794352, "eval_sn": 0.930278, "eval_sp": 0.862233, "update_flag": false, "test_avg_loss": 3.938874, "test_total_loss": 47.266487, "test_acc": 0.705556, "test_prec": 0.57377, "test_recall": 0.564516, "test_f1": 0.569106, "test_roc_auc": 0.670312, "test_pr_auc": 0.588639, "test_confusion_matrix": {"tn": 92, "fp": 26, "fn": 27, "tp": 35}, "test_mcc2": 0.345526, "test_mcc": 0.345526, "test_sn": 0.564516, "test_sp": 0.779661, "lr": 0.00016961616161616163, "cur_epoch_step": 2099, "train_global_avg_loss": 0.3131628115448025, "train_cur_epoch_loss": 228.81968918634084, "train_cur_epoch_avg_loss": 0.10901366802588892, "train_cur_epoch_time": 852.0369446277618, "train_cur_epoch_avg_time": 0.4059251760970757, "epoch": 8, "step": 16792} ################################################## Training, Epoch: 0009, Batch: 000208, Sample Num: 3328, Cur Loss: 0.05118110, Cur Avg Loss: 0.05868334, Log Avg loss: 0.10676502, Global Avg Loss: 0.31004918, Time: 0.4049 Steps: 17000, Updated lr: 0.000169 Training, Epoch: 0009, Batch: 001208, Sample Num: 19328, Cur Loss: 0.07842915, Cur Avg Loss: 0.09044114, Log Avg loss: 0.09704677, Global Avg Loss: 0.29821571, Time: 0.4066 Steps: 18000, Updated lr: 0.000167 ***** Running evaluation checkpoint-18891 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-18891 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.396436, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.808163, "eval_total_loss": 723.306111, "eval_acc": 0.907014, "eval_prec": 0.903574, "eval_recall": 0.911276, "eval_f1": 0.907409, "eval_roc_auc": 0.962112, "eval_pr_auc": 0.962825, "eval_confusion_matrix": {"tn": 6461, "fp": 696, "fn": 635, "tp": 6522}, "eval_mcc2": 0.814058, "eval_mcc": 0.814058, "eval_sn": 0.911276, "eval_sp": 0.902753, "update_flag": true, "test_avg_loss": 5.469304, "test_total_loss": 65.631654, "test_acc": 0.694444, "test_prec": 0.566038, "test_recall": 0.483871, "test_f1": 0.521739, "test_roc_auc": 0.638737, "test_pr_auc": 0.589894, "test_confusion_matrix": {"tn": 95, "fp": 23, "fn": 32, "tp": 30}, "test_mcc2": 0.301251, "test_mcc": 0.301251, "test_sn": 0.483871, "test_sp": 0.805085, "lr": 0.00016557768157768159, "cur_epoch_step": 2099, "train_global_avg_loss": 0.28895459678890606, "train_cur_epoch_loss": 200.01135647890078, "train_cur_epoch_avg_loss": 0.09528887874173453, "train_cur_epoch_time": 852.396436214447, "train_cur_epoch_avg_time": 0.4060964441231286, "epoch": 9, "step": 18891} ################################################## Training, Epoch: 0010, Batch: 000109, Sample Num: 1744, Cur Loss: 0.24397898, Cur Avg Loss: 0.07791728, Log Avg loss: 0.09925144, Global Avg Loss: 0.28774391, Time: 0.4027 Steps: 19000, Updated lr: 0.000165 Training, Epoch: 0010, Batch: 001109, Sample Num: 17744, Cur Loss: 0.01241237, Cur Avg Loss: 0.07682440, Log Avg loss: 0.07670528, Global Avg Loss: 0.27719198, Time: 0.4059 Steps: 20000, Updated lr: 0.000163 ***** Running evaluation checkpoint-20990 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-20990 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 853.005806, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.779794, "eval_total_loss": 697.915399, "eval_acc": 0.898491, "eval_prec": 0.870775, "eval_recall": 0.935867, "eval_f1": 0.902148, "eval_roc_auc": 0.957939, "eval_pr_auc": 0.957153, "eval_confusion_matrix": {"tn": 6163, "fp": 994, "fn": 459, "tp": 6698}, "eval_mcc2": 0.799218, "eval_mcc": 0.799218, "eval_sn": 0.935867, "eval_sp": 0.861115, "update_flag": false, "test_avg_loss": 5.143652, "test_total_loss": 61.723824, "test_acc": 0.677778, "test_prec": 0.533333, "test_recall": 0.516129, "test_f1": 0.52459, "test_roc_auc": 0.634226, "test_pr_auc": 0.589862, "test_confusion_matrix": {"tn": 90, "fp": 28, "fn": 30, "tp": 32}, "test_mcc2": 0.281078, "test_mcc": 0.281078, "test_sn": 0.516129, "test_sp": 0.762712, "lr": 0.00016153920153920155, "cur_epoch_step": 2099, "train_global_avg_loss": 0.2687642710914823, "train_cur_epoch_loss": 182.72076227098887, "train_cur_epoch_avg_loss": 0.08705133981466835, "train_cur_epoch_time": 853.0058062076569, "train_cur_epoch_avg_time": 0.4063867585553391, "epoch": 10, "step": 20990} ################################################## Training, Epoch: 0011, Batch: 000010, Sample Num: 160, Cur Loss: 0.03087013, Cur Avg Loss: 0.02489711, Log Avg loss: 0.09777147, Global Avg Loss: 0.26864814, Time: 0.4054 Steps: 21000, Updated lr: 0.000162 Training, Epoch: 0011, Batch: 001010, Sample Num: 16160, Cur Loss: 0.05915420, Cur Avg Loss: 0.06967151, Log Avg loss: 0.07011925, Global Avg Loss: 0.25962410, Time: 0.4075 Steps: 22000, Updated lr: 0.000160 Training, Epoch: 0011, Batch: 002010, Sample Num: 32160, Cur Loss: 0.02700784, Cur Avg Loss: 0.08048988, Log Avg loss: 0.09141644, Global Avg Loss: 0.25231073, Time: 0.4073 Steps: 23000, Updated lr: 0.000158 ***** Running evaluation checkpoint-23089 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-23089 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 854.599443, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.757477, "eval_total_loss": 677.941518, "eval_acc": 0.900866, "eval_prec": 0.87641, "eval_recall": 0.933352, "eval_f1": 0.903985, "eval_roc_auc": 0.960002, "eval_pr_auc": 0.958769, "eval_confusion_matrix": {"tn": 6215, "fp": 942, "fn": 477, "tp": 6680}, "eval_mcc2": 0.80343, "eval_mcc": 0.80343, "eval_sn": 0.933352, "eval_sp": 0.868381, "update_flag": false, "test_avg_loss": 5.425139, "test_total_loss": 65.101666, "test_acc": 0.666667, "test_prec": 0.516129, "test_recall": 0.516129, "test_f1": 0.516129, "test_roc_auc": 0.637917, "test_pr_auc": 0.565599, "test_confusion_matrix": {"tn": 88, "fp": 30, "fn": 30, "tp": 32}, "test_mcc2": 0.261892, "test_mcc": 0.261892, "test_sn": 0.516129, "test_sp": 0.745763, "lr": 0.0001575007215007215, "cur_epoch_step": 2099, "train_global_avg_loss": 0.25166085112132475, "train_cur_epoch_loss": 169.23534133005342, "train_cur_epoch_avg_loss": 0.08062665141974913, "train_cur_epoch_time": 854.5994429588318, "train_cur_epoch_avg_time": 0.407145994739796, "epoch": 11, "step": 23089} ################################################## Training, Epoch: 0012, Batch: 000911, Sample Num: 14576, Cur Loss: 0.00029330, Cur Avg Loss: 0.05520845, Log Avg loss: 0.05774557, Global Avg Loss: 0.24420385, Time: 0.4070 Steps: 24000, Updated lr: 0.000156 Training, Epoch: 0012, Batch: 001911, Sample Num: 30576, Cur Loss: 0.03184273, Cur Avg Loss: 0.06543167, Log Avg loss: 0.07474502, Global Avg Loss: 0.23742549, Time: 0.4055 Steps: 25000, Updated lr: 0.000154 ***** Running evaluation checkpoint-25188 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-25188 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 851.818565, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.039577, "eval_total_loss": 930.421268, "eval_acc": 0.907224, "eval_prec": 0.899192, "eval_recall": 0.917284, "eval_f1": 0.908148, "eval_roc_auc": 0.962154, "eval_pr_auc": 0.96257, "eval_confusion_matrix": {"tn": 6421, "fp": 736, "fn": 592, "tp": 6565}, "eval_mcc2": 0.814612, "eval_mcc": 0.814612, "eval_sn": 0.917284, "eval_sp": 0.897164, "update_flag": true, "test_avg_loss": 7.284875, "test_total_loss": 87.418503, "test_acc": 0.7, "test_prec": 0.595238, "test_recall": 0.403226, "test_f1": 0.480769, "test_roc_auc": 0.638874, "test_pr_auc": 0.538216, "test_confusion_matrix": {"tn": 101, "fp": 17, "fn": 37, "tp": 25}, "test_mcc2": 0.291164, "test_mcc": 0.291164, "test_sn": 0.403226, "test_sp": 0.855932, "lr": 0.00015346224146224147, "cur_epoch_step": 2099, "train_global_avg_loss": 0.23613671614988035, "train_cur_epoch_loss": 137.21421484291864, "train_cur_epoch_avg_loss": 0.06537123146399172, "train_cur_epoch_time": 851.8185648918152, "train_cur_epoch_avg_time": 0.4058211362038186, "epoch": 12, "step": 25188} ################################################## Training, Epoch: 0013, Batch: 000812, Sample Num: 12992, Cur Loss: 0.36328930, Cur Avg Loss: 0.06541257, Log Avg loss: 0.06528930, Global Avg Loss: 0.23080487, Time: 0.4049 Steps: 26000, Updated lr: 0.000152 Training, Epoch: 0013, Batch: 001812, Sample Num: 28992, Cur Loss: 0.00055299, Cur Avg Loss: 0.06335020, Log Avg loss: 0.06167556, Global Avg Loss: 0.22454082, Time: 0.4052 Steps: 27000, Updated lr: 0.000150 ***** Running evaluation checkpoint-27287 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-27287 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.926320, Avg time per batch (s): 0.400000 {"eval_avg_loss": 0.895148, "eval_total_loss": 801.157712, "eval_acc": 0.906944, "eval_prec": 0.89513, "eval_recall": 0.921895, "eval_f1": 0.908315, "eval_roc_auc": 0.96136, "eval_pr_auc": 0.960065, "eval_confusion_matrix": {"tn": 6384, "fp": 773, "fn": 559, "tp": 6598}, "eval_mcc2": 0.814253, "eval_mcc": 0.814253, "eval_sn": 0.921895, "eval_sp": 0.891994, "update_flag": true, "test_avg_loss": 6.42454, "test_total_loss": 77.094476, "test_acc": 0.688889, "test_prec": 0.555556, "test_recall": 0.483871, "test_f1": 0.517241, "test_roc_auc": 0.652064, "test_pr_auc": 0.553883, "test_confusion_matrix": {"tn": 94, "fp": 24, "fn": 32, "tp": 30}, "test_mcc2": 0.290843, "test_mcc": 0.290843, "test_sn": 0.483871, "test_sp": 0.79661, "lr": 0.00014942376142376143, "cur_epoch_step": 2099, "train_global_avg_loss": 0.22298833599679294, "train_cur_epoch_loss": 136.87111796130148, "train_cur_epoch_avg_loss": 0.06520777415974344, "train_cur_epoch_time": 849.9263203144073, "train_cur_epoch_avg_time": 0.4049196380726095, "epoch": 13, "step": 27287} ################################################## Training, Epoch: 0014, Batch: 000713, Sample Num: 11408, Cur Loss: 0.00035967, Cur Avg Loss: 0.04934561, Log Avg loss: 0.05726397, Global Avg Loss: 0.21856665, Time: 0.4056 Steps: 28000, Updated lr: 0.000148 Training, Epoch: 0014, Batch: 001713, Sample Num: 27408, Cur Loss: 0.09704811, Cur Avg Loss: 0.05549460, Log Avg loss: 0.05987883, Global Avg Loss: 0.21309465, Time: 0.4063 Steps: 29000, Updated lr: 0.000146 ***** Running evaluation checkpoint-29386 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-29386 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.894786, Avg time per batch (s): 0.400000 {"eval_avg_loss": 0.956904, "eval_total_loss": 856.4291, "eval_acc": 0.909529, "eval_prec": 0.899645, "eval_recall": 0.921895, "eval_f1": 0.910634, "eval_roc_auc": 0.960145, "eval_pr_auc": 0.958944, "eval_confusion_matrix": {"tn": 6421, "fp": 736, "fn": 559, "tp": 6598}, "eval_mcc2": 0.819309, "eval_mcc": 0.819309, "eval_sn": 0.921895, "eval_sp": 0.897164, "update_flag": true, "test_avg_loss": 7.0808, "test_total_loss": 84.969603, "test_acc": 0.716667, "test_prec": 0.622222, "test_recall": 0.451613, "test_f1": 0.523364, "test_roc_auc": 0.648032, "test_pr_auc": 0.561583, "test_confusion_matrix": {"tn": 101, "fp": 17, "fn": 34, "tp": 28}, "test_mcc2": 0.337499, "test_mcc": 0.337499, "test_sn": 0.451613, "test_sp": 0.855932, "lr": 0.0001453852813852814, "cur_epoch_step": 2099, "train_global_avg_loss": 0.21117687544052638, "train_cur_epoch_loss": 120.96093735081786, "train_cur_epoch_avg_loss": 0.05762788820906044, "train_cur_epoch_time": 849.8947858810425, "train_cur_epoch_avg_time": 0.40490461452169724, "epoch": 14, "step": 29386} ################################################## Training, Epoch: 0015, Batch: 000614, Sample Num: 9824, Cur Loss: 0.00054974, Cur Avg Loss: 0.03741980, Log Avg loss: 0.04887445, Global Avg Loss: 0.20762065, Time: 0.4066 Steps: 30000, Updated lr: 0.000144 Training, Epoch: 0015, Batch: 001614, Sample Num: 25824, Cur Loss: 0.07493325, Cur Avg Loss: 0.04694109, Log Avg loss: 0.05278716, Global Avg Loss: 0.20262602, Time: 0.4051 Steps: 31000, Updated lr: 0.000142 ***** Running evaluation checkpoint-31485 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-31485 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 851.911523, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.09514, "eval_total_loss": 980.149853, "eval_acc": 0.906525, "eval_prec": 0.891325, "eval_recall": 0.925947, "eval_f1": 0.908306, "eval_roc_auc": 0.960288, "eval_pr_auc": 0.959422, "eval_confusion_matrix": {"tn": 6349, "fp": 808, "fn": 530, "tp": 6627}, "eval_mcc2": 0.813664, "eval_mcc": 0.813664, "eval_sn": 0.925947, "eval_sp": 0.887104, "update_flag": false, "test_avg_loss": 8.179267, "test_total_loss": 98.151202, "test_acc": 0.655556, "test_prec": 0.5, "test_recall": 0.451613, "test_f1": 0.474576, "test_roc_auc": 0.62213, "test_pr_auc": 0.544371, "test_confusion_matrix": {"tn": 90, "fp": 28, "fn": 34, "tp": 28}, "test_mcc2": 0.219991, "test_mcc": 0.219991, "test_sn": 0.451613, "test_sp": 0.762712, "lr": 0.00014134680134680137, "cur_epoch_step": 2099, "train_global_avg_loss": 0.20018369184048349, "train_cur_epoch_loss": 97.13987590231395, "train_cur_epoch_avg_loss": 0.046279121439882775, "train_cur_epoch_time": 851.9115233421326, "train_cur_epoch_avg_time": 0.40586542322159724, "epoch": 15, "step": 31485} ################################################## Training, Epoch: 0016, Batch: 000515, Sample Num: 8240, Cur Loss: 0.00614715, Cur Avg Loss: 0.04100793, Log Avg loss: 0.04249604, Global Avg Loss: 0.19762196, Time: 0.4048 Steps: 32000, Updated lr: 0.000140 Training, Epoch: 0016, Batch: 001515, Sample Num: 24240, Cur Loss: 0.00083612, Cur Avg Loss: 0.03889870, Log Avg loss: 0.03781245, Global Avg Loss: 0.19277924, Time: 0.4079 Steps: 33000, Updated lr: 0.000138 ***** Running evaluation checkpoint-33584 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-33584 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 854.863812, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.239214, "eval_total_loss": 1109.096965, "eval_acc": 0.911346, "eval_prec": 0.906405, "eval_recall": 0.917424, "eval_f1": 0.911881, "eval_roc_auc": 0.962371, "eval_pr_auc": 0.959479, "eval_confusion_matrix": {"tn": 6479, "fp": 678, "fn": 591, "tp": 6566}, "eval_mcc2": 0.822752, "eval_mcc": 0.822752, "eval_sn": 0.917424, "eval_sp": 0.905268, "update_flag": true, "test_avg_loss": 9.633122, "test_total_loss": 115.597464, "test_acc": 0.666667, "test_prec": 0.518519, "test_recall": 0.451613, "test_f1": 0.482759, "test_roc_auc": 0.612903, "test_pr_auc": 0.521625, "test_confusion_matrix": {"tn": 92, "fp": 26, "fn": 34, "tp": 28}, "test_mcc2": 0.239818, "test_mcc": 0.239818, "test_sn": 0.451613, "test_sp": 0.779661, "lr": 0.0001373083213083213, "cur_epoch_step": 2099, "train_global_avg_loss": 0.19023807997669903, "train_cur_epoch_loss": 86.17214033983629, "train_cur_epoch_avg_loss": 0.04105390201993153, "train_cur_epoch_time": 854.8638117313385, "train_cur_epoch_avg_time": 0.40727194460759336, "epoch": 16, "step": 33584} ################################################## Training, Epoch: 0017, Batch: 000416, Sample Num: 6656, Cur Loss: 4.78899384, Cur Avg Loss: 0.04807436, Log Avg loss: 0.04723954, Global Avg Loss: 0.18849867, Time: 0.4087 Steps: 34000, Updated lr: 0.000137 Training, Epoch: 0017, Batch: 001416, Sample Num: 22656, Cur Loss: 0.00000402, Cur Avg Loss: 0.04074711, Log Avg loss: 0.03769897, Global Avg Loss: 0.18419010, Time: 0.4074 Steps: 35000, Updated lr: 0.000135 ***** Running evaluation checkpoint-35683 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-35683 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 855.259135, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.074062, "eval_total_loss": 961.285249, "eval_acc": 0.901705, "eval_prec": 0.877594, "eval_recall": 0.933631, "eval_f1": 0.904746, "eval_roc_auc": 0.960162, "eval_pr_auc": 0.956137, "eval_confusion_matrix": {"tn": 6225, "fp": 932, "fn": 475, "tp": 6682}, "eval_mcc2": 0.805052, "eval_mcc": 0.805052, "eval_sn": 0.933631, "eval_sp": 0.869778, "update_flag": false, "test_avg_loss": 7.748186, "test_total_loss": 92.978227, "test_acc": 0.661111, "test_prec": 0.507937, "test_recall": 0.516129, "test_f1": 0.512, "test_roc_auc": 0.633338, "test_pr_auc": 0.542964, "test_confusion_matrix": {"tn": 87, "fp": 31, "fn": 30, "tp": 32}, "test_mcc2": 0.25247, "test_mcc": 0.25247, "test_sn": 0.516129, "test_sp": 0.737288, "lr": 0.00013326984126984127, "cur_epoch_step": 2099, "train_global_avg_loss": 0.18166313887498228, "train_cur_epoch_loss": 93.33010653853214, "train_cur_epoch_avg_loss": 0.044464081247514116, "train_cur_epoch_time": 855.2591347694397, "train_cur_epoch_avg_time": 0.40746028335847534, "epoch": 17, "step": 35683} ################################################## Training, Epoch: 0018, Batch: 000317, Sample Num: 5072, Cur Loss: 0.00038281, Cur Avg Loss: 0.04039041, Log Avg loss: 0.04843596, Global Avg Loss: 0.18041915, Time: 0.4053 Steps: 36000, Updated lr: 0.000133 Training, Epoch: 0018, Batch: 001317, Sample Num: 21072, Cur Loss: 0.01828199, Cur Avg Loss: 0.03573571, Log Avg loss: 0.03426017, Global Avg Loss: 0.17646891, Time: 0.4052 Steps: 37000, Updated lr: 0.000131 ***** Running evaluation checkpoint-37782 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-37782 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 850.402728, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.132033, "eval_total_loss": 1013.169224, "eval_acc": 0.895068, "eval_prec": 0.864509, "eval_recall": 0.936985, "eval_f1": 0.899289, "eval_roc_auc": 0.955114, "eval_pr_auc": 0.94616, "eval_confusion_matrix": {"tn": 6106, "fp": 1051, "fn": 451, "tp": 6706}, "eval_mcc2": 0.792927, "eval_mcc": 0.792927, "eval_sn": 0.936985, "eval_sp": 0.853151, "update_flag": false, "test_avg_loss": 7.82254, "test_total_loss": 93.87048, "test_acc": 0.661111, "test_prec": 0.507692, "test_recall": 0.532258, "test_f1": 0.519685, "test_roc_auc": 0.637097, "test_pr_auc": 0.549294, "test_confusion_matrix": {"tn": 86, "fp": 32, "fn": 29, "tp": 33}, "test_mcc2": 0.25828, "test_mcc": 0.25828, "test_sn": 0.532258, "test_sp": 0.728814, "lr": 0.00012923136123136123, "cur_epoch_step": 2099, "train_global_avg_loss": 0.17353323472906598, "train_cur_epoch_loss": 74.14689005758312, "train_cur_epoch_avg_loss": 0.035324864248491246, "train_cur_epoch_time": 850.4027278423309, "train_cur_epoch_avg_time": 0.4051466068805769, "epoch": 18, "step": 37782} ################################################## Training, Epoch: 0019, Batch: 000218, Sample Num: 3488, Cur Loss: 0.00000006, Cur Avg Loss: 0.02365802, Log Avg loss: 0.03224041, Global Avg Loss: 0.17267342, Time: 0.4020 Steps: 38000, Updated lr: 0.000129 Training, Epoch: 0019, Batch: 001218, Sample Num: 19488, Cur Loss: 0.00012800, Cur Avg Loss: 0.02570654, Log Avg loss: 0.02615312, Global Avg Loss: 0.16891649, Time: 0.4058 Steps: 39000, Updated lr: 0.000127 ***** Running evaluation checkpoint-39881 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-39881 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.660769, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.37344, "eval_total_loss": 1229.228556, "eval_acc": 0.901216, "eval_prec": 0.878676, "eval_recall": 0.930977, "eval_f1": 0.904071, "eval_roc_auc": 0.959281, "eval_pr_auc": 0.951453, "eval_confusion_matrix": {"tn": 6237, "fp": 920, "fn": 494, "tp": 6663}, "eval_mcc2": 0.803856, "eval_mcc": 0.803856, "eval_sn": 0.930977, "eval_sp": 0.871455, "update_flag": false, "test_avg_loss": 8.658156, "test_total_loss": 103.897874, "test_acc": 0.705556, "test_prec": 0.581818, "test_recall": 0.516129, "test_f1": 0.547009, "test_roc_auc": 0.671132, "test_pr_auc": 0.601011, "test_confusion_matrix": {"tn": 95, "fp": 23, "fn": 30, "tp": 32}, "test_mcc2": 0.331356, "test_mcc": 0.331356, "test_sn": 0.516129, "test_sp": 0.805085, "lr": 0.00012519288119288119, "cur_epoch_step": 2099, "train_global_avg_loss": 0.16602785426822833, "train_cur_epoch_loss": 64.92418153764028, "train_cur_epoch_avg_loss": 0.030931005973149253, "train_cur_epoch_time": 849.6607689857483, "train_cur_epoch_avg_time": 0.40479312481455376, "epoch": 19, "step": 39881} ################################################## Training, Epoch: 0020, Batch: 000119, Sample Num: 1904, Cur Loss: 0.01030638, Cur Avg Loss: 0.01792710, Log Avg loss: 0.03574694, Global Avg Loss: 0.16558725, Time: 0.4040 Steps: 40000, Updated lr: 0.000125 Training, Epoch: 0020, Batch: 001119, Sample Num: 17904, Cur Loss: 0.00001499, Cur Avg Loss: 0.03119762, Log Avg loss: 0.03277681, Global Avg Loss: 0.16234798, Time: 0.4056 Steps: 41000, Updated lr: 0.000123 ***** Running evaluation checkpoint-41980 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-41980 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.368581, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.371285, "eval_total_loss": 1227.30004, "eval_acc": 0.906455, "eval_prec": 0.89226, "eval_recall": 0.924549, "eval_f1": 0.908118, "eval_roc_auc": 0.960598, "eval_pr_auc": 0.953555, "eval_confusion_matrix": {"tn": 6358, "fp": 799, "fn": 540, "tp": 6617}, "eval_mcc2": 0.813443, "eval_mcc": 0.813443, "eval_sn": 0.924549, "eval_sp": 0.888361, "update_flag": false, "test_avg_loss": 9.654276, "test_total_loss": 115.851317, "test_acc": 0.711111, "test_prec": 0.604167, "test_recall": 0.467742, "test_f1": 0.527273, "test_roc_auc": 0.642701, "test_pr_auc": 0.529114, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 33, "tp": 29}, "test_mcc2": 0.329593, "test_mcc": 0.329593, "test_sn": 0.467742, "test_sp": 0.838983, "lr": 0.00012115440115440116, "cur_epoch_step": 2099, "train_global_avg_loss": 0.15933260755145562, "train_cur_epoch_loss": 67.42600893888121, "train_cur_epoch_avg_loss": 0.032122919932768564, "train_cur_epoch_time": 849.3685805797577, "train_cur_epoch_avg_time": 0.4046539211909279, "epoch": 20, "step": 41980} ################################################## Training, Epoch: 0021, Batch: 000020, Sample Num: 320, Cur Loss: 0.00021404, Cur Avg Loss: 0.02437829, Log Avg loss: 0.03300344, Global Avg Loss: 0.15926834, Time: 0.4016 Steps: 42000, Updated lr: 0.000121 Training, Epoch: 0021, Batch: 001020, Sample Num: 16320, Cur Loss: 0.00012741, Cur Avg Loss: 0.02654094, Log Avg loss: 0.02658419, Global Avg Loss: 0.15618267, Time: 0.4040 Steps: 43000, Updated lr: 0.000119 Training, Epoch: 0021, Batch: 002020, Sample Num: 32320, Cur Loss: 0.00345512, Cur Avg Loss: 0.02560766, Log Avg loss: 0.02465572, Global Avg Loss: 0.15319342, Time: 0.4090 Steps: 44000, Updated lr: 0.000117 ***** Running evaluation checkpoint-44079 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-44079 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 850.263905, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.499871, "eval_total_loss": 1342.384271, "eval_acc": 0.90897, "eval_prec": 0.904505, "eval_recall": 0.914489, "eval_f1": 0.90947, "eval_roc_auc": 0.959921, "eval_pr_auc": 0.954113, "eval_confusion_matrix": {"tn": 6466, "fp": 691, "fn": 612, "tp": 6545}, "eval_mcc2": 0.81799, "eval_mcc": 0.81799, "eval_sn": 0.914489, "eval_sp": 0.903451, "update_flag": false, "test_avg_loss": 10.192019, "test_total_loss": 122.304229, "test_acc": 0.694444, "test_prec": 0.566038, "test_recall": 0.483871, "test_f1": 0.521739, "test_roc_auc": 0.659513, "test_pr_auc": 0.56437, "test_confusion_matrix": {"tn": 95, "fp": 23, "fn": 32, "tp": 30}, "test_mcc2": 0.301251, "test_mcc": 0.301251, "test_sn": 0.483871, "test_sp": 0.805085, "lr": 0.0001171159211159211, "cur_epoch_step": 2099, "train_global_avg_loss": 0.15296388496429533, "train_cur_epoch_loss": 53.712220331057964, "train_cur_epoch_avg_loss": 0.025589433221085263, "train_cur_epoch_time": 850.2639050483704, "train_cur_epoch_avg_time": 0.40508046929412594, "epoch": 21, "step": 44079} ################################################## Training, Epoch: 0022, Batch: 000921, Sample Num: 14736, Cur Loss: 0.06945743, Cur Avg Loss: 0.02263191, Log Avg loss: 0.02282873, Global Avg Loss: 0.15029642, Time: 0.4084 Steps: 45000, Updated lr: 0.000115 Training, Epoch: 0022, Batch: 001921, Sample Num: 30736, Cur Loss: 0.00077053, Cur Avg Loss: 0.02503533, Log Avg loss: 0.02724888, Global Avg Loss: 0.14762148, Time: 0.4092 Steps: 46000, Updated lr: 0.000113 ***** Running evaluation checkpoint-46178 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-46178 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 856.774312, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.624996, "eval_total_loss": 1454.371008, "eval_acc": 0.912673, "eval_prec": 0.923441, "eval_recall": 0.899958, "eval_f1": 0.911548, "eval_roc_auc": 0.962768, "eval_pr_auc": 0.959014, "eval_confusion_matrix": {"tn": 6623, "fp": 534, "fn": 716, "tp": 6441}, "eval_mcc2": 0.825613, "eval_mcc": 0.825613, "eval_sn": 0.899958, "eval_sp": 0.925388, "update_flag": false, "test_avg_loss": 11.097855, "test_total_loss": 133.174259, "test_acc": 0.694444, "test_prec": 0.571429, "test_recall": 0.451613, "test_f1": 0.504505, "test_roc_auc": 0.65097, "test_pr_auc": 0.562641, "test_confusion_matrix": {"tn": 97, "fp": 21, "fn": 34, "tp": 28}, "test_mcc2": 0.292142, "test_mcc": 0.292142, "test_sn": 0.451613, "test_sp": 0.822034, "lr": 0.00011307744107744109, "cur_epoch_step": 2099, "train_global_avg_loss": 0.1470986471578416, "train_cur_epoch_loss": 50.226243113623454, "train_cur_epoch_avg_loss": 0.023928653222307507, "train_cur_epoch_time": 856.7743120193481, "train_cur_epoch_avg_time": 0.40818214007591624, "epoch": 22, "step": 46178} ################################################## Training, Epoch: 0023, Batch: 000822, Sample Num: 13152, Cur Loss: 0.00000304, Cur Avg Loss: 0.02024665, Log Avg loss: 0.01877612, Global Avg Loss: 0.14488009, Time: 0.4039 Steps: 47000, Updated lr: 0.000111 Training, Epoch: 0023, Batch: 001822, Sample Num: 29152, Cur Loss: 0.00009682, Cur Avg Loss: 0.02137034, Log Avg loss: 0.02229402, Global Avg Loss: 0.14232621, Time: 0.4051 Steps: 48000, Updated lr: 0.000110 ***** Running evaluation checkpoint-48277 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-48277 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 847.743153, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.757614, "eval_total_loss": 1573.064813, "eval_acc": 0.912393, "eval_prec": 0.917409, "eval_recall": 0.906385, "eval_f1": 0.911864, "eval_roc_auc": 0.961332, "eval_pr_auc": 0.955969, "eval_confusion_matrix": {"tn": 6573, "fp": 584, "fn": 670, "tp": 6487}, "eval_mcc2": 0.824846, "eval_mcc": 0.824846, "eval_sn": 0.906385, "eval_sp": 0.918402, "update_flag": false, "test_avg_loss": 11.489512, "test_total_loss": 137.874142, "test_acc": 0.694444, "test_prec": 0.568627, "test_recall": 0.467742, "test_f1": 0.513274, "test_roc_auc": 0.653636, "test_pr_auc": 0.554074, "test_confusion_matrix": {"tn": 96, "fp": 22, "fn": 33, "tp": 29}, "test_mcc2": 0.296639, "test_mcc": 0.296639, "test_sn": 0.467742, "test_sp": 0.813559, "lr": 0.00010903896103896105, "cur_epoch_step": 2099, "train_global_avg_loss": 0.1416137835316888, "train_cur_epoch_loss": 43.96729910451606, "train_cur_epoch_avg_loss": 0.02094678375632018, "train_cur_epoch_time": 847.7431530952454, "train_cur_epoch_avg_time": 0.40387953934980725, "epoch": 23, "step": 48277} ################################################## Training, Epoch: 0024, Batch: 000723, Sample Num: 11568, Cur Loss: 0.00013058, Cur Avg Loss: 0.01598318, Log Avg loss: 0.01658637, Global Avg Loss: 0.13976009, Time: 0.4082 Steps: 49000, Updated lr: 0.000108 Training, Epoch: 0024, Batch: 001723, Sample Num: 27568, Cur Loss: 0.01171990, Cur Avg Loss: 0.01915791, Log Avg loss: 0.02145325, Global Avg Loss: 0.13739395, Time: 0.4087 Steps: 50000, Updated lr: 0.000106 ***** Running evaluation checkpoint-50376 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-50376 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 853.793442, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.428329, "eval_total_loss": 1278.354218, "eval_acc": 0.898491, "eval_prec": 0.869239, "eval_recall": 0.938103, "eval_f1": 0.902359, "eval_roc_auc": 0.955961, "eval_pr_auc": 0.942976, "eval_confusion_matrix": {"tn": 6147, "fp": 1010, "fn": 443, "tp": 6714}, "eval_mcc2": 0.799495, "eval_mcc": 0.799495, "eval_sn": 0.938103, "eval_sp": 0.858879, "update_flag": false, "test_avg_loss": 9.244621, "test_total_loss": 110.935456, "test_acc": 0.672222, "test_prec": 0.522388, "test_recall": 0.564516, "test_f1": 0.542636, "test_roc_auc": 0.672977, "test_pr_auc": 0.592666, "test_confusion_matrix": {"tn": 86, "fp": 32, "fn": 27, "tp": 35}, "test_mcc2": 0.288348, "test_mcc": 0.288348, "test_sn": 0.564516, "test_sp": 0.728814, "lr": 0.00010500048100048101, "cur_epoch_step": 2099, "train_global_avg_loss": 0.1367628495843487, "train_cur_epoch_loss": 52.87668310179926, "train_cur_epoch_avg_loss": 0.025191368795521325, "train_cur_epoch_time": 853.7934420108795, "train_cur_epoch_avg_time": 0.40676200191085254, "epoch": 24, "step": 50376} ################################################## Training, Epoch: 0025, Batch: 000624, Sample Num: 9984, Cur Loss: 0.00005074, Cur Avg Loss: 0.02608769, Log Avg loss: 0.03614632, Global Avg Loss: 0.13540871, Time: 0.4078 Steps: 51000, Updated lr: 0.000104 Training, Epoch: 0025, Batch: 001624, Sample Num: 25984, Cur Loss: 0.00000077, Cur Avg Loss: 0.02181333, Log Avg loss: 0.01914613, Global Avg Loss: 0.13317289, Time: 0.4050 Steps: 52000, Updated lr: 0.000102 ***** Running evaluation checkpoint-52475 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-52475 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 851.456581, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.537751, "eval_total_loss": 1376.286916, "eval_acc": 0.90932, "eval_prec": 0.898517, "eval_recall": 0.922873, "eval_f1": 0.910532, "eval_roc_auc": 0.959623, "eval_pr_auc": 0.951664, "eval_confusion_matrix": {"tn": 6411, "fp": 746, "fn": 552, "tp": 6605}, "eval_mcc2": 0.81894, "eval_mcc": 0.81894, "eval_sn": 0.922873, "eval_sp": 0.895766, "update_flag": false, "test_avg_loss": 11.092452, "test_total_loss": 133.109427, "test_acc": 0.688889, "test_prec": 0.553571, "test_recall": 0.5, "test_f1": 0.525424, "test_roc_auc": 0.645025, "test_pr_auc": 0.56856, "test_confusion_matrix": {"tn": 93, "fp": 25, "fn": 31, "tp": 31}, "test_mcc2": 0.295753, "test_mcc": 0.295753, "test_sn": 0.5, "test_sp": 0.788136, "lr": 0.00010096200096200097, "cur_epoch_step": 2099, "train_global_avg_loss": 0.13214733319937103, "train_cur_epoch_loss": 44.865998975834025, "train_cur_epoch_avg_loss": 0.02137493995990187, "train_cur_epoch_time": 851.4565811157227, "train_cur_epoch_avg_time": 0.4056486808555134, "epoch": 25, "step": 52475} ################################################## Training, Epoch: 0026, Batch: 000525, Sample Num: 8400, Cur Loss: 0.00007909, Cur Avg Loss: 0.02122604, Log Avg loss: 0.02058482, Global Avg Loss: 0.13104858, Time: 0.4043 Steps: 53000, Updated lr: 0.000100 Training, Epoch: 0026, Batch: 001525, Sample Num: 24400, Cur Loss: 0.00018518, Cur Avg Loss: 0.01585444, Log Avg loss: 0.01303435, Global Avg Loss: 0.12886314, Time: 0.4119 Steps: 54000, Updated lr: 0.000098 ***** Running evaluation checkpoint-54574 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-54574 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.872018, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.622841, "eval_total_loss": 1452.442699, "eval_acc": 0.912184, "eval_prec": 0.909722, "eval_recall": 0.915188, "eval_f1": 0.912447, "eval_roc_auc": 0.962345, "eval_pr_auc": 0.954294, "eval_confusion_matrix": {"tn": 6507, "fp": 650, "fn": 607, "tp": 6550}, "eval_mcc2": 0.824383, "eval_mcc": 0.824383, "eval_sn": 0.915188, "eval_sp": 0.90918, "update_flag": true, "test_avg_loss": 12.0051, "test_total_loss": 144.061206, "test_acc": 0.711111, "test_prec": 0.604167, "test_recall": 0.467742, "test_f1": 0.527273, "test_roc_auc": 0.644956, "test_pr_auc": 0.542197, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 33, "tp": 29}, "test_mcc2": 0.329593, "test_mcc": 0.329593, "test_sn": 0.467742, "test_sp": 0.838983, "lr": 9.692352092352093e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.12775503686919515, "train_cur_epoch_loss": 37.672072462459525, "train_cur_epoch_avg_loss": 0.017947628614797297, "train_cur_epoch_time": 852.872017621994, "train_cur_epoch_avg_time": 0.4063230193530224, "epoch": 26, "step": 54574} ################################################## Training, Epoch: 0027, Batch: 000426, Sample Num: 6816, Cur Loss: 0.04054213, Cur Avg Loss: 0.01160486, Log Avg loss: 0.01843772, Global Avg Loss: 0.12685540, Time: 0.4052 Steps: 55000, Updated lr: 0.000096 Training, Epoch: 0027, Batch: 001426, Sample Num: 22816, Cur Loss: 0.00051485, Cur Avg Loss: 0.01268546, Log Avg loss: 0.01314580, Global Avg Loss: 0.12482487, Time: 0.4047 Steps: 56000, Updated lr: 0.000094 ***** Running evaluation checkpoint-56673 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-56673 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.044215, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.732468, "eval_total_loss": 1550.558485, "eval_acc": 0.913092, "eval_prec": 0.915998, "eval_recall": 0.909599, "eval_f1": 0.912787, "eval_roc_auc": 0.961386, "eval_pr_auc": 0.955516, "eval_confusion_matrix": {"tn": 6560, "fp": 597, "fn": 647, "tp": 6510}, "eval_mcc2": 0.826204, "eval_mcc": 0.826204, "eval_sn": 0.909599, "eval_sp": 0.916585, "update_flag": true, "test_avg_loss": 11.689109, "test_total_loss": 140.269305, "test_acc": 0.716667, "test_prec": 0.622222, "test_recall": 0.451613, "test_f1": 0.523364, "test_roc_auc": 0.644341, "test_pr_auc": 0.569029, "test_confusion_matrix": {"tn": 101, "fp": 17, "fn": 34, "tp": 28}, "test_mcc2": 0.337499, "test_mcc": 0.337499, "test_sn": 0.451613, "test_sp": 0.855932, "lr": 9.288504088504088e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.12348480730790642, "train_cur_epoch_loss": 26.151102461509012, "train_cur_epoch_avg_loss": 0.012458838714392097, "train_cur_epoch_time": 849.0442147254944, "train_cur_epoch_avg_time": 0.4044993876729368, "epoch": 27, "step": 56673} ################################################## Training, Epoch: 0028, Batch: 000327, Sample Num: 5232, Cur Loss: 0.00078544, Cur Avg Loss: 0.01162492, Log Avg loss: 0.01186298, Global Avg Loss: 0.12284308, Time: 0.4050 Steps: 57000, Updated lr: 0.000092 Training, Epoch: 0028, Batch: 001327, Sample Num: 21232, Cur Loss: 0.00000025, Cur Avg Loss: 0.01194141, Log Avg loss: 0.01204490, Global Avg Loss: 0.12093277, Time: 0.4034 Steps: 58000, Updated lr: 0.000090 ***** Running evaluation checkpoint-58772 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-58772 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.185251, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.687155, "eval_total_loss": 1510.004161, "eval_acc": 0.910996, "eval_prec": 0.905109, "eval_recall": 0.918262, "eval_f1": 0.911638, "eval_roc_auc": 0.96039, "eval_pr_auc": 0.952306, "eval_confusion_matrix": {"tn": 6468, "fp": 689, "fn": 585, "tp": 6572}, "eval_mcc2": 0.822079, "eval_mcc": 0.822079, "eval_sn": 0.918262, "eval_sp": 0.903731, "update_flag": false, "test_avg_loss": 12.045862, "test_total_loss": 144.550344, "test_acc": 0.688889, "test_prec": 0.557692, "test_recall": 0.467742, "test_f1": 0.508772, "test_roc_auc": 0.627529, "test_pr_auc": 0.520458, "test_confusion_matrix": {"tn": 95, "fp": 23, "fn": 33, "tp": 29}, "test_mcc2": 0.286034, "test_mcc": 0.286034, "test_sn": 0.467742, "test_sp": 0.805085, "lr": 8.884656084656085e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.11954820430147903, "train_cur_epoch_loss": 27.832578645520044, "train_cur_epoch_avg_loss": 0.013259923127927606, "train_cur_epoch_time": 849.1852505207062, "train_cur_epoch_avg_time": 0.40456657957156084, "epoch": 28, "step": 58772} ################################################## Training, Epoch: 0029, Batch: 000228, Sample Num: 3648, Cur Loss: 0.00000328, Cur Avg Loss: 0.00314889, Log Avg loss: 0.01270427, Global Avg Loss: 0.11909839, Time: 0.4030 Steps: 59000, Updated lr: 0.000088 Training, Epoch: 0029, Batch: 001228, Sample Num: 19648, Cur Loss: 0.00002071, Cur Avg Loss: 0.00813593, Log Avg loss: 0.00927298, Global Avg Loss: 0.11726797, Time: 0.4054 Steps: 60000, Updated lr: 0.000086 ***** Running evaluation checkpoint-60871 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-60871 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 848.836171, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.937885, "eval_total_loss": 1734.407172, "eval_acc": 0.910787, "eval_prec": 0.908901, "eval_recall": 0.913092, "eval_f1": 0.910992, "eval_roc_auc": 0.959209, "eval_pr_auc": 0.949521, "eval_confusion_matrix": {"tn": 6502, "fp": 655, "fn": 622, "tp": 6535}, "eval_mcc2": 0.821582, "eval_mcc": 0.821582, "eval_sn": 0.913092, "eval_sp": 0.908481, "update_flag": false, "test_avg_loss": 13.224507, "test_total_loss": 158.694083, "test_acc": 0.683333, "test_prec": 0.54902, "test_recall": 0.451613, "test_f1": 0.495575, "test_roc_auc": 0.643384, "test_pr_auc": 0.528326, "test_confusion_matrix": {"tn": 95, "fp": 23, "fn": 34, "tp": 28}, "test_mcc2": 0.270694, "test_mcc": 0.270694, "test_sn": 0.451613, "test_sp": 0.805085, "lr": 8.480808080808081e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.11572757104887593, "train_cur_epoch_loss": 18.365914109577087, "train_cur_epoch_avg_loss": 0.00874983997597765, "train_cur_epoch_time": 848.8361713886261, "train_cur_epoch_avg_time": 0.40440027221945024, "epoch": 29, "step": 60871} ################################################## Training, Epoch: 0030, Batch: 000129, Sample Num: 2064, Cur Loss: 0.00000136, Cur Avg Loss: 0.00806871, Log Avg loss: 0.00941585, Global Avg Loss: 0.11549990, Time: 0.4023 Steps: 61000, Updated lr: 0.000085 Training, Epoch: 0030, Batch: 001129, Sample Num: 18064, Cur Loss: 0.00001876, Cur Avg Loss: 0.00819977, Log Avg loss: 0.00821667, Global Avg Loss: 0.11376952, Time: 0.4051 Steps: 62000, Updated lr: 0.000083 ***** Running evaluation checkpoint-62970 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-62970 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 848.991176, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.809265, "eval_total_loss": 1619.292154, "eval_acc": 0.915537, "eval_prec": 0.920413, "eval_recall": 0.909739, "eval_f1": 0.915045, "eval_roc_auc": 0.961033, "eval_pr_auc": 0.952993, "eval_confusion_matrix": {"tn": 6594, "fp": 563, "fn": 646, "tp": 6511}, "eval_mcc2": 0.83113, "eval_mcc": 0.83113, "eval_sn": 0.909739, "eval_sp": 0.921336, "update_flag": true, "test_avg_loss": 12.094611, "test_total_loss": 145.135335, "test_acc": 0.727778, "test_prec": 0.638298, "test_recall": 0.483871, "test_f1": 0.550459, "test_roc_auc": 0.662042, "test_pr_auc": 0.578641, "test_confusion_matrix": {"tn": 101, "fp": 17, "fn": 32, "tp": 30}, "test_mcc2": 0.367612, "test_mcc": 0.367612, "test_sn": 0.483871, "test_sp": 0.855932, "lr": 8.076960076960077e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.11216508733436313, "train_cur_epoch_loss": 18.582572128698775, "train_cur_epoch_avg_loss": 0.008853059613482027, "train_cur_epoch_time": 848.9911758899689, "train_cur_epoch_avg_time": 0.40447411905191466, "epoch": 30, "step": 62970} ################################################## Training, Epoch: 0031, Batch: 000030, Sample Num: 480, Cur Loss: 0.00002879, Cur Avg Loss: 0.00577924, Log Avg loss: 0.00949841, Global Avg Loss: 0.11211443, Time: 0.4065 Steps: 63000, Updated lr: 0.000081 Training, Epoch: 0031, Batch: 001030, Sample Num: 16480, Cur Loss: 0.00000736, Cur Avg Loss: 0.00566984, Log Avg loss: 0.00566656, Global Avg Loss: 0.11045118, Time: 0.4035 Steps: 64000, Updated lr: 0.000079 Training, Epoch: 0031, Batch: 002030, Sample Num: 32480, Cur Loss: 0.00017957, Cur Avg Loss: 0.00756653, Log Avg loss: 0.00952013, Global Avg Loss: 0.10889839, Time: 0.4050 Steps: 65000, Updated lr: 0.000077 ***** Running evaluation checkpoint-65069 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-65069 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.464780, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.753849, "eval_total_loss": 1569.694664, "eval_acc": 0.908272, "eval_prec": 0.89106, "eval_recall": 0.930278, "eval_f1": 0.910247, "eval_roc_auc": 0.958784, "eval_pr_auc": 0.946321, "eval_confusion_matrix": {"tn": 6343, "fp": 814, "fn": 499, "tp": 6658}, "eval_mcc2": 0.817335, "eval_mcc": 0.817335, "eval_sn": 0.930278, "eval_sp": 0.886265, "update_flag": false, "test_avg_loss": 13.105933, "test_total_loss": 157.271199, "test_acc": 0.688889, "test_prec": 0.56, "test_recall": 0.451613, "test_f1": 0.5, "test_roc_auc": 0.639147, "test_pr_auc": 0.541722, "test_confusion_matrix": {"tn": 96, "fp": 22, "fn": 34, "tp": 28}, "test_mcc2": 0.281325, "test_mcc": 0.281325, "test_sn": 0.451613, "test_sp": 0.813559, "lr": 7.673112073112073e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.10878532644752065, "train_cur_epoch_loss": 15.516857168853935, "train_cur_epoch_avg_loss": 0.007392499842236272, "train_cur_epoch_time": 849.4647798538208, "train_cur_epoch_avg_time": 0.4046997521933401, "epoch": 31, "step": 65069} ################################################## Training, Epoch: 0032, Batch: 000931, Sample Num: 14896, Cur Loss: 0.00029948, Cur Avg Loss: 0.00437211, Log Avg loss: 0.00422723, Global Avg Loss: 0.10731247, Time: 0.4059 Steps: 66000, Updated lr: 0.000075 Training, Epoch: 0032, Batch: 001931, Sample Num: 30896, Cur Loss: 0.00000013, Cur Avg Loss: 0.00598277, Log Avg loss: 0.00748229, Global Avg Loss: 0.10582246, Time: 0.4050 Steps: 67000, Updated lr: 0.000073 ***** Running evaluation checkpoint-67168 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-67168 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 850.187085, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.871762, "eval_total_loss": 1675.227195, "eval_acc": 0.911346, "eval_prec": 0.908662, "eval_recall": 0.914629, "eval_f1": 0.911636, "eval_roc_auc": 0.960607, "eval_pr_auc": 0.951548, "eval_confusion_matrix": {"tn": 6499, "fp": 658, "fn": 611, "tp": 6546}, "eval_mcc2": 0.822709, "eval_mcc": 0.822709, "eval_sn": 0.914629, "eval_sp": 0.908062, "update_flag": false, "test_avg_loss": 13.651496, "test_total_loss": 163.817951, "test_acc": 0.711111, "test_prec": 0.613636, "test_recall": 0.435484, "test_f1": 0.509434, "test_roc_auc": 0.640172, "test_pr_auc": 0.565673, "test_confusion_matrix": {"tn": 101, "fp": 17, "fn": 35, "tp": 27}, "test_mcc2": 0.322222, "test_mcc": 0.322222, "test_sn": 0.435484, "test_sp": 0.855932, "lr": 7.26926406926407e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.1055932948788229, "train_cur_epoch_loss": 13.938023807033812, "train_cur_epoch_avg_loss": 0.006640316249182378, "train_cur_epoch_time": 850.1870853900909, "train_cur_epoch_avg_time": 0.4050438710767465, "epoch": 32, "step": 67168} ################################################## Training, Epoch: 0033, Batch: 000832, Sample Num: 13312, Cur Loss: 0.00055998, Cur Avg Loss: 0.00573871, Log Avg loss: 0.00715991, Global Avg Loss: 0.10437154, Time: 0.4038 Steps: 68000, Updated lr: 0.000071 Training, Epoch: 0033, Batch: 001832, Sample Num: 29312, Cur Loss: 0.00145543, Cur Avg Loss: 0.00966839, Log Avg loss: 0.01293787, Global Avg Loss: 0.10304642, Time: 0.4058 Steps: 69000, Updated lr: 0.000069 ***** Running evaluation checkpoint-69267 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-69267 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 848.412454, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.921943, "eval_total_loss": 1720.13906, "eval_acc": 0.910717, "eval_prec": 0.902506, "eval_recall": 0.920917, "eval_f1": 0.911618, "eval_roc_auc": 0.958487, "eval_pr_auc": 0.947386, "eval_confusion_matrix": {"tn": 6445, "fp": 712, "fn": 566, "tp": 6591}, "eval_mcc2": 0.821605, "eval_mcc": 0.821605, "eval_sn": 0.920917, "eval_sp": 0.900517, "update_flag": false, "test_avg_loss": 13.324765, "test_total_loss": 159.897179, "test_acc": 0.688889, "test_prec": 0.56, "test_recall": 0.451613, "test_f1": 0.5, "test_roc_auc": 0.642838, "test_pr_auc": 0.552836, "test_confusion_matrix": {"tn": 96, "fp": 22, "fn": 34, "tp": 28}, "test_mcc2": 0.281325, "test_mcc": 0.281325, "test_sn": 0.451613, "test_sp": 0.813559, "lr": 6.865416065416065e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.10266673007007066, "train_cur_epoch_loss": 18.925961342786927, "train_cur_epoch_avg_loss": 0.009016656189989007, "train_cur_epoch_time": 848.4124541282654, "train_cur_epoch_avg_time": 0.4041984059686829, "epoch": 33, "step": 69267} ################################################## Training, Epoch: 0034, Batch: 000733, Sample Num: 11728, Cur Loss: 0.00000073, Cur Avg Loss: 0.00348425, Log Avg loss: 0.00376743, Global Avg Loss: 0.10162815, Time: 0.4052 Steps: 70000, Updated lr: 0.000067 Training, Epoch: 0034, Batch: 001733, Sample Num: 27728, Cur Loss: 0.00006465, Cur Avg Loss: 0.00456549, Log Avg loss: 0.00535804, Global Avg Loss: 0.10027223, Time: 0.4064 Steps: 71000, Updated lr: 0.000065 ***** Running evaluation checkpoint-71366 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-71366 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 850.029224, Avg time per batch (s): 0.400000 {"eval_avg_loss": 2.085353, "eval_total_loss": 1866.390734, "eval_acc": 0.915537, "eval_prec": 0.91852, "eval_recall": 0.911974, "eval_f1": 0.915235, "eval_roc_auc": 0.962889, "eval_pr_auc": 0.955569, "eval_confusion_matrix": {"tn": 6578, "fp": 579, "fn": 630, "tp": 6527}, "eval_mcc2": 0.831096, "eval_mcc": 0.831096, "eval_sn": 0.911974, "eval_sp": 0.9191, "update_flag": true, "test_avg_loss": 14.695804, "test_total_loss": 176.349645, "test_acc": 0.7, "test_prec": 0.590909, "test_recall": 0.419355, "test_f1": 0.490566, "test_roc_auc": 0.651517, "test_pr_auc": 0.547001, "test_confusion_matrix": {"tn": 100, "fp": 18, "fn": 36, "tp": 26}, "test_mcc2": 0.295017, "test_mcc": 0.295017, "test_sn": 0.419355, "test_sp": 0.847458, "lr": 6.461568061568061e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.09980092405574607, "train_cur_epoch_loss": 10.976354398768535, "train_cur_epoch_avg_loss": 0.005229325583024552, "train_cur_epoch_time": 850.0292236804962, "train_cur_epoch_avg_time": 0.4049686630207224, "epoch": 34, "step": 71366} ################################################## Training, Epoch: 0035, Batch: 000634, Sample Num: 10144, Cur Loss: 0.00008208, Cur Avg Loss: 0.00247944, Log Avg loss: 0.00463633, Global Avg Loss: 0.09894395, Time: 0.4051 Steps: 72000, Updated lr: 0.000063 Training, Epoch: 0035, Batch: 001634, Sample Num: 26144, Cur Loss: 0.00000110, Cur Avg Loss: 0.00502634, Log Avg loss: 0.00664107, Global Avg Loss: 0.09767953, Time: 0.4063 Steps: 73000, Updated lr: 0.000061 ***** Running evaluation checkpoint-73465 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-73465 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 850.397160, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.826975, "eval_total_loss": 1635.142204, "eval_acc": 0.907363, "eval_prec": 0.885903, "eval_recall": 0.935168, "eval_f1": 0.909869, "eval_roc_auc": 0.956279, "eval_pr_auc": 0.939918, "eval_confusion_matrix": {"tn": 6295, "fp": 862, "fn": 464, "tp": 6693}, "eval_mcc2": 0.81599, "eval_mcc": 0.81599, "eval_sn": 0.935168, "eval_sp": 0.879558, "update_flag": false, "test_avg_loss": 13.061035, "test_total_loss": 156.732415, "test_acc": 0.7, "test_prec": 0.574074, "test_recall": 0.5, "test_f1": 0.534483, "test_roc_auc": 0.644068, "test_pr_auc": 0.553401, "test_confusion_matrix": {"tn": 95, "fp": 23, "fn": 31, "tp": 31}, "test_mcc2": 0.316355, "test_mcc": 0.316355, "test_sn": 0.5, "test_sp": 0.805085, "lr": 6.057720057720058e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.09710544209173297, "train_cur_epoch_loss": 11.458557106780372, "train_cur_epoch_avg_loss": 0.0054590553152836455, "train_cur_epoch_time": 850.3971598148346, "train_cur_epoch_avg_time": 0.4051439541757192, "epoch": 35, "step": 73465} ################################################## Training, Epoch: 0036, Batch: 000535, Sample Num: 8560, Cur Loss: 0.00004581, Cur Avg Loss: 0.00281965, Log Avg loss: 0.00475403, Global Avg Loss: 0.09642378, Time: 0.4048 Steps: 74000, Updated lr: 0.000060 Training, Epoch: 0036, Batch: 001535, Sample Num: 24560, Cur Loss: 0.00153938, Cur Avg Loss: 0.00434634, Log Avg loss: 0.00516312, Global Avg Loss: 0.09520697, Time: 0.4048 Steps: 75000, Updated lr: 0.000058 ***** Running evaluation checkpoint-75564 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-75564 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.122379, Avg time per batch (s): 0.400000 {"eval_avg_loss": 2.215424, "eval_total_loss": 1982.804507, "eval_acc": 0.914629, "eval_prec": 0.923868, "eval_recall": 0.903731, "eval_f1": 0.913688, "eval_roc_auc": 0.962257, "eval_pr_auc": 0.954859, "eval_confusion_matrix": {"tn": 6624, "fp": 533, "fn": 689, "tp": 6468}, "eval_mcc2": 0.829455, "eval_mcc": 0.829455, "eval_sn": 0.903731, "eval_sp": 0.925527, "update_flag": false, "test_avg_loss": 15.330119, "test_total_loss": 183.961432, "test_acc": 0.716667, "test_prec": 0.648649, "test_recall": 0.387097, "test_f1": 0.484848, "test_roc_auc": 0.643043, "test_pr_auc": 0.549468, "test_confusion_matrix": {"tn": 105, "fp": 13, "fn": 38, "tp": 24}, "test_mcc2": 0.325637, "test_mcc": 0.325637, "test_sn": 0.387097, "test_sp": 0.889831, "lr": 5.6538720538720547e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.09452838938389804, "train_cur_epoch_loss": 9.091912135689881, "train_cur_epoch_avg_loss": 0.0043315446096664515, "train_cur_epoch_time": 849.1223793029785, "train_cur_epoch_avg_time": 0.40453662663314843, "epoch": 36, "step": 75564} ################################################## Training, Epoch: 0037, Batch: 000436, Sample Num: 6976, Cur Loss: 0.00000000, Cur Avg Loss: 0.00359480, Log Avg loss: 0.00398762, Global Avg Loss: 0.09400672, Time: 0.4054 Steps: 76000, Updated lr: 0.000056 Training, Epoch: 0037, Batch: 001436, Sample Num: 22976, Cur Loss: 0.00000000, Cur Avg Loss: 0.00250416, Log Avg loss: 0.00202863, Global Avg Loss: 0.09281220, Time: 0.4045 Steps: 77000, Updated lr: 0.000054 ***** Running evaluation checkpoint-77663 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-77663 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 851.576496, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.838884, "eval_total_loss": 1645.801188, "eval_acc": 0.908621, "eval_prec": 0.891028, "eval_recall": 0.931116, "eval_f1": 0.910631, "eval_roc_auc": 0.956172, "eval_pr_auc": 0.940288, "eval_confusion_matrix": {"tn": 6342, "fp": 815, "fn": 493, "tp": 6664}, "eval_mcc2": 0.81807, "eval_mcc": 0.81807, "eval_sn": 0.931116, "eval_sp": 0.886125, "update_flag": false, "test_avg_loss": 13.212102, "test_total_loss": 158.545224, "test_acc": 0.711111, "test_prec": 0.592593, "test_recall": 0.516129, "test_f1": 0.551724, "test_roc_auc": 0.653021, "test_pr_auc": 0.563145, "test_confusion_matrix": {"tn": 96, "fp": 22, "fn": 30, "tp": 32}, "test_mcc2": 0.341868, "test_mcc": 0.341868, "test_sn": 0.516129, "test_sp": 0.813559, "lr": 5.2500240500240507e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.09204443974712896, "train_cur_epoch_loss": 5.504108676381851, "train_cur_epoch_avg_loss": 0.002622252823431087, "train_cur_epoch_time": 851.576495885849, "train_cur_epoch_avg_time": 0.40570581033151454, "epoch": 37, "step": 77663} ################################################## Training, Epoch: 0038, Batch: 000337, Sample Num: 5392, Cur Loss: 0.00000000, Cur Avg Loss: 0.00201669, Log Avg loss: 0.00258777, Global Avg Loss: 0.09165547, Time: 0.4033 Steps: 78000, Updated lr: 0.000052 Training, Epoch: 0038, Batch: 001337, Sample Num: 21392, Cur Loss: 0.00002563, Cur Avg Loss: 0.00425621, Log Avg loss: 0.00501093, Global Avg Loss: 0.09055871, Time: 0.4048 Steps: 79000, Updated lr: 0.000050 ***** Running evaluation checkpoint-79762 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-79762 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 848.702103, Avg time per batch (s): 0.400000 {"eval_avg_loss": 2.13565, "eval_total_loss": 1911.407044, "eval_acc": 0.914489, "eval_prec": 0.919887, "eval_recall": 0.908062, "eval_f1": 0.913936, "eval_roc_auc": 0.961306, "eval_pr_auc": 0.95305, "eval_confusion_matrix": {"tn": 6591, "fp": 566, "fn": 658, "tp": 6499}, "eval_mcc2": 0.829047, "eval_mcc": 0.829047, "eval_sn": 0.908062, "eval_sp": 0.920917, "update_flag": false, "test_avg_loss": 15.534083, "test_total_loss": 186.408991, "test_acc": 0.727778, "test_prec": 0.658537, "test_recall": 0.435484, "test_f1": 0.524272, "test_roc_auc": 0.636003, "test_pr_auc": 0.555754, "test_confusion_matrix": {"tn": 104, "fp": 14, "fn": 35, "tp": 27}, "test_mcc2": 0.358986, "test_mcc": 0.358986, "test_sn": 0.435484, "test_sp": 0.881356, "lr": 4.8461760461760467e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.08972113520175344, "train_cur_epoch_loss": 7.889861880959712, "train_cur_epoch_avg_loss": 0.0037588670228488386, "train_cur_epoch_time": 848.7021028995514, "train_cur_epoch_avg_time": 0.40433639966629414, "epoch": 38, "step": 79762} ################################################## Training, Epoch: 0039, Batch: 000238, Sample Num: 3808, Cur Loss: 0.00000149, Cur Avg Loss: 0.00115439, Log Avg loss: 0.00247405, Global Avg Loss: 0.08945765, Time: 0.4027 Steps: 80000, Updated lr: 0.000048 Training, Epoch: 0039, Batch: 001238, Sample Num: 19808, Cur Loss: 0.00003325, Cur Avg Loss: 0.00346648, Log Avg loss: 0.00401676, Global Avg Loss: 0.08840282, Time: 0.4031 Steps: 81000, Updated lr: 0.000046 ***** Running evaluation checkpoint-81861 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-81861 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 847.521859, Avg time per batch (s): 0.400000 {"eval_avg_loss": 2.00385, "eval_total_loss": 1793.445598, "eval_acc": 0.912114, "eval_prec": 0.912229, "eval_recall": 0.911974, "eval_f1": 0.912102, "eval_roc_auc": 0.960924, "eval_pr_auc": 0.952804, "eval_confusion_matrix": {"tn": 6529, "fp": 628, "fn": 630, "tp": 6527}, "eval_mcc2": 0.824228, "eval_mcc": 0.824228, "eval_sn": 0.911974, "eval_sp": 0.912254, "update_flag": false, "test_avg_loss": 13.880733, "test_total_loss": 166.568795, "test_acc": 0.733333, "test_prec": 0.666667, "test_recall": 0.451613, "test_f1": 0.538462, "test_roc_auc": 0.644068, "test_pr_auc": 0.568678, "test_confusion_matrix": {"tn": 104, "fp": 14, "fn": 34, "tp": 28}, "test_mcc2": 0.37409, "test_mcc": 0.37409, "test_sn": 0.451613, "test_sp": 0.881356, "lr": 4.4423280423280427e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.08750570420426582, "train_cur_epoch_loss": 6.9672659031241855, "train_cur_epoch_avg_loss": 0.003319326299725672, "train_cur_epoch_time": 847.5218586921692, "train_cur_epoch_avg_time": 0.40377411085858467, "epoch": 39, "step": 81861} ################################################## Training, Epoch: 0040, Batch: 000139, Sample Num: 2224, Cur Loss: 0.00009054, Cur Avg Loss: 0.00069146, Log Avg loss: 0.00277188, Global Avg Loss: 0.08735854, Time: 0.4040 Steps: 82000, Updated lr: 0.000044 Training, Epoch: 0040, Batch: 001139, Sample Num: 18224, Cur Loss: 0.00000001, Cur Avg Loss: 0.00235310, Log Avg loss: 0.00258407, Global Avg Loss: 0.08633716, Time: 0.4042 Steps: 83000, Updated lr: 0.000042 ***** Running evaluation checkpoint-83960 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-83960 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 850.426338, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.889775, "eval_total_loss": 1691.348699, "eval_acc": 0.910158, "eval_prec": 0.901848, "eval_recall": 0.920497, "eval_f1": 0.911077, "eval_roc_auc": 0.958915, "eval_pr_auc": 0.946763, "eval_confusion_matrix": {"tn": 6440, "fp": 717, "fn": 569, "tp": 6588}, "eval_mcc2": 0.820491, "eval_mcc": 0.820491, "eval_sn": 0.920497, "eval_sp": 0.899818, "update_flag": false, "test_avg_loss": 13.199942, "test_total_loss": 158.399306, "test_acc": 0.705556, "test_prec": 0.584906, "test_recall": 0.5, "test_f1": 0.53913, "test_roc_auc": 0.654524, "test_pr_auc": 0.55926, "test_confusion_matrix": {"tn": 96, "fp": 22, "fn": 31, "tp": 31}, "test_mcc2": 0.326901, "test_mcc": 0.326901, "test_sn": 0.5, "test_sp": 0.813559, "lr": 4.0384800384800387e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.0853622087517106, "train_cur_epoch_loss": 3.706594928203492, "train_cur_epoch_avg_loss": 0.0017658861020502581, "train_cur_epoch_time": 850.4263381958008, "train_cur_epoch_avg_time": 0.40515785526241105, "epoch": 40, "step": 83960} ################################################## Training, Epoch: 0041, Batch: 000040, Sample Num: 640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00047741, Log Avg loss: 0.00104551, Global Avg Loss: 0.08532179, Time: 0.4045 Steps: 84000, Updated lr: 0.000040 Training, Epoch: 0041, Batch: 001040, Sample Num: 16640, Cur Loss: 0.00000052, Cur Avg Loss: 0.00296621, Log Avg loss: 0.00306576, Global Avg Loss: 0.08435407, Time: 0.4056 Steps: 85000, Updated lr: 0.000038 Training, Epoch: 0041, Batch: 002040, Sample Num: 32640, Cur Loss: 0.00001009, Cur Avg Loss: 0.00191503, Log Avg loss: 0.00082182, Global Avg Loss: 0.08338276, Time: 0.4057 Steps: 86000, Updated lr: 0.000036 ***** Running evaluation checkpoint-86059 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-86059 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 850.369032, Avg time per batch (s): 0.410000 {"eval_avg_loss": 2.250106, "eval_total_loss": 2013.844486, "eval_acc": 0.915118, "eval_prec": 0.921897, "eval_recall": 0.907084, "eval_f1": 0.914431, "eval_roc_auc": 0.961601, "eval_pr_auc": 0.952959, "eval_confusion_matrix": {"tn": 6607, "fp": 550, "fn": 665, "tp": 6492}, "eval_mcc2": 0.830343, "eval_mcc": 0.830343, "eval_sn": 0.907084, "eval_sp": 0.923152, "update_flag": false, "test_avg_loss": 15.691475, "test_total_loss": 188.297695, "test_acc": 0.738889, "test_prec": 0.674419, "test_recall": 0.467742, "test_f1": 0.552381, "test_roc_auc": 0.641197, "test_pr_auc": 0.550143, "test_confusion_matrix": {"tn": 104, "fp": 14, "fn": 33, "tp": 29}, "test_mcc2": 0.389036, "test_mcc": 0.389036, "test_sn": 0.467742, "test_sp": 0.881356, "lr": 3.634632034632035e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.08332700522428431, "train_cur_epoch_loss": 4.027695803047896, "train_cur_epoch_avg_loss": 0.0019188641272262488, "train_cur_epoch_time": 850.3690323829651, "train_cur_epoch_avg_time": 0.40513055377940216, "epoch": 41, "step": 86059} ################################################## Training, Epoch: 0042, Batch: 000941, Sample Num: 15056, Cur Loss: 0.00000221, Cur Avg Loss: 0.00236191, Log Avg loss: 0.00234358, Global Avg Loss: 0.08245128, Time: 0.4085 Steps: 87000, Updated lr: 0.000035 Training, Epoch: 0042, Batch: 001941, Sample Num: 31056, Cur Loss: 0.00000003, Cur Avg Loss: 0.00220211, Log Avg loss: 0.00205173, Global Avg Loss: 0.08153765, Time: 0.4057 Steps: 88000, Updated lr: 0.000033 ***** Running evaluation checkpoint-88158 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-88158 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 853.712240, Avg time per batch (s): 0.410000 {"eval_avg_loss": 1.954404, "eval_total_loss": 1749.191219, "eval_acc": 0.909459, "eval_prec": 0.898979, "eval_recall": 0.922593, "eval_f1": 0.910633, "eval_roc_auc": 0.959293, "eval_pr_auc": 0.94729, "eval_confusion_matrix": {"tn": 6415, "fp": 742, "fn": 554, "tp": 6603}, "eval_mcc2": 0.819201, "eval_mcc": 0.819201, "eval_sn": 0.922593, "eval_sp": 0.896325, "update_flag": false, "test_avg_loss": 14.119479, "test_total_loss": 169.43375, "test_acc": 0.716667, "test_prec": 0.612245, "test_recall": 0.483871, "test_f1": 0.540541, "test_roc_auc": 0.645708, "test_pr_auc": 0.553926, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 32, "tp": 30}, "test_mcc2": 0.344675, "test_mcc": 0.344675, "test_sn": 0.483871, "test_sp": 0.838983, "lr": 3.2307840307840307e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.08139184937844937, "train_cur_epoch_loss": 4.303914908626932, "train_cur_epoch_avg_loss": 0.0020504596992029214, "train_cur_epoch_time": 853.7122399806976, "train_cur_epoch_avg_time": 0.4067233158555015, "epoch": 42, "step": 88158} ################################################## Training, Epoch: 0043, Batch: 000842, Sample Num: 13472, Cur Loss: 0.00000000, Cur Avg Loss: 0.00103344, Log Avg loss: 0.00089978, Global Avg Loss: 0.08063160, Time: 0.4052 Steps: 89000, Updated lr: 0.000031 Training, Epoch: 0043, Batch: 001842, Sample Num: 29472, Cur Loss: 0.00000936, Cur Avg Loss: 0.00155064, Log Avg loss: 0.00198612, Global Avg Loss: 0.07975777, Time: 0.4063 Steps: 90000, Updated lr: 0.000029 ***** Running evaluation checkpoint-90257 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-90257 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 851.199307, Avg time per batch (s): 0.410000 {"eval_avg_loss": 2.021194, "eval_total_loss": 1808.969039, "eval_acc": 0.90918, "eval_prec": 0.897192, "eval_recall": 0.92427, "eval_f1": 0.91053, "eval_roc_auc": 0.958088, "eval_pr_auc": 0.944263, "eval_confusion_matrix": {"tn": 6399, "fp": 758, "fn": 542, "tp": 6615}, "eval_mcc2": 0.818733, "eval_mcc": 0.818733, "eval_sn": 0.92427, "eval_sp": 0.89409, "update_flag": false, "test_avg_loss": 14.293844, "test_total_loss": 171.526132, "test_acc": 0.7, "test_prec": 0.576923, "test_recall": 0.483871, "test_f1": 0.526316, "test_roc_auc": 0.643521, "test_pr_auc": 0.538688, "test_confusion_matrix": {"tn": 96, "fp": 22, "fn": 32, "tp": 30}, "test_mcc2": 0.311828, "test_mcc": 0.311828, "test_sn": 0.483871, "test_sp": 0.813559, "lr": 2.8269360269360273e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.07953286066521019, "train_cur_epoch_loss": 3.0547475545245852, "train_cur_epoch_avg_loss": 0.0014553347091589258, "train_cur_epoch_time": 851.1993072032928, "train_cur_epoch_avg_time": 0.4055261111020928, "epoch": 43, "step": 90257} ################################################## Training, Epoch: 0044, Batch: 000743, Sample Num: 11888, Cur Loss: 0.00000843, Cur Avg Loss: 0.00218228, Log Avg loss: 0.00181990, Global Avg Loss: 0.07890131, Time: 0.4046 Steps: 91000, Updated lr: 0.000027 Training, Epoch: 0044, Batch: 001743, Sample Num: 27888, Cur Loss: 0.00011140, Cur Avg Loss: 0.00216618, Log Avg loss: 0.00215422, Global Avg Loss: 0.07806710, Time: 0.4044 Steps: 92000, Updated lr: 0.000025 ***** Running evaluation checkpoint-92356 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-92356 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 848.467386, Avg time per batch (s): 0.400000 {"eval_avg_loss": 1.999232, "eval_total_loss": 1789.312403, "eval_acc": 0.910996, "eval_prec": 0.903553, "eval_recall": 0.920218, "eval_f1": 0.911809, "eval_roc_auc": 0.959643, "eval_pr_auc": 0.947357, "eval_confusion_matrix": {"tn": 6454, "fp": 703, "fn": 571, "tp": 6586}, "eval_mcc2": 0.822132, "eval_mcc": 0.822132, "eval_sn": 0.920218, "eval_sp": 0.901774, "update_flag": false, "test_avg_loss": 14.407629, "test_total_loss": 172.891544, "test_acc": 0.711111, "test_prec": 0.6, "test_recall": 0.483871, "test_f1": 0.535714, "test_roc_auc": 0.646255, "test_pr_auc": 0.553574, "test_confusion_matrix": {"tn": 98, "fp": 20, "fn": 32, "tp": 30}, "test_mcc2": 0.33353, "test_mcc": 0.33353, "test_sn": 0.483871, "test_sp": 0.830508, "lr": 2.4230880230880233e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.07776776958172144, "train_cur_epoch_loss": 3.922722429575087, "train_cur_epoch_avg_loss": 0.0018688529916984693, "train_cur_epoch_time": 848.4673857688904, "train_cur_epoch_avg_time": 0.4042245763548787, "epoch": 44, "step": 92356} ################################################## Training, Epoch: 0045, Batch: 000644, Sample Num: 10304, Cur Loss: 0.00000498, Cur Avg Loss: 0.00216159, Log Avg loss: 0.00153914, Global Avg Loss: 0.07724422, Time: 0.4036 Steps: 93000, Updated lr: 0.000023 Training, Epoch: 0045, Batch: 001644, Sample Num: 26304, Cur Loss: 0.00000001, Cur Avg Loss: 0.00140769, Log Avg loss: 0.00092218, Global Avg Loss: 0.07643228, Time: 0.4057 Steps: 94000, Updated lr: 0.000021 ***** Running evaluation checkpoint-94455 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-94455 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.672894, Avg time per batch (s): 0.400000 {"eval_avg_loss": 2.064834, "eval_total_loss": 1848.026116, "eval_acc": 0.912254, "eval_prec": 0.90579, "eval_recall": 0.920218, "eval_f1": 0.912947, "eval_roc_auc": 0.959462, "eval_pr_auc": 0.946831, "eval_confusion_matrix": {"tn": 6472, "fp": 685, "fn": 571, "tp": 6586}, "eval_mcc2": 0.824612, "eval_mcc": 0.824612, "eval_sn": 0.920218, "eval_sp": 0.90429, "update_flag": false, "test_avg_loss": 14.716132, "test_total_loss": 176.593581, "test_acc": 0.705556, "test_prec": 0.588235, "test_recall": 0.483871, "test_f1": 0.530973, "test_roc_auc": 0.645298, "test_pr_auc": 0.55451, "test_confusion_matrix": {"tn": 97, "fp": 21, "fn": 32, "tp": 30}, "test_mcc2": 0.322584, "test_mcc": 0.322584, "test_sn": 0.483871, "test_sp": 0.822034, "lr": 2.0192400192400193e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.0760656193529835, "train_cur_epoch_loss": 2.457948496569883, "train_cur_epoch_avg_loss": 0.001171009288503994, "train_cur_epoch_time": 849.6728937625885, "train_cur_epoch_avg_time": 0.4047989012685033, "epoch": 45, "step": 94455} ################################################## Training, Epoch: 0046, Batch: 000545, Sample Num: 8720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00093424, Log Avg loss: 0.00065287, Global Avg Loss: 0.07563460, Time: 0.4039 Steps: 95000, Updated lr: 0.000019 Training, Epoch: 0046, Batch: 001545, Sample Num: 24720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00061764, Log Avg loss: 0.00044508, Global Avg Loss: 0.07485138, Time: 0.4066 Steps: 96000, Updated lr: 0.000017 ***** Running evaluation checkpoint-96554 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-96554 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 849.531672, Avg time per batch (s): 0.400000 {"eval_avg_loss": 2.090754, "eval_total_loss": 1871.225085, "eval_acc": 0.911765, "eval_prec": 0.90503, "eval_recall": 0.920078, "eval_f1": 0.912492, "eval_roc_auc": 0.959134, "eval_pr_auc": 0.946466, "eval_confusion_matrix": {"tn": 6466, "fp": 691, "fn": 572, "tp": 6585}, "eval_mcc2": 0.823643, "eval_mcc": 0.823643, "eval_sn": 0.920078, "eval_sp": 0.903451, "update_flag": false, "test_avg_loss": 14.972745, "test_total_loss": 179.672936, "test_acc": 0.716667, "test_prec": 0.612245, "test_recall": 0.483871, "test_f1": 0.540541, "test_roc_auc": 0.642838, "test_pr_auc": 0.552663, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 32, "tp": 30}, "test_mcc2": 0.344675, "test_mcc": 0.344675, "test_sn": 0.483871, "test_sp": 0.838983, "lr": 1.6153920153920153e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.0744349764554205, "train_cur_epoch_loss": 2.216640690603599, "train_cur_epoch_avg_loss": 0.0010560460650803234, "train_cur_epoch_time": 849.5316717624664, "train_cur_epoch_avg_time": 0.404731620658631, "epoch": 46, "step": 96554} ################################################## Training, Epoch: 0047, Batch: 000446, Sample Num: 7136, Cur Loss: 0.00000009, Cur Avg Loss: 0.00279407, Log Avg loss: 0.00250855, Global Avg Loss: 0.07410558, Time: 0.4026 Steps: 97000, Updated lr: 0.000015 Training, Epoch: 0047, Batch: 001446, Sample Num: 23136, Cur Loss: 0.00000000, Cur Avg Loss: 0.00118351, Log Avg loss: 0.00046520, Global Avg Loss: 0.07335414, Time: 0.4083 Steps: 98000, Updated lr: 0.000013 ***** Running evaluation checkpoint-98653 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-98653 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 851.621919, Avg time per batch (s): 0.410000 {"eval_avg_loss": 2.079877, "eval_total_loss": 1861.489611, "eval_acc": 0.912114, "eval_prec": 0.905653, "eval_recall": 0.920078, "eval_f1": 0.912808, "eval_roc_auc": 0.959462, "eval_pr_auc": 0.946932, "eval_confusion_matrix": {"tn": 6471, "fp": 686, "fn": 572, "tp": 6585}, "eval_mcc2": 0.824333, "eval_mcc": 0.824333, "eval_sn": 0.920078, "eval_sp": 0.90415, "update_flag": false, "test_avg_loss": 14.959666, "test_total_loss": 179.515988, "test_acc": 0.716667, "test_prec": 0.612245, "test_recall": 0.483871, "test_f1": 0.540541, "test_roc_auc": 0.643111, "test_pr_auc": 0.553639, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 32, "tp": 30}, "test_mcc2": 0.344675, "test_mcc": 0.344675, "test_sn": 0.483871, "test_sp": 0.838983, "lr": 1.2115440115440117e-05, "cur_epoch_step": 2099, "train_global_avg_loss": 0.0728725471119654, "train_cur_epoch_loss": 2.1006735600382225, "train_cur_epoch_avg_loss": 0.0010007973130244034, "train_cur_epoch_time": 851.6219191551208, "train_cur_epoch_avg_time": 0.4057274507647074, "epoch": 47, "step": 98653} ################################################## Training, Epoch: 0048, Batch: 000347, Sample Num: 5552, Cur Loss: 0.00000194, Cur Avg Loss: 0.00290859, Log Avg loss: 0.00139860, Global Avg Loss: 0.07262732, Time: 0.4072 Steps: 99000, Updated lr: 0.000011 Training, Epoch: 0048, Batch: 001347, Sample Num: 21552, Cur Loss: 0.00000618, Cur Avg Loss: 0.00093987, Log Avg loss: 0.00025672, Global Avg Loss: 0.07190361, Time: 0.4059 Steps: 100000, Updated lr: 0.000010 ***** Running evaluation checkpoint-100752 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-100752 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 853.397438, Avg time per batch (s): 0.410000 {"eval_avg_loss": 2.112541, "eval_total_loss": 1890.724572, "eval_acc": 0.912324, "eval_prec": 0.907034, "eval_recall": 0.918821, "eval_f1": 0.91289, "eval_roc_auc": 0.959509, "eval_pr_auc": 0.946982, "eval_confusion_matrix": {"tn": 6483, "fp": 674, "fn": 581, "tp": 6576}, "eval_mcc2": 0.824717, "eval_mcc": 0.824717, "eval_sn": 0.918821, "eval_sp": 0.905826, "update_flag": false, "test_avg_loss": 15.141685, "test_total_loss": 181.700226, "test_acc": 0.716667, "test_prec": 0.612245, "test_recall": 0.483871, "test_f1": 0.540541, "test_roc_auc": 0.642701, "test_pr_auc": 0.552562, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 32, "tp": 30}, "test_mcc2": 0.344675, "test_mcc": 0.344675, "test_sn": 0.483871, "test_sp": 0.838983, "lr": 8.076960076960077e-06, "cur_epoch_step": 2099, "train_global_avg_loss": 0.07137144864294653, "train_cur_epoch_loss": 1.7208034374024637, "train_cur_epoch_avg_loss": 0.000819820599048339, "train_cur_epoch_time": 853.3974375724792, "train_cur_epoch_avg_time": 0.4065733385290516, "epoch": 48, "step": 100752} ################################################## Training, Epoch: 0049, Batch: 000248, Sample Num: 3968, Cur Loss: 0.00000006, Cur Avg Loss: 0.00052642, Log Avg loss: 0.00058536, Global Avg Loss: 0.07119749, Time: 0.4050 Steps: 101000, Updated lr: 0.000008 Training, Epoch: 0049, Batch: 001248, Sample Num: 19968, Cur Loss: 0.00043464, Cur Avg Loss: 0.00108155, Log Avg loss: 0.00121922, Global Avg Loss: 0.07051143, Time: 0.4062 Steps: 102000, Updated lr: 0.000006 ***** Running evaluation checkpoint-102851 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-102851 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.608160, Avg time per batch (s): 0.410000 {"eval_avg_loss": 2.149715, "eval_total_loss": 1923.995354, "eval_acc": 0.912393, "eval_prec": 0.906935, "eval_recall": 0.9191, "eval_f1": 0.912977, "eval_roc_auc": 0.959485, "eval_pr_auc": 0.946705, "eval_confusion_matrix": {"tn": 6482, "fp": 675, "fn": 579, "tp": 6578}, "eval_mcc2": 0.824861, "eval_mcc": 0.824861, "eval_sn": 0.9191, "eval_sp": 0.905687, "update_flag": false, "test_avg_loss": 15.374666, "test_total_loss": 184.495992, "test_acc": 0.716667, "test_prec": 0.612245, "test_recall": 0.483871, "test_f1": 0.540541, "test_roc_auc": 0.641812, "test_pr_auc": 0.551556, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 32, "tp": 30}, "test_mcc2": 0.344675, "test_mcc": 0.344675, "test_sn": 0.483871, "test_sp": 0.838983, "lr": 4.038480038480038e-06, "cur_epoch_step": 2099, "train_global_avg_loss": 0.06993139361224328, "train_cur_epoch_loss": 1.6975707386658883, "train_cur_epoch_avg_loss": 0.0008087521384782698, "train_cur_epoch_time": 852.6081600189209, "train_cur_epoch_avg_time": 0.4061973130152077, "epoch": 49, "step": 102851} ################################################## Training, Epoch: 0050, Batch: 000149, Sample Num: 2384, Cur Loss: 0.00000261, Cur Avg Loss: 0.00008721, Log Avg loss: 0.00036079, Global Avg Loss: 0.06983036, Time: 0.4046 Steps: 103000, Updated lr: 0.000004 Training, Epoch: 0050, Batch: 001149, Sample Num: 18384, Cur Loss: 0.00000009, Cur Avg Loss: 0.00038533, Log Avg loss: 0.00042975, Global Avg Loss: 0.06916304, Time: 0.4063 Steps: 104000, Updated lr: 0.000002 ***** Running evaluation checkpoint-104950 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## ***** Running testing checkpoint-104950 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## Epoch Time: 852.561331, Avg time per batch (s): 0.410000 {"eval_avg_loss": 2.169038, "eval_total_loss": 1941.289164, "eval_acc": 0.913022, "eval_prec": 0.908513, "eval_recall": 0.918541, "eval_f1": 0.9135, "eval_roc_auc": 0.959801, "eval_pr_auc": 0.947442, "eval_confusion_matrix": {"tn": 6495, "fp": 662, "fn": 583, "tp": 6574}, "eval_mcc2": 0.826095, "eval_mcc": 0.826095, "eval_sn": 0.918541, "eval_sp": 0.907503, "update_flag": false, "test_avg_loss": 15.476873, "test_total_loss": 185.722471, "test_acc": 0.716667, "test_prec": 0.612245, "test_recall": 0.483871, "test_f1": 0.540541, "test_roc_auc": 0.641949, "test_pr_auc": 0.551947, "test_confusion_matrix": {"tn": 99, "fp": 19, "fn": 32, "tp": 30}, "test_mcc2": 0.344675, "test_mcc": 0.344675, "test_sn": 0.483871, "test_sp": 0.838983, "lr": 0.0, "cur_epoch_step": 2099, "train_global_avg_loss": 0.06854740841428907, "train_cur_epoch_loss": 1.5367486667894301, "train_cur_epoch_avg_loss": 0.0007321337145256932, "train_cur_epoch_time": 852.5613307952881, "train_cur_epoch_avg_time": 0.40617500276097573, "epoch": 50, "step": 104950} ################################################## #########################Best Metric######################### {"epoch": 34, "global_step": 71366, "eval_avg_loss": 2.085353, "eval_total_loss": 1866.390734, "eval_acc": 0.915537, "eval_prec": 0.91852, "eval_recall": 0.911974, "eval_f1": 0.915235, "eval_roc_auc": 0.962889, "eval_pr_auc": 0.955569, "eval_confusion_matrix": {"tn": 6578, "fp": 579, "fn": 630, "tp": 6527}, "eval_mcc2": 0.831096, "eval_mcc": 0.831096, "eval_sn": 0.911974, "eval_sp": 0.9191, "update_flag": true, "test_avg_loss": 14.695804, "test_total_loss": 176.349645, "test_acc": 0.7, "test_prec": 0.590909, "test_recall": 0.419355, "test_f1": 0.490566, "test_roc_auc": 0.651517, "test_pr_auc": 0.547001, "test_confusion_matrix": {"tn": 100, "fp": 18, "fn": 36, "tp": 26}, "test_mcc2": 0.295017, "test_mcc": 0.295017, "test_sn": 0.419355, "test_sp": 0.847458} ################################################## Total Time: 871913.791441, Avg time per epoch(50 epochs): 17438.280000 ++++++++++++Validation+++++++++++++ best f1 global step: 71366 checkpoint path: ../models/DeepAbBindv2_nucl/protein_protein_gene/binary_class/lucatriple2/matrix/20250127174006/checkpoint-71366 ***** Running evaluation checkpoint-71366 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 14314 ################################################## {"evaluation_avg_loss_71366": 2.085353, "evaluation_total_loss_71366": 1866.390734, "evaluation_acc_71366": 0.915537, "evaluation_prec_71366": 0.91852, "evaluation_recall_71366": 0.911974, "evaluation_f1_71366": 0.915235, "evaluation_roc_auc_71366": 0.962889, "evaluation_pr_auc_71366": 0.955569, "evaluation_confusion_matrix_71366": {"tn": 6578, "fp": 579, "fn": 630, "tp": 6527}, "evaluation_mcc2_71366": 0.831096, "evaluation_mcc_71366": 0.831096, "evaluation_sn_71366": 0.911974, "evaluation_sp_71366": 0.9191} ++++++++++++Testing+++++++++++++ best f1 global step: 71366 checkpoint path: ../models/DeepAbBindv2_nucl/protein_protein_gene/binary_class/lucatriple2/matrix/20250127174006/checkpoint-71366 ***** Running testing checkpoint-71366 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [180] ################################################## {"evaluation_avg_loss_71366": 14.695804, "evaluation_total_loss_71366": 176.349645, "evaluation_acc_71366": 0.7, "evaluation_prec_71366": 0.590909, "evaluation_recall_71366": 0.419355, "evaluation_f1_71366": 0.490566, "evaluation_roc_auc_71366": 0.651517, "evaluation_pr_auc_71366": 0.547001, "evaluation_confusion_matrix_71366": {"tn": 100, "fp": 18, "fn": 36, "tp": 26}, "evaluation_mcc2_71366": 0.295017, "evaluation_mcc_71366": 0.295017, "evaluation_sn_71366": 0.419355, "evaluation_sp_71366": 0.847458}