{"adam_epsilon": 1e-08, "alphabet": "gene_prot", "append_eos": true, "asl_gamma_neg": 4.0, "asl_gamma_pos": 1.0, "best_metric_type": "f1", "beta1": 0.9, "beta2": 0.98, "buffer_size": 1024, "cache_dir": null, "classifier_activate_func": "gelu", "classifier_size": 128, "codes_file": null, "config_path": "../config/luca_base/luca_base_config.json", "cross_atten": false, "dataset_name": "RdRP", "dataset_type": "protein", "delete_old": false, "dev_data_dir": "../dataset/RdRP/protein/binary_class/dev/", "device": "cuda", "do_eval": true, "do_lower_case": false, "do_metrics": true, "do_predict": true, "do_train": true, "dropout_prob": 0.1, "early_stop_epoch": -1, "emb_activate_func": "gelu", "embedding_complete": true, "embedding_complete_seg_overlap": true, "embedding_fixed_len_a_time": 3072, "embedding_input_size": 2560, "embedding_input_size_a": null, "embedding_input_size_b": null, "eval_all_checkpoints": false, "evaluate_during_training": true, "evaluate_steps": -1, "evaluate_strategy": "epoch", "fc_activate_func": "gelu", "focal_loss_alpha": 0.7, "focal_loss_gamma": 2.0, "focal_loss_reduce": false, "fp16": false, "fp16_embedding": false, "fp16_opt_level": "O1", "fusion_type": "concat", "gradient_accumulation_steps": 1, "hidden_size": 1024, "ignore_index": -100, "input_mode": "single", "input_type": "matrix", "intermediate_size": 4096, "label_filepath": "../dataset/RdRP/protein/binary_class/label.txt", "label_size": 2, "label_type": "RdRP", "learning_rate": 0.0001, "llm_dir": "..", "llm_dirpath": "../llm/models/lucagplm/v2.0/token_level,span_level,seq_level,structure_level/lucaone_gplm/20231125113045/checkpoint-step17600000", "llm_step": "17600000", "llm_task_level": "token_level,span_level,seq_level,structure_level", "llm_time_str": "20231125113045", "llm_type": "lucaone_gplm", "llm_version": "v2.0", "lmdb_path": null, "local_rank": -1, "log_dir": "../logs/RdRP/protein/binary_class/luca_base/matrix/20250331145853", "logging_steps": 200, "loss_reduction": "mean", "loss_type": "bce", "lr_decay_rate": 0.9, "lr_update_strategy": "step", "matrix_add_special_token": true, "matrix_dirpath": "../matrices/RdRP/protein/binary_class/luca_base/v2.0/lucaone_gplm/20231125113045/17600000", "matrix_embedding_exists": false, "matrix_encoder": false, "matrix_encoder_act": false, "matrix_fc_size": "128", "matrix_max_length": 4096, "matrix_max_length_a": null, "matrix_max_length_b": null, "matrix_pooling_type": "value_attention", "max_grad_norm": 1.0, "max_sentence_length": null, "max_sentences": null, "max_steps": -1, "model_dirpath": null, "model_type": "luca_base", "n_gpu": 1, "no_cuda": false, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "non_ignore": false, "not_append_eos": false, "not_matrix_encoder_shared": false, "not_prepend_bos": false, "not_save_emb_to_disk": false, "not_seq_encoder_shared": false, "num_attention_heads": 0, "num_hidden_layers": 0, "num_train_epochs": 10, "output_dir": "../models/RdRP/protein/binary_class/luca_base/matrix/20250331145853", "output_mode": "binary_class", "overwrite_cache": false, "overwrite_output_dir": true, "per_gpu_eval_batch_size": 16, "per_gpu_train_batch_size": 16, "pos_weight": 40.0, "position_embedding_type": "absolute", "prepend_bos": true, "save_all": true, "save_steps": -1, "seed": 1221, "self_atten": false, "seq_fc_size": "null", "seq_max_length": 4096, "seq_max_length_a": null, "seq_max_length_b": null, "seq_pooling_type": "value_attention", "seq_subword": false, "seq_vocab_path": "gene_prot", "sigmoid": true, "task_level_type": "seq_level", "task_type": "binary_class", "tb_log_dir": "../tb-logs/RdRP/protein/binary_class/luca_base/matrix/20250331145853", "test_data_dir": "../dataset/RdRP/protein/binary_class/test/", "time_str": "20250331145859", "train_data_dir": "../dataset/RdRP/protein/binary_class/train/", "trunc_type": "right", "vector_dirpath": "../vectors/RdRP/protein/binary_class/luca_base/v2.0/lucaone_gplm/20231125113045/17600000", "vector_fc_size": "null", "vocab_size": 39, "warmup_steps": 200, "weight": null, "weight_decay": 0.01, "worker_num": 0} ################################################## n_gpu: 1 ################################################## Inputs: Input Name List: protein,embedding_matrix ################################################## Encoder Config: {'llm_type': 'lucaone_gplm', 'llm_version': 'v2.0', 'llm_step': '17600000', 'llm_dirpath': '../llm/models/lucagplm/v2.0/token_level,span_level,seq_level,structure_level/lucaone_gplm/20231125113045/checkpoint-step17600000', 'input_type': 'matrix', 'trunc_type': 'right', 'seq_max_length': 4096, 'atom_seq_max_length': None, 'vector_dirpath': '../vectors/RdRP/protein/binary_class/luca_base/v2.0/lucaone_gplm/20231125113045/17600000', 'matrix_dirpath': '../matrices/RdRP/protein/binary_class/luca_base/v2.0/lucaone_gplm/20231125113045/17600000', 'local_rank': -1, 'max_sentence_length': None, 'max_sentences': None, 'matrix_add_special_token': True, 'embedding_complete': True, 'embedding_complete_seg_overlap': True, 'embedding_fixed_len_a_time': 3072, 'matrix_embedding_exists': False, 'save_emb_to_disk': True, 'fp16_embedding': False} ################################################## Model Config: LucaConfig { "alphabet": "gene_prot", "attention_probs_dropout_prob": 0.1, "classifier_activate_func": "gelu", "classifier_dropout_prob": 0.1, "classifier_size": 128, "cls_token_id": 2, "cross_atten": false, "directionality": "bidi", "emb_activate_func": "gelu", "embedding_input_size": 2560, "fc_activate_func": "gelu", "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 1024, "ignore_index": -100, "initializer_range": 0.02, "intermediate_size": 4096, "kernel_size": 7, "layer_norm_eps": 1e-12, "loss_reduction": "mean", "matrix_fc_size": [ 128 ], "matrix_max_length": 4096, "matrix_pooling_type": "value_attention", "max_position_embeddings": 4098, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "num_attention_heads": 8, "num_hidden_layers": 4, "pad_token_id": 0, "pos_weight": 40.0, "position_embedding_type": "absolute", "self_atten": false, "sep_token_id": 3, "seq_fc_size": null, "seq_max_length": 4096, "seq_pooling_type": "value_attention", "token_dropout": null, "transformers_version": "4.29.0", "type_vocab_size": 2, "use_luca_layer_norm_v2": true, "vector_fc_size": null, "vocab_size": 39 } ################################################## Mode Architecture: LucaBase( (matrix_pooler): GlobalMaskValueAttentionPooling1D (2560 -> 2560) (linear): ModuleList( (0): ModuleList( (0): Linear(in_features=2560, out_features=128, bias=True) (1): GELU(approximate='none') ) ) (dropout): Dropout(p=0.1, inplace=False) (hidden_layer): Linear(in_features=128, out_features=128, bias=True) (hidden_act): GELU(approximate='none') (classifier): Linear(in_features=128, out_features=1, bias=True) (output): Sigmoid() (loss_fct): MaskedBCEWithLogitsLoss( (criterion): BCEWithLogitsLoss() ) ) ################################################## Model parameters: 20005249 ################################################## {"total_num": "19.080000M", "total_size": "76.310000MB", "param_sum": "19.080000M", "param_size": "76.310000MB", "buffer_sum": "0.000000M", "buffer_size": "0.000000MB", "trainable_num": "19.078492M", "trainable_size": "76.313969MB"} ################################################## Train dataset len: 190846, batch size: 16, batch num: 11928 Train dataset t_total: 119280, max_steps: -1 ***** Running training ***** Train Dataset Num examples = 190846 Train Dataset Num Epochs = 10 Logging Steps = 200 Saving Steps = -1 Evaluating Strategy = epoch Train Dataset Instantaneous batch size per GPU = 16 Train Dataset Total train batch size (w. parallel, distributed & accumulation) = 16 Train Dataset Gradient Accumulation steps = 1 Train Dataset Total optimization steps = 119280 ################################################## Training, Epoch: 0001, Batch: 000200, Sample Num: 3200, Cur Loss: 0.57323194, Cur Avg Loss: 1.36142579, Log Avg loss: 1.36142579, Global Avg Loss: 1.36142579, Time: 0.0826 Steps: 200, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000400, Sample Num: 6400, Cur Loss: 0.00815655, Cur Avg Loss: 0.92817494, Log Avg loss: 0.49492408, Global Avg Loss: 0.92817494, Time: 0.1047 Steps: 400, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000600, Sample Num: 9600, Cur Loss: 0.14095490, Cur Avg Loss: 0.64991484, Log Avg loss: 0.09339465, Global Avg Loss: 0.64991484, Time: 0.2289 Steps: 600, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000800, Sample Num: 12800, Cur Loss: 0.03953269, Cur Avg Loss: 0.49757446, Log Avg loss: 0.04055332, Global Avg Loss: 0.49757446, Time: 0.2270 Steps: 800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001000, Sample Num: 16000, Cur Loss: 0.00003171, Cur Avg Loss: 0.42543469, Log Avg loss: 0.13687561, Global Avg Loss: 0.42543469, Time: 0.1534 Steps: 1000, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001200, Sample Num: 19200, Cur Loss: 0.00366507, Cur Avg Loss: 0.36445138, Log Avg loss: 0.05953482, Global Avg Loss: 0.36445138, Time: 0.1048 Steps: 1200, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001400, Sample Num: 22400, Cur Loss: 0.00002003, Cur Avg Loss: 0.31951343, Log Avg loss: 0.04988572, Global Avg Loss: 0.31951343, Time: 0.0701 Steps: 1400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001600, Sample Num: 25600, Cur Loss: 0.00000060, Cur Avg Loss: 0.28050597, Log Avg loss: 0.00745375, Global Avg Loss: 0.28050597, Time: 0.1277 Steps: 1600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001800, Sample Num: 28800, Cur Loss: 0.00002402, Cur Avg Loss: 0.25172016, Log Avg loss: 0.02143374, Global Avg Loss: 0.25172016, Time: 0.0814 Steps: 1800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002000, Sample Num: 32000, Cur Loss: 0.00001258, Cur Avg Loss: 0.22719786, Log Avg loss: 0.00649716, Global Avg Loss: 0.22719786, Time: 0.1132 Steps: 2000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002200, Sample Num: 35200, Cur Loss: 0.00001097, Cur Avg Loss: 0.20701132, Log Avg loss: 0.00514593, Global Avg Loss: 0.20701132, Time: 0.1313 Steps: 2200, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002400, Sample Num: 38400, Cur Loss: 0.00000000, Cur Avg Loss: 0.21117975, Log Avg loss: 0.25703239, Global Avg Loss: 0.21117975, Time: 0.0540 Steps: 2400, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002600, Sample Num: 41600, Cur Loss: 0.00023973, Cur Avg Loss: 0.20344000, Log Avg loss: 0.11056308, Global Avg Loss: 0.20344000, Time: 0.1260 Steps: 2600, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002800, Sample Num: 44800, Cur Loss: 0.00000519, Cur Avg Loss: 0.19245408, Log Avg loss: 0.04963703, Global Avg Loss: 0.19245408, Time: 0.1252 Steps: 2800, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003000, Sample Num: 48000, Cur Loss: 0.00000340, Cur Avg Loss: 0.18133668, Log Avg loss: 0.02569315, Global Avg Loss: 0.18133668, Time: 0.1236 Steps: 3000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003200, Sample Num: 51200, Cur Loss: 0.00000018, Cur Avg Loss: 0.17147168, Log Avg loss: 0.02349662, Global Avg Loss: 0.17147168, Time: 0.3854 Steps: 3200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003400, Sample Num: 54400, Cur Loss: 0.04319482, Cur Avg Loss: 0.16530535, Log Avg loss: 0.06664419, Global Avg Loss: 0.16530535, Time: 0.1724 Steps: 3400, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003600, Sample Num: 57600, Cur Loss: 0.00044358, Cur Avg Loss: 0.16559233, Log Avg loss: 0.17047099, Global Avg Loss: 0.16559233, Time: 0.1230 Steps: 3600, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003800, Sample Num: 60800, Cur Loss: 0.00002217, Cur Avg Loss: 0.16216308, Log Avg loss: 0.10043642, Global Avg Loss: 0.16216308, Time: 0.0824 Steps: 3800, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004000, Sample Num: 64000, Cur Loss: 0.00000125, Cur Avg Loss: 0.15862343, Log Avg loss: 0.09137022, Global Avg Loss: 0.15862343, Time: 0.0793 Steps: 4000, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004200, Sample Num: 67200, Cur Loss: 0.00000012, Cur Avg Loss: 0.15139774, Log Avg loss: 0.00688398, Global Avg Loss: 0.15139774, Time: 0.1119 Steps: 4200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004400, Sample Num: 70400, Cur Loss: 0.00000507, Cur Avg Loss: 0.14489712, Log Avg loss: 0.00838403, Global Avg Loss: 0.14489712, Time: 0.2226 Steps: 4400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 004600, Sample Num: 73600, Cur Loss: 0.00000000, Cur Avg Loss: 0.13862362, Log Avg loss: 0.00060670, Global Avg Loss: 0.13862362, Time: 0.1282 Steps: 4600, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 004800, Sample Num: 76800, Cur Loss: 0.00000024, Cur Avg Loss: 0.13287370, Log Avg loss: 0.00062553, Global Avg Loss: 0.13287370, Time: 0.0873 Steps: 4800, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005000, Sample Num: 80000, Cur Loss: 0.00001091, Cur Avg Loss: 0.12766093, Log Avg loss: 0.00255449, Global Avg Loss: 0.12766093, Time: 0.2112 Steps: 5000, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005200, Sample Num: 83200, Cur Loss: 0.00000036, Cur Avg Loss: 0.12656271, Log Avg loss: 0.09910709, Global Avg Loss: 0.12656271, Time: 0.1386 Steps: 5200, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005400, Sample Num: 86400, Cur Loss: 0.00000083, Cur Avg Loss: 0.13726362, Log Avg loss: 0.41548726, Global Avg Loss: 0.13726362, Time: 0.0983 Steps: 5400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005600, Sample Num: 89600, Cur Loss: 0.00000006, Cur Avg Loss: 0.13269528, Log Avg loss: 0.00935024, Global Avg Loss: 0.13269528, Time: 0.1229 Steps: 5600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 005800, Sample Num: 92800, Cur Loss: 0.00000000, Cur Avg Loss: 0.12842253, Log Avg loss: 0.00878533, Global Avg Loss: 0.12842253, Time: 0.1908 Steps: 5800, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006000, Sample Num: 96000, Cur Loss: 0.00000030, Cur Avg Loss: 0.12976237, Log Avg loss: 0.16861777, Global Avg Loss: 0.12976237, Time: 0.1961 Steps: 6000, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006200, Sample Num: 99200, Cur Loss: 0.00000006, Cur Avg Loss: 0.12896628, Log Avg loss: 0.10508364, Global Avg Loss: 0.12896628, Time: 0.0897 Steps: 6200, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006400, Sample Num: 102400, Cur Loss: 0.00000113, Cur Avg Loss: 0.12509181, Log Avg loss: 0.00498330, Global Avg Loss: 0.12509181, Time: 0.0992 Steps: 6400, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006600, Sample Num: 105600, Cur Loss: 0.00000012, Cur Avg Loss: 0.12165054, Log Avg loss: 0.01152980, Global Avg Loss: 0.12165054, Time: 0.3669 Steps: 6600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006800, Sample Num: 108800, Cur Loss: 0.00000000, Cur Avg Loss: 0.11812967, Log Avg loss: 0.00194085, Global Avg Loss: 0.11812967, Time: 0.2061 Steps: 6800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007000, Sample Num: 112000, Cur Loss: 0.00000000, Cur Avg Loss: 0.11488288, Log Avg loss: 0.00449228, Global Avg Loss: 0.11488288, Time: 0.0978 Steps: 7000, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007200, Sample Num: 115200, Cur Loss: 0.00000000, Cur Avg Loss: 0.11372900, Log Avg loss: 0.07334295, Global Avg Loss: 0.11372900, Time: 0.1083 Steps: 7200, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007400, Sample Num: 118400, Cur Loss: 0.00000221, Cur Avg Loss: 0.11088384, Log Avg loss: 0.00845828, Global Avg Loss: 0.11088384, Time: 0.1049 Steps: 7400, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007600, Sample Num: 121600, Cur Loss: 0.00000089, Cur Avg Loss: 0.10797406, Log Avg loss: 0.00031216, Global Avg Loss: 0.10797406, Time: 0.1986 Steps: 7600, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007800, Sample Num: 124800, Cur Loss: 0.00000060, Cur Avg Loss: 0.10933920, Log Avg loss: 0.16121432, Global Avg Loss: 0.10933920, Time: 0.0822 Steps: 7800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 008000, Sample Num: 128000, Cur Loss: 0.00000000, Cur Avg Loss: 0.10709909, Log Avg loss: 0.01973498, Global Avg Loss: 0.10709909, Time: 0.2321 Steps: 8000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008200, Sample Num: 131200, Cur Loss: 0.00000000, Cur Avg Loss: 0.10456780, Log Avg loss: 0.00331635, Global Avg Loss: 0.10456780, Time: 0.2223 Steps: 8200, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008400, Sample Num: 134400, Cur Loss: 0.00000006, Cur Avg Loss: 0.10589469, Log Avg loss: 0.16029698, Global Avg Loss: 0.10589469, Time: 0.2228 Steps: 8400, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008600, Sample Num: 137600, Cur Loss: 0.28002954, Cur Avg Loss: 0.10815560, Log Avg loss: 0.20311388, Global Avg Loss: 0.10815560, Time: 0.3100 Steps: 8600, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008800, Sample Num: 140800, Cur Loss: 0.00002354, Cur Avg Loss: 0.10890894, Log Avg loss: 0.14130237, Global Avg Loss: 0.10890894, Time: 0.0823 Steps: 8800, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 009000, Sample Num: 144000, Cur Loss: 0.00000149, Cur Avg Loss: 0.10681985, Log Avg loss: 0.01489997, Global Avg Loss: 0.10681985, Time: 0.0943 Steps: 9000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 009200, Sample Num: 147200, Cur Loss: 0.00001663, Cur Avg Loss: 0.10548868, Log Avg loss: 0.04558628, Global Avg Loss: 0.10548868, Time: 0.1337 Steps: 9200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009400, Sample Num: 150400, Cur Loss: 0.00000012, Cur Avg Loss: 0.10376539, Log Avg loss: 0.02449412, Global Avg Loss: 0.10376539, Time: 0.1228 Steps: 9400, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009600, Sample Num: 153600, Cur Loss: 0.00000328, Cur Avg Loss: 0.10319409, Log Avg loss: 0.07634253, Global Avg Loss: 0.10319409, Time: 0.0893 Steps: 9600, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009800, Sample Num: 156800, Cur Loss: 0.00000012, Cur Avg Loss: 0.10215155, Log Avg loss: 0.05210979, Global Avg Loss: 0.10215155, Time: 0.2021 Steps: 9800, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010000, Sample Num: 160000, Cur Loss: 0.00000280, Cur Avg Loss: 0.10092731, Log Avg loss: 0.04093940, Global Avg Loss: 0.10092731, Time: 0.1036 Steps: 10000, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010200, Sample Num: 163200, Cur Loss: 0.00000113, Cur Avg Loss: 0.10201211, Log Avg loss: 0.15625253, Global Avg Loss: 0.10201211, Time: 0.1080 Steps: 10200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010400, Sample Num: 166400, Cur Loss: 0.00000268, Cur Avg Loss: 0.10013109, Log Avg loss: 0.00419892, Global Avg Loss: 0.10013109, Time: 0.2238 Steps: 10400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 010600, Sample Num: 169600, Cur Loss: 0.00000030, Cur Avg Loss: 0.09833755, Log Avg loss: 0.00507347, Global Avg Loss: 0.09833755, Time: 0.0700 Steps: 10600, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 010800, Sample Num: 172800, Cur Loss: 0.00000030, Cur Avg Loss: 0.09748582, Log Avg loss: 0.05234406, Global Avg Loss: 0.09748582, Time: 0.1707 Steps: 10800, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011000, Sample Num: 176000, Cur Loss: 0.00000000, Cur Avg Loss: 0.09575383, Log Avg loss: 0.00222638, Global Avg Loss: 0.09575383, Time: 0.0931 Steps: 11000, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011200, Sample Num: 179200, Cur Loss: 0.00000018, Cur Avg Loss: 0.09411447, Log Avg loss: 0.00394955, Global Avg Loss: 0.09411447, Time: 0.0978 Steps: 11200, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011400, Sample Num: 182400, Cur Loss: 0.00000042, Cur Avg Loss: 0.09248103, Log Avg loss: 0.00100851, Global Avg Loss: 0.09248103, Time: 0.1819 Steps: 11400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011600, Sample Num: 185600, Cur Loss: 0.00000149, Cur Avg Loss: 0.09284522, Log Avg loss: 0.11360385, Global Avg Loss: 0.09284522, Time: 0.1060 Steps: 11600, Updated lr: 0.000090 Training, Epoch: 0001, Batch: 011800, Sample Num: 188800, Cur Loss: 0.00000387, Cur Avg Loss: 0.09152396, Log Avg loss: 0.01489115, Global Avg Loss: 0.09152396, Time: 0.3081 Steps: 11800, Updated lr: 0.000090 ***** Running evaluation checkpoint-11928 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-11928 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1868.720525, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.038675, "eval_total_loss": 53.874395, "eval_acc": 0.999551, "eval_prec": 0.992565, "eval_recall": 0.988889, "eval_f1": 0.990724, "eval_roc_auc": 0.999991, "eval_pr_auc": 0.999672, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 6, "tp": 534}, "eval_mcc2": 0.990495, "eval_mcc": 0.990495, "eval_sn": 0.988889, "eval_sp": 0.999816, "update_flag": true, "test_avg_loss": 0.033361, "test_total_loss": 46.472451, "test_acc": 0.999641, "test_prec": 0.994424, "test_recall": 0.990741, "test_f1": 0.992579, "test_roc_auc": 0.999996, "test_pr_auc": 0.999836, "test_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 5, "tp": 535}, "test_mcc2": 0.992397, "test_mcc": 0.992397, "test_sn": 0.990741, "test_sp": 0.999862, "lr": 9.015115888478334e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.09058745659924682, "train_cur_epoch_loss": 1080.527182315816, "train_cur_epoch_avg_loss": 0.09058745659924682, "train_cur_epoch_time": 1868.7205245494843, "train_cur_epoch_avg_time": 0.15666671064298157, "epoch": 1, "step": 11928} ################################################## Training, Epoch: 0002, Batch: 000072, Sample Num: 1152, Cur Loss: 0.00000030, Cur Avg Loss: 0.00065231, Log Avg loss: 0.00295707, Global Avg Loss: 0.09004785, Time: 0.0864 Steps: 12000, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000272, Sample Num: 4352, Cur Loss: 0.00000030, Cur Avg Loss: 0.00599575, Log Avg loss: 0.00791939, Global Avg Loss: 0.08870148, Time: 0.2102 Steps: 12200, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000472, Sample Num: 7552, Cur Loss: 0.00000101, Cur Avg Loss: 0.05294038, Log Avg loss: 0.11678507, Global Avg Loss: 0.08915444, Time: 0.1776 Steps: 12400, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000672, Sample Num: 10752, Cur Loss: 0.00000060, Cur Avg Loss: 0.04018418, Log Avg loss: 0.01007956, Global Avg Loss: 0.08789928, Time: 0.2137 Steps: 12600, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000872, Sample Num: 13952, Cur Loss: 0.00000000, Cur Avg Loss: 0.03108680, Log Avg loss: 0.00051959, Global Avg Loss: 0.08653397, Time: 0.1266 Steps: 12800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001072, Sample Num: 17152, Cur Loss: 0.00343921, Cur Avg Loss: 0.05898877, Log Avg loss: 0.18064135, Global Avg Loss: 0.08798178, Time: 0.1101 Steps: 13000, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001272, Sample Num: 20352, Cur Loss: 0.00000000, Cur Avg Loss: 0.05323007, Log Avg loss: 0.02236346, Global Avg Loss: 0.08698756, Time: 0.1845 Steps: 13200, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001472, Sample Num: 23552, Cur Loss: 0.00000000, Cur Avg Loss: 0.04716824, Log Avg loss: 0.00861501, Global Avg Loss: 0.08581782, Time: 0.1204 Steps: 13400, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001672, Sample Num: 26752, Cur Loss: 0.00000000, Cur Avg Loss: 0.04164937, Log Avg loss: 0.00103046, Global Avg Loss: 0.08457095, Time: 0.1258 Steps: 13600, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001872, Sample Num: 29952, Cur Loss: 0.00000006, Cur Avg Loss: 0.03720301, Log Avg loss: 0.00003147, Global Avg Loss: 0.08334574, Time: 0.1510 Steps: 13800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002072, Sample Num: 33152, Cur Loss: 0.00000286, Cur Avg Loss: 0.03425452, Log Avg loss: 0.00665665, Global Avg Loss: 0.08225018, Time: 0.0846 Steps: 14000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002272, Sample Num: 36352, Cur Loss: 0.00001043, Cur Avg Loss: 0.03175283, Log Avg loss: 0.00583533, Global Avg Loss: 0.08117392, Time: 0.2097 Steps: 14200, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002472, Sample Num: 39552, Cur Loss: 0.00000048, Cur Avg Loss: 0.03223849, Log Avg loss: 0.03775561, Global Avg Loss: 0.08057088, Time: 0.1170 Steps: 14400, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002672, Sample Num: 42752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03592935, Log Avg loss: 0.08154828, Global Avg Loss: 0.08058427, Time: 0.0871 Steps: 14600, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002872, Sample Num: 45952, Cur Loss: 0.00000000, Cur Avg Loss: 0.03350774, Log Avg loss: 0.00115503, Global Avg Loss: 0.07951091, Time: 0.1049 Steps: 14800, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003072, Sample Num: 49152, Cur Loss: 0.00000006, Cur Avg Loss: 0.03134729, Log Avg loss: 0.00032332, Global Avg Loss: 0.07845507, Time: 0.0942 Steps: 15000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003272, Sample Num: 52352, Cur Loss: 0.00000000, Cur Avg Loss: 0.02944111, Log Avg loss: 0.00016210, Global Avg Loss: 0.07742490, Time: 0.1683 Steps: 15200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003472, Sample Num: 55552, Cur Loss: 0.00000000, Cur Avg Loss: 0.02822547, Log Avg loss: 0.00833758, Global Avg Loss: 0.07652766, Time: 0.0813 Steps: 15400, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003672, Sample Num: 58752, Cur Loss: 0.00000024, Cur Avg Loss: 0.03332690, Log Avg loss: 0.12188783, Global Avg Loss: 0.07710920, Time: 0.1286 Steps: 15600, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003872, Sample Num: 61952, Cur Loss: 0.00000137, Cur Avg Loss: 0.03189480, Log Avg loss: 0.00560151, Global Avg Loss: 0.07620404, Time: 0.2328 Steps: 15800, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004072, Sample Num: 65152, Cur Loss: 0.00002092, Cur Avg Loss: 0.03043219, Log Avg loss: 0.00211589, Global Avg Loss: 0.07527794, Time: 0.1055 Steps: 16000, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004272, Sample Num: 68352, Cur Loss: 0.00000060, Cur Avg Loss: 0.02909561, Log Avg loss: 0.00188291, Global Avg Loss: 0.07437183, Time: 0.2795 Steps: 16200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004472, Sample Num: 71552, Cur Loss: 0.00000000, Cur Avg Loss: 0.02779992, Log Avg loss: 0.00012398, Global Avg Loss: 0.07346637, Time: 0.2291 Steps: 16400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 004672, Sample Num: 74752, Cur Loss: 0.00000221, Cur Avg Loss: 0.02661063, Log Avg loss: 0.00001811, Global Avg Loss: 0.07258145, Time: 0.0810 Steps: 16600, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 004872, Sample Num: 77952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02552377, Log Avg loss: 0.00013471, Global Avg Loss: 0.07171899, Time: 0.1187 Steps: 16800, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005072, Sample Num: 81152, Cur Loss: 0.00000018, Cur Avg Loss: 0.02592814, Log Avg loss: 0.03577872, Global Avg Loss: 0.07129616, Time: 0.1110 Steps: 17000, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005272, Sample Num: 84352, Cur Loss: 0.00000006, Cur Avg Loss: 0.03761005, Log Avg loss: 0.33386325, Global Avg Loss: 0.07434927, Time: 0.1306 Steps: 17200, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005472, Sample Num: 87552, Cur Loss: 0.00000477, Cur Avg Loss: 0.03629741, Log Avg loss: 0.00169625, Global Avg Loss: 0.07351417, Time: 0.1495 Steps: 17400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005672, Sample Num: 90752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03501873, Log Avg loss: 0.00003393, Global Avg Loss: 0.07267917, Time: 0.1053 Steps: 17600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 005872, Sample Num: 93952, Cur Loss: 0.00000000, Cur Avg Loss: 0.03523525, Log Avg loss: 0.04137586, Global Avg Loss: 0.07232745, Time: 0.4038 Steps: 17800, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006072, Sample Num: 97152, Cur Loss: 0.00000006, Cur Avg Loss: 0.03407481, Log Avg loss: 0.00000412, Global Avg Loss: 0.07152386, Time: 0.4038 Steps: 18000, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006272, Sample Num: 100352, Cur Loss: 0.00000006, Cur Avg Loss: 0.03508285, Log Avg loss: 0.06568692, Global Avg Loss: 0.07145971, Time: 0.2114 Steps: 18200, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006472, Sample Num: 103552, Cur Loss: 0.00000030, Cur Avg Loss: 0.03407220, Log Avg loss: 0.00237845, Global Avg Loss: 0.07070883, Time: 0.2077 Steps: 18400, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006672, Sample Num: 106752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03305894, Log Avg loss: 0.00026986, Global Avg Loss: 0.06995142, Time: 0.1120 Steps: 18600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006872, Sample Num: 109952, Cur Loss: 0.00000036, Cur Avg Loss: 0.03214284, Log Avg loss: 0.00158173, Global Avg Loss: 0.06922409, Time: 0.1254 Steps: 18800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007072, Sample Num: 113152, Cur Loss: 0.00000000, Cur Avg Loss: 0.03151262, Log Avg loss: 0.00985802, Global Avg Loss: 0.06859918, Time: 0.1798 Steps: 19000, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007272, Sample Num: 116352, Cur Loss: 0.00000000, Cur Avg Loss: 0.03074446, Log Avg loss: 0.00358237, Global Avg Loss: 0.06792192, Time: 0.1302 Steps: 19200, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007472, Sample Num: 119552, Cur Loss: 0.00000000, Cur Avg Loss: 0.03001180, Log Avg loss: 0.00337239, Global Avg Loss: 0.06725646, Time: 0.0526 Steps: 19400, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007672, Sample Num: 122752, Cur Loss: 0.00000006, Cur Avg Loss: 0.02942815, Log Avg loss: 0.00762306, Global Avg Loss: 0.06664796, Time: 0.2304 Steps: 19600, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007872, Sample Num: 125952, Cur Loss: 0.00000048, Cur Avg Loss: 0.02933491, Log Avg loss: 0.02575812, Global Avg Loss: 0.06623493, Time: 0.1410 Steps: 19800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 008072, Sample Num: 129152, Cur Loss: 0.00000000, Cur Avg Loss: 0.02874277, Log Avg loss: 0.00543600, Global Avg Loss: 0.06562694, Time: 0.0559 Steps: 20000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008272, Sample Num: 132352, Cur Loss: 0.00000000, Cur Avg Loss: 0.02903130, Log Avg loss: 0.04067660, Global Avg Loss: 0.06537991, Time: 0.0870 Steps: 20200, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008472, Sample Num: 135552, Cur Loss: 0.00000000, Cur Avg Loss: 0.03359818, Log Avg loss: 0.22248421, Global Avg Loss: 0.06692015, Time: 0.1809 Steps: 20400, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008672, Sample Num: 138752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03284301, Log Avg loss: 0.00085412, Global Avg Loss: 0.06627873, Time: 0.0936 Steps: 20600, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008872, Sample Num: 141952, Cur Loss: 0.00000012, Cur Avg Loss: 0.03230269, Log Avg loss: 0.00887434, Global Avg Loss: 0.06572676, Time: 0.0931 Steps: 20800, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 009072, Sample Num: 145152, Cur Loss: 0.00000000, Cur Avg Loss: 0.03159140, Log Avg loss: 0.00003850, Global Avg Loss: 0.06510116, Time: 0.1120 Steps: 21000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 009272, Sample Num: 148352, Cur Loss: 0.00001091, Cur Avg Loss: 0.03154026, Log Avg loss: 0.02922085, Global Avg Loss: 0.06476267, Time: 0.1273 Steps: 21200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009472, Sample Num: 151552, Cur Loss: 0.00000054, Cur Avg Loss: 0.03090821, Log Avg loss: 0.00160629, Global Avg Loss: 0.06417242, Time: 0.2732 Steps: 21400, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009672, Sample Num: 154752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03030031, Log Avg loss: 0.00151001, Global Avg Loss: 0.06359221, Time: 0.1250 Steps: 21600, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009872, Sample Num: 157952, Cur Loss: 0.00000656, Cur Avg Loss: 0.03174447, Log Avg loss: 0.10158405, Global Avg Loss: 0.06394076, Time: 0.2299 Steps: 21800, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010072, Sample Num: 161152, Cur Loss: 0.00000745, Cur Avg Loss: 0.03153761, Log Avg loss: 0.02132713, Global Avg Loss: 0.06355336, Time: 0.1374 Steps: 22000, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010272, Sample Num: 164352, Cur Loss: 0.00000697, Cur Avg Loss: 0.03095516, Log Avg loss: 0.00162295, Global Avg Loss: 0.06299543, Time: 0.4042 Steps: 22200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010472, Sample Num: 167552, Cur Loss: 0.00000012, Cur Avg Loss: 0.03044663, Log Avg loss: 0.00432825, Global Avg Loss: 0.06247162, Time: 0.0987 Steps: 22400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 010672, Sample Num: 170752, Cur Loss: 0.00000030, Cur Avg Loss: 0.02989967, Log Avg loss: 0.00126088, Global Avg Loss: 0.06192993, Time: 0.2094 Steps: 22600, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 010872, Sample Num: 173952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02934971, Log Avg loss: 0.00000377, Global Avg Loss: 0.06138672, Time: 0.1084 Steps: 22800, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011072, Sample Num: 177152, Cur Loss: 0.00000000, Cur Avg Loss: 0.02882195, Log Avg loss: 0.00013287, Global Avg Loss: 0.06085408, Time: 0.1821 Steps: 23000, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011272, Sample Num: 180352, Cur Loss: 0.00000000, Cur Avg Loss: 0.02836120, Log Avg loss: 0.00285414, Global Avg Loss: 0.06035408, Time: 0.2172 Steps: 23200, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011472, Sample Num: 183552, Cur Loss: 0.00000000, Cur Avg Loss: 0.02802777, Log Avg loss: 0.00923562, Global Avg Loss: 0.05991717, Time: 0.2298 Steps: 23400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011672, Sample Num: 186752, Cur Loss: 0.00000000, Cur Avg Loss: 0.02756746, Log Avg loss: 0.00116452, Global Avg Loss: 0.05941926, Time: 0.1009 Steps: 23600, Updated lr: 0.000080 Training, Epoch: 0002, Batch: 011872, Sample Num: 189952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02715978, Log Avg loss: 0.00336717, Global Avg Loss: 0.05894824, Time: 0.2756 Steps: 23800, Updated lr: 0.000080 ***** Running evaluation checkpoint-23856 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-23856 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1862.675031, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.04232, "eval_total_loss": 58.951537, "eval_acc": 0.999686, "eval_prec": 0.994434, "eval_recall": 0.992593, "eval_f1": 0.993513, "eval_roc_auc": 0.999994, "eval_pr_auc": 0.999762, "eval_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 4, "tp": 536}, "eval_mcc2": 0.993352, "eval_mcc": 0.993352, "eval_sn": 0.992593, "eval_sp": 0.999862, "update_flag": true, "test_avg_loss": 0.033912, "test_total_loss": 47.238892, "test_acc": 0.999731, "test_prec": 0.998134, "test_recall": 0.990741, "test_f1": 0.994424, "test_roc_auc": 0.999998, "test_pr_auc": 0.999915, "test_confusion_matrix": {"tn": 21743, "fp": 1, "fn": 5, "tp": 535}, "test_mcc2": 0.994293, "test_mcc": 0.994293, "test_sn": 0.990741, "test_sp": 0.999954, "lr": 8.013436345314075e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.05880986214662483, "train_cur_epoch_loss": 322.4408890540461, "train_cur_epoch_avg_loss": 0.027032267694001185, "train_cur_epoch_time": 1862.6750309467316, "train_cur_epoch_avg_time": 0.15615987851666094, "epoch": 2, "step": 23856} ################################################## Training, Epoch: 0003, Batch: 000144, Sample Num: 2304, Cur Loss: 0.00000000, Cur Avg Loss: 0.00011062, Log Avg loss: 0.00007976, Global Avg Loss: 0.05845767, Time: 0.0768 Steps: 24000, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000344, Sample Num: 5504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02200753, Log Avg loss: 0.03777331, Global Avg Loss: 0.05828672, Time: 0.2390 Steps: 24200, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000544, Sample Num: 8704, Cur Loss: 0.00000000, Cur Avg Loss: 0.01678301, Log Avg loss: 0.00779683, Global Avg Loss: 0.05787287, Time: 0.1034 Steps: 24400, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000744, Sample Num: 11904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01228127, Log Avg loss: 0.00003652, Global Avg Loss: 0.05740266, Time: 0.0966 Steps: 24600, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000944, Sample Num: 15104, Cur Loss: 0.00000000, Cur Avg Loss: 0.05244552, Log Avg loss: 0.20185656, Global Avg Loss: 0.05856761, Time: 0.1496 Steps: 24800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001144, Sample Num: 18304, Cur Loss: 0.00000006, Cur Avg Loss: 0.04329492, Log Avg loss: 0.00010410, Global Avg Loss: 0.05809990, Time: 0.1008 Steps: 25000, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001344, Sample Num: 21504, Cur Loss: 0.00000000, Cur Avg Loss: 0.03685528, Log Avg loss: 0.00002052, Global Avg Loss: 0.05763895, Time: 0.2022 Steps: 25200, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001544, Sample Num: 24704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03209032, Log Avg loss: 0.00006978, Global Avg Loss: 0.05718565, Time: 0.1062 Steps: 25400, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001744, Sample Num: 27904, Cur Loss: 0.00000000, Cur Avg Loss: 0.03016650, Log Avg loss: 0.01531463, Global Avg Loss: 0.05685853, Time: 0.1220 Steps: 25600, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001944, Sample Num: 31104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02706305, Log Avg loss: 0.00000093, Global Avg Loss: 0.05641778, Time: 0.1095 Steps: 25800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002144, Sample Num: 34304, Cur Loss: 0.00000000, Cur Avg Loss: 0.03100799, Log Avg loss: 0.06935285, Global Avg Loss: 0.05651728, Time: 0.0708 Steps: 26000, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002344, Sample Num: 37504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02852312, Log Avg loss: 0.00188535, Global Avg Loss: 0.05610024, Time: 0.1136 Steps: 26200, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002544, Sample Num: 40704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02636374, Log Avg loss: 0.00105582, Global Avg Loss: 0.05568324, Time: 0.1069 Steps: 26400, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002744, Sample Num: 43904, Cur Loss: 0.00000894, Cur Avg Loss: 0.02910051, Log Avg loss: 0.06391212, Global Avg Loss: 0.05574511, Time: 0.2201 Steps: 26600, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002944, Sample Num: 47104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02713460, Log Avg loss: 0.00016243, Global Avg Loss: 0.05533031, Time: 0.1074 Steps: 26800, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 003144, Sample Num: 50304, Cur Loss: 0.00000012, Cur Avg Loss: 0.02541051, Log Avg loss: 0.00003183, Global Avg Loss: 0.05492069, Time: 0.2133 Steps: 27000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003344, Sample Num: 53504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02397059, Log Avg loss: 0.00133504, Global Avg Loss: 0.05452668, Time: 0.1604 Steps: 27200, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003544, Sample Num: 56704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03429968, Log Avg loss: 0.20700210, Global Avg Loss: 0.05563964, Time: 0.1739 Steps: 27400, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003744, Sample Num: 59904, Cur Loss: 0.00000000, Cur Avg Loss: 0.03269010, Log Avg loss: 0.00416825, Global Avg Loss: 0.05526666, Time: 0.1231 Steps: 27600, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003944, Sample Num: 63104, Cur Loss: 0.00000000, Cur Avg Loss: 0.03103474, Log Avg loss: 0.00004650, Global Avg Loss: 0.05486939, Time: 0.1101 Steps: 27800, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 004144, Sample Num: 66304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02953821, Log Avg loss: 0.00002660, Global Avg Loss: 0.05447766, Time: 0.0915 Steps: 28000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 004344, Sample Num: 69504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02822398, Log Avg loss: 0.00099321, Global Avg Loss: 0.05409834, Time: 0.2341 Steps: 28200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004544, Sample Num: 72704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02698179, Log Avg loss: 0.00000135, Global Avg Loss: 0.05371737, Time: 0.1258 Steps: 28400, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004744, Sample Num: 75904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02584434, Log Avg loss: 0.00000143, Global Avg Loss: 0.05334173, Time: 0.1701 Steps: 28600, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004944, Sample Num: 79104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02480245, Log Avg loss: 0.00008888, Global Avg Loss: 0.05297192, Time: 0.1998 Steps: 28800, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005144, Sample Num: 82304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02391277, Log Avg loss: 0.00191991, Global Avg Loss: 0.05261984, Time: 0.0618 Steps: 29000, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005344, Sample Num: 85504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02993841, Log Avg loss: 0.18491793, Global Avg Loss: 0.05352599, Time: 0.3473 Steps: 29200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005544, Sample Num: 88704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02886300, Log Avg loss: 0.00012782, Global Avg Loss: 0.05316274, Time: 0.2185 Steps: 29400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 005744, Sample Num: 91904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02785813, Log Avg loss: 0.00000322, Global Avg Loss: 0.05280355, Time: 0.1473 Steps: 29600, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 005944, Sample Num: 95104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02692085, Log Avg loss: 0.00000212, Global Avg Loss: 0.05244918, Time: 0.0757 Steps: 29800, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006144, Sample Num: 98304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02630095, Log Avg loss: 0.00787748, Global Avg Loss: 0.05215204, Time: 0.1287 Steps: 30000, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006344, Sample Num: 101504, Cur Loss: 0.00000018, Cur Avg Loss: 0.02547190, Log Avg loss: 0.00000379, Global Avg Loss: 0.05180668, Time: 0.0742 Steps: 30200, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006544, Sample Num: 104704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02473031, Log Avg loss: 0.00120702, Global Avg Loss: 0.05147379, Time: 0.1225 Steps: 30400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006744, Sample Num: 107904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02399695, Log Avg loss: 0.00000121, Global Avg Loss: 0.05113737, Time: 0.1077 Steps: 30600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 006944, Sample Num: 111104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02330592, Log Avg loss: 0.00000458, Global Avg Loss: 0.05080534, Time: 0.1133 Steps: 30800, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007144, Sample Num: 114304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02308259, Log Avg loss: 0.01532846, Global Avg Loss: 0.05057645, Time: 0.2404 Steps: 31000, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007344, Sample Num: 117504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02251649, Log Avg loss: 0.00229554, Global Avg Loss: 0.05026696, Time: 0.1095 Steps: 31200, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007544, Sample Num: 120704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02191962, Log Avg loss: 0.00000236, Global Avg Loss: 0.04994680, Time: 0.1010 Steps: 31400, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007744, Sample Num: 123904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02214383, Log Avg loss: 0.03060113, Global Avg Loss: 0.04982436, Time: 0.0870 Steps: 31600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007944, Sample Num: 127104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02166795, Log Avg loss: 0.00324167, Global Avg Loss: 0.04953139, Time: 0.1269 Steps: 31800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008144, Sample Num: 130304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02117043, Log Avg loss: 0.00140909, Global Avg Loss: 0.04923063, Time: 0.1234 Steps: 32000, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008344, Sample Num: 133504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02293923, Log Avg loss: 0.09496475, Global Avg Loss: 0.04951469, Time: 0.4021 Steps: 32200, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008544, Sample Num: 136704, Cur Loss: 0.00000006, Cur Avg Loss: 0.02611128, Log Avg loss: 0.15844945, Global Avg Loss: 0.05018713, Time: 0.1186 Steps: 32400, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008744, Sample Num: 139904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02551995, Log Avg loss: 0.00025812, Global Avg Loss: 0.04988081, Time: 0.1686 Steps: 32600, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008944, Sample Num: 143104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02495505, Log Avg loss: 0.00025767, Global Avg Loss: 0.04957823, Time: 0.2498 Steps: 32800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 009144, Sample Num: 146304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02440968, Log Avg loss: 0.00002091, Global Avg Loss: 0.04927789, Time: 0.0823 Steps: 33000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009344, Sample Num: 149504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02388806, Log Avg loss: 0.00003944, Global Avg Loss: 0.04898127, Time: 0.1280 Steps: 33200, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009544, Sample Num: 152704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02339835, Log Avg loss: 0.00051911, Global Avg Loss: 0.04869108, Time: 0.2794 Steps: 33400, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009744, Sample Num: 155904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02291812, Log Avg loss: 0.00000169, Global Avg Loss: 0.04840126, Time: 0.0709 Steps: 33600, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009944, Sample Num: 159104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02366280, Log Avg loss: 0.05994319, Global Avg Loss: 0.04846955, Time: 0.2229 Steps: 33800, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 010144, Sample Num: 162304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02319641, Log Avg loss: 0.00000766, Global Avg Loss: 0.04818448, Time: 0.2499 Steps: 34000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 010344, Sample Num: 165504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02277941, Log Avg loss: 0.00162913, Global Avg Loss: 0.04791223, Time: 0.1168 Steps: 34200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010544, Sample Num: 168704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02236847, Log Avg loss: 0.00111457, Global Avg Loss: 0.04764015, Time: 0.2941 Steps: 34400, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010744, Sample Num: 171904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02195215, Log Avg loss: 0.00000379, Global Avg Loss: 0.04736480, Time: 0.1334 Steps: 34600, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010944, Sample Num: 175104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02155099, Log Avg loss: 0.00000074, Global Avg Loss: 0.04709259, Time: 0.1520 Steps: 34800, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011144, Sample Num: 178304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02117435, Log Avg loss: 0.00056443, Global Avg Loss: 0.04682671, Time: 0.1184 Steps: 35000, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011344, Sample Num: 181504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02080104, Log Avg loss: 0.00000051, Global Avg Loss: 0.04656066, Time: 0.1248 Steps: 35200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011544, Sample Num: 184704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02044322, Log Avg loss: 0.00014741, Global Avg Loss: 0.04629843, Time: 0.1056 Steps: 35400, Updated lr: 0.000070 Training, Epoch: 0003, Batch: 011744, Sample Num: 187904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02010077, Log Avg loss: 0.00033468, Global Avg Loss: 0.04604021, Time: 0.0865 Steps: 35600, Updated lr: 0.000070 ***** Running evaluation checkpoint-35784 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-35784 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1866.352637, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.051992, "eval_total_loss": 72.425042, "eval_acc": 0.999596, "eval_prec": 0.98895, "eval_recall": 0.994444, "eval_f1": 0.99169, "eval_roc_auc": 0.999994, "eval_pr_auc": 0.999767, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 3, "tp": 537}, "eval_mcc2": 0.991487, "eval_mcc": 0.991487, "eval_sn": 0.994444, "eval_sp": 0.999724, "update_flag": false, "test_avg_loss": 0.030679, "test_total_loss": 42.736244, "test_acc": 0.999686, "test_prec": 0.994434, "test_recall": 0.992593, "test_f1": 0.993513, "test_roc_auc": 0.999998, "test_pr_auc": 0.999935, "test_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 4, "tp": 536}, "test_mcc2": 0.993352, "test_mcc": 0.993352, "test_sn": 0.992593, "test_sp": 0.999862, "lr": 7.011756802149816e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.04581382059902257, "train_cur_epoch_loss": 236.433684945531, "train_cur_epoch_avg_loss": 0.019821737503817154, "train_cur_epoch_time": 1866.3526372909546, "train_cur_epoch_avg_time": 0.15646819561460049, "epoch": 3, "step": 35784} ################################################## Training, Epoch: 0004, Batch: 000016, Sample Num: 256, Cur Loss: 0.00000203, Cur Avg Loss: 0.00000249, Log Avg loss: 0.00185150, Global Avg Loss: 0.04579335, Time: 0.2381 Steps: 35800, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000216, Sample Num: 3456, Cur Loss: 0.00000066, Cur Avg Loss: 0.00000038, Log Avg loss: 0.00000021, Global Avg Loss: 0.04553894, Time: 0.0639 Steps: 36000, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000416, Sample Num: 6656, Cur Loss: 0.00000000, Cur Avg Loss: 0.00632021, Log Avg loss: 0.01314562, Global Avg Loss: 0.04535997, Time: 0.1354 Steps: 36200, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000616, Sample Num: 9856, Cur Loss: 0.00000000, Cur Avg Loss: 0.00440632, Log Avg loss: 0.00042542, Global Avg Loss: 0.04511308, Time: 0.1239 Steps: 36400, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000816, Sample Num: 13056, Cur Loss: 0.00000000, Cur Avg Loss: 0.00332644, Log Avg loss: 0.00000041, Global Avg Loss: 0.04486656, Time: 0.1602 Steps: 36600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001016, Sample Num: 16256, Cur Loss: 0.00000000, Cur Avg Loss: 0.04368802, Log Avg loss: 0.20836328, Global Avg Loss: 0.04575513, Time: 0.2695 Steps: 36800, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001216, Sample Num: 19456, Cur Loss: 0.00000000, Cur Avg Loss: 0.03650469, Log Avg loss: 0.00001336, Global Avg Loss: 0.04550788, Time: 0.2805 Steps: 37000, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001416, Sample Num: 22656, Cur Loss: 0.00000000, Cur Avg Loss: 0.03134890, Log Avg loss: 0.00000172, Global Avg Loss: 0.04526322, Time: 0.0893 Steps: 37200, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001616, Sample Num: 25856, Cur Loss: 0.00000000, Cur Avg Loss: 0.03251355, Log Avg loss: 0.04075923, Global Avg Loss: 0.04523913, Time: 0.1021 Steps: 37400, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001816, Sample Num: 29056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02893276, Log Avg loss: 0.00000004, Global Avg Loss: 0.04499850, Time: 0.1017 Steps: 37600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002016, Sample Num: 32256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02679020, Log Avg loss: 0.00733568, Global Avg Loss: 0.04479923, Time: 0.1227 Steps: 37800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002216, Sample Num: 35456, Cur Loss: 0.00000000, Cur Avg Loss: 0.03151599, Log Avg loss: 0.07915200, Global Avg Loss: 0.04498003, Time: 0.2871 Steps: 38000, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002416, Sample Num: 38656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02890876, Log Avg loss: 0.00002061, Global Avg Loss: 0.04474464, Time: 0.1373 Steps: 38200, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002616, Sample Num: 41856, Cur Loss: 0.00000000, Cur Avg Loss: 0.03439810, Log Avg loss: 0.10070941, Global Avg Loss: 0.04503612, Time: 0.1265 Steps: 38400, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002816, Sample Num: 45056, Cur Loss: 0.00000000, Cur Avg Loss: 0.03283390, Log Avg loss: 0.01237411, Global Avg Loss: 0.04486689, Time: 0.1843 Steps: 38600, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003016, Sample Num: 48256, Cur Loss: 0.00000000, Cur Avg Loss: 0.03066289, Log Avg loss: 0.00009507, Global Avg Loss: 0.04463611, Time: 0.2476 Steps: 38800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003216, Sample Num: 51456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02875605, Log Avg loss: 0.00000097, Global Avg Loss: 0.04440721, Time: 0.1422 Steps: 39000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003416, Sample Num: 54656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02707721, Log Avg loss: 0.00008135, Global Avg Loss: 0.04418106, Time: 0.1320 Steps: 39200, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003616, Sample Num: 57856, Cur Loss: 0.00000000, Cur Avg Loss: 0.03382722, Log Avg loss: 0.14911740, Global Avg Loss: 0.04471373, Time: 0.1160 Steps: 39400, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003816, Sample Num: 61056, Cur Loss: 0.00000000, Cur Avg Loss: 0.03208734, Log Avg loss: 0.00063034, Global Avg Loss: 0.04449109, Time: 0.1490 Steps: 39600, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004016, Sample Num: 64256, Cur Loss: 0.00000000, Cur Avg Loss: 0.03048957, Log Avg loss: 0.00000404, Global Avg Loss: 0.04426753, Time: 0.0475 Steps: 39800, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004216, Sample Num: 67456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02905319, Log Avg loss: 0.00021075, Global Avg Loss: 0.04404725, Time: 0.1003 Steps: 40000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004416, Sample Num: 70656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02773937, Log Avg loss: 0.00004401, Global Avg Loss: 0.04382833, Time: 0.4267 Steps: 40200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 004616, Sample Num: 73856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02653749, Log Avg loss: 0.00000004, Global Avg Loss: 0.04361136, Time: 0.2204 Steps: 40400, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 004816, Sample Num: 77056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02543547, Log Avg loss: 0.00000072, Global Avg Loss: 0.04339653, Time: 0.1813 Steps: 40600, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005016, Sample Num: 80256, Cur Loss: 0.00017672, Cur Avg Loss: 0.02442182, Log Avg loss: 0.00001331, Global Avg Loss: 0.04318386, Time: 0.2035 Steps: 40800, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005216, Sample Num: 83456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02348556, Log Avg loss: 0.00000416, Global Avg Loss: 0.04297323, Time: 0.0770 Steps: 41000, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005416, Sample Num: 86656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02305972, Log Avg loss: 0.01195375, Global Avg Loss: 0.04282265, Time: 0.1056 Steps: 41200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005616, Sample Num: 89856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02224301, Log Avg loss: 0.00012639, Global Avg Loss: 0.04261639, Time: 0.1773 Steps: 41400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 005816, Sample Num: 93056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02147812, Log Avg loss: 0.00000017, Global Avg Loss: 0.04241150, Time: 0.3393 Steps: 41600, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006016, Sample Num: 96256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02076409, Log Avg loss: 0.00000005, Global Avg Loss: 0.04220858, Time: 0.1030 Steps: 41800, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006216, Sample Num: 99456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02009603, Log Avg loss: 0.00000063, Global Avg Loss: 0.04200759, Time: 0.0886 Steps: 42000, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006416, Sample Num: 102656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01949081, Log Avg loss: 0.00068087, Global Avg Loss: 0.04181173, Time: 0.0766 Steps: 42200, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006616, Sample Num: 105856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01890169, Log Avg loss: 0.00000246, Global Avg Loss: 0.04161451, Time: 0.4047 Steps: 42400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006816, Sample Num: 109056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01834706, Log Avg loss: 0.00000013, Global Avg Loss: 0.04141914, Time: 0.0699 Steps: 42600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007016, Sample Num: 112256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01782406, Log Avg loss: 0.00000004, Global Avg Loss: 0.04122559, Time: 0.0637 Steps: 42800, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007216, Sample Num: 115456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01733203, Log Avg loss: 0.00007183, Global Avg Loss: 0.04103418, Time: 0.0955 Steps: 43000, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007416, Sample Num: 118656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01687393, Log Avg loss: 0.00034559, Global Avg Loss: 0.04084581, Time: 0.2001 Steps: 43200, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007616, Sample Num: 121856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01643982, Log Avg loss: 0.00034280, Global Avg Loss: 0.04065916, Time: 0.2184 Steps: 43400, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007816, Sample Num: 125056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01601918, Log Avg loss: 0.00000143, Global Avg Loss: 0.04047265, Time: 0.1218 Steps: 43600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 008016, Sample Num: 128256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01568881, Log Avg loss: 0.00277799, Global Avg Loss: 0.04030053, Time: 0.1499 Steps: 43800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008216, Sample Num: 131456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01665767, Log Avg loss: 0.05548951, Global Avg Loss: 0.04036957, Time: 0.1222 Steps: 44000, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008416, Sample Num: 134656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01843513, Log Avg loss: 0.09145325, Global Avg Loss: 0.04060072, Time: 0.1497 Steps: 44200, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008616, Sample Num: 137856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01800731, Log Avg loss: 0.00000460, Global Avg Loss: 0.04041785, Time: 0.2172 Steps: 44400, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008816, Sample Num: 141056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01759882, Log Avg loss: 0.00000110, Global Avg Loss: 0.04023661, Time: 0.1166 Steps: 44600, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 009016, Sample Num: 144256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01720843, Log Avg loss: 0.00000018, Global Avg Loss: 0.04005699, Time: 0.0864 Steps: 44800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 009216, Sample Num: 147456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01683501, Log Avg loss: 0.00000116, Global Avg Loss: 0.03987896, Time: 0.1221 Steps: 45000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009416, Sample Num: 150656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01647745, Log Avg loss: 0.00000090, Global Avg Loss: 0.03970251, Time: 0.0687 Steps: 45200, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009616, Sample Num: 153856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01613497, Log Avg loss: 0.00001084, Global Avg Loss: 0.03952766, Time: 0.1294 Steps: 45400, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009816, Sample Num: 157056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01767952, Log Avg loss: 0.09194191, Global Avg Loss: 0.03975754, Time: 0.1328 Steps: 45600, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010016, Sample Num: 160256, Cur Loss: 0.00020310, Cur Avg Loss: 0.01732713, Log Avg loss: 0.00003150, Global Avg Loss: 0.03958407, Time: 0.1782 Steps: 45800, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010216, Sample Num: 163456, Cur Loss: 0.00003096, Cur Avg Loss: 0.01698800, Log Avg loss: 0.00000432, Global Avg Loss: 0.03941198, Time: 0.0733 Steps: 46000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010416, Sample Num: 166656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01666283, Log Avg loss: 0.00005358, Global Avg Loss: 0.03924160, Time: 0.1164 Steps: 46200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 010616, Sample Num: 169856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01635346, Log Avg loss: 0.00024097, Global Avg Loss: 0.03907349, Time: 0.0747 Steps: 46400, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 010816, Sample Num: 173056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01605106, Log Avg loss: 0.00000018, Global Avg Loss: 0.03890580, Time: 0.0700 Steps: 46600, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011016, Sample Num: 176256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01575965, Log Avg loss: 0.00000018, Global Avg Loss: 0.03873953, Time: 0.1712 Steps: 46800, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011216, Sample Num: 179456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01547909, Log Avg loss: 0.00002537, Global Avg Loss: 0.03857479, Time: 0.1560 Steps: 47000, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011416, Sample Num: 182656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01520791, Log Avg loss: 0.00000010, Global Avg Loss: 0.03841134, Time: 0.0702 Steps: 47200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011616, Sample Num: 185856, Cur Loss: 0.00000149, Cur Avg Loss: 0.01494611, Log Avg loss: 0.00000262, Global Avg Loss: 0.03824928, Time: 0.1935 Steps: 47400, Updated lr: 0.000060 Training, Epoch: 0004, Batch: 011816, Sample Num: 189056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01469408, Log Avg loss: 0.00005628, Global Avg Loss: 0.03808880, Time: 0.1167 Steps: 47600, Updated lr: 0.000060 ***** Running evaluation checkpoint-47712 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-47712 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1867.943863, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.071767, "eval_total_loss": 99.971875, "eval_acc": 0.999551, "eval_prec": 0.98893, "eval_recall": 0.992593, "eval_f1": 0.990758, "eval_roc_auc": 0.999992, "eval_pr_auc": 0.999674, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 4, "tp": 536}, "eval_mcc2": 0.99053, "eval_mcc": 0.99053, "eval_sn": 0.992593, "eval_sp": 0.999724, "update_flag": false, "test_avg_loss": 0.033998, "test_total_loss": 47.359616, "test_acc": 0.999776, "test_prec": 0.996289, "test_recall": 0.994444, "test_f1": 0.995366, "test_roc_auc": 0.999998, "test_pr_auc": 0.999929, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 3, "tp": 537}, "test_mcc2": 0.995252, "test_mcc": 0.995252, "test_sn": 0.994444, "test_sp": 0.999908, "lr": 6.010077258985556e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.03799950185168793, "train_cur_epoch_loss": 173.63047603230478, "train_cur_epoch_avg_loss": 0.0145565456096835, "train_cur_epoch_time": 1867.9438631534576, "train_cur_epoch_avg_time": 0.15660159818523287, "epoch": 4, "step": 47712} ################################################## Training, Epoch: 0005, Batch: 000088, Sample Num: 1408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000017, Log Avg loss: 0.00002632, Global Avg Loss: 0.03792954, Time: 0.1404 Steps: 47800, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000288, Sample Num: 4608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000007, Log Avg loss: 0.00000002, Global Avg Loss: 0.03777151, Time: 0.1007 Steps: 48000, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000488, Sample Num: 7808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000092, Log Avg loss: 0.00000214, Global Avg Loss: 0.03761479, Time: 0.0973 Steps: 48200, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000688, Sample Num: 11008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000073, Log Avg loss: 0.00000029, Global Avg Loss: 0.03745935, Time: 0.0961 Steps: 48400, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000888, Sample Num: 14208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000058, Log Avg loss: 0.00000004, Global Avg Loss: 0.03730520, Time: 0.0818 Steps: 48600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001088, Sample Num: 17408, Cur Loss: 0.00000000, Cur Avg Loss: 0.02926358, Log Avg loss: 0.15919133, Global Avg Loss: 0.03780473, Time: 0.0683 Steps: 48800, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001288, Sample Num: 20608, Cur Loss: 0.00000000, Cur Avg Loss: 0.02471955, Log Avg loss: 0.00000002, Global Avg Loss: 0.03765043, Time: 0.1018 Steps: 49000, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001488, Sample Num: 23808, Cur Loss: 0.00000000, Cur Avg Loss: 0.02139705, Log Avg loss: 0.00000015, Global Avg Loss: 0.03749738, Time: 0.2404 Steps: 49200, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001688, Sample Num: 27008, Cur Loss: 0.00000000, Cur Avg Loss: 0.02522556, Log Avg loss: 0.05370967, Global Avg Loss: 0.03756302, Time: 0.1017 Steps: 49400, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001888, Sample Num: 30208, Cur Loss: 0.00000000, Cur Avg Loss: 0.02255336, Log Avg loss: 0.00000000, Global Avg Loss: 0.03741155, Time: 0.2937 Steps: 49600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002088, Sample Num: 33408, Cur Loss: 0.00000000, Cur Avg Loss: 0.02768297, Log Avg loss: 0.07610653, Global Avg Loss: 0.03756695, Time: 0.2062 Steps: 49800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002288, Sample Num: 36608, Cur Loss: 0.00000000, Cur Avg Loss: 0.02526450, Log Avg loss: 0.00001566, Global Avg Loss: 0.03741675, Time: 0.2106 Steps: 50000, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002488, Sample Num: 39808, Cur Loss: 0.00000000, Cur Avg Loss: 0.02323360, Log Avg loss: 0.00000012, Global Avg Loss: 0.03726768, Time: 0.4032 Steps: 50200, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002688, Sample Num: 43008, Cur Loss: 0.00000316, Cur Avg Loss: 0.02237429, Log Avg loss: 0.01168443, Global Avg Loss: 0.03716616, Time: 0.1137 Steps: 50400, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002888, Sample Num: 46208, Cur Loss: 0.00000000, Cur Avg Loss: 0.02082486, Log Avg loss: 0.00000056, Global Avg Loss: 0.03701926, Time: 0.1188 Steps: 50600, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003088, Sample Num: 49408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01947611, Log Avg loss: 0.00000005, Global Avg Loss: 0.03687351, Time: 0.1050 Steps: 50800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003288, Sample Num: 52608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01829156, Log Avg loss: 0.00000210, Global Avg Loss: 0.03672892, Time: 0.1453 Steps: 51000, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003488, Sample Num: 55808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01724332, Log Avg loss: 0.00001035, Global Avg Loss: 0.03658549, Time: 0.2026 Steps: 51200, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003688, Sample Num: 59008, Cur Loss: 0.00007749, Cur Avg Loss: 0.02430381, Log Avg loss: 0.14743881, Global Avg Loss: 0.03701682, Time: 0.2177 Steps: 51400, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003888, Sample Num: 62208, Cur Loss: 0.00000000, Cur Avg Loss: 0.02305571, Log Avg loss: 0.00004075, Global Avg Loss: 0.03687350, Time: 0.0787 Steps: 51600, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 004088, Sample Num: 65408, Cur Loss: 0.00000000, Cur Avg Loss: 0.02192775, Log Avg loss: 0.00000009, Global Avg Loss: 0.03673114, Time: 0.1333 Steps: 51800, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 004288, Sample Num: 68608, Cur Loss: 0.00000000, Cur Avg Loss: 0.02090501, Log Avg loss: 0.00000032, Global Avg Loss: 0.03658986, Time: 0.4403 Steps: 52000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004488, Sample Num: 71808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01997342, Log Avg loss: 0.00000004, Global Avg Loss: 0.03644967, Time: 0.1952 Steps: 52200, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004688, Sample Num: 75008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01912131, Log Avg loss: 0.00000001, Global Avg Loss: 0.03631055, Time: 0.2861 Steps: 52400, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004888, Sample Num: 78208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01833895, Log Avg loss: 0.00000038, Global Avg Loss: 0.03617249, Time: 0.2708 Steps: 52600, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005088, Sample Num: 81408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01761808, Log Avg loss: 0.00000004, Global Avg Loss: 0.03603547, Time: 0.1109 Steps: 52800, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005288, Sample Num: 84608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01695174, Log Avg loss: 0.00000001, Global Avg Loss: 0.03589949, Time: 0.0620 Steps: 53000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005488, Sample Num: 87808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01633664, Log Avg loss: 0.00007348, Global Avg Loss: 0.03576481, Time: 0.1386 Steps: 53200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 005688, Sample Num: 91008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01576223, Log Avg loss: 0.00000042, Global Avg Loss: 0.03563086, Time: 0.1671 Steps: 53400, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 005888, Sample Num: 94208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01522684, Log Avg loss: 0.00000019, Global Avg Loss: 0.03549791, Time: 0.1107 Steps: 53600, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006088, Sample Num: 97408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01472661, Log Avg loss: 0.00000002, Global Avg Loss: 0.03536595, Time: 0.4022 Steps: 53800, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006288, Sample Num: 100608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01425821, Log Avg loss: 0.00000006, Global Avg Loss: 0.03523496, Time: 0.2216 Steps: 54000, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006488, Sample Num: 103808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01383375, Log Avg loss: 0.00048878, Global Avg Loss: 0.03510675, Time: 0.2340 Steps: 54200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006688, Sample Num: 107008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01342006, Log Avg loss: 0.00000001, Global Avg Loss: 0.03497768, Time: 0.1027 Steps: 54400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 006888, Sample Num: 110208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01303040, Log Avg loss: 0.00000014, Global Avg Loss: 0.03484955, Time: 0.0801 Steps: 54600, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007088, Sample Num: 113408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01266277, Log Avg loss: 0.00000140, Global Avg Loss: 0.03472237, Time: 0.1386 Steps: 54800, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007288, Sample Num: 116608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01231529, Log Avg loss: 0.00000058, Global Avg Loss: 0.03459611, Time: 0.1022 Steps: 55000, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007488, Sample Num: 119808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01198635, Log Avg loss: 0.00000000, Global Avg Loss: 0.03447076, Time: 0.1494 Steps: 55200, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007688, Sample Num: 123008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01167460, Log Avg loss: 0.00000241, Global Avg Loss: 0.03434633, Time: 0.1107 Steps: 55400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007888, Sample Num: 126208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01137859, Log Avg loss: 0.00000014, Global Avg Loss: 0.03422278, Time: 0.0773 Steps: 55600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008088, Sample Num: 129408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01111365, Log Avg loss: 0.00066425, Global Avg Loss: 0.03410250, Time: 0.0709 Steps: 55800, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008288, Sample Num: 132608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01135158, Log Avg loss: 0.02097371, Global Avg Loss: 0.03405561, Time: 0.1227 Steps: 56000, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008488, Sample Num: 135808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01244616, Log Avg loss: 0.05780527, Global Avg Loss: 0.03414013, Time: 0.0958 Steps: 56200, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008688, Sample Num: 139008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01217108, Log Avg loss: 0.00049686, Global Avg Loss: 0.03402083, Time: 0.1054 Steps: 56400, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008888, Sample Num: 142208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01189721, Log Avg loss: 0.00000025, Global Avg Loss: 0.03390061, Time: 0.1061 Steps: 56600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 009088, Sample Num: 145408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01163539, Log Avg loss: 0.00000009, Global Avg Loss: 0.03378124, Time: 0.1922 Steps: 56800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009288, Sample Num: 148608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01138484, Log Avg loss: 0.00000010, Global Avg Loss: 0.03366271, Time: 0.1334 Steps: 57000, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009488, Sample Num: 151808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01114486, Log Avg loss: 0.00000008, Global Avg Loss: 0.03354501, Time: 0.1992 Steps: 57200, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009688, Sample Num: 155008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01093079, Log Avg loss: 0.00077538, Global Avg Loss: 0.03343083, Time: 0.1215 Steps: 57400, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009888, Sample Num: 158208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01171871, Log Avg loss: 0.04988530, Global Avg Loss: 0.03348797, Time: 0.2172 Steps: 57600, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 010088, Sample Num: 161408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01148856, Log Avg loss: 0.00011002, Global Avg Loss: 0.03337247, Time: 0.0664 Steps: 57800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 010288, Sample Num: 164608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01126522, Log Avg loss: 0.00000015, Global Avg Loss: 0.03325739, Time: 0.0796 Steps: 58000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010488, Sample Num: 167808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01105049, Log Avg loss: 0.00000456, Global Avg Loss: 0.03314312, Time: 0.0985 Steps: 58200, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010688, Sample Num: 171008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01084374, Log Avg loss: 0.00000198, Global Avg Loss: 0.03302963, Time: 0.2876 Steps: 58400, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010888, Sample Num: 174208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01064456, Log Avg loss: 0.00000001, Global Avg Loss: 0.03291690, Time: 0.0980 Steps: 58600, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011088, Sample Num: 177408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01045255, Log Avg loss: 0.00000002, Global Avg Loss: 0.03280493, Time: 0.0848 Steps: 58800, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011288, Sample Num: 180608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01026736, Log Avg loss: 0.00000029, Global Avg Loss: 0.03269373, Time: 0.1715 Steps: 59000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011488, Sample Num: 183808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01008861, Log Avg loss: 0.00000002, Global Avg Loss: 0.03258328, Time: 0.0980 Steps: 59200, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 011688, Sample Num: 187008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00991599, Log Avg loss: 0.00000041, Global Avg Loss: 0.03247357, Time: 0.2513 Steps: 59400, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 011888, Sample Num: 190208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00974956, Log Avg loss: 0.00002321, Global Avg Loss: 0.03236468, Time: 0.2138 Steps: 59600, Updated lr: 0.000050 ***** Running evaluation checkpoint-59640 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-59640 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1864.747385, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.080767, "eval_total_loss": 112.508149, "eval_acc": 0.999596, "eval_prec": 0.990758, "eval_recall": 0.992593, "eval_f1": 0.991674, "eval_roc_auc": 0.999993, "eval_pr_auc": 0.999743, "eval_confusion_matrix": {"tn": 21739, "fp": 5, "fn": 4, "tp": 536}, "eval_mcc2": 0.991468, "eval_mcc": 0.991468, "eval_sn": 0.992593, "eval_sp": 0.99977, "update_flag": false, "test_avg_loss": 0.038481, "test_total_loss": 53.603561, "test_acc": 0.999776, "test_prec": 0.996289, "test_recall": 0.994444, "test_f1": 0.995366, "test_roc_auc": 0.999998, "test_pr_auc": 0.999912, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 3, "tp": 537}, "test_mcc2": 0.995252, "test_mcc": 0.995252, "test_sn": 0.994444, "test_sp": 0.999908, "lr": 5.008397715821297e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.0323429735731545, "train_cur_epoch_loss": 115.90271155519761, "train_cur_epoch_avg_loss": 0.009716860459020591, "train_cur_epoch_time": 1864.7473850250244, "train_cur_epoch_avg_time": 0.15633361712148092, "epoch": 5, "step": 59640} ################################################## Training, Epoch: 0006, Batch: 000160, Sample Num: 2560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03225644, Time: 0.1128 Steps: 59800, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000360, Sample Num: 5760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03214892, Time: 0.3679 Steps: 60000, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000560, Sample Num: 8960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000003, Global Avg Loss: 0.03204211, Time: 0.1981 Steps: 60200, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000760, Sample Num: 12160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000001, Global Avg Loss: 0.03193601, Time: 0.3931 Steps: 60400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 000960, Sample Num: 15360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01285237, Log Avg loss: 0.06169133, Global Avg Loss: 0.03203421, Time: 0.0796 Steps: 60600, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001160, Sample Num: 18560, Cur Loss: 0.00000000, Cur Avg Loss: 0.01072452, Log Avg loss: 0.00051084, Global Avg Loss: 0.03193052, Time: 0.1400 Steps: 60800, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001360, Sample Num: 21760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00914738, Log Avg loss: 0.00000000, Global Avg Loss: 0.03182583, Time: 0.3181 Steps: 61000, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001560, Sample Num: 24960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00797466, Log Avg loss: 0.00000011, Global Avg Loss: 0.03172182, Time: 0.0935 Steps: 61200, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001760, Sample Num: 28160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00944425, Log Avg loss: 0.02090705, Global Avg Loss: 0.03168659, Time: 0.0821 Steps: 61400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001960, Sample Num: 31360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00848055, Log Avg loss: 0.00000000, Global Avg Loss: 0.03158371, Time: 0.0975 Steps: 61600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002160, Sample Num: 34560, Cur Loss: 0.00000006, Cur Avg Loss: 0.01070308, Log Avg loss: 0.03248384, Global Avg Loss: 0.03158663, Time: 0.1698 Steps: 61800, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002360, Sample Num: 37760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00979605, Log Avg loss: 0.00000015, Global Avg Loss: 0.03148474, Time: 0.1092 Steps: 62000, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002560, Sample Num: 40960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00934345, Log Avg loss: 0.00400274, Global Avg Loss: 0.03139637, Time: 0.1429 Steps: 62200, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002760, Sample Num: 44160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00866714, Log Avg loss: 0.00001039, Global Avg Loss: 0.03129577, Time: 0.0812 Steps: 62400, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002960, Sample Num: 47360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00808153, Log Avg loss: 0.00000010, Global Avg Loss: 0.03119579, Time: 0.0985 Steps: 62600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 003160, Sample Num: 50560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00757005, Log Avg loss: 0.00000012, Global Avg Loss: 0.03109644, Time: 0.1644 Steps: 62800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003360, Sample Num: 53760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00711945, Log Avg loss: 0.00000004, Global Avg Loss: 0.03099772, Time: 0.1080 Steps: 63000, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003560, Sample Num: 56960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01421016, Log Avg loss: 0.13333413, Global Avg Loss: 0.03132157, Time: 0.1058 Steps: 63200, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003760, Sample Num: 60160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01370059, Log Avg loss: 0.00463021, Global Avg Loss: 0.03123737, Time: 0.0765 Steps: 63400, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003960, Sample Num: 63360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01300866, Log Avg loss: 0.00000035, Global Avg Loss: 0.03113914, Time: 0.4035 Steps: 63600, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 004160, Sample Num: 66560, Cur Loss: 0.00000000, Cur Avg Loss: 0.01238324, Log Avg loss: 0.00000001, Global Avg Loss: 0.03104152, Time: 0.1724 Steps: 63800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 004360, Sample Num: 69760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01181521, Log Avg loss: 0.00000002, Global Avg Loss: 0.03094452, Time: 0.0862 Steps: 64000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004560, Sample Num: 72960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01129700, Log Avg loss: 0.00000001, Global Avg Loss: 0.03084812, Time: 0.0923 Steps: 64200, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004760, Sample Num: 76160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01082233, Log Avg loss: 0.00000000, Global Avg Loss: 0.03075232, Time: 0.0671 Steps: 64400, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004960, Sample Num: 79360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01038595, Log Avg loss: 0.00000016, Global Avg Loss: 0.03065711, Time: 0.1864 Steps: 64600, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005160, Sample Num: 82560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00998341, Log Avg loss: 0.00000043, Global Avg Loss: 0.03056249, Time: 0.0845 Steps: 64800, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005360, Sample Num: 85760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00961090, Log Avg loss: 0.00000005, Global Avg Loss: 0.03046845, Time: 0.3574 Steps: 65000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005560, Sample Num: 88960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00926518, Log Avg loss: 0.00000002, Global Avg Loss: 0.03037499, Time: 0.1365 Steps: 65200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 005760, Sample Num: 92160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00894348, Log Avg loss: 0.00000000, Global Avg Loss: 0.03028210, Time: 0.1084 Steps: 65400, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 005960, Sample Num: 95360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00864336, Log Avg loss: 0.00000000, Global Avg Loss: 0.03018978, Time: 0.0889 Steps: 65600, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006160, Sample Num: 98560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00836273, Log Avg loss: 0.00000002, Global Avg Loss: 0.03009801, Time: 0.1238 Steps: 65800, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006360, Sample Num: 101760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00809975, Log Avg loss: 0.00000000, Global Avg Loss: 0.03000681, Time: 0.0674 Steps: 66000, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006560, Sample Num: 104960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00785527, Log Avg loss: 0.00008066, Global Avg Loss: 0.02991640, Time: 0.2333 Steps: 66200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006760, Sample Num: 108160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00762286, Log Avg loss: 0.00000000, Global Avg Loss: 0.02982629, Time: 0.1208 Steps: 66400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 006960, Sample Num: 111360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00740382, Log Avg loss: 0.00000000, Global Avg Loss: 0.02973672, Time: 0.1435 Steps: 66600, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007160, Sample Num: 114560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00719702, Log Avg loss: 0.00000041, Global Avg Loss: 0.02964769, Time: 0.1129 Steps: 66800, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007360, Sample Num: 117760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00700145, Log Avg loss: 0.00000002, Global Avg Loss: 0.02955919, Time: 0.1248 Steps: 67000, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007560, Sample Num: 120960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00681622, Log Avg loss: 0.00000001, Global Avg Loss: 0.02947121, Time: 0.0874 Steps: 67200, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007760, Sample Num: 124160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00664057, Log Avg loss: 0.00000090, Global Avg Loss: 0.02938377, Time: 0.1553 Steps: 67400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007960, Sample Num: 127360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00647372, Log Avg loss: 0.00000003, Global Avg Loss: 0.02929683, Time: 0.0727 Steps: 67600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008160, Sample Num: 130560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00638756, Log Avg loss: 0.00295843, Global Avg Loss: 0.02921914, Time: 0.0657 Steps: 67800, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008360, Sample Num: 133760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00623554, Log Avg loss: 0.00003276, Global Avg Loss: 0.02913329, Time: 0.1299 Steps: 68000, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008560, Sample Num: 136960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00612186, Log Avg loss: 0.00137014, Global Avg Loss: 0.02905188, Time: 0.2073 Steps: 68200, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008760, Sample Num: 140160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00598209, Log Avg loss: 0.00000003, Global Avg Loss: 0.02896693, Time: 0.2695 Steps: 68400, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008960, Sample Num: 143360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00584856, Log Avg loss: 0.00000001, Global Avg Loss: 0.02888248, Time: 0.1216 Steps: 68600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 009160, Sample Num: 146560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00572086, Log Avg loss: 0.00000003, Global Avg Loss: 0.02879852, Time: 0.1824 Steps: 68800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009360, Sample Num: 149760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00559862, Log Avg loss: 0.00000000, Global Avg Loss: 0.02871504, Time: 0.0918 Steps: 69000, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009560, Sample Num: 152960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00548150, Log Avg loss: 0.00000003, Global Avg Loss: 0.02863205, Time: 0.1064 Steps: 69200, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009760, Sample Num: 156160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00536947, Log Avg loss: 0.00001470, Global Avg Loss: 0.02854958, Time: 0.1615 Steps: 69400, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009960, Sample Num: 159360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00544822, Log Avg loss: 0.00929102, Global Avg Loss: 0.02849424, Time: 0.1360 Steps: 69600, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 010160, Sample Num: 162560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00534114, Log Avg loss: 0.00000855, Global Avg Loss: 0.02841262, Time: 0.0901 Steps: 69800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 010360, Sample Num: 165760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00523805, Log Avg loss: 0.00000102, Global Avg Loss: 0.02833144, Time: 0.1963 Steps: 70000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010560, Sample Num: 168960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00513886, Log Avg loss: 0.00000103, Global Avg Loss: 0.02825073, Time: 0.1124 Steps: 70200, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010760, Sample Num: 172160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00504335, Log Avg loss: 0.00000032, Global Avg Loss: 0.02817047, Time: 0.1795 Steps: 70400, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010960, Sample Num: 175360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00495132, Log Avg loss: 0.00000002, Global Avg Loss: 0.02809067, Time: 0.3897 Steps: 70600, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011160, Sample Num: 178560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00486259, Log Avg loss: 0.00000040, Global Avg Loss: 0.02801132, Time: 0.1209 Steps: 70800, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011360, Sample Num: 181760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00477698, Log Avg loss: 0.00000003, Global Avg Loss: 0.02793242, Time: 0.1059 Steps: 71000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011560, Sample Num: 184960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00469434, Log Avg loss: 0.00000001, Global Avg Loss: 0.02785395, Time: 0.0680 Steps: 71200, Updated lr: 0.000040 Training, Epoch: 0006, Batch: 011760, Sample Num: 188160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00461450, Log Avg loss: 0.00000005, Global Avg Loss: 0.02777593, Time: 0.1233 Steps: 71400, Updated lr: 0.000040 ***** Running evaluation checkpoint-71568 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-71568 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1867.079925, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.080852, "eval_total_loss": 112.626971, "eval_acc": 0.999506, "eval_prec": 0.988909, "eval_recall": 0.990741, "eval_f1": 0.989824, "eval_roc_auc": 0.999994, "eval_pr_auc": 0.999759, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 5, "tp": 535}, "eval_mcc2": 0.989572, "eval_mcc": 0.989572, "eval_sn": 0.990741, "eval_sp": 0.999724, "update_flag": false, "test_avg_loss": 0.03, "test_total_loss": 41.790655, "test_acc": 0.999731, "test_prec": 0.994444, "test_recall": 0.994444, "test_f1": 0.994444, "test_roc_auc": 0.999998, "test_pr_auc": 0.999939, "test_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 3, "tp": 537}, "test_mcc2": 0.994306, "test_mcc": 0.994306, "test_sn": 0.994444, "test_sp": 0.999862, "lr": 4.006718172657038e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.02771072978953648, "train_cur_epoch_loss": 54.26656567461137, "train_cur_epoch_avg_loss": 0.004549510871446292, "train_cur_epoch_time": 1867.0799250602722, "train_cur_epoch_avg_time": 0.15652916876762846, "epoch": 6, "step": 71568} ################################################## Training, Epoch: 0007, Batch: 000032, Sample Num: 512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000005, Log Avg loss: 0.00000004, Global Avg Loss: 0.02769835, Time: 0.2216 Steps: 71600, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000232, Sample Num: 3712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000005, Log Avg loss: 0.00000006, Global Avg Loss: 0.02762119, Time: 0.0622 Steps: 71800, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000432, Sample Num: 6912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000025, Log Avg loss: 0.00000048, Global Avg Loss: 0.02754447, Time: 0.2277 Steps: 72000, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000632, Sample Num: 10112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000017, Log Avg loss: 0.00000000, Global Avg Loss: 0.02746817, Time: 0.3422 Steps: 72200, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000832, Sample Num: 13312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000013, Log Avg loss: 0.00000000, Global Avg Loss: 0.02739229, Time: 0.1188 Steps: 72400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001032, Sample Num: 16512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00197246, Log Avg loss: 0.01017736, Global Avg Loss: 0.02734486, Time: 0.1724 Steps: 72600, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001232, Sample Num: 19712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00165226, Log Avg loss: 0.00000001, Global Avg Loss: 0.02726974, Time: 0.1808 Steps: 72800, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001432, Sample Num: 22912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00142150, Log Avg loss: 0.00000000, Global Avg Loss: 0.02719503, Time: 0.1225 Steps: 73000, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001632, Sample Num: 26112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00141675, Log Avg loss: 0.00138273, Global Avg Loss: 0.02712450, Time: 0.1966 Steps: 73200, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001832, Sample Num: 29312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00126208, Log Avg loss: 0.00000000, Global Avg Loss: 0.02705059, Time: 0.1085 Steps: 73400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002032, Sample Num: 32512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00113786, Log Avg loss: 0.00000000, Global Avg Loss: 0.02697709, Time: 0.2109 Steps: 73600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002232, Sample Num: 35712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00415100, Log Avg loss: 0.03476448, Global Avg Loss: 0.02699819, Time: 0.0824 Steps: 73800, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002432, Sample Num: 38912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00380963, Log Avg loss: 0.00000000, Global Avg Loss: 0.02692522, Time: 0.1269 Steps: 74000, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002632, Sample Num: 42112, Cur Loss: 0.00000000, Cur Avg Loss: 0.01028160, Log Avg loss: 0.08898078, Global Avg Loss: 0.02709249, Time: 0.0872 Steps: 74200, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002832, Sample Num: 45312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00955635, Log Avg loss: 0.00001196, Global Avg Loss: 0.02701969, Time: 0.0853 Steps: 74400, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003032, Sample Num: 48512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00892598, Log Avg loss: 0.00000000, Global Avg Loss: 0.02694725, Time: 0.1584 Steps: 74600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003232, Sample Num: 51712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00837363, Log Avg loss: 0.00000000, Global Avg Loss: 0.02687520, Time: 0.1986 Steps: 74800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003432, Sample Num: 54912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00788566, Log Avg loss: 0.00000010, Global Avg Loss: 0.02680353, Time: 0.0711 Steps: 75000, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003632, Sample Num: 58112, Cur Loss: 0.00000000, Cur Avg Loss: 0.01156220, Log Avg loss: 0.07465156, Global Avg Loss: 0.02693079, Time: 0.1596 Steps: 75200, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003832, Sample Num: 61312, Cur Loss: 0.00000000, Cur Avg Loss: 0.01096133, Log Avg loss: 0.00004960, Global Avg Loss: 0.02685949, Time: 0.1079 Steps: 75400, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004032, Sample Num: 64512, Cur Loss: 0.00000000, Cur Avg Loss: 0.01041762, Log Avg loss: 0.00000000, Global Avg Loss: 0.02678843, Time: 0.1613 Steps: 75600, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004232, Sample Num: 67712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00992529, Log Avg loss: 0.00000000, Global Avg Loss: 0.02671775, Time: 0.1035 Steps: 75800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004432, Sample Num: 70912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00947740, Log Avg loss: 0.00000003, Global Avg Loss: 0.02664744, Time: 0.2213 Steps: 76000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 004632, Sample Num: 74112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00906818, Log Avg loss: 0.00000000, Global Avg Loss: 0.02657750, Time: 0.1801 Steps: 76200, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 004832, Sample Num: 77312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00869285, Log Avg loss: 0.00000000, Global Avg Loss: 0.02650792, Time: 0.1672 Steps: 76400, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005032, Sample Num: 80512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00834734, Log Avg loss: 0.00000003, Global Avg Loss: 0.02643871, Time: 0.1185 Steps: 76600, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005232, Sample Num: 83712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00802826, Log Avg loss: 0.00000000, Global Avg Loss: 0.02636986, Time: 0.0767 Steps: 76800, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005432, Sample Num: 86912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00773267, Log Avg loss: 0.00000005, Global Avg Loss: 0.02630137, Time: 0.0870 Steps: 77000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005632, Sample Num: 90112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00745807, Log Avg loss: 0.00000000, Global Avg Loss: 0.02623323, Time: 0.1742 Steps: 77200, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 005832, Sample Num: 93312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00720231, Log Avg loss: 0.00000001, Global Avg Loss: 0.02616544, Time: 0.0552 Steps: 77400, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006032, Sample Num: 96512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00696350, Log Avg loss: 0.00000000, Global Avg Loss: 0.02609801, Time: 0.2877 Steps: 77600, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006232, Sample Num: 99712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00674003, Log Avg loss: 0.00000000, Global Avg Loss: 0.02603092, Time: 0.1482 Steps: 77800, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006432, Sample Num: 102912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00653048, Log Avg loss: 0.00000114, Global Avg Loss: 0.02596417, Time: 0.0977 Steps: 78000, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006632, Sample Num: 106112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00633355, Log Avg loss: 0.00000004, Global Avg Loss: 0.02589777, Time: 0.0831 Steps: 78200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 006832, Sample Num: 109312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00614814, Log Avg loss: 0.00000001, Global Avg Loss: 0.02583170, Time: 0.1103 Steps: 78400, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007032, Sample Num: 112512, Cur Loss: 0.00005156, Cur Avg Loss: 0.00597328, Log Avg loss: 0.00000026, Global Avg Loss: 0.02576598, Time: 0.4023 Steps: 78600, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007232, Sample Num: 115712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00580809, Log Avg loss: 0.00000000, Global Avg Loss: 0.02570058, Time: 0.2511 Steps: 78800, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007432, Sample Num: 118912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00565179, Log Avg loss: 0.00000000, Global Avg Loss: 0.02563551, Time: 0.1568 Steps: 79000, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007632, Sample Num: 122112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00550369, Log Avg loss: 0.00000006, Global Avg Loss: 0.02557078, Time: 0.0794 Steps: 79200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007832, Sample Num: 125312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00536315, Log Avg loss: 0.00000039, Global Avg Loss: 0.02550637, Time: 0.1935 Steps: 79400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008032, Sample Num: 128512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00527487, Log Avg loss: 0.00181749, Global Avg Loss: 0.02544685, Time: 0.0998 Steps: 79600, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008232, Sample Num: 131712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00518821, Log Avg loss: 0.00170800, Global Avg Loss: 0.02538735, Time: 0.1497 Steps: 79800, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008432, Sample Num: 134912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00507941, Log Avg loss: 0.00060109, Global Avg Loss: 0.02532539, Time: 0.0869 Steps: 80000, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008632, Sample Num: 138112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00496187, Log Avg loss: 0.00000675, Global Avg Loss: 0.02526225, Time: 0.2136 Steps: 80200, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008832, Sample Num: 141312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00484951, Log Avg loss: 0.00000007, Global Avg Loss: 0.02519941, Time: 0.1562 Steps: 80400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 009032, Sample Num: 144512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00474214, Log Avg loss: 0.00000035, Global Avg Loss: 0.02513688, Time: 0.0926 Steps: 80600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009232, Sample Num: 147712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00463940, Log Avg loss: 0.00000000, Global Avg Loss: 0.02507466, Time: 0.1345 Steps: 80800, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009432, Sample Num: 150912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00454103, Log Avg loss: 0.00000000, Global Avg Loss: 0.02501275, Time: 0.1082 Steps: 81000, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009632, Sample Num: 154112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00444674, Log Avg loss: 0.00000017, Global Avg Loss: 0.02495114, Time: 0.0796 Steps: 81200, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009832, Sample Num: 157312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00493552, Log Avg loss: 0.02847512, Global Avg Loss: 0.02495980, Time: 0.0910 Steps: 81400, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 010032, Sample Num: 160512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00483716, Log Avg loss: 0.00000183, Global Avg Loss: 0.02489863, Time: 0.3578 Steps: 81600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 010232, Sample Num: 163712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00474262, Log Avg loss: 0.00000032, Global Avg Loss: 0.02483775, Time: 0.1257 Steps: 81800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010432, Sample Num: 166912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00465336, Log Avg loss: 0.00008691, Global Avg Loss: 0.02477738, Time: 0.0961 Steps: 82000, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010632, Sample Num: 170112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00456583, Log Avg loss: 0.00000002, Global Avg Loss: 0.02471710, Time: 0.1038 Steps: 82200, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010832, Sample Num: 173312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00448152, Log Avg loss: 0.00000000, Global Avg Loss: 0.02465710, Time: 0.2010 Steps: 82400, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011032, Sample Num: 176512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00440028, Log Avg loss: 0.00000000, Global Avg Loss: 0.02459740, Time: 0.2172 Steps: 82600, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011232, Sample Num: 179712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00432197, Log Avg loss: 0.00000260, Global Avg Loss: 0.02453799, Time: 0.2168 Steps: 82800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011432, Sample Num: 182912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00424636, Log Avg loss: 0.00000000, Global Avg Loss: 0.02447887, Time: 0.2170 Steps: 83000, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 011632, Sample Num: 186112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00417335, Log Avg loss: 0.00000003, Global Avg Loss: 0.02442002, Time: 0.2344 Steps: 83200, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 011832, Sample Num: 189312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00410280, Log Avg loss: 0.00000001, Global Avg Loss: 0.02436146, Time: 0.3107 Steps: 83400, Updated lr: 0.000030 ***** Running evaluation checkpoint-83496 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-83496 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1862.403648, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.093132, "eval_total_loss": 129.73273, "eval_acc": 0.999506, "eval_prec": 0.990724, "eval_recall": 0.988889, "eval_f1": 0.989805, "eval_roc_auc": 0.999993, "eval_pr_auc": 0.999734, "eval_confusion_matrix": {"tn": 21739, "fp": 5, "fn": 6, "tp": 534}, "eval_mcc2": 0.989553, "eval_mcc": 0.989553, "eval_sn": 0.988889, "eval_sp": 0.99977, "update_flag": false, "test_avg_loss": 0.03744, "test_total_loss": 52.153813, "test_acc": 0.999776, "test_prec": 0.996289, "test_recall": 0.994444, "test_f1": 0.995366, "test_roc_auc": 0.999999, "test_pr_auc": 0.999943, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 3, "tp": 537}, "test_mcc2": 0.995252, "test_mcc": 0.995252, "test_sn": 0.994444, "test_sp": 0.999908, "lr": 3.005038629492778e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.024333451832396755, "train_cur_epoch_loss": 48.54438462025112, "train_cur_epoch_avg_loss": 0.0040697840895582765, "train_cur_epoch_time": 1862.4036481380463, "train_cur_epoch_avg_time": 0.15613712677213668, "epoch": 7, "step": 83496} ################################################## Training, Epoch: 0008, Batch: 000104, Sample Num: 1664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000001, Global Avg Loss: 0.02430318, Time: 0.3033 Steps: 83600, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000304, Sample Num: 4864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000001, Global Avg Loss: 0.02424518, Time: 0.1654 Steps: 83800, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000504, Sample Num: 8064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000002, Global Avg Loss: 0.02418745, Time: 0.0952 Steps: 84000, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000704, Sample Num: 11264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000000, Global Avg Loss: 0.02413000, Time: 0.1359 Steps: 84200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 000904, Sample Num: 14464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000000, Global Avg Loss: 0.02407282, Time: 0.1041 Steps: 84400, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001104, Sample Num: 17664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002478, Log Avg loss: 0.00013677, Global Avg Loss: 0.02401623, Time: 0.1752 Steps: 84600, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001304, Sample Num: 20864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002098, Log Avg loss: 0.00000000, Global Avg Loss: 0.02395959, Time: 0.2271 Steps: 84800, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001504, Sample Num: 24064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001819, Log Avg loss: 0.00000000, Global Avg Loss: 0.02390321, Time: 0.2448 Steps: 85000, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001704, Sample Num: 27264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00004066, Log Avg loss: 0.00020960, Global Avg Loss: 0.02384760, Time: 0.2191 Steps: 85200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001904, Sample Num: 30464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00003639, Log Avg loss: 0.00000000, Global Avg Loss: 0.02379175, Time: 0.0869 Steps: 85400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002104, Sample Num: 33664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00067946, Log Avg loss: 0.00680156, Global Avg Loss: 0.02375205, Time: 0.2104 Steps: 85600, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002304, Sample Num: 36864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00181122, Log Avg loss: 0.01371730, Global Avg Loss: 0.02372866, Time: 0.1245 Steps: 85800, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002504, Sample Num: 40064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00166655, Log Avg loss: 0.00000000, Global Avg Loss: 0.02367348, Time: 0.1954 Steps: 86000, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002704, Sample Num: 43264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00187555, Log Avg loss: 0.00449222, Global Avg Loss: 0.02362897, Time: 0.1087 Steps: 86200, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002904, Sample Num: 46464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00174923, Log Avg loss: 0.00004138, Global Avg Loss: 0.02357437, Time: 0.0933 Steps: 86400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 003104, Sample Num: 49664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00163653, Log Avg loss: 0.00000001, Global Avg Loss: 0.02351993, Time: 0.1086 Steps: 86600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003304, Sample Num: 52864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00153746, Log Avg loss: 0.00000002, Global Avg Loss: 0.02346573, Time: 0.1572 Steps: 86800, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003504, Sample Num: 56064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00144971, Log Avg loss: 0.00000005, Global Avg Loss: 0.02341179, Time: 0.1281 Steps: 87000, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003704, Sample Num: 59264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00508999, Log Avg loss: 0.06886769, Global Avg Loss: 0.02351605, Time: 0.2110 Steps: 87200, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003904, Sample Num: 62464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00482923, Log Avg loss: 0.00000001, Global Avg Loss: 0.02346223, Time: 0.3774 Steps: 87400, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 004104, Sample Num: 65664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00459389, Log Avg loss: 0.00000000, Global Avg Loss: 0.02340867, Time: 0.2484 Steps: 87600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 004304, Sample Num: 68864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00438042, Log Avg loss: 0.00000000, Global Avg Loss: 0.02335534, Time: 0.0705 Steps: 87800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004504, Sample Num: 72064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00418591, Log Avg loss: 0.00000000, Global Avg Loss: 0.02330226, Time: 0.2987 Steps: 88000, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004704, Sample Num: 75264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00400794, Log Avg loss: 0.00000000, Global Avg Loss: 0.02324942, Time: 0.2083 Steps: 88200, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004904, Sample Num: 78464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00384448, Log Avg loss: 0.00000002, Global Avg Loss: 0.02319682, Time: 0.1035 Steps: 88400, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005104, Sample Num: 81664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00369383, Log Avg loss: 0.00000000, Global Avg Loss: 0.02314446, Time: 0.1208 Steps: 88600, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005304, Sample Num: 84864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00355455, Log Avg loss: 0.00000004, Global Avg Loss: 0.02309233, Time: 0.1068 Steps: 88800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005504, Sample Num: 88064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00342539, Log Avg loss: 0.00000000, Global Avg Loss: 0.02304044, Time: 0.1439 Steps: 89000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 005704, Sample Num: 91264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00330528, Log Avg loss: 0.00000000, Global Avg Loss: 0.02298878, Time: 0.4043 Steps: 89200, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 005904, Sample Num: 94464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00319332, Log Avg loss: 0.00000001, Global Avg Loss: 0.02293735, Time: 0.1545 Steps: 89400, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006104, Sample Num: 97664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00308869, Log Avg loss: 0.00000001, Global Avg Loss: 0.02288615, Time: 0.1714 Steps: 89600, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006304, Sample Num: 100864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00299070, Log Avg loss: 0.00000000, Global Avg Loss: 0.02283518, Time: 0.0711 Steps: 89800, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006504, Sample Num: 104064, Cur Loss: 0.00000972, Cur Avg Loss: 0.00289874, Log Avg loss: 0.00000041, Global Avg Loss: 0.02278444, Time: 0.1114 Steps: 90000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006704, Sample Num: 107264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00281227, Log Avg loss: 0.00000001, Global Avg Loss: 0.02273392, Time: 0.0985 Steps: 90200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 006904, Sample Num: 110464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00273080, Log Avg loss: 0.00000000, Global Avg Loss: 0.02268362, Time: 0.1968 Steps: 90400, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007104, Sample Num: 113664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00265392, Log Avg loss: 0.00000000, Global Avg Loss: 0.02263355, Time: 0.1333 Steps: 90600, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007304, Sample Num: 116864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00258125, Log Avg loss: 0.00000000, Global Avg Loss: 0.02258369, Time: 0.1220 Steps: 90800, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007504, Sample Num: 120064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00251245, Log Avg loss: 0.00000000, Global Avg Loss: 0.02253406, Time: 0.1558 Steps: 91000, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007704, Sample Num: 123264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00244723, Log Avg loss: 0.00000000, Global Avg Loss: 0.02248464, Time: 0.2830 Steps: 91200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007904, Sample Num: 126464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00238530, Log Avg loss: 0.00000000, Global Avg Loss: 0.02243544, Time: 0.1465 Steps: 91400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008104, Sample Num: 129664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00232730, Log Avg loss: 0.00003503, Global Avg Loss: 0.02238653, Time: 0.0639 Steps: 91600, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008304, Sample Num: 132864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00227964, Log Avg loss: 0.00034831, Global Avg Loss: 0.02233852, Time: 0.0911 Steps: 91800, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008504, Sample Num: 136064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00222619, Log Avg loss: 0.00000702, Global Avg Loss: 0.02228997, Time: 0.1716 Steps: 92000, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008704, Sample Num: 139264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00217503, Log Avg loss: 0.00000001, Global Avg Loss: 0.02224162, Time: 0.1926 Steps: 92200, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008904, Sample Num: 142464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00212618, Log Avg loss: 0.00000004, Global Avg Loss: 0.02219348, Time: 0.1342 Steps: 92400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 009104, Sample Num: 145664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00207947, Log Avg loss: 0.00000000, Global Avg Loss: 0.02214554, Time: 0.1191 Steps: 92600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009304, Sample Num: 148864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00203477, Log Avg loss: 0.00000001, Global Avg Loss: 0.02209782, Time: 0.1207 Steps: 92800, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009504, Sample Num: 152064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00199195, Log Avg loss: 0.00000004, Global Avg Loss: 0.02205029, Time: 0.2561 Steps: 93000, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009704, Sample Num: 155264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00195090, Log Avg loss: 0.00000000, Global Avg Loss: 0.02200298, Time: 0.2873 Steps: 93200, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009904, Sample Num: 158464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00193299, Log Avg loss: 0.00106428, Global Avg Loss: 0.02195814, Time: 0.1443 Steps: 93400, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 010104, Sample Num: 161664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00189475, Log Avg loss: 0.00000068, Global Avg Loss: 0.02191122, Time: 0.0568 Steps: 93600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 010304, Sample Num: 164864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00185797, Log Avg loss: 0.00000005, Global Avg Loss: 0.02186450, Time: 0.2121 Steps: 93800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010504, Sample Num: 168064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00182270, Log Avg loss: 0.00000563, Global Avg Loss: 0.02181800, Time: 0.2108 Steps: 94000, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010704, Sample Num: 171264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00178864, Log Avg loss: 0.00000000, Global Avg Loss: 0.02177167, Time: 0.0852 Steps: 94200, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010904, Sample Num: 174464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00175584, Log Avg loss: 0.00000000, Global Avg Loss: 0.02172555, Time: 0.1129 Steps: 94400, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011104, Sample Num: 177664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00172421, Log Avg loss: 0.00000000, Global Avg Loss: 0.02167961, Time: 0.1575 Steps: 94600, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011304, Sample Num: 180864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00169371, Log Avg loss: 0.00000008, Global Avg Loss: 0.02163388, Time: 0.0954 Steps: 94800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011504, Sample Num: 184064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00166426, Log Avg loss: 0.00000002, Global Avg Loss: 0.02158833, Time: 0.1966 Steps: 95000, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 011704, Sample Num: 187264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00163582, Log Avg loss: 0.00000000, Global Avg Loss: 0.02154298, Time: 0.0763 Steps: 95200, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 011904, Sample Num: 190464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00160834, Log Avg loss: 0.00000001, Global Avg Loss: 0.02149782, Time: 0.0969 Steps: 95400, Updated lr: 0.000020 ***** Running evaluation checkpoint-95424 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-95424 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1861.330375, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.090563, "eval_total_loss": 126.154038, "eval_acc": 0.999461, "eval_prec": 0.988889, "eval_recall": 0.988889, "eval_f1": 0.988889, "eval_roc_auc": 0.999994, "eval_pr_auc": 0.999743, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 6, "tp": 534}, "eval_mcc2": 0.988613, "eval_mcc": 0.988613, "eval_sn": 0.988889, "eval_sp": 0.999724, "update_flag": false, "test_avg_loss": 0.032228, "test_total_loss": 44.892911, "test_acc": 0.999776, "test_prec": 0.996289, "test_recall": 0.994444, "test_f1": 0.995366, "test_roc_auc": 0.999999, "test_pr_auc": 0.999956, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 3, "tp": 537}, "test_mcc2": 0.995252, "test_mcc": 0.995252, "test_sn": 0.994444, "test_sp": 0.999908, "lr": 2.003359086328519e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.021492408260784364, "train_cur_epoch_loss": 19.14567167928658, "train_cur_epoch_avg_loss": 0.0016051032594975334, "train_cur_epoch_time": 1861.330374956131, "train_cur_epoch_avg_time": 0.15604714746446438, "epoch": 8, "step": 95424} ################################################## Training, Epoch: 0009, Batch: 000176, Sample Num: 2816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02145284, Time: 0.1475 Steps: 95600, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000376, Sample Num: 6016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02140805, Time: 0.0937 Steps: 95800, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000576, Sample Num: 9216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02136345, Time: 0.3566 Steps: 96000, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000776, Sample Num: 12416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000001, Global Avg Loss: 0.02131904, Time: 0.2834 Steps: 96200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 000976, Sample Num: 15616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000003, Log Avg loss: 0.00000012, Global Avg Loss: 0.02127481, Time: 0.0959 Steps: 96400, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001176, Sample Num: 18816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000177, Log Avg loss: 0.00001026, Global Avg Loss: 0.02123078, Time: 0.1364 Steps: 96600, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001376, Sample Num: 22016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000151, Log Avg loss: 0.00000002, Global Avg Loss: 0.02118692, Time: 0.1064 Steps: 96800, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001576, Sample Num: 25216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00013980, Log Avg loss: 0.00109123, Global Avg Loss: 0.02114548, Time: 0.1341 Steps: 97000, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001776, Sample Num: 28416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00012406, Log Avg loss: 0.00000000, Global Avg Loss: 0.02110197, Time: 0.4024 Steps: 97200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001976, Sample Num: 31616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00011150, Log Avg loss: 0.00000000, Global Avg Loss: 0.02105864, Time: 0.1407 Steps: 97400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002176, Sample Num: 34816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00206770, Log Avg loss: 0.02139496, Global Avg Loss: 0.02105933, Time: 0.1186 Steps: 97600, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002376, Sample Num: 38016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00189365, Log Avg loss: 0.00000000, Global Avg Loss: 0.02101627, Time: 0.1660 Steps: 97800, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002576, Sample Num: 41216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00396223, Log Avg loss: 0.02853696, Global Avg Loss: 0.02103162, Time: 0.0952 Steps: 98000, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002776, Sample Num: 44416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00367737, Log Avg loss: 0.00000831, Global Avg Loss: 0.02098880, Time: 0.1298 Steps: 98200, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002976, Sample Num: 47616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00343023, Log Avg loss: 0.00000000, Global Avg Loss: 0.02094614, Time: 0.1071 Steps: 98400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 003176, Sample Num: 50816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00321422, Log Avg loss: 0.00000000, Global Avg Loss: 0.02090365, Time: 0.1111 Steps: 98600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003376, Sample Num: 54016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00302381, Log Avg loss: 0.00000000, Global Avg Loss: 0.02086134, Time: 0.0889 Steps: 98800, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003576, Sample Num: 57216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00714797, Log Avg loss: 0.07676379, Global Avg Loss: 0.02097427, Time: 0.1733 Steps: 99000, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003776, Sample Num: 60416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00676947, Log Avg loss: 0.00000201, Global Avg Loss: 0.02093199, Time: 0.2951 Steps: 99200, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003976, Sample Num: 63616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00642896, Log Avg loss: 0.00000000, Global Avg Loss: 0.02088987, Time: 0.3374 Steps: 99400, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 004176, Sample Num: 66816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00612106, Log Avg loss: 0.00000000, Global Avg Loss: 0.02084792, Time: 0.2049 Steps: 99600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 004376, Sample Num: 70016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00584130, Log Avg loss: 0.00000000, Global Avg Loss: 0.02080614, Time: 0.0864 Steps: 99800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004576, Sample Num: 73216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00558600, Log Avg loss: 0.00000000, Global Avg Loss: 0.02076453, Time: 0.1806 Steps: 100000, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004776, Sample Num: 76416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00535208, Log Avg loss: 0.00000000, Global Avg Loss: 0.02072308, Time: 0.0864 Steps: 100200, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004976, Sample Num: 79616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00513697, Log Avg loss: 0.00000001, Global Avg Loss: 0.02068180, Time: 0.1031 Steps: 100400, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005176, Sample Num: 82816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00493847, Log Avg loss: 0.00000000, Global Avg Loss: 0.02064069, Time: 0.2199 Steps: 100600, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005376, Sample Num: 86016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00475475, Log Avg loss: 0.00000000, Global Avg Loss: 0.02059973, Time: 0.1542 Steps: 100800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005576, Sample Num: 89216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00458421, Log Avg loss: 0.00000000, Global Avg Loss: 0.02055894, Time: 0.2255 Steps: 101000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 005776, Sample Num: 92416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00442547, Log Avg loss: 0.00000001, Global Avg Loss: 0.02051831, Time: 0.2129 Steps: 101200, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 005976, Sample Num: 95616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00427737, Log Avg loss: 0.00000000, Global Avg Loss: 0.02047784, Time: 0.0750 Steps: 101400, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006176, Sample Num: 98816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00413885, Log Avg loss: 0.00000001, Global Avg Loss: 0.02043753, Time: 0.1498 Steps: 101600, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006376, Sample Num: 102016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00400903, Log Avg loss: 0.00000000, Global Avg Loss: 0.02039738, Time: 0.1023 Steps: 101800, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006576, Sample Num: 105216, Cur Loss: 0.00000215, Cur Avg Loss: 0.00388710, Log Avg loss: 0.00000009, Global Avg Loss: 0.02035738, Time: 0.0976 Steps: 102000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006776, Sample Num: 108416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00377238, Log Avg loss: 0.00000031, Global Avg Loss: 0.02031755, Time: 0.0873 Steps: 102200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 006976, Sample Num: 111616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00366422, Log Avg loss: 0.00000000, Global Avg Loss: 0.02027786, Time: 0.1508 Steps: 102400, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007176, Sample Num: 114816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00356210, Log Avg loss: 0.00000001, Global Avg Loss: 0.02023834, Time: 0.1576 Steps: 102600, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007376, Sample Num: 118016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00346551, Log Avg loss: 0.00000000, Global Avg Loss: 0.02019896, Time: 0.1317 Steps: 102800, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007576, Sample Num: 121216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00337403, Log Avg loss: 0.00000001, Global Avg Loss: 0.02015974, Time: 0.1163 Steps: 103000, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007776, Sample Num: 124416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00328725, Log Avg loss: 0.00000000, Global Avg Loss: 0.02012067, Time: 0.0912 Steps: 103200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007976, Sample Num: 127616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00320482, Log Avg loss: 0.00000000, Global Avg Loss: 0.02008175, Time: 0.1505 Steps: 103400, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008176, Sample Num: 130816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00312649, Log Avg loss: 0.00000263, Global Avg Loss: 0.02004299, Time: 0.0734 Steps: 103600, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008376, Sample Num: 134016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00306371, Log Avg loss: 0.00049727, Global Avg Loss: 0.02000533, Time: 0.4034 Steps: 103800, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008576, Sample Num: 137216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00299226, Log Avg loss: 0.00000006, Global Avg Loss: 0.01996686, Time: 0.1739 Steps: 104000, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008776, Sample Num: 140416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00292407, Log Avg loss: 0.00000001, Global Avg Loss: 0.01992853, Time: 0.1241 Steps: 104200, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008976, Sample Num: 143616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00285892, Log Avg loss: 0.00000001, Global Avg Loss: 0.01989036, Time: 0.3468 Steps: 104400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009176, Sample Num: 146816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00279660, Log Avg loss: 0.00000000, Global Avg Loss: 0.01985233, Time: 0.1267 Steps: 104600, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009376, Sample Num: 150016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00273695, Log Avg loss: 0.00000000, Global Avg Loss: 0.01981444, Time: 0.1327 Steps: 104800, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009576, Sample Num: 153216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00267980, Log Avg loss: 0.00000085, Global Avg Loss: 0.01977670, Time: 0.3541 Steps: 105000, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009776, Sample Num: 156416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00262555, Log Avg loss: 0.00002769, Global Avg Loss: 0.01973915, Time: 0.1491 Steps: 105200, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009976, Sample Num: 159616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00266607, Log Avg loss: 0.00464698, Global Avg Loss: 0.01971052, Time: 0.1623 Steps: 105400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 010176, Sample Num: 162816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00261379, Log Avg loss: 0.00000620, Global Avg Loss: 0.01967320, Time: 0.0814 Steps: 105600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010376, Sample Num: 166016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00256344, Log Avg loss: 0.00000114, Global Avg Loss: 0.01963601, Time: 0.3372 Steps: 105800, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010576, Sample Num: 169216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00251497, Log Avg loss: 0.00000056, Global Avg Loss: 0.01959896, Time: 0.4039 Steps: 106000, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010776, Sample Num: 172416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00246829, Log Avg loss: 0.00000000, Global Avg Loss: 0.01956205, Time: 0.0865 Steps: 106200, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010976, Sample Num: 175616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00242332, Log Avg loss: 0.00000007, Global Avg Loss: 0.01952528, Time: 0.1313 Steps: 106400, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 011176, Sample Num: 178816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00237995, Log Avg loss: 0.00000001, Global Avg Loss: 0.01948865, Time: 0.1305 Steps: 106600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 011376, Sample Num: 182016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00233811, Log Avg loss: 0.00000001, Global Avg Loss: 0.01945215, Time: 0.1003 Steps: 106800, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 011576, Sample Num: 185216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00229771, Log Avg loss: 0.00000001, Global Avg Loss: 0.01941579, Time: 0.1476 Steps: 107000, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 011776, Sample Num: 188416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00225869, Log Avg loss: 0.00000001, Global Avg Loss: 0.01937957, Time: 0.2776 Steps: 107200, Updated lr: 0.000010 ***** Running evaluation checkpoint-107352 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-107352 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1864.412487, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.085801, "eval_total_loss": 119.521364, "eval_acc": 0.999551, "eval_prec": 0.98893, "eval_recall": 0.992593, "eval_f1": 0.990758, "eval_roc_auc": 0.999993, "eval_pr_auc": 0.999736, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 4, "tp": 536}, "eval_mcc2": 0.99053, "eval_mcc": 0.99053, "eval_sn": 0.992593, "eval_sp": 0.999724, "update_flag": false, "test_avg_loss": 0.026189, "test_total_loss": 36.481733, "test_acc": 0.999776, "test_prec": 0.996289, "test_recall": 0.994444, "test_f1": 0.995366, "test_roc_auc": 0.999999, "test_pr_auc": 0.999956, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 3, "tp": 537}, "test_mcc2": 0.995252, "test_mcc": 0.995252, "test_sn": 0.994444, "test_sp": 0.999908, "lr": 1.0016795431642594e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.019352130428835136, "train_cur_epoch_loss": 26.598339919220223, "train_cur_epoch_avg_loss": 0.002229907773241132, "train_cur_epoch_time": 1864.4124870300293, "train_cur_epoch_avg_time": 0.1563055404954753, "epoch": 9, "step": 107352} ################################################## Training, Epoch: 0010, Batch: 000048, Sample Num: 768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000002, Global Avg Loss: 0.01934348, Time: 0.1428 Steps: 107400, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000248, Sample Num: 3968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.01930753, Time: 0.1181 Steps: 107600, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000448, Sample Num: 7168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.01927171, Time: 0.1224 Steps: 107800, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000648, Sample Num: 10368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.01923602, Time: 0.0767 Steps: 108000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 000848, Sample Num: 13568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.01920046, Time: 0.0952 Steps: 108200, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001048, Sample Num: 16768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000378, Log Avg loss: 0.00001982, Global Avg Loss: 0.01916507, Time: 0.0960 Steps: 108400, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001248, Sample Num: 19968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000318, Log Avg loss: 0.00000000, Global Avg Loss: 0.01912978, Time: 0.2152 Steps: 108600, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001448, Sample Num: 23168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000274, Log Avg loss: 0.00000000, Global Avg Loss: 0.01909461, Time: 0.1153 Steps: 108800, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001648, Sample Num: 26368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000548, Log Avg loss: 0.00002536, Global Avg Loss: 0.01905962, Time: 0.0814 Steps: 109000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001848, Sample Num: 29568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000489, Log Avg loss: 0.00000000, Global Avg Loss: 0.01902472, Time: 0.1952 Steps: 109200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002048, Sample Num: 32768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00043736, Log Avg loss: 0.00443339, Global Avg Loss: 0.01899804, Time: 0.1104 Steps: 109400, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002248, Sample Num: 35968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00041415, Log Avg loss: 0.00017650, Global Avg Loss: 0.01896369, Time: 0.1133 Steps: 109600, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002448, Sample Num: 39168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00038032, Log Avg loss: 0.00000000, Global Avg Loss: 0.01892915, Time: 0.0659 Steps: 109800, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002648, Sample Num: 42368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00635433, Log Avg loss: 0.07947629, Global Avg Loss: 0.01903924, Time: 0.0795 Steps: 110000, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002848, Sample Num: 45568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00590889, Log Avg loss: 0.00001123, Global Avg Loss: 0.01900470, Time: 0.0810 Steps: 110200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 003048, Sample Num: 48768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00552117, Log Avg loss: 0.00000000, Global Avg Loss: 0.01897028, Time: 0.0873 Steps: 110400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003248, Sample Num: 51968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00518120, Log Avg loss: 0.00000002, Global Avg Loss: 0.01893597, Time: 0.4034 Steps: 110600, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003448, Sample Num: 55168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00488066, Log Avg loss: 0.00000001, Global Avg Loss: 0.01890179, Time: 0.1094 Steps: 110800, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003648, Sample Num: 58368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00747500, Log Avg loss: 0.05220140, Global Avg Loss: 0.01896179, Time: 0.1217 Steps: 111000, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003848, Sample Num: 61568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00708649, Log Avg loss: 0.00000001, Global Avg Loss: 0.01892769, Time: 0.1040 Steps: 111200, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 004048, Sample Num: 64768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00673637, Log Avg loss: 0.00000001, Global Avg Loss: 0.01889370, Time: 0.1120 Steps: 111400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 004248, Sample Num: 67968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00641921, Log Avg loss: 0.00000000, Global Avg Loss: 0.01885985, Time: 0.1098 Steps: 111600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004448, Sample Num: 71168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00613058, Log Avg loss: 0.00000000, Global Avg Loss: 0.01882611, Time: 0.2025 Steps: 111800, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004648, Sample Num: 74368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00586678, Log Avg loss: 0.00000000, Global Avg Loss: 0.01879249, Time: 0.1226 Steps: 112000, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004848, Sample Num: 77568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00562476, Log Avg loss: 0.00000000, Global Avg Loss: 0.01875899, Time: 0.2172 Steps: 112200, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005048, Sample Num: 80768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00540190, Log Avg loss: 0.00000001, Global Avg Loss: 0.01872561, Time: 0.2072 Steps: 112400, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005248, Sample Num: 83968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00519604, Log Avg loss: 0.00000000, Global Avg Loss: 0.01869235, Time: 0.1189 Steps: 112600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005448, Sample Num: 87168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00500529, Log Avg loss: 0.00000000, Global Avg Loss: 0.01865921, Time: 0.4023 Steps: 112800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 005648, Sample Num: 90368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00482805, Log Avg loss: 0.00000000, Global Avg Loss: 0.01862618, Time: 0.1165 Steps: 113000, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 005848, Sample Num: 93568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00466293, Log Avg loss: 0.00000000, Global Avg Loss: 0.01859327, Time: 0.0878 Steps: 113200, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006048, Sample Num: 96768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00450873, Log Avg loss: 0.00000001, Global Avg Loss: 0.01856048, Time: 0.0656 Steps: 113400, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006248, Sample Num: 99968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00436441, Log Avg loss: 0.00000000, Global Avg Loss: 0.01852781, Time: 0.1401 Steps: 113600, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006448, Sample Num: 103168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00422904, Log Avg loss: 0.00000001, Global Avg Loss: 0.01849524, Time: 0.1270 Steps: 113800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006648, Sample Num: 106368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00410181, Log Avg loss: 0.00000000, Global Avg Loss: 0.01846280, Time: 0.2214 Steps: 114000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 006848, Sample Num: 109568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00398201, Log Avg loss: 0.00000003, Global Avg Loss: 0.01843046, Time: 0.2934 Steps: 114200, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007048, Sample Num: 112768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00386903, Log Avg loss: 0.00000034, Global Avg Loss: 0.01839824, Time: 0.1363 Steps: 114400, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007248, Sample Num: 115968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00376226, Log Avg loss: 0.00000000, Global Avg Loss: 0.01836613, Time: 0.0870 Steps: 114600, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007448, Sample Num: 119168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00366124, Log Avg loss: 0.00000000, Global Avg Loss: 0.01833414, Time: 0.1063 Steps: 114800, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007648, Sample Num: 122368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00356549, Log Avg loss: 0.00000002, Global Avg Loss: 0.01830225, Time: 0.2283 Steps: 115000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007848, Sample Num: 125568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00347463, Log Avg loss: 0.00000000, Global Avg Loss: 0.01827048, Time: 0.2550 Steps: 115200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008048, Sample Num: 128768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00338833, Log Avg loss: 0.00000204, Global Avg Loss: 0.01823881, Time: 0.1366 Steps: 115400, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008248, Sample Num: 131968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00330635, Log Avg loss: 0.00000721, Global Avg Loss: 0.01820727, Time: 0.1728 Steps: 115600, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008448, Sample Num: 135168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00325993, Log Avg loss: 0.00134571, Global Avg Loss: 0.01817815, Time: 0.4496 Steps: 115800, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008648, Sample Num: 138368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00318454, Log Avg loss: 0.00000000, Global Avg Loss: 0.01814681, Time: 0.0878 Steps: 116000, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008848, Sample Num: 141568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00311256, Log Avg loss: 0.00000000, Global Avg Loss: 0.01811557, Time: 0.1200 Steps: 116200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 009048, Sample Num: 144768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00304375, Log Avg loss: 0.00000000, Global Avg Loss: 0.01808445, Time: 0.0993 Steps: 116400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009248, Sample Num: 147968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00297793, Log Avg loss: 0.00000000, Global Avg Loss: 0.01805343, Time: 0.1024 Steps: 116600, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009448, Sample Num: 151168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00291490, Log Avg loss: 0.00000044, Global Avg Loss: 0.01802252, Time: 0.0914 Steps: 116800, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009648, Sample Num: 154368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00285452, Log Avg loss: 0.00000213, Global Avg Loss: 0.01799171, Time: 0.1262 Steps: 117000, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009848, Sample Num: 157568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00280557, Log Avg loss: 0.00044441, Global Avg Loss: 0.01796177, Time: 0.1299 Steps: 117200, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 010048, Sample Num: 160768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00274973, Log Avg loss: 0.00000019, Global Avg Loss: 0.01793117, Time: 0.3182 Steps: 117400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 010248, Sample Num: 163968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00269607, Log Avg loss: 0.00000000, Global Avg Loss: 0.01790067, Time: 0.0838 Steps: 117600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010448, Sample Num: 167168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00264447, Log Avg loss: 0.00000026, Global Avg Loss: 0.01787028, Time: 0.2834 Steps: 117800, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010648, Sample Num: 170368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00259480, Log Avg loss: 0.00000000, Global Avg Loss: 0.01783999, Time: 0.0742 Steps: 118000, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010848, Sample Num: 173568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00254696, Log Avg loss: 0.00000001, Global Avg Loss: 0.01780981, Time: 0.0865 Steps: 118200, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011048, Sample Num: 176768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00250085, Log Avg loss: 0.00000000, Global Avg Loss: 0.01777972, Time: 0.0808 Steps: 118400, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011248, Sample Num: 179968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00245638, Log Avg loss: 0.00000003, Global Avg Loss: 0.01774974, Time: 0.2209 Steps: 118600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011448, Sample Num: 183168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00241347, Log Avg loss: 0.00000000, Global Avg Loss: 0.01771986, Time: 0.1800 Steps: 118800, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 011648, Sample Num: 186368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00237203, Log Avg loss: 0.00000000, Global Avg Loss: 0.01769008, Time: 0.0945 Steps: 119000, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 011848, Sample Num: 189568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00233199, Log Avg loss: 0.00000003, Global Avg Loss: 0.01766040, Time: 0.2248 Steps: 119200, Updated lr: 0.000000 ***** Running evaluation checkpoint-119280 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-119280 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1865.804111, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.084038, "eval_total_loss": 117.065265, "eval_acc": 0.999551, "eval_prec": 0.98893, "eval_recall": 0.992593, "eval_f1": 0.990758, "eval_roc_auc": 0.999993, "eval_pr_auc": 0.99974, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 4, "tp": 536}, "eval_mcc2": 0.99053, "eval_mcc": 0.99053, "eval_sn": 0.992593, "eval_sp": 0.999724, "update_flag": false, "test_avg_loss": 0.024306, "test_total_loss": 33.858145, "test_acc": 0.999776, "test_prec": 0.996289, "test_recall": 0.994444, "test_f1": 0.995366, "test_roc_auc": 0.999999, "test_pr_auc": 0.999959, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 3, "tp": 537}, "test_mcc2": 0.995252, "test_mcc": 0.995252, "test_sn": 0.994444, "test_sp": 0.999908, "lr": 0.0, "cur_epoch_step": 11928, "train_global_avg_loss": 0.017648552168122715, "train_cur_epoch_loss": 27.629396817365944, "train_cur_epoch_avg_loss": 0.00231634782171076, "train_cur_epoch_time": 1865.8041107654572, "train_cur_epoch_avg_time": 0.15642220915203364, "epoch": 10, "step": 119280} ################################################## #########################Best Metric######################### {"epoch": 2, "global_step": 23856, "eval_avg_loss": 0.04232, "eval_total_loss": 58.951537, "eval_acc": 0.999686, "eval_prec": 0.994434, "eval_recall": 0.992593, "eval_f1": 0.993513, "eval_roc_auc": 0.999994, "eval_pr_auc": 0.999762, "eval_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 4, "tp": 536}, "eval_mcc2": 0.993352, "eval_mcc": 0.993352, "eval_sn": 0.992593, "eval_sp": 0.999862, "update_flag": true, "test_avg_loss": 0.033912, "test_total_loss": 47.238892, "test_acc": 0.999731, "test_prec": 0.998134, "test_recall": 0.990741, "test_f1": 0.994424, "test_roc_auc": 0.999998, "test_pr_auc": 0.999915, "test_confusion_matrix": {"tn": 21743, "fp": 1, "fn": 5, "tp": 535}, "test_mcc2": 0.994293, "test_mcc": 0.994293, "test_sn": 0.990741, "test_sp": 0.999954} ################################################## Total Time: 140089.625974, Avg time per epoch(10 epochs): 14008.960000 ++++++++++++Validation+++++++++++++ best f1 global step: 23856 checkpoint path: ../models/RdRP/protein/binary_class/luca_base/matrix/20250331145853/checkpoint-23856 ***** Running evaluation checkpoint-23856 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## {"evaluation_avg_loss_23856": 0.04232, "evaluation_total_loss_23856": 58.951537, "evaluation_acc_23856": 0.999686, "evaluation_prec_23856": 0.994434, "evaluation_recall_23856": 0.992593, "evaluation_f1_23856": 0.993513, "evaluation_roc_auc_23856": 0.999994, "evaluation_pr_auc_23856": 0.999762, "evaluation_confusion_matrix_23856": {"tn": 21741, "fp": 3, "fn": 4, "tp": 536}, "evaluation_mcc2_23856": 0.993352, "evaluation_mcc_23856": 0.993352, "evaluation_sn_23856": 0.992593, "evaluation_sp_23856": 0.999862} ++++++++++++Testing+++++++++++++ best f1 global step: 23856 checkpoint path: ../models/RdRP/protein/binary_class/luca_base/matrix/20250331145853/checkpoint-23856 ***** Running testing checkpoint-23856 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## {"evaluation_avg_loss_23856": 0.033912, "evaluation_total_loss_23856": 47.238892, "evaluation_acc_23856": 0.999731, "evaluation_prec_23856": 0.998134, "evaluation_recall_23856": 0.990741, "evaluation_f1_23856": 0.994424, "evaluation_roc_auc_23856": 0.999998, "evaluation_pr_auc_23856": 0.999915, "evaluation_confusion_matrix_23856": {"tn": 21743, "fp": 1, "fn": 5, "tp": 535}, "evaluation_mcc2_23856": 0.994293, "evaluation_mcc_23856": 0.994293, "evaluation_sn_23856": 0.990741, "evaluation_sp_23856": 0.999954}