{"adam_epsilon": 1e-08, "alphabet": "gene_prot", "append_eos": true, "asl_gamma_neg": 4.0, "asl_gamma_pos": 1.0, "best_metric_type": "f1", "beta1": 0.9, "beta2": 0.98, "buffer_size": 1024, "cache_dir": null, "classifier_activate_func": "gelu", "classifier_size": 128, "codes_file": null, "config_path": "../config/luca_base/luca_base_config.json", "cross_atten": false, "dataset_name": "RdRP", "dataset_type": "protein", "delete_old": false, "dev_data_dir": "../dataset/RdRP/protein/binary_class/dev/", "device": "cuda", "do_eval": true, "do_lower_case": false, "do_metrics": true, "do_predict": true, "do_train": true, "dropout_prob": 0.1, "early_stop_epoch": -1, "emb_activate_func": "gelu", "embedding_complete": true, "embedding_complete_seg_overlap": true, "embedding_fixed_len_a_time": 3072, "embedding_input_size": 2560, "embedding_input_size_a": null, "embedding_input_size_b": null, "eval_all_checkpoints": false, "evaluate_during_training": true, "evaluate_steps": -1, "evaluate_strategy": "epoch", "fc_activate_func": "gelu", "focal_loss_alpha": 0.7, "focal_loss_gamma": 2.0, "focal_loss_reduce": false, "fp16": false, "fp16_embedding": false, "fp16_opt_level": "O1", "fusion_type": "concat", "gradient_accumulation_steps": 1, "hidden_size": 1024, "ignore_index": -100, "input_mode": "single", "input_type": "matrix", "intermediate_size": 4096, "label_filepath": "../dataset/RdRP/protein/binary_class/label.txt", "label_size": 2, "label_type": "RdRP", "learning_rate": 0.0001, "llm_dir": "..", "llm_dirpath": "../llm/models/lucaone_virus/v1.0/token_level,span_level,seq_level/lucaone_virus/20240815023346/checkpoint-step3800000", "llm_step": "3800000", "llm_task_level": "token_level,span_level,seq_level", "llm_time_str": "20240815023346", "llm_type": "lucaone_virus", "llm_version": "v1.0", "lmdb_path": null, "local_rank": -1, "log_dir": "../logs/RdRP/protein/binary_class/luca_base/matrix/20250329135221", "logging_steps": 200, "loss_reduction": "mean", "loss_type": "bce", "lr_decay_rate": 0.9, "lr_update_strategy": "step", "matrix_add_special_token": true, "matrix_dirpath": "../matrices/RdRP/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000", "matrix_embedding_exists": false, "matrix_encoder": false, "matrix_encoder_act": false, "matrix_fc_size": "128", "matrix_max_length": 4096, "matrix_max_length_a": null, "matrix_max_length_b": null, "matrix_pooling_type": "value_attention", "max_grad_norm": 1.0, "max_sentence_length": null, "max_sentences": null, "max_steps": -1, "model_dirpath": null, "model_type": "luca_base", "n_gpu": 1, "no_cuda": false, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "non_ignore": false, "not_append_eos": false, "not_matrix_encoder_shared": false, "not_prepend_bos": false, "not_save_emb_to_disk": false, "not_seq_encoder_shared": false, "num_attention_heads": 0, "num_hidden_layers": 0, "num_train_epochs": 10, "output_dir": "../models/RdRP/protein/binary_class/luca_base/matrix/20250329135221", "output_mode": "binary_class", "overwrite_cache": false, "overwrite_output_dir": true, "per_gpu_eval_batch_size": 16, "per_gpu_train_batch_size": 16, "pos_weight": 40.0, "position_embedding_type": "absolute", "prepend_bos": true, "save_all": true, "save_steps": -1, "seed": 1221, "self_atten": false, "seq_fc_size": "null", "seq_max_length": 4096, "seq_max_length_a": null, "seq_max_length_b": null, "seq_pooling_type": "value_attention", "seq_subword": false, "seq_vocab_path": "gene_prot", "sigmoid": true, "task_level_type": "seq_level", "task_type": "binary_class", "tb_log_dir": "../tb-logs/RdRP/protein/binary_class/luca_base/matrix/20250329135221", "test_data_dir": "../dataset/RdRP/protein/binary_class/test/", "time_str": "20250329135226", "train_data_dir": "../dataset/RdRP/protein/binary_class/train/", "trunc_type": "right", "vector_dirpath": "../vectors/RdRP/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000", "vector_fc_size": "null", "vocab_size": 39, "warmup_steps": 200, "weight": null, "weight_decay": 0.01, "worker_num": 0} ################################################## n_gpu: 1 ################################################## Inputs: Input Name List: protein,embedding_matrix ################################################## Encoder Config: {'llm_type': 'lucaone_virus', 'llm_version': 'v1.0', 'llm_step': '3800000', 'llm_dirpath': '../llm/models/lucaone_virus/v1.0/token_level,span_level,seq_level/lucaone_virus/20240815023346/checkpoint-step3800000', 'input_type': 'matrix', 'trunc_type': 'right', 'seq_max_length': 4096, 'atom_seq_max_length': None, 'vector_dirpath': '../vectors/RdRP/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000', 'matrix_dirpath': '../matrices/RdRP/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000', 'local_rank': -1, 'max_sentence_length': None, 'max_sentences': None, 'matrix_add_special_token': True, 'embedding_complete': True, 'embedding_complete_seg_overlap': True, 'embedding_fixed_len_a_time': 3072, 'matrix_embedding_exists': False, 'save_emb_to_disk': True, 'fp16_embedding': False} ################################################## Model Config: LucaConfig { "alphabet": "gene_prot", "attention_probs_dropout_prob": 0.1, "classifier_activate_func": "gelu", "classifier_dropout_prob": 0.1, "classifier_size": 128, "cls_token_id": 2, "cross_atten": false, "directionality": "bidi", "emb_activate_func": "gelu", "embedding_input_size": 2560, "fc_activate_func": "gelu", "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 1024, "ignore_index": -100, "initializer_range": 0.02, "intermediate_size": 4096, "kernel_size": 7, "layer_norm_eps": 1e-12, "loss_reduction": "mean", "matrix_fc_size": [ 128 ], "matrix_max_length": 4096, "matrix_pooling_type": "value_attention", "max_position_embeddings": 4098, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "num_attention_heads": 8, "num_hidden_layers": 4, "pad_token_id": 0, "pos_weight": 40.0, "position_embedding_type": "absolute", "self_atten": false, "sep_token_id": 3, "seq_fc_size": null, "seq_max_length": 4096, "seq_pooling_type": "value_attention", "token_dropout": null, "transformers_version": "4.29.0", "type_vocab_size": 2, "use_luca_layer_norm_v2": true, "vector_fc_size": null, "vocab_size": 39 } ################################################## Mode Architecture: LucaBase( (matrix_pooler): GlobalMaskValueAttentionPooling1D (2560 -> 2560) (linear): ModuleList( (0): ModuleList( (0): Linear(in_features=2560, out_features=128, bias=True) (1): GELU(approximate='none') ) ) (dropout): Dropout(p=0.1, inplace=False) (hidden_layer): Linear(in_features=128, out_features=128, bias=True) (hidden_act): GELU(approximate='none') (classifier): Linear(in_features=128, out_features=1, bias=True) (output): Sigmoid() (loss_fct): MaskedBCEWithLogitsLoss( (criterion): BCEWithLogitsLoss() ) ) ################################################## Model parameters: 20005249 ################################################## {"total_num": "19.080000M", "total_size": "76.310000MB", "param_sum": "19.080000M", "param_size": "76.310000MB", "buffer_sum": "0.000000M", "buffer_size": "0.000000MB", "trainable_num": "19.078492M", "trainable_size": "76.313969MB"} ################################################## Train dataset len: 190846, batch size: 16, batch num: 11928 Train dataset t_total: 119280, max_steps: -1 ***** Running training ***** Train Dataset Num examples = 190846 Train Dataset Num Epochs = 10 Logging Steps = 200 Saving Steps = -1 Evaluating Strategy = epoch Train Dataset Instantaneous batch size per GPU = 16 Train Dataset Total train batch size (w. parallel, distributed & accumulation) = 16 Train Dataset Gradient Accumulation steps = 1 Train Dataset Total optimization steps = 119280 ################################################## Training, Epoch: 0001, Batch: 000200, Sample Num: 3200, Cur Loss: 0.61183280, Cur Avg Loss: 1.36863929, Log Avg loss: 1.36863929, Global Avg Loss: 1.36863929, Time: 0.0819 Steps: 200, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000400, Sample Num: 6400, Cur Loss: 0.00578785, Cur Avg Loss: 0.95366812, Log Avg loss: 0.53869696, Global Avg Loss: 0.95366812, Time: 0.1054 Steps: 400, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000600, Sample Num: 9600, Cur Loss: 0.03005599, Cur Avg Loss: 0.66835461, Log Avg loss: 0.09772757, Global Avg Loss: 0.66835461, Time: 0.2294 Steps: 600, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000800, Sample Num: 12800, Cur Loss: 0.00321190, Cur Avg Loss: 0.52574687, Log Avg loss: 0.09792366, Global Avg Loss: 0.52574687, Time: 0.2271 Steps: 800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001000, Sample Num: 16000, Cur Loss: 0.00326839, Cur Avg Loss: 0.48591509, Log Avg loss: 0.32658797, Global Avg Loss: 0.48591509, Time: 0.1538 Steps: 1000, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001200, Sample Num: 19200, Cur Loss: 0.00014234, Cur Avg Loss: 0.45627440, Log Avg loss: 0.30807093, Global Avg Loss: 0.45627440, Time: 0.1054 Steps: 1200, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001400, Sample Num: 22400, Cur Loss: 0.00033352, Cur Avg Loss: 0.41538424, Log Avg loss: 0.17004333, Global Avg Loss: 0.41538424, Time: 0.0706 Steps: 1400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001600, Sample Num: 25600, Cur Loss: 0.00013784, Cur Avg Loss: 0.36585112, Log Avg loss: 0.01911924, Global Avg Loss: 0.36585112, Time: 0.1278 Steps: 1600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001800, Sample Num: 28800, Cur Loss: 0.00011757, Cur Avg Loss: 0.33369223, Log Avg loss: 0.07642115, Global Avg Loss: 0.33369223, Time: 0.0816 Steps: 1800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002000, Sample Num: 32000, Cur Loss: 1.39376032, Cur Avg Loss: 0.30151043, Log Avg loss: 0.01187418, Global Avg Loss: 0.30151043, Time: 0.1142 Steps: 2000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002200, Sample Num: 35200, Cur Loss: 0.00010037, Cur Avg Loss: 0.28814974, Log Avg loss: 0.15454287, Global Avg Loss: 0.28814974, Time: 0.1329 Steps: 2200, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002400, Sample Num: 38400, Cur Loss: 0.00000334, Cur Avg Loss: 0.26438796, Log Avg loss: 0.00300839, Global Avg Loss: 0.26438796, Time: 0.0538 Steps: 2400, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002600, Sample Num: 41600, Cur Loss: 0.00001448, Cur Avg Loss: 0.25326191, Log Avg loss: 0.11974924, Global Avg Loss: 0.25326191, Time: 0.1264 Steps: 2600, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002800, Sample Num: 44800, Cur Loss: 0.00000614, Cur Avg Loss: 0.24356425, Log Avg loss: 0.11749468, Global Avg Loss: 0.24356425, Time: 0.1257 Steps: 2800, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003000, Sample Num: 48000, Cur Loss: 0.00001395, Cur Avg Loss: 0.22835394, Log Avg loss: 0.01540968, Global Avg Loss: 0.22835394, Time: 0.1247 Steps: 3000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003200, Sample Num: 51200, Cur Loss: 0.00006703, Cur Avg Loss: 0.22111099, Log Avg loss: 0.11246665, Global Avg Loss: 0.22111099, Time: 0.3872 Steps: 3200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003400, Sample Num: 54400, Cur Loss: 0.00000769, Cur Avg Loss: 0.20995997, Log Avg loss: 0.03154368, Global Avg Loss: 0.20995997, Time: 0.1727 Steps: 3400, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003600, Sample Num: 57600, Cur Loss: 0.00001013, Cur Avg Loss: 0.21045121, Log Avg loss: 0.21880233, Global Avg Loss: 0.21045121, Time: 0.1232 Steps: 3600, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003800, Sample Num: 60800, Cur Loss: 0.00194994, Cur Avg Loss: 0.20078741, Log Avg loss: 0.02683906, Global Avg Loss: 0.20078741, Time: 0.0834 Steps: 3800, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004000, Sample Num: 64000, Cur Loss: 0.00001341, Cur Avg Loss: 0.19293611, Log Avg loss: 0.04376136, Global Avg Loss: 0.19293611, Time: 0.0795 Steps: 4000, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004200, Sample Num: 67200, Cur Loss: 0.00000507, Cur Avg Loss: 0.18437226, Log Avg loss: 0.01309519, Global Avg Loss: 0.18437226, Time: 0.1118 Steps: 4200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004400, Sample Num: 70400, Cur Loss: 0.00006717, Cur Avg Loss: 0.17721387, Log Avg loss: 0.02688777, Global Avg Loss: 0.17721387, Time: 0.2216 Steps: 4400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 004600, Sample Num: 73600, Cur Loss: 0.00000167, Cur Avg Loss: 0.16951139, Log Avg loss: 0.00005677, Global Avg Loss: 0.16951139, Time: 0.1284 Steps: 4600, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 004800, Sample Num: 76800, Cur Loss: 0.00000304, Cur Avg Loss: 0.16252260, Log Avg loss: 0.00178048, Global Avg Loss: 0.16252260, Time: 0.0877 Steps: 4800, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005000, Sample Num: 80000, Cur Loss: 0.00007397, Cur Avg Loss: 0.16143175, Log Avg loss: 0.13525137, Global Avg Loss: 0.16143175, Time: 0.2122 Steps: 5000, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005200, Sample Num: 83200, Cur Loss: 0.00000703, Cur Avg Loss: 0.15569948, Log Avg loss: 0.01239266, Global Avg Loss: 0.15569948, Time: 0.1393 Steps: 5200, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005400, Sample Num: 86400, Cur Loss: 0.00000840, Cur Avg Loss: 0.15729139, Log Avg loss: 0.19868102, Global Avg Loss: 0.15729139, Time: 0.0993 Steps: 5400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005600, Sample Num: 89600, Cur Loss: 0.00000048, Cur Avg Loss: 0.15196752, Log Avg loss: 0.00822302, Global Avg Loss: 0.15196752, Time: 0.1235 Steps: 5600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 005800, Sample Num: 92800, Cur Loss: 0.00000042, Cur Avg Loss: 0.14690408, Log Avg loss: 0.00512795, Global Avg Loss: 0.14690408, Time: 0.1911 Steps: 5800, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006000, Sample Num: 96000, Cur Loss: 0.00000072, Cur Avg Loss: 0.14500248, Log Avg loss: 0.08985582, Global Avg Loss: 0.14500248, Time: 0.1959 Steps: 6000, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006200, Sample Num: 99200, Cur Loss: 0.00000119, Cur Avg Loss: 0.14567911, Log Avg loss: 0.16597817, Global Avg Loss: 0.14567911, Time: 0.0908 Steps: 6200, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006400, Sample Num: 102400, Cur Loss: 0.00000042, Cur Avg Loss: 0.14125251, Log Avg loss: 0.00402795, Global Avg Loss: 0.14125251, Time: 0.1012 Steps: 6400, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006600, Sample Num: 105600, Cur Loss: 0.00000030, Cur Avg Loss: 0.13782305, Log Avg loss: 0.02808016, Global Avg Loss: 0.13782305, Time: 0.3689 Steps: 6600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006800, Sample Num: 108800, Cur Loss: 0.00000471, Cur Avg Loss: 0.13385385, Log Avg loss: 0.00287048, Global Avg Loss: 0.13385385, Time: 0.2069 Steps: 6800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007000, Sample Num: 112000, Cur Loss: 0.00000018, Cur Avg Loss: 0.13003005, Log Avg loss: 0.00002077, Global Avg Loss: 0.13003005, Time: 0.0991 Steps: 7000, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007200, Sample Num: 115200, Cur Loss: 0.00000423, Cur Avg Loss: 0.12699824, Log Avg loss: 0.02088498, Global Avg Loss: 0.12699824, Time: 0.1091 Steps: 7200, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007400, Sample Num: 118400, Cur Loss: 0.00000018, Cur Avg Loss: 0.12357014, Log Avg loss: 0.00015848, Global Avg Loss: 0.12357014, Time: 0.1060 Steps: 7400, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007600, Sample Num: 121600, Cur Loss: 0.00000125, Cur Avg Loss: 0.12041211, Log Avg loss: 0.00356506, Global Avg Loss: 0.12041211, Time: 0.1995 Steps: 7600, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007800, Sample Num: 124800, Cur Loss: 0.00000370, Cur Avg Loss: 0.12252377, Log Avg loss: 0.20276655, Global Avg Loss: 0.12252377, Time: 0.0822 Steps: 7800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 008000, Sample Num: 128000, Cur Loss: 0.00000376, Cur Avg Loss: 0.12105378, Log Avg loss: 0.06372421, Global Avg Loss: 0.12105378, Time: 0.2314 Steps: 8000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008200, Sample Num: 131200, Cur Loss: 0.00000089, Cur Avg Loss: 0.11838963, Log Avg loss: 0.01182360, Global Avg Loss: 0.11838963, Time: 0.2224 Steps: 8200, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008400, Sample Num: 134400, Cur Loss: 0.00008005, Cur Avg Loss: 0.11941384, Log Avg loss: 0.16140665, Global Avg Loss: 0.11941384, Time: 0.2225 Steps: 8400, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008600, Sample Num: 137600, Cur Loss: 0.00476873, Cur Avg Loss: 0.12046037, Log Avg loss: 0.16441451, Global Avg Loss: 0.12046037, Time: 0.3101 Steps: 8600, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008800, Sample Num: 140800, Cur Loss: 0.00000262, Cur Avg Loss: 0.12028712, Log Avg loss: 0.11283731, Global Avg Loss: 0.12028712, Time: 0.0828 Steps: 8800, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 009000, Sample Num: 144000, Cur Loss: 0.00000441, Cur Avg Loss: 0.11773462, Log Avg loss: 0.00542486, Global Avg Loss: 0.11773462, Time: 0.0958 Steps: 9000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 009200, Sample Num: 147200, Cur Loss: 0.00000733, Cur Avg Loss: 0.11761939, Log Avg loss: 0.11243401, Global Avg Loss: 0.11761939, Time: 0.1346 Steps: 9200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009400, Sample Num: 150400, Cur Loss: 0.00000411, Cur Avg Loss: 0.11677003, Log Avg loss: 0.07769923, Global Avg Loss: 0.11677003, Time: 0.1231 Steps: 9400, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009600, Sample Num: 153600, Cur Loss: 0.00000107, Cur Avg Loss: 0.11556924, Log Avg loss: 0.05913232, Global Avg Loss: 0.11556924, Time: 0.0899 Steps: 9600, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009800, Sample Num: 156800, Cur Loss: 0.00000495, Cur Avg Loss: 0.11684995, Log Avg loss: 0.17832400, Global Avg Loss: 0.11684995, Time: 0.2020 Steps: 9800, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010000, Sample Num: 160000, Cur Loss: 0.00000697, Cur Avg Loss: 0.11499012, Log Avg loss: 0.02385822, Global Avg Loss: 0.11499012, Time: 0.1050 Steps: 10000, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010200, Sample Num: 163200, Cur Loss: 0.00000554, Cur Avg Loss: 0.11276479, Log Avg loss: 0.00149857, Global Avg Loss: 0.11276479, Time: 0.1087 Steps: 10200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010400, Sample Num: 166400, Cur Loss: 0.00000376, Cur Avg Loss: 0.11181597, Log Avg loss: 0.06342601, Global Avg Loss: 0.11181597, Time: 0.2245 Steps: 10400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 010600, Sample Num: 169600, Cur Loss: 0.00000197, Cur Avg Loss: 0.10975132, Log Avg loss: 0.00238946, Global Avg Loss: 0.10975132, Time: 0.0701 Steps: 10600, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 010800, Sample Num: 172800, Cur Loss: 0.00000155, Cur Avg Loss: 0.11060468, Log Avg loss: 0.15583301, Global Avg Loss: 0.11060468, Time: 0.1709 Steps: 10800, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011000, Sample Num: 176000, Cur Loss: 0.00000978, Cur Avg Loss: 0.10883023, Log Avg loss: 0.01300966, Global Avg Loss: 0.10883023, Time: 0.0947 Steps: 11000, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011200, Sample Num: 179200, Cur Loss: 0.00000244, Cur Avg Loss: 0.10786992, Log Avg loss: 0.05505313, Global Avg Loss: 0.10786992, Time: 0.0987 Steps: 11200, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011400, Sample Num: 182400, Cur Loss: 0.00000077, Cur Avg Loss: 0.10606057, Log Avg loss: 0.00473680, Global Avg Loss: 0.10606057, Time: 0.1812 Steps: 11400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011600, Sample Num: 185600, Cur Loss: 0.00000072, Cur Avg Loss: 0.10428637, Log Avg loss: 0.00315703, Global Avg Loss: 0.10428637, Time: 0.1076 Steps: 11600, Updated lr: 0.000090 Training, Epoch: 0001, Batch: 011800, Sample Num: 188800, Cur Loss: 0.00006735, Cur Avg Loss: 0.10414723, Log Avg loss: 0.09607732, Global Avg Loss: 0.10414723, Time: 0.3102 Steps: 11800, Updated lr: 0.000090 ***** Running evaluation checkpoint-11928 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-11928 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1874.314038, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.036683, "eval_total_loss": 51.099171, "eval_acc": 0.999596, "eval_prec": 0.992579, "eval_recall": 0.990741, "eval_f1": 0.991659, "eval_roc_auc": 0.999886, "eval_pr_auc": 0.998551, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 5, "tp": 535}, "eval_mcc2": 0.991452, "eval_mcc": 0.991452, "eval_sn": 0.990741, "eval_sp": 0.999816, "update_flag": true, "test_avg_loss": 0.037712, "test_total_loss": 52.533318, "test_acc": 0.999686, "test_prec": 0.996276, "test_recall": 0.990741, "test_f1": 0.9935, "test_roc_auc": 0.999997, "test_pr_auc": 0.999881, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 5, "tp": 535}, "test_mcc2": 0.993344, "test_mcc": 0.993344, "test_sn": 0.990741, "test_sp": 0.999908, "lr": 9.015115888478334e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.10310147945626465, "train_cur_epoch_loss": 1229.7944469543247, "train_cur_epoch_avg_loss": 0.10310147945626465, "train_cur_epoch_time": 1874.3140377998352, "train_cur_epoch_avg_time": 0.1571356503856334, "epoch": 1, "step": 11928} ################################################## Training, Epoch: 0002, Batch: 000072, Sample Num: 1152, Cur Loss: 0.00645245, Cur Avg Loss: 0.00013996, Log Avg loss: 0.00433581, Global Avg Loss: 0.10248371, Time: 0.0866 Steps: 12000, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000272, Sample Num: 4352, Cur Loss: 0.00000244, Cur Avg Loss: 0.00017802, Log Avg loss: 0.00019172, Global Avg Loss: 0.10080679, Time: 0.2108 Steps: 12200, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000472, Sample Num: 7552, Cur Loss: 0.00000077, Cur Avg Loss: 0.03717585, Log Avg loss: 0.08749291, Global Avg Loss: 0.10059205, Time: 0.1776 Steps: 12400, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000672, Sample Num: 10752, Cur Loss: 0.00005540, Cur Avg Loss: 0.02611595, Log Avg loss: 0.00001459, Global Avg Loss: 0.09899558, Time: 0.2141 Steps: 12600, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000872, Sample Num: 13952, Cur Loss: 0.00000006, Cur Avg Loss: 0.02034546, Log Avg loss: 0.00095661, Global Avg Loss: 0.09746373, Time: 0.1269 Steps: 12800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001072, Sample Num: 17152, Cur Loss: 15.06140232, Cur Avg Loss: 0.06840562, Log Avg loss: 0.27794792, Global Avg Loss: 0.10024041, Time: 0.1095 Steps: 13000, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001272, Sample Num: 20352, Cur Loss: 0.00000006, Cur Avg Loss: 0.06644457, Log Avg loss: 0.05593332, Global Avg Loss: 0.09956909, Time: 0.1842 Steps: 13200, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001472, Sample Num: 23552, Cur Loss: 0.00000125, Cur Avg Loss: 0.05846118, Log Avg loss: 0.00768687, Global Avg Loss: 0.09819771, Time: 0.1209 Steps: 13400, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001672, Sample Num: 26752, Cur Loss: 0.00001764, Cur Avg Loss: 0.05608592, Log Avg loss: 0.03860395, Global Avg Loss: 0.09732133, Time: 0.1247 Steps: 13600, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001872, Sample Num: 29952, Cur Loss: 0.00000030, Cur Avg Loss: 0.05024681, Log Avg loss: 0.00143188, Global Avg Loss: 0.09593163, Time: 0.1532 Steps: 13800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002072, Sample Num: 33152, Cur Loss: 0.00135153, Cur Avg Loss: 0.04729592, Log Avg loss: 0.01967554, Global Avg Loss: 0.09484226, Time: 0.0844 Steps: 14000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002272, Sample Num: 36352, Cur Loss: 0.00000244, Cur Avg Loss: 0.04627054, Log Avg loss: 0.03564765, Global Avg Loss: 0.09400853, Time: 0.2101 Steps: 14200, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002472, Sample Num: 39552, Cur Loss: 0.00000817, Cur Avg Loss: 0.04569057, Log Avg loss: 0.03910215, Global Avg Loss: 0.09324594, Time: 0.1163 Steps: 14400, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002672, Sample Num: 42752, Cur Loss: 0.00003362, Cur Avg Loss: 0.05079303, Log Avg loss: 0.11385945, Global Avg Loss: 0.09352832, Time: 0.0870 Steps: 14600, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002872, Sample Num: 45952, Cur Loss: 0.00000143, Cur Avg Loss: 0.04727867, Log Avg loss: 0.00032678, Global Avg Loss: 0.09226884, Time: 0.1034 Steps: 14800, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003072, Sample Num: 49152, Cur Loss: 0.00000036, Cur Avg Loss: 0.04431972, Log Avg loss: 0.00182913, Global Avg Loss: 0.09106297, Time: 0.0932 Steps: 15000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003272, Sample Num: 52352, Cur Loss: 0.00000060, Cur Avg Loss: 0.04162407, Log Avg loss: 0.00021890, Global Avg Loss: 0.08986766, Time: 0.1670 Steps: 15200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003472, Sample Num: 55552, Cur Loss: 0.00000054, Cur Avg Loss: 0.03939024, Log Avg loss: 0.00284475, Global Avg Loss: 0.08873749, Time: 0.0812 Steps: 15400, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003672, Sample Num: 58752, Cur Loss: 0.00000006, Cur Avg Loss: 0.05192582, Log Avg loss: 0.26954352, Global Avg Loss: 0.09105552, Time: 0.1302 Steps: 15600, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003872, Sample Num: 61952, Cur Loss: 0.00000060, Cur Avg Loss: 0.04960243, Log Avg loss: 0.00694504, Global Avg Loss: 0.08999083, Time: 0.2333 Steps: 15800, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004072, Sample Num: 65152, Cur Loss: 0.00000030, Cur Avg Loss: 0.04717104, Log Avg loss: 0.00009937, Global Avg Loss: 0.08886718, Time: 0.1048 Steps: 16000, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004272, Sample Num: 68352, Cur Loss: 0.00000525, Cur Avg Loss: 0.04498111, Log Avg loss: 0.00039398, Global Avg Loss: 0.08777492, Time: 0.2781 Steps: 16200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004472, Sample Num: 71552, Cur Loss: 0.00000006, Cur Avg Loss: 0.04297055, Log Avg loss: 0.00002519, Global Avg Loss: 0.08670480, Time: 0.2277 Steps: 16400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 004672, Sample Num: 74752, Cur Loss: 0.00000024, Cur Avg Loss: 0.04113120, Log Avg loss: 0.00000320, Global Avg Loss: 0.08566021, Time: 0.0807 Steps: 16600, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 004872, Sample Num: 77952, Cur Loss: 0.00000006, Cur Avg Loss: 0.03950981, Log Avg loss: 0.00163412, Global Avg Loss: 0.08465989, Time: 0.1180 Steps: 16800, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005072, Sample Num: 81152, Cur Loss: 0.00000000, Cur Avg Loss: 0.04027235, Log Avg loss: 0.05884800, Global Avg Loss: 0.08435623, Time: 0.1092 Steps: 17000, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005272, Sample Num: 84352, Cur Loss: 0.00000012, Cur Avg Loss: 0.03996969, Log Avg loss: 0.03229420, Global Avg Loss: 0.08375085, Time: 0.1317 Steps: 17200, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005472, Sample Num: 87552, Cur Loss: 0.00000036, Cur Avg Loss: 0.03851557, Log Avg loss: 0.00018490, Global Avg Loss: 0.08279032, Time: 0.1474 Steps: 17400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005672, Sample Num: 90752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03715790, Log Avg loss: 0.00001213, Global Avg Loss: 0.08184966, Time: 0.1036 Steps: 17600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 005872, Sample Num: 93952, Cur Loss: 0.00000048, Cur Avg Loss: 0.03602427, Log Avg loss: 0.00387454, Global Avg Loss: 0.08097354, Time: 0.4082 Steps: 17800, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006072, Sample Num: 97152, Cur Loss: 0.00000346, Cur Avg Loss: 0.03483905, Log Avg loss: 0.00004100, Global Avg Loss: 0.08007429, Time: 0.4083 Steps: 18000, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006272, Sample Num: 100352, Cur Loss: 0.00000060, Cur Avg Loss: 0.03956028, Log Avg loss: 0.18289660, Global Avg Loss: 0.08120420, Time: 0.2057 Steps: 18200, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006472, Sample Num: 103552, Cur Loss: 0.00000000, Cur Avg Loss: 0.03840135, Log Avg loss: 0.00205729, Global Avg Loss: 0.08034391, Time: 0.2084 Steps: 18400, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006672, Sample Num: 106752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03727345, Log Avg loss: 0.00077468, Global Avg Loss: 0.07948833, Time: 0.1119 Steps: 18600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006872, Sample Num: 109952, Cur Loss: 0.00000006, Cur Avg Loss: 0.03625742, Log Avg loss: 0.00236273, Global Avg Loss: 0.07866784, Time: 0.1254 Steps: 18800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007072, Sample Num: 113152, Cur Loss: 0.00000006, Cur Avg Loss: 0.03551809, Log Avg loss: 0.01011482, Global Avg Loss: 0.07794623, Time: 0.1773 Steps: 19000, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007272, Sample Num: 116352, Cur Loss: 0.00000000, Cur Avg Loss: 0.03454138, Log Avg loss: 0.00000486, Global Avg Loss: 0.07713434, Time: 0.1293 Steps: 19200, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007472, Sample Num: 119552, Cur Loss: 0.00000006, Cur Avg Loss: 0.03361703, Log Avg loss: 0.00000772, Global Avg Loss: 0.07633922, Time: 0.0529 Steps: 19400, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007672, Sample Num: 122752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03284321, Log Avg loss: 0.00393325, Global Avg Loss: 0.07560039, Time: 0.2337 Steps: 19600, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007872, Sample Num: 125952, Cur Loss: 0.00000107, Cur Avg Loss: 0.03672153, Log Avg loss: 0.18549383, Global Avg Loss: 0.07671042, Time: 0.1404 Steps: 19800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 008072, Sample Num: 129152, Cur Loss: 0.00000000, Cur Avg Loss: 0.03591463, Log Avg loss: 0.00415520, Global Avg Loss: 0.07598487, Time: 0.0560 Steps: 20000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008272, Sample Num: 132352, Cur Loss: 0.00000000, Cur Avg Loss: 0.03749927, Log Avg loss: 0.10145506, Global Avg Loss: 0.07623705, Time: 0.0869 Steps: 20200, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008472, Sample Num: 135552, Cur Loss: 0.00001228, Cur Avg Loss: 0.04057071, Log Avg loss: 0.16760573, Global Avg Loss: 0.07713282, Time: 0.1827 Steps: 20400, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008672, Sample Num: 138752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03967538, Log Avg loss: 0.00174891, Global Avg Loss: 0.07640094, Time: 0.0931 Steps: 20600, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008872, Sample Num: 141952, Cur Loss: 0.00000018, Cur Avg Loss: 0.03881870, Log Avg loss: 0.00167337, Global Avg Loss: 0.07568240, Time: 0.0937 Steps: 20800, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 009072, Sample Num: 145152, Cur Loss: 0.00000006, Cur Avg Loss: 0.03796319, Log Avg loss: 0.00001260, Global Avg Loss: 0.07496174, Time: 0.1116 Steps: 21000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 009272, Sample Num: 148352, Cur Loss: 0.00000012, Cur Avg Loss: 0.03716421, Log Avg loss: 0.00092261, Global Avg Loss: 0.07426326, Time: 0.1272 Steps: 21200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009472, Sample Num: 151552, Cur Loss: 0.00000018, Cur Avg Loss: 0.03682619, Log Avg loss: 0.02115555, Global Avg Loss: 0.07376692, Time: 0.2579 Steps: 21400, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009672, Sample Num: 154752, Cur Loss: 0.00000018, Cur Avg Loss: 0.03616616, Log Avg loss: 0.00490701, Global Avg Loss: 0.07312933, Time: 0.1256 Steps: 21600, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009872, Sample Num: 157952, Cur Loss: 0.00000000, Cur Avg Loss: 0.03639549, Log Avg loss: 0.04748584, Global Avg Loss: 0.07289407, Time: 0.2285 Steps: 21800, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010072, Sample Num: 161152, Cur Loss: 0.00000036, Cur Avg Loss: 0.03567507, Log Avg loss: 0.00011525, Global Avg Loss: 0.07223244, Time: 0.1369 Steps: 22000, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010272, Sample Num: 164352, Cur Loss: 0.00000149, Cur Avg Loss: 0.03498055, Log Avg loss: 0.00000427, Global Avg Loss: 0.07158174, Time: 0.4015 Steps: 22200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010472, Sample Num: 167552, Cur Loss: 0.00000185, Cur Avg Loss: 0.03431262, Log Avg loss: 0.00000802, Global Avg Loss: 0.07094269, Time: 0.0989 Steps: 22400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 010672, Sample Num: 170752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03391971, Log Avg loss: 0.01334710, Global Avg Loss: 0.07043299, Time: 0.2069 Steps: 22600, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 010872, Sample Num: 173952, Cur Loss: 0.00000000, Cur Avg Loss: 0.03331663, Log Avg loss: 0.00113596, Global Avg Loss: 0.06982512, Time: 0.1084 Steps: 22800, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011072, Sample Num: 177152, Cur Loss: 0.00000006, Cur Avg Loss: 0.03272198, Log Avg loss: 0.00039718, Global Avg Loss: 0.06922140, Time: 0.1830 Steps: 23000, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011272, Sample Num: 180352, Cur Loss: 0.00000000, Cur Avg Loss: 0.03216887, Log Avg loss: 0.00154859, Global Avg Loss: 0.06863802, Time: 0.2185 Steps: 23200, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011472, Sample Num: 183552, Cur Loss: 0.00000006, Cur Avg Loss: 0.03160806, Log Avg loss: 0.00000080, Global Avg Loss: 0.06805137, Time: 0.2311 Steps: 23400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011672, Sample Num: 186752, Cur Loss: 0.00000000, Cur Avg Loss: 0.03106647, Log Avg loss: 0.00000059, Global Avg Loss: 0.06747467, Time: 0.1009 Steps: 23600, Updated lr: 0.000080 Training, Epoch: 0002, Batch: 011872, Sample Num: 189952, Cur Loss: 0.00000000, Cur Avg Loss: 0.03135501, Log Avg loss: 0.04819458, Global Avg Loss: 0.06731265, Time: 0.2757 Steps: 23800, Updated lr: 0.000080 ***** Running evaluation checkpoint-23856 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-23856 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1860.840643, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.060708, "eval_total_loss": 84.566099, "eval_acc": 0.999731, "eval_prec": 0.996283, "eval_recall": 0.992593, "eval_f1": 0.994434, "eval_roc_auc": 0.999981, "eval_pr_auc": 0.999406, "eval_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 4, "tp": 536}, "eval_mcc2": 0.994298, "eval_mcc": 0.994298, "eval_sn": 0.992593, "eval_sp": 0.999908, "update_flag": true, "test_avg_loss": 0.059266, "test_total_loss": 82.558123, "test_acc": 0.999731, "test_prec": 1.0, "test_recall": 0.988889, "test_f1": 0.994413, "test_roc_auc": 0.999995, "test_pr_auc": 0.99982, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 6, "tp": 534}, "test_mcc2": 0.994292, "test_mcc": 0.994292, "test_sn": 0.988889, "test_sp": 1.0, "lr": 8.013436345314075e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.06715465431274786, "train_cur_epoch_loss": 372.24698633057994, "train_cur_epoch_avg_loss": 0.03120782916923038, "train_cur_epoch_time": 1860.8406434059143, "train_cur_epoch_avg_time": 0.15600609015810818, "epoch": 2, "step": 23856} ################################################## Training, Epoch: 0003, Batch: 000144, Sample Num: 2304, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001062, Log Avg loss: 0.00000894, Global Avg Loss: 0.06675179, Time: 0.0769 Steps: 24000, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000344, Sample Num: 5504, Cur Loss: 0.00038603, Cur Avg Loss: 0.02821123, Log Avg loss: 0.04851567, Global Avg Loss: 0.06660108, Time: 0.2378 Steps: 24200, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000544, Sample Num: 8704, Cur Loss: 0.00000000, Cur Avg Loss: 0.01784784, Log Avg loss: 0.00002282, Global Avg Loss: 0.06605535, Time: 0.1030 Steps: 24400, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000744, Sample Num: 11904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01305127, Log Avg loss: 0.00000458, Global Avg Loss: 0.06551836, Time: 0.0957 Steps: 24600, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000944, Sample Num: 15104, Cur Loss: 0.00000000, Cur Avg Loss: 0.04863363, Log Avg loss: 0.18100000, Global Avg Loss: 0.06644966, Time: 0.1479 Steps: 24800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001144, Sample Num: 18304, Cur Loss: 0.00000006, Cur Avg Loss: 0.04595892, Log Avg loss: 0.03333430, Global Avg Loss: 0.06618474, Time: 0.1003 Steps: 25000, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001344, Sample Num: 21504, Cur Loss: 0.00000179, Cur Avg Loss: 0.03911996, Log Avg loss: 0.00000111, Global Avg Loss: 0.06565947, Time: 0.1873 Steps: 25200, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001544, Sample Num: 24704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03405351, Log Avg loss: 0.00000697, Global Avg Loss: 0.06514252, Time: 0.1058 Steps: 25400, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001744, Sample Num: 27904, Cur Loss: 0.00000626, Cur Avg Loss: 0.04107315, Log Avg loss: 0.09526477, Global Avg Loss: 0.06537785, Time: 0.1206 Steps: 25600, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001944, Sample Num: 31104, Cur Loss: 0.00000000, Cur Avg Loss: 0.03684761, Log Avg loss: 0.00000087, Global Avg Loss: 0.06487105, Time: 0.1083 Steps: 25800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002144, Sample Num: 34304, Cur Loss: 0.00003445, Cur Avg Loss: 0.04164162, Log Avg loss: 0.08823941, Global Avg Loss: 0.06505081, Time: 0.0695 Steps: 26000, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002344, Sample Num: 37504, Cur Loss: 0.00000000, Cur Avg Loss: 0.03814474, Log Avg loss: 0.00065821, Global Avg Loss: 0.06455926, Time: 0.1117 Steps: 26200, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002544, Sample Num: 40704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03524357, Log Avg loss: 0.00124185, Global Avg Loss: 0.06407959, Time: 0.1061 Steps: 26400, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002744, Sample Num: 43904, Cur Loss: 0.00000316, Cur Avg Loss: 0.04432711, Log Avg loss: 0.15986975, Global Avg Loss: 0.06479981, Time: 0.2213 Steps: 26600, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002944, Sample Num: 47104, Cur Loss: 0.00000077, Cur Avg Loss: 0.04133110, Log Avg loss: 0.00022591, Global Avg Loss: 0.06431792, Time: 0.1070 Steps: 26800, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 003144, Sample Num: 50304, Cur Loss: 0.00000000, Cur Avg Loss: 0.03870215, Log Avg loss: 0.00000399, Global Avg Loss: 0.06384152, Time: 0.2107 Steps: 27000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003344, Sample Num: 53504, Cur Loss: 0.00000000, Cur Avg Loss: 0.03647162, Log Avg loss: 0.00140764, Global Avg Loss: 0.06338245, Time: 0.1599 Steps: 27200, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003544, Sample Num: 56704, Cur Loss: 0.00000000, Cur Avg Loss: 0.05196555, Log Avg loss: 0.31102406, Global Avg Loss: 0.06519005, Time: 0.1754 Steps: 27400, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003744, Sample Num: 59904, Cur Loss: 0.00000018, Cur Avg Loss: 0.04925630, Log Avg loss: 0.00124842, Global Avg Loss: 0.06472670, Time: 0.1233 Steps: 27600, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003944, Sample Num: 63104, Cur Loss: 0.00000000, Cur Avg Loss: 0.04675858, Log Avg loss: 0.00000122, Global Avg Loss: 0.06426105, Time: 0.1100 Steps: 27800, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 004144, Sample Num: 66304, Cur Loss: 0.00000000, Cur Avg Loss: 0.04450207, Log Avg loss: 0.00000371, Global Avg Loss: 0.06380207, Time: 0.0922 Steps: 28000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 004344, Sample Num: 69504, Cur Loss: 0.00000000, Cur Avg Loss: 0.04245592, Log Avg loss: 0.00005958, Global Avg Loss: 0.06335000, Time: 0.2353 Steps: 28200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004544, Sample Num: 72704, Cur Loss: 0.00000000, Cur Avg Loss: 0.04058727, Log Avg loss: 0.00000037, Global Avg Loss: 0.06290387, Time: 0.1252 Steps: 28400, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004744, Sample Num: 75904, Cur Loss: 0.00000000, Cur Avg Loss: 0.03887618, Log Avg loss: 0.00000014, Global Avg Loss: 0.06246399, Time: 0.1687 Steps: 28600, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004944, Sample Num: 79104, Cur Loss: 0.00000864, Cur Avg Loss: 0.03781754, Log Avg loss: 0.01270659, Global Avg Loss: 0.06211845, Time: 0.1992 Steps: 28800, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005144, Sample Num: 82304, Cur Loss: 0.00000000, Cur Avg Loss: 0.03634722, Log Avg loss: 0.00000080, Global Avg Loss: 0.06169005, Time: 0.0621 Steps: 29000, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005344, Sample Num: 85504, Cur Loss: 0.00000018, Cur Avg Loss: 0.03500856, Log Avg loss: 0.00057828, Global Avg Loss: 0.06127148, Time: 0.3482 Steps: 29200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005544, Sample Num: 88704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03374575, Log Avg loss: 0.00000353, Global Avg Loss: 0.06085469, Time: 0.2189 Steps: 29400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 005744, Sample Num: 91904, Cur Loss: 0.00000000, Cur Avg Loss: 0.03257089, Log Avg loss: 0.00000390, Global Avg Loss: 0.06044354, Time: 0.1459 Steps: 29600, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 005944, Sample Num: 95104, Cur Loss: 0.00000024, Cur Avg Loss: 0.03147519, Log Avg loss: 0.00000657, Global Avg Loss: 0.06003792, Time: 0.0764 Steps: 29800, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006144, Sample Num: 98304, Cur Loss: 0.00000006, Cur Avg Loss: 0.03528742, Log Avg loss: 0.14858703, Global Avg Loss: 0.06062825, Time: 0.1280 Steps: 30000, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006344, Sample Num: 101504, Cur Loss: 0.00000024, Cur Avg Loss: 0.03417504, Log Avg loss: 0.00000266, Global Avg Loss: 0.06022675, Time: 0.0748 Steps: 30200, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006544, Sample Num: 104704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03315139, Log Avg loss: 0.00068124, Global Avg Loss: 0.05983501, Time: 0.1232 Steps: 30400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006744, Sample Num: 107904, Cur Loss: 0.00000000, Cur Avg Loss: 0.03216838, Log Avg loss: 0.00000409, Global Avg Loss: 0.05944395, Time: 0.1084 Steps: 30600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 006944, Sample Num: 111104, Cur Loss: 0.00000000, Cur Avg Loss: 0.03124190, Log Avg loss: 0.00000092, Global Avg Loss: 0.05905796, Time: 0.1126 Steps: 30800, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007144, Sample Num: 114304, Cur Loss: 0.00000000, Cur Avg Loss: 0.03103671, Log Avg loss: 0.02391281, Global Avg Loss: 0.05883122, Time: 0.2440 Steps: 31000, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007344, Sample Num: 117504, Cur Loss: 0.00000000, Cur Avg Loss: 0.03019152, Log Avg loss: 0.00000115, Global Avg Loss: 0.05845410, Time: 0.1093 Steps: 31200, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007544, Sample Num: 120704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02939119, Log Avg loss: 0.00000293, Global Avg Loss: 0.05808180, Time: 0.1000 Steps: 31400, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007744, Sample Num: 123904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02863458, Log Avg loss: 0.00009533, Global Avg Loss: 0.05771480, Time: 0.0878 Steps: 31600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007944, Sample Num: 127104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02934881, Log Avg loss: 0.05700383, Global Avg Loss: 0.05771033, Time: 0.1267 Steps: 31800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008144, Sample Num: 130304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02862984, Log Avg loss: 0.00007255, Global Avg Loss: 0.05735009, Time: 0.1242 Steps: 32000, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008344, Sample Num: 133504, Cur Loss: 0.00000012, Cur Avg Loss: 0.02804297, Log Avg loss: 0.00414538, Global Avg Loss: 0.05701963, Time: 0.4018 Steps: 32200, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008544, Sample Num: 136704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03223884, Log Avg loss: 0.20729055, Global Avg Loss: 0.05794722, Time: 0.1186 Steps: 32400, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008744, Sample Num: 139904, Cur Loss: 0.00000006, Cur Avg Loss: 0.03150147, Log Avg loss: 0.00000102, Global Avg Loss: 0.05759173, Time: 0.1678 Steps: 32600, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008944, Sample Num: 143104, Cur Loss: 0.00000000, Cur Avg Loss: 0.03079808, Log Avg loss: 0.00004595, Global Avg Loss: 0.05724084, Time: 0.2446 Steps: 32800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 009144, Sample Num: 146304, Cur Loss: 0.00000006, Cur Avg Loss: 0.03012447, Log Avg loss: 0.00000060, Global Avg Loss: 0.05689393, Time: 0.0830 Steps: 33000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009344, Sample Num: 149504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02948530, Log Avg loss: 0.00026273, Global Avg Loss: 0.05655277, Time: 0.1279 Steps: 33200, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009544, Sample Num: 152704, Cur Loss: 0.00000006, Cur Avg Loss: 0.02887049, Log Avg loss: 0.00014639, Global Avg Loss: 0.05621501, Time: 0.2745 Steps: 33400, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009744, Sample Num: 155904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02828185, Log Avg loss: 0.00019192, Global Avg Loss: 0.05588154, Time: 0.0698 Steps: 33600, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009944, Sample Num: 159104, Cur Loss: 0.00000006, Cur Avg Loss: 0.02859349, Log Avg loss: 0.04377652, Global Avg Loss: 0.05580991, Time: 0.2216 Steps: 33800, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 010144, Sample Num: 162304, Cur Loss: 0.00000006, Cur Avg Loss: 0.02802975, Log Avg loss: 0.00000067, Global Avg Loss: 0.05548162, Time: 0.2468 Steps: 34000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 010344, Sample Num: 165504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02749295, Log Avg loss: 0.00026658, Global Avg Loss: 0.05515873, Time: 0.1176 Steps: 34200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010544, Sample Num: 168704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02697148, Log Avg loss: 0.00000103, Global Avg Loss: 0.05483804, Time: 0.2931 Steps: 34400, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010744, Sample Num: 171904, Cur Loss: 0.00000006, Cur Avg Loss: 0.02646942, Log Avg loss: 0.00000068, Global Avg Loss: 0.05452107, Time: 0.1328 Steps: 34600, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010944, Sample Num: 175104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02598572, Log Avg loss: 0.00000151, Global Avg Loss: 0.05420774, Time: 0.1505 Steps: 34800, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011144, Sample Num: 178304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02551939, Log Avg loss: 0.00000146, Global Avg Loss: 0.05389798, Time: 0.1186 Steps: 35000, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011344, Sample Num: 181504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02506947, Log Avg loss: 0.00000006, Global Avg Loss: 0.05359175, Time: 0.1251 Steps: 35200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011544, Sample Num: 184704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02463514, Log Avg loss: 0.00000006, Global Avg Loss: 0.05328897, Time: 0.1055 Steps: 35400, Updated lr: 0.000070 Training, Epoch: 0003, Batch: 011744, Sample Num: 187904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02421738, Log Avg loss: 0.00010409, Global Avg Loss: 0.05299018, Time: 0.0876 Steps: 35600, Updated lr: 0.000070 ***** Running evaluation checkpoint-35784 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-35784 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1860.063487, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.082544, "eval_total_loss": 114.984221, "eval_acc": 0.999686, "eval_prec": 0.998131, "eval_recall": 0.988889, "eval_f1": 0.993488, "eval_roc_auc": 0.999994, "eval_pr_auc": 0.999771, "eval_confusion_matrix": {"tn": 21743, "fp": 1, "fn": 6, "tp": 534}, "eval_mcc2": 0.993339, "eval_mcc": 0.993339, "eval_sn": 0.988889, "eval_sp": 0.999954, "update_flag": false, "test_avg_loss": 0.080777, "test_total_loss": 112.522777, "test_acc": 0.999731, "test_prec": 1.0, "test_recall": 0.988889, "test_f1": 0.994413, "test_roc_auc": 0.999999, "test_pr_auc": 0.999952, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 6, "tp": 534}, "test_mcc2": 0.994292, "test_mcc": 0.994292, "test_sn": 0.988889, "test_sp": 1.0, "lr": 7.011756802149816e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.05271777671136802, "train_cur_epoch_loss": 284.4114885546737, "train_cur_epoch_avg_loss": 0.02384402150860779, "train_cur_epoch_time": 1860.063486814499, "train_cur_epoch_avg_time": 0.1559409361849848, "epoch": 3, "step": 35784} ################################################## Training, Epoch: 0004, Batch: 000016, Sample Num: 256, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000007, Log Avg loss: 0.00001304, Global Avg Loss: 0.05269422, Time: 0.2397 Steps: 35800, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000216, Sample Num: 3456, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000016, Log Avg loss: 0.00000017, Global Avg Loss: 0.05240147, Time: 0.0642 Steps: 36000, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000416, Sample Num: 6656, Cur Loss: 0.00000000, Cur Avg Loss: 0.00003518, Log Avg loss: 0.00007300, Global Avg Loss: 0.05211236, Time: 0.1364 Steps: 36200, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000616, Sample Num: 9856, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002377, Log Avg loss: 0.00000003, Global Avg Loss: 0.05182603, Time: 0.1224 Steps: 36400, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000816, Sample Num: 13056, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001799, Log Avg loss: 0.00000018, Global Avg Loss: 0.05154283, Time: 0.1583 Steps: 36600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001016, Sample Num: 16256, Cur Loss: 0.00000000, Cur Avg Loss: 0.03166663, Log Avg loss: 0.16079308, Global Avg Loss: 0.05213658, Time: 0.2661 Steps: 36800, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001216, Sample Num: 19456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02646188, Log Avg loss: 0.00002178, Global Avg Loss: 0.05185488, Time: 0.2925 Steps: 37000, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001416, Sample Num: 22656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02272449, Log Avg loss: 0.00000113, Global Avg Loss: 0.05157610, Time: 0.0899 Steps: 37200, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001616, Sample Num: 25856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02629571, Log Avg loss: 0.05157995, Global Avg Loss: 0.05157612, Time: 0.1008 Steps: 37400, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001816, Sample Num: 29056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02340000, Log Avg loss: 0.00000265, Global Avg Loss: 0.05130179, Time: 0.0996 Steps: 37600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002016, Sample Num: 32256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02184105, Log Avg loss: 0.00768583, Global Avg Loss: 0.05107102, Time: 0.1233 Steps: 37800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002216, Sample Num: 35456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02940479, Log Avg loss: 0.10564727, Global Avg Loss: 0.05135826, Time: 0.2876 Steps: 38000, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002416, Sample Num: 38656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02697090, Log Avg loss: 0.00000342, Global Avg Loss: 0.05108939, Time: 0.1386 Steps: 38200, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002616, Sample Num: 41856, Cur Loss: 0.00000000, Cur Avg Loss: 0.03285376, Log Avg loss: 0.10391875, Global Avg Loss: 0.05136454, Time: 0.1248 Steps: 38400, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002816, Sample Num: 45056, Cur Loss: 0.00000000, Cur Avg Loss: 0.03052559, Log Avg loss: 0.00007304, Global Avg Loss: 0.05109878, Time: 0.1899 Steps: 38600, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003016, Sample Num: 48256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02850573, Log Avg loss: 0.00006607, Global Avg Loss: 0.05083573, Time: 0.2477 Steps: 38800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003216, Sample Num: 51456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02673304, Log Avg loss: 0.00000088, Global Avg Loss: 0.05057504, Time: 0.1416 Steps: 39000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003416, Sample Num: 54656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02517547, Log Avg loss: 0.00012983, Global Avg Loss: 0.05031766, Time: 0.1305 Steps: 39200, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003616, Sample Num: 57856, Cur Loss: 0.00000000, Cur Avg Loss: 0.04168332, Log Avg loss: 0.32363738, Global Avg Loss: 0.05170507, Time: 0.1163 Steps: 39400, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003816, Sample Num: 61056, Cur Loss: 0.00000000, Cur Avg Loss: 0.03949874, Log Avg loss: 0.00000145, Global Avg Loss: 0.05144394, Time: 0.1473 Steps: 39600, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004016, Sample Num: 64256, Cur Loss: 0.00000155, Cur Avg Loss: 0.03753177, Log Avg loss: 0.00000196, Global Avg Loss: 0.05118544, Time: 0.0474 Steps: 39800, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004216, Sample Num: 67456, Cur Loss: 0.00000000, Cur Avg Loss: 0.03575139, Log Avg loss: 0.00000143, Global Avg Loss: 0.05092952, Time: 0.1001 Steps: 40000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004416, Sample Num: 70656, Cur Loss: 0.00000000, Cur Avg Loss: 0.03413224, Log Avg loss: 0.00000058, Global Avg Loss: 0.05067614, Time: 0.4223 Steps: 40200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 004616, Sample Num: 73856, Cur Loss: 0.00000000, Cur Avg Loss: 0.03265337, Log Avg loss: 0.00000001, Global Avg Loss: 0.05042527, Time: 0.2238 Steps: 40400, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 004816, Sample Num: 77056, Cur Loss: 0.00000000, Cur Avg Loss: 0.03132993, Log Avg loss: 0.00078483, Global Avg Loss: 0.05018074, Time: 0.1813 Steps: 40600, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005016, Sample Num: 80256, Cur Loss: 0.00000006, Cur Avg Loss: 0.03008075, Log Avg loss: 0.00000047, Global Avg Loss: 0.04993475, Time: 0.2053 Steps: 40800, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005216, Sample Num: 83456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02892737, Log Avg loss: 0.00000065, Global Avg Loss: 0.04969117, Time: 0.0765 Steps: 41000, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005416, Sample Num: 86656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02786130, Log Avg loss: 0.00005817, Global Avg Loss: 0.04945024, Time: 0.1041 Steps: 41200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005616, Sample Num: 89856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02686911, Log Avg loss: 0.00000049, Global Avg Loss: 0.04921135, Time: 0.1771 Steps: 41400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 005816, Sample Num: 93056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02594516, Log Avg loss: 0.00000065, Global Avg Loss: 0.04897476, Time: 0.3350 Steps: 41600, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006016, Sample Num: 96256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02508262, Log Avg loss: 0.00000003, Global Avg Loss: 0.04874043, Time: 0.1033 Steps: 41800, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006216, Sample Num: 99456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02669485, Log Avg loss: 0.07519068, Global Avg Loss: 0.04886638, Time: 0.0891 Steps: 42000, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006416, Sample Num: 102656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02586272, Log Avg loss: 0.00000010, Global Avg Loss: 0.04863479, Time: 0.0764 Steps: 42200, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006616, Sample Num: 105856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02508090, Log Avg loss: 0.00000010, Global Avg Loss: 0.04840538, Time: 0.4075 Steps: 42400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006816, Sample Num: 109056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02434496, Log Avg loss: 0.00000006, Global Avg Loss: 0.04817813, Time: 0.0699 Steps: 42600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007016, Sample Num: 112256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02365097, Log Avg loss: 0.00000004, Global Avg Loss: 0.04795299, Time: 0.0635 Steps: 42800, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007216, Sample Num: 115456, Cur Loss: 0.00000006, Cur Avg Loss: 0.02299547, Log Avg loss: 0.00000050, Global Avg Loss: 0.04772996, Time: 0.0963 Steps: 43000, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007416, Sample Num: 118656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02237532, Log Avg loss: 0.00000010, Global Avg Loss: 0.04750899, Time: 0.1996 Steps: 43200, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007616, Sample Num: 121856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02178773, Log Avg loss: 0.00000003, Global Avg Loss: 0.04729005, Time: 0.2109 Steps: 43400, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007816, Sample Num: 125056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02123035, Log Avg loss: 0.00000546, Global Avg Loss: 0.04707315, Time: 0.1226 Steps: 43600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 008016, Sample Num: 128256, Cur Loss: 0.00000012, Cur Avg Loss: 0.02070066, Log Avg loss: 0.00000004, Global Avg Loss: 0.04685821, Time: 0.1495 Steps: 43800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008216, Sample Num: 131456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02019850, Log Avg loss: 0.00007198, Global Avg Loss: 0.04664554, Time: 0.1224 Steps: 44000, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008416, Sample Num: 134656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02502400, Log Avg loss: 0.22325565, Global Avg Loss: 0.04744468, Time: 0.1517 Steps: 44200, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008616, Sample Num: 137856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02444313, Log Avg loss: 0.00000002, Global Avg Loss: 0.04723097, Time: 0.2139 Steps: 44400, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008816, Sample Num: 141056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02388861, Log Avg loss: 0.00000004, Global Avg Loss: 0.04701917, Time: 0.1163 Steps: 44600, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 009016, Sample Num: 144256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02335870, Log Avg loss: 0.00000015, Global Avg Loss: 0.04680926, Time: 0.0872 Steps: 44800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 009216, Sample Num: 147456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02285178, Log Avg loss: 0.00000002, Global Avg Loss: 0.04660122, Time: 0.1222 Steps: 45000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009416, Sample Num: 150656, Cur Loss: 0.00000012, Cur Avg Loss: 0.02236651, Log Avg loss: 0.00000515, Global Avg Loss: 0.04639504, Time: 0.0679 Steps: 45200, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009616, Sample Num: 153856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02190212, Log Avg loss: 0.00003886, Global Avg Loss: 0.04619083, Time: 0.1291 Steps: 45400, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009816, Sample Num: 157056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02375207, Log Avg loss: 0.11269725, Global Avg Loss: 0.04648253, Time: 0.1316 Steps: 45600, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010016, Sample Num: 160256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02327784, Log Avg loss: 0.00000258, Global Avg Loss: 0.04627956, Time: 0.1782 Steps: 45800, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010216, Sample Num: 163456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02282214, Log Avg loss: 0.00000097, Global Avg Loss: 0.04607835, Time: 0.0717 Steps: 46000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010416, Sample Num: 166656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02238394, Log Avg loss: 0.00000050, Global Avg Loss: 0.04587887, Time: 0.1157 Steps: 46200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 010616, Sample Num: 169856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02196224, Log Avg loss: 0.00000002, Global Avg Loss: 0.04568112, Time: 0.0747 Steps: 46400, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 010816, Sample Num: 173056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02155613, Log Avg loss: 0.00000001, Global Avg Loss: 0.04548506, Time: 0.0706 Steps: 46600, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011016, Sample Num: 176256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02116477, Log Avg loss: 0.00000005, Global Avg Loss: 0.04529068, Time: 0.1702 Steps: 46800, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011216, Sample Num: 179456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02078737, Log Avg loss: 0.00000013, Global Avg Loss: 0.04509796, Time: 0.1562 Steps: 47000, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011416, Sample Num: 182656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02042319, Log Avg loss: 0.00000000, Global Avg Loss: 0.04490687, Time: 0.0701 Steps: 47200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011616, Sample Num: 185856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02007155, Log Avg loss: 0.00000000, Global Avg Loss: 0.04471739, Time: 0.1951 Steps: 47400, Updated lr: 0.000060 Training, Epoch: 0004, Batch: 011816, Sample Num: 189056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01973218, Log Avg loss: 0.00002141, Global Avg Loss: 0.04452959, Time: 0.1177 Steps: 47600, Updated lr: 0.000060 ***** Running evaluation checkpoint-47712 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-47712 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1866.278110, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.080624, "eval_total_loss": 112.308681, "eval_acc": 0.999776, "eval_prec": 0.998138, "eval_recall": 0.992593, "eval_f1": 0.995357, "eval_roc_auc": 0.999994, "eval_pr_auc": 0.999769, "eval_confusion_matrix": {"tn": 21743, "fp": 1, "fn": 4, "tp": 536}, "eval_mcc2": 0.995247, "eval_mcc": 0.995247, "eval_sn": 0.992593, "eval_sp": 0.999954, "update_flag": true, "test_avg_loss": 0.086911, "test_total_loss": 121.06695, "test_acc": 0.999776, "test_prec": 1.0, "test_recall": 0.990741, "test_f1": 0.995349, "test_roc_auc": 0.999998, "test_pr_auc": 0.999939, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 5, "tp": 535}, "test_mcc2": 0.995245, "test_mcc": 0.995245, "test_sn": 0.990741, "test_sp": 1.0, "lr": 6.010077258985556e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.044425057464852376, "train_cur_epoch_loss": 233.15541992344052, "train_cur_epoch_avg_loss": 0.019546899725305208, "train_cur_epoch_time": 1866.278109550476, "train_cur_epoch_avg_time": 0.15646194748075756, "epoch": 4, "step": 47712} ################################################## Training, Epoch: 0005, Batch: 000088, Sample Num: 1408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000001, Global Avg Loss: 0.04434327, Time: 0.1388 Steps: 47800, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000288, Sample Num: 4608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.04415851, Time: 0.0993 Steps: 48000, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000488, Sample Num: 7808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000001, Global Avg Loss: 0.04397528, Time: 0.0957 Steps: 48200, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000688, Sample Num: 11008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.04379356, Time: 0.0949 Steps: 48400, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000888, Sample Num: 14208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000012, Log Avg loss: 0.00000052, Global Avg Loss: 0.04361334, Time: 0.0813 Steps: 48600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001088, Sample Num: 17408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00277901, Log Avg loss: 0.01511730, Global Avg Loss: 0.04349656, Time: 0.0685 Steps: 48800, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001288, Sample Num: 20608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00234749, Log Avg loss: 0.00000003, Global Avg Loss: 0.04331902, Time: 0.1014 Steps: 49000, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001488, Sample Num: 23808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00203198, Log Avg loss: 0.00000004, Global Avg Loss: 0.04314293, Time: 0.2407 Steps: 49200, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001688, Sample Num: 27008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00191724, Log Avg loss: 0.00106363, Global Avg Loss: 0.04297256, Time: 0.1006 Steps: 49400, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001888, Sample Num: 30208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00171463, Log Avg loss: 0.00000459, Global Avg Loss: 0.04279931, Time: 0.2968 Steps: 49600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002088, Sample Num: 33408, Cur Loss: 0.00000006, Cur Avg Loss: 0.00355337, Log Avg loss: 0.02091102, Global Avg Loss: 0.04271140, Time: 0.2067 Steps: 49800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002288, Sample Num: 36608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00324295, Log Avg loss: 0.00000217, Global Avg Loss: 0.04254056, Time: 0.2107 Steps: 50000, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002488, Sample Num: 39808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00298249, Log Avg loss: 0.00000285, Global Avg Loss: 0.04237109, Time: 0.4025 Steps: 50200, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002688, Sample Num: 43008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00307643, Log Avg loss: 0.00424504, Global Avg Loss: 0.04221980, Time: 0.1132 Steps: 50400, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002888, Sample Num: 46208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00286344, Log Avg loss: 0.00000083, Global Avg Loss: 0.04205292, Time: 0.1195 Steps: 50600, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003088, Sample Num: 49408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00267799, Log Avg loss: 0.00000012, Global Avg Loss: 0.04188736, Time: 0.1059 Steps: 50800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003288, Sample Num: 52608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00261709, Log Avg loss: 0.00167685, Global Avg Loss: 0.04172967, Time: 0.1512 Steps: 51000, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003488, Sample Num: 55808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00246901, Log Avg loss: 0.00003451, Global Avg Loss: 0.04156680, Time: 0.2023 Steps: 51200, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003688, Sample Num: 59008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00531053, Log Avg loss: 0.05486658, Global Avg Loss: 0.04161855, Time: 0.2168 Steps: 51400, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003888, Sample Num: 62208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00509744, Log Avg loss: 0.00116814, Global Avg Loss: 0.04146177, Time: 0.0782 Steps: 51600, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 004088, Sample Num: 65408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00484806, Log Avg loss: 0.00000000, Global Avg Loss: 0.04130168, Time: 0.1335 Steps: 51800, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 004288, Sample Num: 68608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00462193, Log Avg loss: 0.00000000, Global Avg Loss: 0.04114283, Time: 0.4379 Steps: 52000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004488, Sample Num: 71808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00441597, Log Avg loss: 0.00000000, Global Avg Loss: 0.04098520, Time: 0.1942 Steps: 52200, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004688, Sample Num: 75008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00422757, Log Avg loss: 0.00000000, Global Avg Loss: 0.04082876, Time: 0.2900 Steps: 52400, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004888, Sample Num: 78208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00405460, Log Avg loss: 0.00000018, Global Avg Loss: 0.04067352, Time: 0.2713 Steps: 52600, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005088, Sample Num: 81408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00389522, Log Avg loss: 0.00000000, Global Avg Loss: 0.04051946, Time: 0.1094 Steps: 52800, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005288, Sample Num: 84608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00374790, Log Avg loss: 0.00000000, Global Avg Loss: 0.04036655, Time: 0.0615 Steps: 53000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005488, Sample Num: 87808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00361131, Log Avg loss: 0.00000002, Global Avg Loss: 0.04021480, Time: 0.1439 Steps: 53200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 005688, Sample Num: 91008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00348434, Log Avg loss: 0.00000009, Global Avg Loss: 0.04006418, Time: 0.1668 Steps: 53400, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 005888, Sample Num: 94208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00336598, Log Avg loss: 0.00000000, Global Avg Loss: 0.03991469, Time: 0.1102 Steps: 53600, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006088, Sample Num: 97408, Cur Loss: 0.00000030, Cur Avg Loss: 0.00325541, Log Avg loss: 0.00000009, Global Avg Loss: 0.03976631, Time: 0.4010 Steps: 53800, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006288, Sample Num: 100608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00315489, Log Avg loss: 0.00009501, Global Avg Loss: 0.03961938, Time: 0.2211 Steps: 54000, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006488, Sample Num: 103808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00305764, Log Avg loss: 0.00000018, Global Avg Loss: 0.03947318, Time: 0.2339 Steps: 54200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006688, Sample Num: 107008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00296620, Log Avg loss: 0.00000000, Global Avg Loss: 0.03932806, Time: 0.1011 Steps: 54400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 006888, Sample Num: 110208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00288008, Log Avg loss: 0.00000009, Global Avg Loss: 0.03918400, Time: 0.0794 Steps: 54600, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007088, Sample Num: 113408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00279881, Log Avg loss: 0.00000000, Global Avg Loss: 0.03904099, Time: 0.1386 Steps: 54800, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007288, Sample Num: 116608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00272201, Log Avg loss: 0.00000000, Global Avg Loss: 0.03889902, Time: 0.1023 Steps: 55000, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007488, Sample Num: 119808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00264930, Log Avg loss: 0.00000000, Global Avg Loss: 0.03875809, Time: 0.1548 Steps: 55200, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007688, Sample Num: 123008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00258038, Log Avg loss: 0.00000000, Global Avg Loss: 0.03861816, Time: 0.1101 Steps: 55400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007888, Sample Num: 126208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00251496, Log Avg loss: 0.00000000, Global Avg Loss: 0.03847925, Time: 0.0770 Steps: 55600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008088, Sample Num: 129408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00245277, Log Avg loss: 0.00000000, Global Avg Loss: 0.03834133, Time: 0.0702 Steps: 55800, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008288, Sample Num: 132608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00239358, Log Avg loss: 0.00000005, Global Avg Loss: 0.03820440, Time: 0.1230 Steps: 56000, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008488, Sample Num: 135808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00476697, Log Avg loss: 0.10312016, Global Avg Loss: 0.03843542, Time: 0.0954 Steps: 56200, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008688, Sample Num: 139008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00465723, Log Avg loss: 0.00000001, Global Avg Loss: 0.03829912, Time: 0.1051 Steps: 56400, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008888, Sample Num: 142208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00455243, Log Avg loss: 0.00000000, Global Avg Loss: 0.03816379, Time: 0.1062 Steps: 56600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 009088, Sample Num: 145408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00445225, Log Avg loss: 0.00000000, Global Avg Loss: 0.03802941, Time: 0.1921 Steps: 56800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009288, Sample Num: 148608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00435638, Log Avg loss: 0.00000000, Global Avg Loss: 0.03789597, Time: 0.1347 Steps: 57000, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009488, Sample Num: 151808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00426456, Log Avg loss: 0.00000044, Global Avg Loss: 0.03776347, Time: 0.1984 Steps: 57200, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009688, Sample Num: 155008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00417652, Log Avg loss: 0.00000000, Global Avg Loss: 0.03763189, Time: 0.1209 Steps: 57400, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009888, Sample Num: 158208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00409228, Log Avg loss: 0.00001176, Global Avg Loss: 0.03750126, Time: 0.2167 Steps: 57600, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 010088, Sample Num: 161408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00401115, Log Avg loss: 0.00000001, Global Avg Loss: 0.03737150, Time: 0.0660 Steps: 57800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 010288, Sample Num: 164608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00393318, Log Avg loss: 0.00000044, Global Avg Loss: 0.03724264, Time: 0.0785 Steps: 58000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010488, Sample Num: 167808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00385818, Log Avg loss: 0.00000008, Global Avg Loss: 0.03711465, Time: 0.0985 Steps: 58200, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010688, Sample Num: 171008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00378598, Log Avg loss: 0.00000001, Global Avg Loss: 0.03698755, Time: 0.2945 Steps: 58400, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010888, Sample Num: 174208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00371644, Log Avg loss: 0.00000000, Global Avg Loss: 0.03686131, Time: 0.0972 Steps: 58600, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011088, Sample Num: 177408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00364940, Log Avg loss: 0.00000000, Global Avg Loss: 0.03673593, Time: 0.0847 Steps: 58800, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011288, Sample Num: 180608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00358474, Log Avg loss: 0.00000000, Global Avg Loss: 0.03661141, Time: 0.1709 Steps: 59000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011488, Sample Num: 183808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00352233, Log Avg loss: 0.00000000, Global Avg Loss: 0.03648772, Time: 0.0969 Steps: 59200, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 011688, Sample Num: 187008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00346206, Log Avg loss: 0.00000014, Global Avg Loss: 0.03636486, Time: 0.2496 Steps: 59400, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 011888, Sample Num: 190208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00340382, Log Avg loss: 0.00000002, Global Avg Loss: 0.03624283, Time: 0.2123 Steps: 59600, Updated lr: 0.000050 ***** Running evaluation checkpoint-59640 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-59640 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1866.275959, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.060583, "eval_total_loss": 84.392163, "eval_acc": 0.999776, "eval_prec": 0.994455, "eval_recall": 0.996296, "eval_f1": 0.995375, "eval_roc_auc": 0.999996, "eval_pr_auc": 0.999834, "eval_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 2, "tp": 538}, "eval_mcc2": 0.99526, "eval_mcc": 0.99526, "eval_sn": 0.996296, "eval_sp": 0.999862, "update_flag": true, "test_avg_loss": 0.056199, "test_total_loss": 78.284853, "test_acc": 0.99982, "test_prec": 1.0, "test_recall": 0.992593, "test_f1": 0.996283, "test_roc_auc": 0.999999, "test_pr_auc": 0.99998, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 4, "tp": 536}, "test_mcc2": 0.996198, "test_mcc": 0.996198, "test_sn": 0.992593, "test_sp": 1.0, "lr": 5.008397715821297e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.03621852701047791, "train_cur_epoch_loss": 40.464609141864315, "train_cur_epoch_avg_loss": 0.0033924051929799057, "train_cur_epoch_time": 1866.275958776474, "train_cur_epoch_avg_time": 0.15646176716771243, "epoch": 5, "step": 59640} ################################################## Training, Epoch: 0006, Batch: 000160, Sample Num: 2560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03612162, Time: 0.1118 Steps: 59800, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000360, Sample Num: 5760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03600122, Time: 0.3677 Steps: 60000, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000560, Sample Num: 8960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03588161, Time: 0.2001 Steps: 60200, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000760, Sample Num: 12160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03576280, Time: 0.3977 Steps: 60400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 000960, Sample Num: 15360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000003, Log Avg loss: 0.00000014, Global Avg Loss: 0.03564477, Time: 0.0800 Steps: 60600, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001160, Sample Num: 18560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00003153, Log Avg loss: 0.00018276, Global Avg Loss: 0.03552812, Time: 0.1408 Steps: 60800, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001360, Sample Num: 21760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002691, Log Avg loss: 0.00000008, Global Avg Loss: 0.03541163, Time: 0.3195 Steps: 61000, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001560, Sample Num: 24960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002346, Log Avg loss: 0.00000000, Global Avg Loss: 0.03529591, Time: 0.0932 Steps: 61200, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001760, Sample Num: 28160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00006953, Log Avg loss: 0.00042892, Global Avg Loss: 0.03518233, Time: 0.0828 Steps: 61400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001960, Sample Num: 31360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00006244, Log Avg loss: 0.00000000, Global Avg Loss: 0.03506811, Time: 0.0968 Steps: 61600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002160, Sample Num: 34560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00443407, Log Avg loss: 0.04727602, Global Avg Loss: 0.03510761, Time: 0.1709 Steps: 61800, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002360, Sample Num: 37760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00405830, Log Avg loss: 0.00000000, Global Avg Loss: 0.03499436, Time: 0.1074 Steps: 62000, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002560, Sample Num: 40960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00403188, Log Avg loss: 0.00372016, Global Avg Loss: 0.03489380, Time: 0.1424 Steps: 62200, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002760, Sample Num: 44160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00373973, Log Avg loss: 0.00000023, Global Avg Loss: 0.03478196, Time: 0.0811 Steps: 62400, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002960, Sample Num: 47360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00348705, Log Avg loss: 0.00000001, Global Avg Loss: 0.03467084, Time: 0.0976 Steps: 62600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 003160, Sample Num: 50560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00326635, Log Avg loss: 0.00000002, Global Avg Loss: 0.03456042, Time: 0.1644 Steps: 62800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003360, Sample Num: 53760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00307194, Log Avg loss: 0.00000017, Global Avg Loss: 0.03445071, Time: 0.1075 Steps: 63000, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003560, Sample Num: 56960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00401423, Log Avg loss: 0.01984474, Global Avg Loss: 0.03440449, Time: 0.1054 Steps: 63200, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003760, Sample Num: 60160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00388928, Log Avg loss: 0.00166528, Global Avg Loss: 0.03430121, Time: 0.0767 Steps: 63400, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003960, Sample Num: 63360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00369286, Log Avg loss: 0.00000000, Global Avg Loss: 0.03419334, Time: 0.4080 Steps: 63600, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 004160, Sample Num: 66560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00351531, Log Avg loss: 0.00000001, Global Avg Loss: 0.03408615, Time: 0.1718 Steps: 63800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 004360, Sample Num: 69760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00335668, Log Avg loss: 0.00005713, Global Avg Loss: 0.03397981, Time: 0.0857 Steps: 64000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004560, Sample Num: 72960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00320955, Log Avg loss: 0.00000207, Global Avg Loss: 0.03387396, Time: 0.0915 Steps: 64200, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004760, Sample Num: 76160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00307470, Log Avg loss: 0.00000000, Global Avg Loss: 0.03376877, Time: 0.0655 Steps: 64400, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004960, Sample Num: 79360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00295072, Log Avg loss: 0.00000001, Global Avg Loss: 0.03366422, Time: 0.1851 Steps: 64600, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005160, Sample Num: 82560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00283635, Log Avg loss: 0.00000000, Global Avg Loss: 0.03356032, Time: 0.0845 Steps: 64800, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005360, Sample Num: 85760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00273051, Log Avg loss: 0.00000000, Global Avg Loss: 0.03345705, Time: 0.3559 Steps: 65000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005560, Sample Num: 88960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00263229, Log Avg loss: 0.00000000, Global Avg Loss: 0.03335442, Time: 0.1372 Steps: 65200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 005760, Sample Num: 92160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00254089, Log Avg loss: 0.00000000, Global Avg Loss: 0.03325242, Time: 0.1075 Steps: 65400, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 005960, Sample Num: 95360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00245563, Log Avg loss: 0.00000000, Global Avg Loss: 0.03315104, Time: 0.0893 Steps: 65600, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006160, Sample Num: 98560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00237590, Log Avg loss: 0.00000000, Global Avg Loss: 0.03305028, Time: 0.1220 Steps: 65800, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006360, Sample Num: 101760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00230119, Log Avg loss: 0.00000000, Global Avg Loss: 0.03295013, Time: 0.0677 Steps: 66000, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006560, Sample Num: 104960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00223103, Log Avg loss: 0.00000000, Global Avg Loss: 0.03285058, Time: 0.2314 Steps: 66200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006760, Sample Num: 108160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00216502, Log Avg loss: 0.00000000, Global Avg Loss: 0.03275163, Time: 0.1205 Steps: 66400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 006960, Sample Num: 111360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00210281, Log Avg loss: 0.00000000, Global Avg Loss: 0.03265328, Time: 0.1437 Steps: 66600, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007160, Sample Num: 114560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00204407, Log Avg loss: 0.00000000, Global Avg Loss: 0.03255552, Time: 0.1112 Steps: 66800, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007360, Sample Num: 117760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00198853, Log Avg loss: 0.00000000, Global Avg Loss: 0.03245834, Time: 0.1246 Steps: 67000, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007560, Sample Num: 120960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00193592, Log Avg loss: 0.00000000, Global Avg Loss: 0.03236173, Time: 0.0869 Steps: 67200, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007760, Sample Num: 124160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00188602, Log Avg loss: 0.00000000, Global Avg Loss: 0.03226570, Time: 0.1535 Steps: 67400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007960, Sample Num: 127360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00183864, Log Avg loss: 0.00000000, Global Avg Loss: 0.03217024, Time: 0.0723 Steps: 67600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008160, Sample Num: 130560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00179357, Log Avg loss: 0.00000000, Global Avg Loss: 0.03207535, Time: 0.0662 Steps: 67800, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008360, Sample Num: 133760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00175066, Log Avg loss: 0.00000000, Global Avg Loss: 0.03198101, Time: 0.1290 Steps: 68000, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008560, Sample Num: 136960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00180908, Log Avg loss: 0.00425106, Global Avg Loss: 0.03189969, Time: 0.2101 Steps: 68200, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008760, Sample Num: 140160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00176778, Log Avg loss: 0.00000000, Global Avg Loss: 0.03180641, Time: 0.2655 Steps: 68400, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008960, Sample Num: 143360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00172832, Log Avg loss: 0.00000000, Global Avg Loss: 0.03171368, Time: 0.1212 Steps: 68600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 009160, Sample Num: 146560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00169059, Log Avg loss: 0.00000000, Global Avg Loss: 0.03162149, Time: 0.1816 Steps: 68800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009360, Sample Num: 149760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00165446, Log Avg loss: 0.00000000, Global Avg Loss: 0.03152984, Time: 0.0919 Steps: 69000, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009560, Sample Num: 152960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00161986, Log Avg loss: 0.00000036, Global Avg Loss: 0.03143871, Time: 0.1049 Steps: 69200, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009760, Sample Num: 156160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00158666, Log Avg loss: 0.00000000, Global Avg Loss: 0.03134811, Time: 0.1607 Steps: 69400, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009960, Sample Num: 159360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00158580, Log Avg loss: 0.00154363, Global Avg Loss: 0.03126246, Time: 0.1348 Steps: 69600, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 010160, Sample Num: 162560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00155458, Log Avg loss: 0.00000000, Global Avg Loss: 0.03117289, Time: 0.0893 Steps: 69800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 010360, Sample Num: 165760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00152457, Log Avg loss: 0.00000002, Global Avg Loss: 0.03108382, Time: 0.1968 Steps: 70000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010560, Sample Num: 168960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00149570, Log Avg loss: 0.00000000, Global Avg Loss: 0.03099526, Time: 0.1131 Steps: 70200, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010760, Sample Num: 172160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146790, Log Avg loss: 0.00000000, Global Avg Loss: 0.03090721, Time: 0.1791 Steps: 70400, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010960, Sample Num: 175360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00144111, Log Avg loss: 0.00000000, Global Avg Loss: 0.03081965, Time: 0.3906 Steps: 70600, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011160, Sample Num: 178560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00141528, Log Avg loss: 0.00000000, Global Avg Loss: 0.03073259, Time: 0.1204 Steps: 70800, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011360, Sample Num: 181760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00139037, Log Avg loss: 0.00000010, Global Avg Loss: 0.03064602, Time: 0.1053 Steps: 71000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011560, Sample Num: 184960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00136631, Log Avg loss: 0.00000000, Global Avg Loss: 0.03055994, Time: 0.0685 Steps: 71200, Updated lr: 0.000040 Training, Epoch: 0006, Batch: 011760, Sample Num: 188160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00134308, Log Avg loss: 0.00000000, Global Avg Loss: 0.03047434, Time: 0.1227 Steps: 71400, Updated lr: 0.000040 ***** Running evaluation checkpoint-71568 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-71568 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1862.797725, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.056105, "eval_total_loss": 78.153577, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999996, "eval_pr_auc": 0.999844, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.036852, "test_total_loss": 51.33434, "test_acc": 0.99982, "test_prec": 1.0, "test_recall": 0.992593, "test_f1": 0.996283, "test_roc_auc": 0.999999, "test_pr_auc": 0.99997, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 4, "tp": 536}, "test_mcc2": 0.996198, "test_mcc": 0.996198, "test_sn": 0.992593, "test_sp": 1.0, "lr": 4.006718172657038e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.030402799374123213, "train_cur_epoch_loss": 15.794594702346757, "train_cur_epoch_avg_loss": 0.001324161192349661, "train_cur_epoch_time": 1862.797725200653, "train_cur_epoch_avg_time": 0.1561701647552526, "epoch": 6, "step": 71568} ################################################## Training, Epoch: 0007, Batch: 000032, Sample Num: 512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000003, Global Avg Loss: 0.03038921, Time: 0.2203 Steps: 71600, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000232, Sample Num: 3712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03030456, Time: 0.0620 Steps: 71800, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000432, Sample Num: 6912, Cur Loss: 0.00000006, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03022038, Time: 0.2270 Steps: 72000, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000632, Sample Num: 10112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03013667, Time: 0.3440 Steps: 72200, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000832, Sample Num: 13312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.03005342, Time: 0.1180 Steps: 72400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001032, Sample Num: 16512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000282, Log Avg loss: 0.00001457, Global Avg Loss: 0.02997067, Time: 0.1731 Steps: 72600, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001232, Sample Num: 19712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000239, Log Avg loss: 0.00000014, Global Avg Loss: 0.02988833, Time: 0.1815 Steps: 72800, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001432, Sample Num: 22912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000206, Log Avg loss: 0.00000001, Global Avg Loss: 0.02980645, Time: 0.1229 Steps: 73000, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001632, Sample Num: 26112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000246, Log Avg loss: 0.00000532, Global Avg Loss: 0.02972502, Time: 0.1960 Steps: 73200, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001832, Sample Num: 29312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000219, Log Avg loss: 0.00000000, Global Avg Loss: 0.02964403, Time: 0.1082 Steps: 73400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002032, Sample Num: 32512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000197, Log Avg loss: 0.00000000, Global Avg Loss: 0.02956347, Time: 0.2117 Steps: 73600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002232, Sample Num: 35712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00302433, Log Avg loss: 0.03373144, Global Avg Loss: 0.02957477, Time: 0.0831 Steps: 73800, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002432, Sample Num: 38912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00277562, Log Avg loss: 0.00000000, Global Avg Loss: 0.02949484, Time: 0.1301 Steps: 74000, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002632, Sample Num: 42112, Cur Loss: 0.00000000, Cur Avg Loss: 0.01101231, Log Avg loss: 0.11117047, Global Avg Loss: 0.02971499, Time: 0.0875 Steps: 74200, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002832, Sample Num: 45312, Cur Loss: 0.00000000, Cur Avg Loss: 0.01023460, Log Avg loss: 0.00000000, Global Avg Loss: 0.02963511, Time: 0.0854 Steps: 74400, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003032, Sample Num: 48512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00955950, Log Avg loss: 0.00000001, Global Avg Loss: 0.02955566, Time: 0.1595 Steps: 74600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003232, Sample Num: 51712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00896796, Log Avg loss: 0.00000030, Global Avg Loss: 0.02947663, Time: 0.2013 Steps: 74800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003432, Sample Num: 54912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00844535, Log Avg loss: 0.00000001, Global Avg Loss: 0.02939803, Time: 0.0698 Steps: 75000, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003632, Sample Num: 58112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00809442, Log Avg loss: 0.00207240, Global Avg Loss: 0.02932535, Time: 0.1601 Steps: 75200, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003832, Sample Num: 61312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00778368, Log Avg loss: 0.00214062, Global Avg Loss: 0.02925324, Time: 0.1081 Steps: 75400, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004032, Sample Num: 64512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00739758, Log Avg loss: 0.00000000, Global Avg Loss: 0.02917585, Time: 0.1627 Steps: 75600, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004232, Sample Num: 67712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00704798, Log Avg loss: 0.00000000, Global Avg Loss: 0.02909887, Time: 0.1035 Steps: 75800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004432, Sample Num: 70912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00672993, Log Avg loss: 0.00000000, Global Avg Loss: 0.02902230, Time: 0.2205 Steps: 76000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 004632, Sample Num: 74112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00643935, Log Avg loss: 0.00000000, Global Avg Loss: 0.02894612, Time: 0.1804 Steps: 76200, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 004832, Sample Num: 77312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00617282, Log Avg loss: 0.00000000, Global Avg Loss: 0.02887035, Time: 0.1661 Steps: 76400, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005032, Sample Num: 80512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00592748, Log Avg loss: 0.00000000, Global Avg Loss: 0.02879497, Time: 0.1185 Steps: 76600, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005232, Sample Num: 83712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00570089, Log Avg loss: 0.00000000, Global Avg Loss: 0.02871998, Time: 0.0765 Steps: 76800, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005432, Sample Num: 86912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00549099, Log Avg loss: 0.00000000, Global Avg Loss: 0.02864538, Time: 0.0876 Steps: 77000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005632, Sample Num: 90112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00529600, Log Avg loss: 0.00000000, Global Avg Loss: 0.02857117, Time: 0.1748 Steps: 77200, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 005832, Sample Num: 93312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00511438, Log Avg loss: 0.00000000, Global Avg Loss: 0.02849735, Time: 0.0553 Steps: 77400, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006032, Sample Num: 96512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00494480, Log Avg loss: 0.00000000, Global Avg Loss: 0.02842390, Time: 0.2866 Steps: 77600, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006232, Sample Num: 99712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00478611, Log Avg loss: 0.00000000, Global Avg Loss: 0.02835083, Time: 0.1497 Steps: 77800, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006432, Sample Num: 102912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00463729, Log Avg loss: 0.00000000, Global Avg Loss: 0.02827814, Time: 0.0980 Steps: 78000, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006632, Sample Num: 106112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00449745, Log Avg loss: 0.00000000, Global Avg Loss: 0.02820581, Time: 0.0833 Steps: 78200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 006832, Sample Num: 109312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00436579, Log Avg loss: 0.00000000, Global Avg Loss: 0.02813386, Time: 0.1113 Steps: 78400, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007032, Sample Num: 112512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00424162, Log Avg loss: 0.00000000, Global Avg Loss: 0.02806227, Time: 0.4078 Steps: 78600, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007232, Sample Num: 115712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00412432, Log Avg loss: 0.00000000, Global Avg Loss: 0.02799105, Time: 0.2495 Steps: 78800, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007432, Sample Num: 118912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00401333, Log Avg loss: 0.00000000, Global Avg Loss: 0.02792018, Time: 0.1563 Steps: 79000, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007632, Sample Num: 122112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00390816, Log Avg loss: 0.00000000, Global Avg Loss: 0.02784968, Time: 0.0794 Steps: 79200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007832, Sample Num: 125312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00380836, Log Avg loss: 0.00000000, Global Avg Loss: 0.02777953, Time: 0.1950 Steps: 79400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008032, Sample Num: 128512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00371353, Log Avg loss: 0.00000000, Global Avg Loss: 0.02770973, Time: 0.1005 Steps: 79600, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008232, Sample Num: 131712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00362331, Log Avg loss: 0.00000000, Global Avg Loss: 0.02764028, Time: 0.1478 Steps: 79800, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008432, Sample Num: 134912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00353738, Log Avg loss: 0.00000064, Global Avg Loss: 0.02757118, Time: 0.0874 Steps: 80000, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008632, Sample Num: 138112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00345542, Log Avg loss: 0.00000000, Global Avg Loss: 0.02750243, Time: 0.2125 Steps: 80200, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008832, Sample Num: 141312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00337717, Log Avg loss: 0.00000000, Global Avg Loss: 0.02743401, Time: 0.1660 Steps: 80400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 009032, Sample Num: 144512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00330239, Log Avg loss: 0.00000000, Global Avg Loss: 0.02736594, Time: 0.0927 Steps: 80600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009232, Sample Num: 147712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00323085, Log Avg loss: 0.00000002, Global Avg Loss: 0.02729820, Time: 0.1351 Steps: 80800, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009432, Sample Num: 150912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00316234, Log Avg loss: 0.00000002, Global Avg Loss: 0.02723080, Time: 0.1082 Steps: 81000, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009632, Sample Num: 154112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00309668, Log Avg loss: 0.00000001, Global Avg Loss: 0.02716373, Time: 0.0788 Steps: 81200, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009832, Sample Num: 157312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00303466, Log Avg loss: 0.00004779, Global Avg Loss: 0.02709710, Time: 0.0913 Steps: 81400, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 010032, Sample Num: 160512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00297416, Log Avg loss: 0.00000000, Global Avg Loss: 0.02703069, Time: 0.3604 Steps: 81600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 010232, Sample Num: 163712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00291602, Log Avg loss: 0.00000000, Global Avg Loss: 0.02696460, Time: 0.1246 Steps: 81800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010432, Sample Num: 166912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00286012, Log Avg loss: 0.00000000, Global Avg Loss: 0.02689883, Time: 0.0965 Steps: 82000, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010632, Sample Num: 170112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00280632, Log Avg loss: 0.00000000, Global Avg Loss: 0.02683339, Time: 0.1034 Steps: 82200, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010832, Sample Num: 173312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00275450, Log Avg loss: 0.00000000, Global Avg Loss: 0.02676826, Time: 0.1992 Steps: 82400, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011032, Sample Num: 176512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00270456, Log Avg loss: 0.00000000, Global Avg Loss: 0.02670344, Time: 0.2191 Steps: 82600, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011232, Sample Num: 179712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00265642, Log Avg loss: 0.00000077, Global Avg Loss: 0.02663894, Time: 0.2160 Steps: 82800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011432, Sample Num: 182912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00260995, Log Avg loss: 0.00000000, Global Avg Loss: 0.02657475, Time: 0.2170 Steps: 83000, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 011632, Sample Num: 186112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00256507, Log Avg loss: 0.00000000, Global Avg Loss: 0.02651087, Time: 0.2338 Steps: 83200, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 011832, Sample Num: 189312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00252171, Log Avg loss: 0.00000000, Global Avg Loss: 0.02644730, Time: 0.3060 Steps: 83400, Updated lr: 0.000030 ***** Running evaluation checkpoint-83496 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-83496 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1864.307018, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.066045, "eval_total_loss": 92.001109, "eval_acc": 0.99982, "eval_prec": 0.998141, "eval_recall": 0.994444, "eval_f1": 0.996289, "eval_roc_auc": 0.999996, "eval_pr_auc": 0.999857, "eval_confusion_matrix": {"tn": 21743, "fp": 1, "fn": 3, "tp": 537}, "eval_mcc2": 0.996199, "eval_mcc": 0.996199, "eval_sn": 0.994444, "eval_sp": 0.999954, "update_flag": true, "test_avg_loss": 0.054296, "test_total_loss": 75.634388, "test_acc": 0.99982, "test_prec": 1.0, "test_recall": 0.992593, "test_f1": 0.996283, "test_roc_auc": 0.999999, "test_pr_auc": 0.999969, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 4, "tp": 536}, "test_mcc2": 0.996198, "test_mcc": 0.996198, "test_sn": 0.992593, "test_sp": 1.0, "lr": 3.005038629492778e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.026416887731420306, "train_cur_epoch_loss": 29.83691241541908, "train_cur_epoch_avg_loss": 0.0025014178752028066, "train_cur_epoch_time": 1864.3070178031921, "train_cur_epoch_avg_time": 0.15629669834030785, "epoch": 7, "step": 83496} ################################################## Training, Epoch: 0008, Batch: 000104, Sample Num: 1664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02638402, Time: 0.3045 Steps: 83600, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000304, Sample Num: 4864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02632106, Time: 0.1662 Steps: 83800, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000504, Sample Num: 8064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02625839, Time: 0.0956 Steps: 84000, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000704, Sample Num: 11264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02619601, Time: 0.1380 Steps: 84200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 000904, Sample Num: 14464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02613394, Time: 0.1042 Steps: 84400, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001104, Sample Num: 17664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000002, Global Avg Loss: 0.02607216, Time: 0.1755 Steps: 84600, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001304, Sample Num: 20864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02601067, Time: 0.2264 Steps: 84800, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001504, Sample Num: 24064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02594946, Time: 0.2433 Steps: 85000, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001704, Sample Num: 27264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000007, Log Avg loss: 0.00000055, Global Avg Loss: 0.02588855, Time: 0.2184 Steps: 85200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001904, Sample Num: 30464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000006, Log Avg loss: 0.00000000, Global Avg Loss: 0.02582792, Time: 0.0881 Steps: 85400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002104, Sample Num: 33664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00047940, Log Avg loss: 0.00504273, Global Avg Loss: 0.02577936, Time: 0.2124 Steps: 85600, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002304, Sample Num: 36864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00048007, Log Avg loss: 0.00048713, Global Avg Loss: 0.02572040, Time: 0.1250 Steps: 85800, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002504, Sample Num: 40064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00044173, Log Avg loss: 0.00000000, Global Avg Loss: 0.02566059, Time: 0.1974 Steps: 86000, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002704, Sample Num: 43264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00048152, Log Avg loss: 0.00097977, Global Avg Loss: 0.02560332, Time: 0.1081 Steps: 86200, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002904, Sample Num: 46464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00044836, Log Avg loss: 0.00000000, Global Avg Loss: 0.02554406, Time: 0.0932 Steps: 86400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 003104, Sample Num: 49664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00041947, Log Avg loss: 0.00000000, Global Avg Loss: 0.02548506, Time: 0.1084 Steps: 86600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003304, Sample Num: 52864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00039418, Log Avg loss: 0.00000169, Global Avg Loss: 0.02542635, Time: 0.1564 Steps: 86800, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003504, Sample Num: 56064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00037168, Log Avg loss: 0.00000000, Global Avg Loss: 0.02536789, Time: 0.1272 Steps: 87000, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003704, Sample Num: 59264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00045677, Log Avg loss: 0.00194742, Global Avg Loss: 0.02531418, Time: 0.2085 Steps: 87200, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003904, Sample Num: 62464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00043337, Log Avg loss: 0.00000005, Global Avg Loss: 0.02525625, Time: 0.3557 Steps: 87400, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 004104, Sample Num: 65664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00041225, Log Avg loss: 0.00000000, Global Avg Loss: 0.02519859, Time: 0.2477 Steps: 87600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 004304, Sample Num: 68864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00039309, Log Avg loss: 0.00000001, Global Avg Loss: 0.02514119, Time: 0.0702 Steps: 87800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004504, Sample Num: 72064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00037564, Log Avg loss: 0.00000000, Global Avg Loss: 0.02508405, Time: 0.2988 Steps: 88000, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004704, Sample Num: 75264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00035967, Log Avg loss: 0.00000001, Global Avg Loss: 0.02502717, Time: 0.2081 Steps: 88200, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004904, Sample Num: 78464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00034500, Log Avg loss: 0.00000000, Global Avg Loss: 0.02497055, Time: 0.1035 Steps: 88400, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005104, Sample Num: 81664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00033148, Log Avg loss: 0.00000000, Global Avg Loss: 0.02491418, Time: 0.1221 Steps: 88600, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005304, Sample Num: 84864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00031898, Log Avg loss: 0.00000000, Global Avg Loss: 0.02485807, Time: 0.1057 Steps: 88800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005504, Sample Num: 88064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00030739, Log Avg loss: 0.00000000, Global Avg Loss: 0.02480221, Time: 0.1416 Steps: 89000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 005704, Sample Num: 91264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00029661, Log Avg loss: 0.00000000, Global Avg Loss: 0.02474660, Time: 0.4040 Steps: 89200, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 005904, Sample Num: 94464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00028656, Log Avg loss: 0.00000000, Global Avg Loss: 0.02469123, Time: 0.1536 Steps: 89400, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006104, Sample Num: 97664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00027717, Log Avg loss: 0.00000000, Global Avg Loss: 0.02463612, Time: 0.1706 Steps: 89600, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006304, Sample Num: 100864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00026838, Log Avg loss: 0.00000000, Global Avg Loss: 0.02458125, Time: 0.0706 Steps: 89800, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006504, Sample Num: 104064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00026013, Log Avg loss: 0.00000000, Global Avg Loss: 0.02452663, Time: 0.1101 Steps: 90000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006704, Sample Num: 107264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00025237, Log Avg loss: 0.00000000, Global Avg Loss: 0.02447224, Time: 0.0985 Steps: 90200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 006904, Sample Num: 110464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00024506, Log Avg loss: 0.00000000, Global Avg Loss: 0.02441810, Time: 0.1966 Steps: 90400, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007104, Sample Num: 113664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00023816, Log Avg loss: 0.00000000, Global Avg Loss: 0.02436420, Time: 0.1331 Steps: 90600, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007304, Sample Num: 116864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00023164, Log Avg loss: 0.00000000, Global Avg Loss: 0.02431053, Time: 0.1221 Steps: 90800, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007504, Sample Num: 120064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00022546, Log Avg loss: 0.00000000, Global Avg Loss: 0.02425710, Time: 0.1566 Steps: 91000, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007704, Sample Num: 123264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00021961, Log Avg loss: 0.00000000, Global Avg Loss: 0.02420391, Time: 0.2792 Steps: 91200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007904, Sample Num: 126464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00021405, Log Avg loss: 0.00000000, Global Avg Loss: 0.02415094, Time: 0.1455 Steps: 91400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008104, Sample Num: 129664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00020877, Log Avg loss: 0.00000000, Global Avg Loss: 0.02409821, Time: 0.0643 Steps: 91600, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008304, Sample Num: 132864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00020374, Log Avg loss: 0.00000000, Global Avg Loss: 0.02404571, Time: 0.0907 Steps: 91800, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008504, Sample Num: 136064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00019905, Log Avg loss: 0.00000444, Global Avg Loss: 0.02399345, Time: 0.1710 Steps: 92000, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008704, Sample Num: 139264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00019448, Log Avg loss: 0.00000000, Global Avg Loss: 0.02394140, Time: 0.1824 Steps: 92200, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008904, Sample Num: 142464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00019011, Log Avg loss: 0.00000000, Global Avg Loss: 0.02388958, Time: 0.1338 Steps: 92400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 009104, Sample Num: 145664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00018594, Log Avg loss: 0.00000000, Global Avg Loss: 0.02383798, Time: 0.1196 Steps: 92600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009304, Sample Num: 148864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00018194, Log Avg loss: 0.00000000, Global Avg Loss: 0.02378661, Time: 0.1215 Steps: 92800, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009504, Sample Num: 152064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017811, Log Avg loss: 0.00000001, Global Avg Loss: 0.02373545, Time: 0.2551 Steps: 93000, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009704, Sample Num: 155264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017444, Log Avg loss: 0.00000000, Global Avg Loss: 0.02368452, Time: 0.2869 Steps: 93200, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009904, Sample Num: 158464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017092, Log Avg loss: 0.00000004, Global Avg Loss: 0.02363380, Time: 0.1445 Steps: 93400, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 010104, Sample Num: 161664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00016754, Log Avg loss: 0.00000008, Global Avg Loss: 0.02358330, Time: 0.0579 Steps: 93600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 010304, Sample Num: 164864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00016428, Log Avg loss: 0.00000000, Global Avg Loss: 0.02353302, Time: 0.1961 Steps: 93800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010504, Sample Num: 168064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00016116, Log Avg loss: 0.00000000, Global Avg Loss: 0.02348295, Time: 0.2097 Steps: 94000, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010704, Sample Num: 171264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015815, Log Avg loss: 0.00000000, Global Avg Loss: 0.02343309, Time: 0.0855 Steps: 94200, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010904, Sample Num: 174464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015524, Log Avg loss: 0.00000000, Global Avg Loss: 0.02338345, Time: 0.1122 Steps: 94400, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011104, Sample Num: 177664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015245, Log Avg loss: 0.00000000, Global Avg Loss: 0.02333401, Time: 0.1570 Steps: 94600, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011304, Sample Num: 180864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014975, Log Avg loss: 0.00000002, Global Avg Loss: 0.02328478, Time: 0.0952 Steps: 94800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011504, Sample Num: 184064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014715, Log Avg loss: 0.00000000, Global Avg Loss: 0.02323576, Time: 0.1953 Steps: 95000, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 011704, Sample Num: 187264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014463, Log Avg loss: 0.00000000, Global Avg Loss: 0.02318695, Time: 0.0765 Steps: 95200, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 011904, Sample Num: 190464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014220, Log Avg loss: 0.00000000, Global Avg Loss: 0.02313834, Time: 0.0974 Steps: 95400, Updated lr: 0.000020 ***** Running evaluation checkpoint-95424 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-95424 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1858.962072, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.064149, "eval_total_loss": 89.35926, "eval_acc": 0.999776, "eval_prec": 0.994455, "eval_recall": 0.996296, "eval_f1": 0.995375, "eval_roc_auc": 0.999996, "eval_pr_auc": 0.999857, "eval_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 2, "tp": 538}, "eval_mcc2": 0.99526, "eval_mcc": 0.99526, "eval_sn": 0.996296, "eval_sp": 0.999862, "update_flag": false, "test_avg_loss": 0.040678, "test_total_loss": 56.66451, "test_acc": 0.999865, "test_prec": 1.0, "test_recall": 0.994444, "test_f1": 0.997214, "test_roc_auc": 0.999999, "test_pr_auc": 0.999976, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 3, "tp": 537}, "test_mcc2": 0.99715, "test_mcc": 0.99715, "test_sn": 0.994444, "test_sp": 1.0, "lr": 2.003359086328519e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.02313251647565682, "train_cur_epoch_loss": 1.6927941504060868, "train_cur_epoch_avg_loss": 0.00014191768531238153, "train_cur_epoch_time": 1858.9620718955994, "train_cur_epoch_avg_time": 0.1558485975767605, "epoch": 8, "step": 95424} ################################################## Training, Epoch: 0009, Batch: 000176, Sample Num: 2816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02308993, Time: 0.1463 Steps: 95600, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000376, Sample Num: 6016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02304172, Time: 0.0932 Steps: 95800, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000576, Sample Num: 9216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02299372, Time: 0.3521 Steps: 96000, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000776, Sample Num: 12416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02294592, Time: 0.2841 Steps: 96200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 000976, Sample Num: 15616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02289831, Time: 0.0971 Steps: 96400, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001176, Sample Num: 18816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000115, Log Avg loss: 0.00000673, Global Avg Loss: 0.02285092, Time: 0.1369 Steps: 96600, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001376, Sample Num: 22016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000098, Log Avg loss: 0.00000003, Global Avg Loss: 0.02280370, Time: 0.1060 Steps: 96800, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001576, Sample Num: 25216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000088, Log Avg loss: 0.00000020, Global Avg Loss: 0.02275669, Time: 0.1347 Steps: 97000, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001776, Sample Num: 28416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000078, Log Avg loss: 0.00000000, Global Avg Loss: 0.02270986, Time: 0.4036 Steps: 97200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001976, Sample Num: 31616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000070, Log Avg loss: 0.00000000, Global Avg Loss: 0.02266323, Time: 0.1413 Steps: 97400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002176, Sample Num: 34816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00064913, Log Avg loss: 0.00705561, Global Avg Loss: 0.02263125, Time: 0.1173 Steps: 97600, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002376, Sample Num: 38016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00059449, Log Avg loss: 0.00000000, Global Avg Loss: 0.02258497, Time: 0.1681 Steps: 97800, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002576, Sample Num: 41216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00056759, Log Avg loss: 0.00024795, Global Avg Loss: 0.02253938, Time: 0.0961 Steps: 98000, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002776, Sample Num: 44416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00052670, Log Avg loss: 0.00000012, Global Avg Loss: 0.02249348, Time: 0.1304 Steps: 98200, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002976, Sample Num: 47616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00049131, Log Avg loss: 0.00000000, Global Avg Loss: 0.02244776, Time: 0.1062 Steps: 98400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 003176, Sample Num: 50816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00046037, Log Avg loss: 0.00000002, Global Avg Loss: 0.02240223, Time: 0.1097 Steps: 98600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003376, Sample Num: 54016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00043310, Log Avg loss: 0.00000000, Global Avg Loss: 0.02235688, Time: 0.0892 Steps: 98800, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003576, Sample Num: 57216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00040890, Log Avg loss: 0.00000052, Global Avg Loss: 0.02231171, Time: 0.1746 Steps: 99000, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003776, Sample Num: 60416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00038740, Log Avg loss: 0.00000283, Global Avg Loss: 0.02226673, Time: 0.2809 Steps: 99200, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003976, Sample Num: 63616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00036791, Log Avg loss: 0.00000000, Global Avg Loss: 0.02222193, Time: 0.3389 Steps: 99400, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 004176, Sample Num: 66816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00035029, Log Avg loss: 0.00000000, Global Avg Loss: 0.02217731, Time: 0.2050 Steps: 99600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 004376, Sample Num: 70016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00033428, Log Avg loss: 0.00000000, Global Avg Loss: 0.02213287, Time: 0.0868 Steps: 99800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004576, Sample Num: 73216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00031967, Log Avg loss: 0.00000000, Global Avg Loss: 0.02208860, Time: 0.1813 Steps: 100000, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004776, Sample Num: 76416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00030628, Log Avg loss: 0.00000000, Global Avg Loss: 0.02204451, Time: 0.0866 Steps: 100200, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004976, Sample Num: 79616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00029397, Log Avg loss: 0.00000000, Global Avg Loss: 0.02200060, Time: 0.1032 Steps: 100400, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005176, Sample Num: 82816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00028261, Log Avg loss: 0.00000000, Global Avg Loss: 0.02195686, Time: 0.2183 Steps: 100600, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005376, Sample Num: 86016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00027210, Log Avg loss: 0.00000000, Global Avg Loss: 0.02191329, Time: 0.1472 Steps: 100800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005576, Sample Num: 89216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00026234, Log Avg loss: 0.00000000, Global Avg Loss: 0.02186990, Time: 0.2221 Steps: 101000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 005776, Sample Num: 92416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00025326, Log Avg loss: 0.00000000, Global Avg Loss: 0.02182668, Time: 0.2126 Steps: 101200, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 005976, Sample Num: 95616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00024478, Log Avg loss: 0.00000000, Global Avg Loss: 0.02178363, Time: 0.0744 Steps: 101400, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006176, Sample Num: 98816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00023685, Log Avg loss: 0.00000000, Global Avg Loss: 0.02174075, Time: 0.1498 Steps: 101600, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006376, Sample Num: 102016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00022942, Log Avg loss: 0.00000000, Global Avg Loss: 0.02169804, Time: 0.1023 Steps: 101800, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006576, Sample Num: 105216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00022245, Log Avg loss: 0.00000000, Global Avg Loss: 0.02165549, Time: 0.0986 Steps: 102000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006776, Sample Num: 108416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00021588, Log Avg loss: 0.00000000, Global Avg Loss: 0.02161311, Time: 0.0879 Steps: 102200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 006976, Sample Num: 111616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00020969, Log Avg loss: 0.00000000, Global Avg Loss: 0.02157090, Time: 0.1547 Steps: 102400, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007176, Sample Num: 114816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00020385, Log Avg loss: 0.00000000, Global Avg Loss: 0.02152885, Time: 0.1591 Steps: 102600, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007376, Sample Num: 118016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00019832, Log Avg loss: 0.00000000, Global Avg Loss: 0.02148697, Time: 0.1311 Steps: 102800, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007576, Sample Num: 121216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00019308, Log Avg loss: 0.00000000, Global Avg Loss: 0.02144524, Time: 0.1160 Steps: 103000, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007776, Sample Num: 124416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00018812, Log Avg loss: 0.00000000, Global Avg Loss: 0.02140368, Time: 0.0914 Steps: 103200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007976, Sample Num: 127616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00018340, Log Avg loss: 0.00000000, Global Avg Loss: 0.02136228, Time: 0.1525 Steps: 103400, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008176, Sample Num: 130816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017891, Log Avg loss: 0.00000000, Global Avg Loss: 0.02132104, Time: 0.0740 Steps: 103600, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008376, Sample Num: 134016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017464, Log Avg loss: 0.00000001, Global Avg Loss: 0.02127996, Time: 0.4048 Steps: 103800, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008576, Sample Num: 137216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017057, Log Avg loss: 0.00000001, Global Avg Loss: 0.02123904, Time: 0.1636 Steps: 104000, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008776, Sample Num: 140416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00016668, Log Avg loss: 0.00000000, Global Avg Loss: 0.02119827, Time: 0.1245 Steps: 104200, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008976, Sample Num: 143616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00016297, Log Avg loss: 0.00000000, Global Avg Loss: 0.02115766, Time: 0.3467 Steps: 104400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009176, Sample Num: 146816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015942, Log Avg loss: 0.00000000, Global Avg Loss: 0.02111721, Time: 0.1273 Steps: 104600, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009376, Sample Num: 150016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015608, Log Avg loss: 0.00000285, Global Avg Loss: 0.02107691, Time: 0.1332 Steps: 104800, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009576, Sample Num: 153216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015282, Log Avg loss: 0.00000000, Global Avg Loss: 0.02103677, Time: 0.3558 Steps: 105000, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009776, Sample Num: 156416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014969, Log Avg loss: 0.00000000, Global Avg Loss: 0.02099677, Time: 0.1496 Steps: 105200, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009976, Sample Num: 159616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014669, Log Avg loss: 0.00000003, Global Avg Loss: 0.02095693, Time: 0.1629 Steps: 105400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 010176, Sample Num: 162816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014381, Log Avg loss: 0.00000000, Global Avg Loss: 0.02091724, Time: 0.0810 Steps: 105600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010376, Sample Num: 166016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014104, Log Avg loss: 0.00000001, Global Avg Loss: 0.02087770, Time: 0.3384 Steps: 105800, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010576, Sample Num: 169216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00013837, Log Avg loss: 0.00000000, Global Avg Loss: 0.02083831, Time: 0.4037 Steps: 106000, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010776, Sample Num: 172416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00013580, Log Avg loss: 0.00000001, Global Avg Loss: 0.02079906, Time: 0.0856 Steps: 106200, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010976, Sample Num: 175616, Cur Loss: 0.00000000, Cur Avg Loss: 0.00013333, Log Avg loss: 0.00000000, Global Avg Loss: 0.02075997, Time: 0.1309 Steps: 106400, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 011176, Sample Num: 178816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00013094, Log Avg loss: 0.00000000, Global Avg Loss: 0.02072102, Time: 0.1326 Steps: 106600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 011376, Sample Num: 182016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00012864, Log Avg loss: 0.00000000, Global Avg Loss: 0.02068222, Time: 0.1012 Steps: 106800, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 011576, Sample Num: 185216, Cur Loss: 0.00000000, Cur Avg Loss: 0.00012642, Log Avg loss: 0.00000000, Global Avg Loss: 0.02064356, Time: 0.1471 Steps: 107000, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 011776, Sample Num: 188416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00012427, Log Avg loss: 0.00000001, Global Avg Loss: 0.02060504, Time: 0.2761 Steps: 107200, Updated lr: 0.000010 ***** Running evaluation checkpoint-107352 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-107352 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1865.527787, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.062794, "eval_total_loss": 87.472095, "eval_acc": 0.999776, "eval_prec": 0.994455, "eval_recall": 0.996296, "eval_f1": 0.995375, "eval_roc_auc": 0.999997, "eval_pr_auc": 0.999867, "eval_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 2, "tp": 538}, "eval_mcc2": 0.99526, "eval_mcc": 0.99526, "eval_sn": 0.996296, "eval_sp": 0.999862, "update_flag": false, "test_avg_loss": 0.038565, "test_total_loss": 53.720608, "test_acc": 0.999865, "test_prec": 1.0, "test_recall": 0.994444, "test_f1": 0.997214, "test_roc_auc": 0.999999, "test_pr_auc": 0.99998, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 3, "tp": 537}, "test_mcc2": 0.99715, "test_mcc": 0.99715, "test_sn": 0.994444, "test_sp": 1.0, "lr": 1.0016795431642594e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.020575868603267454, "train_cur_epoch_loss": 1.4633941248906694, "train_cur_epoch_avg_loss": 0.0001226856241524706, "train_cur_epoch_time": 1865.5277872085571, "train_cur_epoch_avg_time": 0.15639904319320566, "epoch": 9, "step": 107352} ################################################## Training, Epoch: 0010, Batch: 000048, Sample Num: 768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02056667, Time: 0.1424 Steps: 107400, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000248, Sample Num: 3968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02052844, Time: 0.1202 Steps: 107600, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000448, Sample Num: 7168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02049036, Time: 0.1227 Steps: 107800, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000648, Sample Num: 10368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02045241, Time: 0.0762 Steps: 108000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 000848, Sample Num: 13568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000000, Log Avg loss: 0.00000000, Global Avg Loss: 0.02041461, Time: 0.0964 Steps: 108200, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001048, Sample Num: 16768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000469, Log Avg loss: 0.00002456, Global Avg Loss: 0.02037699, Time: 0.0958 Steps: 108400, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001248, Sample Num: 19968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000394, Log Avg loss: 0.00000000, Global Avg Loss: 0.02033946, Time: 0.2161 Steps: 108600, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001448, Sample Num: 23168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000339, Log Avg loss: 0.00000000, Global Avg Loss: 0.02030207, Time: 0.1146 Steps: 108800, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001648, Sample Num: 26368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000299, Log Avg loss: 0.00000006, Global Avg Loss: 0.02026482, Time: 0.0810 Steps: 109000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001848, Sample Num: 29568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000267, Log Avg loss: 0.00000000, Global Avg Loss: 0.02022771, Time: 0.1968 Steps: 109200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002048, Sample Num: 32768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00043604, Log Avg loss: 0.00444045, Global Avg Loss: 0.02019885, Time: 0.1095 Steps: 109400, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002248, Sample Num: 35968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00039868, Log Avg loss: 0.00001606, Global Avg Loss: 0.02016202, Time: 0.1126 Steps: 109600, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002448, Sample Num: 39168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00036611, Log Avg loss: 0.00000000, Global Avg Loss: 0.02012529, Time: 0.0662 Steps: 109800, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002648, Sample Num: 42368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00034072, Log Avg loss: 0.00002999, Global Avg Loss: 0.02008875, Time: 0.0790 Steps: 110000, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002848, Sample Num: 45568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00031679, Log Avg loss: 0.00000000, Global Avg Loss: 0.02005229, Time: 0.0803 Steps: 110200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 003048, Sample Num: 48768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00029601, Log Avg loss: 0.00000000, Global Avg Loss: 0.02001597, Time: 0.0868 Steps: 110400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003248, Sample Num: 51968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00027785, Log Avg loss: 0.00000109, Global Avg Loss: 0.01997977, Time: 0.4039 Steps: 110600, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003448, Sample Num: 55168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00026173, Log Avg loss: 0.00000000, Global Avg Loss: 0.01994371, Time: 0.1114 Steps: 110800, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003648, Sample Num: 58368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00024757, Log Avg loss: 0.00000351, Global Avg Loss: 0.01990778, Time: 0.1202 Steps: 111000, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003848, Sample Num: 61568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00023471, Log Avg loss: 0.00000001, Global Avg Loss: 0.01987198, Time: 0.1052 Steps: 111200, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 004048, Sample Num: 64768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00022311, Log Avg loss: 0.00000000, Global Avg Loss: 0.01983630, Time: 0.1138 Steps: 111400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 004248, Sample Num: 67968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00021261, Log Avg loss: 0.00000001, Global Avg Loss: 0.01980075, Time: 0.1102 Steps: 111600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004448, Sample Num: 71168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00020305, Log Avg loss: 0.00000000, Global Avg Loss: 0.01976533, Time: 0.2039 Steps: 111800, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004648, Sample Num: 74368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00019431, Log Avg loss: 0.00000000, Global Avg Loss: 0.01973003, Time: 0.1228 Steps: 112000, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004848, Sample Num: 77568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00018629, Log Avg loss: 0.00000000, Global Avg Loss: 0.01969486, Time: 0.2181 Steps: 112200, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005048, Sample Num: 80768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017891, Log Avg loss: 0.00000000, Global Avg Loss: 0.01965982, Time: 0.2051 Steps: 112400, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005248, Sample Num: 83968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00017209, Log Avg loss: 0.00000000, Global Avg Loss: 0.01962490, Time: 0.1166 Steps: 112600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005448, Sample Num: 87168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00016578, Log Avg loss: 0.00000000, Global Avg Loss: 0.01959010, Time: 0.4055 Steps: 112800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 005648, Sample Num: 90368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015991, Log Avg loss: 0.00000000, Global Avg Loss: 0.01955543, Time: 0.1183 Steps: 113000, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 005848, Sample Num: 93568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00015444, Log Avg loss: 0.00000000, Global Avg Loss: 0.01952088, Time: 0.0867 Steps: 113200, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006048, Sample Num: 96768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014933, Log Avg loss: 0.00000000, Global Avg Loss: 0.01948645, Time: 0.0654 Steps: 113400, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006248, Sample Num: 99968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014455, Log Avg loss: 0.00000000, Global Avg Loss: 0.01945215, Time: 0.1402 Steps: 113600, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006448, Sample Num: 103168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014007, Log Avg loss: 0.00000000, Global Avg Loss: 0.01941796, Time: 0.1249 Steps: 113800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006648, Sample Num: 106368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00013585, Log Avg loss: 0.00000000, Global Avg Loss: 0.01938389, Time: 0.2207 Steps: 114000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 006848, Sample Num: 109568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00013189, Log Avg loss: 0.00000000, Global Avg Loss: 0.01934995, Time: 0.3047 Steps: 114200, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007048, Sample Num: 112768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00012814, Log Avg loss: 0.00000000, Global Avg Loss: 0.01931612, Time: 0.1357 Steps: 114400, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007248, Sample Num: 115968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00012461, Log Avg loss: 0.00000000, Global Avg Loss: 0.01928241, Time: 0.0873 Steps: 114600, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007448, Sample Num: 119168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00012126, Log Avg loss: 0.00000000, Global Avg Loss: 0.01924881, Time: 0.1062 Steps: 114800, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007648, Sample Num: 122368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00011809, Log Avg loss: 0.00000000, Global Avg Loss: 0.01921534, Time: 0.2287 Steps: 115000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007848, Sample Num: 125568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00011508, Log Avg loss: 0.00000000, Global Avg Loss: 0.01918198, Time: 0.2534 Steps: 115200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008048, Sample Num: 128768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00011222, Log Avg loss: 0.00000000, Global Avg Loss: 0.01914873, Time: 0.1387 Steps: 115400, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008248, Sample Num: 131968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00010950, Log Avg loss: 0.00000000, Global Avg Loss: 0.01911560, Time: 0.1744 Steps: 115600, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008448, Sample Num: 135168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00010691, Log Avg loss: 0.00000000, Global Avg Loss: 0.01908259, Time: 0.4465 Steps: 115800, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008648, Sample Num: 138368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00010443, Log Avg loss: 0.00000000, Global Avg Loss: 0.01904969, Time: 0.0880 Steps: 116000, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008848, Sample Num: 141568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00010207, Log Avg loss: 0.00000000, Global Avg Loss: 0.01901690, Time: 0.1186 Steps: 116200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 009048, Sample Num: 144768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00009982, Log Avg loss: 0.00000000, Global Avg Loss: 0.01898423, Time: 0.0995 Steps: 116400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009248, Sample Num: 147968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00009766, Log Avg loss: 0.00000000, Global Avg Loss: 0.01895166, Time: 0.1025 Steps: 116600, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009448, Sample Num: 151168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00009559, Log Avg loss: 0.00000004, Global Avg Loss: 0.01891921, Time: 0.0908 Steps: 116800, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009648, Sample Num: 154368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00009361, Log Avg loss: 0.00000001, Global Avg Loss: 0.01888687, Time: 0.1252 Steps: 117000, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009848, Sample Num: 157568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00009171, Log Avg loss: 0.00000000, Global Avg Loss: 0.01885464, Time: 0.1298 Steps: 117200, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 010048, Sample Num: 160768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00008988, Log Avg loss: 0.00000000, Global Avg Loss: 0.01882252, Time: 0.3117 Steps: 117400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 010248, Sample Num: 163968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00008813, Log Avg loss: 0.00000000, Global Avg Loss: 0.01879051, Time: 0.0831 Steps: 117600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010448, Sample Num: 167168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00008644, Log Avg loss: 0.00000000, Global Avg Loss: 0.01875861, Time: 0.2818 Steps: 117800, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010648, Sample Num: 170368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00008482, Log Avg loss: 0.00000000, Global Avg Loss: 0.01872681, Time: 0.0740 Steps: 118000, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010848, Sample Num: 173568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00008326, Log Avg loss: 0.00000000, Global Avg Loss: 0.01869513, Time: 0.0871 Steps: 118200, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011048, Sample Num: 176768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00008175, Log Avg loss: 0.00000000, Global Avg Loss: 0.01866355, Time: 0.0811 Steps: 118400, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011248, Sample Num: 179968, Cur Loss: 0.00000000, Cur Avg Loss: 0.00008030, Log Avg loss: 0.00000001, Global Avg Loss: 0.01863207, Time: 0.2225 Steps: 118600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011448, Sample Num: 183168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00007889, Log Avg loss: 0.00000000, Global Avg Loss: 0.01860071, Time: 0.1802 Steps: 118800, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 011648, Sample Num: 186368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00007754, Log Avg loss: 0.00000000, Global Avg Loss: 0.01856944, Time: 0.0944 Steps: 119000, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 011848, Sample Num: 189568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00007623, Log Avg loss: 0.00000000, Global Avg Loss: 0.01853829, Time: 0.2251 Steps: 119200, Updated lr: 0.000000 ***** Running evaluation checkpoint-119280 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-119280 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 1866.868841, Avg time per batch (s): 0.160000 {"eval_avg_loss": 0.062093, "eval_total_loss": 86.494874, "eval_acc": 0.999776, "eval_prec": 0.994455, "eval_recall": 0.996296, "eval_f1": 0.995375, "eval_roc_auc": 0.999997, "eval_pr_auc": 0.999867, "eval_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 2, "tp": 538}, "eval_mcc2": 0.99526, "eval_mcc": 0.99526, "eval_sn": 0.996296, "eval_sp": 0.999862, "update_flag": false, "test_avg_loss": 0.037232, "test_total_loss": 51.864235, "test_acc": 0.999865, "test_prec": 1.0, "test_recall": 0.994444, "test_f1": 0.997214, "test_roc_auc": 0.999999, "test_pr_auc": 0.99998, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 3, "tp": 537}, "test_mcc2": 0.99715, "test_mcc": 0.99715, "test_sn": 0.994444, "test_sp": 1.0, "lr": 0.0, "cur_epoch_step": 11928, "train_global_avg_loss": 0.018525853555390468, "train_cur_epoch_loss": 0.9031657890069482, "train_cur_epoch_avg_loss": 7.571812449756441e-05, "train_cur_epoch_time": 1866.868840932846, "train_cur_epoch_avg_time": 0.15651147224453774, "epoch": 10, "step": 119280} ################################################## #########################Best Metric######################### {"epoch": 7, "global_step": 83496, "eval_avg_loss": 0.066045, "eval_total_loss": 92.001109, "eval_acc": 0.99982, "eval_prec": 0.998141, "eval_recall": 0.994444, "eval_f1": 0.996289, "eval_roc_auc": 0.999996, "eval_pr_auc": 0.999857, "eval_confusion_matrix": {"tn": 21743, "fp": 1, "fn": 3, "tp": 537}, "eval_mcc2": 0.996199, "eval_mcc": 0.996199, "eval_sn": 0.994444, "eval_sp": 0.999954, "update_flag": true, "test_avg_loss": 0.054296, "test_total_loss": 75.634388, "test_acc": 0.99982, "test_prec": 1.0, "test_recall": 0.992593, "test_f1": 0.996283, "test_roc_auc": 0.999999, "test_pr_auc": 0.999969, "test_confusion_matrix": {"tn": 21744, "fp": 0, "fn": 4, "tp": 536}, "test_mcc2": 0.996198, "test_mcc": 0.996198, "test_sn": 0.992593, "test_sp": 1.0} ################################################## Total Time: 127432.900497, Avg time per epoch(10 epochs): 12743.290000 ++++++++++++Validation+++++++++++++ best f1 global step: 83496 checkpoint path: ../models/RdRP/protein/binary_class/luca_base/matrix/20250329135221/checkpoint-83496 ***** Running evaluation checkpoint-83496 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## {"evaluation_avg_loss_83496": 0.066045, "evaluation_total_loss_83496": 92.001109, "evaluation_acc_83496": 0.99982, "evaluation_prec_83496": 0.998141, "evaluation_recall_83496": 0.994444, "evaluation_f1_83496": 0.996289, "evaluation_roc_auc_83496": 0.999996, "evaluation_pr_auc_83496": 0.999857, "evaluation_confusion_matrix_83496": {"tn": 21743, "fp": 1, "fn": 3, "tp": 537}, "evaluation_mcc2_83496": 0.996199, "evaluation_mcc_83496": 0.996199, "evaluation_sn_83496": 0.994444, "evaluation_sp_83496": 0.999954} ++++++++++++Testing+++++++++++++ best f1 global step: 83496 checkpoint path: ../models/RdRP/protein/binary_class/luca_base/matrix/20250329135221/checkpoint-83496 ***** Running testing checkpoint-83496 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## {"evaluation_avg_loss_83496": 0.054296, "evaluation_total_loss_83496": 75.634388, "evaluation_acc_83496": 0.99982, "evaluation_prec_83496": 1.0, "evaluation_recall_83496": 0.992593, "evaluation_f1_83496": 0.996283, "evaluation_roc_auc_83496": 0.999999, "evaluation_pr_auc_83496": 0.999969, "evaluation_confusion_matrix_83496": {"tn": 21744, "fp": 0, "fn": 4, "tp": 536}, "evaluation_mcc2_83496": 0.996198, "evaluation_mcc_83496": 0.996198, "evaluation_sn_83496": 0.992593, "evaluation_sp_83496": 1.0}