{"adam_epsilon": 1e-08, "alphabet": "gene_prot", "append_eos": true, "asl_gamma_neg": 4.0, "asl_gamma_pos": 1.0, "best_metric_type": "f1", "beta1": 0.9, "beta2": 0.98, "buffer_size": 1024, "cache_dir": null, "classifier_activate_func": "gelu", "classifier_size": 128, "codes_file": null, "config_path": "../config/luca_base/luca_base_config.json", "cross_atten": false, "dataset_name": "RdRP", "dataset_type": "protein", "delete_old": false, "dev_data_dir": "../dataset/RdRP/protein/binary_class/dev/", "device": "cuda", "do_eval": true, "do_lower_case": false, "do_metrics": true, "do_predict": true, "do_train": true, "dropout_prob": 0.1, "early_stop_epoch": -1, "emb_activate_func": "gelu", "embedding_complete": true, "embedding_complete_seg_overlap": true, "embedding_fixed_len_a_time": 3072, "embedding_input_size": 1152, "embedding_input_size_a": null, "embedding_input_size_b": null, "eval_all_checkpoints": false, "evaluate_during_training": true, "evaluate_steps": -1, "evaluate_strategy": "epoch", "fc_activate_func": "gelu", "focal_loss_alpha": 0.7, "focal_loss_gamma": 2.0, "focal_loss_reduce": false, "fp16": false, "fp16_embedding": false, "fp16_opt_level": "O1", "fusion_type": "concat", "gradient_accumulation_steps": 1, "hidden_size": 1024, "ignore_index": -100, "input_mode": "single", "input_type": "seq_matrix", "intermediate_size": 4096, "label_filepath": "../dataset/RdRP/protein/binary_class/label.txt", "label_size": 2, "label_type": "RdRP", "learning_rate": 0.0001, "llm_dir": "..", "llm_dirpath": null, "llm_step": null, "llm_task_level": "token_level,span_level,seq_level,structure_level", "llm_time_str": null, "llm_type": "esmc", "llm_version": "600M", "lmdb_path": null, "local_rank": -1, "log_dir": "../logs/RdRP/protein/binary_class/luca_base/seq_matrix/20250415141827", "logging_steps": 200, "loss_reduction": "mean", "loss_type": "bce", "lr_decay_rate": 0.9, "lr_update_strategy": "step", "matrix_add_special_token": false, "matrix_dirpath": "../matrices/RdRP/protein/binary_class/luca_base/600M/esmc//", "matrix_embedding_exists": false, "matrix_encoder": false, "matrix_encoder_act": false, "matrix_fc_size": "128", "matrix_max_length": 4096, "matrix_max_length_a": null, "matrix_max_length_b": null, "matrix_pooling_type": "value_attention", "max_grad_norm": 1.0, "max_sentence_length": null, "max_sentences": null, "max_steps": -1, "model_dirpath": null, "model_type": "luca_base", "n_gpu": 1, "no_cuda": false, "no_position_embeddings": true, "no_token_embeddings": false, "no_token_type_embeddings": true, "non_ignore": false, "not_append_eos": false, "not_matrix_encoder_shared": false, "not_prepend_bos": false, "not_save_emb_to_disk": false, "not_seq_encoder_shared": false, "num_attention_heads": 4, "num_hidden_layers": 2, "num_train_epochs": 10, "output_dir": "../models/RdRP/protein/binary_class/luca_base/seq_matrix/20250415141827", "output_mode": "binary_class", "overwrite_cache": false, "overwrite_output_dir": true, "per_gpu_eval_batch_size": 16, "per_gpu_train_batch_size": 16, "pos_weight": 40.0, "position_embedding_type": "absolute", "prepend_bos": true, "save_all": true, "save_steps": -1, "seed": 1221, "self_atten": false, "seq_fc_size": "128", "seq_max_length": 4096, "seq_max_length_a": null, "seq_max_length_b": null, "seq_pooling_type": "value_attention", "seq_subword": false, "seq_vocab_path": "gene_prot", "sigmoid": true, "task_level_type": "seq_level", "task_type": "binary_class", "tb_log_dir": "../tb-logs/RdRP/protein/binary_class/luca_base/seq_matrix/20250415141827", "test_data_dir": "../dataset/RdRP/protein/binary_class/test/", "time_str": "20250415141831", "train_data_dir": "../dataset/RdRP/protein/binary_class/train/", "trunc_type": "right", "vector_dirpath": "../vectors/RdRP/protein/binary_class/luca_base/600M/esmc//", "vector_fc_size": "null", "vocab_size": 39, "warmup_steps": 200, "weight": null, "weight_decay": 0.01, "worker_num": 0} ################################################## n_gpu: 1 ################################################## Inputs: Input Name List: protein,seq,embedding_matrix ################################################## Encoder Config: {'llm_type': 'esmc', 'llm_version': '600M', 'llm_step': None, 'llm_dirpath': None, 'input_type': 'seq_matrix', 'trunc_type': 'right', 'seq_max_length': 4096, 'atom_seq_max_length': None, 'vector_dirpath': '../vectors/RdRP/protein/binary_class/luca_base/600M/esmc//', 'matrix_dirpath': '../matrices/RdRP/protein/binary_class/luca_base/600M/esmc//', 'local_rank': -1, 'max_sentence_length': None, 'max_sentences': None, 'matrix_add_special_token': False, 'embedding_complete': True, 'embedding_complete_seg_overlap': True, 'embedding_fixed_len_a_time': 3072, 'matrix_embedding_exists': False, 'save_emb_to_disk': True, 'fp16_embedding': False} ################################################## Model Config: LucaConfig { "_attn_implementation_autoset": true, "alphabet": "gene_prot", "attention_probs_dropout_prob": 0.1, "classifier_activate_func": "gelu", "classifier_dropout_prob": 0.1, "classifier_size": 128, "cls_token_id": 2, "cross_atten": false, "directionality": "bidi", "emb_activate_func": "gelu", "embedding_input_size": 1152, "fc_activate_func": "gelu", "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 1024, "ignore_index": -100, "initializer_range": 0.02, "intermediate_size": 4096, "kernel_size": 7, "layer_norm_eps": 1e-12, "loss_reduction": "mean", "matrix_fc_size": [ 128 ], "matrix_max_length": 4096, "matrix_pooling_type": "value_attention", "max_position_embeddings": 4096, "no_position_embeddings": true, "no_token_embeddings": false, "no_token_type_embeddings": true, "num_attention_heads": 4, "num_hidden_layers": 2, "pad_token_id": 0, "pos_weight": 40.0, "position_embedding_type": "absolute", "self_atten": false, "sep_token_id": 3, "seq_fc_size": [ 128 ], "seq_max_length": 4096, "seq_pooling_type": "value_attention", "token_dropout": null, "transformers_version": "4.46.3", "type_vocab_size": 2, "use_luca_layer_norm_v2": true, "vector_fc_size": null, "vocab_size": 39 } ################################################## Mode Architecture: LucaBase( (seq_encoder): LucaTransformer( (embeddings): LucaEmbeddings( (word_embeddings): Embedding(39, 1024, padding_idx=0) (dropout): Dropout(p=0.1, inplace=False) ) (encoder): ModuleList( (0-1): 2 x LucaTransformerLayer( (pre_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) (self_attn): LucaMultiHeadAttention( (k_proj): Linear(in_features=1024, out_features=1024, bias=True) (v_proj): Linear(in_features=1024, out_features=1024, bias=True) (q_proj): Linear(in_features=1024, out_features=1024, bias=True) (out_proj): Linear(in_features=1024, out_features=1024, bias=True) (rot_emb): RotaryEmbedding() ) (post_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) (fc1): Linear(in_features=1024, out_features=4096, bias=True) (fc2): Linear(in_features=4096, out_features=1024, bias=True) ) ) (last_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) ) (seq_pooler): GlobalMaskValueAttentionPooling1D (1024 -> 1024) (matrix_pooler): GlobalMaskValueAttentionPooling1D (1152 -> 1152) (linear): ModuleList( (0): ModuleList( (0): Linear(in_features=1024, out_features=128, bias=True) (1): GELU(approximate='none') ) (1): ModuleList( (0): Linear(in_features=1152, out_features=128, bias=True) (1): GELU(approximate='none') ) ) (dropout): Dropout(p=0.1, inplace=False) (hidden_layer): Linear(in_features=256, out_features=128, bias=True) (hidden_act): GELU(approximate='none') (classifier): Linear(in_features=128, out_features=1, bias=True) (output): Sigmoid() (loss_fct): MaskedBCEWithLogitsLoss( (criterion): BCEWithLogitsLoss() ) ) ################################################## Model parameters: 32673281 ################################################## {"total_num": "31.160000M", "total_size": "124.640000MB", "param_sum": "31.160000M", "param_size": "124.640000MB", "buffer_sum": "0.000000M", "buffer_size": "0.000000MB", "trainable_num": "31.159669M", "trainable_size": "124.638676MB"} ################################################## Train dataset len: 190846, batch size: 16, batch num: 11928 Train dataset t_total: 119280, max_steps: -1 ***** Running training ***** Train Dataset Num examples = 190846 Train Dataset Num Epochs = 10 Logging Steps = 200 Saving Steps = -1 Evaluating Strategy = epoch Train Dataset Instantaneous batch size per GPU = 16 Train Dataset Total train batch size (w. parallel, distributed & accumulation) = 16 Train Dataset Gradient Accumulation steps = 1 Train Dataset Total optimization steps = 119280 ################################################## Training, Epoch: 0001, Batch: 000200, Sample Num: 3200, Cur Loss: 0.01474746, Cur Avg Loss: 1.64773840, Log Avg loss: 1.64773840, Global Avg Loss: 1.64773840, Time: 0.1872 Steps: 200, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000400, Sample Num: 6400, Cur Loss: 0.00852779, Cur Avg Loss: 2.12392889, Log Avg loss: 2.60011937, Global Avg Loss: 2.12392889, Time: 0.2504 Steps: 400, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000600, Sample Num: 9600, Cur Loss: 11.23127174, Cur Avg Loss: 2.05278935, Log Avg loss: 1.91051028, Global Avg Loss: 2.05278935, Time: 0.6151 Steps: 600, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000800, Sample Num: 12800, Cur Loss: 0.27513665, Cur Avg Loss: 1.95115059, Log Avg loss: 1.64623429, Global Avg Loss: 1.95115059, Time: 0.6104 Steps: 800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001000, Sample Num: 16000, Cur Loss: 0.00064135, Cur Avg Loss: 1.72174180, Log Avg loss: 0.80410667, Global Avg Loss: 1.72174180, Time: 0.3857 Steps: 1000, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001200, Sample Num: 19200, Cur Loss: 0.00102410, Cur Avg Loss: 1.48677646, Log Avg loss: 0.31194977, Global Avg Loss: 1.48677646, Time: 0.2485 Steps: 1200, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001400, Sample Num: 22400, Cur Loss: 0.02107602, Cur Avg Loss: 1.28917519, Log Avg loss: 0.10356752, Global Avg Loss: 1.28917519, Time: 0.1595 Steps: 1400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001600, Sample Num: 25600, Cur Loss: 0.00003135, Cur Avg Loss: 1.14696505, Log Avg loss: 0.15149411, Global Avg Loss: 1.14696505, Time: 0.3425 Steps: 1600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001800, Sample Num: 28800, Cur Loss: 0.00088066, Cur Avg Loss: 1.03213541, Log Avg loss: 0.11349825, Global Avg Loss: 1.03213541, Time: 0.1872 Steps: 1800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002000, Sample Num: 32000, Cur Loss: 0.02804235, Cur Avg Loss: 0.93343898, Log Avg loss: 0.04517118, Global Avg Loss: 0.93343898, Time: 0.2705 Steps: 2000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002200, Sample Num: 35200, Cur Loss: 0.11300424, Cur Avg Loss: 0.85077901, Log Avg loss: 0.02417923, Global Avg Loss: 0.85077901, Time: 0.3230 Steps: 2200, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002400, Sample Num: 38400, Cur Loss: 0.00021130, Cur Avg Loss: 0.78330434, Log Avg loss: 0.04108297, Global Avg Loss: 0.78330434, Time: 0.1184 Steps: 2400, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002600, Sample Num: 41600, Cur Loss: 0.00000006, Cur Avg Loss: 0.72347448, Log Avg loss: 0.00551617, Global Avg Loss: 0.72347448, Time: 0.3034 Steps: 2600, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 002800, Sample Num: 44800, Cur Loss: 0.00000095, Cur Avg Loss: 0.67548212, Log Avg loss: 0.05158147, Global Avg Loss: 0.67548212, Time: 0.3002 Steps: 2800, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003000, Sample Num: 48000, Cur Loss: 0.00000024, Cur Avg Loss: 0.63097690, Log Avg loss: 0.00790383, Global Avg Loss: 0.63097690, Time: 0.2990 Steps: 3000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003200, Sample Num: 51200, Cur Loss: 0.00001389, Cur Avg Loss: 0.59304711, Log Avg loss: 0.02410033, Global Avg Loss: 0.59304711, Time: 1.2526 Steps: 3200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003400, Sample Num: 54400, Cur Loss: 0.03870186, Cur Avg Loss: 0.55938129, Log Avg loss: 0.02072814, Global Avg Loss: 0.55938129, Time: 0.4397 Steps: 3400, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003600, Sample Num: 57600, Cur Loss: 0.00000000, Cur Avg Loss: 0.52879146, Log Avg loss: 0.00876427, Global Avg Loss: 0.52879146, Time: 0.2933 Steps: 3600, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 003800, Sample Num: 60800, Cur Loss: 0.00000924, Cur Avg Loss: 0.50162474, Log Avg loss: 0.01262377, Global Avg Loss: 0.50162474, Time: 0.1893 Steps: 3800, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004000, Sample Num: 64000, Cur Loss: 0.00000006, Cur Avg Loss: 0.47748874, Log Avg loss: 0.01890479, Global Avg Loss: 0.47748874, Time: 0.1801 Steps: 4000, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004200, Sample Num: 67200, Cur Loss: 0.00000000, Cur Avg Loss: 0.45516737, Log Avg loss: 0.00873994, Global Avg Loss: 0.45516737, Time: 0.2622 Steps: 4200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 004400, Sample Num: 70400, Cur Loss: 0.01025162, Cur Avg Loss: 0.43583692, Log Avg loss: 0.02989744, Global Avg Loss: 0.43583692, Time: 0.5952 Steps: 4400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 004600, Sample Num: 73600, Cur Loss: 0.00000000, Cur Avg Loss: 0.41711027, Log Avg loss: 0.00512405, Global Avg Loss: 0.41711027, Time: 0.3105 Steps: 4600, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 004800, Sample Num: 76800, Cur Loss: 0.00000000, Cur Avg Loss: 0.40006968, Log Avg loss: 0.00813615, Global Avg Loss: 0.40006968, Time: 0.2375 Steps: 4800, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005000, Sample Num: 80000, Cur Loss: 0.00000000, Cur Avg Loss: 0.38982205, Log Avg loss: 0.14387899, Global Avg Loss: 0.38982205, Time: 0.5614 Steps: 5000, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005200, Sample Num: 83200, Cur Loss: 0.00000212, Cur Avg Loss: 0.37564742, Log Avg loss: 0.02128147, Global Avg Loss: 0.37564742, Time: 0.3365 Steps: 5200, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005400, Sample Num: 86400, Cur Loss: 0.00000000, Cur Avg Loss: 0.36626950, Log Avg loss: 0.12244358, Global Avg Loss: 0.36626950, Time: 0.2283 Steps: 5400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 005600, Sample Num: 89600, Cur Loss: 0.00000000, Cur Avg Loss: 0.35323204, Log Avg loss: 0.00122060, Global Avg Loss: 0.35323204, Time: 0.2926 Steps: 5600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 005800, Sample Num: 92800, Cur Loss: 0.00002375, Cur Avg Loss: 0.34107601, Log Avg loss: 0.00070734, Global Avg Loss: 0.34107601, Time: 0.4952 Steps: 5800, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006000, Sample Num: 96000, Cur Loss: 0.00011243, Cur Avg Loss: 0.33033814, Log Avg loss: 0.01893985, Global Avg Loss: 0.33033814, Time: 0.5066 Steps: 6000, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006200, Sample Num: 99200, Cur Loss: 0.00000000, Cur Avg Loss: 0.32364054, Log Avg loss: 0.12271272, Global Avg Loss: 0.32364054, Time: 0.2075 Steps: 6200, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006400, Sample Num: 102400, Cur Loss: 0.00000000, Cur Avg Loss: 0.31357179, Log Avg loss: 0.00144044, Global Avg Loss: 0.31357179, Time: 0.2548 Steps: 6400, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006600, Sample Num: 105600, Cur Loss: 0.00001866, Cur Avg Loss: 0.30413027, Log Avg loss: 0.00200172, Global Avg Loss: 0.30413027, Time: 1.1526 Steps: 6600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 006800, Sample Num: 108800, Cur Loss: 0.00000054, Cur Avg Loss: 0.29530078, Log Avg loss: 0.00392735, Global Avg Loss: 0.29530078, Time: 0.5511 Steps: 6800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007000, Sample Num: 112000, Cur Loss: 0.00000000, Cur Avg Loss: 0.28693473, Log Avg loss: 0.00248910, Global Avg Loss: 0.28693473, Time: 0.2286 Steps: 7000, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007200, Sample Num: 115200, Cur Loss: 0.00000000, Cur Avg Loss: 0.28295362, Log Avg loss: 0.14361474, Global Avg Loss: 0.28295362, Time: 0.2566 Steps: 7200, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007400, Sample Num: 118400, Cur Loss: 0.00000000, Cur Avg Loss: 0.27530858, Log Avg loss: 0.00008732, Global Avg Loss: 0.27530858, Time: 0.2505 Steps: 7400, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007600, Sample Num: 121600, Cur Loss: 0.00000143, Cur Avg Loss: 0.26944488, Log Avg loss: 0.05248769, Global Avg Loss: 0.26944488, Time: 0.5208 Steps: 7600, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 007800, Sample Num: 124800, Cur Loss: 0.00000047, Cur Avg Loss: 0.26254340, Log Avg loss: 0.00028729, Global Avg Loss: 0.26254340, Time: 0.1892 Steps: 7800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 008000, Sample Num: 128000, Cur Loss: 0.00000063, Cur Avg Loss: 0.25598813, Log Avg loss: 0.00033280, Global Avg Loss: 0.25598813, Time: 0.6713 Steps: 8000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008200, Sample Num: 131200, Cur Loss: 0.00000000, Cur Avg Loss: 0.24976062, Log Avg loss: 0.00066008, Global Avg Loss: 0.24976062, Time: 0.5973 Steps: 8200, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008400, Sample Num: 134400, Cur Loss: 0.00000032, Cur Avg Loss: 0.24651049, Log Avg loss: 0.11325498, Global Avg Loss: 0.24651049, Time: 0.5999 Steps: 8400, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008600, Sample Num: 137600, Cur Loss: 0.00002490, Cur Avg Loss: 0.24442535, Log Avg loss: 0.15684947, Global Avg Loss: 0.24442535, Time: 0.9037 Steps: 8600, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 008800, Sample Num: 140800, Cur Loss: 0.00000000, Cur Avg Loss: 0.23936723, Log Avg loss: 0.02186823, Global Avg Loss: 0.23936723, Time: 0.1896 Steps: 8800, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 009000, Sample Num: 144000, Cur Loss: 0.00000000, Cur Avg Loss: 0.23413277, Log Avg loss: 0.00381663, Global Avg Loss: 0.23413277, Time: 0.2190 Steps: 9000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 009200, Sample Num: 147200, Cur Loss: 0.00000009, Cur Avg Loss: 0.23144946, Log Avg loss: 0.11070049, Global Avg Loss: 0.23144946, Time: 0.3287 Steps: 9200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009400, Sample Num: 150400, Cur Loss: 0.00000000, Cur Avg Loss: 0.23175094, Log Avg loss: 0.24561895, Global Avg Loss: 0.23175094, Time: 0.2932 Steps: 9400, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009600, Sample Num: 153600, Cur Loss: 0.00000073, Cur Avg Loss: 0.22694030, Log Avg loss: 0.00083999, Global Avg Loss: 0.22694030, Time: 0.2380 Steps: 9600, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 009800, Sample Num: 156800, Cur Loss: 0.00000051, Cur Avg Loss: 0.22526913, Log Avg loss: 0.14505330, Global Avg Loss: 0.22526913, Time: 0.5240 Steps: 9800, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010000, Sample Num: 160000, Cur Loss: 0.00000000, Cur Avg Loss: 0.22092211, Log Avg loss: 0.00791792, Global Avg Loss: 0.22092211, Time: 0.2477 Steps: 10000, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010200, Sample Num: 163200, Cur Loss: 0.00000000, Cur Avg Loss: 0.21666662, Log Avg loss: 0.00389199, Global Avg Loss: 0.21666662, Time: 0.2580 Steps: 10200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 010400, Sample Num: 166400, Cur Loss: 0.00000054, Cur Avg Loss: 0.21250161, Log Avg loss: 0.00008613, Global Avg Loss: 0.21250161, Time: 0.5988 Steps: 10400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 010600, Sample Num: 169600, Cur Loss: 0.00000167, Cur Avg Loss: 0.20852870, Log Avg loss: 0.00193739, Global Avg Loss: 0.20852870, Time: 0.1555 Steps: 10600, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 010800, Sample Num: 172800, Cur Loss: 0.00000035, Cur Avg Loss: 0.20466812, Log Avg loss: 0.00005765, Global Avg Loss: 0.20466812, Time: 0.4284 Steps: 10800, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011000, Sample Num: 176000, Cur Loss: 0.00000000, Cur Avg Loss: 0.20096521, Log Avg loss: 0.00100798, Global Avg Loss: 0.20096521, Time: 0.2178 Steps: 11000, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011200, Sample Num: 179200, Cur Loss: 0.00000209, Cur Avg Loss: 0.19748638, Log Avg loss: 0.00615093, Global Avg Loss: 0.19748638, Time: 0.2708 Steps: 11200, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011400, Sample Num: 182400, Cur Loss: 0.00000006, Cur Avg Loss: 0.19404834, Log Avg loss: 0.00151779, Global Avg Loss: 0.19404834, Time: 0.4624 Steps: 11400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 011600, Sample Num: 185600, Cur Loss: 0.00000040, Cur Avg Loss: 0.19131078, Log Avg loss: 0.03527009, Global Avg Loss: 0.19131078, Time: 0.2534 Steps: 11600, Updated lr: 0.000090 Training, Epoch: 0001, Batch: 011800, Sample Num: 188800, Cur Loss: 0.00000000, Cur Avg Loss: 0.18830393, Log Avg loss: 0.01390650, Global Avg Loss: 0.18830393, Time: 0.9074 Steps: 11800, Updated lr: 0.000090 ***** Running evaluation checkpoint-11928 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-11928 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4891.307250, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.001592, "eval_total_loss": 2.217617, "eval_acc": 0.999731, "eval_prec": 0.989011, "eval_recall": 1.0, "eval_f1": 0.994475, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999928, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 0, "tp": 540}, "eval_mcc2": 0.994353, "eval_mcc": 0.994353, "eval_sn": 1.0, "eval_sp": 0.999724, "update_flag": true, "test_avg_loss": 0.001329, "test_total_loss": 1.85087, "test_acc": 0.999731, "test_prec": 0.989011, "test_recall": 1.0, "test_f1": 0.994475, "test_roc_auc": 0.999999, "test_pr_auc": 0.99998, "test_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 0, "tp": 540}, "test_mcc2": 0.994353, "test_mcc": 0.994353, "test_sn": 1.0, "test_sp": 0.999724, "lr": 9.015115888478334e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.18632415336853556, "train_cur_epoch_loss": 2222.474501379892, "train_cur_epoch_avg_loss": 0.18632415336853556, "train_cur_epoch_time": 4891.307250499725, "train_cur_epoch_avg_time": 0.4100693536636255, "epoch": 1, "step": 11928} ################################################## Training, Epoch: 0002, Batch: 000072, Sample Num: 1152, Cur Loss: 0.00000055, Cur Avg Loss: 0.00016857, Log Avg loss: 0.00250136, Global Avg Loss: 0.18520722, Time: 0.2001 Steps: 12000, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000272, Sample Num: 4352, Cur Loss: 0.00000006, Cur Avg Loss: 0.03375824, Log Avg loss: 0.04585052, Global Avg Loss: 0.18292268, Time: 0.5649 Steps: 12200, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000472, Sample Num: 7552, Cur Loss: 0.00000242, Cur Avg Loss: 0.03093696, Log Avg loss: 0.02710002, Global Avg Loss: 0.18040942, Time: 0.4624 Steps: 12400, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000672, Sample Num: 10752, Cur Loss: 0.00000002, Cur Avg Loss: 0.02209979, Log Avg loss: 0.00124405, Global Avg Loss: 0.17756552, Time: 0.5662 Steps: 12600, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000872, Sample Num: 13952, Cur Loss: 0.00000000, Cur Avg Loss: 0.01705064, Log Avg loss: 0.00008551, Global Avg Loss: 0.17479240, Time: 0.3136 Steps: 12800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001072, Sample Num: 17152, Cur Loss: 0.01721842, Cur Avg Loss: 0.06771999, Log Avg loss: 0.28863834, Global Avg Loss: 0.17654387, Time: 0.2618 Steps: 13000, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001272, Sample Num: 20352, Cur Loss: 0.00000000, Cur Avg Loss: 0.05992018, Log Avg loss: 0.01811320, Global Avg Loss: 0.17414341, Time: 0.4762 Steps: 13200, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001472, Sample Num: 23552, Cur Loss: 0.00000000, Cur Avg Loss: 0.05228203, Log Avg loss: 0.00370343, Global Avg Loss: 0.17159953, Time: 0.2715 Steps: 13400, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001672, Sample Num: 26752, Cur Loss: 0.00000000, Cur Avg Loss: 0.04619180, Log Avg loss: 0.00136771, Global Avg Loss: 0.16909612, Time: 0.3032 Steps: 13600, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001872, Sample Num: 29952, Cur Loss: 0.00000000, Cur Avg Loss: 0.04136628, Log Avg loss: 0.00102487, Global Avg Loss: 0.16666030, Time: 0.3807 Steps: 13800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002072, Sample Num: 33152, Cur Loss: 0.00003082, Cur Avg Loss: 0.03784644, Log Avg loss: 0.00490077, Global Avg Loss: 0.16434945, Time: 0.1918 Steps: 14000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002272, Sample Num: 36352, Cur Loss: 0.00000055, Cur Avg Loss: 0.03473258, Log Avg loss: 0.00247304, Global Avg Loss: 0.16206950, Time: 0.5608 Steps: 14200, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002472, Sample Num: 39552, Cur Loss: 0.00069934, Cur Avg Loss: 0.03234332, Log Avg loss: 0.00520134, Global Avg Loss: 0.15989078, Time: 0.2841 Steps: 14400, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002672, Sample Num: 42752, Cur Loss: 0.00000000, Cur Avg Loss: 0.04009911, Log Avg loss: 0.13596059, Global Avg Loss: 0.15956297, Time: 0.2008 Steps: 14600, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 002872, Sample Num: 45952, Cur Loss: 0.00000000, Cur Avg Loss: 0.03735585, Log Avg loss: 0.00070593, Global Avg Loss: 0.15741625, Time: 0.2480 Steps: 14800, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003072, Sample Num: 49152, Cur Loss: 0.00000000, Cur Avg Loss: 0.03509078, Log Avg loss: 0.00256436, Global Avg Loss: 0.15535156, Time: 0.2016 Steps: 15000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003272, Sample Num: 52352, Cur Loss: 0.00000000, Cur Avg Loss: 0.03294619, Log Avg loss: 0.00000524, Global Avg Loss: 0.15330753, Time: 0.4283 Steps: 15200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003472, Sample Num: 55552, Cur Loss: 0.00000000, Cur Avg Loss: 0.03132261, Log Avg loss: 0.00476096, Global Avg Loss: 0.15137835, Time: 0.1885 Steps: 15400, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003672, Sample Num: 58752, Cur Loss: 0.00000000, Cur Avg Loss: 0.02977239, Log Avg loss: 0.00286047, Global Avg Loss: 0.14947428, Time: 0.3208 Steps: 15600, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 003872, Sample Num: 61952, Cur Loss: 0.00000015, Cur Avg Loss: 0.03193967, Log Avg loss: 0.07173090, Global Avg Loss: 0.14849018, Time: 0.6362 Steps: 15800, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004072, Sample Num: 65152, Cur Loss: 0.00000027, Cur Avg Loss: 0.03048051, Log Avg loss: 0.00223119, Global Avg Loss: 0.14666195, Time: 0.2469 Steps: 16000, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004272, Sample Num: 68352, Cur Loss: 0.00000205, Cur Avg Loss: 0.03028884, Log Avg loss: 0.02638651, Global Avg Loss: 0.14517706, Time: 0.7913 Steps: 16200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 004472, Sample Num: 71552, Cur Loss: 0.00000137, Cur Avg Loss: 0.02999254, Log Avg loss: 0.02366346, Global Avg Loss: 0.14369519, Time: 0.6183 Steps: 16400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 004672, Sample Num: 74752, Cur Loss: 0.00000000, Cur Avg Loss: 0.02870868, Log Avg loss: 0.00000165, Global Avg Loss: 0.14196394, Time: 0.1841 Steps: 16600, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 004872, Sample Num: 77952, Cur Loss: 0.00000006, Cur Avg Loss: 0.02963904, Log Avg loss: 0.05137218, Global Avg Loss: 0.14088547, Time: 0.2894 Steps: 16800, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005072, Sample Num: 81152, Cur Loss: 0.00000000, Cur Avg Loss: 0.02854979, Log Avg loss: 0.00201575, Global Avg Loss: 0.13925171, Time: 0.2619 Steps: 17000, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005272, Sample Num: 84352, Cur Loss: 0.00000000, Cur Avg Loss: 0.03108807, Log Avg loss: 0.09545892, Global Avg Loss: 0.13874249, Time: 0.3238 Steps: 17200, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005472, Sample Num: 87552, Cur Loss: 0.00000031, Cur Avg Loss: 0.03041413, Log Avg loss: 0.01264891, Global Avg Loss: 0.13729314, Time: 0.3655 Steps: 17400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 005672, Sample Num: 90752, Cur Loss: 0.00000000, Cur Avg Loss: 0.02934768, Log Avg loss: 0.00016983, Global Avg Loss: 0.13573492, Time: 0.2487 Steps: 17600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 005872, Sample Num: 93952, Cur Loss: 0.00000101, Cur Avg Loss: 0.02871423, Log Avg loss: 0.01074935, Global Avg Loss: 0.13433059, Time: 1.2806 Steps: 17800, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006072, Sample Num: 97152, Cur Loss: 0.00000469, Cur Avg Loss: 0.02776848, Log Avg loss: 0.00000142, Global Avg Loss: 0.13283804, Time: 1.2804 Steps: 18000, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006272, Sample Num: 100352, Cur Loss: 0.00000010, Cur Avg Loss: 0.03162542, Log Avg loss: 0.14872198, Global Avg Loss: 0.13301259, Time: 0.5124 Steps: 18200, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006472, Sample Num: 103552, Cur Loss: 0.00000146, Cur Avg Loss: 0.03074661, Log Avg loss: 0.00318715, Global Avg Loss: 0.13160144, Time: 0.5591 Steps: 18400, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006672, Sample Num: 106752, Cur Loss: 0.00000004, Cur Avg Loss: 0.02984243, Log Avg loss: 0.00058334, Global Avg Loss: 0.13019265, Time: 0.2675 Steps: 18600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 006872, Sample Num: 109952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02900338, Log Avg loss: 0.00101261, Global Avg Loss: 0.12881839, Time: 0.3039 Steps: 18800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007072, Sample Num: 113152, Cur Loss: 0.00000127, Cur Avg Loss: 0.02826863, Log Avg loss: 0.00302270, Global Avg Loss: 0.12749423, Time: 0.4615 Steps: 19000, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007272, Sample Num: 116352, Cur Loss: 0.00000006, Cur Avg Loss: 0.02754715, Log Avg loss: 0.00203549, Global Avg Loss: 0.12618736, Time: 0.3212 Steps: 19200, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007472, Sample Num: 119552, Cur Loss: 0.00000000, Cur Avg Loss: 0.02680989, Log Avg loss: 0.00000313, Global Avg Loss: 0.12488649, Time: 0.1162 Steps: 19400, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007672, Sample Num: 122752, Cur Loss: 0.00000010, Cur Avg Loss: 0.02611130, Log Avg loss: 0.00001190, Global Avg Loss: 0.12361226, Time: 0.6357 Steps: 19600, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 007872, Sample Num: 125952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02544806, Log Avg loss: 0.00000613, Global Avg Loss: 0.12236372, Time: 0.3170 Steps: 19800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 008072, Sample Num: 129152, Cur Loss: 0.00000000, Cur Avg Loss: 0.02548223, Log Avg loss: 0.02682742, Global Avg Loss: 0.12140835, Time: 0.1255 Steps: 20000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008272, Sample Num: 132352, Cur Loss: 0.00000000, Cur Avg Loss: 0.02601560, Log Avg loss: 0.04754221, Global Avg Loss: 0.12067701, Time: 0.1999 Steps: 20200, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008472, Sample Num: 135552, Cur Loss: 0.00000001, Cur Avg Loss: 0.02985953, Log Avg loss: 0.18884463, Global Avg Loss: 0.12134532, Time: 0.4698 Steps: 20400, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008672, Sample Num: 138752, Cur Loss: 0.00000000, Cur Avg Loss: 0.02919741, Log Avg loss: 0.00114996, Global Avg Loss: 0.12017837, Time: 0.2176 Steps: 20600, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 008872, Sample Num: 141952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02856085, Log Avg loss: 0.00095963, Global Avg Loss: 0.11903204, Time: 0.2172 Steps: 20800, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 009072, Sample Num: 145152, Cur Loss: 0.00000000, Cur Avg Loss: 0.02793131, Log Avg loss: 0.00000470, Global Avg Loss: 0.11789844, Time: 0.2650 Steps: 21000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 009272, Sample Num: 148352, Cur Loss: 0.00000000, Cur Avg Loss: 0.02964009, Log Avg loss: 0.10715060, Global Avg Loss: 0.11779705, Time: 0.3137 Steps: 21200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009472, Sample Num: 151552, Cur Loss: 0.00001357, Cur Avg Loss: 0.03069981, Log Avg loss: 0.07982810, Global Avg Loss: 0.11744220, Time: 0.6502 Steps: 21400, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009672, Sample Num: 154752, Cur Loss: 0.00000048, Cur Avg Loss: 0.03065253, Log Avg loss: 0.02841332, Global Avg Loss: 0.11661786, Time: 0.3062 Steps: 21600, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 009872, Sample Num: 157952, Cur Loss: 0.00000116, Cur Avg Loss: 0.03016801, Log Avg loss: 0.00673707, Global Avg Loss: 0.11560978, Time: 0.6256 Steps: 21800, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010072, Sample Num: 161152, Cur Loss: 0.00000006, Cur Avg Loss: 0.02965616, Log Avg loss: 0.00439104, Global Avg Loss: 0.11459870, Time: 0.3360 Steps: 22000, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010272, Sample Num: 164352, Cur Loss: 0.00019428, Cur Avg Loss: 0.02908848, Log Avg loss: 0.00049988, Global Avg Loss: 0.11357078, Time: 1.2801 Steps: 22200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 010472, Sample Num: 167552, Cur Loss: 0.00000000, Cur Avg Loss: 0.02857690, Log Avg loss: 0.00230263, Global Avg Loss: 0.11257731, Time: 0.2289 Steps: 22400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 010672, Sample Num: 170752, Cur Loss: 0.00000120, Cur Avg Loss: 0.02806286, Log Avg loss: 0.00114741, Global Avg Loss: 0.11159121, Time: 0.5476 Steps: 22600, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 010872, Sample Num: 173952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02754667, Log Avg loss: 0.00000278, Global Avg Loss: 0.11061236, Time: 0.2578 Steps: 22800, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011072, Sample Num: 177152, Cur Loss: 0.00000002, Cur Avg Loss: 0.02706657, Log Avg loss: 0.00096858, Global Avg Loss: 0.10965894, Time: 0.4161 Steps: 23000, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011272, Sample Num: 180352, Cur Loss: 0.00000000, Cur Avg Loss: 0.02661411, Log Avg loss: 0.00156597, Global Avg Loss: 0.10872710, Time: 0.5880 Steps: 23200, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011472, Sample Num: 183552, Cur Loss: 0.00000000, Cur Avg Loss: 0.02675515, Log Avg loss: 0.03470413, Global Avg Loss: 0.10809443, Time: 0.6252 Steps: 23400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 011672, Sample Num: 186752, Cur Loss: 0.00000001, Cur Avg Loss: 0.02630166, Log Avg loss: 0.00028914, Global Avg Loss: 0.10718082, Time: 0.2332 Steps: 23600, Updated lr: 0.000080 Training, Epoch: 0002, Batch: 011872, Sample Num: 189952, Cur Loss: 0.00000000, Cur Avg Loss: 0.02651928, Log Avg loss: 0.03921997, Global Avg Loss: 0.10660972, Time: 0.7835 Steps: 23800, Updated lr: 0.000080 ***** Running evaluation checkpoint-23856 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-23856 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4887.717147, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.002436, "eval_total_loss": 3.393751, "eval_acc": 0.999686, "eval_prec": 0.987203, "eval_recall": 1.0, "eval_f1": 0.99356, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999925, "eval_confusion_matrix": {"tn": 21737, "fp": 7, "fn": 0, "tp": 540}, "eval_mcc2": 0.993421, "eval_mcc": 0.993421, "eval_sn": 1.0, "eval_sp": 0.999678, "update_flag": false, "test_avg_loss": 0.003071, "test_total_loss": 4.277524, "test_acc": 0.999776, "test_prec": 0.992634, "test_recall": 0.998148, "test_f1": 0.995383, "test_roc_auc": 0.999999, "test_pr_auc": 0.999973, "test_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 1, "tp": 539}, "test_mcc2": 0.995272, "test_mcc": 0.995272, "test_sn": 0.998148, "test_sp": 0.999816, "lr": 8.013436345314075e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.1063594743310459, "train_cur_epoch_loss": 314.8371182615479, "train_cur_epoch_avg_loss": 0.026394795293557, "train_cur_epoch_time": 4887.7171466350555, "train_cur_epoch_avg_time": 0.40976837245431386, "epoch": 2, "step": 23856} ################################################## Training, Epoch: 0003, Batch: 000144, Sample Num: 2304, Cur Loss: 0.00000000, Cur Avg Loss: 0.00119385, Log Avg loss: 0.00086044, Global Avg Loss: 0.10572848, Time: 0.1782 Steps: 24000, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000344, Sample Num: 5504, Cur Loss: 0.00000000, Cur Avg Loss: 0.05189572, Log Avg loss: 0.08840106, Global Avg Loss: 0.10558528, Time: 0.6459 Steps: 24200, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000544, Sample Num: 8704, Cur Loss: 0.00000000, Cur Avg Loss: 0.03282011, Log Avg loss: 0.00001006, Global Avg Loss: 0.10471991, Time: 0.2469 Steps: 24400, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000744, Sample Num: 11904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02501886, Log Avg loss: 0.00379946, Global Avg Loss: 0.10389942, Time: 0.2231 Steps: 24600, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000944, Sample Num: 15104, Cur Loss: 0.00000000, Cur Avg Loss: 0.06260398, Log Avg loss: 0.20242062, Global Avg Loss: 0.10469394, Time: 0.3718 Steps: 24800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001144, Sample Num: 18304, Cur Loss: 0.00000000, Cur Avg Loss: 0.05206094, Log Avg loss: 0.00229783, Global Avg Loss: 0.10387477, Time: 0.2323 Steps: 25000, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001344, Sample Num: 21504, Cur Loss: 0.00000000, Cur Avg Loss: 0.04444936, Log Avg loss: 0.00091108, Global Avg Loss: 0.10305760, Time: 0.4619 Steps: 25200, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001544, Sample Num: 24704, Cur Loss: 0.00000006, Cur Avg Loss: 0.03894103, Log Avg loss: 0.00192506, Global Avg Loss: 0.10226128, Time: 0.2537 Steps: 25400, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001744, Sample Num: 27904, Cur Loss: 0.00000000, Cur Avg Loss: 0.03495606, Log Avg loss: 0.00419207, Global Avg Loss: 0.10149512, Time: 0.2927 Steps: 25600, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001944, Sample Num: 31104, Cur Loss: 0.00000001, Cur Avg Loss: 0.03135983, Log Avg loss: 0.00000076, Global Avg Loss: 0.10070834, Time: 0.2587 Steps: 25800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002144, Sample Num: 34304, Cur Loss: 0.00000018, Cur Avg Loss: 0.03629153, Log Avg loss: 0.08422762, Global Avg Loss: 0.10058156, Time: 0.1560 Steps: 26000, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002344, Sample Num: 37504, Cur Loss: 0.00000000, Cur Avg Loss: 0.03403608, Log Avg loss: 0.00985771, Global Avg Loss: 0.09988902, Time: 0.2711 Steps: 26200, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002544, Sample Num: 40704, Cur Loss: 0.00000002, Cur Avg Loss: 0.03151225, Log Avg loss: 0.00193288, Global Avg Loss: 0.09914692, Time: 0.2530 Steps: 26400, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002744, Sample Num: 43904, Cur Loss: 0.00001091, Cur Avg Loss: 0.03459066, Log Avg loss: 0.07374804, Global Avg Loss: 0.09895595, Time: 0.5997 Steps: 26600, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 002944, Sample Num: 47104, Cur Loss: 0.00000000, Cur Avg Loss: 0.03232261, Log Avg loss: 0.00120494, Global Avg Loss: 0.09822647, Time: 0.2478 Steps: 26800, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 003144, Sample Num: 50304, Cur Loss: 0.00000000, Cur Avg Loss: 0.03027052, Log Avg loss: 0.00006375, Global Avg Loss: 0.09749934, Time: 0.5677 Steps: 27000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003344, Sample Num: 53504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02852716, Log Avg loss: 0.00112160, Global Avg Loss: 0.09679068, Time: 0.4066 Steps: 27200, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003544, Sample Num: 56704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02757962, Log Avg loss: 0.01173671, Global Avg Loss: 0.09616985, Time: 0.4478 Steps: 27400, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003744, Sample Num: 59904, Cur Loss: 0.00000029, Cur Avg Loss: 0.02635597, Log Avg loss: 0.00467298, Global Avg Loss: 0.09550683, Time: 0.3017 Steps: 27600, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 003944, Sample Num: 63104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02505033, Log Avg loss: 0.00060877, Global Avg Loss: 0.09482411, Time: 0.2636 Steps: 27800, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 004144, Sample Num: 66304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02390237, Log Avg loss: 0.00126457, Global Avg Loss: 0.09415582, Time: 0.2148 Steps: 28000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 004344, Sample Num: 69504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02325637, Log Avg loss: 0.00987127, Global Avg Loss: 0.09355806, Time: 0.6443 Steps: 28200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004544, Sample Num: 72704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02223294, Log Avg loss: 0.00000396, Global Avg Loss: 0.09289923, Time: 0.2783 Steps: 28400, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004744, Sample Num: 75904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02129566, Log Avg loss: 0.00000061, Global Avg Loss: 0.09224959, Time: 0.4309 Steps: 28600, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 004944, Sample Num: 79104, Cur Loss: 0.00000000, Cur Avg Loss: 0.02070898, Log Avg loss: 0.00679310, Global Avg Loss: 0.09165614, Time: 0.5213 Steps: 28800, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005144, Sample Num: 82304, Cur Loss: 0.00000000, Cur Avg Loss: 0.01990519, Log Avg loss: 0.00003554, Global Avg Loss: 0.09102427, Time: 0.1391 Steps: 29000, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005344, Sample Num: 85504, Cur Loss: 0.00000000, Cur Avg Loss: 0.01963881, Log Avg loss: 0.01278749, Global Avg Loss: 0.09048841, Time: 1.0661 Steps: 29200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 005544, Sample Num: 88704, Cur Loss: 0.00000000, Cur Avg Loss: 0.01907762, Log Avg loss: 0.00408242, Global Avg Loss: 0.08990061, Time: 0.5940 Steps: 29400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 005744, Sample Num: 91904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01841340, Log Avg loss: 0.00000123, Global Avg Loss: 0.08929318, Time: 0.3667 Steps: 29600, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 005944, Sample Num: 95104, Cur Loss: 0.00002456, Cur Avg Loss: 0.01779567, Log Avg loss: 0.00005447, Global Avg Loss: 0.08869426, Time: 0.1731 Steps: 29800, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006144, Sample Num: 98304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02130570, Log Avg loss: 0.12562382, Global Avg Loss: 0.08894046, Time: 0.2771 Steps: 30000, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006344, Sample Num: 101504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02063426, Log Avg loss: 0.00000761, Global Avg Loss: 0.08835150, Time: 0.1684 Steps: 30200, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006544, Sample Num: 104704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02007137, Log Avg loss: 0.00221655, Global Avg Loss: 0.08778482, Time: 0.3018 Steps: 30400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 006744, Sample Num: 107904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01949099, Log Avg loss: 0.00050113, Global Avg Loss: 0.08721434, Time: 0.2586 Steps: 30600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 006944, Sample Num: 111104, Cur Loss: 0.00000000, Cur Avg Loss: 0.01921720, Log Avg loss: 0.00998501, Global Avg Loss: 0.08671285, Time: 0.2724 Steps: 30800, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007144, Sample Num: 114304, Cur Loss: 0.00000036, Cur Avg Loss: 0.01941799, Log Avg loss: 0.02638927, Global Avg Loss: 0.08632367, Time: 0.6732 Steps: 31000, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007344, Sample Num: 117504, Cur Loss: 0.00000002, Cur Avg Loss: 0.01889321, Log Avg loss: 0.00014797, Global Avg Loss: 0.08577126, Time: 0.2614 Steps: 31200, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007544, Sample Num: 120704, Cur Loss: 0.00000000, Cur Avg Loss: 0.01839234, Log Avg loss: 0.00000043, Global Avg Loss: 0.08522495, Time: 0.2324 Steps: 31400, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007744, Sample Num: 123904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01791918, Log Avg loss: 0.00007151, Global Avg Loss: 0.08468600, Time: 0.1855 Steps: 31600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 007944, Sample Num: 127104, Cur Loss: 0.00000000, Cur Avg Loss: 0.01752472, Log Avg loss: 0.00225128, Global Avg Loss: 0.08416755, Time: 0.3139 Steps: 31800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008144, Sample Num: 130304, Cur Loss: 0.00000000, Cur Avg Loss: 0.01715263, Log Avg loss: 0.00237351, Global Avg Loss: 0.08365633, Time: 0.3030 Steps: 32000, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008344, Sample Num: 133504, Cur Loss: 0.00000000, Cur Avg Loss: 0.01681380, Log Avg loss: 0.00301647, Global Avg Loss: 0.08315546, Time: 1.2741 Steps: 32200, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008544, Sample Num: 136704, Cur Loss: 0.00000048, Cur Avg Loss: 0.01972626, Log Avg loss: 0.14123410, Global Avg Loss: 0.08351398, Time: 0.2912 Steps: 32400, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008744, Sample Num: 139904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01930288, Log Avg loss: 0.00121605, Global Avg Loss: 0.08300908, Time: 0.4342 Steps: 32600, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 008944, Sample Num: 143104, Cur Loss: 0.00000680, Cur Avg Loss: 0.01887153, Log Avg loss: 0.00001303, Global Avg Loss: 0.08250301, Time: 0.6838 Steps: 32800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 009144, Sample Num: 146304, Cur Loss: 0.00000000, Cur Avg Loss: 0.01846163, Log Avg loss: 0.00013085, Global Avg Loss: 0.08200378, Time: 0.1898 Steps: 33000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009344, Sample Num: 149504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02191145, Log Avg loss: 0.17963697, Global Avg Loss: 0.08259193, Time: 0.2991 Steps: 33200, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009544, Sample Num: 152704, Cur Loss: 0.00000000, Cur Avg Loss: 0.02146384, Log Avg loss: 0.00055190, Global Avg Loss: 0.08210068, Time: 0.7872 Steps: 33400, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009744, Sample Num: 155904, Cur Loss: 0.00000000, Cur Avg Loss: 0.02107212, Log Avg loss: 0.00237893, Global Avg Loss: 0.08162614, Time: 0.1569 Steps: 33600, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 009944, Sample Num: 159104, Cur Loss: 0.00000013, Cur Avg Loss: 0.02084106, Log Avg loss: 0.00958392, Global Avg Loss: 0.08119986, Time: 0.6008 Steps: 33800, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 010144, Sample Num: 162304, Cur Loss: 0.00000000, Cur Avg Loss: 0.02046477, Log Avg loss: 0.00175550, Global Avg Loss: 0.08073254, Time: 0.6876 Steps: 34000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 010344, Sample Num: 165504, Cur Loss: 0.00000000, Cur Avg Loss: 0.02007074, Log Avg loss: 0.00008549, Global Avg Loss: 0.08026092, Time: 0.2824 Steps: 34200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010544, Sample Num: 168704, Cur Loss: 0.00000215, Cur Avg Loss: 0.01970795, Log Avg loss: 0.00094471, Global Avg Loss: 0.07979978, Time: 0.8544 Steps: 34400, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010744, Sample Num: 171904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01934197, Log Avg loss: 0.00004729, Global Avg Loss: 0.07933878, Time: 0.3287 Steps: 34600, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 010944, Sample Num: 175104, Cur Loss: 0.00000000, Cur Avg Loss: 0.01898850, Log Avg loss: 0.00000004, Global Avg Loss: 0.07888281, Time: 0.3444 Steps: 34800, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011144, Sample Num: 178304, Cur Loss: 0.00000000, Cur Avg Loss: 0.01867842, Log Avg loss: 0.00171092, Global Avg Loss: 0.07844183, Time: 0.2909 Steps: 35000, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011344, Sample Num: 181504, Cur Loss: 0.00000000, Cur Avg Loss: 0.01834911, Log Avg loss: 0.00000012, Global Avg Loss: 0.07799613, Time: 0.3062 Steps: 35200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 011544, Sample Num: 184704, Cur Loss: 0.00000012, Cur Avg Loss: 0.01803318, Log Avg loss: 0.00011359, Global Avg Loss: 0.07755612, Time: 0.2511 Steps: 35400, Updated lr: 0.000070 Training, Epoch: 0003, Batch: 011744, Sample Num: 187904, Cur Loss: 0.00000000, Cur Avg Loss: 0.01777751, Log Avg loss: 0.00302025, Global Avg Loss: 0.07713738, Time: 0.2007 Steps: 35600, Updated lr: 0.000070 ***** Running evaluation checkpoint-35784 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-35784 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4895.186811, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.004761, "eval_total_loss": 6.632444, "eval_acc": 0.999731, "eval_prec": 0.990809, "eval_recall": 0.998148, "eval_f1": 0.994465, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999932, "eval_confusion_matrix": {"tn": 21739, "fp": 5, "fn": 1, "tp": 539}, "eval_mcc2": 0.994334, "eval_mcc": 0.994334, "eval_sn": 0.998148, "eval_sp": 0.99977, "update_flag": false, "test_avg_loss": 0.009426, "test_total_loss": 13.13064, "test_acc": 0.99982, "test_prec": 0.994465, "test_recall": 0.998148, "test_f1": 0.996303, "test_roc_auc": 0.999999, "test_pr_auc": 0.999963, "test_confusion_matrix": {"tn": 21741, "fp": 3, "fn": 1, "tp": 539}, "test_mcc2": 0.996213, "test_mcc": 0.996213, "test_sn": 0.998148, "test_sp": 0.999862, "lr": 7.011756802149816e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.07675853772769331, "train_cur_epoch_loss": 209.41589440635107, "train_cur_epoch_avg_loss": 0.01755666452098852, "train_cur_epoch_time": 4895.186811447144, "train_cur_epoch_avg_time": 0.4103946018986539, "epoch": 3, "step": 35784} ################################################## Training, Epoch: 0004, Batch: 000016, Sample Num: 256, Cur Loss: 0.00000024, Cur Avg Loss: 0.00064292, Log Avg loss: 0.00323547, Global Avg Loss: 0.07672452, Time: 0.6669 Steps: 35800, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000216, Sample Num: 3456, Cur Loss: 0.00000000, Cur Avg Loss: 0.00021518, Log Avg loss: 0.00018097, Global Avg Loss: 0.07629928, Time: 0.1454 Steps: 36000, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000416, Sample Num: 6656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01816453, Log Avg loss: 0.03754982, Global Avg Loss: 0.07608519, Time: 0.3329 Steps: 36200, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000616, Sample Num: 9856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01226720, Log Avg loss: 0.00000077, Global Avg Loss: 0.07566715, Time: 0.3000 Steps: 36400, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000816, Sample Num: 13056, Cur Loss: 0.00000000, Cur Avg Loss: 0.00990087, Log Avg loss: 0.00261255, Global Avg Loss: 0.07526794, Time: 0.3988 Steps: 36600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001016, Sample Num: 16256, Cur Loss: 0.00001199, Cur Avg Loss: 0.04348743, Log Avg loss: 0.18052062, Global Avg Loss: 0.07583997, Time: 0.7533 Steps: 36800, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001216, Sample Num: 19456, Cur Loss: 0.00000000, Cur Avg Loss: 0.03637018, Log Avg loss: 0.00021453, Global Avg Loss: 0.07543118, Time: 0.7434 Steps: 37000, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001416, Sample Num: 22656, Cur Loss: 0.00000000, Cur Avg Loss: 0.03123982, Log Avg loss: 0.00004722, Global Avg Loss: 0.07502589, Time: 0.2039 Steps: 37200, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001616, Sample Num: 25856, Cur Loss: 0.00000000, Cur Avg Loss: 0.02839332, Log Avg loss: 0.00824009, Global Avg Loss: 0.07466875, Time: 0.2335 Steps: 37400, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001816, Sample Num: 29056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02526645, Log Avg loss: 0.00000136, Global Avg Loss: 0.07427158, Time: 0.2328 Steps: 37600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002016, Sample Num: 32256, Cur Loss: 0.00000003, Cur Avg Loss: 0.02328872, Log Avg loss: 0.00533093, Global Avg Loss: 0.07390681, Time: 0.3004 Steps: 37800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002216, Sample Num: 35456, Cur Loss: 0.00000106, Cur Avg Loss: 0.02531427, Log Avg loss: 0.04573187, Global Avg Loss: 0.07375852, Time: 0.8222 Steps: 38000, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002416, Sample Num: 38656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02399146, Log Avg loss: 0.00933469, Global Avg Loss: 0.07342123, Time: 0.3369 Steps: 38200, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002616, Sample Num: 41856, Cur Loss: 0.00000763, Cur Avg Loss: 0.02249191, Log Avg loss: 0.00437730, Global Avg Loss: 0.07306162, Time: 0.3047 Steps: 38400, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 002816, Sample Num: 45056, Cur Loss: 0.00000000, Cur Avg Loss: 0.02413861, Log Avg loss: 0.04567744, Global Avg Loss: 0.07291974, Time: 0.4457 Steps: 38600, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003016, Sample Num: 48256, Cur Loss: 0.00000000, Cur Avg Loss: 0.02260444, Log Avg loss: 0.00100342, Global Avg Loss: 0.07254903, Time: 0.6867 Steps: 38800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003216, Sample Num: 51456, Cur Loss: 0.00000000, Cur Avg Loss: 0.02119930, Log Avg loss: 0.00000967, Global Avg Loss: 0.07217704, Time: 0.3519 Steps: 39000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003416, Sample Num: 54656, Cur Loss: 0.00000000, Cur Avg Loss: 0.02002725, Log Avg loss: 0.00118070, Global Avg Loss: 0.07181481, Time: 0.3221 Steps: 39200, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003616, Sample Num: 57856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01933574, Log Avg loss: 0.00752486, Global Avg Loss: 0.07148847, Time: 0.2810 Steps: 39400, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 003816, Sample Num: 61056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01843871, Log Avg loss: 0.00222035, Global Avg Loss: 0.07113863, Time: 0.3680 Steps: 39600, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004016, Sample Num: 64256, Cur Loss: 0.00000012, Cur Avg Loss: 0.01752200, Log Avg loss: 0.00003110, Global Avg Loss: 0.07078130, Time: 0.1043 Steps: 39800, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004216, Sample Num: 67456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01672953, Log Avg loss: 0.00081681, Global Avg Loss: 0.07043148, Time: 0.2307 Steps: 40000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 004416, Sample Num: 70656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01605387, Log Avg loss: 0.00181089, Global Avg Loss: 0.07009008, Time: 1.2271 Steps: 40200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 004616, Sample Num: 73856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01535842, Log Avg loss: 0.00000294, Global Avg Loss: 0.06974312, Time: 0.5945 Steps: 40400, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 004816, Sample Num: 77056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01484812, Log Avg loss: 0.00307048, Global Avg Loss: 0.06941468, Time: 0.4715 Steps: 40600, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005016, Sample Num: 80256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01429446, Log Avg loss: 0.00096231, Global Avg Loss: 0.06907913, Time: 0.5523 Steps: 40800, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005216, Sample Num: 83456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01376771, Log Avg loss: 0.00055689, Global Avg Loss: 0.06874488, Time: 0.1736 Steps: 41000, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005416, Sample Num: 86656, Cur Loss: 0.00000739, Cur Avg Loss: 0.01364603, Log Avg loss: 0.01047241, Global Avg Loss: 0.06846200, Time: 0.2479 Steps: 41200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 005616, Sample Num: 89856, Cur Loss: 0.00000001, Cur Avg Loss: 0.01316014, Log Avg loss: 0.00000236, Global Avg Loss: 0.06813128, Time: 0.4519 Steps: 41400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 005816, Sample Num: 93056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01270763, Log Avg loss: 0.00000103, Global Avg Loss: 0.06780373, Time: 1.0083 Steps: 41600, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006016, Sample Num: 96256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01228519, Log Avg loss: 0.00000073, Global Avg Loss: 0.06747931, Time: 0.2180 Steps: 41800, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006216, Sample Num: 99456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01529256, Log Avg loss: 0.10575425, Global Avg Loss: 0.06766157, Time: 0.2044 Steps: 42000, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006416, Sample Num: 102656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01487926, Log Avg loss: 0.00203401, Global Avg Loss: 0.06735054, Time: 0.1746 Steps: 42200, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006616, Sample Num: 105856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01443093, Log Avg loss: 0.00004853, Global Avg Loss: 0.06703308, Time: 1.2765 Steps: 42400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 006816, Sample Num: 109056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01400762, Log Avg loss: 0.00000433, Global Avg Loss: 0.06671839, Time: 0.1612 Steps: 42600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007016, Sample Num: 112256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01376525, Log Avg loss: 0.00550540, Global Avg Loss: 0.06643235, Time: 0.1404 Steps: 42800, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007216, Sample Num: 115456, Cur Loss: 0.00000006, Cur Avg Loss: 0.01545744, Log Avg loss: 0.07481930, Global Avg Loss: 0.06647136, Time: 0.2219 Steps: 43000, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007416, Sample Num: 118656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01504096, Log Avg loss: 0.00001457, Global Avg Loss: 0.06616369, Time: 0.5216 Steps: 43200, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007616, Sample Num: 121856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01464656, Log Avg loss: 0.00002207, Global Avg Loss: 0.06585889, Time: 0.5229 Steps: 43400, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 007816, Sample Num: 125056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01427206, Log Avg loss: 0.00001099, Global Avg Loss: 0.06555683, Time: 0.2944 Steps: 43600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 008016, Sample Num: 128256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01400147, Log Avg loss: 0.00342680, Global Avg Loss: 0.06527313, Time: 0.3759 Steps: 43800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008216, Sample Num: 131456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01371092, Log Avg loss: 0.00206580, Global Avg Loss: 0.06498583, Time: 0.2951 Steps: 44000, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008416, Sample Num: 134656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01631607, Log Avg loss: 0.12333576, Global Avg Loss: 0.06524985, Time: 0.3764 Steps: 44200, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008616, Sample Num: 137856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01593745, Log Avg loss: 0.00000489, Global Avg Loss: 0.06495596, Time: 0.5832 Steps: 44400, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 008816, Sample Num: 141056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01560096, Log Avg loss: 0.00110501, Global Avg Loss: 0.06466963, Time: 0.2822 Steps: 44600, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 009016, Sample Num: 144256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01525499, Log Avg loss: 0.00000476, Global Avg Loss: 0.06438095, Time: 0.2012 Steps: 44800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 009216, Sample Num: 147456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01576778, Log Avg loss: 0.03888426, Global Avg Loss: 0.06426763, Time: 0.2720 Steps: 45000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009416, Sample Num: 150656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01703915, Log Avg loss: 0.07562370, Global Avg Loss: 0.06431788, Time: 0.1562 Steps: 45200, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009616, Sample Num: 153856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01668486, Log Avg loss: 0.00000489, Global Avg Loss: 0.06403456, Time: 0.3223 Steps: 45400, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 009816, Sample Num: 157056, Cur Loss: 0.00000077, Cur Avg Loss: 0.01643130, Log Avg loss: 0.00424022, Global Avg Loss: 0.06377231, Time: 0.3239 Steps: 45600, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010016, Sample Num: 160256, Cur Loss: 0.00000924, Cur Avg Loss: 0.01610933, Log Avg loss: 0.00030725, Global Avg Loss: 0.06349517, Time: 0.4633 Steps: 45800, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010216, Sample Num: 163456, Cur Loss: 0.00000823, Cur Avg Loss: 0.01579423, Log Avg loss: 0.00001389, Global Avg Loss: 0.06321916, Time: 0.1651 Steps: 46000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 010416, Sample Num: 166656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01551547, Log Avg loss: 0.00127641, Global Avg Loss: 0.06295101, Time: 0.2801 Steps: 46200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 010616, Sample Num: 169856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01523102, Log Avg loss: 0.00041694, Global Avg Loss: 0.06268147, Time: 0.1694 Steps: 46400, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 010816, Sample Num: 173056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01494942, Log Avg loss: 0.00000216, Global Avg Loss: 0.06241246, Time: 0.1541 Steps: 46600, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011016, Sample Num: 176256, Cur Loss: 0.00000000, Cur Avg Loss: 0.01473309, Log Avg loss: 0.00303411, Global Avg Loss: 0.06215870, Time: 0.4396 Steps: 46800, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011216, Sample Num: 179456, Cur Loss: 0.00000000, Cur Avg Loss: 0.01450035, Log Avg loss: 0.00168073, Global Avg Loss: 0.06190135, Time: 0.3940 Steps: 47000, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011416, Sample Num: 182656, Cur Loss: 0.00000000, Cur Avg Loss: 0.01424640, Log Avg loss: 0.00000475, Global Avg Loss: 0.06163908, Time: 0.1574 Steps: 47200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 011616, Sample Num: 185856, Cur Loss: 0.00000000, Cur Avg Loss: 0.01400158, Log Avg loss: 0.00002739, Global Avg Loss: 0.06137911, Time: 0.5089 Steps: 47400, Updated lr: 0.000060 Training, Epoch: 0004, Batch: 011816, Sample Num: 189056, Cur Loss: 0.00000000, Cur Avg Loss: 0.01385859, Log Avg loss: 0.00555393, Global Avg Loss: 0.06114455, Time: 0.2823 Steps: 47600, Updated lr: 0.000060 ***** Running evaluation checkpoint-47712 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-47712 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4900.071111, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.017655, "eval_total_loss": 24.593489, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999935, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.025912, "test_total_loss": 36.095703, "test_acc": 0.99982, "test_prec": 0.996296, "test_recall": 0.996296, "test_f1": 0.996296, "test_roc_auc": 0.999998, "test_pr_auc": 0.99992, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 2, "tp": 538}, "test_mcc2": 0.996204, "test_mcc": 0.996204, "test_sn": 0.996296, "test_sp": 0.999908, "lr": 6.010077258985556e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.06101123223885796, "train_cur_epoch_loss": 164.24039853259512, "train_cur_epoch_avg_loss": 0.013769315772350363, "train_cur_epoch_time": 4900.071110725403, "train_cur_epoch_avg_time": 0.41080408372949384, "epoch": 4, "step": 47712} ################################################## Training, Epoch: 0005, Batch: 000088, Sample Num: 1408, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00243635, Global Avg Loss: 0.06089891, Time: 0.3433 Steps: 47800, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000288, Sample Num: 4608, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000341, Log Avg loss: 0.00000491, Global Avg Loss: 0.06064519, Time: 0.2326 Steps: 48000, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000488, Sample Num: 7808, Cur Loss: 0.00000000, Cur Avg Loss: 0.00020694, Log Avg loss: 0.00050000, Global Avg Loss: 0.06039562, Time: 0.2219 Steps: 48200, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000688, Sample Num: 11008, Cur Loss: 0.00000000, Cur Avg Loss: 0.00014711, Log Avg loss: 0.00000114, Global Avg Loss: 0.06014606, Time: 0.2195 Steps: 48400, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000888, Sample Num: 14208, Cur Loss: 0.00000000, Cur Avg Loss: 0.00021570, Log Avg loss: 0.00045167, Global Avg Loss: 0.05990040, Time: 0.1892 Steps: 48600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001088, Sample Num: 17408, Cur Loss: 0.00000000, Cur Avg Loss: 0.03270236, Log Avg loss: 0.17694311, Global Avg Loss: 0.06038008, Time: 0.1519 Steps: 48800, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001288, Sample Num: 20608, Cur Loss: 0.00000000, Cur Avg Loss: 0.02762838, Log Avg loss: 0.00002594, Global Avg Loss: 0.06013374, Time: 0.2455 Steps: 49000, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001488, Sample Num: 23808, Cur Loss: 0.00000006, Cur Avg Loss: 0.02410505, Log Avg loss: 0.00141480, Global Avg Loss: 0.05989505, Time: 0.6720 Steps: 49200, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001688, Sample Num: 27008, Cur Loss: 0.00000161, Cur Avg Loss: 0.02451605, Log Avg loss: 0.02757391, Global Avg Loss: 0.05976419, Time: 0.2330 Steps: 49400, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001888, Sample Num: 30208, Cur Loss: 0.00000000, Cur Avg Loss: 0.02191902, Log Avg loss: 0.00000009, Global Avg Loss: 0.05952321, Time: 0.8581 Steps: 49600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002088, Sample Num: 33408, Cur Loss: 0.00000000, Cur Avg Loss: 0.02194648, Log Avg loss: 0.02220572, Global Avg Loss: 0.05937334, Time: 0.5523 Steps: 49800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002288, Sample Num: 36608, Cur Loss: 0.00000000, Cur Avg Loss: 0.02224897, Log Avg loss: 0.02540687, Global Avg Loss: 0.05923747, Time: 0.5618 Steps: 50000, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002488, Sample Num: 39808, Cur Loss: 0.00026481, Cur Avg Loss: 0.02046210, Log Avg loss: 0.00002031, Global Avg Loss: 0.05900155, Time: 1.2748 Steps: 50200, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002688, Sample Num: 43008, Cur Loss: 0.00000000, Cur Avg Loss: 0.02690493, Log Avg loss: 0.10705376, Global Avg Loss: 0.05919223, Time: 0.2523 Steps: 50400, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 002888, Sample Num: 46208, Cur Loss: 0.00000000, Cur Avg Loss: 0.02507404, Log Avg loss: 0.00046687, Global Avg Loss: 0.05896011, Time: 0.2890 Steps: 50600, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003088, Sample Num: 49408, Cur Loss: 0.00000000, Cur Avg Loss: 0.02345021, Log Avg loss: 0.00000215, Global Avg Loss: 0.05872800, Time: 0.2498 Steps: 50800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003288, Sample Num: 52608, Cur Loss: 0.00000000, Cur Avg Loss: 0.02203609, Log Avg loss: 0.00020204, Global Avg Loss: 0.05849848, Time: 0.3656 Steps: 51000, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003488, Sample Num: 55808, Cur Loss: 0.00000004, Cur Avg Loss: 0.02084710, Log Avg loss: 0.00130013, Global Avg Loss: 0.05827505, Time: 0.5457 Steps: 51200, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003688, Sample Num: 59008, Cur Loss: 0.00040255, Cur Avg Loss: 0.02257599, Log Avg loss: 0.05272790, Global Avg Loss: 0.05825347, Time: 0.5874 Steps: 51400, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 003888, Sample Num: 62208, Cur Loss: 0.00000000, Cur Avg Loss: 0.02150348, Log Avg loss: 0.00172636, Global Avg Loss: 0.05803437, Time: 0.1800 Steps: 51600, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 004088, Sample Num: 65408, Cur Loss: 0.00000000, Cur Avg Loss: 0.02045208, Log Avg loss: 0.00001287, Global Avg Loss: 0.05781035, Time: 0.3314 Steps: 51800, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 004288, Sample Num: 68608, Cur Loss: 0.00000036, Cur Avg Loss: 0.01950129, Log Avg loss: 0.00006705, Global Avg Loss: 0.05758826, Time: 1.3171 Steps: 52000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004488, Sample Num: 71808, Cur Loss: 0.00000059, Cur Avg Loss: 0.01867572, Log Avg loss: 0.00097549, Global Avg Loss: 0.05737135, Time: 0.5103 Steps: 52200, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004688, Sample Num: 75008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01787899, Log Avg loss: 0.00000052, Global Avg Loss: 0.05715238, Time: 0.8237 Steps: 52400, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 004888, Sample Num: 78208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01722188, Log Avg loss: 0.00181923, Global Avg Loss: 0.05694199, Time: 0.7647 Steps: 52600, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005088, Sample Num: 81408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01654554, Log Avg loss: 0.00001576, Global Avg Loss: 0.05672636, Time: 0.2618 Steps: 52800, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005288, Sample Num: 84608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01592619, Log Avg loss: 0.00016995, Global Avg Loss: 0.05651294, Time: 0.1373 Steps: 53000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 005488, Sample Num: 87808, Cur Loss: 0.00000012, Cur Avg Loss: 0.01535496, Log Avg loss: 0.00025157, Global Avg Loss: 0.05630143, Time: 0.3426 Steps: 53200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 005688, Sample Num: 91008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01481506, Log Avg loss: 0.00000031, Global Avg Loss: 0.05609056, Time: 0.4297 Steps: 53400, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 005888, Sample Num: 94208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01431185, Log Avg loss: 0.00000048, Global Avg Loss: 0.05588127, Time: 0.2568 Steps: 53600, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006088, Sample Num: 97408, Cur Loss: 0.00033470, Cur Avg Loss: 0.01384174, Log Avg loss: 0.00000173, Global Avg Loss: 0.05567354, Time: 1.2759 Steps: 53800, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006288, Sample Num: 100608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01701304, Log Avg loss: 0.11354726, Global Avg Loss: 0.05588789, Time: 0.6058 Steps: 54000, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006488, Sample Num: 103808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01659094, Log Avg loss: 0.00332018, Global Avg Loss: 0.05569391, Time: 0.6416 Steps: 54200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 006688, Sample Num: 107008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01609526, Log Avg loss: 0.00001554, Global Avg Loss: 0.05548921, Time: 0.2346 Steps: 54400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 006888, Sample Num: 110208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01562799, Log Avg loss: 0.00000239, Global Avg Loss: 0.05528596, Time: 0.1813 Steps: 54600, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007088, Sample Num: 113408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01570485, Log Avg loss: 0.01835189, Global Avg Loss: 0.05515117, Time: 0.3385 Steps: 54800, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007288, Sample Num: 116608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01527411, Log Avg loss: 0.00000863, Global Avg Loss: 0.05495065, Time: 0.2345 Steps: 55000, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007488, Sample Num: 119808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01486615, Log Avg loss: 0.00000008, Global Avg Loss: 0.05475155, Time: 0.3521 Steps: 55200, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007688, Sample Num: 123008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01447945, Log Avg loss: 0.00000159, Global Avg Loss: 0.05455390, Time: 0.2623 Steps: 55400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 007888, Sample Num: 126208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01411233, Log Avg loss: 0.00000006, Global Avg Loss: 0.05435766, Time: 0.1792 Steps: 55600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008088, Sample Num: 129408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01384634, Log Avg loss: 0.00335575, Global Avg Loss: 0.05417486, Time: 0.1573 Steps: 55800, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008288, Sample Num: 132608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01352849, Log Avg loss: 0.00067447, Global Avg Loss: 0.05398379, Time: 0.3000 Steps: 56000, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008488, Sample Num: 135808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01711546, Log Avg loss: 0.16575976, Global Avg Loss: 0.05438157, Time: 0.2217 Steps: 56200, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008688, Sample Num: 139008, Cur Loss: 0.00000030, Cur Avg Loss: 0.01672156, Log Avg loss: 0.00000445, Global Avg Loss: 0.05418874, Time: 0.2510 Steps: 56400, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 008888, Sample Num: 142208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01636546, Log Avg loss: 0.00089639, Global Avg Loss: 0.05400043, Time: 0.2523 Steps: 56600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 009088, Sample Num: 145408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01600533, Log Avg loss: 0.00000112, Global Avg Loss: 0.05381029, Time: 0.4462 Steps: 56800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009288, Sample Num: 148608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01597433, Log Avg loss: 0.01456559, Global Avg Loss: 0.05367259, Time: 0.3305 Steps: 57000, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009488, Sample Num: 151808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01853845, Log Avg loss: 0.13761638, Global Avg Loss: 0.05396610, Time: 0.5197 Steps: 57200, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009688, Sample Num: 155008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01820303, Log Avg loss: 0.00229057, Global Avg Loss: 0.05378604, Time: 0.2917 Steps: 57400, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 009888, Sample Num: 158208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01799804, Log Avg loss: 0.00806842, Global Avg Loss: 0.05362730, Time: 0.5951 Steps: 57600, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 010088, Sample Num: 161408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01764168, Log Avg loss: 0.00002341, Global Avg Loss: 0.05344182, Time: 0.1513 Steps: 57800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 010288, Sample Num: 164608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01729901, Log Avg loss: 0.00001464, Global Avg Loss: 0.05325759, Time: 0.1815 Steps: 58000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010488, Sample Num: 167808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01697433, Log Avg loss: 0.00027279, Global Avg Loss: 0.05307551, Time: 0.2286 Steps: 58200, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010688, Sample Num: 171008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01665719, Log Avg loss: 0.00002607, Global Avg Loss: 0.05289383, Time: 0.7471 Steps: 58400, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 010888, Sample Num: 174208, Cur Loss: 0.00000000, Cur Avg Loss: 0.01635122, Log Avg loss: 0.00000029, Global Avg Loss: 0.05271331, Time: 0.2275 Steps: 58600, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011088, Sample Num: 177408, Cur Loss: 0.00000000, Cur Avg Loss: 0.01658062, Log Avg loss: 0.02906898, Global Avg Loss: 0.05263289, Time: 0.1940 Steps: 58800, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011288, Sample Num: 180608, Cur Loss: 0.00000000, Cur Avg Loss: 0.01628684, Log Avg loss: 0.00000006, Global Avg Loss: 0.05245447, Time: 0.4382 Steps: 59000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 011488, Sample Num: 183808, Cur Loss: 0.00000000, Cur Avg Loss: 0.01600330, Log Avg loss: 0.00000004, Global Avg Loss: 0.05227726, Time: 0.2277 Steps: 59200, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 011688, Sample Num: 187008, Cur Loss: 0.00000000, Cur Avg Loss: 0.01573911, Log Avg loss: 0.00056425, Global Avg Loss: 0.05210314, Time: 0.6980 Steps: 59400, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 011888, Sample Num: 190208, Cur Loss: 0.00000002, Cur Avg Loss: 0.01551380, Log Avg loss: 0.00234686, Global Avg Loss: 0.05193617, Time: 0.5736 Steps: 59600, Updated lr: 0.000050 ***** Running evaluation checkpoint-59640 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-59640 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4894.794752, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.025807, "eval_total_loss": 35.948871, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999935, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.034894, "test_total_loss": 48.607901, "test_acc": 0.99982, "test_prec": 0.996296, "test_recall": 0.996296, "test_f1": 0.996296, "test_roc_auc": 0.999998, "test_pr_auc": 0.999914, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 2, "tp": 538}, "test_mcc2": 0.996204, "test_mcc": 0.996204, "test_sn": 0.996296, "test_sp": 0.999908, "lr": 5.008397715821297e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.051901341786457604, "train_cur_epoch_loss": 184.42811156394703, "train_cur_epoch_avg_loss": 0.015461779976856726, "train_cur_epoch_time": 4894.79475235939, "train_cur_epoch_avg_time": 0.410361733095187, "epoch": 5, "step": 59640} ################################################## Training, Epoch: 0006, Batch: 000160, Sample Num: 2560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000093, Log Avg loss: 0.00000074, Global Avg Loss: 0.05176248, Time: 0.2643 Steps: 59800, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000360, Sample Num: 5760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000054, Log Avg loss: 0.00000024, Global Avg Loss: 0.05158994, Time: 1.1530 Steps: 60000, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000560, Sample Num: 8960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000708, Log Avg loss: 0.00001885, Global Avg Loss: 0.05141860, Time: 0.5205 Steps: 60200, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000760, Sample Num: 12160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000528, Log Avg loss: 0.00000023, Global Avg Loss: 0.05124834, Time: 1.2570 Steps: 60400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 000960, Sample Num: 15360, Cur Loss: 0.00000000, Cur Avg Loss: 0.03288379, Log Avg loss: 0.15782212, Global Avg Loss: 0.05160007, Time: 0.1824 Steps: 60600, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001160, Sample Num: 18560, Cur Loss: 0.00000000, Cur Avg Loss: 0.02743517, Log Avg loss: 0.00128181, Global Avg Loss: 0.05143455, Time: 0.3515 Steps: 60800, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001360, Sample Num: 21760, Cur Loss: 0.00000000, Cur Avg Loss: 0.02340062, Log Avg loss: 0.00000022, Global Avg Loss: 0.05126592, Time: 0.9470 Steps: 61000, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001560, Sample Num: 24960, Cur Loss: 0.00000000, Cur Avg Loss: 0.02048310, Log Avg loss: 0.00064398, Global Avg Loss: 0.05110048, Time: 0.2183 Steps: 61200, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001760, Sample Num: 28160, Cur Loss: 0.00000000, Cur Avg Loss: 0.02336484, Log Avg loss: 0.04584236, Global Avg Loss: 0.05108336, Time: 0.1898 Steps: 61400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001960, Sample Num: 31360, Cur Loss: 0.00000000, Cur Avg Loss: 0.02098068, Log Avg loss: 0.00000014, Global Avg Loss: 0.05091750, Time: 0.2271 Steps: 61600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002160, Sample Num: 34560, Cur Loss: 0.00000000, Cur Avg Loss: 0.02583208, Log Avg loss: 0.07337579, Global Avg Loss: 0.05099018, Time: 0.4386 Steps: 61800, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002360, Sample Num: 37760, Cur Loss: 0.00000000, Cur Avg Loss: 0.02377056, Log Avg loss: 0.00150614, Global Avg Loss: 0.05083056, Time: 0.2591 Steps: 62000, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002560, Sample Num: 40960, Cur Loss: 0.00000000, Cur Avg Loss: 0.02468138, Log Avg loss: 0.03542903, Global Avg Loss: 0.05078103, Time: 0.3363 Steps: 62200, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002760, Sample Num: 44160, Cur Loss: 0.00000000, Cur Avg Loss: 0.02966369, Log Avg loss: 0.09343732, Global Avg Loss: 0.05091775, Time: 0.1894 Steps: 62400, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 002960, Sample Num: 47360, Cur Loss: 0.00000000, Cur Avg Loss: 0.02767462, Log Avg loss: 0.00022533, Global Avg Loss: 0.05075580, Time: 0.2287 Steps: 62600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 003160, Sample Num: 50560, Cur Loss: 0.00000000, Cur Avg Loss: 0.02592307, Log Avg loss: 0.00000013, Global Avg Loss: 0.05059415, Time: 0.4192 Steps: 62800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003360, Sample Num: 53760, Cur Loss: 0.00000000, Cur Avg Loss: 0.02438640, Log Avg loss: 0.00010706, Global Avg Loss: 0.05043388, Time: 0.2590 Steps: 63000, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003560, Sample Num: 56960, Cur Loss: 0.00000000, Cur Avg Loss: 0.02929920, Log Avg loss: 0.11183425, Global Avg Loss: 0.05062818, Time: 0.2525 Steps: 63200, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003760, Sample Num: 60160, Cur Loss: 0.00000000, Cur Avg Loss: 0.02797113, Log Avg loss: 0.00433144, Global Avg Loss: 0.05048214, Time: 0.1737 Steps: 63400, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 003960, Sample Num: 63360, Cur Loss: 0.00000000, Cur Avg Loss: 0.02655885, Log Avg loss: 0.00000811, Global Avg Loss: 0.05032341, Time: 1.2744 Steps: 63600, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 004160, Sample Num: 66560, Cur Loss: 0.00000000, Cur Avg Loss: 0.02528231, Log Avg loss: 0.00000677, Global Avg Loss: 0.05016568, Time: 0.3853 Steps: 63800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 004360, Sample Num: 69760, Cur Loss: 0.00000000, Cur Avg Loss: 0.02416504, Log Avg loss: 0.00092589, Global Avg Loss: 0.05001181, Time: 0.1951 Steps: 64000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004560, Sample Num: 72960, Cur Loss: 0.00000000, Cur Avg Loss: 0.02310528, Log Avg loss: 0.00000247, Global Avg Loss: 0.04985601, Time: 0.2151 Steps: 64200, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004760, Sample Num: 76160, Cur Loss: 0.00000000, Cur Avg Loss: 0.02213522, Log Avg loss: 0.00001786, Global Avg Loss: 0.04970124, Time: 0.1505 Steps: 64400, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 004960, Sample Num: 79360, Cur Loss: 0.00000000, Cur Avg Loss: 0.02126032, Log Avg loss: 0.00043760, Global Avg Loss: 0.04954872, Time: 0.4856 Steps: 64600, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005160, Sample Num: 82560, Cur Loss: 0.00000000, Cur Avg Loss: 0.02043762, Log Avg loss: 0.00003465, Global Avg Loss: 0.04939590, Time: 0.1926 Steps: 64800, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005360, Sample Num: 85760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01967549, Log Avg loss: 0.00001260, Global Avg Loss: 0.04924395, Time: 1.0967 Steps: 65000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 005560, Sample Num: 88960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01897676, Log Avg loss: 0.00025069, Global Avg Loss: 0.04909366, Time: 0.3363 Steps: 65200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 005760, Sample Num: 92160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01831785, Log Avg loss: 0.00000020, Global Avg Loss: 0.04894353, Time: 0.2292 Steps: 65400, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 005960, Sample Num: 95360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01770316, Log Avg loss: 0.00000004, Global Avg Loss: 0.04879431, Time: 0.2031 Steps: 65600, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006160, Sample Num: 98560, Cur Loss: 0.00000000, Cur Avg Loss: 0.01934446, Log Avg loss: 0.06825543, Global Avg Loss: 0.04885346, Time: 0.2955 Steps: 65800, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006360, Sample Num: 101760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01873624, Log Avg loss: 0.00000304, Global Avg Loss: 0.04870543, Time: 0.1529 Steps: 66000, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006560, Sample Num: 104960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01823646, Log Avg loss: 0.00234339, Global Avg Loss: 0.04856537, Time: 0.6382 Steps: 66200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 006760, Sample Num: 108160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01769707, Log Avg loss: 0.00000496, Global Avg Loss: 0.04841910, Time: 0.2937 Steps: 66400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 006960, Sample Num: 111360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01718854, Log Avg loss: 0.00000027, Global Avg Loss: 0.04827370, Time: 0.3620 Steps: 66600, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007160, Sample Num: 114560, Cur Loss: 0.00000530, Cur Avg Loss: 0.01786189, Log Avg loss: 0.04129465, Global Avg Loss: 0.04825280, Time: 0.2650 Steps: 66800, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007360, Sample Num: 117760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01737667, Log Avg loss: 0.00000568, Global Avg Loss: 0.04810878, Time: 0.2871 Steps: 67000, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007560, Sample Num: 120960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01691697, Log Avg loss: 0.00000016, Global Avg Loss: 0.04796560, Time: 0.2019 Steps: 67200, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007760, Sample Num: 124160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01648100, Log Avg loss: 0.00000119, Global Avg Loss: 0.04782327, Time: 0.3883 Steps: 67400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 007960, Sample Num: 127360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01611265, Log Avg loss: 0.00182065, Global Avg Loss: 0.04768717, Time: 0.1643 Steps: 67600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008160, Sample Num: 130560, Cur Loss: 0.00000000, Cur Avg Loss: 0.01575662, Log Avg loss: 0.00158676, Global Avg Loss: 0.04755118, Time: 0.1508 Steps: 67800, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008360, Sample Num: 133760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01538601, Log Avg loss: 0.00026495, Global Avg Loss: 0.04741210, Time: 0.3186 Steps: 68000, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008560, Sample Num: 136960, Cur Loss: 0.00000027, Cur Avg Loss: 0.01814674, Log Avg loss: 0.13354516, Global Avg Loss: 0.04766469, Time: 0.5564 Steps: 68200, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008760, Sample Num: 140160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01773540, Log Avg loss: 0.00013024, Global Avg Loss: 0.04752570, Time: 0.7578 Steps: 68400, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 008960, Sample Num: 143360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01733953, Log Avg loss: 0.00000025, Global Avg Loss: 0.04738715, Time: 0.2711 Steps: 68600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 009160, Sample Num: 146560, Cur Loss: 0.00000000, Cur Avg Loss: 0.01696094, Log Avg loss: 0.00000004, Global Avg Loss: 0.04724939, Time: 0.4711 Steps: 68800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009360, Sample Num: 149760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01966603, Log Avg loss: 0.14355944, Global Avg Loss: 0.04752855, Time: 0.2159 Steps: 69000, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009560, Sample Num: 152960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01925472, Log Avg loss: 0.00000543, Global Avg Loss: 0.04739120, Time: 0.2510 Steps: 69200, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009760, Sample Num: 156160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01891230, Log Avg loss: 0.00254466, Global Avg Loss: 0.04726196, Time: 0.4084 Steps: 69400, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 009960, Sample Num: 159360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01859106, Log Avg loss: 0.00291450, Global Avg Loss: 0.04713453, Time: 0.3313 Steps: 69600, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 010160, Sample Num: 162560, Cur Loss: 0.00000000, Cur Avg Loss: 0.01822578, Log Avg loss: 0.00003474, Global Avg Loss: 0.04699957, Time: 0.2072 Steps: 69800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 010360, Sample Num: 165760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01787444, Log Avg loss: 0.00002607, Global Avg Loss: 0.04686536, Time: 0.5166 Steps: 70000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010560, Sample Num: 168960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01753891, Log Avg loss: 0.00015882, Global Avg Loss: 0.04673229, Time: 0.2505 Steps: 70200, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010760, Sample Num: 172160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01721302, Log Avg loss: 0.00000581, Global Avg Loss: 0.04659955, Time: 0.4658 Steps: 70400, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 010960, Sample Num: 175360, Cur Loss: 0.00000000, Cur Avg Loss: 0.01689891, Log Avg loss: 0.00000001, Global Avg Loss: 0.04646754, Time: 1.2511 Steps: 70600, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011160, Sample Num: 178560, Cur Loss: 0.00000000, Cur Avg Loss: 0.01663364, Log Avg loss: 0.00209653, Global Avg Loss: 0.04634220, Time: 0.2939 Steps: 70800, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011360, Sample Num: 181760, Cur Loss: 0.00000000, Cur Avg Loss: 0.01634083, Log Avg loss: 0.00000216, Global Avg Loss: 0.04621166, Time: 0.2533 Steps: 71000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 011560, Sample Num: 184960, Cur Loss: 0.00000000, Cur Avg Loss: 0.01605813, Log Avg loss: 0.00000092, Global Avg Loss: 0.04608185, Time: 0.1552 Steps: 71200, Updated lr: 0.000040 Training, Epoch: 0006, Batch: 011760, Sample Num: 188160, Cur Loss: 0.00000000, Cur Avg Loss: 0.01578671, Log Avg loss: 0.00009864, Global Avg Loss: 0.04595305, Time: 0.3013 Steps: 71400, Updated lr: 0.000040 ***** Running evaluation checkpoint-71568 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-71568 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4899.450492, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.032184, "eval_total_loss": 44.832149, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999935, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.041258, "test_total_loss": 57.47272, "test_acc": 0.99982, "test_prec": 0.996296, "test_recall": 0.996296, "test_f1": 0.996296, "test_roc_auc": 0.999999, "test_pr_auc": 0.999949, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 2, "tp": 538}, "test_mcc2": 0.996204, "test_mcc": 0.996204, "test_sn": 0.996296, "test_sp": 0.999908, "lr": 4.006718172657038e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.04585176545391369, "train_cur_epoch_loss": 186.12312586136386, "train_cur_epoch_avg_loss": 0.015603883791194153, "train_cur_epoch_time": 4899.450491666794, "train_cur_epoch_avg_time": 0.4107520532919847, "epoch": 6, "step": 71568} ################################################## Training, Epoch: 0007, Batch: 000032, Sample Num: 512, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000011, Log Avg loss: 0.00235702, Global Avg Loss: 0.04583127, Time: 0.5973 Steps: 71600, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000232, Sample Num: 3712, Cur Loss: 0.00000000, Cur Avg Loss: 0.00010530, Log Avg loss: 0.00012213, Global Avg Loss: 0.04570395, Time: 0.1380 Steps: 71800, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000432, Sample Num: 6912, Cur Loss: 0.00000000, Cur Avg Loss: 0.00005722, Log Avg loss: 0.00000145, Global Avg Loss: 0.04557700, Time: 0.6230 Steps: 72000, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000632, Sample Num: 10112, Cur Loss: 0.00000000, Cur Avg Loss: 0.00003912, Log Avg loss: 0.00000003, Global Avg Loss: 0.04545075, Time: 1.0425 Steps: 72200, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000832, Sample Num: 13312, Cur Loss: 0.00000000, Cur Avg Loss: 0.00010223, Log Avg loss: 0.00030166, Global Avg Loss: 0.04532602, Time: 0.2545 Steps: 72400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001032, Sample Num: 16512, Cur Loss: 0.00000000, Cur Avg Loss: 0.04025210, Log Avg loss: 0.20727553, Global Avg Loss: 0.04577217, Time: 0.4439 Steps: 72600, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001232, Sample Num: 19712, Cur Loss: 0.00000000, Cur Avg Loss: 0.03371781, Log Avg loss: 0.00000086, Global Avg Loss: 0.04564642, Time: 0.4732 Steps: 72800, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001432, Sample Num: 22912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02900865, Log Avg loss: 0.00000024, Global Avg Loss: 0.04552136, Time: 0.3001 Steps: 73000, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001632, Sample Num: 26112, Cur Loss: 0.00000000, Cur Avg Loss: 0.03298554, Log Avg loss: 0.06146008, Global Avg Loss: 0.04556491, Time: 0.5144 Steps: 73200, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001832, Sample Num: 29312, Cur Loss: 0.00000000, Cur Avg Loss: 0.02938452, Log Avg loss: 0.00000017, Global Avg Loss: 0.04544076, Time: 0.2608 Steps: 73400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002032, Sample Num: 32512, Cur Loss: 0.00000000, Cur Avg Loss: 0.02649240, Log Avg loss: 0.00000063, Global Avg Loss: 0.04531728, Time: 0.5630 Steps: 73600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002232, Sample Num: 35712, Cur Loss: 0.00000000, Cur Avg Loss: 0.03279754, Log Avg loss: 0.09685776, Global Avg Loss: 0.04545695, Time: 0.1896 Steps: 73800, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002432, Sample Num: 38912, Cur Loss: 0.00000000, Cur Avg Loss: 0.03010498, Log Avg loss: 0.00005596, Global Avg Loss: 0.04533425, Time: 0.2902 Steps: 74000, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002632, Sample Num: 42112, Cur Loss: 0.00000000, Cur Avg Loss: 0.03468236, Log Avg loss: 0.09034335, Global Avg Loss: 0.04545557, Time: 0.2019 Steps: 74200, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 002832, Sample Num: 45312, Cur Loss: 0.00000000, Cur Avg Loss: 0.03792678, Log Avg loss: 0.08062333, Global Avg Loss: 0.04555010, Time: 0.1946 Steps: 74400, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003032, Sample Num: 48512, Cur Loss: 0.00000000, Cur Avg Loss: 0.03542502, Log Avg loss: 0.00000013, Global Avg Loss: 0.04542799, Time: 0.3984 Steps: 74600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003232, Sample Num: 51712, Cur Loss: 0.00000000, Cur Avg Loss: 0.03323289, Log Avg loss: 0.00000009, Global Avg Loss: 0.04530652, Time: 0.5220 Steps: 74800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003432, Sample Num: 54912, Cur Loss: 0.00000000, Cur Avg Loss: 0.03130803, Log Avg loss: 0.00020237, Global Avg Loss: 0.04518624, Time: 0.1619 Steps: 75000, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003632, Sample Num: 58112, Cur Loss: 0.00000000, Cur Avg Loss: 0.03632538, Log Avg loss: 0.12242302, Global Avg Loss: 0.04539166, Time: 0.4082 Steps: 75200, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 003832, Sample Num: 61312, Cur Loss: 0.00000000, Cur Avg Loss: 0.03444590, Log Avg loss: 0.00031471, Global Avg Loss: 0.04527209, Time: 0.2595 Steps: 75400, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004032, Sample Num: 64512, Cur Loss: 0.00000000, Cur Avg Loss: 0.03273736, Log Avg loss: 0.00000165, Global Avg Loss: 0.04515233, Time: 0.3797 Steps: 75600, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004232, Sample Num: 67712, Cur Loss: 0.00000000, Cur Avg Loss: 0.03119058, Log Avg loss: 0.00000742, Global Avg Loss: 0.04503321, Time: 0.2473 Steps: 75800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 004432, Sample Num: 70912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02981149, Log Avg loss: 0.00063009, Global Avg Loss: 0.04491636, Time: 0.6046 Steps: 76000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 004632, Sample Num: 74112, Cur Loss: 0.00000000, Cur Avg Loss: 0.02852432, Log Avg loss: 0.00000045, Global Avg Loss: 0.04479847, Time: 0.4691 Steps: 76200, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 004832, Sample Num: 77312, Cur Loss: 0.00000000, Cur Avg Loss: 0.02734379, Log Avg loss: 0.00000277, Global Avg Loss: 0.04468121, Time: 0.4291 Steps: 76400, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005032, Sample Num: 80512, Cur Loss: 0.00000000, Cur Avg Loss: 0.02636405, Log Avg loss: 0.00269356, Global Avg Loss: 0.04457158, Time: 0.2865 Steps: 76600, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005232, Sample Num: 83712, Cur Loss: 0.00000000, Cur Avg Loss: 0.02535927, Log Avg loss: 0.00007892, Global Avg Loss: 0.04445571, Time: 0.1738 Steps: 76800, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005432, Sample Num: 86912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02442702, Log Avg loss: 0.00003943, Global Avg Loss: 0.04434035, Time: 0.2016 Steps: 77000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 005632, Sample Num: 90112, Cur Loss: 0.00000000, Cur Avg Loss: 0.02355968, Log Avg loss: 0.00000289, Global Avg Loss: 0.04422548, Time: 0.4050 Steps: 77200, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 005832, Sample Num: 93312, Cur Loss: 0.00000000, Cur Avg Loss: 0.02275174, Log Avg loss: 0.00000009, Global Avg Loss: 0.04411121, Time: 0.1235 Steps: 77400, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006032, Sample Num: 96512, Cur Loss: 0.00000000, Cur Avg Loss: 0.02199737, Log Avg loss: 0.00000002, Global Avg Loss: 0.04399752, Time: 0.8262 Steps: 77600, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006232, Sample Num: 99712, Cur Loss: 0.00000000, Cur Avg Loss: 0.02459126, Log Avg loss: 0.10282270, Global Avg Loss: 0.04414874, Time: 0.3700 Steps: 77800, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006432, Sample Num: 102912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02390009, Log Avg loss: 0.00236336, Global Avg Loss: 0.04404160, Time: 0.2287 Steps: 78000, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 006632, Sample Num: 106112, Cur Loss: 0.00000000, Cur Avg Loss: 0.02317947, Log Avg loss: 0.00000421, Global Avg Loss: 0.04392897, Time: 0.1922 Steps: 78200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 006832, Sample Num: 109312, Cur Loss: 0.00000000, Cur Avg Loss: 0.02250093, Log Avg loss: 0.00000080, Global Avg Loss: 0.04381691, Time: 0.2620 Steps: 78400, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007032, Sample Num: 112512, Cur Loss: 3.26074624, Cur Avg Loss: 0.02232468, Log Avg loss: 0.01630375, Global Avg Loss: 0.04374690, Time: 1.2743 Steps: 78600, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007232, Sample Num: 115712, Cur Loss: 0.00000000, Cur Avg Loss: 0.02170737, Log Avg loss: 0.00000298, Global Avg Loss: 0.04363587, Time: 0.6110 Steps: 78800, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007432, Sample Num: 118912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02112325, Log Avg loss: 0.00000130, Global Avg Loss: 0.04352541, Time: 0.3956 Steps: 79000, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007632, Sample Num: 122112, Cur Loss: 0.00000000, Cur Avg Loss: 0.02056975, Log Avg loss: 0.00000165, Global Avg Loss: 0.04341550, Time: 0.1815 Steps: 79200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 007832, Sample Num: 125312, Cur Loss: 0.00000000, Cur Avg Loss: 0.02004458, Log Avg loss: 0.00000420, Global Avg Loss: 0.04330615, Time: 0.5088 Steps: 79400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008032, Sample Num: 128512, Cur Loss: 0.00000000, Cur Avg Loss: 0.01964107, Log Avg loss: 0.00383958, Global Avg Loss: 0.04320699, Time: 0.2317 Steps: 79600, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008232, Sample Num: 131712, Cur Loss: 0.00000000, Cur Avg Loss: 0.01916647, Log Avg loss: 0.00010652, Global Avg Loss: 0.04309897, Time: 0.3669 Steps: 79800, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008432, Sample Num: 134912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02143218, Log Avg loss: 0.11468880, Global Avg Loss: 0.04327794, Time: 0.2012 Steps: 80000, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008632, Sample Num: 138112, Cur Loss: 0.00000000, Cur Avg Loss: 0.02093564, Log Avg loss: 0.00000155, Global Avg Loss: 0.04317002, Time: 0.5689 Steps: 80200, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 008832, Sample Num: 141312, Cur Loss: 0.00000000, Cur Avg Loss: 0.02046318, Log Avg loss: 0.00007173, Global Avg Loss: 0.04306281, Time: 0.3727 Steps: 80400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 009032, Sample Num: 144512, Cur Loss: 0.00000000, Cur Avg Loss: 0.02001008, Log Avg loss: 0.00000137, Global Avg Loss: 0.04295596, Time: 0.2168 Steps: 80600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009232, Sample Num: 147712, Cur Loss: 0.00000000, Cur Avg Loss: 0.02013907, Log Avg loss: 0.02596414, Global Avg Loss: 0.04291390, Time: 0.3329 Steps: 80800, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009432, Sample Num: 150912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02222510, Log Avg loss: 0.11851608, Global Avg Loss: 0.04310057, Time: 0.2600 Steps: 81000, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009632, Sample Num: 154112, Cur Loss: 0.00000000, Cur Avg Loss: 0.02176367, Log Avg loss: 0.00000301, Global Avg Loss: 0.04299442, Time: 0.1816 Steps: 81200, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 009832, Sample Num: 157312, Cur Loss: 0.00000000, Cur Avg Loss: 0.02138805, Log Avg loss: 0.00329789, Global Avg Loss: 0.04289688, Time: 0.2092 Steps: 81400, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 010032, Sample Num: 160512, Cur Loss: 0.00000000, Cur Avg Loss: 0.02096247, Log Avg loss: 0.00004085, Global Avg Loss: 0.04279185, Time: 1.1125 Steps: 81600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 010232, Sample Num: 163712, Cur Loss: 0.00000000, Cur Avg Loss: 0.02055305, Log Avg loss: 0.00001660, Global Avg Loss: 0.04268726, Time: 0.3029 Steps: 81800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010432, Sample Num: 166912, Cur Loss: 0.00000000, Cur Avg Loss: 0.02015917, Log Avg loss: 0.00000852, Global Avg Loss: 0.04258317, Time: 0.2236 Steps: 82000, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010632, Sample Num: 170112, Cur Loss: 0.00000000, Cur Avg Loss: 0.01977996, Log Avg loss: 0.00000014, Global Avg Loss: 0.04247956, Time: 0.2482 Steps: 82200, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 010832, Sample Num: 173312, Cur Loss: 0.00000000, Cur Avg Loss: 0.01941475, Log Avg loss: 0.00000013, Global Avg Loss: 0.04237645, Time: 0.5257 Steps: 82400, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011032, Sample Num: 176512, Cur Loss: 0.00000006, Cur Avg Loss: 0.01907114, Log Avg loss: 0.00046137, Global Avg Loss: 0.04227496, Time: 0.5888 Steps: 82600, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011232, Sample Num: 179712, Cur Loss: 0.00000047, Cur Avg Loss: 0.01874260, Log Avg loss: 0.00062057, Global Avg Loss: 0.04217435, Time: 0.5906 Steps: 82800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 011432, Sample Num: 182912, Cur Loss: 0.00000000, Cur Avg Loss: 0.01841471, Log Avg loss: 0.00000001, Global Avg Loss: 0.04207272, Time: 0.5950 Steps: 83000, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 011632, Sample Num: 186112, Cur Loss: 0.00000000, Cur Avg Loss: 0.01809825, Log Avg loss: 0.00000951, Global Avg Loss: 0.04197161, Time: 0.6363 Steps: 83200, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 011832, Sample Num: 189312, Cur Loss: 0.00001491, Cur Avg Loss: 0.01782125, Log Avg loss: 0.00171131, Global Avg Loss: 0.04187506, Time: 0.9094 Steps: 83400, Updated lr: 0.000030 ***** Running evaluation checkpoint-83496 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-83496 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4895.871289, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.030185, "eval_total_loss": 42.047723, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999925, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.038441, "test_total_loss": 53.548473, "test_acc": 0.99982, "test_prec": 0.996296, "test_recall": 0.996296, "test_f1": 0.996296, "test_roc_auc": 0.999998, "test_pr_auc": 0.999927, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 2, "tp": 538}, "test_mcc2": 0.996204, "test_mcc": 0.996204, "test_sn": 0.996296, "test_sp": 0.999908, "lr": 3.005038629492778e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.04182691669746372, "train_cur_epoch_loss": 210.86108656574447, "train_cur_epoch_avg_loss": 0.017677824158764628, "train_cur_epoch_time": 4895.871289491653, "train_cur_epoch_avg_time": 0.4104519860405477, "epoch": 7, "step": 83496} ################################################## Training, Epoch: 0008, Batch: 000104, Sample Num: 1664, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000032, Log Avg loss: 0.00000017, Global Avg Loss: 0.04177488, Time: 0.8811 Steps: 83600, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000304, Sample Num: 4864, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000050, Log Avg loss: 0.00000059, Global Avg Loss: 0.04167518, Time: 0.4256 Steps: 83800, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000504, Sample Num: 8064, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000091, Log Avg loss: 0.00000155, Global Avg Loss: 0.04157596, Time: 0.2216 Steps: 84000, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000704, Sample Num: 11264, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000068, Log Avg loss: 0.00000010, Global Avg Loss: 0.04147721, Time: 0.3073 Steps: 84200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 000904, Sample Num: 14464, Cur Loss: 0.00000000, Cur Avg Loss: 0.00026954, Log Avg loss: 0.00121591, Global Avg Loss: 0.04138180, Time: 0.2502 Steps: 84400, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001104, Sample Num: 17664, Cur Loss: 0.00000000, Cur Avg Loss: 0.04045015, Log Avg loss: 0.22206653, Global Avg Loss: 0.04180895, Time: 0.4531 Steps: 84600, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001304, Sample Num: 20864, Cur Loss: 0.00000000, Cur Avg Loss: 0.03424619, Log Avg loss: 0.00000032, Global Avg Loss: 0.04171035, Time: 0.6244 Steps: 84800, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001504, Sample Num: 24064, Cur Loss: 0.00000002, Cur Avg Loss: 0.02990710, Log Avg loss: 0.00161626, Global Avg Loss: 0.04161601, Time: 0.6816 Steps: 85000, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001704, Sample Num: 27264, Cur Loss: 0.00000000, Cur Avg Loss: 0.02899379, Log Avg loss: 0.02212564, Global Avg Loss: 0.04157025, Time: 0.5930 Steps: 85200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001904, Sample Num: 30464, Cur Loss: 0.00000000, Cur Avg Loss: 0.02594880, Log Avg loss: 0.00000550, Global Avg Loss: 0.04147291, Time: 0.2017 Steps: 85400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002104, Sample Num: 33664, Cur Loss: 0.00000000, Cur Avg Loss: 0.02411715, Log Avg loss: 0.00667984, Global Avg Loss: 0.04139162, Time: 0.5671 Steps: 85600, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002304, Sample Num: 36864, Cur Loss: 0.00000000, Cur Avg Loss: 0.02761740, Log Avg loss: 0.06444003, Global Avg Loss: 0.04144535, Time: 0.2907 Steps: 85800, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002504, Sample Num: 40064, Cur Loss: 0.00000000, Cur Avg Loss: 0.02541156, Log Avg loss: 0.00000027, Global Avg Loss: 0.04134896, Time: 0.5116 Steps: 86000, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002704, Sample Num: 43264, Cur Loss: 0.00000000, Cur Avg Loss: 0.03556770, Log Avg loss: 0.16272265, Global Avg Loss: 0.04163057, Time: 0.2596 Steps: 86200, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 002904, Sample Num: 46464, Cur Loss: 0.00000000, Cur Avg Loss: 0.03312004, Log Avg loss: 0.00002762, Global Avg Loss: 0.04153427, Time: 0.2166 Steps: 86400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 003104, Sample Num: 49664, Cur Loss: 0.00000000, Cur Avg Loss: 0.03098602, Log Avg loss: 0.00000003, Global Avg Loss: 0.04143835, Time: 0.2588 Steps: 86600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003304, Sample Num: 52864, Cur Loss: 0.00000000, Cur Avg Loss: 0.02916662, Log Avg loss: 0.00092954, Global Avg Loss: 0.04134501, Time: 0.3958 Steps: 86800, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003504, Sample Num: 56064, Cur Loss: 0.00000000, Cur Avg Loss: 0.02750204, Log Avg loss: 0.00000324, Global Avg Loss: 0.04124997, Time: 0.3123 Steps: 87000, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003704, Sample Num: 59264, Cur Loss: 0.00000000, Cur Avg Loss: 0.03059246, Log Avg loss: 0.08473666, Global Avg Loss: 0.04134971, Time: 0.5542 Steps: 87200, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 003904, Sample Num: 62464, Cur Loss: 0.00000000, Cur Avg Loss: 0.02902524, Log Avg loss: 0.00000024, Global Avg Loss: 0.04125509, Time: 1.0405 Steps: 87400, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 004104, Sample Num: 65664, Cur Loss: 0.00000000, Cur Avg Loss: 0.02761082, Log Avg loss: 0.00000139, Global Avg Loss: 0.04116090, Time: 0.6862 Steps: 87600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 004304, Sample Num: 68864, Cur Loss: 0.00000000, Cur Avg Loss: 0.02632788, Log Avg loss: 0.00000183, Global Avg Loss: 0.04106715, Time: 0.1593 Steps: 87800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004504, Sample Num: 72064, Cur Loss: 0.00000000, Cur Avg Loss: 0.02517379, Log Avg loss: 0.00033789, Global Avg Loss: 0.04097458, Time: 0.8708 Steps: 88000, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004704, Sample Num: 75264, Cur Loss: 0.00000000, Cur Avg Loss: 0.02410353, Log Avg loss: 0.00000126, Global Avg Loss: 0.04088167, Time: 0.5603 Steps: 88200, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 004904, Sample Num: 78464, Cur Loss: 0.00000000, Cur Avg Loss: 0.02329901, Log Avg loss: 0.00437656, Global Avg Loss: 0.04079908, Time: 0.2449 Steps: 88400, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005104, Sample Num: 81664, Cur Loss: 0.00000000, Cur Avg Loss: 0.02238605, Log Avg loss: 0.00000044, Global Avg Loss: 0.04070698, Time: 0.2917 Steps: 88600, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005304, Sample Num: 84864, Cur Loss: 0.00000042, Cur Avg Loss: 0.02154199, Log Avg loss: 0.00000152, Global Avg Loss: 0.04061530, Time: 0.2521 Steps: 88800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 005504, Sample Num: 88064, Cur Loss: 0.00000000, Cur Avg Loss: 0.02076225, Log Avg loss: 0.00008344, Global Avg Loss: 0.04052422, Time: 0.3355 Steps: 89000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 005704, Sample Num: 91264, Cur Loss: 0.00000000, Cur Avg Loss: 0.02003426, Log Avg loss: 0.00000006, Global Avg Loss: 0.04043336, Time: 1.2743 Steps: 89200, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 005904, Sample Num: 94464, Cur Loss: 0.00000000, Cur Avg Loss: 0.01935559, Log Avg loss: 0.00000005, Global Avg Loss: 0.04034290, Time: 0.3864 Steps: 89400, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006104, Sample Num: 97664, Cur Loss: 0.00000000, Cur Avg Loss: 0.02137078, Log Avg loss: 0.08085913, Global Avg Loss: 0.04043334, Time: 0.4393 Steps: 89600, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006304, Sample Num: 100864, Cur Loss: 0.00000000, Cur Avg Loss: 0.02069285, Log Avg loss: 0.00000242, Global Avg Loss: 0.04034330, Time: 0.1617 Steps: 89800, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006504, Sample Num: 104064, Cur Loss: 0.00000364, Cur Avg Loss: 0.02012244, Log Avg loss: 0.00214320, Global Avg Loss: 0.04025841, Time: 0.2607 Steps: 90000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 006704, Sample Num: 107264, Cur Loss: 0.00000000, Cur Avg Loss: 0.01952223, Log Avg loss: 0.00000315, Global Avg Loss: 0.04016915, Time: 0.2289 Steps: 90200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 006904, Sample Num: 110464, Cur Loss: 0.00000000, Cur Avg Loss: 0.01895670, Log Avg loss: 0.00000009, Global Avg Loss: 0.04008028, Time: 0.5173 Steps: 90400, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007104, Sample Num: 113664, Cur Loss: 0.00000000, Cur Avg Loss: 0.01912957, Log Avg loss: 0.02509734, Global Avg Loss: 0.04004720, Time: 0.2998 Steps: 90600, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007304, Sample Num: 116864, Cur Loss: 0.00000012, Cur Avg Loss: 0.01860587, Log Avg loss: 0.00000389, Global Avg Loss: 0.03995900, Time: 0.2923 Steps: 90800, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007504, Sample Num: 120064, Cur Loss: 0.00000000, Cur Avg Loss: 0.01810998, Log Avg loss: 0.00000008, Global Avg Loss: 0.03987118, Time: 0.3947 Steps: 91000, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007704, Sample Num: 123264, Cur Loss: 0.00000000, Cur Avg Loss: 0.01764007, Log Avg loss: 0.00000924, Global Avg Loss: 0.03978377, Time: 0.8064 Steps: 91200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 007904, Sample Num: 126464, Cur Loss: 0.00000000, Cur Avg Loss: 0.01724457, Log Avg loss: 0.00200957, Global Avg Loss: 0.03970111, Time: 0.3632 Steps: 91400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008104, Sample Num: 129664, Cur Loss: 0.00000000, Cur Avg Loss: 0.01687326, Log Avg loss: 0.00219924, Global Avg Loss: 0.03961923, Time: 0.1416 Steps: 91600, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008304, Sample Num: 132864, Cur Loss: 0.00000000, Cur Avg Loss: 0.01646756, Log Avg loss: 0.00002845, Global Avg Loss: 0.03953297, Time: 0.2091 Steps: 91800, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008504, Sample Num: 136064, Cur Loss: 0.00000000, Cur Avg Loss: 0.01881301, Log Avg loss: 0.11619617, Global Avg Loss: 0.03969963, Time: 0.4399 Steps: 92000, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008704, Sample Num: 139264, Cur Loss: 0.00000000, Cur Avg Loss: 0.01838080, Log Avg loss: 0.00000345, Global Avg Loss: 0.03961352, Time: 0.4487 Steps: 92200, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 008904, Sample Num: 142464, Cur Loss: 0.00000000, Cur Avg Loss: 0.01796795, Log Avg loss: 0.00000057, Global Avg Loss: 0.03952778, Time: 0.3317 Steps: 92400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 009104, Sample Num: 145664, Cur Loss: 0.00000000, Cur Avg Loss: 0.01757333, Log Avg loss: 0.00000470, Global Avg Loss: 0.03944242, Time: 0.2897 Steps: 92600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009304, Sample Num: 148864, Cur Loss: 0.00000000, Cur Avg Loss: 0.01999530, Log Avg loss: 0.13024359, Global Avg Loss: 0.03963811, Time: 0.2911 Steps: 92800, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009504, Sample Num: 152064, Cur Loss: 0.00000000, Cur Avg Loss: 0.01957460, Log Avg loss: 0.00000336, Global Avg Loss: 0.03955287, Time: 0.7152 Steps: 93000, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009704, Sample Num: 155264, Cur Loss: 0.00000000, Cur Avg Loss: 0.01917117, Log Avg loss: 0.00000037, Global Avg Loss: 0.03946800, Time: 0.8261 Steps: 93200, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 009904, Sample Num: 158464, Cur Loss: 0.00000000, Cur Avg Loss: 0.01885208, Log Avg loss: 0.00337004, Global Avg Loss: 0.03939070, Time: 0.3555 Steps: 93400, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 010104, Sample Num: 161664, Cur Loss: 0.00000000, Cur Avg Loss: 0.01847953, Log Avg loss: 0.00003052, Global Avg Loss: 0.03930660, Time: 0.1259 Steps: 93600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 010304, Sample Num: 164864, Cur Loss: 0.00000000, Cur Avg Loss: 0.01812137, Log Avg loss: 0.00002704, Global Avg Loss: 0.03922284, Time: 0.5029 Steps: 93800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010504, Sample Num: 168064, Cur Loss: 0.00000000, Cur Avg Loss: 0.01777634, Log Avg loss: 0.00000070, Global Avg Loss: 0.03913939, Time: 0.5650 Steps: 94000, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010704, Sample Num: 171264, Cur Loss: 0.00000000, Cur Avg Loss: 0.01744421, Log Avg loss: 0.00000075, Global Avg Loss: 0.03905630, Time: 0.1959 Steps: 94200, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 010904, Sample Num: 174464, Cur Loss: 0.00000000, Cur Avg Loss: 0.01712425, Log Avg loss: 0.00000004, Global Avg Loss: 0.03897355, Time: 0.2721 Steps: 94400, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011104, Sample Num: 177664, Cur Loss: 0.00000000, Cur Avg Loss: 0.01681882, Log Avg loss: 0.00016656, Global Avg Loss: 0.03889151, Time: 0.3977 Steps: 94600, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011304, Sample Num: 180864, Cur Loss: 0.00000000, Cur Avg Loss: 0.01653274, Log Avg loss: 0.00064941, Global Avg Loss: 0.03881083, Time: 0.2216 Steps: 94800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 011504, Sample Num: 184064, Cur Loss: 0.00000000, Cur Avg Loss: 0.01624532, Log Avg loss: 0.00000051, Global Avg Loss: 0.03872912, Time: 0.5146 Steps: 95000, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 011704, Sample Num: 187264, Cur Loss: 0.00000000, Cur Avg Loss: 0.01596781, Log Avg loss: 0.00000576, Global Avg Loss: 0.03864777, Time: 0.1725 Steps: 95200, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 011904, Sample Num: 190464, Cur Loss: 0.00000000, Cur Avg Loss: 0.01574436, Log Avg loss: 0.00266800, Global Avg Loss: 0.03857234, Time: 0.2235 Steps: 95400, Updated lr: 0.000020 ***** Running evaluation checkpoint-95424 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-95424 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4879.954697, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.02811, "eval_total_loss": 39.157302, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999922, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.038262, "test_total_loss": 53.299245, "test_acc": 0.99982, "test_prec": 0.996296, "test_recall": 0.996296, "test_f1": 0.996296, "test_roc_auc": 0.999997, "test_pr_auc": 0.999898, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 2, "tp": 538}, "test_mcc2": 0.996204, "test_mcc": 0.996204, "test_sn": 0.996296, "test_sp": 0.999908, "lr": 2.003359086328519e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.0385626376281859, "train_cur_epoch_loss": 187.42089646058236, "train_cur_epoch_avg_loss": 0.015712684143241312, "train_cur_epoch_time": 4879.954697370529, "train_cur_epoch_avg_time": 0.40911759702972245, "epoch": 8, "step": 95424} ################################################## Training, Epoch: 0009, Batch: 000176, Sample Num: 2816, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000147, Log Avg loss: 0.00000129, Global Avg Loss: 0.03849165, Time: 0.3648 Steps: 95600, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000376, Sample Num: 6016, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000083, Log Avg loss: 0.00000026, Global Avg Loss: 0.03841129, Time: 0.2175 Steps: 95800, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000576, Sample Num: 9216, Cur Loss: 0.00000006, Cur Avg Loss: 0.00000054, Log Avg loss: 0.00000002, Global Avg Loss: 0.03833127, Time: 0.9822 Steps: 96000, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000776, Sample Num: 12416, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002350, Log Avg loss: 0.00008960, Global Avg Loss: 0.03825176, Time: 0.8140 Steps: 96200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 000976, Sample Num: 15616, Cur Loss: 0.00000000, Cur Avg Loss: 0.04541303, Log Avg loss: 0.22152442, Global Avg Loss: 0.03863199, Time: 0.2228 Steps: 96400, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001176, Sample Num: 18816, Cur Loss: 0.00000000, Cur Avg Loss: 0.03778231, Log Avg loss: 0.00054441, Global Avg Loss: 0.03855314, Time: 0.3338 Steps: 96600, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001376, Sample Num: 22016, Cur Loss: 0.00000000, Cur Avg Loss: 0.03229071, Log Avg loss: 0.00000009, Global Avg Loss: 0.03847348, Time: 0.2549 Steps: 96800, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001576, Sample Num: 25216, Cur Loss: 0.00000000, Cur Avg Loss: 0.02969268, Log Avg loss: 0.01181820, Global Avg Loss: 0.03841852, Time: 0.3325 Steps: 97000, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001776, Sample Num: 28416, Cur Loss: 0.00000000, Cur Avg Loss: 0.02634891, Log Avg loss: 0.00000006, Global Avg Loss: 0.03833947, Time: 1.2746 Steps: 97200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001976, Sample Num: 31616, Cur Loss: 0.00000000, Cur Avg Loss: 0.02368205, Log Avg loss: 0.00000025, Global Avg Loss: 0.03826075, Time: 0.3537 Steps: 97400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002176, Sample Num: 34816, Cur Loss: 0.00000000, Cur Avg Loss: 0.02441322, Log Avg loss: 0.03163726, Global Avg Loss: 0.03824718, Time: 0.2632 Steps: 97600, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002376, Sample Num: 38016, Cur Loss: 0.00000000, Cur Avg Loss: 0.02242667, Log Avg loss: 0.00081297, Global Avg Loss: 0.03817062, Time: 0.4255 Steps: 97800, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002576, Sample Num: 41216, Cur Loss: 0.00000000, Cur Avg Loss: 0.02549839, Log Avg loss: 0.06199038, Global Avg Loss: 0.03821923, Time: 0.2203 Steps: 98000, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002776, Sample Num: 44416, Cur Loss: 0.00000000, Cur Avg Loss: 0.03174656, Log Avg loss: 0.11222300, Global Avg Loss: 0.03836995, Time: 0.3195 Steps: 98200, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 002976, Sample Num: 47616, Cur Loss: 0.00000000, Cur Avg Loss: 0.02961311, Log Avg loss: 0.00000081, Global Avg Loss: 0.03829197, Time: 0.2536 Steps: 98400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 003176, Sample Num: 50816, Cur Loss: 0.00000000, Cur Avg Loss: 0.02774831, Log Avg loss: 0.00000006, Global Avg Loss: 0.03821430, Time: 0.2629 Steps: 98600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003376, Sample Num: 54016, Cur Loss: 0.00000000, Cur Avg Loss: 0.02614679, Log Avg loss: 0.00071478, Global Avg Loss: 0.03813839, Time: 0.2030 Steps: 98800, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003576, Sample Num: 57216, Cur Loss: 0.00000000, Cur Avg Loss: 0.02967893, Log Avg loss: 0.08930138, Global Avg Loss: 0.03824175, Time: 0.4440 Steps: 99000, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003776, Sample Num: 60416, Cur Loss: 0.00000000, Cur Avg Loss: 0.02811862, Log Avg loss: 0.00022026, Global Avg Loss: 0.03816509, Time: 0.7535 Steps: 99200, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 003976, Sample Num: 63616, Cur Loss: 0.00000000, Cur Avg Loss: 0.02670423, Log Avg loss: 0.00000060, Global Avg Loss: 0.03808830, Time: 1.0188 Steps: 99400, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 004176, Sample Num: 66816, Cur Loss: 0.00000000, Cur Avg Loss: 0.02542532, Log Avg loss: 0.00000055, Global Avg Loss: 0.03801182, Time: 0.5441 Steps: 99600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 004376, Sample Num: 70016, Cur Loss: 0.00000000, Cur Avg Loss: 0.02426922, Log Avg loss: 0.00012990, Global Avg Loss: 0.03793590, Time: 0.2001 Steps: 99800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004576, Sample Num: 73216, Cur Loss: 0.00000000, Cur Avg Loss: 0.02320851, Log Avg loss: 0.00000009, Global Avg Loss: 0.03786003, Time: 0.4723 Steps: 100000, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004776, Sample Num: 76416, Cur Loss: 0.00000000, Cur Avg Loss: 0.02223663, Log Avg loss: 0.00000017, Global Avg Loss: 0.03778446, Time: 0.2000 Steps: 100200, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 004976, Sample Num: 79616, Cur Loss: 0.00000000, Cur Avg Loss: 0.02140196, Log Avg loss: 0.00146986, Global Avg Loss: 0.03771212, Time: 0.2474 Steps: 100400, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005176, Sample Num: 82816, Cur Loss: 0.00000000, Cur Avg Loss: 0.02057501, Log Avg loss: 0.00000051, Global Avg Loss: 0.03763715, Time: 0.5985 Steps: 100600, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005376, Sample Num: 86016, Cur Loss: 0.00000000, Cur Avg Loss: 0.01980958, Log Avg loss: 0.00000036, Global Avg Loss: 0.03756247, Time: 0.3625 Steps: 100800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 005576, Sample Num: 89216, Cur Loss: 0.00000000, Cur Avg Loss: 0.01910033, Log Avg loss: 0.00003575, Global Avg Loss: 0.03748816, Time: 0.6084 Steps: 101000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 005776, Sample Num: 92416, Cur Loss: 0.00000000, Cur Avg Loss: 0.01843897, Log Avg loss: 0.00000004, Global Avg Loss: 0.03741408, Time: 0.5680 Steps: 101200, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 005976, Sample Num: 95616, Cur Loss: 0.00000000, Cur Avg Loss: 0.01782187, Log Avg loss: 0.00000001, Global Avg Loss: 0.03734028, Time: 0.1680 Steps: 101400, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006176, Sample Num: 98816, Cur Loss: 0.00000000, Cur Avg Loss: 0.01912725, Log Avg loss: 0.05813208, Global Avg Loss: 0.03738121, Time: 0.3750 Steps: 101600, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006376, Sample Num: 102016, Cur Loss: 0.00000000, Cur Avg Loss: 0.01859300, Log Avg loss: 0.00209533, Global Avg Loss: 0.03731189, Time: 0.2443 Steps: 101800, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006576, Sample Num: 105216, Cur Loss: 0.00000000, Cur Avg Loss: 0.01802760, Log Avg loss: 0.00000259, Global Avg Loss: 0.03723873, Time: 0.2282 Steps: 102000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 006776, Sample Num: 108416, Cur Loss: 0.00000000, Cur Avg Loss: 0.01749550, Log Avg loss: 0.00000006, Global Avg Loss: 0.03716586, Time: 0.2018 Steps: 102200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 006976, Sample Num: 111616, Cur Loss: 0.00000000, Cur Avg Loss: 0.01699391, Log Avg loss: 0.00000018, Global Avg Loss: 0.03709327, Time: 0.3591 Steps: 102400, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007176, Sample Num: 114816, Cur Loss: 0.00000000, Cur Avg Loss: 0.01677685, Log Avg loss: 0.00920584, Global Avg Loss: 0.03703891, Time: 0.4022 Steps: 102600, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007376, Sample Num: 118016, Cur Loss: 0.00000000, Cur Avg Loss: 0.01632212, Log Avg loss: 0.00000633, Global Avg Loss: 0.03696686, Time: 0.3242 Steps: 102800, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007576, Sample Num: 121216, Cur Loss: 0.00000000, Cur Avg Loss: 0.01589125, Log Avg loss: 0.00000069, Global Avg Loss: 0.03689508, Time: 0.2808 Steps: 103000, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007776, Sample Num: 124416, Cur Loss: 0.00000000, Cur Avg Loss: 0.01548252, Log Avg loss: 0.00000002, Global Avg Loss: 0.03682358, Time: 0.2089 Steps: 103200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 007976, Sample Num: 127616, Cur Loss: 0.00000000, Cur Avg Loss: 0.01512544, Log Avg loss: 0.00124204, Global Avg Loss: 0.03675475, Time: 0.3766 Steps: 103400, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008176, Sample Num: 130816, Cur Loss: 0.00000000, Cur Avg Loss: 0.01480385, Log Avg loss: 0.00197904, Global Avg Loss: 0.03668762, Time: 0.1662 Steps: 103600, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008376, Sample Num: 134016, Cur Loss: 0.00000001, Cur Avg Loss: 0.01448619, Log Avg loss: 0.00150011, Global Avg Loss: 0.03661982, Time: 1.2739 Steps: 103800, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008576, Sample Num: 137216, Cur Loss: 0.00000000, Cur Avg Loss: 0.01749627, Log Avg loss: 0.14355849, Global Avg Loss: 0.03682547, Time: 0.4102 Steps: 104000, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008776, Sample Num: 140416, Cur Loss: 0.00000000, Cur Avg Loss: 0.01709756, Log Avg loss: 0.00000083, Global Avg Loss: 0.03675479, Time: 0.3003 Steps: 104200, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 008976, Sample Num: 143616, Cur Loss: 0.00000000, Cur Avg Loss: 0.01671662, Log Avg loss: 0.00000089, Global Avg Loss: 0.03668438, Time: 1.0581 Steps: 104400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009176, Sample Num: 146816, Cur Loss: 0.00000000, Cur Avg Loss: 0.01635226, Log Avg loss: 0.00000003, Global Avg Loss: 0.03661424, Time: 0.3100 Steps: 104600, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009376, Sample Num: 150016, Cur Loss: 0.00000000, Cur Avg Loss: 0.01879781, Log Avg loss: 0.13099959, Global Avg Loss: 0.03679436, Time: 0.3308 Steps: 104800, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009576, Sample Num: 153216, Cur Loss: 0.00000000, Cur Avg Loss: 0.01840521, Log Avg loss: 0.00000012, Global Avg Loss: 0.03672428, Time: 1.0960 Steps: 105000, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009776, Sample Num: 156416, Cur Loss: 0.00001734, Cur Avg Loss: 0.01808278, Log Avg loss: 0.00264479, Global Avg Loss: 0.03665949, Time: 0.3754 Steps: 105200, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 009976, Sample Num: 159616, Cur Loss: 0.00000000, Cur Avg Loss: 0.01772490, Log Avg loss: 0.00023176, Global Avg Loss: 0.03659037, Time: 0.4098 Steps: 105400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 010176, Sample Num: 162816, Cur Loss: 0.00000000, Cur Avg Loss: 0.01737808, Log Avg loss: 0.00007836, Global Avg Loss: 0.03652122, Time: 0.1799 Steps: 105600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010376, Sample Num: 166016, Cur Loss: 0.00000000, Cur Avg Loss: 0.01704396, Log Avg loss: 0.00004389, Global Avg Loss: 0.03645226, Time: 1.0140 Steps: 105800, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010576, Sample Num: 169216, Cur Loss: 0.00000495, Cur Avg Loss: 0.01672165, Log Avg loss: 0.00000048, Global Avg Loss: 0.03638348, Time: 1.2747 Steps: 106000, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010776, Sample Num: 172416, Cur Loss: 0.00000012, Cur Avg Loss: 0.01641131, Log Avg loss: 0.00000032, Global Avg Loss: 0.03631497, Time: 0.1956 Steps: 106200, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 010976, Sample Num: 175616, Cur Loss: 0.00000000, Cur Avg Loss: 0.01611227, Log Avg loss: 0.00000001, Global Avg Loss: 0.03624670, Time: 0.3226 Steps: 106400, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 011176, Sample Num: 178816, Cur Loss: 0.00000000, Cur Avg Loss: 0.01583077, Log Avg loss: 0.00038203, Global Avg Loss: 0.03617942, Time: 0.3240 Steps: 106600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 011376, Sample Num: 182016, Cur Loss: 0.00000000, Cur Avg Loss: 0.01555245, Log Avg loss: 0.00000003, Global Avg Loss: 0.03611166, Time: 0.2328 Steps: 106800, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 011576, Sample Num: 185216, Cur Loss: 0.00000000, Cur Avg Loss: 0.01528375, Log Avg loss: 0.00000016, Global Avg Loss: 0.03604417, Time: 0.3683 Steps: 107000, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 011776, Sample Num: 188416, Cur Loss: 0.00000001, Cur Avg Loss: 0.01502433, Log Avg loss: 0.00000924, Global Avg Loss: 0.03597694, Time: 0.6893 Steps: 107200, Updated lr: 0.000010 ***** Running evaluation checkpoint-107352 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-107352 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4896.434173, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.023969, "eval_total_loss": 33.388665, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999922, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.035269, "test_total_loss": 49.129152, "test_acc": 0.99982, "test_prec": 0.996296, "test_recall": 0.996296, "test_f1": 0.996296, "test_roc_auc": 0.999998, "test_pr_auc": 0.999905, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 2, "tp": 538}, "test_mcc2": 0.996204, "test_mcc": 0.996204, "test_sn": 0.996296, "test_sp": 0.999908, "lr": 1.0016795431642594e-05, "cur_epoch_step": 11928, "train_global_avg_loss": 0.035930652985462656, "train_cur_epoch_loss": 177.42632626339258, "train_cur_epoch_avg_loss": 0.014874775843678117, "train_cur_epoch_time": 4896.434173107147, "train_cur_epoch_avg_time": 0.4104991761491572, "epoch": 9, "step": 107352} ################################################## Training, Epoch: 0010, Batch: 000048, Sample Num: 768, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000275, Log Avg loss: 0.00249961, Global Avg Loss: 0.03591460, Time: 0.3515 Steps: 107400, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000248, Sample Num: 3968, Cur Loss: 0.00000012, Cur Avg Loss: 0.00000414, Log Avg loss: 0.00000448, Global Avg Loss: 0.03584785, Time: 0.2892 Steps: 107600, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000448, Sample Num: 7168, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000233, Log Avg loss: 0.00000009, Global Avg Loss: 0.03578134, Time: 0.2834 Steps: 107800, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000648, Sample Num: 10368, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000163, Log Avg loss: 0.00000006, Global Avg Loss: 0.03571508, Time: 0.1729 Steps: 108000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 000848, Sample Num: 13568, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000240, Log Avg loss: 0.00000488, Global Avg Loss: 0.03564907, Time: 0.2218 Steps: 108200, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001048, Sample Num: 16768, Cur Loss: 0.00000000, Cur Avg Loss: 0.03376579, Log Avg loss: 0.17692257, Global Avg Loss: 0.03590972, Time: 0.2195 Steps: 108400, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001248, Sample Num: 19968, Cur Loss: 0.00000000, Cur Avg Loss: 0.02835461, Log Avg loss: 0.00000001, Global Avg Loss: 0.03584359, Time: 0.5827 Steps: 108600, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001448, Sample Num: 23168, Cur Loss: 0.00000000, Cur Avg Loss: 0.02460143, Log Avg loss: 0.00118163, Global Avg Loss: 0.03577987, Time: 0.2800 Steps: 108800, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001648, Sample Num: 26368, Cur Loss: 0.00000000, Cur Avg Loss: 0.02189025, Log Avg loss: 0.00226131, Global Avg Loss: 0.03571837, Time: 0.1818 Steps: 109000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001848, Sample Num: 29568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01952119, Log Avg loss: 0.00000007, Global Avg Loss: 0.03565295, Time: 0.5108 Steps: 109200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002048, Sample Num: 32768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01821342, Log Avg loss: 0.00612968, Global Avg Loss: 0.03559898, Time: 0.2534 Steps: 109400, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002248, Sample Num: 35968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01746059, Log Avg loss: 0.00975160, Global Avg Loss: 0.03555181, Time: 0.2727 Steps: 109600, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002448, Sample Num: 39168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01603526, Log Avg loss: 0.00001457, Global Avg Loss: 0.03548708, Time: 0.1519 Steps: 109800, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002648, Sample Num: 42368, Cur Loss: 0.00000000, Cur Avg Loss: 0.02654827, Log Avg loss: 0.15522753, Global Avg Loss: 0.03570479, Time: 0.1807 Steps: 110000, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 002848, Sample Num: 45568, Cur Loss: 0.00000000, Cur Avg Loss: 0.02468597, Log Avg loss: 0.00002908, Global Avg Loss: 0.03564005, Time: 0.1820 Steps: 110200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 003048, Sample Num: 48768, Cur Loss: 0.00000000, Cur Avg Loss: 0.02306617, Log Avg loss: 0.00000019, Global Avg Loss: 0.03557548, Time: 0.2010 Steps: 110400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003248, Sample Num: 51968, Cur Loss: 0.00000000, Cur Avg Loss: 0.02164584, Log Avg loss: 0.00000004, Global Avg Loss: 0.03551115, Time: 1.2755 Steps: 110600, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003448, Sample Num: 55168, Cur Loss: 0.00000000, Cur Avg Loss: 0.02043449, Log Avg loss: 0.00076212, Global Avg Loss: 0.03544843, Time: 0.2628 Steps: 110800, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003648, Sample Num: 58368, Cur Loss: 0.00000000, Cur Avg Loss: 0.02242921, Log Avg loss: 0.05681820, Global Avg Loss: 0.03548693, Time: 0.2742 Steps: 111000, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 003848, Sample Num: 61568, Cur Loss: 0.00000000, Cur Avg Loss: 0.02127051, Log Avg loss: 0.00013579, Global Avg Loss: 0.03542335, Time: 0.2473 Steps: 111200, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 004048, Sample Num: 64768, Cur Loss: 0.00000000, Cur Avg Loss: 0.02021960, Log Avg loss: 0.00000012, Global Avg Loss: 0.03535975, Time: 0.2703 Steps: 111400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 004248, Sample Num: 67968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01926790, Log Avg loss: 0.00000555, Global Avg Loss: 0.03529639, Time: 0.2640 Steps: 111600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004448, Sample Num: 71168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01841850, Log Avg loss: 0.00037722, Global Avg Loss: 0.03523393, Time: 0.5442 Steps: 111800, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004648, Sample Num: 74368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01762598, Log Avg loss: 0.00000029, Global Avg Loss: 0.03517101, Time: 0.2932 Steps: 112000, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 004848, Sample Num: 77568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01690552, Log Avg loss: 0.00016197, Global Avg Loss: 0.03510860, Time: 0.5821 Steps: 112200, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005048, Sample Num: 80768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01623900, Log Avg loss: 0.00008256, Global Avg Loss: 0.03504628, Time: 0.5513 Steps: 112400, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005248, Sample Num: 83968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01562014, Log Avg loss: 0.00000026, Global Avg Loss: 0.03498403, Time: 0.2518 Steps: 112600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 005448, Sample Num: 87168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01504713, Log Avg loss: 0.00001115, Global Avg Loss: 0.03492202, Time: 1.2744 Steps: 112800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 005648, Sample Num: 90368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01451430, Log Avg loss: 0.00000001, Global Avg Loss: 0.03486021, Time: 0.2824 Steps: 113000, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 005848, Sample Num: 93568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01401791, Log Avg loss: 0.00000003, Global Avg Loss: 0.03479862, Time: 0.2006 Steps: 113200, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006048, Sample Num: 96768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01355436, Log Avg loss: 0.00000000, Global Avg Loss: 0.03473725, Time: 0.1500 Steps: 113400, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006248, Sample Num: 99968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01620931, Log Avg loss: 0.09649512, Global Avg Loss: 0.03484598, Time: 0.3481 Steps: 113600, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006448, Sample Num: 103168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01575881, Log Avg loss: 0.00168528, Global Avg Loss: 0.03478770, Time: 0.3030 Steps: 113800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 006648, Sample Num: 106368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01528474, Log Avg loss: 0.00000057, Global Avg Loss: 0.03472667, Time: 0.5919 Steps: 114000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 006848, Sample Num: 109568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01483834, Log Avg loss: 0.00000007, Global Avg Loss: 0.03466585, Time: 0.7609 Steps: 114200, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007048, Sample Num: 112768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01484217, Log Avg loss: 0.01497344, Global Avg Loss: 0.03463143, Time: 0.3338 Steps: 114400, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007248, Sample Num: 115968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01443265, Log Avg loss: 0.00000102, Global Avg Loss: 0.03457099, Time: 0.2023 Steps: 114600, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007448, Sample Num: 119168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01404512, Log Avg loss: 0.00000110, Global Avg Loss: 0.03451076, Time: 0.2533 Steps: 114800, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007648, Sample Num: 122368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01367785, Log Avg loss: 0.00000084, Global Avg Loss: 0.03445075, Time: 0.6289 Steps: 115000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 007848, Sample Num: 125568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01332930, Log Avg loss: 0.00000041, Global Avg Loss: 0.03439094, Time: 0.7107 Steps: 115200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008048, Sample Num: 128768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01307899, Log Avg loss: 0.00325695, Global Avg Loss: 0.03433698, Time: 0.3355 Steps: 115400, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008248, Sample Num: 131968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01276618, Log Avg loss: 0.00017877, Global Avg Loss: 0.03427788, Time: 0.4424 Steps: 115600, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008448, Sample Num: 135168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01540729, Log Avg loss: 0.12432677, Global Avg Loss: 0.03443340, Time: 1.2927 Steps: 115800, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008648, Sample Num: 138368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01505100, Log Avg loss: 0.00000110, Global Avg Loss: 0.03437404, Time: 0.2012 Steps: 116000, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 008848, Sample Num: 141568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01471100, Log Avg loss: 0.00000932, Global Avg Loss: 0.03431489, Time: 0.2884 Steps: 116200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 009048, Sample Num: 144768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01438587, Log Avg loss: 0.00000231, Global Avg Loss: 0.03425593, Time: 0.2319 Steps: 116400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009248, Sample Num: 147968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01407803, Log Avg loss: 0.00015142, Global Avg Loss: 0.03419744, Time: 0.2461 Steps: 116600, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009448, Sample Num: 151168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01642296, Log Avg loss: 0.12485264, Global Avg Loss: 0.03435267, Time: 0.2145 Steps: 116800, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009648, Sample Num: 154368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01608254, Log Avg loss: 0.00000074, Global Avg Loss: 0.03429395, Time: 0.3038 Steps: 117000, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 009848, Sample Num: 157568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01581395, Log Avg loss: 0.00285739, Global Avg Loss: 0.03424030, Time: 0.3193 Steps: 117200, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 010048, Sample Num: 160768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01550186, Log Avg loss: 0.00013458, Global Avg Loss: 0.03418220, Time: 0.8212 Steps: 117400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 010248, Sample Num: 163968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01519983, Log Avg loss: 0.00002580, Global Avg Loss: 0.03412411, Time: 0.1914 Steps: 117600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010448, Sample Num: 167168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01491717, Log Avg loss: 0.00043375, Global Avg Loss: 0.03406691, Time: 0.8151 Steps: 117800, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010648, Sample Num: 170368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01463699, Log Avg loss: 0.00000003, Global Avg Loss: 0.03400917, Time: 0.1665 Steps: 118000, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 010848, Sample Num: 173568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01436713, Log Avg loss: 0.00000003, Global Avg Loss: 0.03395163, Time: 0.2012 Steps: 118200, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011048, Sample Num: 176768, Cur Loss: 0.00000000, Cur Avg Loss: 0.01411150, Log Avg loss: 0.00024608, Global Avg Loss: 0.03389469, Time: 0.1861 Steps: 118400, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011248, Sample Num: 179968, Cur Loss: 0.00000000, Cur Avg Loss: 0.01386069, Log Avg loss: 0.00000575, Global Avg Loss: 0.03383754, Time: 0.6027 Steps: 118600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 011448, Sample Num: 183168, Cur Loss: 0.00000000, Cur Avg Loss: 0.01361854, Log Avg loss: 0.00000018, Global Avg Loss: 0.03378058, Time: 0.4634 Steps: 118800, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 011648, Sample Num: 186368, Cur Loss: 0.00000000, Cur Avg Loss: 0.01338471, Log Avg loss: 0.00000058, Global Avg Loss: 0.03372380, Time: 0.2064 Steps: 119000, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 011848, Sample Num: 189568, Cur Loss: 0.00000000, Cur Avg Loss: 0.01319208, Log Avg loss: 0.00197329, Global Avg Loss: 0.03367053, Time: 0.6045 Steps: 119200, Updated lr: 0.000000 ***** Running evaluation checkpoint-119280 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## ***** Running testing checkpoint-119280 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## Epoch Time: 4891.636017, Avg time per batch (s): 0.410000 {"eval_avg_loss": 0.025941, "eval_total_loss": 36.135352, "eval_acc": 0.999731, "eval_prec": 0.99262, "eval_recall": 0.996296, "eval_f1": 0.994455, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999912, "eval_confusion_matrix": {"tn": 21740, "fp": 4, "fn": 2, "tp": 538}, "eval_mcc2": 0.994319, "eval_mcc": 0.994319, "eval_sn": 0.996296, "eval_sp": 0.999816, "update_flag": false, "test_avg_loss": 0.037871, "test_total_loss": 52.753749, "test_acc": 0.99982, "test_prec": 0.996296, "test_recall": 0.996296, "test_f1": 0.996296, "test_roc_auc": 0.999997, "test_pr_auc": 0.999889, "test_confusion_matrix": {"tn": 21742, "fp": 2, "fn": 2, "tp": 538}, "test_mcc2": 0.996204, "test_mcc": 0.996204, "test_sn": 0.996296, "test_sp": 0.999908, "lr": 0.0, "cur_epoch_step": 11928, "train_global_avg_loss": 0.033647948299251836, "train_cur_epoch_loss": 156.29981383939287, "train_cur_epoch_avg_loss": 0.01310360612335621, "train_cur_epoch_time": 4891.636016845703, "train_cur_epoch_avg_time": 0.41009691623454925, "epoch": 10, "step": 119280} ################################################## #########################Best Metric######################### {"epoch": 1, "global_step": 11928, "eval_avg_loss": 0.001592, "eval_total_loss": 2.217617, "eval_acc": 0.999731, "eval_prec": 0.989011, "eval_recall": 1.0, "eval_f1": 0.994475, "eval_roc_auc": 0.999998, "eval_pr_auc": 0.999928, "eval_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 0, "tp": 540}, "eval_mcc2": 0.994353, "eval_mcc": 0.994353, "eval_sn": 1.0, "eval_sp": 0.999724, "update_flag": true, "test_avg_loss": 0.001329, "test_total_loss": 1.85087, "test_acc": 0.999731, "test_prec": 0.989011, "test_recall": 1.0, "test_f1": 0.994475, "test_roc_auc": 0.999999, "test_pr_auc": 0.99998, "test_confusion_matrix": {"tn": 21738, "fp": 6, "fn": 0, "tp": 540}, "test_mcc2": 0.994353, "test_mcc": 0.994353, "test_sn": 1.0, "test_sp": 0.999724} ################################################## Total Time: 97841.808865, Avg time per epoch(10 epochs): 9784.180000 ++++++++++++Validation+++++++++++++ best f1 global step: 11928 checkpoint path: ../models/RdRP/protein/binary_class/luca_base/seq_matrix/20250415141827/checkpoint-11928 ***** Running evaluation checkpoint-11928 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 22284 ################################################## {"evaluation_avg_loss_11928": 0.001592, "evaluation_total_loss_11928": 2.217617, "evaluation_acc_11928": 0.999731, "evaluation_prec_11928": 0.989011, "evaluation_recall_11928": 1.0, "evaluation_f1_11928": 0.994475, "evaluation_roc_auc_11928": 0.999998, "evaluation_pr_auc_11928": 0.999928, "evaluation_confusion_matrix_11928": {"tn": 21738, "fp": 6, "fn": 0, "tp": 540}, "evaluation_mcc2_11928": 0.994353, "evaluation_mcc_11928": 0.994353, "evaluation_sn_11928": 1.0, "evaluation_sp_11928": 0.999724} ++++++++++++Testing+++++++++++++ best f1 global step: 11928 checkpoint path: ../models/RdRP/protein/binary_class/luca_base/seq_matrix/20250415141827/checkpoint-11928 ***** Running testing checkpoint-11928 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [22284] ################################################## {"evaluation_avg_loss_11928": 0.001329, "evaluation_total_loss_11928": 1.85087, "evaluation_acc_11928": 0.999731, "evaluation_prec_11928": 0.989011, "evaluation_recall_11928": 1.0, "evaluation_f1_11928": 0.994475, "evaluation_roc_auc_11928": 0.999999, "evaluation_pr_auc_11928": 0.99998, "evaluation_confusion_matrix_11928": {"tn": 21738, "fp": 6, "fn": 0, "tp": 540}, "evaluation_mcc2_11928": 0.994353, "evaluation_mcc_11928": 0.994353, "evaluation_sn_11928": 1.0, "evaluation_sp_11928": 0.999724}