{"adam_epsilon": 1e-08, "alphabet": "gene_prot", "append_eos": true, "asl_gamma_neg": 4.0, "asl_gamma_pos": 1.0, "best_metric_type": "acc", "beta1": 0.9, "beta2": 0.98, "buffer_size": 1024, "cache_dir": null, "classifier_activate_func": "gelu", "classifier_size": 128, "codes_file": null, "config_path": "../config/luca_base/luca_base_config.json", "cross_atten": false, "dataset_name": "ViralCapsid", "dataset_type": "protein", "delete_old": false, "dev_data_dir": "../dataset/ViralCapsid/protein/binary_class/dev/", "device": "cuda", "do_eval": true, "do_lower_case": false, "do_metrics": true, "do_predict": true, "do_train": true, "dropout_prob": 0.1, "early_stop_epoch": -1, "emb_activate_func": "gelu", "embedding_complete": true, "embedding_complete_seg_overlap": true, "embedding_fixed_len_a_time": 3072, "embedding_input_size": 2560, "embedding_input_size_a": null, "embedding_input_size_b": null, "eval_all_checkpoints": false, "evaluate_during_training": true, "evaluate_steps": -1, "evaluate_strategy": "epoch", "fc_activate_func": "gelu", "focal_loss_alpha": 0.7, "focal_loss_gamma": 2.0, "focal_loss_reduce": false, "fp16": false, "fp16_embedding": false, "fp16_opt_level": "O1", "fusion_type": "concat", "gradient_accumulation_steps": 1, "hidden_size": 1024, "ignore_index": -100, "input_mode": "single", "input_type": "matrix", "intermediate_size": 4096, "label_filepath": "../dataset/ViralCapsid/protein/binary_class/label.txt", "label_size": 2, "label_type": "ViralCapsid", "learning_rate": 0.0001, "llm_dir": "..", "llm_dirpath": "../llm/models/lucaone_virus/v1.0/token_level,span_level,seq_level/lucaone_virus/20240815023346/checkpoint-step3800000", "llm_step": "3800000", "llm_task_level": "token_level,span_level,seq_level", "llm_time_str": "20240815023346", "llm_type": "lucaone_virus", "llm_version": "v1.0", "local_rank": -1, "log_dir": "../logs/ViralCapsid/protein/binary_class/luca_base/matrix/20250103142554", "logging_steps": 200, "loss_reduction": "mean", "loss_type": "bce", "lr_decay_rate": 0.9, "lr_update_strategy": "step", "matrix_add_special_token": true, "matrix_dirpath": "../matrices/ViralCapsid/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000", "matrix_embedding_exists": false, "matrix_encoder": false, "matrix_encoder_act": false, "matrix_fc_size": "128", "matrix_max_length": 4096, "matrix_max_length_a": null, "matrix_max_length_b": null, "matrix_pooling_type": "value_attention", "max_grad_norm": 1.0, "max_sentence_length": null, "max_sentences": null, "max_steps": -1, "model_dirpath": null, "model_type": "luca_base", "n_gpu": 1, "no_cuda": false, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "non_ignore": false, "not_append_eos": false, "not_matrix_encoder_shared": false, "not_prepend_bos": false, "not_save_emb_to_disk": false, "not_seq_encoder_shared": false, "num_attention_heads": 0, "num_hidden_layers": 0, "num_train_epochs": 10, "output_dir": "../models/ViralCapsid/protein/binary_class/luca_base/matrix/20250103142554", "output_mode": "binary_class", "overwrite_cache": false, "overwrite_output_dir": true, "per_gpu_eval_batch_size": 16, "per_gpu_train_batch_size": 16, "pos_weight": 1.0, "position_embedding_type": "absolute", "prepend_bos": true, "save_all": true, "save_steps": -1, "seed": 1221, "self_atten": false, "seq_fc_size": "null", "seq_max_length": 4096, "seq_max_length_a": null, "seq_max_length_b": null, "seq_pooling_type": "value_attention", "seq_subword": false, "seq_vocab_path": "gene_prot", "sigmoid": true, "task_level_type": "seq_level", "task_type": "binary_class", "tb_log_dir": "../tb-logs/ViralCapsid/protein/binary_class/luca_base/matrix/20250103142554", "test_data_dir": "../dataset/ViralCapsid/protein/binary_class/test/", "time_str": "20250103142804", "train_data_dir": "../dataset/ViralCapsid/protein/binary_class/train/", "trunc_type": "right", "vector_dirpath": "../vectors/ViralCapsid/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000", "vector_fc_size": "null", "vocab_size": 39, "warmup_steps": 200, "weight": null, "weight_decay": 0.01, "worker_num": 0} ################################################## n_gpu: 1 ################################################## Inputs: Input Name List: protein,embedding_matrix ################################################## Encoder Config: {'llm_type': 'lucaone_virus', 'llm_version': 'v1.0', 'llm_step': '3800000', 'llm_dirpath': '../llm/models/lucaone_virus/v1.0/token_level,span_level,seq_level/lucaone_virus/20240815023346/checkpoint-step3800000', 'input_type': 'matrix', 'trunc_type': 'right', 'seq_max_length': 4096, 'atom_seq_max_length': None, 'vector_dirpath': '../vectors/ViralCapsid/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000', 'matrix_dirpath': '../matrices/ViralCapsid/protein/binary_class/luca_base/v1.0/lucaone_virus/20240815023346/3800000', 'local_rank': -1, 'max_sentence_length': None, 'max_sentences': None, 'matrix_add_special_token': True, 'embedding_complete': True, 'embedding_complete_seg_overlap': True, 'embedding_fixed_len_a_time': 3072, 'matrix_embedding_exists': False, 'save_emb_to_disk': True, 'fp16_embedding': False} ################################################## Model Config: LucaConfig { "alphabet": "gene_prot", "attention_probs_dropout_prob": 0.1, "classifier_activate_func": "gelu", "classifier_dropout_prob": 0.1, "classifier_size": 128, "cls_token_id": 2, "cross_atten": false, "directionality": "bidi", "emb_activate_func": "gelu", "embedding_input_size": 2560, "fc_activate_func": "gelu", "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 1024, "ignore_index": -100, "initializer_range": 0.02, "intermediate_size": 4096, "kernel_size": 7, "layer_norm_eps": 1e-12, "loss_reduction": "mean", "matrix_fc_size": [ 128 ], "matrix_max_length": 4096, "matrix_pooling_type": "value_attention", "max_position_embeddings": 4098, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "num_attention_heads": 8, "num_hidden_layers": 4, "pad_token_id": 0, "pos_weight": 1.0, "position_embedding_type": "absolute", "self_atten": false, "sep_token_id": 3, "seq_fc_size": null, "seq_max_length": 4096, "seq_pooling_type": "value_attention", "token_dropout": null, "transformers_version": "4.29.0", "type_vocab_size": 2, "use_luca_layer_norm_v2": true, "vector_fc_size": null, "vocab_size": 39 } ################################################## Mode Architecture: LucaBase( (matrix_pooler): GlobalMaskValueAttentionPooling1D (2560 -> 2560) (linear): ModuleList( (0): ModuleList( (0): Linear(in_features=2560, out_features=128, bias=True) (1): GELU(approximate='none') ) ) (dropout): Dropout(p=0.1, inplace=False) (hidden_layer): Linear(in_features=128, out_features=128, bias=True) (hidden_act): GELU(approximate='none') (classifier): Linear(in_features=128, out_features=1, bias=True) (output): Sigmoid() (loss_fct): MaskedBCEWithLogitsLoss( (criterion): BCEWithLogitsLoss() ) ) ################################################## Model parameters: 20005249 ################################################## {"total_num": "19.080000M", "total_size": "76.310000MB", "param_sum": "19.080000M", "param_size": "76.310000MB", "buffer_sum": "0.000000M", "buffer_size": "0.000000MB", "trainable_num": "19.078492M", "trainable_size": "76.313969MB"} ################################################## Train dataset len: 325113, batch size: 16, batch num: 20320 Train dataset t_total: 203200, max_steps: -1 ***** Running training ***** Train Dataset Num examples = 325113 Train Dataset Num Epochs = 10 Logging Steps = 200 Saving Steps = -1 Evaluating Strategy = epoch Train Dataset Instantaneous batch size per GPU = 16 Train Dataset Total train batch size (w. parallel, distributed & accumulation) = 16 Train Dataset Gradient Accumulation steps = 1 Train Dataset Total optimization steps = 203200 ################################################## Training, Epoch: 0001, Batch: 000200, Sample Num: 3200, Cur Loss: 0.30254388, Cur Avg Loss: 0.60527141, Log Avg loss: 0.60527141, Global Avg Loss: 0.60527141, Time: 0.0453 Steps: 200, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000400, Sample Num: 6400, Cur Loss: 0.25487703, Cur Avg Loss: 0.35105198, Log Avg loss: 0.09683255, Global Avg Loss: 0.35105198, Time: 0.0443 Steps: 400, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000600, Sample Num: 9600, Cur Loss: 0.01333562, Cur Avg Loss: 0.24802494, Log Avg loss: 0.04197087, Global Avg Loss: 0.24802494, Time: 0.0764 Steps: 600, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 000800, Sample Num: 12800, Cur Loss: 0.01222416, Cur Avg Loss: 0.19802354, Log Avg loss: 0.04801934, Global Avg Loss: 0.19802354, Time: 0.1180 Steps: 800, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 001000, Sample Num: 16000, Cur Loss: 0.01280217, Cur Avg Loss: 0.16503268, Log Avg loss: 0.03306921, Global Avg Loss: 0.16503268, Time: 0.0665 Steps: 1000, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 001200, Sample Num: 19200, Cur Loss: 0.00781911, Cur Avg Loss: 0.14435332, Log Avg loss: 0.04095655, Global Avg Loss: 0.14435332, Time: 0.0573 Steps: 1200, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 001400, Sample Num: 22400, Cur Loss: 0.06560751, Cur Avg Loss: 0.12980306, Log Avg loss: 0.04250146, Global Avg Loss: 0.12980306, Time: 0.0622 Steps: 1400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001600, Sample Num: 25600, Cur Loss: 0.00366292, Cur Avg Loss: 0.11668890, Log Avg loss: 0.02488981, Global Avg Loss: 0.11668890, Time: 0.1178 Steps: 1600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 001800, Sample Num: 28800, Cur Loss: 0.00317017, Cur Avg Loss: 0.10659591, Log Avg loss: 0.02585195, Global Avg Loss: 0.10659591, Time: 0.1245 Steps: 1800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002000, Sample Num: 32000, Cur Loss: 0.00068630, Cur Avg Loss: 0.09823771, Log Avg loss: 0.02301393, Global Avg Loss: 0.09823771, Time: 0.0384 Steps: 2000, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002200, Sample Num: 35200, Cur Loss: 0.00087628, Cur Avg Loss: 0.09166550, Log Avg loss: 0.02594337, Global Avg Loss: 0.09166550, Time: 0.1250 Steps: 2200, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002400, Sample Num: 38400, Cur Loss: 0.04968735, Cur Avg Loss: 0.08565797, Log Avg loss: 0.01957524, Global Avg Loss: 0.08565797, Time: 0.1090 Steps: 2400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002600, Sample Num: 41600, Cur Loss: 0.00308879, Cur Avg Loss: 0.08074235, Log Avg loss: 0.02175481, Global Avg Loss: 0.08074235, Time: 0.0502 Steps: 2600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002800, Sample Num: 44800, Cur Loss: 0.00031807, Cur Avg Loss: 0.07640534, Log Avg loss: 0.02002427, Global Avg Loss: 0.07640534, Time: 0.0553 Steps: 2800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003000, Sample Num: 48000, Cur Loss: 0.00408585, Cur Avg Loss: 0.07296893, Log Avg loss: 0.02485917, Global Avg Loss: 0.07296893, Time: 0.0642 Steps: 3000, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003200, Sample Num: 51200, Cur Loss: 0.00172926, Cur Avg Loss: 0.06925865, Log Avg loss: 0.01360443, Global Avg Loss: 0.06925865, Time: 0.2297 Steps: 3200, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003400, Sample Num: 54400, Cur Loss: 0.00027615, Cur Avg Loss: 0.06622573, Log Avg loss: 0.01769899, Global Avg Loss: 0.06622573, Time: 0.1121 Steps: 3400, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003600, Sample Num: 57600, Cur Loss: 0.00028369, Cur Avg Loss: 0.06359974, Log Avg loss: 0.01895801, Global Avg Loss: 0.06359974, Time: 0.2312 Steps: 3600, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 003800, Sample Num: 60800, Cur Loss: 0.40410638, Cur Avg Loss: 0.06129788, Log Avg loss: 0.01986442, Global Avg Loss: 0.06129788, Time: 0.0590 Steps: 3800, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 004000, Sample Num: 64000, Cur Loss: 0.00165107, Cur Avg Loss: 0.05910234, Log Avg loss: 0.01738705, Global Avg Loss: 0.05910234, Time: 0.0776 Steps: 4000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 004200, Sample Num: 67200, Cur Loss: 0.00107892, Cur Avg Loss: 0.05716610, Log Avg loss: 0.01844121, Global Avg Loss: 0.05716610, Time: 0.0365 Steps: 4200, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 004400, Sample Num: 70400, Cur Loss: 0.00064074, Cur Avg Loss: 0.05529631, Log Avg loss: 0.01603075, Global Avg Loss: 0.05529631, Time: 0.0524 Steps: 4400, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 004600, Sample Num: 73600, Cur Loss: 0.00015595, Cur Avg Loss: 0.05371103, Log Avg loss: 0.01883478, Global Avg Loss: 0.05371103, Time: 0.0580 Steps: 4600, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 004800, Sample Num: 76800, Cur Loss: 0.01044154, Cur Avg Loss: 0.05183614, Log Avg loss: 0.00871367, Global Avg Loss: 0.05183614, Time: 0.2243 Steps: 4800, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 005000, Sample Num: 80000, Cur Loss: 0.00157980, Cur Avg Loss: 0.05053396, Log Avg loss: 0.01928171, Global Avg Loss: 0.05053396, Time: 0.1227 Steps: 5000, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 005200, Sample Num: 83200, Cur Loss: 0.00016078, Cur Avg Loss: 0.04904545, Log Avg loss: 0.01183274, Global Avg Loss: 0.04904545, Time: 0.0374 Steps: 5200, Updated lr: 0.000098 Training, Epoch: 0001, Batch: 005400, Sample Num: 86400, Cur Loss: 0.00131254, Cur Avg Loss: 0.04783347, Log Avg loss: 0.01632196, Global Avg Loss: 0.04783347, Time: 0.0843 Steps: 5400, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 005600, Sample Num: 89600, Cur Loss: 0.00029321, Cur Avg Loss: 0.04662595, Log Avg loss: 0.01402281, Global Avg Loss: 0.04662595, Time: 0.2239 Steps: 5600, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 005800, Sample Num: 92800, Cur Loss: 0.00031165, Cur Avg Loss: 0.04543720, Log Avg loss: 0.01215238, Global Avg Loss: 0.04543720, Time: 0.0249 Steps: 5800, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 006000, Sample Num: 96000, Cur Loss: 0.01684300, Cur Avg Loss: 0.04437620, Log Avg loss: 0.01360723, Global Avg Loss: 0.04437620, Time: 0.1057 Steps: 6000, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 006200, Sample Num: 99200, Cur Loss: 0.00015774, Cur Avg Loss: 0.04347282, Log Avg loss: 0.01637123, Global Avg Loss: 0.04347282, Time: 0.1145 Steps: 6200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 006400, Sample Num: 102400, Cur Loss: 0.00127487, Cur Avg Loss: 0.04257303, Log Avg loss: 0.01467970, Global Avg Loss: 0.04257303, Time: 0.0653 Steps: 6400, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 006600, Sample Num: 105600, Cur Loss: 0.00029810, Cur Avg Loss: 0.04154697, Log Avg loss: 0.00871315, Global Avg Loss: 0.04154697, Time: 0.1106 Steps: 6600, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 006800, Sample Num: 108800, Cur Loss: 0.00080594, Cur Avg Loss: 0.04084233, Log Avg loss: 0.01758921, Global Avg Loss: 0.04084233, Time: 0.0378 Steps: 6800, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 007000, Sample Num: 112000, Cur Loss: 0.00013947, Cur Avg Loss: 0.03999552, Log Avg loss: 0.01120395, Global Avg Loss: 0.03999552, Time: 0.0421 Steps: 7000, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 007200, Sample Num: 115200, Cur Loss: 0.00021556, Cur Avg Loss: 0.03917957, Log Avg loss: 0.01062127, Global Avg Loss: 0.03917957, Time: 0.0404 Steps: 7200, Updated lr: 0.000097 Training, Epoch: 0001, Batch: 007400, Sample Num: 118400, Cur Loss: 0.00066298, Cur Avg Loss: 0.03851317, Log Avg loss: 0.01452269, Global Avg Loss: 0.03851317, Time: 0.0360 Steps: 7400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 007600, Sample Num: 121600, Cur Loss: 0.00018461, Cur Avg Loss: 0.03771321, Log Avg loss: 0.00811480, Global Avg Loss: 0.03771321, Time: 0.1496 Steps: 7600, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 007800, Sample Num: 124800, Cur Loss: 0.00065063, Cur Avg Loss: 0.03708736, Log Avg loss: 0.01330513, Global Avg Loss: 0.03708736, Time: 0.0469 Steps: 7800, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 008000, Sample Num: 128000, Cur Loss: 0.00049863, Cur Avg Loss: 0.03664447, Log Avg loss: 0.01937159, Global Avg Loss: 0.03664447, Time: 0.1893 Steps: 8000, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 008200, Sample Num: 131200, Cur Loss: 0.00028966, Cur Avg Loss: 0.03602956, Log Avg loss: 0.01143314, Global Avg Loss: 0.03602956, Time: 0.0550 Steps: 8200, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 008400, Sample Num: 134400, Cur Loss: 0.01183284, Cur Avg Loss: 0.03555290, Log Avg loss: 0.01600968, Global Avg Loss: 0.03555290, Time: 0.0741 Steps: 8400, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 008600, Sample Num: 137600, Cur Loss: 0.00375712, Cur Avg Loss: 0.03510664, Log Avg loss: 0.01636410, Global Avg Loss: 0.03510664, Time: 0.0560 Steps: 8600, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 008800, Sample Num: 140800, Cur Loss: 0.00035392, Cur Avg Loss: 0.03460265, Log Avg loss: 0.01293099, Global Avg Loss: 0.03460265, Time: 0.1001 Steps: 8800, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 009000, Sample Num: 144000, Cur Loss: 0.00019222, Cur Avg Loss: 0.03412161, Log Avg loss: 0.01295576, Global Avg Loss: 0.03412161, Time: 0.1446 Steps: 9000, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 009200, Sample Num: 147200, Cur Loss: 0.00009070, Cur Avg Loss: 0.03350128, Log Avg loss: 0.00558623, Global Avg Loss: 0.03350128, Time: 0.0464 Steps: 9200, Updated lr: 0.000096 Training, Epoch: 0001, Batch: 009400, Sample Num: 150400, Cur Loss: 0.00003713, Cur Avg Loss: 0.03296102, Log Avg loss: 0.00810906, Global Avg Loss: 0.03296102, Time: 0.0646 Steps: 9400, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 009600, Sample Num: 153600, Cur Loss: 0.00126145, Cur Avg Loss: 0.03247207, Log Avg loss: 0.00949145, Global Avg Loss: 0.03247207, Time: 0.1054 Steps: 9600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 009800, Sample Num: 156800, Cur Loss: 0.00197504, Cur Avg Loss: 0.03198293, Log Avg loss: 0.00850454, Global Avg Loss: 0.03198293, Time: 0.1061 Steps: 9800, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 010000, Sample Num: 160000, Cur Loss: 0.00012234, Cur Avg Loss: 0.03157038, Log Avg loss: 0.01135528, Global Avg Loss: 0.03157038, Time: 0.1171 Steps: 10000, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 010200, Sample Num: 163200, Cur Loss: 0.00074751, Cur Avg Loss: 0.03111289, Log Avg loss: 0.00823839, Global Avg Loss: 0.03111289, Time: 0.0635 Steps: 10200, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 010400, Sample Num: 166400, Cur Loss: 0.00079431, Cur Avg Loss: 0.03074729, Log Avg loss: 0.01210193, Global Avg Loss: 0.03074729, Time: 0.1030 Steps: 10400, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 010600, Sample Num: 169600, Cur Loss: 0.00022568, Cur Avg Loss: 0.03043023, Log Avg loss: 0.01394297, Global Avg Loss: 0.03043023, Time: 0.1143 Steps: 10600, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 010800, Sample Num: 172800, Cur Loss: 0.00008983, Cur Avg Loss: 0.02997656, Log Avg loss: 0.00593214, Global Avg Loss: 0.02997656, Time: 0.1134 Steps: 10800, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 011000, Sample Num: 176000, Cur Loss: 0.00241531, Cur Avg Loss: 0.02955801, Log Avg loss: 0.00695630, Global Avg Loss: 0.02955801, Time: 0.0588 Steps: 11000, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 011200, Sample Num: 179200, Cur Loss: 0.00049383, Cur Avg Loss: 0.02927083, Log Avg loss: 0.01347573, Global Avg Loss: 0.02927083, Time: 0.1489 Steps: 11200, Updated lr: 0.000095 Training, Epoch: 0001, Batch: 011400, Sample Num: 182400, Cur Loss: 0.00026508, Cur Avg Loss: 0.02899763, Log Avg loss: 0.01369857, Global Avg Loss: 0.02899763, Time: 0.0676 Steps: 11400, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 011600, Sample Num: 185600, Cur Loss: 0.00012453, Cur Avg Loss: 0.02868150, Log Avg loss: 0.01066186, Global Avg Loss: 0.02868150, Time: 0.1409 Steps: 11600, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 011800, Sample Num: 188800, Cur Loss: 0.00356217, Cur Avg Loss: 0.02843646, Log Avg loss: 0.01422447, Global Avg Loss: 0.02843646, Time: 0.0453 Steps: 11800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 012000, Sample Num: 192000, Cur Loss: 0.00725466, Cur Avg Loss: 0.02806132, Log Avg loss: 0.00592770, Global Avg Loss: 0.02806132, Time: 0.1065 Steps: 12000, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 012200, Sample Num: 195200, Cur Loss: 0.00072146, Cur Avg Loss: 0.02795636, Log Avg loss: 0.02165910, Global Avg Loss: 0.02795636, Time: 0.1171 Steps: 12200, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 012400, Sample Num: 198400, Cur Loss: 0.00013571, Cur Avg Loss: 0.02762957, Log Avg loss: 0.00769545, Global Avg Loss: 0.02762957, Time: 0.0672 Steps: 12400, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 012600, Sample Num: 201600, Cur Loss: 0.00020614, Cur Avg Loss: 0.02736362, Log Avg loss: 0.01087446, Global Avg Loss: 0.02736362, Time: 0.2280 Steps: 12600, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 012800, Sample Num: 204800, Cur Loss: 0.00007880, Cur Avg Loss: 0.02704319, Log Avg loss: 0.00685633, Global Avg Loss: 0.02704319, Time: 0.1589 Steps: 12800, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 013000, Sample Num: 208000, Cur Loss: 0.00014438, Cur Avg Loss: 0.02676824, Log Avg loss: 0.00917088, Global Avg Loss: 0.02676824, Time: 0.0357 Steps: 13000, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 013200, Sample Num: 211200, Cur Loss: 0.00073555, Cur Avg Loss: 0.02650677, Log Avg loss: 0.00951141, Global Avg Loss: 0.02650677, Time: 0.1434 Steps: 13200, Updated lr: 0.000094 Training, Epoch: 0001, Batch: 013400, Sample Num: 214400, Cur Loss: 0.00025831, Cur Avg Loss: 0.02624280, Log Avg loss: 0.00882113, Global Avg Loss: 0.02624280, Time: 0.0383 Steps: 13400, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 013600, Sample Num: 217600, Cur Loss: 0.00491779, Cur Avg Loss: 0.02600394, Log Avg loss: 0.00999987, Global Avg Loss: 0.02600394, Time: 0.1447 Steps: 13600, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 013800, Sample Num: 220800, Cur Loss: 0.00004206, Cur Avg Loss: 0.02582610, Log Avg loss: 0.01373297, Global Avg Loss: 0.02582610, Time: 0.1069 Steps: 13800, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 014000, Sample Num: 224000, Cur Loss: 0.00048940, Cur Avg Loss: 0.02558678, Log Avg loss: 0.00907408, Global Avg Loss: 0.02558678, Time: 0.1078 Steps: 14000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 014200, Sample Num: 227200, Cur Loss: 0.00016300, Cur Avg Loss: 0.02538671, Log Avg loss: 0.01138198, Global Avg Loss: 0.02538671, Time: 0.0280 Steps: 14200, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 014400, Sample Num: 230400, Cur Loss: 0.00021893, Cur Avg Loss: 0.02518959, Log Avg loss: 0.01119356, Global Avg Loss: 0.02518959, Time: 0.0398 Steps: 14400, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 014600, Sample Num: 233600, Cur Loss: 0.06269059, Cur Avg Loss: 0.02493757, Log Avg loss: 0.00679207, Global Avg Loss: 0.02493757, Time: 0.0334 Steps: 14600, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 014800, Sample Num: 236800, Cur Loss: 0.00016395, Cur Avg Loss: 0.02476086, Log Avg loss: 0.01186124, Global Avg Loss: 0.02476086, Time: 0.0438 Steps: 14800, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 015000, Sample Num: 240000, Cur Loss: 0.00025274, Cur Avg Loss: 0.02456508, Log Avg loss: 0.01007772, Global Avg Loss: 0.02456508, Time: 0.0281 Steps: 15000, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 015200, Sample Num: 243200, Cur Loss: 0.00002747, Cur Avg Loss: 0.02430356, Log Avg loss: 0.00468893, Global Avg Loss: 0.02430356, Time: 0.0633 Steps: 15200, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 015400, Sample Num: 246400, Cur Loss: 0.00005343, Cur Avg Loss: 0.02413338, Log Avg loss: 0.01119971, Global Avg Loss: 0.02413338, Time: 0.1102 Steps: 15400, Updated lr: 0.000093 Training, Epoch: 0001, Batch: 015600, Sample Num: 249600, Cur Loss: 0.00068375, Cur Avg Loss: 0.02394083, Log Avg loss: 0.00911479, Global Avg Loss: 0.02394083, Time: 0.0278 Steps: 15600, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 015800, Sample Num: 252800, Cur Loss: 0.00017231, Cur Avg Loss: 0.02377492, Log Avg loss: 0.01083432, Global Avg Loss: 0.02377492, Time: 0.1156 Steps: 15800, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 016000, Sample Num: 256000, Cur Loss: 0.00046731, Cur Avg Loss: 0.02351222, Log Avg loss: 0.00275843, Global Avg Loss: 0.02351222, Time: 0.0963 Steps: 16000, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 016200, Sample Num: 259200, Cur Loss: 0.00000640, Cur Avg Loss: 0.02325861, Log Avg loss: 0.00297010, Global Avg Loss: 0.02325861, Time: 0.1971 Steps: 16200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 016400, Sample Num: 262400, Cur Loss: 0.00011922, Cur Avg Loss: 0.02303661, Log Avg loss: 0.00505409, Global Avg Loss: 0.02303661, Time: 0.0585 Steps: 16400, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 016600, Sample Num: 265600, Cur Loss: 0.00006370, Cur Avg Loss: 0.02288782, Log Avg loss: 0.01068772, Global Avg Loss: 0.02288782, Time: 0.0698 Steps: 16600, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 016800, Sample Num: 268800, Cur Loss: 0.00015530, Cur Avg Loss: 0.02271341, Log Avg loss: 0.00823714, Global Avg Loss: 0.02271341, Time: 0.1041 Steps: 16800, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 017000, Sample Num: 272000, Cur Loss: 0.00005728, Cur Avg Loss: 0.02257402, Log Avg loss: 0.01086482, Global Avg Loss: 0.02257402, Time: 0.0626 Steps: 17000, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 017200, Sample Num: 275200, Cur Loss: 0.00010776, Cur Avg Loss: 0.02252846, Log Avg loss: 0.01865659, Global Avg Loss: 0.02252846, Time: 0.0736 Steps: 17200, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 017400, Sample Num: 278400, Cur Loss: 0.00006758, Cur Avg Loss: 0.02233863, Log Avg loss: 0.00601320, Global Avg Loss: 0.02233863, Time: 0.1407 Steps: 17400, Updated lr: 0.000092 Training, Epoch: 0001, Batch: 017600, Sample Num: 281600, Cur Loss: 0.00096162, Cur Avg Loss: 0.02219244, Log Avg loss: 0.00947324, Global Avg Loss: 0.02219244, Time: 0.0808 Steps: 17600, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 017800, Sample Num: 284800, Cur Loss: 0.00079855, Cur Avg Loss: 0.02208598, Log Avg loss: 0.01271743, Global Avg Loss: 0.02208598, Time: 0.0455 Steps: 17800, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 018000, Sample Num: 288000, Cur Loss: 0.00004465, Cur Avg Loss: 0.02190138, Log Avg loss: 0.00547239, Global Avg Loss: 0.02190138, Time: 0.0434 Steps: 18000, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 018200, Sample Num: 291200, Cur Loss: 0.00013724, Cur Avg Loss: 0.02180395, Log Avg loss: 0.01303508, Global Avg Loss: 0.02180395, Time: 0.0578 Steps: 18200, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 018400, Sample Num: 294400, Cur Loss: 0.00027362, Cur Avg Loss: 0.02167609, Log Avg loss: 0.01004088, Global Avg Loss: 0.02167609, Time: 0.0531 Steps: 18400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 018600, Sample Num: 297600, Cur Loss: 0.03617809, Cur Avg Loss: 0.02150411, Log Avg loss: 0.00568174, Global Avg Loss: 0.02150411, Time: 0.1009 Steps: 18600, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 018800, Sample Num: 300800, Cur Loss: 0.39869627, Cur Avg Loss: 0.02139473, Log Avg loss: 0.01122285, Global Avg Loss: 0.02139473, Time: 0.0306 Steps: 18800, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 019000, Sample Num: 304000, Cur Loss: 0.00014209, Cur Avg Loss: 0.02126930, Log Avg loss: 0.00947877, Global Avg Loss: 0.02126930, Time: 0.0381 Steps: 19000, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 019200, Sample Num: 307200, Cur Loss: 0.00004188, Cur Avg Loss: 0.02116254, Log Avg loss: 0.01102003, Global Avg Loss: 0.02116254, Time: 0.0937 Steps: 19200, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 019400, Sample Num: 310400, Cur Loss: 0.00277110, Cur Avg Loss: 0.02101829, Log Avg loss: 0.00717033, Global Avg Loss: 0.02101829, Time: 0.0547 Steps: 19400, Updated lr: 0.000091 Training, Epoch: 0001, Batch: 019600, Sample Num: 313600, Cur Loss: 0.00021416, Cur Avg Loss: 0.02092977, Log Avg loss: 0.01234386, Global Avg Loss: 0.02092977, Time: 0.0483 Steps: 19600, Updated lr: 0.000090 Training, Epoch: 0001, Batch: 019800, Sample Num: 316800, Cur Loss: 0.00011675, Cur Avg Loss: 0.02077229, Log Avg loss: 0.00533932, Global Avg Loss: 0.02077229, Time: 0.1080 Steps: 19800, Updated lr: 0.000090 Training, Epoch: 0001, Batch: 020000, Sample Num: 320000, Cur Loss: 0.07535560, Cur Avg Loss: 0.02065963, Log Avg loss: 0.00950628, Global Avg Loss: 0.02065963, Time: 0.2310 Steps: 20000, Updated lr: 0.000090 Training, Epoch: 0001, Batch: 020200, Sample Num: 323200, Cur Loss: 0.00002665, Cur Avg Loss: 0.02051262, Log Avg loss: 0.00581141, Global Avg Loss: 0.02051262, Time: 0.0613 Steps: 20200, Updated lr: 0.000090 ***** Running evaluation checkpoint-20320 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-20320 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 1939.219489, Avg time per batch (s): 0.100000 {"eval_avg_loss": 0.007368, "eval_total_loss": 19.968247, "eval_acc": 0.998062, "eval_prec": 0.998388, "eval_recall": 0.997744, "eval_f1": 0.998066, "eval_roc_auc": 0.999916, "eval_pr_auc": 0.999921, "eval_confusion_matrix": {"tn": 21592, "fp": 35, "fn": 49, "tp": 21673}, "eval_mcc2": 0.996125, "eval_mcc": 0.996125, "eval_sn": 0.997744, "eval_sp": 0.998382, "update_flag": true, "test_avg_loss": 0.006984, "test_total_loss": 28.381939, "test_acc": 0.99797, "test_prec": 0.998153, "test_recall": 0.997785, "test_f1": 0.997969, "test_roc_auc": 0.999917, "test_pr_auc": 0.999925, "test_confusion_matrix": {"tn": 32457, "fp": 60, "fn": 72, "tp": 32433}, "test_mcc2": 0.99594, "test_mcc": 0.99594, "test_sn": 0.997785, "test_sp": 0.998155, "lr": 9.008866995073892e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.02052816845249141, "train_cur_epoch_loss": 417.1323829546254, "train_cur_epoch_avg_loss": 0.02052816845249141, "train_cur_epoch_time": 1939.2194890975952, "train_cur_epoch_avg_time": 0.09543402997527535, "epoch": 1, "step": 20320} ################################################## Training, Epoch: 0002, Batch: 000080, Sample Num: 1280, Cur Loss: 0.00015135, Cur Avg Loss: 0.00473398, Log Avg loss: 0.01578076, Global Avg Loss: 0.02046623, Time: 0.1132 Steps: 20400, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000280, Sample Num: 4480, Cur Loss: 0.00031741, Cur Avg Loss: 0.00697737, Log Avg loss: 0.00787472, Global Avg Loss: 0.02034398, Time: 0.0346 Steps: 20600, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000480, Sample Num: 7680, Cur Loss: 0.00015951, Cur Avg Loss: 0.00555942, Log Avg loss: 0.00357429, Global Avg Loss: 0.02018274, Time: 0.1136 Steps: 20800, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000680, Sample Num: 10880, Cur Loss: 0.00004303, Cur Avg Loss: 0.00477100, Log Avg loss: 0.00287882, Global Avg Loss: 0.02001794, Time: 0.0636 Steps: 21000, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 000880, Sample Num: 14080, Cur Loss: 0.00013898, Cur Avg Loss: 0.00578071, Log Avg loss: 0.00921373, Global Avg Loss: 0.01991601, Time: 0.0632 Steps: 21200, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 001080, Sample Num: 17280, Cur Loss: 0.00000879, Cur Avg Loss: 0.00571240, Log Avg loss: 0.00541184, Global Avg Loss: 0.01978046, Time: 0.1826 Steps: 21400, Updated lr: 0.000090 Training, Epoch: 0002, Batch: 001280, Sample Num: 20480, Cur Loss: 0.00002956, Cur Avg Loss: 0.00521881, Log Avg loss: 0.00255341, Global Avg Loss: 0.01962095, Time: 0.2624 Steps: 21600, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001480, Sample Num: 23680, Cur Loss: 0.00019638, Cur Avg Loss: 0.00551437, Log Avg loss: 0.00740596, Global Avg Loss: 0.01950888, Time: 0.1411 Steps: 21800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001680, Sample Num: 26880, Cur Loss: 0.00002488, Cur Avg Loss: 0.00531976, Log Avg loss: 0.00387964, Global Avg Loss: 0.01936680, Time: 0.1026 Steps: 22000, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 001880, Sample Num: 30080, Cur Loss: 0.00000755, Cur Avg Loss: 0.00497621, Log Avg loss: 0.00209035, Global Avg Loss: 0.01921116, Time: 0.0421 Steps: 22200, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002080, Sample Num: 33280, Cur Loss: 0.00023605, Cur Avg Loss: 0.00530169, Log Avg loss: 0.00836122, Global Avg Loss: 0.01911428, Time: 0.1491 Steps: 22400, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002280, Sample Num: 36480, Cur Loss: 0.00003303, Cur Avg Loss: 0.00594228, Log Avg loss: 0.01260444, Global Avg Loss: 0.01905667, Time: 0.0473 Steps: 22600, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002480, Sample Num: 39680, Cur Loss: 0.00018242, Cur Avg Loss: 0.00594659, Log Avg loss: 0.00599576, Global Avg Loss: 0.01894210, Time: 0.1309 Steps: 22800, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002680, Sample Num: 42880, Cur Loss: 0.00003187, Cur Avg Loss: 0.00601631, Log Avg loss: 0.00688076, Global Avg Loss: 0.01883722, Time: 0.0883 Steps: 23000, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 002880, Sample Num: 46080, Cur Loss: 0.00891353, Cur Avg Loss: 0.00602134, Log Avg loss: 0.00608879, Global Avg Loss: 0.01872732, Time: 0.1460 Steps: 23200, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 003080, Sample Num: 49280, Cur Loss: 0.04441684, Cur Avg Loss: 0.00602985, Log Avg loss: 0.00615233, Global Avg Loss: 0.01861984, Time: 0.0997 Steps: 23400, Updated lr: 0.000089 Training, Epoch: 0002, Batch: 003280, Sample Num: 52480, Cur Loss: 0.00001488, Cur Avg Loss: 0.00596287, Log Avg loss: 0.00493140, Global Avg Loss: 0.01850384, Time: 0.1290 Steps: 23600, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003480, Sample Num: 55680, Cur Loss: 0.00116578, Cur Avg Loss: 0.00579602, Log Avg loss: 0.00305975, Global Avg Loss: 0.01837406, Time: 0.1077 Steps: 23800, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003680, Sample Num: 58880, Cur Loss: 0.00001614, Cur Avg Loss: 0.00578901, Log Avg loss: 0.00566702, Global Avg Loss: 0.01826816, Time: 0.0929 Steps: 24000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 003880, Sample Num: 62080, Cur Loss: 0.00001287, Cur Avg Loss: 0.00579613, Log Avg loss: 0.00592711, Global Avg Loss: 0.01816617, Time: 0.1076 Steps: 24200, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 004080, Sample Num: 65280, Cur Loss: 0.00008213, Cur Avg Loss: 0.00576627, Log Avg loss: 0.00518698, Global Avg Loss: 0.01805979, Time: 0.0420 Steps: 24400, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 004280, Sample Num: 68480, Cur Loss: 0.00002517, Cur Avg Loss: 0.00591233, Log Avg loss: 0.00889185, Global Avg Loss: 0.01798525, Time: 0.1160 Steps: 24600, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 004480, Sample Num: 71680, Cur Loss: 0.00011992, Cur Avg Loss: 0.00599108, Log Avg loss: 0.00767634, Global Avg Loss: 0.01790211, Time: 0.3484 Steps: 24800, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 004680, Sample Num: 74880, Cur Loss: 0.00001755, Cur Avg Loss: 0.00584676, Log Avg loss: 0.00261399, Global Avg Loss: 0.01777981, Time: 0.1179 Steps: 25000, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 004880, Sample Num: 78080, Cur Loss: 0.00073168, Cur Avg Loss: 0.00578440, Log Avg loss: 0.00432534, Global Avg Loss: 0.01767303, Time: 0.1047 Steps: 25200, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 005080, Sample Num: 81280, Cur Loss: 0.00081289, Cur Avg Loss: 0.00577675, Log Avg loss: 0.00559012, Global Avg Loss: 0.01757789, Time: 0.0432 Steps: 25400, Updated lr: 0.000088 Training, Epoch: 0002, Batch: 005280, Sample Num: 84480, Cur Loss: 0.00000464, Cur Avg Loss: 0.00580039, Log Avg loss: 0.00640081, Global Avg Loss: 0.01749056, Time: 0.2407 Steps: 25600, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 005480, Sample Num: 87680, Cur Loss: 0.00001554, Cur Avg Loss: 0.00570566, Log Avg loss: 0.00320476, Global Avg Loss: 0.01737982, Time: 0.0553 Steps: 25800, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 005680, Sample Num: 90880, Cur Loss: 0.00016262, Cur Avg Loss: 0.00563026, Log Avg loss: 0.00356429, Global Avg Loss: 0.01727355, Time: 0.1222 Steps: 26000, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 005880, Sample Num: 94080, Cur Loss: 0.00002559, Cur Avg Loss: 0.00558430, Log Avg loss: 0.00427898, Global Avg Loss: 0.01717435, Time: 0.1475 Steps: 26200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 006080, Sample Num: 97280, Cur Loss: 0.00002363, Cur Avg Loss: 0.00579150, Log Avg loss: 0.01188315, Global Avg Loss: 0.01713427, Time: 0.3231 Steps: 26400, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 006280, Sample Num: 100480, Cur Loss: 0.00006463, Cur Avg Loss: 0.00582354, Log Avg loss: 0.00679747, Global Avg Loss: 0.01705655, Time: 0.1055 Steps: 26600, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 006480, Sample Num: 103680, Cur Loss: 0.00003967, Cur Avg Loss: 0.00575654, Log Avg loss: 0.00365277, Global Avg Loss: 0.01695652, Time: 0.2021 Steps: 26800, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 006680, Sample Num: 106880, Cur Loss: 0.00014599, Cur Avg Loss: 0.00585861, Log Avg loss: 0.00916592, Global Avg Loss: 0.01689881, Time: 0.0400 Steps: 27000, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 006880, Sample Num: 110080, Cur Loss: 0.00000560, Cur Avg Loss: 0.00575037, Log Avg loss: 0.00213505, Global Avg Loss: 0.01679026, Time: 0.0651 Steps: 27200, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 007080, Sample Num: 113280, Cur Loss: 0.00002170, Cur Avg Loss: 0.00567699, Log Avg loss: 0.00315271, Global Avg Loss: 0.01669071, Time: 0.2018 Steps: 27400, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 007280, Sample Num: 116480, Cur Loss: 0.00001286, Cur Avg Loss: 0.00564326, Log Avg loss: 0.00444913, Global Avg Loss: 0.01660200, Time: 0.0594 Steps: 27600, Updated lr: 0.000087 Training, Epoch: 0002, Batch: 007480, Sample Num: 119680, Cur Loss: 0.00001502, Cur Avg Loss: 0.00560606, Log Avg loss: 0.00425193, Global Avg Loss: 0.01651315, Time: 0.0609 Steps: 27800, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 007680, Sample Num: 122880, Cur Loss: 0.00000255, Cur Avg Loss: 0.00557655, Log Avg loss: 0.00447283, Global Avg Loss: 0.01642715, Time: 0.0767 Steps: 28000, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 007880, Sample Num: 126080, Cur Loss: 0.00031634, Cur Avg Loss: 0.00566763, Log Avg loss: 0.00916514, Global Avg Loss: 0.01637565, Time: 0.0410 Steps: 28200, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 008080, Sample Num: 129280, Cur Loss: 0.01354444, Cur Avg Loss: 0.00571162, Log Avg loss: 0.00744502, Global Avg Loss: 0.01631276, Time: 0.1640 Steps: 28400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 008280, Sample Num: 132480, Cur Loss: 0.00000542, Cur Avg Loss: 0.00559348, Log Avg loss: 0.00082051, Global Avg Loss: 0.01620442, Time: 0.1152 Steps: 28600, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 008480, Sample Num: 135680, Cur Loss: 0.00000499, Cur Avg Loss: 0.00554093, Log Avg loss: 0.00336536, Global Avg Loss: 0.01611526, Time: 0.0255 Steps: 28800, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 008680, Sample Num: 138880, Cur Loss: 0.00022235, Cur Avg Loss: 0.00559906, Log Avg loss: 0.00806378, Global Avg Loss: 0.01605973, Time: 0.0265 Steps: 29000, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 008880, Sample Num: 142080, Cur Loss: 0.00072361, Cur Avg Loss: 0.00556490, Log Avg loss: 0.00408249, Global Avg Loss: 0.01597770, Time: 0.1084 Steps: 29200, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 009080, Sample Num: 145280, Cur Loss: 0.00001474, Cur Avg Loss: 0.00559648, Log Avg loss: 0.00699869, Global Avg Loss: 0.01591661, Time: 0.1085 Steps: 29400, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 009280, Sample Num: 148480, Cur Loss: 0.00001135, Cur Avg Loss: 0.00547876, Log Avg loss: 0.00013389, Global Avg Loss: 0.01580997, Time: 0.2692 Steps: 29600, Updated lr: 0.000086 Training, Epoch: 0002, Batch: 009480, Sample Num: 151680, Cur Loss: 0.00204983, Cur Avg Loss: 0.00548248, Log Avg loss: 0.00565535, Global Avg Loss: 0.01574182, Time: 0.1575 Steps: 29800, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 009680, Sample Num: 154880, Cur Loss: 0.00002699, Cur Avg Loss: 0.00542603, Log Avg loss: 0.00275006, Global Avg Loss: 0.01565521, Time: 0.0670 Steps: 30000, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 009880, Sample Num: 158080, Cur Loss: 0.00000919, Cur Avg Loss: 0.00540857, Log Avg loss: 0.00456375, Global Avg Loss: 0.01558176, Time: 0.0467 Steps: 30200, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 010080, Sample Num: 161280, Cur Loss: 0.00003737, Cur Avg Loss: 0.00534743, Log Avg loss: 0.00232725, Global Avg Loss: 0.01549456, Time: 0.1315 Steps: 30400, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 010280, Sample Num: 164480, Cur Loss: 0.00002073, Cur Avg Loss: 0.00533017, Log Avg loss: 0.00445994, Global Avg Loss: 0.01542243, Time: 0.0336 Steps: 30600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 010480, Sample Num: 167680, Cur Loss: 0.00000402, Cur Avg Loss: 0.00540808, Log Avg loss: 0.00941271, Global Avg Loss: 0.01538341, Time: 0.1509 Steps: 30800, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 010680, Sample Num: 170880, Cur Loss: 0.00000562, Cur Avg Loss: 0.00533036, Log Avg loss: 0.00125795, Global Avg Loss: 0.01529228, Time: 0.0360 Steps: 31000, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 010880, Sample Num: 174080, Cur Loss: 0.00000737, Cur Avg Loss: 0.00527294, Log Avg loss: 0.00220669, Global Avg Loss: 0.01520840, Time: 0.1982 Steps: 31200, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 011080, Sample Num: 177280, Cur Loss: 0.00000332, Cur Avg Loss: 0.00536726, Log Avg loss: 0.01049848, Global Avg Loss: 0.01517840, Time: 0.0410 Steps: 31400, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 011280, Sample Num: 180480, Cur Loss: 0.00001589, Cur Avg Loss: 0.00543514, Log Avg loss: 0.00919527, Global Avg Loss: 0.01514053, Time: 0.1448 Steps: 31600, Updated lr: 0.000085 Training, Epoch: 0002, Batch: 011480, Sample Num: 183680, Cur Loss: 0.00021169, Cur Avg Loss: 0.00538090, Log Avg loss: 0.00232199, Global Avg Loss: 0.01505991, Time: 0.0412 Steps: 31800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 011680, Sample Num: 186880, Cur Loss: 0.00001191, Cur Avg Loss: 0.00533823, Log Avg loss: 0.00288904, Global Avg Loss: 0.01498384, Time: 0.1595 Steps: 32000, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 011880, Sample Num: 190080, Cur Loss: 0.00024994, Cur Avg Loss: 0.00536101, Log Avg loss: 0.00669117, Global Avg Loss: 0.01493233, Time: 0.1064 Steps: 32200, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 012080, Sample Num: 193280, Cur Loss: 0.00007834, Cur Avg Loss: 0.00531038, Log Avg loss: 0.00230320, Global Avg Loss: 0.01485438, Time: 0.0613 Steps: 32400, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 012280, Sample Num: 196480, Cur Loss: 0.00001490, Cur Avg Loss: 0.00533344, Log Avg loss: 0.00672589, Global Avg Loss: 0.01480451, Time: 0.0285 Steps: 32600, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 012480, Sample Num: 199680, Cur Loss: 0.00001953, Cur Avg Loss: 0.00529365, Log Avg loss: 0.00285106, Global Avg Loss: 0.01473162, Time: 0.0478 Steps: 32800, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 012680, Sample Num: 202880, Cur Loss: 0.00001719, Cur Avg Loss: 0.00529623, Log Avg loss: 0.00545691, Global Avg Loss: 0.01467541, Time: 0.0499 Steps: 33000, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 012880, Sample Num: 206080, Cur Loss: 0.00000142, Cur Avg Loss: 0.00524917, Log Avg loss: 0.00226580, Global Avg Loss: 0.01460065, Time: 0.0663 Steps: 33200, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 013080, Sample Num: 209280, Cur Loss: 0.00003506, Cur Avg Loss: 0.00524911, Log Avg loss: 0.00524518, Global Avg Loss: 0.01454463, Time: 0.0459 Steps: 33400, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 013280, Sample Num: 212480, Cur Loss: 0.00000333, Cur Avg Loss: 0.00517187, Log Avg loss: 0.00012013, Global Avg Loss: 0.01445877, Time: 0.0487 Steps: 33600, Updated lr: 0.000084 Training, Epoch: 0002, Batch: 013480, Sample Num: 215680, Cur Loss: 0.00003028, Cur Avg Loss: 0.00509888, Log Avg loss: 0.00025235, Global Avg Loss: 0.01437471, Time: 0.0698 Steps: 33800, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 013680, Sample Num: 218880, Cur Loss: 0.00000168, Cur Avg Loss: 0.00503438, Log Avg loss: 0.00068732, Global Avg Loss: 0.01429420, Time: 0.0384 Steps: 34000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 013880, Sample Num: 222080, Cur Loss: 0.00000520, Cur Avg Loss: 0.00508316, Log Avg loss: 0.00841956, Global Avg Loss: 0.01425984, Time: 0.0428 Steps: 34200, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 014080, Sample Num: 225280, Cur Loss: 0.00015706, Cur Avg Loss: 0.00505191, Log Avg loss: 0.00288283, Global Avg Loss: 0.01419370, Time: 0.1084 Steps: 34400, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 014280, Sample Num: 228480, Cur Loss: 0.00000451, Cur Avg Loss: 0.00511424, Log Avg loss: 0.00950249, Global Avg Loss: 0.01416658, Time: 0.1473 Steps: 34600, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 014480, Sample Num: 231680, Cur Loss: 0.00001002, Cur Avg Loss: 0.00508412, Log Avg loss: 0.00293329, Global Avg Loss: 0.01410202, Time: 0.1543 Steps: 34800, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 014680, Sample Num: 234880, Cur Loss: 0.00000943, Cur Avg Loss: 0.00507008, Log Avg loss: 0.00405366, Global Avg Loss: 0.01404460, Time: 0.1083 Steps: 35000, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 014880, Sample Num: 238080, Cur Loss: 0.00000492, Cur Avg Loss: 0.00510246, Log Avg loss: 0.00747974, Global Avg Loss: 0.01400730, Time: 0.1124 Steps: 35200, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 015080, Sample Num: 241280, Cur Loss: 0.00001683, Cur Avg Loss: 0.00504699, Log Avg loss: 0.00091991, Global Avg Loss: 0.01393336, Time: 0.0380 Steps: 35400, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 015280, Sample Num: 244480, Cur Loss: 0.00000306, Cur Avg Loss: 0.00498541, Log Avg loss: 0.00034237, Global Avg Loss: 0.01385701, Time: 0.0591 Steps: 35600, Updated lr: 0.000083 Training, Epoch: 0002, Batch: 015480, Sample Num: 247680, Cur Loss: 0.00001839, Cur Avg Loss: 0.00500955, Log Avg loss: 0.00685388, Global Avg Loss: 0.01381788, Time: 0.1210 Steps: 35800, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 015680, Sample Num: 250880, Cur Loss: 0.00010317, Cur Avg Loss: 0.00494776, Log Avg loss: 0.00016476, Global Avg Loss: 0.01374203, Time: 0.0746 Steps: 36000, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 015880, Sample Num: 254080, Cur Loss: 0.00000293, Cur Avg Loss: 0.00493479, Log Avg loss: 0.00391799, Global Avg Loss: 0.01368776, Time: 0.1694 Steps: 36200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 016080, Sample Num: 257280, Cur Loss: 0.00016890, Cur Avg Loss: 0.00487469, Log Avg loss: 0.00010320, Global Avg Loss: 0.01361312, Time: 0.1036 Steps: 36400, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 016280, Sample Num: 260480, Cur Loss: 0.00000241, Cur Avg Loss: 0.00482066, Log Avg loss: 0.00047632, Global Avg Loss: 0.01354133, Time: 0.0852 Steps: 36600, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 016480, Sample Num: 263680, Cur Loss: 0.00000904, Cur Avg Loss: 0.00479234, Log Avg loss: 0.00248744, Global Avg Loss: 0.01348126, Time: 0.1643 Steps: 36800, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 016680, Sample Num: 266880, Cur Loss: 0.00000317, Cur Avg Loss: 0.00478745, Log Avg loss: 0.00438414, Global Avg Loss: 0.01343208, Time: 0.0456 Steps: 37000, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 016880, Sample Num: 270080, Cur Loss: 0.00000059, Cur Avg Loss: 0.00477034, Log Avg loss: 0.00334302, Global Avg Loss: 0.01337784, Time: 0.1515 Steps: 37200, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 017080, Sample Num: 273280, Cur Loss: 0.00000706, Cur Avg Loss: 0.00478578, Log Avg loss: 0.00608943, Global Avg Loss: 0.01333886, Time: 0.0561 Steps: 37400, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 017280, Sample Num: 276480, Cur Loss: 0.00050945, Cur Avg Loss: 0.00481775, Log Avg loss: 0.00754745, Global Avg Loss: 0.01330806, Time: 0.1806 Steps: 37600, Updated lr: 0.000082 Training, Epoch: 0002, Batch: 017480, Sample Num: 279680, Cur Loss: 0.00000150, Cur Avg Loss: 0.00476780, Log Avg loss: 0.00045223, Global Avg Loss: 0.01324004, Time: 0.0659 Steps: 37800, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 017680, Sample Num: 282880, Cur Loss: 0.00000148, Cur Avg Loss: 0.00478198, Log Avg loss: 0.00602171, Global Avg Loss: 0.01320205, Time: 0.0713 Steps: 38000, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 017880, Sample Num: 286080, Cur Loss: 0.00001655, Cur Avg Loss: 0.00477675, Log Avg loss: 0.00431448, Global Avg Loss: 0.01315552, Time: 0.0998 Steps: 38200, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 018080, Sample Num: 289280, Cur Loss: 0.00000196, Cur Avg Loss: 0.00473930, Log Avg loss: 0.00139069, Global Avg Loss: 0.01309424, Time: 0.2314 Steps: 38400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 018280, Sample Num: 292480, Cur Loss: 0.00000186, Cur Avg Loss: 0.00475018, Log Avg loss: 0.00573376, Global Avg Loss: 0.01305610, Time: 0.0483 Steps: 38600, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 018480, Sample Num: 295680, Cur Loss: 0.00000247, Cur Avg Loss: 0.00473227, Log Avg loss: 0.00309564, Global Avg Loss: 0.01300476, Time: 0.1185 Steps: 38800, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 018680, Sample Num: 298880, Cur Loss: 0.00002034, Cur Avg Loss: 0.00472488, Log Avg loss: 0.00404204, Global Avg Loss: 0.01295880, Time: 0.0425 Steps: 39000, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 018880, Sample Num: 302080, Cur Loss: 0.00000818, Cur Avg Loss: 0.00473093, Log Avg loss: 0.00529645, Global Avg Loss: 0.01291970, Time: 0.1934 Steps: 39200, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 019080, Sample Num: 305280, Cur Loss: 0.00000838, Cur Avg Loss: 0.00472597, Log Avg loss: 0.00425685, Global Avg Loss: 0.01287573, Time: 0.0643 Steps: 39400, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 019280, Sample Num: 308480, Cur Loss: 0.00000469, Cur Avg Loss: 0.00470982, Log Avg loss: 0.00316971, Global Avg Loss: 0.01282671, Time: 0.0262 Steps: 39600, Updated lr: 0.000081 Training, Epoch: 0002, Batch: 019480, Sample Num: 311680, Cur Loss: 0.00000539, Cur Avg Loss: 0.00469340, Log Avg loss: 0.00310992, Global Avg Loss: 0.01277788, Time: 0.0616 Steps: 39800, Updated lr: 0.000080 Training, Epoch: 0002, Batch: 019680, Sample Num: 314880, Cur Loss: 0.00001095, Cur Avg Loss: 0.00465885, Log Avg loss: 0.00129392, Global Avg Loss: 0.01272046, Time: 0.0513 Steps: 40000, Updated lr: 0.000080 Training, Epoch: 0002, Batch: 019880, Sample Num: 318080, Cur Loss: 0.00000971, Cur Avg Loss: 0.00463605, Log Avg loss: 0.00239251, Global Avg Loss: 0.01266908, Time: 0.1072 Steps: 40200, Updated lr: 0.000080 Training, Epoch: 0002, Batch: 020080, Sample Num: 321280, Cur Loss: 0.00183610, Cur Avg Loss: 0.00459180, Log Avg loss: 0.00019385, Global Avg Loss: 0.01260732, Time: 0.0478 Steps: 40400, Updated lr: 0.000080 Training, Epoch: 0002, Batch: 020280, Sample Num: 324480, Cur Loss: 0.00482288, Cur Avg Loss: 0.00460915, Log Avg loss: 0.00635060, Global Avg Loss: 0.01257650, Time: 0.0440 Steps: 40600, Updated lr: 0.000080 ***** Running evaluation checkpoint-40640 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-40640 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 1969.152944, Avg time per batch (s): 0.100000 {"eval_avg_loss": 0.00937, "eval_total_loss": 25.394005, "eval_acc": 0.998178, "eval_prec": 0.998802, "eval_recall": 0.99756, "eval_f1": 0.99818, "eval_roc_auc": 0.999884, "eval_pr_auc": 0.999825, "eval_confusion_matrix": {"tn": 21601, "fp": 26, "fn": 53, "tp": 21669}, "eval_mcc2": 0.996356, "eval_mcc": 0.996356, "eval_sn": 0.99756, "eval_sp": 0.998798, "update_flag": true, "test_avg_loss": 0.008306, "test_total_loss": 33.757215, "test_acc": 0.998262, "test_prec": 0.998799, "test_recall": 0.997723, "test_f1": 0.998261, "test_roc_auc": 0.999945, "test_pr_auc": 0.999948, "test_confusion_matrix": {"tn": 32478, "fp": 39, "fn": 74, "tp": 32431}, "test_mcc2": 0.996525, "test_mcc": 0.996525, "test_sn": 0.997723, "test_sp": 0.998801, "lr": 8.007881773399016e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.012564515182661598, "train_cur_epoch_loss": 93.48951406874183, "train_cur_epoch_avg_loss": 0.004600861912831783, "train_cur_epoch_time": 1969.1529443264008, "train_cur_epoch_avg_time": 0.09690713308692916, "epoch": 2, "step": 40640} ################################################## Training, Epoch: 0003, Batch: 000160, Sample Num: 2560, Cur Loss: 0.00000552, Cur Avg Loss: 0.00017269, Log Avg loss: 0.00021813, Global Avg Loss: 0.01251592, Time: 0.0629 Steps: 40800, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000360, Sample Num: 5760, Cur Loss: 0.00000399, Cur Avg Loss: 0.00088381, Log Avg loss: 0.00145271, Global Avg Loss: 0.01246195, Time: 0.0493 Steps: 41000, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000560, Sample Num: 8960, Cur Loss: 0.00000131, Cur Avg Loss: 0.00059564, Log Avg loss: 0.00007695, Global Avg Loss: 0.01240183, Time: 0.0689 Steps: 41200, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000760, Sample Num: 12160, Cur Loss: 0.00000254, Cur Avg Loss: 0.00060654, Log Avg loss: 0.00063703, Global Avg Loss: 0.01234500, Time: 0.1083 Steps: 41400, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 000960, Sample Num: 15360, Cur Loss: 0.00000149, Cur Avg Loss: 0.00049899, Log Avg loss: 0.00009031, Global Avg Loss: 0.01228608, Time: 0.0524 Steps: 41600, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 001160, Sample Num: 18560, Cur Loss: 0.00000001, Cur Avg Loss: 0.00093959, Log Avg loss: 0.00305448, Global Avg Loss: 0.01224191, Time: 0.1089 Steps: 41800, Updated lr: 0.000080 Training, Epoch: 0003, Batch: 001360, Sample Num: 21760, Cur Loss: 0.00000041, Cur Avg Loss: 0.00087876, Log Avg loss: 0.00052597, Global Avg Loss: 0.01218612, Time: 0.0302 Steps: 42000, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001560, Sample Num: 24960, Cur Loss: 0.00001220, Cur Avg Loss: 0.00125872, Log Avg loss: 0.00384239, Global Avg Loss: 0.01214658, Time: 0.0705 Steps: 42200, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001760, Sample Num: 28160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00111724, Log Avg loss: 0.00001377, Global Avg Loss: 0.01208935, Time: 0.0610 Steps: 42400, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 001960, Sample Num: 31360, Cur Loss: 0.00000002, Cur Avg Loss: 0.00101438, Log Avg loss: 0.00010920, Global Avg Loss: 0.01203310, Time: 0.0663 Steps: 42600, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002160, Sample Num: 34560, Cur Loss: 0.00000118, Cur Avg Loss: 0.00183940, Log Avg loss: 0.00992458, Global Avg Loss: 0.01202325, Time: 0.1476 Steps: 42800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002360, Sample Num: 37760, Cur Loss: 0.00000036, Cur Avg Loss: 0.00181830, Log Avg loss: 0.00159038, Global Avg Loss: 0.01197472, Time: 0.1211 Steps: 43000, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002560, Sample Num: 40960, Cur Loss: 0.00000018, Cur Avg Loss: 0.00167835, Log Avg loss: 0.00002693, Global Avg Loss: 0.01191941, Time: 0.2286 Steps: 43200, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002760, Sample Num: 44160, Cur Loss: 0.00000328, Cur Avg Loss: 0.00180834, Log Avg loss: 0.00347222, Global Avg Loss: 0.01188048, Time: 0.1420 Steps: 43400, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 002960, Sample Num: 47360, Cur Loss: 0.00000144, Cur Avg Loss: 0.00198180, Log Avg loss: 0.00437565, Global Avg Loss: 0.01184606, Time: 0.0442 Steps: 43600, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 003160, Sample Num: 50560, Cur Loss: 0.00000001, Cur Avg Loss: 0.00227434, Log Avg loss: 0.00660380, Global Avg Loss: 0.01182212, Time: 0.0631 Steps: 43800, Updated lr: 0.000079 Training, Epoch: 0003, Batch: 003360, Sample Num: 53760, Cur Loss: 0.00000504, Cur Avg Loss: 0.00236166, Log Avg loss: 0.00374129, Global Avg Loss: 0.01178539, Time: 0.0669 Steps: 44000, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 003560, Sample Num: 56960, Cur Loss: 0.00000008, Cur Avg Loss: 0.00241076, Log Avg loss: 0.00323568, Global Avg Loss: 0.01174670, Time: 0.0434 Steps: 44200, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 003760, Sample Num: 60160, Cur Loss: 0.00000186, Cur Avg Loss: 0.00243920, Log Avg loss: 0.00294556, Global Avg Loss: 0.01170706, Time: 0.1097 Steps: 44400, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 003960, Sample Num: 63360, Cur Loss: 0.00000994, Cur Avg Loss: 0.00237708, Log Avg loss: 0.00120922, Global Avg Loss: 0.01165998, Time: 0.0854 Steps: 44600, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 004160, Sample Num: 66560, Cur Loss: 0.00000200, Cur Avg Loss: 0.00226853, Log Avg loss: 0.00011907, Global Avg Loss: 0.01160846, Time: 0.0673 Steps: 44800, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 004360, Sample Num: 69760, Cur Loss: 0.00002574, Cur Avg Loss: 0.00228128, Log Avg loss: 0.00254645, Global Avg Loss: 0.01156818, Time: 0.1639 Steps: 45000, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 004560, Sample Num: 72960, Cur Loss: 0.00000174, Cur Avg Loss: 0.00255105, Log Avg loss: 0.00843224, Global Avg Loss: 0.01155431, Time: 0.0651 Steps: 45200, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 004760, Sample Num: 76160, Cur Loss: 0.00000307, Cur Avg Loss: 0.00260948, Log Avg loss: 0.00394158, Global Avg Loss: 0.01152077, Time: 0.0812 Steps: 45400, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 004960, Sample Num: 79360, Cur Loss: 0.00000521, Cur Avg Loss: 0.00253071, Log Avg loss: 0.00065603, Global Avg Loss: 0.01147312, Time: 0.0706 Steps: 45600, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 005160, Sample Num: 82560, Cur Loss: 0.00002220, Cur Avg Loss: 0.00263048, Log Avg loss: 0.00510471, Global Avg Loss: 0.01144531, Time: 0.0668 Steps: 45800, Updated lr: 0.000078 Training, Epoch: 0003, Batch: 005360, Sample Num: 85760, Cur Loss: 0.00000139, Cur Avg Loss: 0.00258470, Log Avg loss: 0.00140365, Global Avg Loss: 0.01140165, Time: 0.1474 Steps: 46000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 005560, Sample Num: 88960, Cur Loss: 0.00000370, Cur Avg Loss: 0.00249263, Log Avg loss: 0.00002507, Global Avg Loss: 0.01135240, Time: 0.1329 Steps: 46200, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 005760, Sample Num: 92160, Cur Loss: 0.00000061, Cur Avg Loss: 0.00251587, Log Avg loss: 0.00316185, Global Avg Loss: 0.01131710, Time: 0.1847 Steps: 46400, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 005960, Sample Num: 95360, Cur Loss: 0.00000612, Cur Avg Loss: 0.00273447, Log Avg loss: 0.00903014, Global Avg Loss: 0.01130728, Time: 0.1071 Steps: 46600, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 006160, Sample Num: 98560, Cur Loss: 0.00001477, Cur Avg Loss: 0.00286032, Log Avg loss: 0.00661087, Global Avg Loss: 0.01128721, Time: 0.1876 Steps: 46800, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 006360, Sample Num: 101760, Cur Loss: 0.00019953, Cur Avg Loss: 0.00277283, Log Avg loss: 0.00007802, Global Avg Loss: 0.01123951, Time: 0.2257 Steps: 47000, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 006560, Sample Num: 104960, Cur Loss: 0.00000056, Cur Avg Loss: 0.00275961, Log Avg loss: 0.00233931, Global Avg Loss: 0.01120180, Time: 0.1093 Steps: 47200, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 006760, Sample Num: 108160, Cur Loss: 0.00001470, Cur Avg Loss: 0.00274136, Log Avg loss: 0.00214280, Global Avg Loss: 0.01116358, Time: 0.0480 Steps: 47400, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 006960, Sample Num: 111360, Cur Loss: 0.00010701, Cur Avg Loss: 0.00266447, Log Avg loss: 0.00006550, Global Avg Loss: 0.01111695, Time: 0.0531 Steps: 47600, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 007160, Sample Num: 114560, Cur Loss: 0.00002480, Cur Avg Loss: 0.00259230, Log Avg loss: 0.00008073, Global Avg Loss: 0.01107077, Time: 0.0337 Steps: 47800, Updated lr: 0.000077 Training, Epoch: 0003, Batch: 007360, Sample Num: 117760, Cur Loss: 0.00000013, Cur Avg Loss: 0.00263430, Log Avg loss: 0.00413783, Global Avg Loss: 0.01104188, Time: 0.2588 Steps: 48000, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 007560, Sample Num: 120960, Cur Loss: 0.00000308, Cur Avg Loss: 0.00277015, Log Avg loss: 0.00776958, Global Avg Loss: 0.01102830, Time: 0.1115 Steps: 48200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 007760, Sample Num: 124160, Cur Loss: 0.00000094, Cur Avg Loss: 0.00280334, Log Avg loss: 0.00405789, Global Avg Loss: 0.01099950, Time: 0.1094 Steps: 48400, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 007960, Sample Num: 127360, Cur Loss: 0.00000187, Cur Avg Loss: 0.00293985, Log Avg loss: 0.00823653, Global Avg Loss: 0.01098813, Time: 0.1219 Steps: 48600, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 008160, Sample Num: 130560, Cur Loss: 0.00000294, Cur Avg Loss: 0.00287067, Log Avg loss: 0.00011706, Global Avg Loss: 0.01094358, Time: 0.1088 Steps: 48800, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 008360, Sample Num: 133760, Cur Loss: 0.00000529, Cur Avg Loss: 0.00287939, Log Avg loss: 0.00323520, Global Avg Loss: 0.01091211, Time: 0.0730 Steps: 49000, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 008560, Sample Num: 136960, Cur Loss: 0.00000257, Cur Avg Loss: 0.00284734, Log Avg loss: 0.00150763, Global Avg Loss: 0.01087388, Time: 0.1184 Steps: 49200, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 008760, Sample Num: 140160, Cur Loss: 0.00000263, Cur Avg Loss: 0.00278670, Log Avg loss: 0.00019137, Global Avg Loss: 0.01083064, Time: 0.0576 Steps: 49400, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 008960, Sample Num: 143360, Cur Loss: 0.00000379, Cur Avg Loss: 0.00282444, Log Avg loss: 0.00447736, Global Avg Loss: 0.01080502, Time: 0.0802 Steps: 49600, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 009160, Sample Num: 146560, Cur Loss: 0.00000155, Cur Avg Loss: 0.00276443, Log Avg loss: 0.00007609, Global Avg Loss: 0.01076193, Time: 0.1441 Steps: 49800, Updated lr: 0.000076 Training, Epoch: 0003, Batch: 009360, Sample Num: 149760, Cur Loss: 0.00000367, Cur Avg Loss: 0.00270567, Log Avg loss: 0.00001465, Global Avg Loss: 0.01071894, Time: 0.0644 Steps: 50000, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 009560, Sample Num: 152960, Cur Loss: 0.00000238, Cur Avg Loss: 0.00274261, Log Avg loss: 0.00447121, Global Avg Loss: 0.01069405, Time: 0.0201 Steps: 50200, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 009760, Sample Num: 156160, Cur Loss: 0.00000018, Cur Avg Loss: 0.00268682, Log Avg loss: 0.00001996, Global Avg Loss: 0.01065169, Time: 0.0967 Steps: 50400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 009960, Sample Num: 159360, Cur Loss: 0.00001600, Cur Avg Loss: 0.00271041, Log Avg loss: 0.00386168, Global Avg Loss: 0.01062485, Time: 0.0710 Steps: 50600, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 010160, Sample Num: 162560, Cur Loss: 0.00000104, Cur Avg Loss: 0.00265722, Log Avg loss: 0.00000835, Global Avg Loss: 0.01058306, Time: 0.0364 Steps: 50800, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 010360, Sample Num: 165760, Cur Loss: 0.00000002, Cur Avg Loss: 0.00262666, Log Avg loss: 0.00107406, Global Avg Loss: 0.01054577, Time: 0.0469 Steps: 51000, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 010560, Sample Num: 168960, Cur Loss: 0.00000098, Cur Avg Loss: 0.00267271, Log Avg loss: 0.00505845, Global Avg Loss: 0.01052433, Time: 0.0571 Steps: 51200, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 010760, Sample Num: 172160, Cur Loss: 0.00000171, Cur Avg Loss: 0.00262420, Log Avg loss: 0.00006265, Global Avg Loss: 0.01048362, Time: 0.1092 Steps: 51400, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 010960, Sample Num: 175360, Cur Loss: 0.00000001, Cur Avg Loss: 0.00257644, Log Avg loss: 0.00000729, Global Avg Loss: 0.01044302, Time: 0.0443 Steps: 51600, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 011160, Sample Num: 178560, Cur Loss: 0.00000039, Cur Avg Loss: 0.00269543, Log Avg loss: 0.00921585, Global Avg Loss: 0.01043828, Time: 0.1919 Steps: 51800, Updated lr: 0.000075 Training, Epoch: 0003, Batch: 011360, Sample Num: 181760, Cur Loss: 0.00000165, Cur Avg Loss: 0.00271230, Log Avg loss: 0.00365368, Global Avg Loss: 0.01041219, Time: 0.0291 Steps: 52000, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 011560, Sample Num: 184960, Cur Loss: 0.00000013, Cur Avg Loss: 0.00271575, Log Avg loss: 0.00291148, Global Avg Loss: 0.01038345, Time: 0.1612 Steps: 52200, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 011760, Sample Num: 188160, Cur Loss: 0.00000095, Cur Avg Loss: 0.00266970, Log Avg loss: 0.00000798, Global Avg Loss: 0.01034385, Time: 0.2317 Steps: 52400, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 011960, Sample Num: 191360, Cur Loss: 0.00000016, Cur Avg Loss: 0.00270039, Log Avg loss: 0.00450528, Global Avg Loss: 0.01032165, Time: 0.1536 Steps: 52600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 012160, Sample Num: 194560, Cur Loss: 0.00000006, Cur Avg Loss: 0.00265637, Log Avg loss: 0.00002392, Global Avg Loss: 0.01028264, Time: 0.0743 Steps: 52800, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 012360, Sample Num: 197760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00261389, Log Avg loss: 0.00003089, Global Avg Loss: 0.01024395, Time: 0.1516 Steps: 53000, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 012560, Sample Num: 200960, Cur Loss: 0.00214881, Cur Avg Loss: 0.00260579, Log Avg loss: 0.00210528, Global Avg Loss: 0.01021336, Time: 0.1514 Steps: 53200, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 012760, Sample Num: 204160, Cur Loss: 0.00000007, Cur Avg Loss: 0.00264258, Log Avg loss: 0.00495340, Global Avg Loss: 0.01019366, Time: 0.0630 Steps: 53400, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 012960, Sample Num: 207360, Cur Loss: 0.00000045, Cur Avg Loss: 0.00260184, Log Avg loss: 0.00000258, Global Avg Loss: 0.01015563, Time: 0.0641 Steps: 53600, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 013160, Sample Num: 210560, Cur Loss: 0.00000022, Cur Avg Loss: 0.00263063, Log Avg loss: 0.00449583, Global Avg Loss: 0.01013459, Time: 0.1318 Steps: 53800, Updated lr: 0.000074 Training, Epoch: 0003, Batch: 013360, Sample Num: 213760, Cur Loss: 0.00000019, Cur Avg Loss: 0.00259131, Log Avg loss: 0.00000433, Global Avg Loss: 0.01009707, Time: 0.1690 Steps: 54000, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 013560, Sample Num: 216960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00255332, Log Avg loss: 0.00001540, Global Avg Loss: 0.01005987, Time: 0.2314 Steps: 54200, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 013760, Sample Num: 220160, Cur Loss: 0.00000013, Cur Avg Loss: 0.00253839, Log Avg loss: 0.00152594, Global Avg Loss: 0.01002849, Time: 0.1898 Steps: 54400, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 013960, Sample Num: 223360, Cur Loss: 0.00000232, Cur Avg Loss: 0.00255791, Log Avg loss: 0.00390123, Global Avg Loss: 0.01000605, Time: 0.1128 Steps: 54600, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 014160, Sample Num: 226560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00252690, Log Avg loss: 0.00036251, Global Avg Loss: 0.00997085, Time: 0.1867 Steps: 54800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 014360, Sample Num: 229760, Cur Loss: 0.00000019, Cur Avg Loss: 0.00259108, Log Avg loss: 0.00713484, Global Avg Loss: 0.00996054, Time: 0.0755 Steps: 55000, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 014560, Sample Num: 232960, Cur Loss: 0.00000001, Cur Avg Loss: 0.00255675, Log Avg loss: 0.00009182, Global Avg Loss: 0.00992479, Time: 0.0635 Steps: 55200, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 014760, Sample Num: 236160, Cur Loss: 0.00000005, Cur Avg Loss: 0.00254222, Log Avg loss: 0.00148435, Global Avg Loss: 0.00989431, Time: 0.0350 Steps: 55400, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 014960, Sample Num: 239360, Cur Loss: 0.00000001, Cur Avg Loss: 0.00253808, Log Avg loss: 0.00223294, Global Avg Loss: 0.00986676, Time: 0.1731 Steps: 55600, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 015160, Sample Num: 242560, Cur Loss: 0.00000234, Cur Avg Loss: 0.00250468, Log Avg loss: 0.00000635, Global Avg Loss: 0.00983141, Time: 0.1164 Steps: 55800, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 015360, Sample Num: 245760, Cur Loss: 0.00000906, Cur Avg Loss: 0.00247217, Log Avg loss: 0.00000759, Global Avg Loss: 0.00979633, Time: 0.3094 Steps: 56000, Updated lr: 0.000073 Training, Epoch: 0003, Batch: 015560, Sample Num: 248960, Cur Loss: 0.00000001, Cur Avg Loss: 0.00246437, Log Avg loss: 0.00186524, Global Avg Loss: 0.00976810, Time: 0.1188 Steps: 56200, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 015760, Sample Num: 252160, Cur Loss: 0.00000006, Cur Avg Loss: 0.00245327, Log Avg loss: 0.00158966, Global Avg Loss: 0.00973910, Time: 0.0538 Steps: 56400, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 015960, Sample Num: 255360, Cur Loss: 0.00000005, Cur Avg Loss: 0.00242262, Log Avg loss: 0.00000749, Global Avg Loss: 0.00970472, Time: 0.0751 Steps: 56600, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 016160, Sample Num: 258560, Cur Loss: 0.00000007, Cur Avg Loss: 0.00239571, Log Avg loss: 0.00024869, Global Avg Loss: 0.00967142, Time: 0.0565 Steps: 56800, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 016360, Sample Num: 261760, Cur Loss: 0.00000013, Cur Avg Loss: 0.00236651, Log Avg loss: 0.00000661, Global Avg Loss: 0.00963751, Time: 0.1525 Steps: 57000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 016560, Sample Num: 264960, Cur Loss: 0.00000027, Cur Avg Loss: 0.00236389, Log Avg loss: 0.00214993, Global Avg Loss: 0.00961133, Time: 0.0529 Steps: 57200, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 016760, Sample Num: 268160, Cur Loss: 0.00000003, Cur Avg Loss: 0.00233584, Log Avg loss: 0.00001305, Global Avg Loss: 0.00957788, Time: 0.2268 Steps: 57400, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 016960, Sample Num: 271360, Cur Loss: 0.00000001, Cur Avg Loss: 0.00230845, Log Avg loss: 0.00001346, Global Avg Loss: 0.00954467, Time: 0.1160 Steps: 57600, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 017160, Sample Num: 274560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00238963, Log Avg loss: 0.00927380, Global Avg Loss: 0.00954374, Time: 0.1132 Steps: 57800, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 017360, Sample Num: 277760, Cur Loss: 0.00000042, Cur Avg Loss: 0.00236235, Log Avg loss: 0.00002121, Global Avg Loss: 0.00951090, Time: 0.0265 Steps: 58000, Updated lr: 0.000072 Training, Epoch: 0003, Batch: 017560, Sample Num: 280960, Cur Loss: 0.00000104, Cur Avg Loss: 0.00233551, Log Avg loss: 0.00000617, Global Avg Loss: 0.00947824, Time: 0.1775 Steps: 58200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 017760, Sample Num: 284160, Cur Loss: 0.00000221, Cur Avg Loss: 0.00235188, Log Avg loss: 0.00378874, Global Avg Loss: 0.00945875, Time: 0.0417 Steps: 58400, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 017960, Sample Num: 287360, Cur Loss: 0.00000005, Cur Avg Loss: 0.00232878, Log Avg loss: 0.00027812, Global Avg Loss: 0.00942742, Time: 0.1431 Steps: 58600, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 018160, Sample Num: 290560, Cur Loss: 0.00000012, Cur Avg Loss: 0.00230316, Log Avg loss: 0.00000255, Global Avg Loss: 0.00939536, Time: 0.1147 Steps: 58800, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 018360, Sample Num: 293760, Cur Loss: 0.00000116, Cur Avg Loss: 0.00230624, Log Avg loss: 0.00258553, Global Avg Loss: 0.00937228, Time: 0.1269 Steps: 59000, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 018560, Sample Num: 296960, Cur Loss: 0.00000089, Cur Avg Loss: 0.00228554, Log Avg loss: 0.00038517, Global Avg Loss: 0.00934192, Time: 0.1484 Steps: 59200, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 018760, Sample Num: 300160, Cur Loss: 0.00000002, Cur Avg Loss: 0.00226194, Log Avg loss: 0.00007205, Global Avg Loss: 0.00931071, Time: 0.1077 Steps: 59400, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 018960, Sample Num: 303360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00224374, Log Avg loss: 0.00053615, Global Avg Loss: 0.00928126, Time: 0.0615 Steps: 59600, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 019160, Sample Num: 306560, Cur Loss: 0.00000057, Cur Avg Loss: 0.00222341, Log Avg loss: 0.00029643, Global Avg Loss: 0.00925121, Time: 0.1277 Steps: 59800, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 019360, Sample Num: 309760, Cur Loss: 0.00000001, Cur Avg Loss: 0.00224322, Log Avg loss: 0.00414092, Global Avg Loss: 0.00923418, Time: 0.0704 Steps: 60000, Updated lr: 0.000071 Training, Epoch: 0003, Batch: 019560, Sample Num: 312960, Cur Loss: 0.00000001, Cur Avg Loss: 0.00222037, Log Avg loss: 0.00000889, Global Avg Loss: 0.00920353, Time: 0.1608 Steps: 60200, Updated lr: 0.000070 Training, Epoch: 0003, Batch: 019760, Sample Num: 316160, Cur Loss: 0.00000018, Cur Avg Loss: 0.00219797, Log Avg loss: 0.00000672, Global Avg Loss: 0.00917308, Time: 0.1680 Steps: 60400, Updated lr: 0.000070 Training, Epoch: 0003, Batch: 019960, Sample Num: 319360, Cur Loss: 0.00000297, Cur Avg Loss: 0.00217601, Log Avg loss: 0.00000653, Global Avg Loss: 0.00914282, Time: 0.0587 Steps: 60600, Updated lr: 0.000070 Training, Epoch: 0003, Batch: 020160, Sample Num: 322560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00215444, Log Avg loss: 0.00000214, Global Avg Loss: 0.00911275, Time: 0.1864 Steps: 60800, Updated lr: 0.000070 ***** Running evaluation checkpoint-60960 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-60960 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2009.783862, Avg time per batch (s): 0.100000 {"eval_avg_loss": 0.014289, "eval_total_loss": 38.722767, "eval_acc": 0.998385, "eval_prec": 0.998986, "eval_recall": 0.99779, "eval_f1": 0.998388, "eval_roc_auc": 0.999885, "eval_pr_auc": 0.999838, "eval_confusion_matrix": {"tn": 21605, "fp": 22, "fn": 48, "tp": 21674}, "eval_mcc2": 0.996771, "eval_mcc": 0.996771, "eval_sn": 0.99779, "eval_sp": 0.998983, "update_flag": true, "test_avg_loss": 0.011784, "test_total_loss": 47.889335, "test_acc": 0.998308, "test_prec": 0.998922, "test_recall": 0.997693, "test_f1": 0.998307, "test_roc_auc": 0.999951, "test_pr_auc": 0.999955, "test_confusion_matrix": {"tn": 32482, "fp": 35, "fn": 75, "tp": 32430}, "test_mcc2": 0.996617, "test_mcc": 0.996617, "test_sn": 0.997693, "test_sp": 0.998924, "lr": 7.006896551724137e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.009114834057643668, "train_cur_epoch_loss": 45.018387130562175, "train_cur_epoch_avg_loss": 0.0022154718076064063, "train_cur_epoch_time": 2009.7838623523712, "train_cur_epoch_avg_time": 0.09890668613938834, "epoch": 3, "step": 60960} ################################################## Training, Epoch: 0004, Batch: 000040, Sample Num: 640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000035, Log Avg loss: 0.00792416, Global Avg Loss: 0.00910886, Time: 0.1079 Steps: 61000, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000240, Sample Num: 3840, Cur Loss: 0.00004127, Cur Avg Loss: 0.00002890, Log Avg loss: 0.00003461, Global Avg Loss: 0.00907920, Time: 0.0910 Steps: 61200, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000440, Sample Num: 7040, Cur Loss: 0.00000029, Cur Avg Loss: 0.00002036, Log Avg loss: 0.00001012, Global Avg Loss: 0.00904966, Time: 0.0701 Steps: 61400, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000640, Sample Num: 10240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001483, Log Avg loss: 0.00000265, Global Avg Loss: 0.00902029, Time: 0.1597 Steps: 61600, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 000840, Sample Num: 13440, Cur Loss: 0.00000007, Cur Avg Loss: 0.00001217, Log Avg loss: 0.00000365, Global Avg Loss: 0.00899111, Time: 0.0730 Steps: 61800, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 001040, Sample Num: 16640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001091, Log Avg loss: 0.00000565, Global Avg Loss: 0.00896212, Time: 0.0414 Steps: 62000, Updated lr: 0.000070 Training, Epoch: 0004, Batch: 001240, Sample Num: 19840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000959, Log Avg loss: 0.00000273, Global Avg Loss: 0.00893331, Time: 0.1129 Steps: 62200, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001440, Sample Num: 23040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00006077, Log Avg loss: 0.00037804, Global Avg Loss: 0.00890589, Time: 0.0482 Steps: 62400, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001640, Sample Num: 26240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00010256, Log Avg loss: 0.00040345, Global Avg Loss: 0.00887873, Time: 0.0280 Steps: 62600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 001840, Sample Num: 29440, Cur Loss: 0.00000002, Cur Avg Loss: 0.00009149, Log Avg loss: 0.00000072, Global Avg Loss: 0.00885046, Time: 0.0564 Steps: 62800, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002040, Sample Num: 32640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00041701, Log Avg loss: 0.00341181, Global Avg Loss: 0.00883319, Time: 0.0721 Steps: 63000, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002240, Sample Num: 35840, Cur Loss: 0.00000018, Cur Avg Loss: 0.00048838, Log Avg loss: 0.00121638, Global Avg Loss: 0.00880909, Time: 0.1273 Steps: 63200, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002440, Sample Num: 39040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00046374, Log Avg loss: 0.00018769, Global Avg Loss: 0.00878189, Time: 0.1500 Steps: 63400, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002640, Sample Num: 42240, Cur Loss: 0.00000006, Cur Avg Loss: 0.00060832, Log Avg loss: 0.00237231, Global Avg Loss: 0.00876173, Time: 0.0352 Steps: 63600, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 002840, Sample Num: 45440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00113493, Log Avg loss: 0.00808612, Global Avg Loss: 0.00875962, Time: 0.0981 Steps: 63800, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 003040, Sample Num: 48640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00165107, Log Avg loss: 0.00898031, Global Avg Loss: 0.00876031, Time: 0.0463 Steps: 64000, Updated lr: 0.000069 Training, Epoch: 0004, Batch: 003240, Sample Num: 51840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00185917, Log Avg loss: 0.00502228, Global Avg Loss: 0.00874866, Time: 0.0693 Steps: 64200, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003440, Sample Num: 55040, Cur Loss: 0.00000068, Cur Avg Loss: 0.00175228, Log Avg loss: 0.00002063, Global Avg Loss: 0.00872155, Time: 0.1278 Steps: 64400, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003640, Sample Num: 58240, Cur Loss: 0.00000001, Cur Avg Loss: 0.00181103, Log Avg loss: 0.00282156, Global Avg Loss: 0.00870329, Time: 0.0690 Steps: 64600, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 003840, Sample Num: 61440, Cur Loss: 0.00002620, Cur Avg Loss: 0.00196956, Log Avg loss: 0.00485470, Global Avg Loss: 0.00869141, Time: 0.1869 Steps: 64800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 004040, Sample Num: 64640, Cur Loss: 0.00000002, Cur Avg Loss: 0.00198584, Log Avg loss: 0.00229846, Global Avg Loss: 0.00867174, Time: 0.0361 Steps: 65000, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 004240, Sample Num: 67840, Cur Loss: 0.00000003, Cur Avg Loss: 0.00189227, Log Avg loss: 0.00000216, Global Avg Loss: 0.00864515, Time: 0.1502 Steps: 65200, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 004440, Sample Num: 71040, Cur Loss: 0.00000049, Cur Avg Loss: 0.00201740, Log Avg loss: 0.00467015, Global Avg Loss: 0.00863299, Time: 0.1059 Steps: 65400, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 004640, Sample Num: 74240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00237061, Log Avg loss: 0.01021181, Global Avg Loss: 0.00863780, Time: 0.0428 Steps: 65600, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 004840, Sample Num: 77440, Cur Loss: 0.00000015, Cur Avg Loss: 0.00228212, Log Avg loss: 0.00022927, Global Avg Loss: 0.00861225, Time: 0.1655 Steps: 65800, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 005040, Sample Num: 80640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00221128, Log Avg loss: 0.00049687, Global Avg Loss: 0.00858765, Time: 0.0655 Steps: 66000, Updated lr: 0.000068 Training, Epoch: 0004, Batch: 005240, Sample Num: 83840, Cur Loss: 0.00000001, Cur Avg Loss: 0.00212700, Log Avg loss: 0.00000325, Global Avg Loss: 0.00856172, Time: 0.1181 Steps: 66200, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 005440, Sample Num: 87040, Cur Loss: 0.00000012, Cur Avg Loss: 0.00204884, Log Avg loss: 0.00000106, Global Avg Loss: 0.00853593, Time: 0.1276 Steps: 66400, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 005640, Sample Num: 90240, Cur Loss: 0.00000001, Cur Avg Loss: 0.00198062, Log Avg loss: 0.00012510, Global Avg Loss: 0.00851068, Time: 0.1565 Steps: 66600, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 005840, Sample Num: 93440, Cur Loss: 0.00000002, Cur Avg Loss: 0.00214874, Log Avg loss: 0.00688959, Global Avg Loss: 0.00850582, Time: 0.0518 Steps: 66800, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 006040, Sample Num: 96640, Cur Loss: 0.00000715, Cur Avg Loss: 0.00248529, Log Avg loss: 0.01231266, Global Avg Loss: 0.00851719, Time: 0.0439 Steps: 67000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 006240, Sample Num: 99840, Cur Loss: 0.00000007, Cur Avg Loss: 0.00245324, Log Avg loss: 0.00148517, Global Avg Loss: 0.00849626, Time: 0.1072 Steps: 67200, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 006440, Sample Num: 103040, Cur Loss: 0.00011915, Cur Avg Loss: 0.00238349, Log Avg loss: 0.00020741, Global Avg Loss: 0.00847166, Time: 0.0671 Steps: 67400, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 006640, Sample Num: 106240, Cur Loss: 0.00000097, Cur Avg Loss: 0.00231188, Log Avg loss: 0.00000610, Global Avg Loss: 0.00844662, Time: 0.1089 Steps: 67600, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 006840, Sample Num: 109440, Cur Loss: 0.00000006, Cur Avg Loss: 0.00224443, Log Avg loss: 0.00000493, Global Avg Loss: 0.00842171, Time: 0.1145 Steps: 67800, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 007040, Sample Num: 112640, Cur Loss: 0.00000110, Cur Avg Loss: 0.00218069, Log Avg loss: 0.00000070, Global Avg Loss: 0.00839695, Time: 0.2617 Steps: 68000, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 007240, Sample Num: 115840, Cur Loss: 0.00000006, Cur Avg Loss: 0.00214094, Log Avg loss: 0.00074171, Global Avg Loss: 0.00837450, Time: 0.1620 Steps: 68200, Updated lr: 0.000067 Training, Epoch: 0004, Batch: 007440, Sample Num: 119040, Cur Loss: 0.00000002, Cur Avg Loss: 0.00227354, Log Avg loss: 0.00707385, Global Avg Loss: 0.00837069, Time: 0.0476 Steps: 68400, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 007640, Sample Num: 122240, Cur Loss: 0.00000001, Cur Avg Loss: 0.00233898, Log Avg loss: 0.00477318, Global Avg Loss: 0.00836021, Time: 0.2435 Steps: 68600, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 007840, Sample Num: 125440, Cur Loss: 0.00000061, Cur Avg Loss: 0.00242969, Log Avg loss: 0.00589490, Global Avg Loss: 0.00835304, Time: 0.1130 Steps: 68800, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 008040, Sample Num: 128640, Cur Loss: 0.00000024, Cur Avg Loss: 0.00262662, Log Avg loss: 0.01034623, Global Avg Loss: 0.00835882, Time: 0.0451 Steps: 69000, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 008240, Sample Num: 131840, Cur Loss: 0.00000031, Cur Avg Loss: 0.00258401, Log Avg loss: 0.00087112, Global Avg Loss: 0.00833718, Time: 0.1579 Steps: 69200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 008440, Sample Num: 135040, Cur Loss: 0.00000073, Cur Avg Loss: 0.00253001, Log Avg loss: 0.00030540, Global Avg Loss: 0.00831403, Time: 0.1032 Steps: 69400, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 008640, Sample Num: 138240, Cur Loss: 0.00000001, Cur Avg Loss: 0.00250955, Log Avg loss: 0.00164597, Global Avg Loss: 0.00829487, Time: 0.1142 Steps: 69600, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 008840, Sample Num: 141440, Cur Loss: 0.00000006, Cur Avg Loss: 0.00245293, Log Avg loss: 0.00000698, Global Avg Loss: 0.00827112, Time: 0.0635 Steps: 69800, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 009040, Sample Num: 144640, Cur Loss: 0.00000001, Cur Avg Loss: 0.00248732, Log Avg loss: 0.00400724, Global Avg Loss: 0.00825894, Time: 0.0452 Steps: 70000, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 009240, Sample Num: 147840, Cur Loss: 0.00000310, Cur Avg Loss: 0.00243349, Log Avg loss: 0.00000048, Global Avg Loss: 0.00823541, Time: 0.0698 Steps: 70200, Updated lr: 0.000066 Training, Epoch: 0004, Batch: 009440, Sample Num: 151040, Cur Loss: 0.00000007, Cur Avg Loss: 0.00246368, Log Avg loss: 0.00385838, Global Avg Loss: 0.00822297, Time: 0.0443 Steps: 70400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 009640, Sample Num: 154240, Cur Loss: 0.00000058, Cur Avg Loss: 0.00241399, Log Avg loss: 0.00006849, Global Avg Loss: 0.00819987, Time: 0.1107 Steps: 70600, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 009840, Sample Num: 157440, Cur Loss: 0.00000188, Cur Avg Loss: 0.00240135, Log Avg loss: 0.00179223, Global Avg Loss: 0.00818177, Time: 0.1571 Steps: 70800, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 010040, Sample Num: 160640, Cur Loss: 0.00000025, Cur Avg Loss: 0.00235386, Log Avg loss: 0.00001736, Global Avg Loss: 0.00815877, Time: 0.0539 Steps: 71000, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 010240, Sample Num: 163840, Cur Loss: 0.00000007, Cur Avg Loss: 0.00230794, Log Avg loss: 0.00000268, Global Avg Loss: 0.00813586, Time: 0.3748 Steps: 71200, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 010440, Sample Num: 167040, Cur Loss: 0.00000006, Cur Avg Loss: 0.00228430, Log Avg loss: 0.00107401, Global Avg Loss: 0.00811608, Time: 0.2352 Steps: 71400, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 010640, Sample Num: 170240, Cur Loss: 0.00000119, Cur Avg Loss: 0.00230618, Log Avg loss: 0.00344820, Global Avg Loss: 0.00810304, Time: 0.2645 Steps: 71600, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 010840, Sample Num: 173440, Cur Loss: 0.00000054, Cur Avg Loss: 0.00226370, Log Avg loss: 0.00000381, Global Avg Loss: 0.00808048, Time: 0.1111 Steps: 71800, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 011040, Sample Num: 176640, Cur Loss: 0.68581563, Cur Avg Loss: 0.00231604, Log Avg loss: 0.00515284, Global Avg Loss: 0.00807235, Time: 0.0465 Steps: 72000, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 011240, Sample Num: 179840, Cur Loss: 0.00004741, Cur Avg Loss: 0.00241915, Log Avg loss: 0.00811103, Global Avg Loss: 0.00807246, Time: 0.1848 Steps: 72200, Updated lr: 0.000065 Training, Epoch: 0004, Batch: 011440, Sample Num: 183040, Cur Loss: 0.00000002, Cur Avg Loss: 0.00240692, Log Avg loss: 0.00171946, Global Avg Loss: 0.00805491, Time: 0.0534 Steps: 72400, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 011640, Sample Num: 186240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00241646, Log Avg loss: 0.00296219, Global Avg Loss: 0.00804088, Time: 0.1148 Steps: 72600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 011840, Sample Num: 189440, Cur Loss: 0.00000024, Cur Avg Loss: 0.00239496, Log Avg loss: 0.00114346, Global Avg Loss: 0.00802193, Time: 0.1112 Steps: 72800, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 012040, Sample Num: 192640, Cur Loss: 0.00000769, Cur Avg Loss: 0.00235522, Log Avg loss: 0.00000275, Global Avg Loss: 0.00799996, Time: 0.1098 Steps: 73000, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 012240, Sample Num: 195840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00231682, Log Avg loss: 0.00000535, Global Avg Loss: 0.00797812, Time: 0.0654 Steps: 73200, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 012440, Sample Num: 199040, Cur Loss: 0.00000003, Cur Avg Loss: 0.00227959, Log Avg loss: 0.00000122, Global Avg Loss: 0.00795638, Time: 0.0445 Steps: 73400, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 012640, Sample Num: 202240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00232107, Log Avg loss: 0.00490085, Global Avg Loss: 0.00794808, Time: 0.1130 Steps: 73600, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 012840, Sample Num: 205440, Cur Loss: 0.00000435, Cur Avg Loss: 0.00228932, Log Avg loss: 0.00028309, Global Avg Loss: 0.00792731, Time: 0.0362 Steps: 73800, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 013040, Sample Num: 208640, Cur Loss: 0.00000001, Cur Avg Loss: 0.00235161, Log Avg loss: 0.00635044, Global Avg Loss: 0.00792304, Time: 0.1697 Steps: 74000, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 013240, Sample Num: 211840, Cur Loss: 0.00000001, Cur Avg Loss: 0.00231612, Log Avg loss: 0.00000200, Global Avg Loss: 0.00790169, Time: 0.0907 Steps: 74200, Updated lr: 0.000064 Training, Epoch: 0004, Batch: 013440, Sample Num: 215040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00228166, Log Avg loss: 0.00000031, Global Avg Loss: 0.00788045, Time: 0.1062 Steps: 74400, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 013640, Sample Num: 218240, Cur Loss: 0.00000006, Cur Avg Loss: 0.00224821, Log Avg loss: 0.00000034, Global Avg Loss: 0.00785933, Time: 0.0588 Steps: 74600, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 013840, Sample Num: 221440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00225282, Log Avg loss: 0.00256723, Global Avg Loss: 0.00784518, Time: 0.0906 Steps: 74800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 014040, Sample Num: 224640, Cur Loss: 0.00000012, Cur Avg Loss: 0.00222532, Log Avg loss: 0.00032227, Global Avg Loss: 0.00782512, Time: 0.0643 Steps: 75000, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 014240, Sample Num: 227840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00226894, Log Avg loss: 0.00533124, Global Avg Loss: 0.00781848, Time: 0.0419 Steps: 75200, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 014440, Sample Num: 231040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00227994, Log Avg loss: 0.00306287, Global Avg Loss: 0.00780587, Time: 0.0599 Steps: 75400, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 014640, Sample Num: 234240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00224882, Log Avg loss: 0.00000239, Global Avg Loss: 0.00778523, Time: 0.1124 Steps: 75600, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 014840, Sample Num: 237440, Cur Loss: 0.00000011, Cur Avg Loss: 0.00224045, Log Avg loss: 0.00162735, Global Avg Loss: 0.00776898, Time: 0.0771 Steps: 75800, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 015040, Sample Num: 240640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00221524, Log Avg loss: 0.00034476, Global Avg Loss: 0.00774944, Time: 0.2613 Steps: 76000, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 015240, Sample Num: 243840, Cur Loss: 0.00000001, Cur Avg Loss: 0.00218627, Log Avg loss: 0.00000791, Global Avg Loss: 0.00772912, Time: 0.0795 Steps: 76200, Updated lr: 0.000063 Training, Epoch: 0004, Batch: 015440, Sample Num: 247040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00215807, Log Avg loss: 0.00000934, Global Avg Loss: 0.00770891, Time: 0.1050 Steps: 76400, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 015640, Sample Num: 250240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00213053, Log Avg loss: 0.00000401, Global Avg Loss: 0.00768880, Time: 0.0523 Steps: 76600, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 015840, Sample Num: 253440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00210364, Log Avg loss: 0.00000116, Global Avg Loss: 0.00766878, Time: 0.0757 Steps: 76800, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 016040, Sample Num: 256640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00207773, Log Avg loss: 0.00002562, Global Avg Loss: 0.00764892, Time: 0.0442 Steps: 77000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 016240, Sample Num: 259840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00205214, Log Avg loss: 0.00000026, Global Avg Loss: 0.00762911, Time: 0.0883 Steps: 77200, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 016440, Sample Num: 263040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00204305, Log Avg loss: 0.00130438, Global Avg Loss: 0.00761276, Time: 0.0654 Steps: 77400, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 016640, Sample Num: 266240, Cur Loss: 0.00000001, Cur Avg Loss: 0.00201857, Log Avg loss: 0.00000693, Global Avg Loss: 0.00759316, Time: 0.0729 Steps: 77600, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 016840, Sample Num: 269440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00199519, Log Avg loss: 0.00004920, Global Avg Loss: 0.00757377, Time: 0.0832 Steps: 77800, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 017040, Sample Num: 272640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00197178, Log Avg loss: 0.00000120, Global Avg Loss: 0.00755435, Time: 0.0410 Steps: 78000, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 017240, Sample Num: 275840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00210126, Log Avg loss: 0.01313311, Global Avg Loss: 0.00756862, Time: 0.1097 Steps: 78200, Updated lr: 0.000062 Training, Epoch: 0004, Batch: 017440, Sample Num: 279040, Cur Loss: 0.00000006, Cur Avg Loss: 0.00207719, Log Avg loss: 0.00000223, Global Avg Loss: 0.00754932, Time: 0.0371 Steps: 78400, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 017640, Sample Num: 282240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00205371, Log Avg loss: 0.00000631, Global Avg Loss: 0.00753012, Time: 0.0482 Steps: 78600, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 017840, Sample Num: 285440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00203296, Log Avg loss: 0.00020283, Global Avg Loss: 0.00751153, Time: 0.1136 Steps: 78800, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 018040, Sample Num: 288640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00201043, Log Avg loss: 0.00000083, Global Avg Loss: 0.00749251, Time: 0.0554 Steps: 79000, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 018240, Sample Num: 291840, Cur Loss: 0.00000016, Cur Avg Loss: 0.00200770, Log Avg loss: 0.00176109, Global Avg Loss: 0.00747804, Time: 0.0743 Steps: 79200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 018440, Sample Num: 295040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00199897, Log Avg loss: 0.00120248, Global Avg Loss: 0.00746223, Time: 0.1934 Steps: 79400, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 018640, Sample Num: 298240, Cur Loss: 0.00002444, Cur Avg Loss: 0.00197752, Log Avg loss: 0.00000052, Global Avg Loss: 0.00744348, Time: 0.1598 Steps: 79600, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 018840, Sample Num: 301440, Cur Loss: 0.00005019, Cur Avg Loss: 0.00195675, Log Avg loss: 0.00002052, Global Avg Loss: 0.00742488, Time: 0.0620 Steps: 79800, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 019040, Sample Num: 304640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00193621, Log Avg loss: 0.00000178, Global Avg Loss: 0.00740632, Time: 0.0525 Steps: 80000, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 019240, Sample Num: 307840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00191610, Log Avg loss: 0.00000097, Global Avg Loss: 0.00738786, Time: 0.1032 Steps: 80200, Updated lr: 0.000061 Training, Epoch: 0004, Batch: 019440, Sample Num: 311040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00190866, Log Avg loss: 0.00119347, Global Avg Loss: 0.00737245, Time: 0.1624 Steps: 80400, Updated lr: 0.000060 Training, Epoch: 0004, Batch: 019640, Sample Num: 314240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00188923, Log Avg loss: 0.00000077, Global Avg Loss: 0.00735415, Time: 0.0416 Steps: 80600, Updated lr: 0.000060 Training, Epoch: 0004, Batch: 019840, Sample Num: 317440, Cur Loss: 0.00000001, Cur Avg Loss: 0.00187019, Log Avg loss: 0.00000014, Global Avg Loss: 0.00733595, Time: 0.0731 Steps: 80800, Updated lr: 0.000060 Training, Epoch: 0004, Batch: 020040, Sample Num: 320640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00185153, Log Avg loss: 0.00000031, Global Avg Loss: 0.00731784, Time: 0.0303 Steps: 81000, Updated lr: 0.000060 Training, Epoch: 0004, Batch: 020240, Sample Num: 323840, Cur Loss: 0.00000003, Cur Avg Loss: 0.00186947, Log Avg loss: 0.00366725, Global Avg Loss: 0.00730885, Time: 0.1602 Steps: 81200, Updated lr: 0.000060 ***** Running evaluation checkpoint-81280 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-81280 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2000.960289, Avg time per batch (s): 0.100000 {"eval_avg_loss": 0.015381, "eval_total_loss": 41.68272, "eval_acc": 0.998408, "eval_prec": 0.998986, "eval_recall": 0.997836, "eval_f1": 0.998411, "eval_roc_auc": 0.999853, "eval_pr_auc": 0.999757, "eval_confusion_matrix": {"tn": 21605, "fp": 22, "fn": 47, "tp": 21675}, "eval_mcc2": 0.996817, "eval_mcc": 0.996817, "eval_sn": 0.997836, "eval_sp": 0.998983, "update_flag": true, "test_avg_loss": 0.013435, "test_total_loss": 54.601641, "test_acc": 0.998324, "test_prec": 0.99886, "test_recall": 0.997785, "test_f1": 0.998322, "test_roc_auc": 0.999945, "test_pr_auc": 0.99993, "test_confusion_matrix": {"tn": 32480, "fp": 37, "fn": 72, "tp": 32433}, "test_mcc2": 0.996648, "test_mcc": 0.996648, "test_sn": 0.997785, "test_sp": 0.998862, "lr": 6.005911330049261e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.00731290209692563, "train_cur_epoch_loss": 38.75239828412906, "train_cur_epoch_avg_loss": 0.0019071062147701307, "train_cur_epoch_time": 2000.9602885246277, "train_cur_epoch_avg_time": 0.09847245514392852, "epoch": 4, "step": 81280} ################################################## Training, Epoch: 0005, Batch: 000120, Sample Num: 1920, Cur Loss: 0.00000045, Cur Avg Loss: 0.00000235, Log Avg loss: 0.00457295, Global Avg Loss: 0.00730212, Time: 0.1612 Steps: 81400, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000320, Sample Num: 5120, Cur Loss: 0.00000004, Cur Avg Loss: 0.00101178, Log Avg loss: 0.00161744, Global Avg Loss: 0.00728819, Time: 0.2647 Steps: 81600, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000520, Sample Num: 8320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00062324, Log Avg loss: 0.00000157, Global Avg Loss: 0.00727038, Time: 0.1887 Steps: 81800, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000720, Sample Num: 11520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00045013, Log Avg loss: 0.00000006, Global Avg Loss: 0.00725264, Time: 0.0336 Steps: 82000, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 000920, Sample Num: 14720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00035229, Log Avg loss: 0.00000004, Global Avg Loss: 0.00723500, Time: 0.1088 Steps: 82200, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 001120, Sample Num: 17920, Cur Loss: 0.00000019, Cur Avg Loss: 0.00028939, Log Avg loss: 0.00000006, Global Avg Loss: 0.00721744, Time: 0.0413 Steps: 82400, Updated lr: 0.000060 Training, Epoch: 0005, Batch: 001320, Sample Num: 21120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00024555, Log Avg loss: 0.00000005, Global Avg Loss: 0.00719996, Time: 0.0317 Steps: 82600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001520, Sample Num: 24320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00021331, Log Avg loss: 0.00000053, Global Avg Loss: 0.00718257, Time: 0.1139 Steps: 82800, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001720, Sample Num: 27520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00018852, Log Avg loss: 0.00000011, Global Avg Loss: 0.00716526, Time: 0.1239 Steps: 83000, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 001920, Sample Num: 30720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00016888, Log Avg loss: 0.00000001, Global Avg Loss: 0.00714804, Time: 0.0492 Steps: 83200, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002120, Sample Num: 33920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00040244, Log Avg loss: 0.00264459, Global Avg Loss: 0.00713724, Time: 0.1071 Steps: 83400, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002320, Sample Num: 37120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00036800, Log Avg loss: 0.00000292, Global Avg Loss: 0.00712017, Time: 0.1502 Steps: 83600, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002520, Sample Num: 40320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00033880, Log Avg loss: 0.00000011, Global Avg Loss: 0.00710318, Time: 0.1049 Steps: 83800, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002720, Sample Num: 43520, Cur Loss: 0.00001323, Cur Avg Loss: 0.00043185, Log Avg loss: 0.00160424, Global Avg Loss: 0.00709009, Time: 0.1083 Steps: 84000, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 002920, Sample Num: 46720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00073709, Log Avg loss: 0.00488840, Global Avg Loss: 0.00708486, Time: 0.0996 Steps: 84200, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 003120, Sample Num: 49920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104011, Log Avg loss: 0.00546412, Global Avg Loss: 0.00708102, Time: 0.1135 Steps: 84400, Updated lr: 0.000059 Training, Epoch: 0005, Batch: 003320, Sample Num: 53120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00125631, Log Avg loss: 0.00462906, Global Avg Loss: 0.00707522, Time: 0.0453 Steps: 84600, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003520, Sample Num: 56320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00118793, Log Avg loss: 0.00005281, Global Avg Loss: 0.00705866, Time: 0.1864 Steps: 84800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003720, Sample Num: 59520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112774, Log Avg loss: 0.00006843, Global Avg Loss: 0.00704221, Time: 0.0875 Steps: 85000, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 003920, Sample Num: 62720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00126180, Log Avg loss: 0.00375541, Global Avg Loss: 0.00703449, Time: 0.0708 Steps: 85200, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 004120, Sample Num: 65920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00120068, Log Avg loss: 0.00000259, Global Avg Loss: 0.00701803, Time: 0.1112 Steps: 85400, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 004320, Sample Num: 69120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00114510, Log Avg loss: 0.00000017, Global Avg Loss: 0.00700163, Time: 0.1827 Steps: 85600, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 004520, Sample Num: 72320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146069, Log Avg loss: 0.00827756, Global Avg Loss: 0.00700460, Time: 0.0467 Steps: 85800, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 004720, Sample Num: 75520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00179910, Log Avg loss: 0.00944716, Global Avg Loss: 0.00701028, Time: 0.1073 Steps: 86000, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 004920, Sample Num: 78720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00172600, Log Avg loss: 0.00000066, Global Avg Loss: 0.00699402, Time: 0.1423 Steps: 86200, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 005120, Sample Num: 81920, Cur Loss: 0.00000008, Cur Avg Loss: 0.00165884, Log Avg loss: 0.00000668, Global Avg Loss: 0.00697785, Time: 0.1208 Steps: 86400, Updated lr: 0.000058 Training, Epoch: 0005, Batch: 005320, Sample Num: 85120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00160782, Log Avg loss: 0.00030177, Global Avg Loss: 0.00696243, Time: 0.1204 Steps: 86600, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 005520, Sample Num: 88320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00154957, Log Avg loss: 0.00000004, Global Avg Loss: 0.00694639, Time: 0.1872 Steps: 86800, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 005720, Sample Num: 91520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00155361, Log Avg loss: 0.00166518, Global Avg Loss: 0.00693425, Time: 0.1323 Steps: 87000, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 005920, Sample Num: 94720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00189688, Log Avg loss: 0.01171441, Global Avg Loss: 0.00694521, Time: 0.0448 Steps: 87200, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 006120, Sample Num: 97920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00204700, Log Avg loss: 0.00649045, Global Avg Loss: 0.00694417, Time: 0.0827 Steps: 87400, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 006320, Sample Num: 101120, Cur Loss: 0.00000025, Cur Avg Loss: 0.00198281, Log Avg loss: 0.00001866, Global Avg Loss: 0.00692836, Time: 0.2047 Steps: 87600, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 006520, Sample Num: 104320, Cur Loss: 0.00000004, Cur Avg Loss: 0.00192922, Log Avg loss: 0.00023598, Global Avg Loss: 0.00691311, Time: 0.1533 Steps: 87800, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 006720, Sample Num: 107520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00187190, Log Avg loss: 0.00000324, Global Avg Loss: 0.00689741, Time: 0.1983 Steps: 88000, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 006920, Sample Num: 110720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00181781, Log Avg loss: 0.00000015, Global Avg Loss: 0.00688177, Time: 0.1273 Steps: 88200, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 007120, Sample Num: 113920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00176675, Log Avg loss: 0.00000005, Global Avg Loss: 0.00686620, Time: 0.1247 Steps: 88400, Updated lr: 0.000057 Training, Epoch: 0005, Batch: 007320, Sample Num: 117120, Cur Loss: 0.00000001, Cur Avg Loss: 0.00177322, Log Avg loss: 0.00200389, Global Avg Loss: 0.00685522, Time: 0.0438 Steps: 88600, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 007520, Sample Num: 120320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00187004, Log Avg loss: 0.00541351, Global Avg Loss: 0.00685198, Time: 0.0565 Steps: 88800, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 007720, Sample Num: 123520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00194064, Log Avg loss: 0.00459523, Global Avg Loss: 0.00684690, Time: 0.0572 Steps: 89000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 007920, Sample Num: 126720, Cur Loss: 0.00000006, Cur Avg Loss: 0.00215315, Log Avg loss: 0.01035605, Global Avg Loss: 0.00685477, Time: 0.0594 Steps: 89200, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 008120, Sample Num: 129920, Cur Loss: 0.00000001, Cur Avg Loss: 0.00217975, Log Avg loss: 0.00323311, Global Avg Loss: 0.00684667, Time: 0.1108 Steps: 89400, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 008320, Sample Num: 133120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00213148, Log Avg loss: 0.00017150, Global Avg Loss: 0.00683177, Time: 0.2587 Steps: 89600, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 008520, Sample Num: 136320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00208145, Log Avg loss: 0.00000040, Global Avg Loss: 0.00681656, Time: 0.0670 Steps: 89800, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 008720, Sample Num: 139520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00203382, Log Avg loss: 0.00000493, Global Avg Loss: 0.00680142, Time: 0.1138 Steps: 90000, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 008920, Sample Num: 142720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00198825, Log Avg loss: 0.00000118, Global Avg Loss: 0.00678634, Time: 0.2269 Steps: 90200, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 009120, Sample Num: 145920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00200480, Log Avg loss: 0.00274295, Global Avg Loss: 0.00677739, Time: 0.1378 Steps: 90400, Updated lr: 0.000056 Training, Epoch: 0005, Batch: 009320, Sample Num: 149120, Cur Loss: 0.00000002, Cur Avg Loss: 0.00196178, Log Avg loss: 0.00000013, Global Avg Loss: 0.00676243, Time: 0.1022 Steps: 90600, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 009520, Sample Num: 152320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00203477, Log Avg loss: 0.00543598, Global Avg Loss: 0.00675951, Time: 0.2513 Steps: 90800, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 009720, Sample Num: 155520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00199291, Log Avg loss: 0.00000020, Global Avg Loss: 0.00674466, Time: 0.1425 Steps: 91000, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 009920, Sample Num: 158720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00195274, Log Avg loss: 0.00000080, Global Avg Loss: 0.00672987, Time: 0.1787 Steps: 91200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 010120, Sample Num: 161920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00191416, Log Avg loss: 0.00000045, Global Avg Loss: 0.00671514, Time: 0.1493 Steps: 91400, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 010320, Sample Num: 165120, Cur Loss: 0.00000024, Cur Avg Loss: 0.00187710, Log Avg loss: 0.00000170, Global Avg Loss: 0.00670048, Time: 0.0700 Steps: 91600, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 010520, Sample Num: 168320, Cur Loss: 0.00000007, Cur Avg Loss: 0.00188542, Log Avg loss: 0.00231519, Global Avg Loss: 0.00669093, Time: 0.0647 Steps: 91800, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 010720, Sample Num: 171520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00185029, Log Avg loss: 0.00000247, Global Avg Loss: 0.00667639, Time: 0.1994 Steps: 92000, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 010920, Sample Num: 174720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00181641, Log Avg loss: 0.00000034, Global Avg Loss: 0.00666191, Time: 0.0814 Steps: 92200, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 011120, Sample Num: 177920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00209046, Log Avg loss: 0.01705364, Global Avg Loss: 0.00668440, Time: 0.0556 Steps: 92400, Updated lr: 0.000055 Training, Epoch: 0005, Batch: 011320, Sample Num: 181120, Cur Loss: 0.00000194, Cur Avg Loss: 0.00209293, Log Avg loss: 0.00223017, Global Avg Loss: 0.00667478, Time: 0.0641 Steps: 92600, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 011520, Sample Num: 184320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00211447, Log Avg loss: 0.00333375, Global Avg Loss: 0.00666758, Time: 0.1805 Steps: 92800, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 011720, Sample Num: 187520, Cur Loss: 0.00000001, Cur Avg Loss: 0.00207840, Log Avg loss: 0.00000055, Global Avg Loss: 0.00665324, Time: 0.0745 Steps: 93000, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 011920, Sample Num: 190720, Cur Loss: 0.00000001, Cur Avg Loss: 0.00204355, Log Avg loss: 0.00000112, Global Avg Loss: 0.00663897, Time: 0.0694 Steps: 93200, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 012120, Sample Num: 193920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00200983, Log Avg loss: 0.00000010, Global Avg Loss: 0.00662475, Time: 0.0487 Steps: 93400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 012320, Sample Num: 197120, Cur Loss: 0.00000001, Cur Avg Loss: 0.00197720, Log Avg loss: 0.00000007, Global Avg Loss: 0.00661060, Time: 0.1767 Steps: 93600, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 012520, Sample Num: 200320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00194562, Log Avg loss: 0.00000007, Global Avg Loss: 0.00659650, Time: 0.0442 Steps: 93800, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 012720, Sample Num: 203520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00200148, Log Avg loss: 0.00549834, Global Avg Loss: 0.00659416, Time: 0.1245 Steps: 94000, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 012920, Sample Num: 206720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00197056, Log Avg loss: 0.00000408, Global Avg Loss: 0.00658017, Time: 0.2477 Steps: 94200, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 013120, Sample Num: 209920, Cur Loss: 0.00000001, Cur Avg Loss: 0.00203994, Log Avg loss: 0.00652199, Global Avg Loss: 0.00658005, Time: 0.1963 Steps: 94400, Updated lr: 0.000054 Training, Epoch: 0005, Batch: 013320, Sample Num: 213120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00202714, Log Avg loss: 0.00118740, Global Avg Loss: 0.00656865, Time: 0.0665 Steps: 94600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 013520, Sample Num: 216320, Cur Loss: 0.00000007, Cur Avg Loss: 0.00199715, Log Avg loss: 0.00000012, Global Avg Loss: 0.00655479, Time: 0.1639 Steps: 94800, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 013720, Sample Num: 219520, Cur Loss: 0.00000006, Cur Avg Loss: 0.00196955, Log Avg loss: 0.00010386, Global Avg Loss: 0.00654121, Time: 0.0402 Steps: 95000, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 013920, Sample Num: 222720, Cur Loss: 0.00000002, Cur Avg Loss: 0.00198650, Log Avg loss: 0.00314933, Global Avg Loss: 0.00653408, Time: 0.1161 Steps: 95200, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 014120, Sample Num: 225920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00195838, Log Avg loss: 0.00000090, Global Avg Loss: 0.00652039, Time: 0.1109 Steps: 95400, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 014320, Sample Num: 229120, Cur Loss: 0.00110370, Cur Avg Loss: 0.00197089, Log Avg loss: 0.00285438, Global Avg Loss: 0.00651272, Time: 0.1168 Steps: 95600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 014520, Sample Num: 232320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00194398, Log Avg loss: 0.00001703, Global Avg Loss: 0.00649916, Time: 0.1521 Steps: 95800, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 014720, Sample Num: 235520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00191757, Log Avg loss: 0.00000047, Global Avg Loss: 0.00648562, Time: 0.1907 Steps: 96000, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 014920, Sample Num: 238720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00189191, Log Avg loss: 0.00000323, Global Avg Loss: 0.00647214, Time: 0.0456 Steps: 96200, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 015120, Sample Num: 241920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00186689, Log Avg loss: 0.00000037, Global Avg Loss: 0.00645871, Time: 0.1675 Steps: 96400, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 015320, Sample Num: 245120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00184252, Log Avg loss: 0.00000004, Global Avg Loss: 0.00644534, Time: 0.0381 Steps: 96600, Updated lr: 0.000053 Training, Epoch: 0005, Batch: 015520, Sample Num: 248320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00181878, Log Avg loss: 0.00000003, Global Avg Loss: 0.00643203, Time: 0.0613 Steps: 96800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 015720, Sample Num: 251520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00182384, Log Avg loss: 0.00221695, Global Avg Loss: 0.00642334, Time: 0.0403 Steps: 97000, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 015920, Sample Num: 254720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00180093, Log Avg loss: 0.00000003, Global Avg Loss: 0.00641012, Time: 0.1586 Steps: 97200, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 016120, Sample Num: 257920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00177864, Log Avg loss: 0.00000382, Global Avg Loss: 0.00639696, Time: 0.0520 Steps: 97400, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 016320, Sample Num: 261120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00175684, Log Avg loss: 0.00000004, Global Avg Loss: 0.00638386, Time: 0.2293 Steps: 97600, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 016520, Sample Num: 264320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00173716, Log Avg loss: 0.00013137, Global Avg Loss: 0.00637107, Time: 0.1122 Steps: 97800, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 016720, Sample Num: 267520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00171638, Log Avg loss: 0.00000008, Global Avg Loss: 0.00635807, Time: 0.2767 Steps: 98000, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 016920, Sample Num: 270720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00169610, Log Avg loss: 0.00000029, Global Avg Loss: 0.00634512, Time: 0.1996 Steps: 98200, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 017120, Sample Num: 273920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00178657, Log Avg loss: 0.00944103, Global Avg Loss: 0.00635141, Time: 0.0495 Steps: 98400, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 017320, Sample Num: 277120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00182234, Log Avg loss: 0.00488363, Global Avg Loss: 0.00634843, Time: 0.0874 Steps: 98600, Updated lr: 0.000052 Training, Epoch: 0005, Batch: 017520, Sample Num: 280320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00180155, Log Avg loss: 0.00000098, Global Avg Loss: 0.00633558, Time: 0.1179 Steps: 98800, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 017720, Sample Num: 283520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00178122, Log Avg loss: 0.00000033, Global Avg Loss: 0.00632279, Time: 0.0666 Steps: 99000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 017920, Sample Num: 286720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00176134, Log Avg loss: 0.00000030, Global Avg Loss: 0.00631004, Time: 0.2005 Steps: 99200, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 018120, Sample Num: 289920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00174190, Log Avg loss: 0.00000037, Global Avg Loss: 0.00629734, Time: 0.0975 Steps: 99400, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 018320, Sample Num: 293120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00172289, Log Avg loss: 0.00000012, Global Avg Loss: 0.00628470, Time: 0.1319 Steps: 99600, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 018520, Sample Num: 296320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00176445, Log Avg loss: 0.00557126, Global Avg Loss: 0.00628327, Time: 0.0699 Steps: 99800, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 018720, Sample Num: 299520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00174561, Log Avg loss: 0.00000175, Global Avg Loss: 0.00627071, Time: 0.1024 Steps: 100000, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 018920, Sample Num: 302720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00176388, Log Avg loss: 0.00347386, Global Avg Loss: 0.00626512, Time: 0.1219 Steps: 100200, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 019120, Sample Num: 305920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00174544, Log Avg loss: 0.00000046, Global Avg Loss: 0.00625264, Time: 0.1162 Steps: 100400, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 019320, Sample Num: 309120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00172737, Log Avg loss: 0.00000011, Global Avg Loss: 0.00624021, Time: 0.0680 Steps: 100600, Updated lr: 0.000051 Training, Epoch: 0005, Batch: 019520, Sample Num: 312320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00170969, Log Avg loss: 0.00000221, Global Avg Loss: 0.00622784, Time: 0.0485 Steps: 100800, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 019720, Sample Num: 315520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00169245, Log Avg loss: 0.00000914, Global Avg Loss: 0.00621552, Time: 0.0651 Steps: 101000, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 019920, Sample Num: 318720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00167546, Log Avg loss: 0.00000030, Global Avg Loss: 0.00620324, Time: 0.1687 Steps: 101200, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 020120, Sample Num: 321920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00165880, Log Avg loss: 0.00000001, Global Avg Loss: 0.00619100, Time: 0.0353 Steps: 101400, Updated lr: 0.000050 Training, Epoch: 0005, Batch: 020320, Sample Num: 325113, Cur Loss: 0.00000000, Cur Avg Loss: 0.00173632, Log Avg loss: 0.00953494, Global Avg Loss: 0.00619759, Time: 0.0830 Steps: 101600, Updated lr: 0.000050 ***** Running evaluation checkpoint-101600 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-101600 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2076.685311, Avg time per batch (s): 0.100000 {"eval_avg_loss": 0.019012, "eval_total_loss": 51.523208, "eval_acc": 0.998316, "eval_prec": 0.999078, "eval_recall": 0.99756, "eval_f1": 0.998318, "eval_roc_auc": 0.999831, "eval_pr_auc": 0.999718, "eval_confusion_matrix": {"tn": 21607, "fp": 20, "fn": 53, "tp": 21669}, "eval_mcc2": 0.996633, "eval_mcc": 0.996633, "eval_sn": 0.99756, "eval_sp": 0.999075, "update_flag": false, "test_avg_loss": 0.016071, "test_total_loss": 65.310799, "test_acc": 0.998324, "test_prec": 0.99886, "test_recall": 0.997785, "test_f1": 0.998322, "test_roc_auc": 0.999932, "test_pr_auc": 0.999902, "test_confusion_matrix": {"tn": 32480, "fp": 37, "fn": 72, "tp": 32433}, "test_mcc2": 0.996648, "test_mcc": 0.996648, "test_sn": 0.997785, "test_sp": 0.998862, "lr": 5.0049261083743846e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.006197586635193756, "train_cur_epoch_loss": 35.282119697558635, "train_cur_epoch_avg_loss": 0.0017363247882656808, "train_cur_epoch_time": 2076.685311317444, "train_cur_epoch_avg_time": 0.10219908028137027, "epoch": 5, "step": 101600} ################################################## Training, Epoch: 0006, Batch: 000200, Sample Num: 3200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000105, Log Avg loss: 0.00000105, Global Avg Loss: 0.00618541, Time: 0.1509 Steps: 101800, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000400, Sample Num: 6400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000232, Log Avg loss: 0.00000360, Global Avg Loss: 0.00617329, Time: 0.0467 Steps: 102000, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000600, Sample Num: 9600, Cur Loss: 0.00000003, Cur Avg Loss: 0.00000161, Log Avg loss: 0.00000017, Global Avg Loss: 0.00616121, Time: 0.0404 Steps: 102200, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 000800, Sample Num: 12800, Cur Loss: 0.00000006, Cur Avg Loss: 0.00000121, Log Avg loss: 0.00000001, Global Avg Loss: 0.00614918, Time: 0.1789 Steps: 102400, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 001000, Sample Num: 16000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000097, Log Avg loss: 0.00000001, Global Avg Loss: 0.00613719, Time: 0.0436 Steps: 102600, Updated lr: 0.000050 Training, Epoch: 0006, Batch: 001200, Sample Num: 19200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000081, Log Avg loss: 0.00000003, Global Avg Loss: 0.00612525, Time: 0.1662 Steps: 102800, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001400, Sample Num: 22400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000070, Log Avg loss: 0.00000000, Global Avg Loss: 0.00611336, Time: 0.1201 Steps: 103000, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001600, Sample Num: 25600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002621, Log Avg loss: 0.00020484, Global Avg Loss: 0.00610191, Time: 0.1023 Steps: 103200, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 001800, Sample Num: 28800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002331, Log Avg loss: 0.00000005, Global Avg Loss: 0.00609010, Time: 0.1047 Steps: 103400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 002000, Sample Num: 32000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00002099, Log Avg loss: 0.00000015, Global Avg Loss: 0.00607835, Time: 0.0370 Steps: 103600, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 002200, Sample Num: 35200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001942, Log Avg loss: 0.00000369, Global Avg Loss: 0.00606664, Time: 0.1205 Steps: 103800, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 002400, Sample Num: 38400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001781, Log Avg loss: 0.00000007, Global Avg Loss: 0.00605498, Time: 0.0609 Steps: 104000, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 002600, Sample Num: 41600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001644, Log Avg loss: 0.00000008, Global Avg Loss: 0.00604335, Time: 0.2353 Steps: 104200, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 002800, Sample Num: 44800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00033102, Log Avg loss: 0.00442049, Global Avg Loss: 0.00604025, Time: 0.0437 Steps: 104400, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 003000, Sample Num: 48000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00073854, Log Avg loss: 0.00644381, Global Avg Loss: 0.00604102, Time: 0.0435 Steps: 104600, Updated lr: 0.000049 Training, Epoch: 0006, Batch: 003200, Sample Num: 51200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00069239, Log Avg loss: 0.00000024, Global Avg Loss: 0.00602949, Time: 0.1907 Steps: 104800, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 003400, Sample Num: 54400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00087951, Log Avg loss: 0.00387345, Global Avg Loss: 0.00602538, Time: 0.0501 Steps: 105000, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 003600, Sample Num: 57600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00083066, Log Avg loss: 0.00000015, Global Avg Loss: 0.00601393, Time: 0.0647 Steps: 105200, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 003800, Sample Num: 60800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00097596, Log Avg loss: 0.00359129, Global Avg Loss: 0.00600933, Time: 0.0493 Steps: 105400, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 004000, Sample Num: 64000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00092774, Log Avg loss: 0.00001171, Global Avg Loss: 0.00599797, Time: 0.0922 Steps: 105600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 004200, Sample Num: 67200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088357, Log Avg loss: 0.00000004, Global Avg Loss: 0.00598663, Time: 0.1583 Steps: 105800, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 004400, Sample Num: 70400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108857, Log Avg loss: 0.00539363, Global Avg Loss: 0.00598551, Time: 0.0321 Steps: 106000, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 004600, Sample Num: 73600, Cur Loss: 0.00000018, Cur Avg Loss: 0.00136360, Log Avg loss: 0.00741422, Global Avg Loss: 0.00598820, Time: 0.0690 Steps: 106200, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 004800, Sample Num: 76800, Cur Loss: 0.00000001, Cur Avg Loss: 0.00130680, Log Avg loss: 0.00000046, Global Avg Loss: 0.00597695, Time: 0.2034 Steps: 106400, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 005000, Sample Num: 80000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00125454, Log Avg loss: 0.00000025, Global Avg Loss: 0.00596574, Time: 0.0462 Steps: 106600, Updated lr: 0.000048 Training, Epoch: 0006, Batch: 005200, Sample Num: 83200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00120630, Log Avg loss: 0.00000037, Global Avg Loss: 0.00595457, Time: 0.2327 Steps: 106800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 005400, Sample Num: 86400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00116164, Log Avg loss: 0.00000056, Global Avg Loss: 0.00594344, Time: 0.1229 Steps: 107000, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 005600, Sample Num: 89600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112041, Log Avg loss: 0.00000706, Global Avg Loss: 0.00593236, Time: 0.1095 Steps: 107200, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 005800, Sample Num: 92800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00132720, Log Avg loss: 0.00711727, Global Avg Loss: 0.00593457, Time: 0.1332 Steps: 107400, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 006000, Sample Num: 96000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00157256, Log Avg loss: 0.00868820, Global Avg Loss: 0.00593969, Time: 0.1505 Steps: 107600, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 006200, Sample Num: 99200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00174734, Log Avg loss: 0.00699052, Global Avg Loss: 0.00594164, Time: 0.1629 Steps: 107800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 006400, Sample Num: 102400, Cur Loss: 0.00000007, Cur Avg Loss: 0.00169632, Log Avg loss: 0.00011494, Global Avg Loss: 0.00593085, Time: 0.1232 Steps: 108000, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 006600, Sample Num: 105600, Cur Loss: 0.00000001, Cur Avg Loss: 0.00164492, Log Avg loss: 0.00000004, Global Avg Loss: 0.00591988, Time: 0.1159 Steps: 108200, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 006800, Sample Num: 108800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00159655, Log Avg loss: 0.00000042, Global Avg Loss: 0.00590896, Time: 0.0695 Steps: 108400, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 007000, Sample Num: 112000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00155094, Log Avg loss: 0.00000001, Global Avg Loss: 0.00589808, Time: 0.0550 Steps: 108600, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 007200, Sample Num: 115200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00150786, Log Avg loss: 0.00000001, Global Avg Loss: 0.00588724, Time: 0.0897 Steps: 108800, Updated lr: 0.000047 Training, Epoch: 0006, Batch: 007400, Sample Num: 118400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146716, Log Avg loss: 0.00000219, Global Avg Loss: 0.00587644, Time: 0.2446 Steps: 109000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 007600, Sample Num: 121600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00162931, Log Avg loss: 0.00762888, Global Avg Loss: 0.00587965, Time: 0.1242 Steps: 109200, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 007800, Sample Num: 124800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00174991, Log Avg loss: 0.00633266, Global Avg Loss: 0.00588048, Time: 0.1077 Steps: 109400, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 008000, Sample Num: 128000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00196557, Log Avg loss: 0.01037631, Global Avg Loss: 0.00588868, Time: 0.3492 Steps: 109600, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 008200, Sample Num: 131200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00192280, Log Avg loss: 0.00021169, Global Avg Loss: 0.00587834, Time: 0.1511 Steps: 109800, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 008400, Sample Num: 134400, Cur Loss: 0.00000001, Cur Avg Loss: 0.00187716, Log Avg loss: 0.00000620, Global Avg Loss: 0.00586766, Time: 0.1094 Steps: 110000, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 008600, Sample Num: 137600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00184072, Log Avg loss: 0.00031000, Global Avg Loss: 0.00585758, Time: 0.1522 Steps: 110200, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 008800, Sample Num: 140800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00179888, Log Avg loss: 0.00000003, Global Avg Loss: 0.00584697, Time: 0.1237 Steps: 110400, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 009000, Sample Num: 144000, Cur Loss: 0.00000001, Cur Avg Loss: 0.00175892, Log Avg loss: 0.00000051, Global Avg Loss: 0.00583639, Time: 0.0429 Steps: 110600, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 009200, Sample Num: 147200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00172068, Log Avg loss: 0.00000002, Global Avg Loss: 0.00582586, Time: 0.0628 Steps: 110800, Updated lr: 0.000046 Training, Epoch: 0006, Batch: 009400, Sample Num: 150400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00168407, Log Avg loss: 0.00000001, Global Avg Loss: 0.00581536, Time: 0.1611 Steps: 111000, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 009600, Sample Num: 153600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00175838, Log Avg loss: 0.00525104, Global Avg Loss: 0.00581435, Time: 0.1073 Steps: 111200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 009800, Sample Num: 156800, Cur Loss: 0.00000001, Cur Avg Loss: 0.00172250, Log Avg loss: 0.00000009, Global Avg Loss: 0.00580391, Time: 0.1118 Steps: 111400, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 010000, Sample Num: 160000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00168805, Log Avg loss: 0.00000008, Global Avg Loss: 0.00579351, Time: 0.1115 Steps: 111600, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 010200, Sample Num: 163200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00165496, Log Avg loss: 0.00000058, Global Avg Loss: 0.00578314, Time: 0.1155 Steps: 111800, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 010400, Sample Num: 166400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00162314, Log Avg loss: 0.00000002, Global Avg Loss: 0.00577282, Time: 0.1647 Steps: 112000, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 010600, Sample Num: 169600, Cur Loss: 0.00000191, Cur Avg Loss: 0.00165103, Log Avg loss: 0.00310136, Global Avg Loss: 0.00576805, Time: 0.2004 Steps: 112200, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 010800, Sample Num: 172800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00162046, Log Avg loss: 0.00000023, Global Avg Loss: 0.00575779, Time: 0.1102 Steps: 112400, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 011000, Sample Num: 176000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00159100, Log Avg loss: 0.00000033, Global Avg Loss: 0.00574757, Time: 0.1634 Steps: 112600, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 011200, Sample Num: 179200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00182703, Log Avg loss: 0.01480835, Global Avg Loss: 0.00576363, Time: 0.2602 Steps: 112800, Updated lr: 0.000045 Training, Epoch: 0006, Batch: 011400, Sample Num: 182400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00180168, Log Avg loss: 0.00038241, Global Avg Loss: 0.00575411, Time: 0.0398 Steps: 113000, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 011600, Sample Num: 185600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00183410, Log Avg loss: 0.00368210, Global Avg Loss: 0.00575045, Time: 0.0432 Steps: 113200, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 011800, Sample Num: 188800, Cur Loss: 0.00000001, Cur Avg Loss: 0.00180302, Log Avg loss: 0.00000037, Global Avg Loss: 0.00574030, Time: 0.0388 Steps: 113400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 012000, Sample Num: 192000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00177297, Log Avg loss: 0.00000006, Global Avg Loss: 0.00573020, Time: 0.0819 Steps: 113600, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 012200, Sample Num: 195200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00174391, Log Avg loss: 0.00000006, Global Avg Loss: 0.00572013, Time: 0.0857 Steps: 113800, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 012400, Sample Num: 198400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00171578, Log Avg loss: 0.00000001, Global Avg Loss: 0.00571009, Time: 0.0531 Steps: 114000, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 012600, Sample Num: 201600, Cur Loss: 0.00000001, Cur Avg Loss: 0.00168855, Log Avg loss: 0.00000002, Global Avg Loss: 0.00570009, Time: 0.1138 Steps: 114200, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 012800, Sample Num: 204800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00173248, Log Avg loss: 0.00450031, Global Avg Loss: 0.00569799, Time: 0.0951 Steps: 114400, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 013000, Sample Num: 208000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00170586, Log Avg loss: 0.00000192, Global Avg Loss: 0.00568805, Time: 0.1409 Steps: 114600, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 013200, Sample Num: 211200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00178891, Log Avg loss: 0.00718735, Global Avg Loss: 0.00569067, Time: 0.0757 Steps: 114800, Updated lr: 0.000044 Training, Epoch: 0006, Batch: 013400, Sample Num: 214400, Cur Loss: 0.00000001, Cur Avg Loss: 0.00176221, Log Avg loss: 0.00000030, Global Avg Loss: 0.00568077, Time: 0.1892 Steps: 115000, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 013600, Sample Num: 217600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00173630, Log Avg loss: 0.00000002, Global Avg Loss: 0.00567091, Time: 0.1668 Steps: 115200, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 013800, Sample Num: 220800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00171234, Log Avg loss: 0.00008328, Global Avg Loss: 0.00566122, Time: 0.1220 Steps: 115400, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 014000, Sample Num: 224000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00173050, Log Avg loss: 0.00298301, Global Avg Loss: 0.00565659, Time: 0.1114 Steps: 115600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 014200, Sample Num: 227200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00170613, Log Avg loss: 0.00000029, Global Avg Loss: 0.00564682, Time: 0.0546 Steps: 115800, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 014400, Sample Num: 230400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00168882, Log Avg loss: 0.00045990, Global Avg Loss: 0.00563788, Time: 0.0621 Steps: 116000, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 014600, Sample Num: 233600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00166569, Log Avg loss: 0.00000063, Global Avg Loss: 0.00562817, Time: 0.1606 Steps: 116200, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 014800, Sample Num: 236800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00164319, Log Avg loss: 0.00000083, Global Avg Loss: 0.00561851, Time: 0.0359 Steps: 116400, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 015000, Sample Num: 240000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00162129, Log Avg loss: 0.00000013, Global Avg Loss: 0.00560887, Time: 0.1574 Steps: 116600, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 015200, Sample Num: 243200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00159995, Log Avg loss: 0.00000002, Global Avg Loss: 0.00559926, Time: 0.1599 Steps: 116800, Updated lr: 0.000043 Training, Epoch: 0006, Batch: 015400, Sample Num: 246400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00157918, Log Avg loss: 0.00000005, Global Avg Loss: 0.00558969, Time: 0.2028 Steps: 117000, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 015600, Sample Num: 249600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00155893, Log Avg loss: 0.00000002, Global Avg Loss: 0.00558015, Time: 0.1507 Steps: 117200, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 015800, Sample Num: 252800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00153920, Log Avg loss: 0.00000039, Global Avg Loss: 0.00557065, Time: 0.0353 Steps: 117400, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 016000, Sample Num: 256000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00151997, Log Avg loss: 0.00000071, Global Avg Loss: 0.00556118, Time: 0.1132 Steps: 117600, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 016200, Sample Num: 259200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00150121, Log Avg loss: 0.00000001, Global Avg Loss: 0.00555173, Time: 0.0325 Steps: 117800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 016400, Sample Num: 262400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00148290, Log Avg loss: 0.00000000, Global Avg Loss: 0.00554232, Time: 0.1941 Steps: 118000, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 016600, Sample Num: 265600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146503, Log Avg loss: 0.00000002, Global Avg Loss: 0.00553295, Time: 0.0409 Steps: 118200, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 016800, Sample Num: 268800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00144759, Log Avg loss: 0.00000001, Global Avg Loss: 0.00552360, Time: 0.1355 Steps: 118400, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 017000, Sample Num: 272000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00143056, Log Avg loss: 0.00000003, Global Avg Loss: 0.00551429, Time: 0.0391 Steps: 118600, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 017200, Sample Num: 275200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00156813, Log Avg loss: 0.01326127, Global Avg Loss: 0.00552733, Time: 0.0309 Steps: 118800, Updated lr: 0.000042 Training, Epoch: 0006, Batch: 017400, Sample Num: 278400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00155010, Log Avg loss: 0.00000001, Global Avg Loss: 0.00551804, Time: 0.0472 Steps: 119000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 017600, Sample Num: 281600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00153249, Log Avg loss: 0.00000002, Global Avg Loss: 0.00550878, Time: 0.1225 Steps: 119200, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 017800, Sample Num: 284800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00151527, Log Avg loss: 0.00000005, Global Avg Loss: 0.00549955, Time: 0.2695 Steps: 119400, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 018000, Sample Num: 288000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00149844, Log Avg loss: 0.00000037, Global Avg Loss: 0.00549036, Time: 0.1234 Steps: 119600, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 018200, Sample Num: 291200, Cur Loss: 0.00000006, Cur Avg Loss: 0.00148197, Log Avg loss: 0.00000020, Global Avg Loss: 0.00548119, Time: 0.1220 Steps: 119800, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 018400, Sample Num: 294400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146587, Log Avg loss: 0.00000001, Global Avg Loss: 0.00547206, Time: 0.1181 Steps: 120000, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 018600, Sample Num: 297600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00147151, Log Avg loss: 0.00199064, Global Avg Loss: 0.00546626, Time: 0.0754 Steps: 120200, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 018800, Sample Num: 300800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00147976, Log Avg loss: 0.00224677, Global Avg Loss: 0.00546092, Time: 0.1148 Steps: 120400, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 019000, Sample Num: 304000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146418, Log Avg loss: 0.00000001, Global Avg Loss: 0.00545186, Time: 0.0615 Steps: 120600, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 019200, Sample Num: 307200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00144893, Log Avg loss: 0.00000000, Global Avg Loss: 0.00544283, Time: 0.2074 Steps: 120800, Updated lr: 0.000041 Training, Epoch: 0006, Batch: 019400, Sample Num: 310400, Cur Loss: 0.00000000, Cur Avg Loss: 0.00143399, Log Avg loss: 0.00000001, Global Avg Loss: 0.00543384, Time: 0.0419 Steps: 121000, Updated lr: 0.000040 Training, Epoch: 0006, Batch: 019600, Sample Num: 313600, Cur Loss: 0.00000000, Cur Avg Loss: 0.00141936, Log Avg loss: 0.00000001, Global Avg Loss: 0.00542487, Time: 0.1775 Steps: 121200, Updated lr: 0.000040 Training, Epoch: 0006, Batch: 019800, Sample Num: 316800, Cur Loss: 0.00000000, Cur Avg Loss: 0.00140503, Log Avg loss: 0.00000046, Global Avg Loss: 0.00541593, Time: 0.0977 Steps: 121400, Updated lr: 0.000040 Training, Epoch: 0006, Batch: 020000, Sample Num: 320000, Cur Loss: 0.00000000, Cur Avg Loss: 0.00139098, Log Avg loss: 0.00000003, Global Avg Loss: 0.00540703, Time: 0.0751 Steps: 121600, Updated lr: 0.000040 Training, Epoch: 0006, Batch: 020200, Sample Num: 323200, Cur Loss: 0.00000000, Cur Avg Loss: 0.00143027, Log Avg loss: 0.00535923, Global Avg Loss: 0.00540695, Time: 0.1191 Steps: 121800, Updated lr: 0.000040 ***** Running evaluation checkpoint-121920 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-121920 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2137.873027, Avg time per batch (s): 0.110000 {"eval_avg_loss": 0.021561, "eval_total_loss": 58.43064, "eval_acc": 0.998316, "eval_prec": 0.999078, "eval_recall": 0.99756, "eval_f1": 0.998318, "eval_roc_auc": 0.999793, "eval_pr_auc": 0.999635, "eval_confusion_matrix": {"tn": 21607, "fp": 20, "fn": 53, "tp": 21669}, "eval_mcc2": 0.996633, "eval_mcc": 0.996633, "eval_sn": 0.99756, "eval_sp": 0.999075, "update_flag": false, "test_avg_loss": 0.018024, "test_total_loss": 73.247695, "test_acc": 0.998385, "test_prec": 0.998861, "test_recall": 0.997908, "test_f1": 0.998384, "test_roc_auc": 0.999875, "test_pr_auc": 0.999783, "test_confusion_matrix": {"tn": 32480, "fp": 37, "fn": 68, "tp": 32437}, "test_mcc2": 0.996771, "test_mcc": 0.996771, "test_sn": 0.997908, "test_sp": 0.998862, "lr": 4.003940886699508e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.005403173489345848, "train_cur_epoch_loss": 29.08010968535659, "train_cur_epoch_avg_loss": 0.0014311077601061313, "train_cur_epoch_time": 2137.8730273246765, "train_cur_epoch_avg_time": 0.1052102867777892, "epoch": 6, "step": 121920} ################################################## Training, Epoch: 0007, Batch: 000080, Sample Num: 1280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000003, Log Avg loss: 0.00094374, Global Avg Loss: 0.00539963, Time: 0.2060 Steps: 122000, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000280, Sample Num: 4480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000016, Log Avg loss: 0.00000021, Global Avg Loss: 0.00539079, Time: 0.1359 Steps: 122200, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000480, Sample Num: 7680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000010, Log Avg loss: 0.00000002, Global Avg Loss: 0.00538198, Time: 0.0292 Steps: 122400, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000680, Sample Num: 10880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000054, Log Avg loss: 0.00000161, Global Avg Loss: 0.00537321, Time: 0.0942 Steps: 122600, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 000880, Sample Num: 14080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000042, Log Avg loss: 0.00000000, Global Avg Loss: 0.00536446, Time: 0.1543 Steps: 122800, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 001080, Sample Num: 17280, Cur Loss: 0.00000001, Cur Avg Loss: 0.00000034, Log Avg loss: 0.00000001, Global Avg Loss: 0.00535573, Time: 0.0417 Steps: 123000, Updated lr: 0.000040 Training, Epoch: 0007, Batch: 001280, Sample Num: 20480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000029, Log Avg loss: 0.00000001, Global Avg Loss: 0.00534704, Time: 0.1786 Steps: 123200, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001480, Sample Num: 23680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000028, Log Avg loss: 0.00000020, Global Avg Loss: 0.00533837, Time: 0.1372 Steps: 123400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001680, Sample Num: 26880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000025, Log Avg loss: 0.00000001, Global Avg Loss: 0.00532974, Time: 0.2496 Steps: 123600, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 001880, Sample Num: 30080, Cur Loss: 0.00000001, Cur Avg Loss: 0.00000022, Log Avg loss: 0.00000004, Global Avg Loss: 0.00532113, Time: 0.0985 Steps: 123800, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002080, Sample Num: 33280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000021, Log Avg loss: 0.00000007, Global Avg Loss: 0.00531254, Time: 0.2516 Steps: 124000, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002280, Sample Num: 36480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000019, Log Avg loss: 0.00000000, Global Avg Loss: 0.00530399, Time: 0.0691 Steps: 124200, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002480, Sample Num: 39680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000018, Log Avg loss: 0.00000000, Global Avg Loss: 0.00529546, Time: 0.1204 Steps: 124400, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002680, Sample Num: 42880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00007057, Log Avg loss: 0.00094345, Global Avg Loss: 0.00528848, Time: 0.0979 Steps: 124600, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 002880, Sample Num: 46080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00036179, Log Avg loss: 0.00426411, Global Avg Loss: 0.00528683, Time: 0.0937 Steps: 124800, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 003080, Sample Num: 49280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00056245, Log Avg loss: 0.00345199, Global Avg Loss: 0.00528390, Time: 0.0804 Steps: 125000, Updated lr: 0.000039 Training, Epoch: 0007, Batch: 003280, Sample Num: 52480, Cur Loss: 0.00000006, Cur Avg Loss: 0.00072904, Log Avg loss: 0.00329456, Global Avg Loss: 0.00528072, Time: 0.0906 Steps: 125200, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003480, Sample Num: 55680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00068883, Log Avg loss: 0.00002941, Global Avg Loss: 0.00527234, Time: 0.0800 Steps: 125400, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003680, Sample Num: 58880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00065140, Log Avg loss: 0.00000010, Global Avg Loss: 0.00526395, Time: 0.0373 Steps: 125600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 003880, Sample Num: 62080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00079594, Log Avg loss: 0.00345538, Global Avg Loss: 0.00526107, Time: 0.2135 Steps: 125800, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 004080, Sample Num: 65280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00075713, Log Avg loss: 0.00000429, Global Avg Loss: 0.00525273, Time: 0.0575 Steps: 126000, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 004280, Sample Num: 68480, Cur Loss: 0.00000027, Cur Avg Loss: 0.00072186, Log Avg loss: 0.00000225, Global Avg Loss: 0.00524441, Time: 0.2046 Steps: 126200, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 004480, Sample Num: 71680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107863, Log Avg loss: 0.00871350, Global Avg Loss: 0.00524990, Time: 0.0763 Steps: 126400, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 004680, Sample Num: 74880, Cur Loss: 0.00000002, Cur Avg Loss: 0.00114674, Log Avg loss: 0.00267254, Global Avg Loss: 0.00524583, Time: 0.1011 Steps: 126600, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 004880, Sample Num: 78080, Cur Loss: 0.00000012, Cur Avg Loss: 0.00113465, Log Avg loss: 0.00085172, Global Avg Loss: 0.00523890, Time: 0.1155 Steps: 126800, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 005080, Sample Num: 81280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109004, Log Avg loss: 0.00000154, Global Avg Loss: 0.00523065, Time: 0.0994 Steps: 127000, Updated lr: 0.000038 Training, Epoch: 0007, Batch: 005280, Sample Num: 84480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104875, Log Avg loss: 0.00000004, Global Avg Loss: 0.00522242, Time: 0.1562 Steps: 127200, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 005480, Sample Num: 87680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00101049, Log Avg loss: 0.00000022, Global Avg Loss: 0.00521423, Time: 0.1670 Steps: 127400, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 005680, Sample Num: 90880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00097497, Log Avg loss: 0.00000174, Global Avg Loss: 0.00520606, Time: 0.0475 Steps: 127600, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 005880, Sample Num: 94080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112922, Log Avg loss: 0.00550998, Global Avg Loss: 0.00520653, Time: 0.0335 Steps: 127800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 006080, Sample Num: 97280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00144945, Log Avg loss: 0.01086436, Global Avg Loss: 0.00521537, Time: 0.2362 Steps: 128000, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 006280, Sample Num: 100480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00140333, Log Avg loss: 0.00000113, Global Avg Loss: 0.00520724, Time: 0.0440 Steps: 128200, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 006480, Sample Num: 103680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00136558, Log Avg loss: 0.00018013, Global Avg Loss: 0.00519941, Time: 0.0497 Steps: 128400, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 006680, Sample Num: 106880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00132469, Log Avg loss: 0.00000002, Global Avg Loss: 0.00519132, Time: 0.0879 Steps: 128600, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 006880, Sample Num: 110080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00128618, Log Avg loss: 0.00000001, Global Avg Loss: 0.00518326, Time: 0.1326 Steps: 128800, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 007080, Sample Num: 113280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00124985, Log Avg loss: 0.00000001, Global Avg Loss: 0.00517522, Time: 0.1014 Steps: 129000, Updated lr: 0.000037 Training, Epoch: 0007, Batch: 007280, Sample Num: 116480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00121552, Log Avg loss: 0.00000026, Global Avg Loss: 0.00516721, Time: 0.1916 Steps: 129200, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 007480, Sample Num: 119680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00132624, Log Avg loss: 0.00535644, Global Avg Loss: 0.00516751, Time: 0.0991 Steps: 129400, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 007680, Sample Num: 122880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00132928, Log Avg loss: 0.00144292, Global Avg Loss: 0.00516176, Time: 0.1776 Steps: 129600, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 007880, Sample Num: 126080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00148833, Log Avg loss: 0.00759588, Global Avg Loss: 0.00516551, Time: 0.1020 Steps: 129800, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 008080, Sample Num: 129280, Cur Loss: 0.00000001, Cur Avg Loss: 0.00173307, Log Avg loss: 0.01137599, Global Avg Loss: 0.00517506, Time: 0.2552 Steps: 130000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 008280, Sample Num: 132480, Cur Loss: 0.00000001, Cur Avg Loss: 0.00169163, Log Avg loss: 0.00001733, Global Avg Loss: 0.00516714, Time: 0.1089 Steps: 130200, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 008480, Sample Num: 135680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00165173, Log Avg loss: 0.00000002, Global Avg Loss: 0.00515921, Time: 0.1929 Steps: 130400, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 008680, Sample Num: 138880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00161368, Log Avg loss: 0.00000008, Global Avg Loss: 0.00515131, Time: 0.0809 Steps: 130600, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 008880, Sample Num: 142080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00157734, Log Avg loss: 0.00000021, Global Avg Loss: 0.00514344, Time: 0.0476 Steps: 130800, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 009080, Sample Num: 145280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00154260, Log Avg loss: 0.00000005, Global Avg Loss: 0.00513559, Time: 0.0656 Steps: 131000, Updated lr: 0.000036 Training, Epoch: 0007, Batch: 009280, Sample Num: 148480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00150935, Log Avg loss: 0.00000000, Global Avg Loss: 0.00512776, Time: 0.1070 Steps: 131200, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 009480, Sample Num: 151680, Cur Loss: 0.00000095, Cur Avg Loss: 0.00159632, Log Avg loss: 0.00563187, Global Avg Loss: 0.00512852, Time: 0.0469 Steps: 131400, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 009680, Sample Num: 154880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00156335, Log Avg loss: 0.00000060, Global Avg Loss: 0.00512073, Time: 0.0445 Steps: 131600, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 009880, Sample Num: 158080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00153171, Log Avg loss: 0.00000006, Global Avg Loss: 0.00511296, Time: 0.0504 Steps: 131800, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 010080, Sample Num: 161280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00150132, Log Avg loss: 0.00000001, Global Avg Loss: 0.00510521, Time: 0.0560 Steps: 132000, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 010280, Sample Num: 164480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00147211, Log Avg loss: 0.00000002, Global Avg Loss: 0.00509749, Time: 0.1594 Steps: 132200, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 010480, Sample Num: 167680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00151677, Log Avg loss: 0.00381242, Global Avg Loss: 0.00509555, Time: 0.1697 Steps: 132400, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 010680, Sample Num: 170880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00148839, Log Avg loss: 0.00000110, Global Avg Loss: 0.00508787, Time: 0.2423 Steps: 132600, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 010880, Sample Num: 174080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146103, Log Avg loss: 0.00000002, Global Avg Loss: 0.00508020, Time: 0.1157 Steps: 132800, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 011080, Sample Num: 177280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00158863, Log Avg loss: 0.00853000, Global Avg Loss: 0.00508539, Time: 0.0623 Steps: 133000, Updated lr: 0.000035 Training, Epoch: 0007, Batch: 011280, Sample Num: 180480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00159674, Log Avg loss: 0.00204632, Global Avg Loss: 0.00508083, Time: 0.0399 Steps: 133200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 011480, Sample Num: 183680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00165404, Log Avg loss: 0.00488543, Global Avg Loss: 0.00508053, Time: 0.1139 Steps: 133400, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 011680, Sample Num: 186880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00162571, Log Avg loss: 0.00000003, Global Avg Loss: 0.00507293, Time: 0.2703 Steps: 133600, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 011880, Sample Num: 190080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00159835, Log Avg loss: 0.00000001, Global Avg Loss: 0.00506535, Time: 0.0958 Steps: 133800, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 012080, Sample Num: 193280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00157188, Log Avg loss: 0.00000001, Global Avg Loss: 0.00505779, Time: 0.0487 Steps: 134000, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 012280, Sample Num: 196480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00154628, Log Avg loss: 0.00000004, Global Avg Loss: 0.00505025, Time: 0.0686 Steps: 134200, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 012480, Sample Num: 199680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00152150, Log Avg loss: 0.00000002, Global Avg Loss: 0.00504273, Time: 0.1197 Steps: 134400, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 012680, Sample Num: 202880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00157415, Log Avg loss: 0.00485899, Global Avg Loss: 0.00504246, Time: 0.0742 Steps: 134600, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 012880, Sample Num: 206080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00154972, Log Avg loss: 0.00000085, Global Avg Loss: 0.00503498, Time: 0.4620 Steps: 134800, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 013080, Sample Num: 209280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00163663, Log Avg loss: 0.00723367, Global Avg Loss: 0.00503824, Time: 0.0387 Steps: 135000, Updated lr: 0.000034 Training, Epoch: 0007, Batch: 013280, Sample Num: 212480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00161277, Log Avg loss: 0.00005257, Global Avg Loss: 0.00503086, Time: 0.0819 Steps: 135200, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 013480, Sample Num: 215680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00158884, Log Avg loss: 0.00000002, Global Avg Loss: 0.00502343, Time: 0.2847 Steps: 135400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 013680, Sample Num: 218880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00156561, Log Avg loss: 0.00000001, Global Avg Loss: 0.00501602, Time: 0.1647 Steps: 135600, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 013880, Sample Num: 222080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00158538, Log Avg loss: 0.00293730, Global Avg Loss: 0.00501296, Time: 0.1604 Steps: 135800, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 014080, Sample Num: 225280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00156295, Log Avg loss: 0.00000651, Global Avg Loss: 0.00500560, Time: 0.0848 Steps: 136000, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 014280, Sample Num: 228480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00154106, Log Avg loss: 0.00000017, Global Avg Loss: 0.00499825, Time: 0.1031 Steps: 136200, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 014480, Sample Num: 231680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00151985, Log Avg loss: 0.00000500, Global Avg Loss: 0.00499093, Time: 0.2012 Steps: 136400, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 014680, Sample Num: 234880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00149916, Log Avg loss: 0.00000158, Global Avg Loss: 0.00498362, Time: 0.1128 Steps: 136600, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 014880, Sample Num: 238080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00149935, Log Avg loss: 0.00151296, Global Avg Loss: 0.00497855, Time: 0.1784 Steps: 136800, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 015080, Sample Num: 241280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00147946, Log Avg loss: 0.00000001, Global Avg Loss: 0.00497128, Time: 0.1842 Steps: 137000, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 015280, Sample Num: 244480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00146010, Log Avg loss: 0.00000000, Global Avg Loss: 0.00496403, Time: 0.1028 Steps: 137200, Updated lr: 0.000033 Training, Epoch: 0007, Batch: 015480, Sample Num: 247680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00144123, Log Avg loss: 0.00000007, Global Avg Loss: 0.00495681, Time: 0.3093 Steps: 137400, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 015680, Sample Num: 250880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00142285, Log Avg loss: 0.00000002, Global Avg Loss: 0.00494960, Time: 0.1884 Steps: 137600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 015880, Sample Num: 254080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00140494, Log Avg loss: 0.00000032, Global Avg Loss: 0.00494242, Time: 0.1093 Steps: 137800, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 016080, Sample Num: 257280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00138747, Log Avg loss: 0.00000076, Global Avg Loss: 0.00493526, Time: 0.0989 Steps: 138000, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 016280, Sample Num: 260480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00137043, Log Avg loss: 0.00000000, Global Avg Loss: 0.00492811, Time: 0.1448 Steps: 138200, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 016480, Sample Num: 263680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00135380, Log Avg loss: 0.00000041, Global Avg Loss: 0.00492099, Time: 0.0486 Steps: 138400, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 016680, Sample Num: 266880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00133757, Log Avg loss: 0.00000001, Global Avg Loss: 0.00491389, Time: 0.0749 Steps: 138600, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 016880, Sample Num: 270080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00132172, Log Avg loss: 0.00000000, Global Avg Loss: 0.00490681, Time: 0.1551 Steps: 138800, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 017080, Sample Num: 273280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00137684, Log Avg loss: 0.00602902, Global Avg Loss: 0.00490843, Time: 0.1372 Steps: 139000, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 017280, Sample Num: 276480, Cur Loss: 0.00000006, Cur Avg Loss: 0.00140763, Log Avg loss: 0.00403710, Global Avg Loss: 0.00490717, Time: 0.2582 Steps: 139200, Updated lr: 0.000032 Training, Epoch: 0007, Batch: 017480, Sample Num: 279680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00139152, Log Avg loss: 0.00000001, Global Avg Loss: 0.00490013, Time: 0.1095 Steps: 139400, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 017680, Sample Num: 282880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00137578, Log Avg loss: 0.00000003, Global Avg Loss: 0.00489311, Time: 0.0435 Steps: 139600, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 017880, Sample Num: 286080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00136040, Log Avg loss: 0.00000038, Global Avg Loss: 0.00488611, Time: 0.0604 Steps: 139800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 018080, Sample Num: 289280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00134535, Log Avg loss: 0.00000001, Global Avg Loss: 0.00487913, Time: 0.0428 Steps: 140000, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 018280, Sample Num: 292480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00133063, Log Avg loss: 0.00000001, Global Avg Loss: 0.00487217, Time: 0.1815 Steps: 140200, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 018480, Sample Num: 295680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00136322, Log Avg loss: 0.00434209, Global Avg Loss: 0.00487142, Time: 0.0587 Steps: 140400, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 018680, Sample Num: 298880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00134863, Log Avg loss: 0.00000001, Global Avg Loss: 0.00486449, Time: 0.1592 Steps: 140600, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 018880, Sample Num: 302080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00136163, Log Avg loss: 0.00257631, Global Avg Loss: 0.00486124, Time: 0.1175 Steps: 140800, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 019080, Sample Num: 305280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00134736, Log Avg loss: 0.00000002, Global Avg Loss: 0.00485434, Time: 0.1893 Steps: 141000, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 019280, Sample Num: 308480, Cur Loss: 0.00000000, Cur Avg Loss: 0.00133338, Log Avg loss: 0.00000002, Global Avg Loss: 0.00484747, Time: 0.0741 Steps: 141200, Updated lr: 0.000031 Training, Epoch: 0007, Batch: 019480, Sample Num: 311680, Cur Loss: 0.00000000, Cur Avg Loss: 0.00131969, Log Avg loss: 0.00000001, Global Avg Loss: 0.00484061, Time: 0.1915 Steps: 141400, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 019680, Sample Num: 314880, Cur Loss: 0.00000000, Cur Avg Loss: 0.00130651, Log Avg loss: 0.00002260, Global Avg Loss: 0.00483381, Time: 0.0703 Steps: 141600, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 019880, Sample Num: 318080, Cur Loss: 0.00000000, Cur Avg Loss: 0.00129337, Log Avg loss: 0.00000000, Global Avg Loss: 0.00482699, Time: 0.3323 Steps: 141800, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 020080, Sample Num: 321280, Cur Loss: 0.00000000, Cur Avg Loss: 0.00128048, Log Avg loss: 0.00000001, Global Avg Loss: 0.00482019, Time: 0.0727 Steps: 142000, Updated lr: 0.000030 Training, Epoch: 0007, Batch: 020280, Sample Num: 324480, Cur Loss: 0.00000010, Cur Avg Loss: 0.00131708, Log Avg loss: 0.00499103, Global Avg Loss: 0.00482043, Time: 0.0724 Steps: 142200, Updated lr: 0.000030 ***** Running evaluation checkpoint-142240 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-142240 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2294.954785, Avg time per batch (s): 0.110000 {"eval_avg_loss": 0.020628, "eval_total_loss": 55.902307, "eval_acc": 0.998316, "eval_prec": 0.998848, "eval_recall": 0.99779, "eval_f1": 0.998319, "eval_roc_auc": 0.999794, "eval_pr_auc": 0.999636, "eval_confusion_matrix": {"tn": 21602, "fp": 25, "fn": 48, "tp": 21674}, "eval_mcc2": 0.996633, "eval_mcc": 0.996633, "eval_sn": 0.99779, "eval_sp": 0.998844, "update_flag": false, "test_avg_loss": 0.016871, "test_total_loss": 68.565408, "test_acc": 0.998447, "test_prec": 0.998738, "test_recall": 0.998154, "test_f1": 0.998446, "test_roc_auc": 0.99986, "test_pr_auc": 0.999753, "test_confusion_matrix": {"tn": 32476, "fp": 41, "fn": 60, "tp": 32445}, "test_mcc2": 0.996894, "test_mcc": 0.996894, "test_sn": 0.998154, "test_sp": 0.998739, "lr": 3.0029556650246303e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.0048190751513062305, "train_cur_epoch_loss": 26.71033770074925, "train_cur_epoch_avg_loss": 0.0013144851230683686, "train_cur_epoch_time": 2294.954785346985, "train_cur_epoch_avg_time": 0.112940688255265, "epoch": 7, "step": 142240} ################################################## Training, Epoch: 0008, Batch: 000160, Sample Num: 2560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000002, Global Avg Loss: 0.00481366, Time: 0.1280 Steps: 142400, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000360, Sample Num: 5760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000002, Log Avg loss: 0.00000002, Global Avg Loss: 0.00480691, Time: 0.0410 Steps: 142600, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000560, Sample Num: 8960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000001, Global Avg Loss: 0.00480018, Time: 0.0365 Steps: 142800, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000760, Sample Num: 12160, Cur Loss: 0.00000011, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000000, Global Avg Loss: 0.00479346, Time: 0.1219 Steps: 143000, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 000960, Sample Num: 15360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000000, Global Avg Loss: 0.00478677, Time: 0.2800 Steps: 143200, Updated lr: 0.000030 Training, Epoch: 0008, Batch: 001160, Sample Num: 18560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000001, Global Avg Loss: 0.00478009, Time: 0.1560 Steps: 143400, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001360, Sample Num: 21760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000002, Global Avg Loss: 0.00477343, Time: 0.2018 Steps: 143600, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001560, Sample Num: 24960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000005, Log Avg loss: 0.00000029, Global Avg Loss: 0.00476680, Time: 0.1509 Steps: 143800, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001760, Sample Num: 28160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000004, Log Avg loss: 0.00000000, Global Avg Loss: 0.00476018, Time: 0.0742 Steps: 144000, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 001960, Sample Num: 31360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000004, Log Avg loss: 0.00000001, Global Avg Loss: 0.00475357, Time: 0.0747 Steps: 144200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 002160, Sample Num: 34560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000513, Log Avg loss: 0.00005504, Global Avg Loss: 0.00474707, Time: 0.0737 Steps: 144400, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 002360, Sample Num: 37760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000470, Log Avg loss: 0.00000002, Global Avg Loss: 0.00474050, Time: 0.0613 Steps: 144600, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 002560, Sample Num: 40960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000433, Log Avg loss: 0.00000001, Global Avg Loss: 0.00473395, Time: 0.0472 Steps: 144800, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 002760, Sample Num: 44160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00001190, Log Avg loss: 0.00010883, Global Avg Loss: 0.00472757, Time: 0.0457 Steps: 145000, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 002960, Sample Num: 47360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00034457, Log Avg loss: 0.00493533, Global Avg Loss: 0.00472786, Time: 0.1174 Steps: 145200, Updated lr: 0.000029 Training, Epoch: 0008, Batch: 003160, Sample Num: 50560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00032276, Log Avg loss: 0.00000001, Global Avg Loss: 0.00472136, Time: 0.0459 Steps: 145400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 003360, Sample Num: 53760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00052528, Log Avg loss: 0.00372507, Global Avg Loss: 0.00471999, Time: 0.1706 Steps: 145600, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 003560, Sample Num: 56960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00049587, Log Avg loss: 0.00000172, Global Avg Loss: 0.00471352, Time: 0.1667 Steps: 145800, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 003760, Sample Num: 60160, Cur Loss: 0.47540909, Cur Avg Loss: 0.00059594, Log Avg loss: 0.00237719, Global Avg Loss: 0.00471031, Time: 0.1126 Steps: 146000, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 003960, Sample Num: 63360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00056603, Log Avg loss: 0.00000382, Global Avg Loss: 0.00470388, Time: 0.0736 Steps: 146200, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 004160, Sample Num: 66560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00053915, Log Avg loss: 0.00000698, Global Avg Loss: 0.00469746, Time: 0.0922 Steps: 146400, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 004360, Sample Num: 69760, Cur Loss: 0.00000001, Cur Avg Loss: 0.00051461, Log Avg loss: 0.00000407, Global Avg Loss: 0.00469106, Time: 0.1188 Steps: 146600, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 004560, Sample Num: 72960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00083341, Log Avg loss: 0.00778320, Global Avg Loss: 0.00469527, Time: 0.0462 Steps: 146800, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 004760, Sample Num: 76160, Cur Loss: 0.00000012, Cur Avg Loss: 0.00095580, Log Avg loss: 0.00374642, Global Avg Loss: 0.00469398, Time: 0.0926 Steps: 147000, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 004960, Sample Num: 79360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00091731, Log Avg loss: 0.00000128, Global Avg Loss: 0.00468760, Time: 0.0706 Steps: 147200, Updated lr: 0.000028 Training, Epoch: 0008, Batch: 005160, Sample Num: 82560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088183, Log Avg loss: 0.00000176, Global Avg Loss: 0.00468124, Time: 0.0566 Steps: 147400, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 005360, Sample Num: 85760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00084893, Log Avg loss: 0.00000010, Global Avg Loss: 0.00467490, Time: 0.1227 Steps: 147600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 005560, Sample Num: 88960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00081846, Log Avg loss: 0.00000185, Global Avg Loss: 0.00466858, Time: 0.1522 Steps: 147800, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 005760, Sample Num: 92160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00079017, Log Avg loss: 0.00000398, Global Avg Loss: 0.00466227, Time: 0.2731 Steps: 148000, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 005960, Sample Num: 95360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108321, Log Avg loss: 0.00952277, Global Avg Loss: 0.00466883, Time: 0.0823 Steps: 148200, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 006160, Sample Num: 98560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00127065, Log Avg loss: 0.00685628, Global Avg Loss: 0.00467178, Time: 0.0903 Steps: 148400, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 006360, Sample Num: 101760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00123079, Log Avg loss: 0.00000301, Global Avg Loss: 0.00466550, Time: 0.0645 Steps: 148600, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 006560, Sample Num: 104960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00119328, Log Avg loss: 0.00000056, Global Avg Loss: 0.00465923, Time: 0.0487 Steps: 148800, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 006760, Sample Num: 108160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00115799, Log Avg loss: 0.00000032, Global Avg Loss: 0.00465297, Time: 0.0553 Steps: 149000, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 006960, Sample Num: 111360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112471, Log Avg loss: 0.00000006, Global Avg Loss: 0.00464674, Time: 0.1198 Steps: 149200, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 007160, Sample Num: 114560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109330, Log Avg loss: 0.00000001, Global Avg Loss: 0.00464052, Time: 0.2290 Steps: 149400, Updated lr: 0.000027 Training, Epoch: 0008, Batch: 007360, Sample Num: 117760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00106360, Log Avg loss: 0.00000033, Global Avg Loss: 0.00463431, Time: 0.0532 Steps: 149600, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 007560, Sample Num: 120960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00117375, Log Avg loss: 0.00522732, Global Avg Loss: 0.00463511, Time: 0.1219 Steps: 149800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 007760, Sample Num: 124160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00114352, Log Avg loss: 0.00000085, Global Avg Loss: 0.00462893, Time: 0.1781 Steps: 150000, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 007960, Sample Num: 127360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00138714, Log Avg loss: 0.01083960, Global Avg Loss: 0.00463720, Time: 0.1092 Steps: 150200, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 008160, Sample Num: 130560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00135314, Log Avg loss: 0.00000002, Global Avg Loss: 0.00463103, Time: 0.0303 Steps: 150400, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 008360, Sample Num: 133760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00132079, Log Avg loss: 0.00000074, Global Avg Loss: 0.00462488, Time: 0.1000 Steps: 150600, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 008560, Sample Num: 136960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00128993, Log Avg loss: 0.00000001, Global Avg Loss: 0.00461875, Time: 0.0950 Steps: 150800, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 008760, Sample Num: 140160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00126153, Log Avg loss: 0.00004600, Global Avg Loss: 0.00461269, Time: 0.0874 Steps: 151000, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 008960, Sample Num: 143360, Cur Loss: 0.00000007, Cur Avg Loss: 0.00123337, Log Avg loss: 0.00000002, Global Avg Loss: 0.00460659, Time: 0.1147 Steps: 151200, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 009160, Sample Num: 146560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00120644, Log Avg loss: 0.00000001, Global Avg Loss: 0.00460050, Time: 0.1380 Steps: 151400, Updated lr: 0.000026 Training, Epoch: 0008, Batch: 009360, Sample Num: 149760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00118066, Log Avg loss: 0.00000001, Global Avg Loss: 0.00459443, Time: 0.1834 Steps: 151600, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 009560, Sample Num: 152960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00127372, Log Avg loss: 0.00562874, Global Avg Loss: 0.00459580, Time: 0.0551 Steps: 151800, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 009760, Sample Num: 156160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00124762, Log Avg loss: 0.00000002, Global Avg Loss: 0.00458975, Time: 0.1128 Steps: 152000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 009960, Sample Num: 159360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00122256, Log Avg loss: 0.00000000, Global Avg Loss: 0.00458372, Time: 0.0664 Steps: 152200, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 010160, Sample Num: 162560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00119850, Log Avg loss: 0.00000003, Global Avg Loss: 0.00457770, Time: 0.0667 Steps: 152400, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 010360, Sample Num: 165760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00117536, Log Avg loss: 0.00000013, Global Avg Loss: 0.00457170, Time: 0.0981 Steps: 152600, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 010560, Sample Num: 168960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00117002, Log Avg loss: 0.00089303, Global Avg Loss: 0.00456689, Time: 0.2120 Steps: 152800, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 010760, Sample Num: 172160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00114827, Log Avg loss: 0.00000001, Global Avg Loss: 0.00456092, Time: 0.1270 Steps: 153000, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 010960, Sample Num: 175360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112735, Log Avg loss: 0.00000211, Global Avg Loss: 0.00455497, Time: 0.0351 Steps: 153200, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 011160, Sample Num: 178560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00125594, Log Avg loss: 0.00830239, Global Avg Loss: 0.00455985, Time: 0.1440 Steps: 153400, Updated lr: 0.000025 Training, Epoch: 0008, Batch: 011360, Sample Num: 181760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00123392, Log Avg loss: 0.00000505, Global Avg Loss: 0.00455392, Time: 0.0897 Steps: 153600, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 011560, Sample Num: 184960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00128441, Log Avg loss: 0.00415252, Global Avg Loss: 0.00455340, Time: 0.1043 Steps: 153800, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 011760, Sample Num: 188160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00126257, Log Avg loss: 0.00000012, Global Avg Loss: 0.00454749, Time: 0.2540 Steps: 154000, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 011960, Sample Num: 191360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00124146, Log Avg loss: 0.00000004, Global Avg Loss: 0.00454159, Time: 0.0464 Steps: 154200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 012160, Sample Num: 194560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00122104, Log Avg loss: 0.00000002, Global Avg Loss: 0.00453571, Time: 0.1904 Steps: 154400, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 012360, Sample Num: 197760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00120128, Log Avg loss: 0.00000002, Global Avg Loss: 0.00452984, Time: 0.1126 Steps: 154600, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 012560, Sample Num: 200960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00118215, Log Avg loss: 0.00000001, Global Avg Loss: 0.00452399, Time: 0.0767 Steps: 154800, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 012760, Sample Num: 204160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00123079, Log Avg loss: 0.00428534, Global Avg Loss: 0.00452368, Time: 0.0703 Steps: 155000, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 012960, Sample Num: 207360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00121180, Log Avg loss: 0.00000001, Global Avg Loss: 0.00451785, Time: 0.2255 Steps: 155200, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 013160, Sample Num: 210560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00122896, Log Avg loss: 0.00234068, Global Avg Loss: 0.00451505, Time: 0.1310 Steps: 155400, Updated lr: 0.000024 Training, Epoch: 0008, Batch: 013360, Sample Num: 213760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00121314, Log Avg loss: 0.00017221, Global Avg Loss: 0.00450946, Time: 0.1760 Steps: 155600, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 013560, Sample Num: 216960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00119524, Log Avg loss: 0.00000001, Global Avg Loss: 0.00450368, Time: 0.0837 Steps: 155800, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 013760, Sample Num: 220160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00117787, Log Avg loss: 0.00000004, Global Avg Loss: 0.00449790, Time: 0.0867 Steps: 156000, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 013960, Sample Num: 223360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00120474, Log Avg loss: 0.00305307, Global Avg Loss: 0.00449605, Time: 0.1188 Steps: 156200, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 014160, Sample Num: 226560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00118772, Log Avg loss: 0.00000001, Global Avg Loss: 0.00449030, Time: 0.0440 Steps: 156400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 014360, Sample Num: 229760, Cur Loss: 0.00000006, Cur Avg Loss: 0.00117118, Log Avg loss: 0.00000021, Global Avg Loss: 0.00448457, Time: 0.1550 Steps: 156600, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 014560, Sample Num: 232960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00115509, Log Avg loss: 0.00000003, Global Avg Loss: 0.00447885, Time: 0.1551 Steps: 156800, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 014760, Sample Num: 236160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00113944, Log Avg loss: 0.00000019, Global Avg Loss: 0.00447314, Time: 0.0708 Steps: 157000, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 014960, Sample Num: 239360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112929, Log Avg loss: 0.00038006, Global Avg Loss: 0.00446794, Time: 0.0490 Steps: 157200, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 015160, Sample Num: 242560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00111439, Log Avg loss: 0.00000001, Global Avg Loss: 0.00446226, Time: 0.0758 Steps: 157400, Updated lr: 0.000023 Training, Epoch: 0008, Batch: 015360, Sample Num: 245760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109988, Log Avg loss: 0.00000003, Global Avg Loss: 0.00445660, Time: 0.1117 Steps: 157600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 015560, Sample Num: 248960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108575, Log Avg loss: 0.00000000, Global Avg Loss: 0.00445095, Time: 0.1008 Steps: 157800, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 015760, Sample Num: 252160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107197, Log Avg loss: 0.00000007, Global Avg Loss: 0.00444531, Time: 0.1546 Steps: 158000, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 015960, Sample Num: 255360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00105854, Log Avg loss: 0.00000010, Global Avg Loss: 0.00443969, Time: 0.1553 Steps: 158200, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 016160, Sample Num: 258560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104544, Log Avg loss: 0.00000007, Global Avg Loss: 0.00443409, Time: 0.1287 Steps: 158400, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 016360, Sample Num: 261760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00103266, Log Avg loss: 0.00000001, Global Avg Loss: 0.00442850, Time: 0.0763 Steps: 158600, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 016560, Sample Num: 264960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00102165, Log Avg loss: 0.00012113, Global Avg Loss: 0.00442307, Time: 0.1286 Steps: 158800, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 016760, Sample Num: 268160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00100946, Log Avg loss: 0.00000001, Global Avg Loss: 0.00441751, Time: 0.1030 Steps: 159000, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 016960, Sample Num: 271360, Cur Loss: 0.00000001, Cur Avg Loss: 0.00099755, Log Avg loss: 0.00000001, Global Avg Loss: 0.00441196, Time: 0.0919 Steps: 159200, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 017160, Sample Num: 274560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108944, Log Avg loss: 0.00888143, Global Avg Loss: 0.00441757, Time: 0.0338 Steps: 159400, Updated lr: 0.000022 Training, Epoch: 0008, Batch: 017360, Sample Num: 277760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107689, Log Avg loss: 0.00000041, Global Avg Loss: 0.00441203, Time: 0.1602 Steps: 159600, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 017560, Sample Num: 280960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00106463, Log Avg loss: 0.00000000, Global Avg Loss: 0.00440651, Time: 0.0726 Steps: 159800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 017760, Sample Num: 284160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00105264, Log Avg loss: 0.00000007, Global Avg Loss: 0.00440100, Time: 0.0415 Steps: 160000, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 017960, Sample Num: 287360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104092, Log Avg loss: 0.00000051, Global Avg Loss: 0.00439551, Time: 0.0594 Steps: 160200, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 018160, Sample Num: 290560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00102946, Log Avg loss: 0.00000000, Global Avg Loss: 0.00439003, Time: 0.0724 Steps: 160400, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 018360, Sample Num: 293760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00101825, Log Avg loss: 0.00000000, Global Avg Loss: 0.00438456, Time: 0.0503 Steps: 160600, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 018560, Sample Num: 296960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00101732, Log Avg loss: 0.00093243, Global Avg Loss: 0.00438027, Time: 0.1518 Steps: 160800, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 018760, Sample Num: 300160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00100648, Log Avg loss: 0.00000015, Global Avg Loss: 0.00437482, Time: 0.0485 Steps: 161000, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 018960, Sample Num: 303360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00100419, Log Avg loss: 0.00078999, Global Avg Loss: 0.00437038, Time: 0.0644 Steps: 161200, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 019160, Sample Num: 306560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00099371, Log Avg loss: 0.00000007, Global Avg Loss: 0.00436496, Time: 0.0862 Steps: 161400, Updated lr: 0.000021 Training, Epoch: 0008, Batch: 019360, Sample Num: 309760, Cur Loss: 0.00000000, Cur Avg Loss: 0.00098345, Log Avg loss: 0.00000000, Global Avg Loss: 0.00435956, Time: 0.1509 Steps: 161600, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 019560, Sample Num: 312960, Cur Loss: 0.00000000, Cur Avg Loss: 0.00097339, Log Avg loss: 0.00000001, Global Avg Loss: 0.00435417, Time: 0.0472 Steps: 161800, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 019760, Sample Num: 316160, Cur Loss: 0.00000000, Cur Avg Loss: 0.00096361, Log Avg loss: 0.00000751, Global Avg Loss: 0.00434880, Time: 0.0621 Steps: 162000, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 019960, Sample Num: 319360, Cur Loss: 0.00000000, Cur Avg Loss: 0.00095396, Log Avg loss: 0.00000001, Global Avg Loss: 0.00434344, Time: 0.0655 Steps: 162200, Updated lr: 0.000020 Training, Epoch: 0008, Batch: 020160, Sample Num: 322560, Cur Loss: 0.00000000, Cur Avg Loss: 0.00094450, Log Avg loss: 0.00000000, Global Avg Loss: 0.00433809, Time: 0.0515 Steps: 162400, Updated lr: 0.000020 ***** Running evaluation checkpoint-162560 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-162560 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2307.001555, Avg time per batch (s): 0.110000 {"eval_avg_loss": 0.021658, "eval_total_loss": 58.692253, "eval_acc": 0.998224, "eval_prec": 0.998848, "eval_recall": 0.997606, "eval_f1": 0.998227, "eval_roc_auc": 0.999771, "eval_pr_auc": 0.99959, "eval_confusion_matrix": {"tn": 21602, "fp": 25, "fn": 52, "tp": 21670}, "eval_mcc2": 0.996448, "eval_mcc": 0.996448, "eval_sn": 0.997606, "eval_sp": 0.998844, "update_flag": false, "test_avg_loss": 0.017325, "test_total_loss": 70.4097, "test_acc": 0.998462, "test_prec": 0.998799, "test_recall": 0.998123, "test_f1": 0.998461, "test_roc_auc": 0.999859, "test_pr_auc": 0.999753, "test_confusion_matrix": {"tn": 32478, "fp": 39, "fn": 61, "tp": 32444}, "test_mcc2": 0.996924, "test_mcc": 0.996924, "test_sn": 0.998123, "test_sp": 0.998801, "lr": 2.001970443349754e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.004340016390168304, "train_cur_epoch_loss": 20.04781486395934, "train_cur_epoch_avg_loss": 0.0009866050622027234, "train_cur_epoch_time": 2307.001554965973, "train_cur_epoch_avg_time": 0.11353354109084512, "epoch": 8, "step": 162560} ################################################## Training, Epoch: 0009, Batch: 000040, Sample Num: 640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00503395, Global Avg Loss: 0.00433895, Time: 0.0706 Steps: 162600, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000240, Sample Num: 3840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000008, Log Avg loss: 0.00000009, Global Avg Loss: 0.00433362, Time: 0.0391 Steps: 162800, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000440, Sample Num: 7040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000022, Log Avg loss: 0.00000039, Global Avg Loss: 0.00432830, Time: 0.0423 Steps: 163000, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000640, Sample Num: 10240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000015, Log Avg loss: 0.00000001, Global Avg Loss: 0.00432300, Time: 0.0461 Steps: 163200, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 000840, Sample Num: 13440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000012, Log Avg loss: 0.00000000, Global Avg Loss: 0.00431771, Time: 0.0849 Steps: 163400, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 001040, Sample Num: 16640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000010, Log Avg loss: 0.00000001, Global Avg Loss: 0.00431243, Time: 0.0651 Steps: 163600, Updated lr: 0.000020 Training, Epoch: 0009, Batch: 001240, Sample Num: 19840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000008, Log Avg loss: 0.00000001, Global Avg Loss: 0.00430716, Time: 0.0481 Steps: 163800, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001440, Sample Num: 23040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000007, Log Avg loss: 0.00000000, Global Avg Loss: 0.00430191, Time: 0.0437 Steps: 164000, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001640, Sample Num: 26240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000006, Log Avg loss: 0.00000001, Global Avg Loss: 0.00429667, Time: 0.0596 Steps: 164200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 001840, Sample Num: 29440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000006, Log Avg loss: 0.00000000, Global Avg Loss: 0.00429144, Time: 0.0434 Steps: 164400, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 002040, Sample Num: 32640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000072, Log Avg loss: 0.00000685, Global Avg Loss: 0.00428624, Time: 0.1238 Steps: 164600, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 002240, Sample Num: 35840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000069, Log Avg loss: 0.00000034, Global Avg Loss: 0.00428104, Time: 0.0873 Steps: 164800, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 002440, Sample Num: 39040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000064, Log Avg loss: 0.00000010, Global Avg Loss: 0.00427585, Time: 0.0875 Steps: 165000, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 002640, Sample Num: 42240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000064, Log Avg loss: 0.00000070, Global Avg Loss: 0.00427067, Time: 0.0439 Steps: 165200, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 002840, Sample Num: 45440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00031050, Log Avg loss: 0.00440057, Global Avg Loss: 0.00427083, Time: 0.0315 Steps: 165400, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 003040, Sample Num: 48640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00029050, Log Avg loss: 0.00000645, Global Avg Loss: 0.00426568, Time: 0.1182 Steps: 165600, Updated lr: 0.000019 Training, Epoch: 0009, Batch: 003240, Sample Num: 51840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00048441, Log Avg loss: 0.00343184, Global Avg Loss: 0.00426467, Time: 0.2073 Steps: 165800, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 003440, Sample Num: 55040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00045627, Log Avg loss: 0.00000049, Global Avg Loss: 0.00425953, Time: 0.1345 Steps: 166000, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 003640, Sample Num: 58240, Cur Loss: 0.00000006, Cur Avg Loss: 0.00043120, Log Avg loss: 0.00000002, Global Avg Loss: 0.00425441, Time: 0.0645 Steps: 166200, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 003840, Sample Num: 61440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00052010, Log Avg loss: 0.00213801, Global Avg Loss: 0.00425186, Time: 0.0770 Steps: 166400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 004040, Sample Num: 64640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00049440, Log Avg loss: 0.00000103, Global Avg Loss: 0.00424676, Time: 0.0821 Steps: 166600, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 004240, Sample Num: 67840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00047110, Log Avg loss: 0.00000043, Global Avg Loss: 0.00424167, Time: 0.0520 Steps: 166800, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 004440, Sample Num: 71040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00078780, Log Avg loss: 0.00750175, Global Avg Loss: 0.00424557, Time: 0.0284 Steps: 167000, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 004640, Sample Num: 74240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088212, Log Avg loss: 0.00297598, Global Avg Loss: 0.00424406, Time: 0.0959 Steps: 167200, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 004840, Sample Num: 77440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00084571, Log Avg loss: 0.00000120, Global Avg Loss: 0.00423899, Time: 0.1091 Steps: 167400, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 005040, Sample Num: 80640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00081227, Log Avg loss: 0.00000289, Global Avg Loss: 0.00423393, Time: 0.0661 Steps: 167600, Updated lr: 0.000018 Training, Epoch: 0009, Batch: 005240, Sample Num: 83840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00078153, Log Avg loss: 0.00000690, Global Avg Loss: 0.00422889, Time: 0.0497 Steps: 167800, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 005440, Sample Num: 87040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00075280, Log Avg loss: 0.00000012, Global Avg Loss: 0.00422386, Time: 0.2468 Steps: 168000, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 005640, Sample Num: 90240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00072621, Log Avg loss: 0.00000282, Global Avg Loss: 0.00421884, Time: 0.0369 Steps: 168200, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 005840, Sample Num: 93440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00083552, Log Avg loss: 0.00391831, Global Avg Loss: 0.00421848, Time: 0.1179 Steps: 168400, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 006040, Sample Num: 96640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00103058, Log Avg loss: 0.00672625, Global Avg Loss: 0.00422146, Time: 0.1123 Steps: 168600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 006240, Sample Num: 99840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00110691, Log Avg loss: 0.00341197, Global Avg Loss: 0.00422050, Time: 0.0330 Steps: 168800, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 006440, Sample Num: 103040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107271, Log Avg loss: 0.00000578, Global Avg Loss: 0.00421551, Time: 0.0433 Steps: 169000, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 006640, Sample Num: 106240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104040, Log Avg loss: 0.00000006, Global Avg Loss: 0.00421053, Time: 0.1130 Steps: 169200, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 006840, Sample Num: 109440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00100999, Log Avg loss: 0.00000038, Global Avg Loss: 0.00420556, Time: 0.1133 Steps: 169400, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 007040, Sample Num: 112640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00098130, Log Avg loss: 0.00000002, Global Avg Loss: 0.00420060, Time: 0.2671 Steps: 169600, Updated lr: 0.000017 Training, Epoch: 0009, Batch: 007240, Sample Num: 115840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00095419, Log Avg loss: 0.00000006, Global Avg Loss: 0.00419565, Time: 0.1645 Steps: 169800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 007440, Sample Num: 119040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00101149, Log Avg loss: 0.00308557, Global Avg Loss: 0.00419434, Time: 0.0756 Steps: 170000, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 007640, Sample Num: 122240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00098501, Log Avg loss: 0.00000004, Global Avg Loss: 0.00418942, Time: 0.1364 Steps: 170200, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 007840, Sample Num: 125440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00105953, Log Avg loss: 0.00390620, Global Avg Loss: 0.00418908, Time: 0.0405 Steps: 170400, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 008040, Sample Num: 128640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00124344, Log Avg loss: 0.00845251, Global Avg Loss: 0.00419408, Time: 0.0299 Steps: 170600, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 008240, Sample Num: 131840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00121326, Log Avg loss: 0.00000029, Global Avg Loss: 0.00418917, Time: 0.0589 Steps: 170800, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 008440, Sample Num: 135040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00118451, Log Avg loss: 0.00000000, Global Avg Loss: 0.00418427, Time: 0.1187 Steps: 171000, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 008640, Sample Num: 138240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00115709, Log Avg loss: 0.00000000, Global Avg Loss: 0.00417938, Time: 0.1095 Steps: 171200, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 008840, Sample Num: 141440, Cur Loss: 0.00000001, Cur Avg Loss: 0.00113092, Log Avg loss: 0.00000000, Global Avg Loss: 0.00417451, Time: 0.0523 Steps: 171400, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 009040, Sample Num: 144640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00110590, Log Avg loss: 0.00000001, Global Avg Loss: 0.00416964, Time: 0.1144 Steps: 171600, Updated lr: 0.000016 Training, Epoch: 0009, Batch: 009240, Sample Num: 147840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108196, Log Avg loss: 0.00000001, Global Avg Loss: 0.00416479, Time: 0.1085 Steps: 171800, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 009440, Sample Num: 151040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00116789, Log Avg loss: 0.00513789, Global Avg Loss: 0.00416592, Time: 0.1160 Steps: 172000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 009640, Sample Num: 154240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00114373, Log Avg loss: 0.00000361, Global Avg Loss: 0.00416108, Time: 0.1746 Steps: 172200, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 009840, Sample Num: 157440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112049, Log Avg loss: 0.00000001, Global Avg Loss: 0.00415626, Time: 0.1339 Steps: 172400, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 010040, Sample Num: 160640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109817, Log Avg loss: 0.00000007, Global Avg Loss: 0.00415144, Time: 0.1953 Steps: 172600, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 010240, Sample Num: 163840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107672, Log Avg loss: 0.00000002, Global Avg Loss: 0.00414664, Time: 0.0862 Steps: 172800, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 010440, Sample Num: 167040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00105609, Log Avg loss: 0.00000001, Global Avg Loss: 0.00414184, Time: 0.1193 Steps: 173000, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 010640, Sample Num: 170240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00105028, Log Avg loss: 0.00074669, Global Avg Loss: 0.00413792, Time: 0.0585 Steps: 173200, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 010840, Sample Num: 173440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00103091, Log Avg loss: 0.00000040, Global Avg Loss: 0.00413315, Time: 0.1143 Steps: 173400, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 011040, Sample Num: 176640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00115490, Log Avg loss: 0.00787550, Global Avg Loss: 0.00413746, Time: 0.1125 Steps: 173600, Updated lr: 0.000015 Training, Epoch: 0009, Batch: 011240, Sample Num: 179840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00113436, Log Avg loss: 0.00000019, Global Avg Loss: 0.00413270, Time: 0.0693 Steps: 173800, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 011440, Sample Num: 183040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00111497, Log Avg loss: 0.00002566, Global Avg Loss: 0.00412798, Time: 0.0841 Steps: 174000, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 011640, Sample Num: 186240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00115840, Log Avg loss: 0.00364238, Global Avg Loss: 0.00412742, Time: 0.1480 Steps: 174200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 011840, Sample Num: 189440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00113883, Log Avg loss: 0.00000002, Global Avg Loss: 0.00412269, Time: 0.2115 Steps: 174400, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 012040, Sample Num: 192640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00111992, Log Avg loss: 0.00000000, Global Avg Loss: 0.00411797, Time: 0.0581 Steps: 174600, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 012240, Sample Num: 195840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00110162, Log Avg loss: 0.00000000, Global Avg Loss: 0.00411325, Time: 0.2375 Steps: 174800, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 012440, Sample Num: 199040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108391, Log Avg loss: 0.00000008, Global Avg Loss: 0.00410855, Time: 0.2483 Steps: 175000, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 012640, Sample Num: 202240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00106676, Log Avg loss: 0.00000000, Global Avg Loss: 0.00410386, Time: 0.0595 Steps: 175200, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 012840, Sample Num: 205440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109043, Log Avg loss: 0.00258650, Global Avg Loss: 0.00410213, Time: 0.1742 Steps: 175400, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 013040, Sample Num: 208640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107371, Log Avg loss: 0.00000012, Global Avg Loss: 0.00409746, Time: 0.2043 Steps: 175600, Updated lr: 0.000014 Training, Epoch: 0009, Batch: 013240, Sample Num: 211840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109933, Log Avg loss: 0.00276975, Global Avg Loss: 0.00409595, Time: 0.2665 Steps: 175800, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 013440, Sample Num: 215040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108297, Log Avg loss: 0.00000006, Global Avg Loss: 0.00409130, Time: 0.1726 Steps: 176000, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 013640, Sample Num: 218240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00106709, Log Avg loss: 0.00000002, Global Avg Loss: 0.00408665, Time: 0.1110 Steps: 176200, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 013840, Sample Num: 221440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109242, Log Avg loss: 0.00281993, Global Avg Loss: 0.00408522, Time: 0.1551 Steps: 176400, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 014040, Sample Num: 224640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107686, Log Avg loss: 0.00000019, Global Avg Loss: 0.00408059, Time: 0.1270 Steps: 176600, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 014240, Sample Num: 227840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00106174, Log Avg loss: 0.00000003, Global Avg Loss: 0.00407597, Time: 0.1343 Steps: 176800, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 014440, Sample Num: 231040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104703, Log Avg loss: 0.00000007, Global Avg Loss: 0.00407137, Time: 0.0649 Steps: 177000, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 014640, Sample Num: 234240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00103273, Log Avg loss: 0.00000022, Global Avg Loss: 0.00406677, Time: 0.1837 Steps: 177200, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 014840, Sample Num: 237440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00101915, Log Avg loss: 0.00002520, Global Avg Loss: 0.00406222, Time: 0.0339 Steps: 177400, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 015040, Sample Num: 240640, Cur Loss: 0.00000001, Cur Avg Loss: 0.00100560, Log Avg loss: 0.00000001, Global Avg Loss: 0.00405764, Time: 0.3211 Steps: 177600, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 015240, Sample Num: 243840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00099240, Log Avg loss: 0.00000000, Global Avg Loss: 0.00405308, Time: 0.0463 Steps: 177800, Updated lr: 0.000013 Training, Epoch: 0009, Batch: 015440, Sample Num: 247040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00097955, Log Avg loss: 0.00000000, Global Avg Loss: 0.00404852, Time: 0.2386 Steps: 178000, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 015640, Sample Num: 250240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00096702, Log Avg loss: 0.00000000, Global Avg Loss: 0.00404398, Time: 0.1035 Steps: 178200, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 015840, Sample Num: 253440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00095481, Log Avg loss: 0.00000000, Global Avg Loss: 0.00403945, Time: 0.0902 Steps: 178400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 016040, Sample Num: 256640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00094291, Log Avg loss: 0.00000011, Global Avg Loss: 0.00403492, Time: 0.1698 Steps: 178600, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 016240, Sample Num: 259840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00093130, Log Avg loss: 0.00000000, Global Avg Loss: 0.00403041, Time: 0.0781 Steps: 178800, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 016440, Sample Num: 263040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00091997, Log Avg loss: 0.00000000, Global Avg Loss: 0.00402591, Time: 0.1156 Steps: 179000, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 016640, Sample Num: 266240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00090899, Log Avg loss: 0.00000631, Global Avg Loss: 0.00402142, Time: 0.0754 Steps: 179200, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 016840, Sample Num: 269440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00089819, Log Avg loss: 0.00000001, Global Avg Loss: 0.00401694, Time: 0.0405 Steps: 179400, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 017040, Sample Num: 272640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088765, Log Avg loss: 0.00000039, Global Avg Loss: 0.00401246, Time: 0.0640 Steps: 179600, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 017240, Sample Num: 275840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00097539, Log Avg loss: 0.00845030, Global Avg Loss: 0.00401740, Time: 0.2302 Steps: 179800, Updated lr: 0.000012 Training, Epoch: 0009, Batch: 017440, Sample Num: 279040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00096420, Log Avg loss: 0.00000001, Global Avg Loss: 0.00401294, Time: 0.2339 Steps: 180000, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 017640, Sample Num: 282240, Cur Loss: 0.00001693, Cur Avg Loss: 0.00095327, Log Avg loss: 0.00000009, Global Avg Loss: 0.00400848, Time: 0.1111 Steps: 180200, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 017840, Sample Num: 285440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00094258, Log Avg loss: 0.00000000, Global Avg Loss: 0.00400404, Time: 0.1519 Steps: 180400, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 018040, Sample Num: 288640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00093213, Log Avg loss: 0.00000006, Global Avg Loss: 0.00399961, Time: 0.0496 Steps: 180600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 018240, Sample Num: 291840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00092191, Log Avg loss: 0.00000000, Global Avg Loss: 0.00399518, Time: 0.1236 Steps: 180800, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 018440, Sample Num: 295040, Cur Loss: 0.00000001, Cur Avg Loss: 0.00091196, Log Avg loss: 0.00000430, Global Avg Loss: 0.00399077, Time: 0.0672 Steps: 181000, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 018640, Sample Num: 298240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00090218, Log Avg loss: 0.00000004, Global Avg Loss: 0.00398637, Time: 0.0364 Steps: 181200, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 018840, Sample Num: 301440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00089652, Log Avg loss: 0.00036955, Global Avg Loss: 0.00398238, Time: 0.2423 Steps: 181400, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 019040, Sample Num: 304640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088711, Log Avg loss: 0.00000014, Global Avg Loss: 0.00397799, Time: 0.0488 Steps: 181600, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 019240, Sample Num: 307840, Cur Loss: 0.00000000, Cur Avg Loss: 0.00087788, Log Avg loss: 0.00000003, Global Avg Loss: 0.00397362, Time: 0.0683 Steps: 181800, Updated lr: 0.000011 Training, Epoch: 0009, Batch: 019440, Sample Num: 311040, Cur Loss: 0.00000000, Cur Avg Loss: 0.00086885, Log Avg loss: 0.00000000, Global Avg Loss: 0.00396925, Time: 0.1658 Steps: 182000, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 019640, Sample Num: 314240, Cur Loss: 0.00000000, Cur Avg Loss: 0.00086005, Log Avg loss: 0.00000487, Global Avg Loss: 0.00396490, Time: 0.0474 Steps: 182200, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 019840, Sample Num: 317440, Cur Loss: 0.00000000, Cur Avg Loss: 0.00085138, Log Avg loss: 0.00000000, Global Avg Loss: 0.00396055, Time: 0.1231 Steps: 182400, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 020040, Sample Num: 320640, Cur Loss: 0.00000000, Cur Avg Loss: 0.00084289, Log Avg loss: 0.00000017, Global Avg Loss: 0.00395621, Time: 0.1134 Steps: 182600, Updated lr: 0.000010 Training, Epoch: 0009, Batch: 020240, Sample Num: 323840, Cur Loss: 0.00000001, Cur Avg Loss: 0.00086704, Log Avg loss: 0.00328731, Global Avg Loss: 0.00395548, Time: 0.0567 Steps: 182800, Updated lr: 0.000010 ***** Running evaluation checkpoint-182880 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-182880 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2119.307521, Avg time per batch (s): 0.100000 {"eval_avg_loss": 0.023041, "eval_total_loss": 62.441366, "eval_acc": 0.998224, "eval_prec": 0.998848, "eval_recall": 0.997606, "eval_f1": 0.998227, "eval_roc_auc": 0.999769, "eval_pr_auc": 0.999589, "eval_confusion_matrix": {"tn": 21602, "fp": 25, "fn": 52, "tp": 21670}, "eval_mcc2": 0.996448, "eval_mcc": 0.996448, "eval_sn": 0.997606, "eval_sp": 0.998844, "update_flag": false, "test_avg_loss": 0.018446, "test_total_loss": 74.963701, "test_acc": 0.998462, "test_prec": 0.99883, "test_recall": 0.998093, "test_f1": 0.998461, "test_roc_auc": 0.999812, "test_pr_auc": 0.99966, "test_confusion_matrix": {"tn": 32479, "fp": 38, "fn": 62, "tp": 32443}, "test_mcc2": 0.996924, "test_mcc": 0.996924, "test_sn": 0.998093, "test_sp": 0.998831, "lr": 1.000985221674877e-05, "cur_epoch_step": 20320, "train_global_avg_loss": 0.003953757300143678, "train_cur_epoch_loss": 17.550070664515033, "train_cur_epoch_avg_loss": 0.0008636845799466059, "train_cur_epoch_time": 2119.307520866394, "train_cur_epoch_avg_time": 0.10429662996389734, "epoch": 9, "step": 182880} ################################################## Training, Epoch: 0010, Batch: 000120, Sample Num: 1920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000552, Global Avg Loss: 0.00395116, Time: 0.1062 Steps: 183000, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000320, Sample Num: 5120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000005, Log Avg loss: 0.00000008, Global Avg Loss: 0.00394685, Time: 0.1209 Steps: 183200, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000520, Sample Num: 8320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000003, Log Avg loss: 0.00000000, Global Avg Loss: 0.00394255, Time: 0.1674 Steps: 183400, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000720, Sample Num: 11520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000003, Log Avg loss: 0.00000001, Global Avg Loss: 0.00393825, Time: 0.0523 Steps: 183600, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 000920, Sample Num: 14720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000002, Log Avg loss: 0.00000000, Global Avg Loss: 0.00393397, Time: 0.1010 Steps: 183800, Updated lr: 0.000010 Training, Epoch: 0010, Batch: 001120, Sample Num: 17920, Cur Loss: 0.00000232, Cur Avg Loss: 0.00000002, Log Avg loss: 0.00000001, Global Avg Loss: 0.00392969, Time: 0.0629 Steps: 184000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001320, Sample Num: 21120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000002, Log Avg loss: 0.00000000, Global Avg Loss: 0.00392542, Time: 0.2036 Steps: 184200, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001520, Sample Num: 24320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000002, Log Avg loss: 0.00000001, Global Avg Loss: 0.00392117, Time: 0.1838 Steps: 184400, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001720, Sample Num: 27520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000000, Global Avg Loss: 0.00391692, Time: 0.1541 Steps: 184600, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 001920, Sample Num: 30720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000001, Log Avg loss: 0.00000001, Global Avg Loss: 0.00391268, Time: 0.0939 Steps: 184800, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 002120, Sample Num: 33920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000003, Log Avg loss: 0.00000019, Global Avg Loss: 0.00390845, Time: 0.0596 Steps: 185000, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 002320, Sample Num: 37120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000003, Log Avg loss: 0.00000001, Global Avg Loss: 0.00390423, Time: 0.1309 Steps: 185200, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 002520, Sample Num: 40320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000004, Log Avg loss: 0.00000015, Global Avg Loss: 0.00390002, Time: 0.1298 Steps: 185400, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 002720, Sample Num: 43520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00000005, Log Avg loss: 0.00000017, Global Avg Loss: 0.00389582, Time: 0.1825 Steps: 185600, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 002920, Sample Num: 46720, Cur Loss: 0.00000001, Cur Avg Loss: 0.00031161, Log Avg loss: 0.00454884, Global Avg Loss: 0.00389652, Time: 0.0964 Steps: 185800, Updated lr: 0.000009 Training, Epoch: 0010, Batch: 003120, Sample Num: 49920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00029164, Log Avg loss: 0.00000003, Global Avg Loss: 0.00389233, Time: 0.1531 Steps: 186000, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 003320, Sample Num: 53120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00046056, Log Avg loss: 0.00309577, Global Avg Loss: 0.00389147, Time: 0.1537 Steps: 186200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 003520, Sample Num: 56320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00043439, Log Avg loss: 0.00000002, Global Avg Loss: 0.00388730, Time: 0.0823 Steps: 186400, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 003720, Sample Num: 59520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00041104, Log Avg loss: 0.00000000, Global Avg Loss: 0.00388313, Time: 0.0469 Steps: 186600, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 003920, Sample Num: 62720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00052267, Log Avg loss: 0.00259892, Global Avg Loss: 0.00388176, Time: 0.0444 Steps: 186800, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 004120, Sample Num: 65920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00049729, Log Avg loss: 0.00000003, Global Avg Loss: 0.00387760, Time: 0.2341 Steps: 187000, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 004320, Sample Num: 69120, Cur Loss: 0.00000001, Cur Avg Loss: 0.00047427, Log Avg loss: 0.00000005, Global Avg Loss: 0.00387346, Time: 0.0752 Steps: 187200, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 004520, Sample Num: 72320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00076815, Log Avg loss: 0.00711589, Global Avg Loss: 0.00387692, Time: 0.0787 Steps: 187400, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 004720, Sample Num: 75520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00074681, Log Avg loss: 0.00026458, Global Avg Loss: 0.00387307, Time: 0.1243 Steps: 187600, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 004920, Sample Num: 78720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00071646, Log Avg loss: 0.00000024, Global Avg Loss: 0.00386895, Time: 0.0880 Steps: 187800, Updated lr: 0.000008 Training, Epoch: 0010, Batch: 005120, Sample Num: 81920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00068849, Log Avg loss: 0.00000028, Global Avg Loss: 0.00386483, Time: 0.2482 Steps: 188000, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 005320, Sample Num: 85120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00066261, Log Avg loss: 0.00000006, Global Avg Loss: 0.00386072, Time: 0.0489 Steps: 188200, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 005520, Sample Num: 88320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00063860, Log Avg loss: 0.00000001, Global Avg Loss: 0.00385663, Time: 0.3456 Steps: 188400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 005720, Sample Num: 91520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00062365, Log Avg loss: 0.00021098, Global Avg Loss: 0.00385276, Time: 0.1179 Steps: 188600, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 005920, Sample Num: 94720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00089583, Log Avg loss: 0.00868036, Global Avg Loss: 0.00385787, Time: 0.0383 Steps: 188800, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 006120, Sample Num: 97920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00105643, Log Avg loss: 0.00580998, Global Avg Loss: 0.00385994, Time: 0.1107 Steps: 189000, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 006320, Sample Num: 101120, Cur Loss: 0.00000006, Cur Avg Loss: 0.00102300, Log Avg loss: 0.00000011, Global Avg Loss: 0.00385586, Time: 0.1056 Steps: 189200, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 006520, Sample Num: 104320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00099163, Log Avg loss: 0.00000048, Global Avg Loss: 0.00385179, Time: 0.0965 Steps: 189400, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 006720, Sample Num: 107520, Cur Loss: 0.00000001, Cur Avg Loss: 0.00096212, Log Avg loss: 0.00000000, Global Avg Loss: 0.00384772, Time: 0.3842 Steps: 189600, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 006920, Sample Num: 110720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00093432, Log Avg loss: 0.00000013, Global Avg Loss: 0.00384367, Time: 0.0377 Steps: 189800, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 007120, Sample Num: 113920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00090807, Log Avg loss: 0.00000000, Global Avg Loss: 0.00383962, Time: 0.1337 Steps: 190000, Updated lr: 0.000007 Training, Epoch: 0010, Batch: 007320, Sample Num: 117120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088327, Log Avg loss: 0.00000034, Global Avg Loss: 0.00383559, Time: 0.0684 Steps: 190200, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 007520, Sample Num: 120320, Cur Loss: 0.00000850, Cur Avg Loss: 0.00094876, Log Avg loss: 0.00334552, Global Avg Loss: 0.00383507, Time: 0.0413 Steps: 190400, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 007720, Sample Num: 123520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00092418, Log Avg loss: 0.00000026, Global Avg Loss: 0.00383105, Time: 0.0386 Steps: 190600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 007920, Sample Num: 126720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00124288, Log Avg loss: 0.01354449, Global Avg Loss: 0.00384123, Time: 0.1523 Steps: 190800, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 008120, Sample Num: 129920, Cur Loss: 0.00000018, Cur Avg Loss: 0.00121227, Log Avg loss: 0.00000028, Global Avg Loss: 0.00383721, Time: 0.0292 Steps: 191000, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 008320, Sample Num: 133120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00118313, Log Avg loss: 0.00000004, Global Avg Loss: 0.00383319, Time: 0.1387 Steps: 191200, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 008520, Sample Num: 136320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00115536, Log Avg loss: 0.00000004, Global Avg Loss: 0.00382919, Time: 0.0433 Steps: 191400, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 008720, Sample Num: 139520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00112886, Log Avg loss: 0.00000000, Global Avg Loss: 0.00382519, Time: 0.2545 Steps: 191600, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 008920, Sample Num: 142720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00110355, Log Avg loss: 0.00000000, Global Avg Loss: 0.00382120, Time: 0.0574 Steps: 191800, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 009120, Sample Num: 145920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107935, Log Avg loss: 0.00000000, Global Avg Loss: 0.00381722, Time: 0.0314 Steps: 192000, Updated lr: 0.000006 Training, Epoch: 0010, Batch: 009320, Sample Num: 149120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00105619, Log Avg loss: 0.00000000, Global Avg Loss: 0.00381325, Time: 0.2302 Steps: 192200, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 009520, Sample Num: 152320, Cur Loss: 0.00003141, Cur Avg Loss: 0.00115548, Log Avg loss: 0.00578252, Global Avg Loss: 0.00381530, Time: 0.0832 Steps: 192400, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 009720, Sample Num: 155520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00113171, Log Avg loss: 0.00000000, Global Avg Loss: 0.00381134, Time: 0.0638 Steps: 192600, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 009920, Sample Num: 158720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00110889, Log Avg loss: 0.00000000, Global Avg Loss: 0.00380738, Time: 0.1597 Steps: 192800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 010120, Sample Num: 161920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00108698, Log Avg loss: 0.00000017, Global Avg Loss: 0.00380344, Time: 0.1458 Steps: 193000, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 010320, Sample Num: 165120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00106591, Log Avg loss: 0.00000003, Global Avg Loss: 0.00379950, Time: 0.1125 Steps: 193200, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 010520, Sample Num: 168320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104578, Log Avg loss: 0.00000715, Global Avg Loss: 0.00379558, Time: 0.1191 Steps: 193400, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 010720, Sample Num: 171520, Cur Loss: 0.00000006, Cur Avg Loss: 0.00102628, Log Avg loss: 0.00000012, Global Avg Loss: 0.00379166, Time: 0.0634 Steps: 193600, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 010920, Sample Num: 174720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00100748, Log Avg loss: 0.00000005, Global Avg Loss: 0.00378774, Time: 0.1262 Steps: 193800, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 011120, Sample Num: 177920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109290, Log Avg loss: 0.00575677, Global Avg Loss: 0.00378977, Time: 0.0344 Steps: 194000, Updated lr: 0.000005 Training, Epoch: 0010, Batch: 011320, Sample Num: 181120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107360, Log Avg loss: 0.00000034, Global Avg Loss: 0.00378587, Time: 0.0796 Steps: 194200, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 011520, Sample Num: 184320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00111579, Log Avg loss: 0.00350369, Global Avg Loss: 0.00378558, Time: 0.1809 Steps: 194400, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 011720, Sample Num: 187520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00109676, Log Avg loss: 0.00000076, Global Avg Loss: 0.00378169, Time: 0.1777 Steps: 194600, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 011920, Sample Num: 190720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00107836, Log Avg loss: 0.00000012, Global Avg Loss: 0.00377781, Time: 0.1820 Steps: 194800, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 012120, Sample Num: 193920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00106056, Log Avg loss: 0.00000000, Global Avg Loss: 0.00377393, Time: 0.1559 Steps: 195000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 012320, Sample Num: 197120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00104335, Log Avg loss: 0.00000001, Global Avg Loss: 0.00377007, Time: 0.0841 Steps: 195200, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 012520, Sample Num: 200320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00102668, Log Avg loss: 0.00000000, Global Avg Loss: 0.00376621, Time: 0.2499 Steps: 195400, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 012720, Sample Num: 203520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00103177, Log Avg loss: 0.00135025, Global Avg Loss: 0.00376374, Time: 0.1353 Steps: 195600, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 012920, Sample Num: 206720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00101580, Log Avg loss: 0.00000000, Global Avg Loss: 0.00375989, Time: 0.1092 Steps: 195800, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 013120, Sample Num: 209920, Cur Loss: 0.00000012, Cur Avg Loss: 0.00102498, Log Avg loss: 0.00161861, Global Avg Loss: 0.00375771, Time: 0.2108 Steps: 196000, Updated lr: 0.000004 Training, Epoch: 0010, Batch: 013320, Sample Num: 213120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00100959, Log Avg loss: 0.00000001, Global Avg Loss: 0.00375388, Time: 0.1070 Steps: 196200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 013520, Sample Num: 216320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00099466, Log Avg loss: 0.00000004, Global Avg Loss: 0.00375006, Time: 0.0457 Steps: 196400, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 013720, Sample Num: 219520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00098016, Log Avg loss: 0.00000001, Global Avg Loss: 0.00374624, Time: 0.1197 Steps: 196600, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 013920, Sample Num: 222720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00100218, Log Avg loss: 0.00251279, Global Avg Loss: 0.00374499, Time: 0.0599 Steps: 196800, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 014120, Sample Num: 225920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00098799, Log Avg loss: 0.00000000, Global Avg Loss: 0.00374119, Time: 0.0555 Steps: 197000, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 014320, Sample Num: 229120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00097419, Log Avg loss: 0.00000003, Global Avg Loss: 0.00373739, Time: 0.0653 Steps: 197200, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 014520, Sample Num: 232320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00096077, Log Avg loss: 0.00000003, Global Avg Loss: 0.00373360, Time: 0.0290 Steps: 197400, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 014720, Sample Num: 235520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00094772, Log Avg loss: 0.00000004, Global Avg Loss: 0.00372983, Time: 0.3001 Steps: 197600, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 014920, Sample Num: 238720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00093504, Log Avg loss: 0.00000195, Global Avg Loss: 0.00372606, Time: 0.0885 Steps: 197800, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 015120, Sample Num: 241920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00092267, Log Avg loss: 0.00000000, Global Avg Loss: 0.00372229, Time: 0.0569 Steps: 198000, Updated lr: 0.000003 Training, Epoch: 0010, Batch: 015320, Sample Num: 245120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00091063, Log Avg loss: 0.00000000, Global Avg Loss: 0.00371854, Time: 0.0574 Steps: 198200, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 015520, Sample Num: 248320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00089889, Log Avg loss: 0.00000009, Global Avg Loss: 0.00371479, Time: 0.1771 Steps: 198400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 015720, Sample Num: 251520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088746, Log Avg loss: 0.00000000, Global Avg Loss: 0.00371105, Time: 0.1618 Steps: 198600, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 015920, Sample Num: 254720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00087631, Log Avg loss: 0.00000015, Global Avg Loss: 0.00370731, Time: 0.0353 Steps: 198800, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 016120, Sample Num: 257920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00086544, Log Avg loss: 0.00000037, Global Avg Loss: 0.00370359, Time: 0.0448 Steps: 199000, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 016320, Sample Num: 261120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00085484, Log Avg loss: 0.00000001, Global Avg Loss: 0.00369987, Time: 0.0645 Steps: 199200, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 016520, Sample Num: 264320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00084452, Log Avg loss: 0.00000321, Global Avg Loss: 0.00369616, Time: 0.0707 Steps: 199400, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 016720, Sample Num: 267520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00083442, Log Avg loss: 0.00000003, Global Avg Loss: 0.00369246, Time: 0.0628 Steps: 199600, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 016920, Sample Num: 270720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00082456, Log Avg loss: 0.00000002, Global Avg Loss: 0.00368876, Time: 0.0574 Steps: 199800, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 017120, Sample Num: 273920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00089145, Log Avg loss: 0.00655002, Global Avg Loss: 0.00369162, Time: 0.0951 Steps: 200000, Updated lr: 0.000002 Training, Epoch: 0010, Batch: 017320, Sample Num: 277120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00088115, Log Avg loss: 0.00000013, Global Avg Loss: 0.00368794, Time: 0.1580 Steps: 200200, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 017520, Sample Num: 280320, Cur Loss: 0.00000001, Cur Avg Loss: 0.00087110, Log Avg loss: 0.00000000, Global Avg Loss: 0.00368426, Time: 0.1844 Steps: 200400, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 017720, Sample Num: 283520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00086126, Log Avg loss: 0.00000002, Global Avg Loss: 0.00368058, Time: 0.0447 Steps: 200600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 017920, Sample Num: 286720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00085165, Log Avg loss: 0.00000021, Global Avg Loss: 0.00367692, Time: 0.2545 Steps: 200800, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 018120, Sample Num: 289920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00084225, Log Avg loss: 0.00000002, Global Avg Loss: 0.00367326, Time: 0.0614 Steps: 201000, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 018320, Sample Num: 293120, Cur Loss: 0.00000001, Cur Avg Loss: 0.00083306, Log Avg loss: 0.00000000, Global Avg Loss: 0.00366961, Time: 0.1439 Steps: 201200, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 018520, Sample Num: 296320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00082407, Log Avg loss: 0.00000067, Global Avg Loss: 0.00366596, Time: 0.2454 Steps: 201400, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 018720, Sample Num: 299520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00081527, Log Avg loss: 0.00000003, Global Avg Loss: 0.00366233, Time: 0.0721 Steps: 201600, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 018920, Sample Num: 302720, Cur Loss: 0.00000001, Cur Avg Loss: 0.00080668, Log Avg loss: 0.00000331, Global Avg Loss: 0.00365870, Time: 0.1138 Steps: 201800, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 019120, Sample Num: 305920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00079824, Log Avg loss: 0.00000000, Global Avg Loss: 0.00365508, Time: 0.0366 Steps: 202000, Updated lr: 0.000001 Training, Epoch: 0010, Batch: 019320, Sample Num: 309120, Cur Loss: 0.00000000, Cur Avg Loss: 0.00078998, Log Avg loss: 0.00000000, Global Avg Loss: 0.00365146, Time: 0.0622 Steps: 202200, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 019520, Sample Num: 312320, Cur Loss: 0.00000000, Cur Avg Loss: 0.00078189, Log Avg loss: 0.00000000, Global Avg Loss: 0.00364785, Time: 0.2131 Steps: 202400, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 019720, Sample Num: 315520, Cur Loss: 0.00000000, Cur Avg Loss: 0.00077415, Log Avg loss: 0.00001851, Global Avg Loss: 0.00364427, Time: 0.1107 Steps: 202600, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 019920, Sample Num: 318720, Cur Loss: 0.00000000, Cur Avg Loss: 0.00076637, Log Avg loss: 0.00000000, Global Avg Loss: 0.00364068, Time: 0.2302 Steps: 202800, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 020120, Sample Num: 321920, Cur Loss: 0.00000000, Cur Avg Loss: 0.00075876, Log Avg loss: 0.00000003, Global Avg Loss: 0.00363709, Time: 0.0463 Steps: 203000, Updated lr: 0.000000 Training, Epoch: 0010, Batch: 020320, Sample Num: 325113, Cur Loss: 0.00000000, Cur Avg Loss: 0.00077614, Log Avg loss: 0.00252513, Global Avg Loss: 0.00363600, Time: 0.0361 Steps: 203200, Updated lr: 0.000000 ***** Running evaluation checkpoint-203200 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## ***** Running testing checkpoint-203200 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## Epoch Time: 2044.537080, Avg time per batch (s): 0.100000 {"eval_avg_loss": 0.023115, "eval_total_loss": 62.641887, "eval_acc": 0.998293, "eval_prec": 0.998848, "eval_recall": 0.997744, "eval_f1": 0.998296, "eval_roc_auc": 0.999724, "eval_pr_auc": 0.999497, "eval_confusion_matrix": {"tn": 21602, "fp": 25, "fn": 49, "tp": 21673}, "eval_mcc2": 0.996586, "eval_mcc": 0.996586, "eval_sn": 0.997744, "eval_sp": 0.998844, "update_flag": false, "test_avg_loss": 0.018668, "test_total_loss": 75.865633, "test_acc": 0.998462, "test_prec": 0.998707, "test_recall": 0.998216, "test_f1": 0.998461, "test_roc_auc": 0.999782, "test_pr_auc": 0.999598, "test_confusion_matrix": {"tn": 32475, "fp": 42, "fn": 58, "tp": 32447}, "test_mcc2": 0.996924, "test_mcc": 0.996924, "test_sn": 0.998216, "test_sp": 0.998708, "lr": 0.0, "cur_epoch_step": 20320, "train_global_avg_loss": 0.0036359956394206657, "train_cur_epoch_loss": 15.771178880001683, "train_cur_epoch_avg_loss": 0.0007761406929134686, "train_cur_epoch_time": 2044.5370798110962, "train_cur_epoch_avg_time": 0.10061698227416813, "epoch": 10, "step": 203200} ################################################## #########################Best Metric######################### {"epoch": 4, "global_step": 81280, "eval_avg_loss": 0.015381, "eval_total_loss": 41.68272, "eval_acc": 0.998408, "eval_prec": 0.998986, "eval_recall": 0.997836, "eval_f1": 0.998411, "eval_roc_auc": 0.999853, "eval_pr_auc": 0.999757, "eval_confusion_matrix": {"tn": 21605, "fp": 22, "fn": 47, "tp": 21675}, "eval_mcc2": 0.996817, "eval_mcc": 0.996817, "eval_sn": 0.997836, "eval_sp": 0.998983, "update_flag": true, "test_avg_loss": 0.013435, "test_total_loss": 54.601641, "test_acc": 0.998324, "test_prec": 0.99886, "test_recall": 0.997785, "test_f1": 0.998322, "test_roc_auc": 0.999945, "test_pr_auc": 0.99993, "test_confusion_matrix": {"tn": 32480, "fp": 37, "fn": 72, "tp": 32433}, "test_mcc2": 0.996648, "test_mcc": 0.996648, "test_sn": 0.997785, "test_sp": 0.998862} ################################################## Total Time: 690900.223548, Avg time per epoch(10 epochs): 69090.020000 ++++++++++++Validation+++++++++++++ best acc global step: 81280 checkpoint path: ../models/ViralCapsid/protein/binary_class/luca_base/matrix/20250103142554/checkpoint-81280 ***** Running evaluation checkpoint-81280 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 43349 ################################################## {"evaluation_avg_loss_81280": 0.015381, "evaluation_total_loss_81280": 41.68272, "evaluation_acc_81280": 0.998408, "evaluation_prec_81280": 0.998986, "evaluation_recall_81280": 0.997836, "evaluation_f1_81280": 0.998411, "evaluation_roc_auc_81280": 0.999853, "evaluation_pr_auc_81280": 0.999757, "evaluation_confusion_matrix_81280": {"tn": 21605, "fp": 22, "fn": 47, "tp": 21675}, "evaluation_mcc2_81280": 0.996817, "evaluation_mcc_81280": 0.996817, "evaluation_sn_81280": 0.997836, "evaluation_sp_81280": 0.998983} ++++++++++++Testing+++++++++++++ best acc global step: 81280 checkpoint path: ../models/ViralCapsid/protein/binary_class/luca_base/matrix/20250103142554/checkpoint-81280 ***** Running testing checkpoint-81280 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [65022] ################################################## {"evaluation_avg_loss_81280": 0.013435, "evaluation_total_loss_81280": 54.601641, "evaluation_acc_81280": 0.998324, "evaluation_prec_81280": 0.99886, "evaluation_recall_81280": 0.997785, "evaluation_f1_81280": 0.998322, "evaluation_roc_auc_81280": 0.999945, "evaluation_pr_auc_81280": 0.99993, "evaluation_confusion_matrix_81280": {"tn": 32480, "fp": 37, "fn": 72, "tp": 32433}, "evaluation_mcc2_81280": 0.996648, "evaluation_mcc_81280": 0.996648, "evaluation_sn_81280": 0.997785, "evaluation_sp_81280": 0.998862}