{"adam_epsilon": 1e-08, "alphabet": "gene_prot", "append_eos": true, "asl_gamma_neg": 4.0, "asl_gamma_pos": 1.0, "best_metric_type": "f1", "beta1": 0.9, "beta2": 0.98, "buffer_size": 512, "cache_dir": null, "classifier_activate_func": "gelu", "classifier_size": 128, "codes_file": null, "config_path": "../config/luca_base/luca_base_config.json", "cross_atten": false, "dataset_name": "VirusEC4", "dataset_type": "protein", "delete_old": false, "dev_data_dir": "../dataset/VirusEC4/protein/multi_label/dev/", "device": "cuda", "do_eval": true, "do_lower_case": false, "do_metrics": true, "do_predict": true, "do_train": true, "dropout_prob": 0.1, "early_stop_epoch": -1, "emb_activate_func": "gelu", "embedding_complete": true, "embedding_complete_seg_overlap": true, "embedding_fixed_len_a_time": 3072, "embedding_input_size": 2560, "embedding_input_size_a": null, "embedding_input_size_b": null, "eval_all_checkpoints": false, "evaluate_during_training": true, "evaluate_steps": -1, "evaluate_strategy": "epoch", "fc_activate_func": "gelu", "focal_loss_alpha": 0.7, "focal_loss_gamma": 2.0, "focal_loss_reduce": false, "fp16": false, "fp16_embedding": false, "fp16_opt_level": "O1", "fusion_type": "concat", "gradient_accumulation_steps": 1, "hidden_size": 1024, "ignore_index": -100, "input_mode": "single", "input_type": "matrix", "intermediate_size": 4096, "label_filepath": "../dataset/VirusEC4/protein/multi_label/label.txt", "label_size": 70, "label_type": "VirusEC4", "learning_rate": 0.0001, "llm_dir": "..", "llm_dirpath": "../llm/models/lucagplm/v2.0/token_level,span_level,seq_level,structure_level/lucaone_gplm/20231125113045/checkpoint-step17600000", "llm_step": "17600000", "llm_task_level": "token_level,span_level,seq_level,structure_level", "llm_time_str": "20231125113045", "llm_type": "lucaone_gplm", "llm_version": "v2.0", "lmdb_path": null, "local_rank": -1, "log_dir": "../logs/VirusEC4/protein/multi_label/luca_base/matrix/20250508171231", "logging_steps": 200, "loss_reduction": "mean", "loss_type": "bce", "lr_decay_rate": 0.9, "lr_update_strategy": "step", "matrix_add_special_token": true, "matrix_dirpath": "../matrices/VirusEC4/protein/multi_label/luca_base/v2.0/lucaone_gplm/20231125113045/17600000", "matrix_embedding_exists": false, "matrix_encoder": false, "matrix_encoder_act": false, "matrix_fc_size": "128", "matrix_max_length": 10000, "matrix_max_length_a": null, "matrix_max_length_b": null, "matrix_pooling_type": "value_attention", "max_grad_norm": 1.0, "max_sentence_length": null, "max_sentences": null, "max_steps": -1, "model_dirpath": null, "model_type": "luca_base", "n_gpu": 1, "no_cuda": false, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "non_ignore": true, "not_append_eos": false, "not_matrix_encoder_shared": false, "not_prepend_bos": false, "not_save_emb_to_disk": false, "not_seq_encoder_shared": false, "num_attention_heads": 0, "num_hidden_layers": 0, "num_train_epochs": 50, "output_dir": "../models/VirusEC4/protein/multi_label/luca_base/matrix/20250508171231", "output_mode": "multi_label", "overwrite_cache": false, "overwrite_output_dir": true, "per_gpu_eval_batch_size": 16, "per_gpu_train_batch_size": 16, "pos_weight": 1.0, "position_embedding_type": "absolute", "prepend_bos": true, "save_all": true, "save_steps": -1, "seed": 1221, "self_atten": false, "seq_fc_size": "null", "seq_max_length": 10000, "seq_max_length_a": null, "seq_max_length_b": null, "seq_pooling_type": "value_attention", "seq_subword": false, "seq_vocab_path": "gene_prot", "sigmoid": true, "task_level_type": "seq_level", "task_type": "multi_label", "tb_log_dir": "../tb-logs/VirusEC4/protein/multi_label/luca_base/matrix/20250508171231", "test_data_dir": "../dataset/VirusEC4/protein/multi_label/test/", "time_str": "20250508171241", "train_data_dir": "../dataset/VirusEC4/protein/multi_label/train/", "trunc_type": "right", "vector_dirpath": "../vectors/VirusEC4/protein/multi_label/luca_base/v2.0/lucaone_gplm/20231125113045/17600000", "vector_fc_size": "null", "vocab_size": 39, "warmup_steps": 1000, "weight": [1749.35, 1715.049019607843, 1666.047619047619, 1521.1739130434783, 1445.7438016528927, 1356.0852713178294, 1305.4850746268658, 1258.525179856115, 1240.6737588652481, 1150.8881578947369, 993.9488636363636, 920.7105263157895, 874.675, 813.6511627906976, 777.4888888888889, 774.0486725663717, 763.9082969432314, 650.3159851301115, 631.5342960288808, 629.2625899280575, 620.3368794326241, 615.9683098591549, 585.0668896321071, 567.9707792207793, 544.9688473520249, 539.9228395061729, 519.0949554896142, 510.01457725947523, 484.584487534626, 366.74004192872115, 354.1194331983806, 343.0098039215686, 331.3162878787879, 328.8251879699248, 328.2082551594747, 314.0664272890485, 312.38392857142856, 199.01592718998862, 186.49786780383795, 181.84511434511435, 172.01081612586037, 170.66829268292682, 156.05263157894737, 149.0076660988075, 134.4619523443505, 128.62867647058823, 127.68978102189782, 127.4107793153678, 127.0406681190995, 100.07723112128147, 94.0510752688172, 80.20861989912883, 78.0611334225792, 74.85451433461704, 65.00743218134522, 63.7983223924143, 44.866632469864065, 42.24462690171456, 41.661109788044776, 29.93924353927777, 27.193377895227734, 27.004476690336524, 25.790210821170575, 22.048777413662716, 21.104475811316203, 17.876047414674023, 10.421482187537233, 9.041970331317517, 17.4935, 7.06893764900796], "weight_decay": 0.01, "worker_num": 0} ################################################## n_gpu: 1 ################################################## Inputs: Input Name List: protein,embedding_matrix ################################################## Encoder Config: {'llm_type': 'lucaone_gplm', 'llm_version': 'v2.0', 'llm_step': '17600000', 'llm_dirpath': '../llm/models/lucagplm/v2.0/token_level,span_level,seq_level,structure_level/lucaone_gplm/20231125113045/checkpoint-step17600000', 'input_type': 'matrix', 'trunc_type': 'right', 'seq_max_length': 10000, 'atom_seq_max_length': None, 'vector_dirpath': '../vectors/VirusEC4/protein/multi_label/luca_base/v2.0/lucaone_gplm/20231125113045/17600000', 'matrix_dirpath': '../matrices/VirusEC4/protein/multi_label/luca_base/v2.0/lucaone_gplm/20231125113045/17600000', 'local_rank': -1, 'max_sentence_length': None, 'max_sentences': None, 'matrix_add_special_token': True, 'embedding_complete': True, 'embedding_complete_seg_overlap': True, 'embedding_fixed_len_a_time': 3072, 'matrix_embedding_exists': False, 'save_emb_to_disk': True, 'fp16_embedding': False} ################################################## Model Config: LucaConfig { "_attn_implementation_autoset": true, "alphabet": "gene_prot", "attention_probs_dropout_prob": 0.1, "classifier_activate_func": "gelu", "classifier_dropout_prob": 0.1, "classifier_size": 128, "cls_token_id": 2, "cross_atten": false, "directionality": "bidi", "emb_activate_func": "gelu", "embedding_input_size": 2560, "fc_activate_func": "gelu", "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 1024, "id2label": {}, "ignore_index": -100, "initializer_range": 0.02, "intermediate_size": 4096, "kernel_size": 7, "label2id": {}, "layer_norm_eps": 1e-12, "loss_reduction": "mean", "matrix_fc_size": [ 128 ], "matrix_max_length": 10000, "matrix_pooling_type": "value_attention", "max_position_embeddings": 10002, "no_position_embeddings": true, "no_token_embeddings": true, "no_token_type_embeddings": true, "num_attention_heads": 8, "num_hidden_layers": 4, "pad_token_id": 0, "pos_weight": 1.0, "position_embedding_type": "absolute", "self_atten": false, "sep_token_id": 3, "seq_fc_size": null, "seq_max_length": 10000, "seq_pooling_type": "value_attention", "token_dropout": null, "transformers_version": "4.46.3", "type_vocab_size": 2, "use_luca_layer_norm_v2": true, "vector_fc_size": null, "vocab_size": 39, "weight": [ 1749.35, 1715.049019607843, 1666.047619047619, 1521.1739130434783, 1445.7438016528927, 1356.0852713178294, 1305.4850746268658, 1258.525179856115, 1240.6737588652481, 1150.8881578947369, 993.9488636363636, 920.7105263157895, 874.675, 813.6511627906976, 777.4888888888889, 774.0486725663717, 763.9082969432314, 650.3159851301115, 631.5342960288808, 629.2625899280575, 620.3368794326241, 615.9683098591549, 585.0668896321071, 567.9707792207793, 544.9688473520249, 539.9228395061729, 519.0949554896142, 510.01457725947523, 484.584487534626, 366.74004192872115, 354.1194331983806, 343.0098039215686, 331.3162878787879, 328.8251879699248, 328.2082551594747, 314.0664272890485, 312.38392857142856, 199.01592718998862, 186.49786780383795, 181.84511434511435, 172.01081612586037, 170.66829268292682, 156.05263157894737, 149.0076660988075, 134.4619523443505, 128.62867647058823, 127.68978102189782, 127.4107793153678, 127.0406681190995, 100.07723112128147, 94.0510752688172, 80.20861989912883, 78.0611334225792, 74.85451433461704, 65.00743218134522, 63.7983223924143, 44.866632469864065, 42.24462690171456, 41.661109788044776, 29.93924353927777, 27.193377895227734, 27.004476690336524, 25.790210821170575, 22.048777413662716, 21.104475811316203, 17.876047414674023, 10.421482187537233, 9.041970331317517, 17.4935, 7.06893764900796 ] } ################################################## Mode Architecture: LucaBase( (matrix_pooler): GlobalMaskValueAttentionPooling1D (2560 -> 2560) (linear): ModuleList( (0): ModuleList( (0): Linear(in_features=2560, out_features=128, bias=True) (1): GELU(approximate='none') ) ) (dropout): Dropout(p=0.1, inplace=False) (hidden_layer): Linear(in_features=128, out_features=128, bias=True) (hidden_act): GELU(approximate='none') (classifier): Linear(in_features=128, out_features=70, bias=True) (output): Sigmoid() (loss_fct): MaskedBCEWithLogitsLoss( (criterion): BCEWithLogitsLoss() ) ) ################################################## Model parameters: 20014150 ################################################## {"total_num": "19.090000M", "total_size": "76.350000MB", "param_sum": "19.090000M", "param_size": "76.350000MB", "buffer_sum": "0.000000M", "buffer_size": "0.000000MB", "trainable_num": "19.086981M", "trainable_size": "76.347923MB"} ################################################## Train dataset len: 79578, batch size: 16, batch num: 4974 Train dataset t_total: 248700, max_steps: -1 ***** Running training ***** Train Dataset Num examples = 79578 Train Dataset Num Epochs = 50 Logging Steps = 200 Saving Steps = -1 Evaluating Strategy = epoch Train Dataset Instantaneous batch size per GPU = 16 Train Dataset Total train batch size (w. parallel, distributed & accumulation) = 16 Train Dataset Gradient Accumulation steps = 1 Train Dataset Total optimization steps = 248700 ################################################## Training, Epoch: 0001, Batch: 000200, Sample Num: 3200, Cur Loss: 0.68710357, Cur Avg Loss: 0.69173493, Log Avg loss: 0.69173493, Global Avg Loss: 0.69173493, Time: 0.2261 Steps: 200, Updated lr: 0.000020 Training, Epoch: 0001, Batch: 000400, Sample Num: 6400, Cur Loss: 0.49359611, Cur Avg Loss: 0.66323552, Log Avg loss: 0.63473610, Global Avg Loss: 0.66323552, Time: 0.2195 Steps: 400, Updated lr: 0.000040 Training, Epoch: 0001, Batch: 000600, Sample Num: 9600, Cur Loss: 0.09289155, Cur Avg Loss: 0.51450549, Log Avg loss: 0.21704542, Global Avg Loss: 0.51450549, Time: 0.2226 Steps: 600, Updated lr: 0.000060 Training, Epoch: 0001, Batch: 000800, Sample Num: 12800, Cur Loss: 0.08599862, Cur Avg Loss: 0.40758232, Log Avg loss: 0.08681282, Global Avg Loss: 0.40758232, Time: 0.2432 Steps: 800, Updated lr: 0.000080 Training, Epoch: 0001, Batch: 001000, Sample Num: 16000, Cur Loss: 0.08545030, Cur Avg Loss: 0.34232936, Log Avg loss: 0.08131750, Global Avg Loss: 0.34232936, Time: 0.2189 Steps: 1000, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 001200, Sample Num: 19200, Cur Loss: 0.06878120, Cur Avg Loss: 0.29835252, Log Avg loss: 0.07846833, Global Avg Loss: 0.29835252, Time: 0.2168 Steps: 1200, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 001400, Sample Num: 22400, Cur Loss: 0.05119146, Cur Avg Loss: 0.26565241, Log Avg loss: 0.06945173, Global Avg Loss: 0.26565241, Time: 0.1164 Steps: 1400, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 001600, Sample Num: 25600, Cur Loss: 0.05858688, Cur Avg Loss: 0.23992992, Log Avg loss: 0.05987256, Global Avg Loss: 0.23992992, Time: 0.2413 Steps: 1600, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 001800, Sample Num: 28800, Cur Loss: 0.05697928, Cur Avg Loss: 0.21879631, Log Avg loss: 0.04972741, Global Avg Loss: 0.21879631, Time: 0.2218 Steps: 1800, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 002000, Sample Num: 32000, Cur Loss: 0.04644882, Cur Avg Loss: 0.20134921, Log Avg loss: 0.04432526, Global Avg Loss: 0.20134921, Time: 0.2180 Steps: 2000, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 002200, Sample Num: 35200, Cur Loss: 0.03809260, Cur Avg Loss: 0.18679807, Log Avg loss: 0.04128674, Global Avg Loss: 0.18679807, Time: 0.2207 Steps: 2200, Updated lr: 0.000100 Training, Epoch: 0001, Batch: 002400, Sample Num: 38400, Cur Loss: 0.03866476, Cur Avg Loss: 0.17427772, Log Avg loss: 0.03655388, Global Avg Loss: 0.17427772, Time: 0.2593 Steps: 2400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002600, Sample Num: 41600, Cur Loss: 0.03390218, Cur Avg Loss: 0.16350518, Log Avg loss: 0.03423466, Global Avg Loss: 0.16350518, Time: 0.2280 Steps: 2600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 002800, Sample Num: 44800, Cur Loss: 0.02507691, Cur Avg Loss: 0.15408283, Log Avg loss: 0.03159225, Global Avg Loss: 0.15408283, Time: 0.2221 Steps: 2800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003000, Sample Num: 48000, Cur Loss: 0.03387703, Cur Avg Loss: 0.14571317, Log Avg loss: 0.02853791, Global Avg Loss: 0.14571317, Time: 0.2276 Steps: 3000, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003200, Sample Num: 51200, Cur Loss: 0.04278360, Cur Avg Loss: 0.13825204, Log Avg loss: 0.02633511, Global Avg Loss: 0.13825204, Time: 0.2600 Steps: 3200, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003400, Sample Num: 54400, Cur Loss: 0.02477257, Cur Avg Loss: 0.13157843, Log Avg loss: 0.02480060, Global Avg Loss: 0.13157843, Time: 0.2207 Steps: 3400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003600, Sample Num: 57600, Cur Loss: 0.02074098, Cur Avg Loss: 0.12551018, Log Avg loss: 0.02235007, Global Avg Loss: 0.12551018, Time: 0.2342 Steps: 3600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 003800, Sample Num: 60800, Cur Loss: 0.01538565, Cur Avg Loss: 0.12001139, Log Avg loss: 0.02103316, Global Avg Loss: 0.12001139, Time: 0.2197 Steps: 3800, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 004000, Sample Num: 64000, Cur Loss: 0.00567909, Cur Avg Loss: 0.11500165, Log Avg loss: 0.01981662, Global Avg Loss: 0.11500165, Time: 0.2592 Steps: 4000, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 004200, Sample Num: 67200, Cur Loss: 0.01298327, Cur Avg Loss: 0.11041612, Log Avg loss: 0.01870552, Global Avg Loss: 0.11041612, Time: 0.2208 Steps: 4200, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 004400, Sample Num: 70400, Cur Loss: 0.00979189, Cur Avg Loss: 0.10617208, Log Avg loss: 0.01704715, Global Avg Loss: 0.10617208, Time: 0.2198 Steps: 4400, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 004600, Sample Num: 73600, Cur Loss: 0.02270590, Cur Avg Loss: 0.10225718, Log Avg loss: 0.01612947, Global Avg Loss: 0.10225718, Time: 0.2202 Steps: 4600, Updated lr: 0.000099 Training, Epoch: 0001, Batch: 004800, Sample Num: 76800, Cur Loss: 0.03392740, Cur Avg Loss: 0.09862668, Log Avg loss: 0.01512515, Global Avg Loss: 0.09862668, Time: 0.7135 Steps: 4800, Updated lr: 0.000098 ***** Running evaluation checkpoint-4974 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-4974 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1089.950976, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.013135, "eval_total_loss": 14.002116, "eval_acc": 0.996368, "eval_jaccard": 0.778525, "eval_prec": 0.78838, "eval_recall": 0.780142, "eval_f1": 0.782478, "eval_pr_auc": 0.903979, "eval_roc_auc": 0.985365, "eval_fmax": 0.90612, "eval_pmax": 0.965311, "eval_rmax": 0.853769, "eval_tmax": 0.18, "update_flag": true, "test_avg_loss": 0.013518, "test_total_loss": 14.409748, "test_acc": 0.996282, "test_jaccard": 0.770549, "test_prec": 0.780757, "test_recall": 0.772326, "test_f1": 0.774741, "test_pr_auc": 0.900239, "test_roc_auc": 0.984455, "test_fmax": 0.901031, "test_pmax": 0.962729, "test_rmax": 0.846765, "test_tmax": 0.18, "lr": 9.839563988696004e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.09568940349646694, "train_cur_epoch_loss": 475.9590929914266, "train_cur_epoch_avg_loss": 0.09568940349646694, "train_cur_epoch_time": 1089.9509763717651, "train_cur_epoch_avg_time": 0.21912966955604446, "epoch": 1, "step": 4974} ################################################## Training, Epoch: 0002, Batch: 000026, Sample Num: 416, Cur Loss: 0.00934371, Cur Avg Loss: 0.01067956, Log Avg loss: 0.01414345, Global Avg Loss: 0.09524735, Time: 0.0774 Steps: 5000, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 000226, Sample Num: 3616, Cur Loss: 0.00910914, Cur Avg Loss: 0.01297828, Log Avg loss: 0.01327712, Global Avg Loss: 0.09209465, Time: 0.2218 Steps: 5200, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 000426, Sample Num: 6816, Cur Loss: 0.02062244, Cur Avg Loss: 0.01283479, Log Avg loss: 0.01267265, Global Avg Loss: 0.08915310, Time: 0.2202 Steps: 5400, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 000626, Sample Num: 10016, Cur Loss: 0.01690722, Cur Avg Loss: 0.01273366, Log Avg loss: 0.01251824, Global Avg Loss: 0.08641614, Time: 0.2215 Steps: 5600, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 000826, Sample Num: 13216, Cur Loss: 0.00860392, Cur Avg Loss: 0.01236820, Log Avg loss: 0.01122429, Global Avg Loss: 0.08382331, Time: 0.1091 Steps: 5800, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 001026, Sample Num: 16416, Cur Loss: 0.00496720, Cur Avg Loss: 0.01204271, Log Avg loss: 0.01069844, Global Avg Loss: 0.08138582, Time: 0.2170 Steps: 6000, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 001226, Sample Num: 19616, Cur Loss: 0.01300684, Cur Avg Loss: 0.01182660, Log Avg loss: 0.01071796, Global Avg Loss: 0.07910621, Time: 0.0855 Steps: 6200, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 001426, Sample Num: 22816, Cur Loss: 0.00767288, Cur Avg Loss: 0.01150614, Log Avg loss: 0.00954171, Global Avg Loss: 0.07693232, Time: 0.2151 Steps: 6400, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 001626, Sample Num: 26016, Cur Loss: 0.01261355, Cur Avg Loss: 0.01127635, Log Avg loss: 0.00963797, Global Avg Loss: 0.07489310, Time: 0.2212 Steps: 6600, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 001826, Sample Num: 29216, Cur Loss: 0.00595180, Cur Avg Loss: 0.01100015, Log Avg loss: 0.00875463, Global Avg Loss: 0.07294785, Time: 0.3024 Steps: 6800, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 002026, Sample Num: 32416, Cur Loss: 0.00293246, Cur Avg Loss: 0.01081707, Log Avg loss: 0.00914556, Global Avg Loss: 0.07112493, Time: 0.2254 Steps: 7000, Updated lr: 0.000098 Training, Epoch: 0002, Batch: 002226, Sample Num: 35616, Cur Loss: 0.00511046, Cur Avg Loss: 0.01059921, Log Avg loss: 0.00839233, Global Avg Loss: 0.06938235, Time: 0.2274 Steps: 7200, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 002426, Sample Num: 38816, Cur Loss: 0.00198442, Cur Avg Loss: 0.01034979, Log Avg loss: 0.00757368, Global Avg Loss: 0.06771185, Time: 0.2216 Steps: 7400, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 002626, Sample Num: 42016, Cur Loss: 0.00659713, Cur Avg Loss: 0.01010187, Log Avg loss: 0.00709468, Global Avg Loss: 0.06611666, Time: 0.2232 Steps: 7600, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 002826, Sample Num: 45216, Cur Loss: 0.00402990, Cur Avg Loss: 0.00989284, Log Avg loss: 0.00714823, Global Avg Loss: 0.06460465, Time: 0.1229 Steps: 7800, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 003026, Sample Num: 48416, Cur Loss: 0.00822419, Cur Avg Loss: 0.00969637, Log Avg loss: 0.00692031, Global Avg Loss: 0.06316254, Time: 0.2200 Steps: 8000, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 003226, Sample Num: 51616, Cur Loss: 0.01197555, Cur Avg Loss: 0.00951456, Log Avg loss: 0.00676374, Global Avg Loss: 0.06178696, Time: 0.2194 Steps: 8200, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 003426, Sample Num: 54816, Cur Loss: 0.00095219, Cur Avg Loss: 0.00934029, Log Avg loss: 0.00652921, Global Avg Loss: 0.06047130, Time: 0.2210 Steps: 8400, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 003626, Sample Num: 58016, Cur Loss: 0.01701835, Cur Avg Loss: 0.00917367, Log Avg loss: 0.00631955, Global Avg Loss: 0.05921196, Time: 0.1292 Steps: 8600, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 003826, Sample Num: 61216, Cur Loss: 0.00674610, Cur Avg Loss: 0.00900890, Log Avg loss: 0.00602170, Global Avg Loss: 0.05800309, Time: 0.2276 Steps: 8800, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 004026, Sample Num: 64416, Cur Loss: 0.00334580, Cur Avg Loss: 0.00888217, Log Avg loss: 0.00645784, Global Avg Loss: 0.05685764, Time: 0.1980 Steps: 9000, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 004226, Sample Num: 67616, Cur Loss: 0.00144992, Cur Avg Loss: 0.00874220, Log Avg loss: 0.00592460, Global Avg Loss: 0.05575040, Time: 0.2240 Steps: 9200, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 004426, Sample Num: 70816, Cur Loss: 0.00690608, Cur Avg Loss: 0.00859552, Log Avg loss: 0.00549602, Global Avg Loss: 0.05468115, Time: 0.2202 Steps: 9400, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 004626, Sample Num: 74016, Cur Loss: 0.00205495, Cur Avg Loss: 0.00845421, Log Avg loss: 0.00532714, Global Avg Loss: 0.05365295, Time: 0.2256 Steps: 9600, Updated lr: 0.000097 Training, Epoch: 0002, Batch: 004826, Sample Num: 77216, Cur Loss: 0.00559091, Cur Avg Loss: 0.00831862, Log Avg loss: 0.00518237, Global Avg Loss: 0.05266375, Time: 0.0860 Steps: 9800, Updated lr: 0.000096 ***** Running evaluation checkpoint-9948 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-9948 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1095.659957, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.004577, "eval_total_loss": 4.879424, "eval_acc": 0.998586, "eval_jaccard": 0.91736, "eval_prec": 0.92105, "eval_recall": 0.919191, "eval_f1": 0.919342, "eval_pr_auc": 0.979521, "eval_roc_auc": 0.997834, "eval_fmax": 0.974327, "eval_pmax": 0.978439, "eval_rmax": 0.970249, "eval_tmax": 0.15, "update_flag": true, "test_avg_loss": 0.004712, "test_total_loss": 5.022764, "test_acc": 0.998609, "test_jaccard": 0.917357, "test_prec": 0.920737, "test_recall": 0.919204, "test_f1": 0.919298, "test_pr_auc": 0.977599, "test_roc_auc": 0.99739, "test_fmax": 0.972486, "test_pmax": 0.987747, "test_rmax": 0.957689, "test_tmax": 0.2, "lr": 9.638756560355269e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.05195731273595275, "train_cur_epoch_loss": 40.91225410583138, "train_cur_epoch_avg_loss": 0.008225221975438556, "train_cur_epoch_time": 1095.6599566936493, "train_cur_epoch_avg_time": 0.2202774339955065, "epoch": 2, "step": 9948} ################################################## Training, Epoch: 0003, Batch: 000052, Sample Num: 832, Cur Loss: 0.00067680, Cur Avg Loss: 0.00453725, Log Avg loss: 0.00501265, Global Avg Loss: 0.05171073, Time: 0.2244 Steps: 10000, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 000252, Sample Num: 4032, Cur Loss: 0.00207104, Cur Avg Loss: 0.00476500, Log Avg loss: 0.00482422, Global Avg Loss: 0.05079139, Time: 0.3962 Steps: 10200, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 000452, Sample Num: 7232, Cur Loss: 0.01879703, Cur Avg Loss: 0.00476720, Log Avg loss: 0.00476996, Global Avg Loss: 0.04990636, Time: 0.2172 Steps: 10400, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 000652, Sample Num: 10432, Cur Loss: 0.00345491, Cur Avg Loss: 0.00481464, Log Avg loss: 0.00492187, Global Avg Loss: 0.04905759, Time: 0.2177 Steps: 10600, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 000852, Sample Num: 13632, Cur Loss: 0.00192806, Cur Avg Loss: 0.00470714, Log Avg loss: 0.00435668, Global Avg Loss: 0.04822980, Time: 0.2196 Steps: 10800, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 001052, Sample Num: 16832, Cur Loss: 0.00591104, Cur Avg Loss: 0.00464873, Log Avg loss: 0.00439989, Global Avg Loss: 0.04743289, Time: 0.3009 Steps: 11000, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 001252, Sample Num: 20032, Cur Loss: 0.00066975, Cur Avg Loss: 0.00456891, Log Avg loss: 0.00414910, Global Avg Loss: 0.04665997, Time: 0.2090 Steps: 11200, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 001452, Sample Num: 23232, Cur Loss: 0.00164676, Cur Avg Loss: 0.00443799, Log Avg loss: 0.00361840, Global Avg Loss: 0.04590485, Time: 0.2171 Steps: 11400, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 001652, Sample Num: 26432, Cur Loss: 0.00175405, Cur Avg Loss: 0.00435772, Log Avg loss: 0.00377493, Global Avg Loss: 0.04517847, Time: 0.1717 Steps: 11600, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 001852, Sample Num: 29632, Cur Loss: 0.01490783, Cur Avg Loss: 0.00428997, Log Avg loss: 0.00373042, Global Avg Loss: 0.04447596, Time: 0.2268 Steps: 11800, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 002052, Sample Num: 32832, Cur Loss: 0.00310002, Cur Avg Loss: 0.00426413, Log Avg loss: 0.00402481, Global Avg Loss: 0.04380178, Time: 0.2913 Steps: 12000, Updated lr: 0.000096 Training, Epoch: 0003, Batch: 002252, Sample Num: 36032, Cur Loss: 0.00636087, Cur Avg Loss: 0.00419476, Log Avg loss: 0.00348301, Global Avg Loss: 0.04314082, Time: 0.2145 Steps: 12200, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 002452, Sample Num: 39232, Cur Loss: 0.00337466, Cur Avg Loss: 0.00409157, Log Avg loss: 0.00292964, Global Avg Loss: 0.04249225, Time: 0.2194 Steps: 12400, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 002652, Sample Num: 42432, Cur Loss: 0.00179195, Cur Avg Loss: 0.00401675, Log Avg loss: 0.00309953, Global Avg Loss: 0.04186697, Time: 0.2149 Steps: 12600, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 002852, Sample Num: 45632, Cur Loss: 0.00075012, Cur Avg Loss: 0.00393771, Log Avg loss: 0.00288956, Global Avg Loss: 0.04125794, Time: 0.2234 Steps: 12800, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 003052, Sample Num: 48832, Cur Loss: 0.00125281, Cur Avg Loss: 0.00389152, Log Avg loss: 0.00323284, Global Avg Loss: 0.04067294, Time: 0.2196 Steps: 13000, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 003252, Sample Num: 52032, Cur Loss: 0.00454962, Cur Avg Loss: 0.00383084, Log Avg loss: 0.00290494, Global Avg Loss: 0.04010070, Time: 0.1912 Steps: 13200, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 003452, Sample Num: 55232, Cur Loss: 0.00791524, Cur Avg Loss: 0.00379144, Log Avg loss: 0.00315080, Global Avg Loss: 0.03954921, Time: 0.2354 Steps: 13400, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 003652, Sample Num: 58432, Cur Loss: 0.00178244, Cur Avg Loss: 0.00374763, Log Avg loss: 0.00299146, Global Avg Loss: 0.03901160, Time: 0.2189 Steps: 13600, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 003852, Sample Num: 61632, Cur Loss: 0.00264522, Cur Avg Loss: 0.00369809, Log Avg loss: 0.00279352, Global Avg Loss: 0.03848670, Time: 0.2197 Steps: 13800, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 004052, Sample Num: 64832, Cur Loss: 0.00364506, Cur Avg Loss: 0.00367028, Log Avg loss: 0.00313469, Global Avg Loss: 0.03798167, Time: 0.2193 Steps: 14000, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 004252, Sample Num: 68032, Cur Loss: 0.00127341, Cur Avg Loss: 0.00362645, Log Avg loss: 0.00273838, Global Avg Loss: 0.03748528, Time: 0.2233 Steps: 14200, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 004452, Sample Num: 71232, Cur Loss: 0.00384645, Cur Avg Loss: 0.00358067, Log Avg loss: 0.00260734, Global Avg Loss: 0.03700087, Time: 0.2214 Steps: 14400, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 004652, Sample Num: 74432, Cur Loss: 0.00043779, Cur Avg Loss: 0.00354001, Log Avg loss: 0.00263502, Global Avg Loss: 0.03653010, Time: 0.2130 Steps: 14600, Updated lr: 0.000095 Training, Epoch: 0003, Batch: 004852, Sample Num: 77632, Cur Loss: 0.00127358, Cur Avg Loss: 0.00350359, Log Avg loss: 0.00265645, Global Avg Loss: 0.03607235, Time: 0.2250 Steps: 14800, Updated lr: 0.000094 ***** Running evaluation checkpoint-14922 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-14922 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1090.592690, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.002542, "eval_total_loss": 2.709736, "eval_acc": 0.999278, "eval_jaccard": 0.961972, "eval_prec": 0.964083, "eval_recall": 0.963812, "eval_f1": 0.963448, "eval_pr_auc": 0.99073, "eval_roc_auc": 0.99891, "eval_fmax": 0.987846, "eval_pmax": 0.994486, "eval_rmax": 0.981294, "eval_tmax": 0.27, "update_flag": true, "test_avg_loss": 0.002634, "test_total_loss": 2.808119, "test_acc": 0.999294, "test_jaccard": 0.960914, "test_prec": 0.962794, "test_recall": 0.962657, "test_f1": 0.962264, "test_pr_auc": 0.989752, "test_roc_auc": 0.998558, "test_fmax": 0.987585, "test_pmax": 0.994814, "test_rmax": 0.98046, "test_tmax": 0.27, "lr": 9.437949132014535e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.03579856323000513, "train_cur_epoch_loss": 17.314813420878636, "train_cur_epoch_avg_loss": 0.0034810642181098985, "train_cur_epoch_time": 1090.5926895141602, "train_cur_epoch_avg_time": 0.21925868305471655, "epoch": 3, "step": 14922} ################################################## Training, Epoch: 0004, Batch: 000078, Sample Num: 1248, Cur Loss: 0.00547207, Cur Avg Loss: 0.00252537, Log Avg loss: 0.00256181, Global Avg Loss: 0.03562554, Time: 0.2198 Steps: 15000, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 000278, Sample Num: 4448, Cur Loss: 0.00148351, Cur Avg Loss: 0.00245691, Log Avg loss: 0.00243021, Global Avg Loss: 0.03518876, Time: 0.2116 Steps: 15200, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 000478, Sample Num: 7648, Cur Loss: 0.01456666, Cur Avg Loss: 0.00249009, Log Avg loss: 0.00253622, Global Avg Loss: 0.03476470, Time: 0.3962 Steps: 15400, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 000678, Sample Num: 10848, Cur Loss: 0.00022804, Cur Avg Loss: 0.00256781, Log Avg loss: 0.00275357, Global Avg Loss: 0.03435430, Time: 0.2175 Steps: 15600, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 000878, Sample Num: 14048, Cur Loss: 0.00120923, Cur Avg Loss: 0.00247798, Log Avg loss: 0.00217343, Global Avg Loss: 0.03394695, Time: 0.2169 Steps: 15800, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 001078, Sample Num: 17248, Cur Loss: 0.00099746, Cur Avg Loss: 0.00245315, Log Avg loss: 0.00234414, Global Avg Loss: 0.03355192, Time: 0.2218 Steps: 16000, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 001278, Sample Num: 20448, Cur Loss: 0.00078003, Cur Avg Loss: 0.00239481, Log Avg loss: 0.00208039, Global Avg Loss: 0.03316338, Time: 0.2201 Steps: 16200, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 001478, Sample Num: 23648, Cur Loss: 0.00164125, Cur Avg Loss: 0.00230989, Log Avg loss: 0.00176726, Global Avg Loss: 0.03278050, Time: 0.2194 Steps: 16400, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 001678, Sample Num: 26848, Cur Loss: 0.00296409, Cur Avg Loss: 0.00224943, Log Avg loss: 0.00180259, Global Avg Loss: 0.03240727, Time: 0.1293 Steps: 16600, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 001878, Sample Num: 30048, Cur Loss: 0.00126712, Cur Avg Loss: 0.00223093, Log Avg loss: 0.00207571, Global Avg Loss: 0.03204618, Time: 0.2226 Steps: 16800, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 002078, Sample Num: 33248, Cur Loss: 0.00142237, Cur Avg Loss: 0.00223180, Log Avg loss: 0.00224002, Global Avg Loss: 0.03169552, Time: 0.2190 Steps: 17000, Updated lr: 0.000094 Training, Epoch: 0004, Batch: 002278, Sample Num: 36448, Cur Loss: 0.00434878, Cur Avg Loss: 0.00219968, Log Avg loss: 0.00186592, Global Avg Loss: 0.03134866, Time: 0.2172 Steps: 17200, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 002478, Sample Num: 39648, Cur Loss: 0.00051719, Cur Avg Loss: 0.00213493, Log Avg loss: 0.00139747, Global Avg Loss: 0.03100440, Time: 0.2198 Steps: 17400, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 002678, Sample Num: 42848, Cur Loss: 0.00050901, Cur Avg Loss: 0.00209980, Log Avg loss: 0.00166455, Global Avg Loss: 0.03067099, Time: 0.1301 Steps: 17600, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 002878, Sample Num: 46048, Cur Loss: 0.00157410, Cur Avg Loss: 0.00207378, Log Avg loss: 0.00172525, Global Avg Loss: 0.03034576, Time: 0.2196 Steps: 17800, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 003078, Sample Num: 49248, Cur Loss: 0.00062992, Cur Avg Loss: 0.00204226, Log Avg loss: 0.00158874, Global Avg Loss: 0.03002624, Time: 0.2217 Steps: 18000, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 003278, Sample Num: 52448, Cur Loss: 0.00246555, Cur Avg Loss: 0.00201420, Log Avg loss: 0.00158232, Global Avg Loss: 0.02971366, Time: 0.2113 Steps: 18200, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 003478, Sample Num: 55648, Cur Loss: 0.00243169, Cur Avg Loss: 0.00200204, Log Avg loss: 0.00180279, Global Avg Loss: 0.02941029, Time: 0.2213 Steps: 18400, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 003678, Sample Num: 58848, Cur Loss: 0.00092580, Cur Avg Loss: 0.00198440, Log Avg loss: 0.00167772, Global Avg Loss: 0.02911209, Time: 0.2184 Steps: 18600, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 003878, Sample Num: 62048, Cur Loss: 0.00057561, Cur Avg Loss: 0.00196122, Log Avg loss: 0.00153482, Global Avg Loss: 0.02881871, Time: 0.2206 Steps: 18800, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 004078, Sample Num: 65248, Cur Loss: 0.00058279, Cur Avg Loss: 0.00195041, Log Avg loss: 0.00174088, Global Avg Loss: 0.02853368, Time: 0.2220 Steps: 19000, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 004278, Sample Num: 68448, Cur Loss: 0.00044333, Cur Avg Loss: 0.00192334, Log Avg loss: 0.00137128, Global Avg Loss: 0.02825074, Time: 0.2199 Steps: 19200, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 004478, Sample Num: 71648, Cur Loss: 0.00034170, Cur Avg Loss: 0.00190154, Log Avg loss: 0.00143528, Global Avg Loss: 0.02797429, Time: 0.2242 Steps: 19400, Updated lr: 0.000093 Training, Epoch: 0004, Batch: 004678, Sample Num: 74848, Cur Loss: 0.00103416, Cur Avg Loss: 0.00188112, Log Avg loss: 0.00142400, Global Avg Loss: 0.02770337, Time: 0.2262 Steps: 19600, Updated lr: 0.000092 Training, Epoch: 0004, Batch: 004878, Sample Num: 78048, Cur Loss: 0.00015494, Cur Avg Loss: 0.00186792, Log Avg loss: 0.00155914, Global Avg Loss: 0.02743929, Time: 0.2200 Steps: 19800, Updated lr: 0.000092 ***** Running evaluation checkpoint-19896 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-19896 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1089.876994, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001831, "eval_total_loss": 1.951643, "eval_acc": 0.999564, "eval_jaccard": 0.980464, "eval_prec": 0.982251, "eval_recall": 0.982242, "eval_f1": 0.981813, "eval_pr_auc": 0.993694, "eval_roc_auc": 0.999195, "eval_fmax": 0.992005, "eval_pmax": 0.995849, "eval_rmax": 0.988191, "eval_tmax": 0.24, "update_flag": true, "test_avg_loss": 0.001887, "test_total_loss": 2.011893, "test_acc": 0.999587, "test_jaccard": 0.979757, "test_prec": 0.981484, "test_recall": 0.981264, "test_f1": 0.980952, "test_pr_auc": 0.993115, "test_roc_auc": 0.99892, "test_fmax": 0.9922, "test_pmax": 0.995683, "test_rmax": 0.988741, "test_tmax": 0.21, "lr": 9.237141703673799e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.027314435909397004, "train_cur_epoch_loss": 9.261856335226184, "train_cur_epoch_avg_loss": 0.0018620539475726146, "train_cur_epoch_time": 1089.8769943714142, "train_cur_epoch_avg_time": 0.21911479581250787, "epoch": 4, "step": 19896} ################################################## Training, Epoch: 0005, Batch: 000104, Sample Num: 1664, Cur Loss: 0.00165697, Cur Avg Loss: 0.00151981, Log Avg loss: 0.00154098, Global Avg Loss: 0.02718030, Time: 0.2222 Steps: 20000, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 000304, Sample Num: 4864, Cur Loss: 0.00100970, Cur Avg Loss: 0.00141946, Log Avg loss: 0.00136727, Global Avg Loss: 0.02692473, Time: 0.2207 Steps: 20200, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 000504, Sample Num: 8064, Cur Loss: 0.00029051, Cur Avg Loss: 0.00147591, Log Avg loss: 0.00156173, Global Avg Loss: 0.02667607, Time: 0.2213 Steps: 20400, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 000704, Sample Num: 11264, Cur Loss: 0.00052287, Cur Avg Loss: 0.00145532, Log Avg loss: 0.00140342, Global Avg Loss: 0.02643071, Time: 0.2338 Steps: 20600, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 000904, Sample Num: 14464, Cur Loss: 0.00070662, Cur Avg Loss: 0.00141500, Log Avg loss: 0.00127307, Global Avg Loss: 0.02618881, Time: 0.2169 Steps: 20800, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 001104, Sample Num: 17664, Cur Loss: 0.00058043, Cur Avg Loss: 0.00140380, Log Avg loss: 0.00135317, Global Avg Loss: 0.02595228, Time: 0.2167 Steps: 21000, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 001304, Sample Num: 20864, Cur Loss: 0.00246154, Cur Avg Loss: 0.00136092, Log Avg loss: 0.00112426, Global Avg Loss: 0.02571805, Time: 0.2214 Steps: 21200, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 001504, Sample Num: 24064, Cur Loss: 0.00090374, Cur Avg Loss: 0.00129479, Log Avg loss: 0.00086358, Global Avg Loss: 0.02548577, Time: 0.4241 Steps: 21400, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 001704, Sample Num: 27264, Cur Loss: 0.00031749, Cur Avg Loss: 0.00126463, Log Avg loss: 0.00103784, Global Avg Loss: 0.02525940, Time: 0.2190 Steps: 21600, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 001904, Sample Num: 30464, Cur Loss: 0.00041419, Cur Avg Loss: 0.00125259, Log Avg loss: 0.00115005, Global Avg Loss: 0.02503821, Time: 0.2221 Steps: 21800, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 002104, Sample Num: 33664, Cur Loss: 0.00198105, Cur Avg Loss: 0.00127689, Log Avg loss: 0.00150817, Global Avg Loss: 0.02482430, Time: 0.2206 Steps: 22000, Updated lr: 0.000092 Training, Epoch: 0005, Batch: 002304, Sample Num: 36864, Cur Loss: 0.00019950, Cur Avg Loss: 0.00125621, Log Avg loss: 0.00103862, Global Avg Loss: 0.02461001, Time: 0.2336 Steps: 22200, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 002504, Sample Num: 40064, Cur Loss: 0.00012852, Cur Avg Loss: 0.00121577, Log Avg loss: 0.00074997, Global Avg Loss: 0.02439698, Time: 0.2152 Steps: 22400, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 002704, Sample Num: 43264, Cur Loss: 0.00320155, Cur Avg Loss: 0.00119577, Log Avg loss: 0.00094539, Global Avg Loss: 0.02418944, Time: 0.0997 Steps: 22600, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 002904, Sample Num: 46464, Cur Loss: 0.00026652, Cur Avg Loss: 0.00118067, Log Avg loss: 0.00097647, Global Avg Loss: 0.02398582, Time: 0.2192 Steps: 22800, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 003104, Sample Num: 49664, Cur Loss: 0.00405191, Cur Avg Loss: 0.00116613, Log Avg loss: 0.00095503, Global Avg Loss: 0.02378555, Time: 0.2332 Steps: 23000, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 003304, Sample Num: 52864, Cur Loss: 0.00128687, Cur Avg Loss: 0.00114934, Log Avg loss: 0.00088878, Global Avg Loss: 0.02358817, Time: 0.2184 Steps: 23200, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 003504, Sample Num: 56064, Cur Loss: 0.00035850, Cur Avg Loss: 0.00115075, Log Avg loss: 0.00117406, Global Avg Loss: 0.02339659, Time: 0.2180 Steps: 23400, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 003704, Sample Num: 59264, Cur Loss: 0.00371176, Cur Avg Loss: 0.00114475, Log Avg loss: 0.00103964, Global Avg Loss: 0.02320713, Time: 0.1434 Steps: 23600, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 003904, Sample Num: 62464, Cur Loss: 0.00110163, Cur Avg Loss: 0.00112651, Log Avg loss: 0.00078866, Global Avg Loss: 0.02301874, Time: 0.2376 Steps: 23800, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 004104, Sample Num: 65664, Cur Loss: 0.01140249, Cur Avg Loss: 0.00112830, Log Avg loss: 0.00116322, Global Avg Loss: 0.02283661, Time: 0.2174 Steps: 24000, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 004304, Sample Num: 68864, Cur Loss: 0.00016587, Cur Avg Loss: 0.00111006, Log Avg loss: 0.00073574, Global Avg Loss: 0.02265395, Time: 0.2170 Steps: 24200, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 004504, Sample Num: 72064, Cur Loss: 0.00069333, Cur Avg Loss: 0.00110415, Log Avg loss: 0.00097711, Global Avg Loss: 0.02247628, Time: 0.2177 Steps: 24400, Updated lr: 0.000091 Training, Epoch: 0005, Batch: 004704, Sample Num: 75264, Cur Loss: 0.00008025, Cur Avg Loss: 0.00108933, Log Avg loss: 0.00075550, Global Avg Loss: 0.02229968, Time: 0.2510 Steps: 24600, Updated lr: 0.000090 Training, Epoch: 0005, Batch: 004904, Sample Num: 78464, Cur Loss: 0.00048387, Cur Avg Loss: 0.00108938, Log Avg loss: 0.00109065, Global Avg Loss: 0.02212864, Time: 0.2262 Steps: 24800, Updated lr: 0.000090 ***** Running evaluation checkpoint-24870 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-24870 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1090.119431, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001598, "eval_total_loss": 1.702993, "eval_acc": 0.999638, "eval_jaccard": 0.985774, "eval_prec": 0.987558, "eval_recall": 0.987222, "eval_f1": 0.986969, "eval_pr_auc": 0.994211, "eval_roc_auc": 0.999293, "eval_fmax": 0.993627, "eval_pmax": 0.997516, "eval_rmax": 0.989767, "eval_tmax": 0.3, "update_flag": true, "test_avg_loss": 0.001622, "test_total_loss": 1.729557, "test_acc": 0.999682, "test_jaccard": 0.986158, "test_prec": 0.987648, "test_recall": 0.987512, "test_f1": 0.987208, "test_pr_auc": 0.99406, "test_roc_auc": 0.999048, "test_fmax": 0.993876, "test_pmax": 0.997338, "test_rmax": 0.990439, "test_tmax": 0.26, "lr": 9.036334275333064e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.022068575390039157, "train_cur_epoch_loss": 5.397453096911022, "train_cur_epoch_avg_loss": 0.0010851333126077648, "train_cur_epoch_time": 1090.1194314956665, "train_cur_epoch_avg_time": 0.2191635366899209, "epoch": 5, "step": 24870} ################################################## Training, Epoch: 0006, Batch: 000130, Sample Num: 2080, Cur Loss: 0.00003814, Cur Avg Loss: 0.00089069, Log Avg loss: 0.00085450, Global Avg Loss: 0.02195845, Time: 0.2266 Steps: 25000, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 000330, Sample Num: 5280, Cur Loss: 0.00039413, Cur Avg Loss: 0.00085234, Log Avg loss: 0.00082741, Global Avg Loss: 0.02179074, Time: 0.2201 Steps: 25200, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 000530, Sample Num: 8480, Cur Loss: 0.00072337, Cur Avg Loss: 0.00093061, Log Avg loss: 0.00105977, Global Avg Loss: 0.02162751, Time: 0.1976 Steps: 25400, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 000730, Sample Num: 11680, Cur Loss: 0.00085801, Cur Avg Loss: 0.00091123, Log Avg loss: 0.00085988, Global Avg Loss: 0.02146526, Time: 0.2172 Steps: 25600, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 000930, Sample Num: 14880, Cur Loss: 0.00023308, Cur Avg Loss: 0.00086742, Log Avg loss: 0.00070749, Global Avg Loss: 0.02130435, Time: 0.3916 Steps: 25800, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 001130, Sample Num: 18080, Cur Loss: 0.00012814, Cur Avg Loss: 0.00087123, Log Avg loss: 0.00088898, Global Avg Loss: 0.02114731, Time: 0.1282 Steps: 26000, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 001330, Sample Num: 21280, Cur Loss: 0.00034601, Cur Avg Loss: 0.00083946, Log Avg loss: 0.00065995, Global Avg Loss: 0.02099091, Time: 0.2199 Steps: 26200, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 001530, Sample Num: 24480, Cur Loss: 0.00023034, Cur Avg Loss: 0.00079547, Log Avg loss: 0.00050293, Global Avg Loss: 0.02083570, Time: 0.2254 Steps: 26400, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 001730, Sample Num: 27680, Cur Loss: 0.00017602, Cur Avg Loss: 0.00078278, Log Avg loss: 0.00068571, Global Avg Loss: 0.02068420, Time: 0.2228 Steps: 26600, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 001930, Sample Num: 30880, Cur Loss: 0.00027880, Cur Avg Loss: 0.00078054, Log Avg loss: 0.00076119, Global Avg Loss: 0.02053552, Time: 0.2255 Steps: 26800, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 002130, Sample Num: 34080, Cur Loss: 0.00010093, Cur Avg Loss: 0.00079339, Log Avg loss: 0.00091736, Global Avg Loss: 0.02039020, Time: 0.2202 Steps: 27000, Updated lr: 0.000090 Training, Epoch: 0006, Batch: 002330, Sample Num: 37280, Cur Loss: 0.00008654, Cur Avg Loss: 0.00077440, Log Avg loss: 0.00057219, Global Avg Loss: 0.02024448, Time: 0.2197 Steps: 27200, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 002530, Sample Num: 40480, Cur Loss: 0.00008018, Cur Avg Loss: 0.00075377, Log Avg loss: 0.00051335, Global Avg Loss: 0.02010046, Time: 0.0856 Steps: 27400, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 002730, Sample Num: 43680, Cur Loss: 0.00572664, Cur Avg Loss: 0.00074336, Log Avg loss: 0.00061169, Global Avg Loss: 0.01995923, Time: 0.2228 Steps: 27600, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 002930, Sample Num: 46880, Cur Loss: 0.00127894, Cur Avg Loss: 0.00072996, Log Avg loss: 0.00054702, Global Avg Loss: 0.01981958, Time: 0.2224 Steps: 27800, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 003130, Sample Num: 50080, Cur Loss: 0.00038396, Cur Avg Loss: 0.00071876, Log Avg loss: 0.00055474, Global Avg Loss: 0.01968197, Time: 0.2207 Steps: 28000, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 003330, Sample Num: 53280, Cur Loss: 0.00002334, Cur Avg Loss: 0.00071557, Log Avg loss: 0.00066560, Global Avg Loss: 0.01954710, Time: 0.2217 Steps: 28200, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 003530, Sample Num: 56480, Cur Loss: 0.00018766, Cur Avg Loss: 0.00071797, Log Avg loss: 0.00075801, Global Avg Loss: 0.01941479, Time: 0.2219 Steps: 28400, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 003730, Sample Num: 59680, Cur Loss: 0.00008680, Cur Avg Loss: 0.00071432, Log Avg loss: 0.00064992, Global Avg Loss: 0.01928356, Time: 0.2206 Steps: 28600, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 003930, Sample Num: 62880, Cur Loss: 0.00116036, Cur Avg Loss: 0.00070726, Log Avg loss: 0.00057563, Global Avg Loss: 0.01915365, Time: 0.2203 Steps: 28800, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 004130, Sample Num: 66080, Cur Loss: 0.00020408, Cur Avg Loss: 0.00071162, Log Avg loss: 0.00079712, Global Avg Loss: 0.01902705, Time: 0.1450 Steps: 29000, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 004330, Sample Num: 69280, Cur Loss: 0.00006800, Cur Avg Loss: 0.00069980, Log Avg loss: 0.00045584, Global Avg Loss: 0.01889985, Time: 0.2206 Steps: 29200, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 004530, Sample Num: 72480, Cur Loss: 0.00005379, Cur Avg Loss: 0.00069655, Log Avg loss: 0.00062614, Global Avg Loss: 0.01877554, Time: 0.2269 Steps: 29400, Updated lr: 0.000089 Training, Epoch: 0006, Batch: 004730, Sample Num: 75680, Cur Loss: 0.00018460, Cur Avg Loss: 0.00068892, Log Avg loss: 0.00051617, Global Avg Loss: 0.01865216, Time: 0.3399 Steps: 29600, Updated lr: 0.000088 Training, Epoch: 0006, Batch: 004930, Sample Num: 78880, Cur Loss: 0.00038767, Cur Avg Loss: 0.00069000, Log Avg loss: 0.00071545, Global Avg Loss: 0.01853178, Time: 0.2199 Steps: 29800, Updated lr: 0.000088 ***** Running evaluation checkpoint-29844 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-29844 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1092.765462, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001554, "eval_total_loss": 1.656715, "eval_acc": 0.999655, "eval_jaccard": 0.986333, "eval_prec": 0.98798, "eval_recall": 0.987779, "eval_f1": 0.987482, "eval_pr_auc": 0.994633, "eval_roc_auc": 0.999357, "eval_fmax": 0.993789, "eval_pmax": 0.997008, "eval_rmax": 0.990591, "eval_tmax": 0.26, "update_flag": true, "test_avg_loss": 0.001624, "test_total_loss": 1.73082, "test_acc": 0.999688, "test_jaccard": 0.98698, "test_prec": 0.98846, "test_recall": 0.988367, "test_f1": 0.988037, "test_pr_auc": 0.994436, "test_roc_auc": 0.999093, "test_fmax": 0.994096, "test_pmax": 0.997643, "test_rmax": 0.990574, "test_tmax": 0.26, "lr": 8.83552684699233e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.018505435622240142, "train_cur_epoch_loss": 3.4307507598609845, "train_cur_epoch_avg_loss": 0.0006897367832450713, "train_cur_epoch_time": 1092.7654616832733, "train_cur_epoch_avg_time": 0.21969550898336818, "epoch": 6, "step": 29844} ################################################## Training, Epoch: 0007, Batch: 000156, Sample Num: 2496, Cur Loss: 0.00005140, Cur Avg Loss: 0.00050578, Log Avg loss: 0.00053981, Global Avg Loss: 0.01841184, Time: 0.2205 Steps: 30000, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 000356, Sample Num: 5696, Cur Loss: 0.00023466, Cur Avg Loss: 0.00058621, Log Avg loss: 0.00064894, Global Avg Loss: 0.01829420, Time: 0.2202 Steps: 30200, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 000556, Sample Num: 8896, Cur Loss: 0.00007031, Cur Avg Loss: 0.00066783, Log Avg loss: 0.00081310, Global Avg Loss: 0.01817920, Time: 0.2241 Steps: 30400, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 000756, Sample Num: 12096, Cur Loss: 0.00026512, Cur Avg Loss: 0.00066532, Log Avg loss: 0.00065835, Global Avg Loss: 0.01806468, Time: 0.1833 Steps: 30600, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 000956, Sample Num: 15296, Cur Loss: 0.00141579, Cur Avg Loss: 0.00063238, Log Avg loss: 0.00050790, Global Avg Loss: 0.01795067, Time: 0.3969 Steps: 30800, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 001156, Sample Num: 18496, Cur Loss: 0.00003401, Cur Avg Loss: 0.00063066, Log Avg loss: 0.00062243, Global Avg Loss: 0.01783888, Time: 0.2198 Steps: 31000, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 001356, Sample Num: 21696, Cur Loss: 0.00012037, Cur Avg Loss: 0.00059793, Log Avg loss: 0.00040871, Global Avg Loss: 0.01772715, Time: 0.2210 Steps: 31200, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 001556, Sample Num: 24896, Cur Loss: 0.00019708, Cur Avg Loss: 0.00056324, Log Avg loss: 0.00032804, Global Avg Loss: 0.01761633, Time: 0.3103 Steps: 31400, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 001756, Sample Num: 28096, Cur Loss: 0.00008781, Cur Avg Loss: 0.00054865, Log Avg loss: 0.00043515, Global Avg Loss: 0.01750758, Time: 0.2278 Steps: 31600, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 001956, Sample Num: 31296, Cur Loss: 0.00009976, Cur Avg Loss: 0.00054345, Log Avg loss: 0.00049779, Global Avg Loss: 0.01740060, Time: 0.2198 Steps: 31800, Updated lr: 0.000088 Training, Epoch: 0007, Batch: 002156, Sample Num: 34496, Cur Loss: 0.00034308, Cur Avg Loss: 0.00056311, Log Avg loss: 0.00075545, Global Avg Loss: 0.01729657, Time: 0.2271 Steps: 32000, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 002356, Sample Num: 37696, Cur Loss: 0.00015054, Cur Avg Loss: 0.00055684, Log Avg loss: 0.00048921, Global Avg Loss: 0.01719218, Time: 0.2168 Steps: 32200, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 002556, Sample Num: 40896, Cur Loss: 0.00002983, Cur Avg Loss: 0.00054334, Log Avg loss: 0.00038437, Global Avg Loss: 0.01708843, Time: 0.2191 Steps: 32400, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 002756, Sample Num: 44096, Cur Loss: 0.00116833, Cur Avg Loss: 0.00054100, Log Avg loss: 0.00051098, Global Avg Loss: 0.01698672, Time: 0.4392 Steps: 32600, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 002956, Sample Num: 47296, Cur Loss: 0.00005624, Cur Avg Loss: 0.00053333, Log Avg loss: 0.00042766, Global Avg Loss: 0.01688575, Time: 0.2254 Steps: 32800, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 003156, Sample Num: 50496, Cur Loss: 0.00446981, Cur Avg Loss: 0.00052738, Log Avg loss: 0.00043949, Global Avg Loss: 0.01678608, Time: 0.2219 Steps: 33000, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 003356, Sample Num: 53696, Cur Loss: 0.00017725, Cur Avg Loss: 0.00052335, Log Avg loss: 0.00045979, Global Avg Loss: 0.01668773, Time: 0.2255 Steps: 33200, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 003556, Sample Num: 56896, Cur Loss: 0.00008844, Cur Avg Loss: 0.00052313, Log Avg loss: 0.00051935, Global Avg Loss: 0.01659091, Time: 0.2204 Steps: 33400, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 003756, Sample Num: 60096, Cur Loss: 0.00271257, Cur Avg Loss: 0.00051986, Log Avg loss: 0.00046184, Global Avg Loss: 0.01649491, Time: 0.2281 Steps: 33600, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 003956, Sample Num: 63296, Cur Loss: 0.00008682, Cur Avg Loss: 0.00051717, Log Avg loss: 0.00046664, Global Avg Loss: 0.01640006, Time: 0.1149 Steps: 33800, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 004156, Sample Num: 66496, Cur Loss: 0.00027376, Cur Avg Loss: 0.00051650, Log Avg loss: 0.00050318, Global Avg Loss: 0.01630655, Time: 0.2192 Steps: 34000, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 004356, Sample Num: 69696, Cur Loss: 0.00007234, Cur Avg Loss: 0.00051057, Log Avg loss: 0.00038728, Global Avg Loss: 0.01621346, Time: 0.2206 Steps: 34200, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 004556, Sample Num: 72896, Cur Loss: 0.00004494, Cur Avg Loss: 0.00051155, Log Avg loss: 0.00053284, Global Avg Loss: 0.01612229, Time: 0.2191 Steps: 34400, Updated lr: 0.000087 Training, Epoch: 0007, Batch: 004756, Sample Num: 76096, Cur Loss: 0.00006150, Cur Avg Loss: 0.00050723, Log Avg loss: 0.00040881, Global Avg Loss: 0.01603146, Time: 0.2246 Steps: 34600, Updated lr: 0.000086 Training, Epoch: 0007, Batch: 004956, Sample Num: 79296, Cur Loss: 0.00004583, Cur Avg Loss: 0.00051181, Log Avg loss: 0.00062072, Global Avg Loss: 0.01594289, Time: 0.2202 Steps: 34800, Updated lr: 0.000086 ***** Running evaluation checkpoint-34818 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-34818 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1094.166798, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001504, "eval_total_loss": 1.603459, "eval_acc": 0.999683, "eval_jaccard": 0.987249, "eval_prec": 0.988873, "eval_recall": 0.98865, "eval_f1": 0.988373, "eval_pr_auc": 0.995077, "eval_roc_auc": 0.999384, "eval_fmax": 0.994116, "eval_pmax": 0.997509, "eval_rmax": 0.990746, "eval_tmax": 0.24, "update_flag": true, "test_avg_loss": 0.001543, "test_total_loss": 1.644992, "test_acc": 0.999731, "test_jaccard": 0.988302, "test_prec": 0.989558, "test_recall": 0.98958, "test_f1": 0.989231, "test_pr_auc": 0.994939, "test_roc_auc": 0.999136, "test_fmax": 0.994493, "test_pmax": 0.997455, "test_rmax": 0.991548, "test_tmax": 0.23, "lr": 8.634719418651595e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.015935043594384395, "train_cur_epoch_loss": 2.5501271591410273, "train_cur_epoch_avg_loss": 0.000512691427249905, "train_cur_epoch_time": 1094.1667976379395, "train_cur_epoch_avg_time": 0.2199772411817329, "epoch": 7, "step": 34818} ################################################## Training, Epoch: 0008, Batch: 000182, Sample Num: 2912, Cur Loss: 0.00060144, Cur Avg Loss: 0.00047048, Log Avg loss: 0.00049623, Global Avg Loss: 0.01585463, Time: 0.2211 Steps: 35000, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 000382, Sample Num: 6112, Cur Loss: 0.00759656, Cur Avg Loss: 0.00049188, Log Avg loss: 0.00051136, Global Avg Loss: 0.01576745, Time: 0.2154 Steps: 35200, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 000582, Sample Num: 9312, Cur Loss: 0.00005123, Cur Avg Loss: 0.00054843, Log Avg loss: 0.00065644, Global Avg Loss: 0.01568208, Time: 0.2210 Steps: 35400, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 000782, Sample Num: 12512, Cur Loss: 0.00012098, Cur Avg Loss: 0.00055638, Log Avg loss: 0.00057949, Global Avg Loss: 0.01559723, Time: 0.1073 Steps: 35600, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 000982, Sample Num: 15712, Cur Loss: 0.00002752, Cur Avg Loss: 0.00051454, Log Avg loss: 0.00035097, Global Avg Loss: 0.01551206, Time: 0.2212 Steps: 35800, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 001182, Sample Num: 18912, Cur Loss: 0.00002603, Cur Avg Loss: 0.00050837, Log Avg loss: 0.00047809, Global Avg Loss: 0.01542853, Time: 0.2160 Steps: 36000, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 001382, Sample Num: 22112, Cur Loss: 0.00001248, Cur Avg Loss: 0.00047644, Log Avg loss: 0.00028768, Global Avg Loss: 0.01534488, Time: 0.2163 Steps: 36200, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 001582, Sample Num: 25312, Cur Loss: 0.00001615, Cur Avg Loss: 0.00044655, Log Avg loss: 0.00024004, Global Avg Loss: 0.01526189, Time: 0.2107 Steps: 36400, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 001782, Sample Num: 28512, Cur Loss: 0.00009041, Cur Avg Loss: 0.00043832, Log Avg loss: 0.00037326, Global Avg Loss: 0.01518053, Time: 0.2263 Steps: 36600, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 001982, Sample Num: 31712, Cur Loss: 0.00015732, Cur Avg Loss: 0.00043649, Log Avg loss: 0.00042018, Global Avg Loss: 0.01510031, Time: 0.2241 Steps: 36800, Updated lr: 0.000086 Training, Epoch: 0008, Batch: 002182, Sample Num: 34912, Cur Loss: 0.00001057, Cur Avg Loss: 0.00044736, Log Avg loss: 0.00055505, Global Avg Loss: 0.01502169, Time: 0.2201 Steps: 37000, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 002382, Sample Num: 38112, Cur Loss: 0.00029587, Cur Avg Loss: 0.00043975, Log Avg loss: 0.00035674, Global Avg Loss: 0.01494285, Time: 0.2228 Steps: 37200, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 002582, Sample Num: 41312, Cur Loss: 0.00013824, Cur Avg Loss: 0.00043035, Log Avg loss: 0.00031836, Global Avg Loss: 0.01486464, Time: 0.3941 Steps: 37400, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 002782, Sample Num: 44512, Cur Loss: 0.00007313, Cur Avg Loss: 0.00042936, Log Avg loss: 0.00041661, Global Avg Loss: 0.01478779, Time: 0.2210 Steps: 37600, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 002982, Sample Num: 47712, Cur Loss: 0.00006282, Cur Avg Loss: 0.00043688, Log Avg loss: 0.00054152, Global Avg Loss: 0.01471241, Time: 0.2193 Steps: 37800, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 003182, Sample Num: 50912, Cur Loss: 0.00000150, Cur Avg Loss: 0.00042762, Log Avg loss: 0.00028946, Global Avg Loss: 0.01463650, Time: 0.2262 Steps: 38000, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 003382, Sample Num: 54112, Cur Loss: 0.00006346, Cur Avg Loss: 0.00042378, Log Avg loss: 0.00036281, Global Avg Loss: 0.01456177, Time: 0.2200 Steps: 38200, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 003582, Sample Num: 57312, Cur Loss: 0.00012670, Cur Avg Loss: 0.00042108, Log Avg loss: 0.00037543, Global Avg Loss: 0.01448788, Time: 0.2222 Steps: 38400, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 003782, Sample Num: 60512, Cur Loss: 0.00000900, Cur Avg Loss: 0.00041722, Log Avg loss: 0.00034806, Global Avg Loss: 0.01441462, Time: 0.2199 Steps: 38600, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 003982, Sample Num: 63712, Cur Loss: 0.00003298, Cur Avg Loss: 0.00041766, Log Avg loss: 0.00042592, Global Avg Loss: 0.01434251, Time: 0.2265 Steps: 38800, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 004182, Sample Num: 66912, Cur Loss: 0.00000947, Cur Avg Loss: 0.00041555, Log Avg loss: 0.00037350, Global Avg Loss: 0.01427088, Time: 0.1193 Steps: 39000, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 004382, Sample Num: 70112, Cur Loss: 0.00012359, Cur Avg Loss: 0.00041197, Log Avg loss: 0.00033713, Global Avg Loss: 0.01419979, Time: 0.2212 Steps: 39200, Updated lr: 0.000085 Training, Epoch: 0008, Batch: 004582, Sample Num: 73312, Cur Loss: 0.00005116, Cur Avg Loss: 0.00041313, Log Avg loss: 0.00043856, Global Avg Loss: 0.01412993, Time: 0.2292 Steps: 39400, Updated lr: 0.000084 Training, Epoch: 0008, Batch: 004782, Sample Num: 76512, Cur Loss: 0.00021095, Cur Avg Loss: 0.00041475, Log Avg loss: 0.00045196, Global Avg Loss: 0.01406085, Time: 0.2195 Steps: 39600, Updated lr: 0.000084 ***** Running evaluation checkpoint-39792 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-39792 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1095.150860, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001502, "eval_total_loss": 1.600848, "eval_acc": 0.99969, "eval_jaccard": 0.987707, "eval_prec": 0.98899, "eval_recall": 0.989441, "eval_f1": 0.988829, "eval_pr_auc": 0.995251, "eval_roc_auc": 0.999384, "eval_fmax": 0.994327, "eval_pmax": 0.997273, "eval_rmax": 0.991398, "eval_tmax": 0.26, "update_flag": true, "test_avg_loss": 0.001556, "test_total_loss": 1.659224, "test_acc": 0.999719, "test_jaccard": 0.988078, "test_prec": 0.989177, "test_recall": 0.989683, "test_f1": 0.989081, "test_pr_auc": 0.995016, "test_roc_auc": 0.999118, "test_fmax": 0.994451, "test_pmax": 0.997326, "test_rmax": 0.991593, "test_tmax": 0.24, "lr": 8.43391199031086e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.013995557162336178, "train_cur_epoch_loss": 2.0848627344053057, "train_cur_epoch_avg_loss": 0.0004191521379986541, "train_cur_epoch_time": 1095.150859594345, "train_cur_epoch_avg_time": 0.2201750823470738, "epoch": 8, "step": 39792} ################################################## Training, Epoch: 0009, Batch: 000008, Sample Num: 128, Cur Loss: 0.00004202, Cur Avg Loss: 0.00021133, Log Avg loss: 0.00051602, Global Avg Loss: 0.01399279, Time: 0.1029 Steps: 39800, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 000208, Sample Num: 3328, Cur Loss: 0.00045639, Cur Avg Loss: 0.00034862, Log Avg loss: 0.00035411, Global Avg Loss: 0.01392459, Time: 0.3920 Steps: 40000, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 000408, Sample Num: 6528, Cur Loss: 0.00008032, Cur Avg Loss: 0.00038776, Log Avg loss: 0.00042847, Global Avg Loss: 0.01385745, Time: 0.2264 Steps: 40200, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 000608, Sample Num: 9728, Cur Loss: 0.00008837, Cur Avg Loss: 0.00043300, Log Avg loss: 0.00052529, Global Avg Loss: 0.01379145, Time: 0.2575 Steps: 40400, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 000808, Sample Num: 12928, Cur Loss: 0.00007961, Cur Avg Loss: 0.00041912, Log Avg loss: 0.00037693, Global Avg Loss: 0.01372537, Time: 0.0848 Steps: 40600, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 001008, Sample Num: 16128, Cur Loss: 0.00001483, Cur Avg Loss: 0.00038949, Log Avg loss: 0.00026976, Global Avg Loss: 0.01365941, Time: 0.2203 Steps: 40800, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 001208, Sample Num: 19328, Cur Loss: 0.00003091, Cur Avg Loss: 0.00039130, Log Avg loss: 0.00040045, Global Avg Loss: 0.01359473, Time: 0.2186 Steps: 41000, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 001408, Sample Num: 22528, Cur Loss: 0.00002094, Cur Avg Loss: 0.00036177, Log Avg loss: 0.00018341, Global Avg Loss: 0.01352963, Time: 0.2510 Steps: 41200, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 001608, Sample Num: 25728, Cur Loss: 0.00005283, Cur Avg Loss: 0.00034056, Log Avg loss: 0.00019124, Global Avg Loss: 0.01346519, Time: 0.2195 Steps: 41400, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 001808, Sample Num: 28928, Cur Loss: 0.00002578, Cur Avg Loss: 0.00034034, Log Avg loss: 0.00033852, Global Avg Loss: 0.01340208, Time: 0.2087 Steps: 41600, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 002008, Sample Num: 32128, Cur Loss: 0.00005389, Cur Avg Loss: 0.00033919, Log Avg loss: 0.00032880, Global Avg Loss: 0.01333953, Time: 0.1235 Steps: 41800, Updated lr: 0.000084 Training, Epoch: 0009, Batch: 002208, Sample Num: 35328, Cur Loss: 0.00002819, Cur Avg Loss: 0.00036110, Log Avg loss: 0.00058114, Global Avg Loss: 0.01327877, Time: 0.2398 Steps: 42000, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 002408, Sample Num: 38528, Cur Loss: 0.00008618, Cur Avg Loss: 0.00035109, Log Avg loss: 0.00024049, Global Avg Loss: 0.01321698, Time: 0.2206 Steps: 42200, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 002608, Sample Num: 41728, Cur Loss: 0.00008533, Cur Avg Loss: 0.00035227, Log Avg loss: 0.00036649, Global Avg Loss: 0.01315637, Time: 0.2198 Steps: 42400, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 002808, Sample Num: 44928, Cur Loss: 0.00001880, Cur Avg Loss: 0.00034720, Log Avg loss: 0.00028119, Global Avg Loss: 0.01309592, Time: 0.2105 Steps: 42600, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 003008, Sample Num: 48128, Cur Loss: 0.00101062, Cur Avg Loss: 0.00035589, Log Avg loss: 0.00047789, Global Avg Loss: 0.01303696, Time: 0.1530 Steps: 42800, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 003208, Sample Num: 51328, Cur Loss: 0.00002216, Cur Avg Loss: 0.00035050, Log Avg loss: 0.00026935, Global Avg Loss: 0.01297757, Time: 0.2190 Steps: 43000, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 003408, Sample Num: 54528, Cur Loss: 0.00001972, Cur Avg Loss: 0.00034979, Log Avg loss: 0.00033836, Global Avg Loss: 0.01291906, Time: 0.2144 Steps: 43200, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 003608, Sample Num: 57728, Cur Loss: 0.00001894, Cur Avg Loss: 0.00035011, Log Avg loss: 0.00035564, Global Avg Loss: 0.01286116, Time: 0.2212 Steps: 43400, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 003808, Sample Num: 60928, Cur Loss: 0.00001526, Cur Avg Loss: 0.00034617, Log Avg loss: 0.00027501, Global Avg Loss: 0.01280343, Time: 0.2435 Steps: 43600, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 004008, Sample Num: 64128, Cur Loss: 0.00457596, Cur Avg Loss: 0.00034849, Log Avg loss: 0.00039281, Global Avg Loss: 0.01274676, Time: 0.2312 Steps: 43800, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 004208, Sample Num: 67328, Cur Loss: 0.00017030, Cur Avg Loss: 0.00034535, Log Avg loss: 0.00028232, Global Avg Loss: 0.01269010, Time: 0.2343 Steps: 44000, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 004408, Sample Num: 70528, Cur Loss: 0.00030485, Cur Avg Loss: 0.00034003, Log Avg loss: 0.00022811, Global Avg Loss: 0.01263371, Time: 0.2226 Steps: 44200, Updated lr: 0.000083 Training, Epoch: 0009, Batch: 004608, Sample Num: 73728, Cur Loss: 0.00003616, Cur Avg Loss: 0.00034142, Log Avg loss: 0.00037211, Global Avg Loss: 0.01257848, Time: 0.2371 Steps: 44400, Updated lr: 0.000082 Training, Epoch: 0009, Batch: 004808, Sample Num: 76928, Cur Loss: 0.00002209, Cur Avg Loss: 0.00034331, Log Avg loss: 0.00038688, Global Avg Loss: 0.01252381, Time: 0.0841 Steps: 44600, Updated lr: 0.000082 ***** Running evaluation checkpoint-44766 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-44766 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1097.239204, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001641, "eval_total_loss": 1.749652, "eval_acc": 0.999684, "eval_jaccard": 0.987538, "eval_prec": 0.989027, "eval_recall": 0.988883, "eval_f1": 0.988584, "eval_pr_auc": 0.994812, "eval_roc_auc": 0.999372, "eval_fmax": 0.994284, "eval_pmax": 0.996823, "eval_rmax": 0.991759, "eval_tmax": 0.14, "update_flag": false, "test_avg_loss": 0.001694, "test_total_loss": 1.805313, "test_acc": 0.999719, "test_jaccard": 0.988234, "test_prec": 0.989514, "test_recall": 0.989489, "test_f1": 0.98917, "test_pr_auc": 0.995051, "test_roc_auc": 0.999123, "test_fmax": 0.994493, "test_pmax": 0.997978, "test_rmax": 0.991033, "test_tmax": 0.28, "lr": 8.233104561970125e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.012478781554741695, "train_cur_epoch_loss": 1.713924475885534, "train_cur_epoch_avg_loss": 0.00034457669398583313, "train_cur_epoch_time": 1097.2392041683197, "train_cur_epoch_avg_time": 0.2205949344930277, "epoch": 9, "step": 44766} ################################################## Training, Epoch: 0010, Batch: 000034, Sample Num: 544, Cur Loss: 0.00008873, Cur Avg Loss: 0.00010255, Log Avg loss: 0.00033381, Global Avg Loss: 0.01246939, Time: 0.0898 Steps: 44800, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 000234, Sample Num: 3744, Cur Loss: 0.00019048, Cur Avg Loss: 0.00026669, Log Avg loss: 0.00029460, Global Avg Loss: 0.01241528, Time: 0.2227 Steps: 45000, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 000434, Sample Num: 6944, Cur Loss: 0.00002989, Cur Avg Loss: 0.00034026, Log Avg loss: 0.00042633, Global Avg Loss: 0.01236223, Time: 0.2148 Steps: 45200, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 000634, Sample Num: 10144, Cur Loss: 0.00004121, Cur Avg Loss: 0.00035842, Log Avg loss: 0.00039782, Global Avg Loss: 0.01230952, Time: 0.2195 Steps: 45400, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 000834, Sample Num: 13344, Cur Loss: 0.00035895, Cur Avg Loss: 0.00036437, Log Avg loss: 0.00038326, Global Avg Loss: 0.01225722, Time: 0.2280 Steps: 45600, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 001034, Sample Num: 16544, Cur Loss: 0.00070126, Cur Avg Loss: 0.00034408, Log Avg loss: 0.00025946, Global Avg Loss: 0.01220482, Time: 0.2201 Steps: 45800, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 001234, Sample Num: 19744, Cur Loss: 0.00001032, Cur Avg Loss: 0.00033456, Log Avg loss: 0.00028536, Global Avg Loss: 0.01215300, Time: 0.2212 Steps: 46000, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 001434, Sample Num: 22944, Cur Loss: 0.00002019, Cur Avg Loss: 0.00031377, Log Avg loss: 0.00018548, Global Avg Loss: 0.01210119, Time: 0.2202 Steps: 46200, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 001634, Sample Num: 26144, Cur Loss: 0.00002146, Cur Avg Loss: 0.00029344, Log Avg loss: 0.00014765, Global Avg Loss: 0.01204967, Time: 0.2189 Steps: 46400, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 001834, Sample Num: 29344, Cur Loss: 0.00002805, Cur Avg Loss: 0.00029021, Log Avg loss: 0.00026383, Global Avg Loss: 0.01199909, Time: 0.2259 Steps: 46600, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 002034, Sample Num: 32544, Cur Loss: 0.00850793, Cur Avg Loss: 0.00029161, Log Avg loss: 0.00030441, Global Avg Loss: 0.01194911, Time: 0.2561 Steps: 46800, Updated lr: 0.000082 Training, Epoch: 0010, Batch: 002234, Sample Num: 35744, Cur Loss: 0.00004079, Cur Avg Loss: 0.00029983, Log Avg loss: 0.00038349, Global Avg Loss: 0.01189989, Time: 0.2265 Steps: 47000, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 002434, Sample Num: 38944, Cur Loss: 0.00001365, Cur Avg Loss: 0.00029754, Log Avg loss: 0.00027198, Global Avg Loss: 0.01185062, Time: 0.2252 Steps: 47200, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 002634, Sample Num: 42144, Cur Loss: 0.00000633, Cur Avg Loss: 0.00029503, Log Avg loss: 0.00026441, Global Avg Loss: 0.01180173, Time: 0.2196 Steps: 47400, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 002834, Sample Num: 45344, Cur Loss: 0.00002626, Cur Avg Loss: 0.00029056, Log Avg loss: 0.00023180, Global Avg Loss: 0.01175312, Time: 0.2273 Steps: 47600, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 003034, Sample Num: 48544, Cur Loss: 0.00350842, Cur Avg Loss: 0.00029476, Log Avg loss: 0.00035413, Global Avg Loss: 0.01170543, Time: 0.2262 Steps: 47800, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 003234, Sample Num: 51744, Cur Loss: 0.00010238, Cur Avg Loss: 0.00028872, Log Avg loss: 0.00019722, Global Avg Loss: 0.01165748, Time: 0.2290 Steps: 48000, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 003434, Sample Num: 54944, Cur Loss: 0.00002814, Cur Avg Loss: 0.00028598, Log Avg loss: 0.00024157, Global Avg Loss: 0.01161011, Time: 0.2269 Steps: 48200, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 003634, Sample Num: 58144, Cur Loss: 0.00001000, Cur Avg Loss: 0.00029064, Log Avg loss: 0.00037069, Global Avg Loss: 0.01156366, Time: 0.2262 Steps: 48400, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 003834, Sample Num: 61344, Cur Loss: 0.00001470, Cur Avg Loss: 0.00028513, Log Avg loss: 0.00018507, Global Avg Loss: 0.01151684, Time: 0.2249 Steps: 48600, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 004034, Sample Num: 64544, Cur Loss: 0.00009237, Cur Avg Loss: 0.00028788, Log Avg loss: 0.00034060, Global Avg Loss: 0.01147103, Time: 0.2224 Steps: 48800, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 004234, Sample Num: 67744, Cur Loss: 0.00028122, Cur Avg Loss: 0.00028491, Log Avg loss: 0.00022495, Global Avg Loss: 0.01142513, Time: 0.2246 Steps: 49000, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 004434, Sample Num: 70944, Cur Loss: 0.00006702, Cur Avg Loss: 0.00027888, Log Avg loss: 0.00015133, Global Avg Loss: 0.01137930, Time: 0.2198 Steps: 49200, Updated lr: 0.000081 Training, Epoch: 0010, Batch: 004634, Sample Num: 74144, Cur Loss: 0.00004481, Cur Avg Loss: 0.00027982, Log Avg loss: 0.00030054, Global Avg Loss: 0.01133445, Time: 0.2215 Steps: 49400, Updated lr: 0.000080 Training, Epoch: 0010, Batch: 004834, Sample Num: 77344, Cur Loss: 0.00001596, Cur Avg Loss: 0.00028534, Log Avg loss: 0.00041329, Global Avg Loss: 0.01129041, Time: 0.2138 Steps: 49600, Updated lr: 0.000080 ***** Running evaluation checkpoint-49740 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-49740 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1098.546064, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001534, "eval_total_loss": 1.635344, "eval_acc": 0.999708, "eval_jaccard": 0.988001, "eval_prec": 0.989018, "eval_recall": 0.989607, "eval_f1": 0.988986, "eval_pr_auc": 0.995093, "eval_roc_auc": 0.999404, "eval_fmax": 0.994811, "eval_pmax": 0.997386, "eval_rmax": 0.99225, "eval_tmax": 0.21, "update_flag": true, "test_avg_loss": 0.001626, "test_total_loss": 1.733165, "test_acc": 0.999739, "test_jaccard": 0.988544, "test_prec": 0.989549, "test_recall": 0.989915, "test_f1": 0.989415, "test_pr_auc": 0.995255, "test_roc_auc": 0.999121, "test_fmax": 0.994805, "test_pmax": 0.996599, "test_rmax": 0.993017, "test_tmax": 0.09, "lr": 8.03229713362939e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.011259458773455178, "train_cur_epoch_loss": 1.420344312093789, "train_cur_epoch_avg_loss": 0.0002855537418765157, "train_cur_epoch_time": 1098.546064376831, "train_cur_epoch_avg_time": 0.22085767277378993, "epoch": 10, "step": 49740} ################################################## Training, Epoch: 0011, Batch: 000060, Sample Num: 960, Cur Loss: 0.00003438, Cur Avg Loss: 0.00020874, Log Avg loss: 0.00026766, Global Avg Loss: 0.01124614, Time: 0.2188 Steps: 49800, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 000260, Sample Num: 4160, Cur Loss: 0.00005651, Cur Avg Loss: 0.00026966, Log Avg loss: 0.00028794, Global Avg Loss: 0.01120231, Time: 0.2394 Steps: 50000, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 000460, Sample Num: 7360, Cur Loss: 0.00083688, Cur Avg Loss: 0.00032365, Log Avg loss: 0.00039385, Global Avg Loss: 0.01115925, Time: 0.0854 Steps: 50200, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 000660, Sample Num: 10560, Cur Loss: 0.00391232, Cur Avg Loss: 0.00032204, Log Avg loss: 0.00031834, Global Avg Loss: 0.01111623, Time: 0.2341 Steps: 50400, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 000860, Sample Num: 13760, Cur Loss: 0.00003516, Cur Avg Loss: 0.00033352, Log Avg loss: 0.00037141, Global Avg Loss: 0.01107376, Time: 0.2307 Steps: 50600, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 001060, Sample Num: 16960, Cur Loss: 0.00005574, Cur Avg Loss: 0.00032314, Log Avg loss: 0.00027851, Global Avg Loss: 0.01103126, Time: 0.2258 Steps: 50800, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 001260, Sample Num: 20160, Cur Loss: 0.00001364, Cur Avg Loss: 0.00029480, Log Avg loss: 0.00014458, Global Avg Loss: 0.01098857, Time: 0.2201 Steps: 51000, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 001460, Sample Num: 23360, Cur Loss: 0.00000760, Cur Avg Loss: 0.00028075, Log Avg loss: 0.00019222, Global Avg Loss: 0.01094639, Time: 0.2248 Steps: 51200, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 001660, Sample Num: 26560, Cur Loss: 0.00001087, Cur Avg Loss: 0.00026801, Log Avg loss: 0.00017504, Global Avg Loss: 0.01090448, Time: 0.2195 Steps: 51400, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 001860, Sample Num: 29760, Cur Loss: 0.00007687, Cur Avg Loss: 0.00026535, Log Avg loss: 0.00024324, Global Avg Loss: 0.01086316, Time: 0.2187 Steps: 51600, Updated lr: 0.000080 Training, Epoch: 0011, Batch: 002060, Sample Num: 32960, Cur Loss: 0.00001380, Cur Avg Loss: 0.00026421, Log Avg loss: 0.00025366, Global Avg Loss: 0.01082220, Time: 0.3142 Steps: 51800, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 002260, Sample Num: 36160, Cur Loss: 0.00637013, Cur Avg Loss: 0.00026696, Log Avg loss: 0.00029530, Global Avg Loss: 0.01078171, Time: 0.2167 Steps: 52000, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 002460, Sample Num: 39360, Cur Loss: 0.00001423, Cur Avg Loss: 0.00025753, Log Avg loss: 0.00015095, Global Avg Loss: 0.01074098, Time: 0.2254 Steps: 52200, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 002660, Sample Num: 42560, Cur Loss: 0.00000753, Cur Avg Loss: 0.00026043, Log Avg loss: 0.00029606, Global Avg Loss: 0.01070111, Time: 0.2254 Steps: 52400, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 002860, Sample Num: 45760, Cur Loss: 0.00046080, Cur Avg Loss: 0.00025900, Log Avg loss: 0.00023995, Global Avg Loss: 0.01066133, Time: 0.2200 Steps: 52600, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 003060, Sample Num: 48960, Cur Loss: 0.00054031, Cur Avg Loss: 0.00026532, Log Avg loss: 0.00035570, Global Avg Loss: 0.01062230, Time: 0.2197 Steps: 52800, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 003260, Sample Num: 52160, Cur Loss: 0.00000480, Cur Avg Loss: 0.00026107, Log Avg loss: 0.00019615, Global Avg Loss: 0.01058295, Time: 0.2257 Steps: 53000, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 003460, Sample Num: 55360, Cur Loss: 0.00001580, Cur Avg Loss: 0.00026031, Log Avg loss: 0.00024778, Global Avg Loss: 0.01054410, Time: 0.2274 Steps: 53200, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 003660, Sample Num: 58560, Cur Loss: 0.00001021, Cur Avg Loss: 0.00026306, Log Avg loss: 0.00031073, Global Avg Loss: 0.01050577, Time: 0.2195 Steps: 53400, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 003860, Sample Num: 61760, Cur Loss: 0.00002463, Cur Avg Loss: 0.00025803, Log Avg loss: 0.00016591, Global Avg Loss: 0.01046719, Time: 0.2196 Steps: 53600, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 004060, Sample Num: 64960, Cur Loss: 0.00246325, Cur Avg Loss: 0.00026299, Log Avg loss: 0.00035873, Global Avg Loss: 0.01042961, Time: 0.2210 Steps: 53800, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 004260, Sample Num: 68160, Cur Loss: 0.00014158, Cur Avg Loss: 0.00026210, Log Avg loss: 0.00024409, Global Avg Loss: 0.01039189, Time: 0.3930 Steps: 54000, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 004460, Sample Num: 71360, Cur Loss: 0.00110754, Cur Avg Loss: 0.00026046, Log Avg loss: 0.00022553, Global Avg Loss: 0.01035438, Time: 0.2195 Steps: 54200, Updated lr: 0.000079 Training, Epoch: 0011, Batch: 004660, Sample Num: 74560, Cur Loss: 0.00000873, Cur Avg Loss: 0.00026195, Log Avg loss: 0.00029523, Global Avg Loss: 0.01031739, Time: 0.2274 Steps: 54400, Updated lr: 0.000078 Training, Epoch: 0011, Batch: 004860, Sample Num: 77760, Cur Loss: 0.00002815, Cur Avg Loss: 0.00026534, Log Avg loss: 0.00034427, Global Avg Loss: 0.01028086, Time: 0.2199 Steps: 54600, Updated lr: 0.000078 ***** Running evaluation checkpoint-54714 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-54714 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1096.665726, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001592, "eval_total_loss": 1.697039, "eval_acc": 0.999712, "eval_jaccard": 0.98816, "eval_prec": 0.989252, "eval_recall": 0.989724, "eval_f1": 0.989155, "eval_pr_auc": 0.995265, "eval_roc_auc": 0.999405, "eval_fmax": 0.99441, "eval_pmax": 0.997051, "eval_rmax": 0.991783, "eval_tmax": 0.18, "update_flag": true, "test_avg_loss": 0.001652, "test_total_loss": 1.760947, "test_acc": 0.999752, "test_jaccard": 0.98928, "test_prec": 0.990223, "test_recall": 0.990704, "test_f1": 0.990141, "test_pr_auc": 0.995164, "test_roc_auc": 0.999135, "test_fmax": 0.994773, "test_pmax": 0.99815, "test_rmax": 0.99142, "test_tmax": 0.33, "lr": 7.831489705288656e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.010260080134588852, "train_cur_epoch_loss": 1.3245450922335635, "train_cur_epoch_avg_loss": 0.0002662937459255254, "train_cur_epoch_time": 1096.665726184845, "train_cur_epoch_avg_time": 0.22047963936164958, "epoch": 11, "step": 54714} ################################################## Training, Epoch: 0012, Batch: 000086, Sample Num: 1376, Cur Loss: 0.00008149, Cur Avg Loss: 0.00031322, Log Avg loss: 0.00030963, Global Avg Loss: 0.01024447, Time: 0.2201 Steps: 54800, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 000286, Sample Num: 4576, Cur Loss: 0.00000364, Cur Avg Loss: 0.00021910, Log Avg loss: 0.00017863, Global Avg Loss: 0.01020787, Time: 0.2177 Steps: 55000, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 000486, Sample Num: 7776, Cur Loss: 0.00003954, Cur Avg Loss: 0.00024347, Log Avg loss: 0.00027832, Global Avg Loss: 0.01017189, Time: 0.2169 Steps: 55200, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 000686, Sample Num: 10976, Cur Loss: 0.00000846, Cur Avg Loss: 0.00025689, Log Avg loss: 0.00028949, Global Avg Loss: 0.01013621, Time: 0.2128 Steps: 55400, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 000886, Sample Num: 14176, Cur Loss: 0.00003847, Cur Avg Loss: 0.00027439, Log Avg loss: 0.00033443, Global Avg Loss: 0.01010096, Time: 0.2215 Steps: 55600, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 001086, Sample Num: 17376, Cur Loss: 0.00003514, Cur Avg Loss: 0.00025705, Log Avg loss: 0.00018020, Global Avg Loss: 0.01006540, Time: 0.2197 Steps: 55800, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 001286, Sample Num: 20576, Cur Loss: 0.00000951, Cur Avg Loss: 0.00023765, Log Avg loss: 0.00013232, Global Avg Loss: 0.01002992, Time: 0.2226 Steps: 56000, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 001486, Sample Num: 23776, Cur Loss: 0.00001394, Cur Avg Loss: 0.00022746, Log Avg loss: 0.00016192, Global Avg Loss: 0.00999480, Time: 0.2191 Steps: 56200, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 001686, Sample Num: 26976, Cur Loss: 0.00001236, Cur Avg Loss: 0.00022436, Log Avg loss: 0.00020134, Global Avg Loss: 0.00996008, Time: 0.2130 Steps: 56400, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 001886, Sample Num: 30176, Cur Loss: 0.00000402, Cur Avg Loss: 0.00022037, Log Avg loss: 0.00018679, Global Avg Loss: 0.00992554, Time: 0.2198 Steps: 56600, Updated lr: 0.000078 Training, Epoch: 0012, Batch: 002086, Sample Num: 33376, Cur Loss: 0.00078472, Cur Avg Loss: 0.00022247, Log Avg loss: 0.00024221, Global Avg Loss: 0.00989145, Time: 0.3945 Steps: 56800, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 002286, Sample Num: 36576, Cur Loss: 0.00001013, Cur Avg Loss: 0.00022542, Log Avg loss: 0.00025616, Global Avg Loss: 0.00985764, Time: 0.2256 Steps: 57000, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 002486, Sample Num: 39776, Cur Loss: 0.00003454, Cur Avg Loss: 0.00021684, Log Avg loss: 0.00011884, Global Avg Loss: 0.00982359, Time: 0.2197 Steps: 57200, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 002686, Sample Num: 42976, Cur Loss: 0.00003895, Cur Avg Loss: 0.00022197, Log Avg loss: 0.00028576, Global Avg Loss: 0.00979035, Time: 0.2203 Steps: 57400, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 002886, Sample Num: 46176, Cur Loss: 0.00002593, Cur Avg Loss: 0.00022771, Log Avg loss: 0.00030470, Global Avg Loss: 0.00975742, Time: 0.2231 Steps: 57600, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 003086, Sample Num: 49376, Cur Loss: 0.00000149, Cur Avg Loss: 0.00022769, Log Avg loss: 0.00022746, Global Avg Loss: 0.00972444, Time: 0.2174 Steps: 57800, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 003286, Sample Num: 52576, Cur Loss: 0.00264922, Cur Avg Loss: 0.00022228, Log Avg loss: 0.00013881, Global Avg Loss: 0.00969139, Time: 0.2184 Steps: 58000, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 003486, Sample Num: 55776, Cur Loss: 0.00000168, Cur Avg Loss: 0.00022280, Log Avg loss: 0.00023130, Global Avg Loss: 0.00965888, Time: 0.2191 Steps: 58200, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 003686, Sample Num: 58976, Cur Loss: 0.00388415, Cur Avg Loss: 0.00022905, Log Avg loss: 0.00033804, Global Avg Loss: 0.00962696, Time: 0.2209 Steps: 58400, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 003886, Sample Num: 62176, Cur Loss: 0.00006622, Cur Avg Loss: 0.00022244, Log Avg loss: 0.00010057, Global Avg Loss: 0.00959444, Time: 0.2202 Steps: 58600, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 004086, Sample Num: 65376, Cur Loss: 0.00005190, Cur Avg Loss: 0.00022633, Log Avg loss: 0.00030188, Global Avg Loss: 0.00956284, Time: 0.2230 Steps: 58800, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 004286, Sample Num: 68576, Cur Loss: 0.00005320, Cur Avg Loss: 0.00022477, Log Avg loss: 0.00019289, Global Avg Loss: 0.00953107, Time: 0.2271 Steps: 59000, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 004486, Sample Num: 71776, Cur Loss: 0.00178078, Cur Avg Loss: 0.00022464, Log Avg loss: 0.00022181, Global Avg Loss: 0.00949962, Time: 0.2189 Steps: 59200, Updated lr: 0.000077 Training, Epoch: 0012, Batch: 004686, Sample Num: 74976, Cur Loss: 0.00006764, Cur Avg Loss: 0.00022334, Log Avg loss: 0.00019438, Global Avg Loss: 0.00946829, Time: 0.2262 Steps: 59400, Updated lr: 0.000076 Training, Epoch: 0012, Batch: 004886, Sample Num: 78176, Cur Loss: 0.00028941, Cur Avg Loss: 0.00022570, Log Avg loss: 0.00028089, Global Avg Loss: 0.00943746, Time: 0.2265 Steps: 59600, Updated lr: 0.000076 ***** Running evaluation checkpoint-59688 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-59688 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1098.973456, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001684, "eval_total_loss": 1.794764, "eval_acc": 0.999711, "eval_jaccard": 0.987965, "eval_prec": 0.988985, "eval_recall": 0.989458, "eval_f1": 0.988894, "eval_pr_auc": 0.99529, "eval_roc_auc": 0.999395, "eval_fmax": 0.994515, "eval_pmax": 0.997394, "eval_rmax": 0.991652, "eval_tmax": 0.17, "update_flag": false, "test_avg_loss": 0.001711, "test_total_loss": 1.82413, "test_acc": 0.999755, "test_jaccard": 0.989091, "test_prec": 0.990023, "test_recall": 0.990433, "test_f1": 0.989922, "test_pr_auc": 0.995127, "test_roc_auc": 0.99914, "test_fmax": 0.994817, "test_pmax": 0.997511, "test_rmax": 0.992137, "test_tmax": 0.16, "lr": 7.630682276947921e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.00942401154776024, "train_cur_epoch_loss": 1.1303767788188281, "train_cur_epoch_avg_loss": 0.00022725709264552235, "train_cur_epoch_time": 1098.9734561443329, "train_cur_epoch_avg_time": 0.22094359793814494, "epoch": 12, "step": 59688} ################################################## Training, Epoch: 0013, Batch: 000112, Sample Num: 1792, Cur Loss: 0.00009461, Cur Avg Loss: 0.00027100, Log Avg loss: 0.00028979, Global Avg Loss: 0.00940687, Time: 0.2290 Steps: 59800, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 000312, Sample Num: 4992, Cur Loss: 0.00007900, Cur Avg Loss: 0.00025653, Log Avg loss: 0.00024843, Global Avg Loss: 0.00937634, Time: 0.2283 Steps: 60000, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 000512, Sample Num: 8192, Cur Loss: 0.00000985, Cur Avg Loss: 0.00029473, Log Avg loss: 0.00035432, Global Avg Loss: 0.00934637, Time: 0.1051 Steps: 60200, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 000712, Sample Num: 11392, Cur Loss: 0.00000707, Cur Avg Loss: 0.00027216, Log Avg loss: 0.00021438, Global Avg Loss: 0.00931613, Time: 0.0853 Steps: 60400, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 000912, Sample Num: 14592, Cur Loss: 0.00002288, Cur Avg Loss: 0.00027399, Log Avg loss: 0.00028052, Global Avg Loss: 0.00928631, Time: 0.2153 Steps: 60600, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 001112, Sample Num: 17792, Cur Loss: 0.00001323, Cur Avg Loss: 0.00025370, Log Avg loss: 0.00016114, Global Avg Loss: 0.00925629, Time: 0.0855 Steps: 60800, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 001312, Sample Num: 20992, Cur Loss: 0.00000396, Cur Avg Loss: 0.00024214, Log Avg loss: 0.00017786, Global Avg Loss: 0.00922653, Time: 0.2516 Steps: 61000, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 001512, Sample Num: 24192, Cur Loss: 0.00001267, Cur Avg Loss: 0.00022310, Log Avg loss: 0.00009826, Global Avg Loss: 0.00919670, Time: 0.2270 Steps: 61200, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 001712, Sample Num: 27392, Cur Loss: 0.00006356, Cur Avg Loss: 0.00021909, Log Avg loss: 0.00018872, Global Avg Loss: 0.00916735, Time: 0.2239 Steps: 61400, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 001912, Sample Num: 30592, Cur Loss: 0.00006289, Cur Avg Loss: 0.00021290, Log Avg loss: 0.00015997, Global Avg Loss: 0.00913811, Time: 0.2252 Steps: 61600, Updated lr: 0.000076 Training, Epoch: 0013, Batch: 002112, Sample Num: 33792, Cur Loss: 0.00234155, Cur Avg Loss: 0.00021590, Log Avg loss: 0.00024455, Global Avg Loss: 0.00910933, Time: 0.6927 Steps: 61800, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 002312, Sample Num: 36992, Cur Loss: 0.00003400, Cur Avg Loss: 0.00022519, Log Avg loss: 0.00032332, Global Avg Loss: 0.00908098, Time: 0.2151 Steps: 62000, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 002512, Sample Num: 40192, Cur Loss: 0.00005186, Cur Avg Loss: 0.00021563, Log Avg loss: 0.00010506, Global Avg Loss: 0.00905212, Time: 0.2252 Steps: 62200, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 002712, Sample Num: 43392, Cur Loss: 0.00001528, Cur Avg Loss: 0.00021789, Log Avg loss: 0.00024629, Global Avg Loss: 0.00902390, Time: 0.2266 Steps: 62400, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 002912, Sample Num: 46592, Cur Loss: 0.00004329, Cur Avg Loss: 0.00022142, Log Avg loss: 0.00026929, Global Avg Loss: 0.00899593, Time: 0.2427 Steps: 62600, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 003112, Sample Num: 49792, Cur Loss: 0.00001714, Cur Avg Loss: 0.00022522, Log Avg loss: 0.00028050, Global Avg Loss: 0.00896817, Time: 0.2214 Steps: 62800, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 003312, Sample Num: 52992, Cur Loss: 0.00004797, Cur Avg Loss: 0.00021999, Log Avg loss: 0.00013864, Global Avg Loss: 0.00894014, Time: 0.2226 Steps: 63000, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 003512, Sample Num: 56192, Cur Loss: 0.00003551, Cur Avg Loss: 0.00022079, Log Avg loss: 0.00023406, Global Avg Loss: 0.00891259, Time: 0.2174 Steps: 63200, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 003712, Sample Num: 59392, Cur Loss: 0.00003250, Cur Avg Loss: 0.00022013, Log Avg loss: 0.00020848, Global Avg Loss: 0.00888513, Time: 0.2584 Steps: 63400, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 003912, Sample Num: 62592, Cur Loss: 0.00000650, Cur Avg Loss: 0.00021487, Log Avg loss: 0.00011729, Global Avg Loss: 0.00885756, Time: 0.2250 Steps: 63600, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 004112, Sample Num: 65792, Cur Loss: 0.00001603, Cur Avg Loss: 0.00021559, Log Avg loss: 0.00022960, Global Avg Loss: 0.00883052, Time: 0.1680 Steps: 63800, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 004312, Sample Num: 68992, Cur Loss: 0.00001229, Cur Avg Loss: 0.00021613, Log Avg loss: 0.00022741, Global Avg Loss: 0.00880363, Time: 0.2202 Steps: 64000, Updated lr: 0.000075 Training, Epoch: 0013, Batch: 004512, Sample Num: 72192, Cur Loss: 0.00000589, Cur Avg Loss: 0.00021775, Log Avg loss: 0.00025259, Global Avg Loss: 0.00877699, Time: 0.2446 Steps: 64200, Updated lr: 0.000074 Training, Epoch: 0013, Batch: 004712, Sample Num: 75392, Cur Loss: 0.00004337, Cur Avg Loss: 0.00021794, Log Avg loss: 0.00022213, Global Avg Loss: 0.00875042, Time: 0.2268 Steps: 64400, Updated lr: 0.000074 Training, Epoch: 0013, Batch: 004912, Sample Num: 78592, Cur Loss: 0.00002630, Cur Avg Loss: 0.00021809, Log Avg loss: 0.00022165, Global Avg Loss: 0.00872402, Time: 0.2229 Steps: 64600, Updated lr: 0.000074 ***** Running evaluation checkpoint-64662 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-64662 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1096.422114, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001641, "eval_total_loss": 1.748883, "eval_acc": 0.999724, "eval_jaccard": 0.988365, "eval_prec": 0.989484, "eval_recall": 0.989829, "eval_f1": 0.989316, "eval_pr_auc": 0.995423, "eval_roc_auc": 0.999417, "eval_fmax": 0.994657, "eval_pmax": 0.997076, "eval_rmax": 0.99225, "eval_tmax": 0.14, "update_flag": true, "test_avg_loss": 0.001698, "test_total_loss": 1.810148, "test_acc": 0.99975, "test_jaccard": 0.988924, "test_prec": 0.989959, "test_recall": 0.99028, "test_f1": 0.989787, "test_pr_auc": 0.995101, "test_roc_auc": 0.999139, "test_fmax": 0.995105, "test_pmax": 0.997399, "test_rmax": 0.992821, "test_tmax": 0.13, "lr": 7.429874848607187e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.008715998900791644, "train_cur_epoch_loss": 1.093519660276236, "train_cur_epoch_avg_loss": 0.00021984713716852354, "train_cur_epoch_time": 1096.4221136569977, "train_cur_epoch_avg_time": 0.22043066217470803, "epoch": 13, "step": 64662} ################################################## Training, Epoch: 0014, Batch: 000138, Sample Num: 2208, Cur Loss: 0.00003028, Cur Avg Loss: 0.00023215, Log Avg loss: 0.00027155, Global Avg Loss: 0.00869793, Time: 0.2264 Steps: 64800, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 000338, Sample Num: 5408, Cur Loss: 0.00000147, Cur Avg Loss: 0.00018413, Log Avg loss: 0.00015100, Global Avg Loss: 0.00867163, Time: 0.2266 Steps: 65000, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 000538, Sample Num: 8608, Cur Loss: 0.00002420, Cur Avg Loss: 0.00023634, Log Avg loss: 0.00032458, Global Avg Loss: 0.00864603, Time: 0.2199 Steps: 65200, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 000738, Sample Num: 11808, Cur Loss: 0.00007154, Cur Avg Loss: 0.00023548, Log Avg loss: 0.00023316, Global Avg Loss: 0.00862030, Time: 0.2289 Steps: 65400, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 000938, Sample Num: 15008, Cur Loss: 0.00000066, Cur Avg Loss: 0.00023614, Log Avg loss: 0.00023857, Global Avg Loss: 0.00859475, Time: 0.2175 Steps: 65600, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 001138, Sample Num: 18208, Cur Loss: 0.00432526, Cur Avg Loss: 0.00022583, Log Avg loss: 0.00017746, Global Avg Loss: 0.00856916, Time: 0.2170 Steps: 65800, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 001338, Sample Num: 21408, Cur Loss: 0.00000297, Cur Avg Loss: 0.00020813, Log Avg loss: 0.00010741, Global Avg Loss: 0.00854352, Time: 0.2322 Steps: 66000, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 001538, Sample Num: 24608, Cur Loss: 0.00001115, Cur Avg Loss: 0.00019309, Log Avg loss: 0.00009254, Global Avg Loss: 0.00851799, Time: 0.2253 Steps: 66200, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 001738, Sample Num: 27808, Cur Loss: 0.00000189, Cur Avg Loss: 0.00018719, Log Avg loss: 0.00014181, Global Avg Loss: 0.00849276, Time: 0.2200 Steps: 66400, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 001938, Sample Num: 31008, Cur Loss: 0.00000774, Cur Avg Loss: 0.00018130, Log Avg loss: 0.00013011, Global Avg Loss: 0.00846765, Time: 0.2043 Steps: 66600, Updated lr: 0.000074 Training, Epoch: 0014, Batch: 002138, Sample Num: 34208, Cur Loss: 0.00003406, Cur Avg Loss: 0.00019171, Log Avg loss: 0.00029258, Global Avg Loss: 0.00844317, Time: 0.2194 Steps: 66800, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 002338, Sample Num: 37408, Cur Loss: 0.00002862, Cur Avg Loss: 0.00019690, Log Avg loss: 0.00025241, Global Avg Loss: 0.00841872, Time: 0.2262 Steps: 67000, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 002538, Sample Num: 40608, Cur Loss: 0.00001036, Cur Avg Loss: 0.00018539, Log Avg loss: 0.00005078, Global Avg Loss: 0.00839382, Time: 0.2295 Steps: 67200, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 002738, Sample Num: 43808, Cur Loss: 0.00460535, Cur Avg Loss: 0.00019851, Log Avg loss: 0.00036506, Global Avg Loss: 0.00836999, Time: 0.2338 Steps: 67400, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 002938, Sample Num: 47008, Cur Loss: 0.00001393, Cur Avg Loss: 0.00020007, Log Avg loss: 0.00022139, Global Avg Loss: 0.00834588, Time: 0.2292 Steps: 67600, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 003138, Sample Num: 50208, Cur Loss: 0.00002082, Cur Avg Loss: 0.00020117, Log Avg loss: 0.00021733, Global Avg Loss: 0.00832191, Time: 0.2340 Steps: 67800, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 003338, Sample Num: 53408, Cur Loss: 0.00002043, Cur Avg Loss: 0.00019861, Log Avg loss: 0.00015845, Global Avg Loss: 0.00829790, Time: 0.2390 Steps: 68000, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 003538, Sample Num: 56608, Cur Loss: 0.00041764, Cur Avg Loss: 0.00020073, Log Avg loss: 0.00023618, Global Avg Loss: 0.00827425, Time: 0.2033 Steps: 68200, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 003738, Sample Num: 59808, Cur Loss: 0.00005712, Cur Avg Loss: 0.00019998, Log Avg loss: 0.00018665, Global Avg Loss: 0.00825061, Time: 0.2400 Steps: 68400, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 003938, Sample Num: 63008, Cur Loss: 0.00001706, Cur Avg Loss: 0.00019721, Log Avg loss: 0.00014536, Global Avg Loss: 0.00822698, Time: 0.2184 Steps: 68600, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 004138, Sample Num: 66208, Cur Loss: 0.00002371, Cur Avg Loss: 0.00019609, Log Avg loss: 0.00017415, Global Avg Loss: 0.00820357, Time: 0.2231 Steps: 68800, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 004338, Sample Num: 69408, Cur Loss: 0.00000349, Cur Avg Loss: 0.00019425, Log Avg loss: 0.00015620, Global Avg Loss: 0.00818024, Time: 0.2167 Steps: 69000, Updated lr: 0.000073 Training, Epoch: 0014, Batch: 004538, Sample Num: 72608, Cur Loss: 0.00000847, Cur Avg Loss: 0.00019386, Log Avg loss: 0.00018531, Global Avg Loss: 0.00815713, Time: 0.1197 Steps: 69200, Updated lr: 0.000072 Training, Epoch: 0014, Batch: 004738, Sample Num: 75808, Cur Loss: 0.00003011, Cur Avg Loss: 0.00019382, Log Avg loss: 0.00019295, Global Avg Loss: 0.00813418, Time: 0.2206 Steps: 69400, Updated lr: 0.000072 Training, Epoch: 0014, Batch: 004938, Sample Num: 79008, Cur Loss: 0.00000844, Cur Avg Loss: 0.00019492, Log Avg loss: 0.00022092, Global Avg Loss: 0.00811144, Time: 0.2119 Steps: 69600, Updated lr: 0.000072 ***** Running evaluation checkpoint-69636 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-69636 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1109.393148, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001703, "eval_total_loss": 1.815342, "eval_acc": 0.999717, "eval_jaccard": 0.988403, "eval_prec": 0.989552, "eval_recall": 0.989961, "eval_f1": 0.989401, "eval_pr_auc": 0.995099, "eval_roc_auc": 0.999355, "eval_fmax": 0.994676, "eval_pmax": 0.997016, "eval_rmax": 0.992347, "eval_tmax": 0.11, "update_flag": true, "test_avg_loss": 0.001759, "test_total_loss": 1.874647, "test_acc": 0.999744, "test_jaccard": 0.988946, "test_prec": 0.989891, "test_recall": 0.990444, "test_f1": 0.989844, "test_pr_auc": 0.995203, "test_roc_auc": 0.999136, "test_fmax": 0.994812, "test_pmax": 0.997442, "test_rmax": 0.992196, "test_tmax": 0.17, "lr": 7.229067420266452e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.008107503041521462, "train_cur_epoch_loss": 0.98016087639893, "train_cur_epoch_avg_loss": 0.00019705687100903297, "train_cur_epoch_time": 1109.3931481838226, "train_cur_epoch_avg_time": 0.22303842947000857, "epoch": 14, "step": 69636} ################################################## Training, Epoch: 0015, Batch: 000164, Sample Num: 2624, Cur Loss: 0.00010036, Cur Avg Loss: 0.00017996, Log Avg loss: 0.00023586, Global Avg Loss: 0.00808888, Time: 0.2113 Steps: 69800, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 000364, Sample Num: 5824, Cur Loss: 0.00000258, Cur Avg Loss: 0.00017396, Log Avg loss: 0.00016903, Global Avg Loss: 0.00806625, Time: 0.2194 Steps: 70000, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 000564, Sample Num: 9024, Cur Loss: 0.00000593, Cur Avg Loss: 0.00021942, Log Avg loss: 0.00030215, Global Avg Loss: 0.00804413, Time: 0.2341 Steps: 70200, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 000764, Sample Num: 12224, Cur Loss: 0.00000733, Cur Avg Loss: 0.00023568, Log Avg loss: 0.00028156, Global Avg Loss: 0.00802208, Time: 0.2222 Steps: 70400, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 000964, Sample Num: 15424, Cur Loss: 0.00000390, Cur Avg Loss: 0.00023146, Log Avg loss: 0.00021532, Global Avg Loss: 0.00799996, Time: 0.2185 Steps: 70600, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 001164, Sample Num: 18624, Cur Loss: 0.00001221, Cur Avg Loss: 0.00021748, Log Avg loss: 0.00015009, Global Avg Loss: 0.00797779, Time: 0.2227 Steps: 70800, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 001364, Sample Num: 21824, Cur Loss: 0.00003123, Cur Avg Loss: 0.00020368, Log Avg loss: 0.00012336, Global Avg Loss: 0.00795566, Time: 0.2226 Steps: 71000, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 001564, Sample Num: 25024, Cur Loss: 0.00000259, Cur Avg Loss: 0.00018713, Log Avg loss: 0.00007430, Global Avg Loss: 0.00793352, Time: 0.2199 Steps: 71200, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 001764, Sample Num: 28224, Cur Loss: 0.00003096, Cur Avg Loss: 0.00018319, Log Avg loss: 0.00015231, Global Avg Loss: 0.00791173, Time: 0.2278 Steps: 71400, Updated lr: 0.000072 Training, Epoch: 0015, Batch: 001964, Sample Num: 31424, Cur Loss: 0.00000667, Cur Avg Loss: 0.00017328, Log Avg loss: 0.00008588, Global Avg Loss: 0.00788987, Time: 0.2134 Steps: 71600, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 002164, Sample Num: 34624, Cur Loss: 0.00018214, Cur Avg Loss: 0.00017660, Log Avg loss: 0.00020927, Global Avg Loss: 0.00786847, Time: 0.2233 Steps: 71800, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 002364, Sample Num: 37824, Cur Loss: 0.00002167, Cur Avg Loss: 0.00018226, Log Avg loss: 0.00024343, Global Avg Loss: 0.00784729, Time: 0.2228 Steps: 72000, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 002564, Sample Num: 41024, Cur Loss: 0.00001130, Cur Avg Loss: 0.00017398, Log Avg loss: 0.00007612, Global Avg Loss: 0.00782576, Time: 0.2260 Steps: 72200, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 002764, Sample Num: 44224, Cur Loss: 0.00000443, Cur Avg Loss: 0.00018427, Log Avg loss: 0.00031614, Global Avg Loss: 0.00780502, Time: 0.2247 Steps: 72400, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 002964, Sample Num: 47424, Cur Loss: 0.00002141, Cur Avg Loss: 0.00018729, Log Avg loss: 0.00022911, Global Avg Loss: 0.00778415, Time: 0.2228 Steps: 72600, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 003164, Sample Num: 50624, Cur Loss: 0.00000342, Cur Avg Loss: 0.00018676, Log Avg loss: 0.00017892, Global Avg Loss: 0.00776326, Time: 0.2218 Steps: 72800, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 003364, Sample Num: 53824, Cur Loss: 0.00013846, Cur Avg Loss: 0.00018312, Log Avg loss: 0.00012555, Global Avg Loss: 0.00774233, Time: 0.2269 Steps: 73000, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 003564, Sample Num: 57024, Cur Loss: 0.00008158, Cur Avg Loss: 0.00018363, Log Avg loss: 0.00019207, Global Avg Loss: 0.00772170, Time: 0.2375 Steps: 73200, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 003764, Sample Num: 60224, Cur Loss: 0.00003775, Cur Avg Loss: 0.00018204, Log Avg loss: 0.00015387, Global Avg Loss: 0.00770108, Time: 0.2342 Steps: 73400, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 003964, Sample Num: 63424, Cur Loss: 0.00005187, Cur Avg Loss: 0.00018180, Log Avg loss: 0.00017713, Global Avg Loss: 0.00768063, Time: 0.2067 Steps: 73600, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 004164, Sample Num: 66624, Cur Loss: 0.00075663, Cur Avg Loss: 0.00017944, Log Avg loss: 0.00013278, Global Avg Loss: 0.00766018, Time: 0.2307 Steps: 73800, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 004364, Sample Num: 69824, Cur Loss: 0.00000984, Cur Avg Loss: 0.00017862, Log Avg loss: 0.00016156, Global Avg Loss: 0.00763991, Time: 0.2236 Steps: 74000, Updated lr: 0.000071 Training, Epoch: 0015, Batch: 004564, Sample Num: 73024, Cur Loss: 0.00000869, Cur Avg Loss: 0.00017953, Log Avg loss: 0.00019935, Global Avg Loss: 0.00761986, Time: 0.2272 Steps: 74200, Updated lr: 0.000070 Training, Epoch: 0015, Batch: 004764, Sample Num: 76224, Cur Loss: 0.00000574, Cur Avg Loss: 0.00018026, Log Avg loss: 0.00019689, Global Avg Loss: 0.00759990, Time: 0.2261 Steps: 74400, Updated lr: 0.000070 Training, Epoch: 0015, Batch: 004964, Sample Num: 79424, Cur Loss: 0.00004868, Cur Avg Loss: 0.00018369, Log Avg loss: 0.00026533, Global Avg Loss: 0.00758024, Time: 0.2181 Steps: 74600, Updated lr: 0.000070 ***** Running evaluation checkpoint-74610 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-74610 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1111.897072, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001692, "eval_total_loss": 1.803378, "eval_acc": 0.999724, "eval_jaccard": 0.988501, "eval_prec": 0.989435, "eval_recall": 0.990073, "eval_f1": 0.98942, "eval_pr_auc": 0.99513, "eval_roc_auc": 0.999407, "eval_fmax": 0.994629, "eval_pmax": 0.997464, "eval_rmax": 0.99181, "eval_tmax": 0.18, "update_flag": true, "test_avg_loss": 0.001801, "test_total_loss": 1.91939, "test_acc": 0.999744, "test_jaccard": 0.989, "test_prec": 0.989954, "test_recall": 0.990461, "test_f1": 0.989874, "test_pr_auc": 0.995, "test_roc_auc": 0.999086, "test_fmax": 0.994808, "test_pmax": 0.997473, "test_rmax": 0.992157, "test_tmax": 0.15, "lr": 7.028259991925716e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.007579292538460395, "train_cur_epoch_loss": 0.9169344951409073, "train_cur_epoch_avg_loss": 0.00018434549560532917, "train_cur_epoch_time": 1111.897071838379, "train_cur_epoch_avg_time": 0.22354183189352209, "epoch": 15, "step": 74610} ################################################## Training, Epoch: 0016, Batch: 000190, Sample Num: 3040, Cur Loss: 0.00003060, Cur Avg Loss: 0.00018302, Log Avg loss: 0.00019942, Global Avg Loss: 0.00756051, Time: 0.2230 Steps: 74800, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 000390, Sample Num: 6240, Cur Loss: 0.00021506, Cur Avg Loss: 0.00019259, Log Avg loss: 0.00020169, Global Avg Loss: 0.00754088, Time: 0.3413 Steps: 75000, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 000590, Sample Num: 9440, Cur Loss: 0.00001567, Cur Avg Loss: 0.00020636, Log Avg loss: 0.00023320, Global Avg Loss: 0.00752145, Time: 0.2277 Steps: 75200, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 000790, Sample Num: 12640, Cur Loss: 0.00004908, Cur Avg Loss: 0.00022622, Log Avg loss: 0.00028480, Global Avg Loss: 0.00750225, Time: 0.2235 Steps: 75400, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 000990, Sample Num: 15840, Cur Loss: 0.00001737, Cur Avg Loss: 0.00021115, Log Avg loss: 0.00015161, Global Avg Loss: 0.00748280, Time: 0.2105 Steps: 75600, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 001190, Sample Num: 19040, Cur Loss: 0.00006293, Cur Avg Loss: 0.00019520, Log Avg loss: 0.00011629, Global Avg Loss: 0.00746337, Time: 0.2209 Steps: 75800, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 001390, Sample Num: 22240, Cur Loss: 0.00001539, Cur Avg Loss: 0.00018273, Log Avg loss: 0.00010854, Global Avg Loss: 0.00744401, Time: 0.3937 Steps: 76000, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 001590, Sample Num: 25440, Cur Loss: 0.00003200, Cur Avg Loss: 0.00017691, Log Avg loss: 0.00013643, Global Avg Loss: 0.00742483, Time: 0.2301 Steps: 76200, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 001790, Sample Num: 28640, Cur Loss: 0.00000067, Cur Avg Loss: 0.00017359, Log Avg loss: 0.00014722, Global Avg Loss: 0.00740578, Time: 0.2258 Steps: 76400, Updated lr: 0.000070 Training, Epoch: 0016, Batch: 001990, Sample Num: 31840, Cur Loss: 0.00066587, Cur Avg Loss: 0.00016778, Log Avg loss: 0.00011581, Global Avg Loss: 0.00738675, Time: 0.0852 Steps: 76600, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 002190, Sample Num: 35040, Cur Loss: 0.00000414, Cur Avg Loss: 0.00017707, Log Avg loss: 0.00026949, Global Avg Loss: 0.00736821, Time: 0.2328 Steps: 76800, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 002390, Sample Num: 38240, Cur Loss: 0.00001238, Cur Avg Loss: 0.00017741, Log Avg loss: 0.00018106, Global Avg Loss: 0.00734955, Time: 0.2255 Steps: 77000, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 002590, Sample Num: 41440, Cur Loss: 0.00006517, Cur Avg Loss: 0.00017940, Log Avg loss: 0.00020321, Global Avg Loss: 0.00733103, Time: 0.2272 Steps: 77200, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 002790, Sample Num: 44640, Cur Loss: 0.00000165, Cur Avg Loss: 0.00017846, Log Avg loss: 0.00016626, Global Avg Loss: 0.00731252, Time: 0.2443 Steps: 77400, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 002990, Sample Num: 47840, Cur Loss: 0.00003530, Cur Avg Loss: 0.00018536, Log Avg loss: 0.00028166, Global Avg Loss: 0.00729440, Time: 0.2234 Steps: 77600, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 003190, Sample Num: 51040, Cur Loss: 0.00000348, Cur Avg Loss: 0.00018046, Log Avg loss: 0.00010725, Global Avg Loss: 0.00727592, Time: 0.2275 Steps: 77800, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 003390, Sample Num: 54240, Cur Loss: 0.00002934, Cur Avg Loss: 0.00017916, Log Avg loss: 0.00015842, Global Avg Loss: 0.00725767, Time: 0.2194 Steps: 78000, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 003590, Sample Num: 57440, Cur Loss: 0.00047424, Cur Avg Loss: 0.00017988, Log Avg loss: 0.00019204, Global Avg Loss: 0.00723960, Time: 0.2394 Steps: 78200, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 003790, Sample Num: 60640, Cur Loss: 0.00000674, Cur Avg Loss: 0.00017834, Log Avg loss: 0.00015066, Global Avg Loss: 0.00722152, Time: 0.2209 Steps: 78400, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 003990, Sample Num: 63840, Cur Loss: 0.00000105, Cur Avg Loss: 0.00017620, Log Avg loss: 0.00013562, Global Avg Loss: 0.00720349, Time: 0.2261 Steps: 78600, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 004190, Sample Num: 67040, Cur Loss: 0.00008761, Cur Avg Loss: 0.00017419, Log Avg loss: 0.00013421, Global Avg Loss: 0.00718554, Time: 0.0856 Steps: 78800, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 004390, Sample Num: 70240, Cur Loss: 0.00000134, Cur Avg Loss: 0.00017186, Log Avg loss: 0.00012307, Global Avg Loss: 0.00716766, Time: 0.2194 Steps: 79000, Updated lr: 0.000069 Training, Epoch: 0016, Batch: 004590, Sample Num: 73440, Cur Loss: 0.00000627, Cur Avg Loss: 0.00017297, Log Avg loss: 0.00019717, Global Avg Loss: 0.00715006, Time: 0.2207 Steps: 79200, Updated lr: 0.000068 Training, Epoch: 0016, Batch: 004790, Sample Num: 76640, Cur Loss: 0.00009364, Cur Avg Loss: 0.00017260, Log Avg loss: 0.00016425, Global Avg Loss: 0.00713247, Time: 0.7024 Steps: 79400, Updated lr: 0.000068 ***** Running evaluation checkpoint-79584 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-79584 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1106.348233, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001687, "eval_total_loss": 1.798084, "eval_acc": 0.999735, "eval_jaccard": 0.988943, "eval_prec": 0.990075, "eval_recall": 0.990344, "eval_f1": 0.989879, "eval_pr_auc": 0.99547, "eval_roc_auc": 0.999424, "eval_fmax": 0.994843, "eval_pmax": 0.99746, "eval_rmax": 0.99224, "eval_tmax": 0.14, "update_flag": true, "test_avg_loss": 0.001768, "test_total_loss": 1.884261, "test_acc": 0.999751, "test_jaccard": 0.989071, "test_prec": 0.990086, "test_recall": 0.990417, "test_f1": 0.989923, "test_pr_auc": 0.99529, "test_roc_auc": 0.999127, "test_fmax": 0.995089, "test_pmax": 0.997356, "test_rmax": 0.992831, "test_tmax": 0.12, "lr": 6.827452563584982e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.00711652673273279, "train_cur_epoch_loss": 0.870647203276441, "train_cur_epoch_avg_loss": 0.00017503964681874566, "train_cur_epoch_time": 1106.3482332229614, "train_cur_epoch_avg_time": 0.22242626321330145, "epoch": 16, "step": 79584} ################################################## Training, Epoch: 0017, Batch: 000016, Sample Num: 256, Cur Loss: 0.00000539, Cur Avg Loss: 0.00006464, Log Avg loss: 0.00022460, Global Avg Loss: 0.00711511, Time: 0.2201 Steps: 79600, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 000216, Sample Num: 3456, Cur Loss: 0.00001131, Cur Avg Loss: 0.00015009, Log Avg loss: 0.00015693, Global Avg Loss: 0.00709767, Time: 0.2171 Steps: 79800, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 000416, Sample Num: 6656, Cur Loss: 0.00002455, Cur Avg Loss: 0.00015975, Log Avg loss: 0.00017018, Global Avg Loss: 0.00708035, Time: 0.2521 Steps: 80000, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 000616, Sample Num: 9856, Cur Loss: 0.00005165, Cur Avg Loss: 0.00017890, Log Avg loss: 0.00021874, Global Avg Loss: 0.00706324, Time: 0.2270 Steps: 80200, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 000816, Sample Num: 13056, Cur Loss: 0.00001649, Cur Avg Loss: 0.00019579, Log Avg loss: 0.00024782, Global Avg Loss: 0.00704629, Time: 0.3402 Steps: 80400, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 001016, Sample Num: 16256, Cur Loss: 0.00000735, Cur Avg Loss: 0.00017708, Log Avg loss: 0.00010072, Global Avg Loss: 0.00702905, Time: 0.0840 Steps: 80600, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 001216, Sample Num: 19456, Cur Loss: 0.00011303, Cur Avg Loss: 0.00017189, Log Avg loss: 0.00014554, Global Avg Loss: 0.00701201, Time: 0.2647 Steps: 80800, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 001416, Sample Num: 22656, Cur Loss: 0.00000412, Cur Avg Loss: 0.00015763, Log Avg loss: 0.00007096, Global Avg Loss: 0.00699487, Time: 0.2210 Steps: 81000, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 001616, Sample Num: 25856, Cur Loss: 0.00000133, Cur Avg Loss: 0.00014850, Log Avg loss: 0.00008380, Global Avg Loss: 0.00697785, Time: 0.1086 Steps: 81200, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 001816, Sample Num: 29056, Cur Loss: 0.00017681, Cur Avg Loss: 0.00014648, Log Avg loss: 0.00013022, Global Avg Loss: 0.00696103, Time: 0.2253 Steps: 81400, Updated lr: 0.000068 Training, Epoch: 0017, Batch: 002016, Sample Num: 32256, Cur Loss: 0.00000321, Cur Avg Loss: 0.00014451, Log Avg loss: 0.00012654, Global Avg Loss: 0.00694428, Time: 0.2496 Steps: 81600, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 002216, Sample Num: 35456, Cur Loss: 0.00000682, Cur Avg Loss: 0.00015700, Log Avg loss: 0.00028298, Global Avg Loss: 0.00692799, Time: 0.2222 Steps: 81800, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 002416, Sample Num: 38656, Cur Loss: 0.00001490, Cur Avg Loss: 0.00015965, Log Avg loss: 0.00018897, Global Avg Loss: 0.00691155, Time: 0.2255 Steps: 82000, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 002616, Sample Num: 41856, Cur Loss: 0.00000396, Cur Avg Loss: 0.00016069, Log Avg loss: 0.00017330, Global Avg Loss: 0.00689516, Time: 0.2210 Steps: 82200, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 002816, Sample Num: 45056, Cur Loss: 0.00004106, Cur Avg Loss: 0.00016511, Log Avg loss: 0.00022287, Global Avg Loss: 0.00687896, Time: 0.2479 Steps: 82400, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 003016, Sample Num: 48256, Cur Loss: 0.00000760, Cur Avg Loss: 0.00017078, Log Avg loss: 0.00025059, Global Avg Loss: 0.00686291, Time: 0.2246 Steps: 82600, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 003216, Sample Num: 51456, Cur Loss: 0.00004459, Cur Avg Loss: 0.00016630, Log Avg loss: 0.00009873, Global Avg Loss: 0.00684658, Time: 0.2274 Steps: 82800, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 003416, Sample Num: 54656, Cur Loss: 0.00000896, Cur Avg Loss: 0.00016693, Log Avg loss: 0.00017704, Global Avg Loss: 0.00683050, Time: 0.2265 Steps: 83000, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 003616, Sample Num: 57856, Cur Loss: 0.00000956, Cur Avg Loss: 0.00016736, Log Avg loss: 0.00017480, Global Avg Loss: 0.00681451, Time: 0.1962 Steps: 83200, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 003816, Sample Num: 61056, Cur Loss: 0.00001269, Cur Avg Loss: 0.00016379, Log Avg loss: 0.00009924, Global Avg Loss: 0.00679840, Time: 0.1548 Steps: 83400, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 004016, Sample Num: 64256, Cur Loss: 0.00000554, Cur Avg Loss: 0.00016319, Log Avg loss: 0.00015178, Global Avg Loss: 0.00678250, Time: 0.2142 Steps: 83600, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 004216, Sample Num: 67456, Cur Loss: 0.00000154, Cur Avg Loss: 0.00016298, Log Avg loss: 0.00015867, Global Avg Loss: 0.00676669, Time: 0.2192 Steps: 83800, Updated lr: 0.000067 Training, Epoch: 0017, Batch: 004416, Sample Num: 70656, Cur Loss: 0.00000309, Cur Avg Loss: 0.00015983, Log Avg loss: 0.00009345, Global Avg Loss: 0.00675080, Time: 0.2358 Steps: 84000, Updated lr: 0.000066 Training, Epoch: 0017, Batch: 004616, Sample Num: 73856, Cur Loss: 0.00000434, Cur Avg Loss: 0.00016104, Log Avg loss: 0.00018774, Global Avg Loss: 0.00673521, Time: 0.2190 Steps: 84200, Updated lr: 0.000066 Training, Epoch: 0017, Batch: 004816, Sample Num: 77056, Cur Loss: 0.00011317, Cur Avg Loss: 0.00016219, Log Avg loss: 0.00018880, Global Avg Loss: 0.00671970, Time: 0.2174 Steps: 84400, Updated lr: 0.000066 ***** Running evaluation checkpoint-84558 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-84558 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1100.991693, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001771, "eval_total_loss": 1.887481, "eval_acc": 0.999736, "eval_jaccard": 0.988873, "eval_prec": 0.989782, "eval_recall": 0.990576, "eval_f1": 0.989827, "eval_pr_auc": 0.995224, "eval_roc_auc": 0.999394, "eval_fmax": 0.994813, "eval_pmax": 0.997735, "eval_rmax": 0.991908, "eval_tmax": 0.14, "update_flag": false, "test_avg_loss": 0.00188, "test_total_loss": 2.003621, "test_acc": 0.99975, "test_jaccard": 0.988942, "test_prec": 0.989705, "test_recall": 0.990484, "test_f1": 0.989783, "test_pr_auc": 0.995109, "test_roc_auc": 0.999093, "test_fmax": 0.994806, "test_pmax": 0.997075, "test_rmax": 0.992548, "test_tmax": 0.08, "lr": 6.626645135244247e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.00670740759638286, "train_cur_epoch_loss": 0.8033080371362935, "train_cur_epoch_avg_loss": 0.00016150141478413622, "train_cur_epoch_time": 1100.9916925430298, "train_cur_epoch_avg_time": 0.22134935515541412, "epoch": 17, "step": 84558} ################################################## Training, Epoch: 0018, Batch: 000042, Sample Num: 672, Cur Loss: 0.00000093, Cur Avg Loss: 0.00009644, Log Avg loss: 0.00013120, Global Avg Loss: 0.00670413, Time: 0.2217 Steps: 84600, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 000242, Sample Num: 3872, Cur Loss: 0.00000260, Cur Avg Loss: 0.00012326, Log Avg loss: 0.00012889, Global Avg Loss: 0.00668862, Time: 0.2207 Steps: 84800, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 000442, Sample Num: 7072, Cur Loss: 0.00002058, Cur Avg Loss: 0.00014448, Log Avg loss: 0.00017016, Global Avg Loss: 0.00667328, Time: 0.2282 Steps: 85000, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 000642, Sample Num: 10272, Cur Loss: 0.00000268, Cur Avg Loss: 0.00017175, Log Avg loss: 0.00023201, Global Avg Loss: 0.00665816, Time: 0.2218 Steps: 85200, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 000842, Sample Num: 13472, Cur Loss: 0.00010653, Cur Avg Loss: 0.00017767, Log Avg loss: 0.00019668, Global Avg Loss: 0.00664303, Time: 0.0860 Steps: 85400, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 001042, Sample Num: 16672, Cur Loss: 0.00000524, Cur Avg Loss: 0.00016958, Log Avg loss: 0.00013552, Global Avg Loss: 0.00662782, Time: 0.2136 Steps: 85600, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 001242, Sample Num: 19872, Cur Loss: 0.00000318, Cur Avg Loss: 0.00015772, Log Avg loss: 0.00009593, Global Avg Loss: 0.00661260, Time: 0.2148 Steps: 85800, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 001442, Sample Num: 23072, Cur Loss: 0.00000360, Cur Avg Loss: 0.00014825, Log Avg loss: 0.00008943, Global Avg Loss: 0.00659743, Time: 0.2227 Steps: 86000, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 001642, Sample Num: 26272, Cur Loss: 0.00000972, Cur Avg Loss: 0.00014043, Log Avg loss: 0.00008411, Global Avg Loss: 0.00658232, Time: 0.2188 Steps: 86200, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 001842, Sample Num: 29472, Cur Loss: 0.00000528, Cur Avg Loss: 0.00013989, Log Avg loss: 0.00013540, Global Avg Loss: 0.00656739, Time: 0.2147 Steps: 86400, Updated lr: 0.000066 Training, Epoch: 0018, Batch: 002042, Sample Num: 32672, Cur Loss: 0.00001183, Cur Avg Loss: 0.00014162, Log Avg loss: 0.00015755, Global Avg Loss: 0.00655259, Time: 0.2216 Steps: 86600, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 002242, Sample Num: 35872, Cur Loss: 0.00000303, Cur Avg Loss: 0.00014723, Log Avg loss: 0.00020449, Global Avg Loss: 0.00653796, Time: 0.2205 Steps: 86800, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 002442, Sample Num: 39072, Cur Loss: 0.00000166, Cur Avg Loss: 0.00014543, Log Avg loss: 0.00012523, Global Avg Loss: 0.00652322, Time: 0.2217 Steps: 87000, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 002642, Sample Num: 42272, Cur Loss: 0.00000941, Cur Avg Loss: 0.00014940, Log Avg loss: 0.00019797, Global Avg Loss: 0.00650871, Time: 0.2366 Steps: 87200, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 002842, Sample Num: 45472, Cur Loss: 0.00000500, Cur Avg Loss: 0.00014794, Log Avg loss: 0.00012854, Global Avg Loss: 0.00649411, Time: 0.2207 Steps: 87400, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 003042, Sample Num: 48672, Cur Loss: 0.00000178, Cur Avg Loss: 0.00015513, Log Avg loss: 0.00025742, Global Avg Loss: 0.00647987, Time: 0.2185 Steps: 87600, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 003242, Sample Num: 51872, Cur Loss: 0.00000428, Cur Avg Loss: 0.00014957, Log Avg loss: 0.00006488, Global Avg Loss: 0.00646526, Time: 0.2115 Steps: 87800, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 003442, Sample Num: 55072, Cur Loss: 0.00000145, Cur Avg Loss: 0.00015068, Log Avg loss: 0.00016872, Global Avg Loss: 0.00645095, Time: 0.2177 Steps: 88000, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 003642, Sample Num: 58272, Cur Loss: 0.00002195, Cur Avg Loss: 0.00015221, Log Avg loss: 0.00017865, Global Avg Loss: 0.00643673, Time: 0.2264 Steps: 88200, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 003842, Sample Num: 61472, Cur Loss: 0.00002952, Cur Avg Loss: 0.00014727, Log Avg loss: 0.00005731, Global Avg Loss: 0.00642229, Time: 0.2228 Steps: 88400, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 004042, Sample Num: 64672, Cur Loss: 0.00002322, Cur Avg Loss: 0.00014850, Log Avg loss: 0.00017211, Global Avg Loss: 0.00640819, Time: 0.2240 Steps: 88600, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 004242, Sample Num: 67872, Cur Loss: 0.00000984, Cur Avg Loss: 0.00014620, Log Avg loss: 0.00009973, Global Avg Loss: 0.00639398, Time: 0.2207 Steps: 88800, Updated lr: 0.000065 Training, Epoch: 0018, Batch: 004442, Sample Num: 71072, Cur Loss: 0.00010423, Cur Avg Loss: 0.00014448, Log Avg loss: 0.00010799, Global Avg Loss: 0.00637985, Time: 0.2206 Steps: 89000, Updated lr: 0.000064 Training, Epoch: 0018, Batch: 004642, Sample Num: 74272, Cur Loss: 0.00000462, Cur Avg Loss: 0.00014700, Log Avg loss: 0.00020301, Global Avg Loss: 0.00636600, Time: 0.3622 Steps: 89200, Updated lr: 0.000064 Training, Epoch: 0018, Batch: 004842, Sample Num: 77472, Cur Loss: 0.00000535, Cur Avg Loss: 0.00015133, Log Avg loss: 0.00025163, Global Avg Loss: 0.00635232, Time: 0.2254 Steps: 89400, Updated lr: 0.000064 ***** Running evaluation checkpoint-89532 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-89532 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1091.430642, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001766, "eval_total_loss": 1.883074, "eval_acc": 0.999727, "eval_jaccard": 0.988665, "eval_prec": 0.98964, "eval_recall": 0.990266, "eval_f1": 0.9896, "eval_pr_auc": 0.995333, "eval_roc_auc": 0.999389, "eval_fmax": 0.994859, "eval_pmax": 0.997098, "eval_rmax": 0.992631, "eval_tmax": 0.07, "update_flag": false, "test_avg_loss": 0.001876, "test_total_loss": 2.000204, "test_acc": 0.999757, "test_jaccard": 0.989127, "test_prec": 0.989954, "test_recall": 0.990588, "test_f1": 0.989968, "test_pr_auc": 0.995087, "test_roc_auc": 0.9991, "test_fmax": 0.994885, "test_pmax": 0.997095, "test_rmax": 0.992685, "test_tmax": 0.1, "lr": 6.425837706903513e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.006343188203380028, "train_cur_epoch_loss": 0.7533546900785044, "train_cur_epoch_avg_loss": 0.00015145852233182638, "train_cur_epoch_time": 1091.4306421279907, "train_cur_epoch_avg_time": 0.21942714960353654, "epoch": 18, "step": 89532} ################################################## Training, Epoch: 0019, Batch: 000068, Sample Num: 1088, Cur Loss: 0.00070344, Cur Avg Loss: 0.00008645, Log Avg loss: 0.00013256, Global Avg Loss: 0.00633844, Time: 0.2184 Steps: 89600, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 000268, Sample Num: 4288, Cur Loss: 0.00000309, Cur Avg Loss: 0.00008570, Log Avg loss: 0.00008544, Global Avg Loss: 0.00632451, Time: 0.2206 Steps: 89800, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 000468, Sample Num: 7488, Cur Loss: 0.00006932, Cur Avg Loss: 0.00013347, Log Avg loss: 0.00019748, Global Avg Loss: 0.00631090, Time: 0.2140 Steps: 90000, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 000668, Sample Num: 10688, Cur Loss: 0.00001148, Cur Avg Loss: 0.00015487, Log Avg loss: 0.00020494, Global Avg Loss: 0.00629736, Time: 0.2211 Steps: 90200, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 000868, Sample Num: 13888, Cur Loss: 0.00005297, Cur Avg Loss: 0.00016181, Log Avg loss: 0.00018502, Global Avg Loss: 0.00628384, Time: 0.3010 Steps: 90400, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 001068, Sample Num: 17088, Cur Loss: 0.00909582, Cur Avg Loss: 0.00015455, Log Avg loss: 0.00012302, Global Avg Loss: 0.00627024, Time: 0.1236 Steps: 90600, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 001268, Sample Num: 20288, Cur Loss: 0.00000897, Cur Avg Loss: 0.00014290, Log Avg loss: 0.00008068, Global Avg Loss: 0.00625660, Time: 0.2169 Steps: 90800, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 001468, Sample Num: 23488, Cur Loss: 0.00002060, Cur Avg Loss: 0.00013512, Log Avg loss: 0.00008578, Global Avg Loss: 0.00624304, Time: 0.2167 Steps: 91000, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 001668, Sample Num: 26688, Cur Loss: 0.00000957, Cur Avg Loss: 0.00012934, Log Avg loss: 0.00008695, Global Avg Loss: 0.00622954, Time: 0.0915 Steps: 91200, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 001868, Sample Num: 29888, Cur Loss: 0.00002715, Cur Avg Loss: 0.00012744, Log Avg loss: 0.00011158, Global Avg Loss: 0.00621615, Time: 0.2232 Steps: 91400, Updated lr: 0.000064 Training, Epoch: 0019, Batch: 002068, Sample Num: 33088, Cur Loss: 0.00015213, Cur Avg Loss: 0.00013615, Log Avg loss: 0.00021747, Global Avg Loss: 0.00620306, Time: 0.2247 Steps: 91600, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 002268, Sample Num: 36288, Cur Loss: 0.00000192, Cur Avg Loss: 0.00014276, Log Avg loss: 0.00021119, Global Avg Loss: 0.00619000, Time: 0.2222 Steps: 91800, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 002468, Sample Num: 39488, Cur Loss: 0.00000653, Cur Avg Loss: 0.00013906, Log Avg loss: 0.00009701, Global Avg Loss: 0.00617676, Time: 0.2188 Steps: 92000, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 002668, Sample Num: 42688, Cur Loss: 0.00096431, Cur Avg Loss: 0.00014396, Log Avg loss: 0.00020448, Global Avg Loss: 0.00616380, Time: 0.2208 Steps: 92200, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 002868, Sample Num: 45888, Cur Loss: 0.00001891, Cur Avg Loss: 0.00014820, Log Avg loss: 0.00020477, Global Avg Loss: 0.00615090, Time: 0.3385 Steps: 92400, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 003068, Sample Num: 49088, Cur Loss: 0.00001473, Cur Avg Loss: 0.00015002, Log Avg loss: 0.00017617, Global Avg Loss: 0.00613800, Time: 0.2082 Steps: 92600, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 003268, Sample Num: 52288, Cur Loss: 0.00000682, Cur Avg Loss: 0.00014453, Log Avg loss: 0.00006025, Global Avg Loss: 0.00612490, Time: 0.2166 Steps: 92800, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 003468, Sample Num: 55488, Cur Loss: 0.00201425, Cur Avg Loss: 0.00014502, Log Avg loss: 0.00015299, Global Avg Loss: 0.00611206, Time: 0.2246 Steps: 93000, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 003668, Sample Num: 58688, Cur Loss: 0.00001388, Cur Avg Loss: 0.00014692, Log Avg loss: 0.00017990, Global Avg Loss: 0.00609933, Time: 0.1878 Steps: 93200, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 003868, Sample Num: 61888, Cur Loss: 0.00000754, Cur Avg Loss: 0.00014261, Log Avg loss: 0.00006352, Global Avg Loss: 0.00608640, Time: 0.2197 Steps: 93400, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 004068, Sample Num: 65088, Cur Loss: 0.00193029, Cur Avg Loss: 0.00014283, Log Avg loss: 0.00014712, Global Avg Loss: 0.00607371, Time: 0.2207 Steps: 93600, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 004268, Sample Num: 68288, Cur Loss: 0.00005864, Cur Avg Loss: 0.00014147, Log Avg loss: 0.00011371, Global Avg Loss: 0.00606100, Time: 0.2111 Steps: 93800, Updated lr: 0.000063 Training, Epoch: 0019, Batch: 004468, Sample Num: 71488, Cur Loss: 0.00092473, Cur Avg Loss: 0.00013933, Log Avg loss: 0.00009372, Global Avg Loss: 0.00604831, Time: 0.2239 Steps: 94000, Updated lr: 0.000062 Training, Epoch: 0019, Batch: 004668, Sample Num: 74688, Cur Loss: 0.00019016, Cur Avg Loss: 0.00014001, Log Avg loss: 0.00015533, Global Avg Loss: 0.00603580, Time: 0.2205 Steps: 94200, Updated lr: 0.000062 Training, Epoch: 0019, Batch: 004868, Sample Num: 77888, Cur Loss: 0.00001376, Cur Avg Loss: 0.00014109, Log Avg loss: 0.00016611, Global Avg Loss: 0.00602336, Time: 0.2206 Steps: 94400, Updated lr: 0.000062 ***** Running evaluation checkpoint-94506 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-94506 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1085.728860, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001793, "eval_total_loss": 1.910902, "eval_acc": 0.999723, "eval_jaccard": 0.988585, "eval_prec": 0.98965, "eval_recall": 0.990051, "eval_f1": 0.989518, "eval_pr_auc": 0.995458, "eval_roc_auc": 0.999388, "eval_fmax": 0.994833, "eval_pmax": 0.997647, "eval_rmax": 0.992035, "eval_tmax": 0.17, "update_flag": false, "test_avg_loss": 0.001899, "test_total_loss": 2.024772, "test_acc": 0.999753, "test_jaccard": 0.989227, "test_prec": 0.990096, "test_recall": 0.99074, "test_f1": 0.990091, "test_pr_auc": 0.99523, "test_roc_auc": 0.999121, "test_fmax": 0.994953, "test_pmax": 0.99815, "test_rmax": 0.991776, "test_tmax": 0.27, "lr": 6.225030278562778e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0060169117603793435, "train_cur_epoch_loss": 0.7159366013886217, "train_cur_epoch_avg_loss": 0.00014393578636683188, "train_cur_epoch_time": 1085.7288596630096, "train_cur_epoch_avg_time": 0.21828083226035577, "epoch": 19, "step": 94506} ################################################## Training, Epoch: 0020, Batch: 000094, Sample Num: 1504, Cur Loss: 0.00000053, Cur Avg Loss: 0.00013970, Log Avg loss: 0.00021131, Global Avg Loss: 0.00601107, Time: 0.2118 Steps: 94600, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 000294, Sample Num: 4704, Cur Loss: 0.00000502, Cur Avg Loss: 0.00012716, Log Avg loss: 0.00012127, Global Avg Loss: 0.00599865, Time: 0.0772 Steps: 94800, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 000494, Sample Num: 7904, Cur Loss: 0.00001274, Cur Avg Loss: 0.00014010, Log Avg loss: 0.00015911, Global Avg Loss: 0.00598635, Time: 0.2181 Steps: 95000, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 000694, Sample Num: 11104, Cur Loss: 0.00001204, Cur Avg Loss: 0.00015126, Log Avg loss: 0.00017882, Global Avg Loss: 0.00597415, Time: 0.2175 Steps: 95200, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 000894, Sample Num: 14304, Cur Loss: 0.00004976, Cur Avg Loss: 0.00016195, Log Avg loss: 0.00019906, Global Avg Loss: 0.00596204, Time: 0.2198 Steps: 95400, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 001094, Sample Num: 17504, Cur Loss: 0.00028214, Cur Avg Loss: 0.00014944, Log Avg loss: 0.00009350, Global Avg Loss: 0.00594977, Time: 0.2178 Steps: 95600, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 001294, Sample Num: 20704, Cur Loss: 0.00000685, Cur Avg Loss: 0.00013971, Log Avg loss: 0.00008653, Global Avg Loss: 0.00593753, Time: 0.2216 Steps: 95800, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 001494, Sample Num: 23904, Cur Loss: 0.00000099, Cur Avg Loss: 0.00013484, Log Avg loss: 0.00010331, Global Avg Loss: 0.00592537, Time: 0.2245 Steps: 96000, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 001694, Sample Num: 27104, Cur Loss: 0.00000716, Cur Avg Loss: 0.00012881, Log Avg loss: 0.00008375, Global Avg Loss: 0.00591323, Time: 0.2195 Steps: 96200, Updated lr: 0.000062 Training, Epoch: 0020, Batch: 001894, Sample Num: 30304, Cur Loss: 0.00003544, Cur Avg Loss: 0.00012754, Log Avg loss: 0.00011681, Global Avg Loss: 0.00590120, Time: 0.2106 Steps: 96400, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 002094, Sample Num: 33504, Cur Loss: 0.00000047, Cur Avg Loss: 0.00012924, Log Avg loss: 0.00014531, Global Avg Loss: 0.00588928, Time: 0.2167 Steps: 96600, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 002294, Sample Num: 36704, Cur Loss: 0.00000158, Cur Avg Loss: 0.00013675, Log Avg loss: 0.00021544, Global Avg Loss: 0.00587756, Time: 0.2190 Steps: 96800, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 002494, Sample Num: 39904, Cur Loss: 0.00004089, Cur Avg Loss: 0.00012939, Log Avg loss: 0.00004492, Global Avg Loss: 0.00586554, Time: 0.2476 Steps: 97000, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 002694, Sample Num: 43104, Cur Loss: 0.00011562, Cur Avg Loss: 0.00013112, Log Avg loss: 0.00015276, Global Avg Loss: 0.00585378, Time: 0.2194 Steps: 97200, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 002894, Sample Num: 46304, Cur Loss: 0.00000168, Cur Avg Loss: 0.00013430, Log Avg loss: 0.00017712, Global Avg Loss: 0.00584212, Time: 0.2198 Steps: 97400, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 003094, Sample Num: 49504, Cur Loss: 0.00000175, Cur Avg Loss: 0.00013625, Log Avg loss: 0.00016442, Global Avg Loss: 0.00583049, Time: 0.2199 Steps: 97600, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 003294, Sample Num: 52704, Cur Loss: 0.00000203, Cur Avg Loss: 0.00013430, Log Avg loss: 0.00010406, Global Avg Loss: 0.00581878, Time: 0.3346 Steps: 97800, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 003494, Sample Num: 55904, Cur Loss: 0.00000208, Cur Avg Loss: 0.00013792, Log Avg loss: 0.00019767, Global Avg Loss: 0.00580731, Time: 0.2130 Steps: 98000, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 003694, Sample Num: 59104, Cur Loss: 0.00000065, Cur Avg Loss: 0.00013957, Log Avg loss: 0.00016832, Global Avg Loss: 0.00579582, Time: 0.2206 Steps: 98200, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 003894, Sample Num: 62304, Cur Loss: 0.00001551, Cur Avg Loss: 0.00013414, Log Avg loss: 0.00003385, Global Avg Loss: 0.00578411, Time: 0.2179 Steps: 98400, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 004094, Sample Num: 65504, Cur Loss: 0.00000132, Cur Avg Loss: 0.00013595, Log Avg loss: 0.00017128, Global Avg Loss: 0.00577273, Time: 0.2209 Steps: 98600, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 004294, Sample Num: 68704, Cur Loss: 0.00004019, Cur Avg Loss: 0.00013527, Log Avg loss: 0.00012135, Global Avg Loss: 0.00576129, Time: 0.2215 Steps: 98800, Updated lr: 0.000061 Training, Epoch: 0020, Batch: 004494, Sample Num: 71904, Cur Loss: 0.00000297, Cur Avg Loss: 0.00013642, Log Avg loss: 0.00016096, Global Avg Loss: 0.00574997, Time: 0.1725 Steps: 99000, Updated lr: 0.000060 Training, Epoch: 0020, Batch: 004694, Sample Num: 75104, Cur Loss: 0.00000498, Cur Avg Loss: 0.00013536, Log Avg loss: 0.00011165, Global Avg Loss: 0.00573861, Time: 0.2065 Steps: 99200, Updated lr: 0.000060 Training, Epoch: 0020, Batch: 004894, Sample Num: 78304, Cur Loss: 0.00000555, Cur Avg Loss: 0.00013633, Log Avg loss: 0.00015895, Global Avg Loss: 0.00572738, Time: 0.2186 Steps: 99400, Updated lr: 0.000060 ***** Running evaluation checkpoint-99480 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-99480 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1090.703389, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001866, "eval_total_loss": 1.988753, "eval_acc": 0.999723, "eval_jaccard": 0.98814, "eval_prec": 0.98922, "eval_recall": 0.989473, "eval_f1": 0.989025, "eval_pr_auc": 0.995073, "eval_roc_auc": 0.999385, "eval_fmax": 0.994801, "eval_pmax": 0.99773, "eval_rmax": 0.991888, "eval_tmax": 0.11, "update_flag": false, "test_avg_loss": 0.001948, "test_total_loss": 2.076217, "test_acc": 0.999762, "test_jaccard": 0.989351, "test_prec": 0.990287, "test_recall": 0.990601, "test_f1": 0.990156, "test_pr_auc": 0.995182, "test_roc_auc": 0.999106, "test_fmax": 0.994877, "test_pmax": 0.998066, "test_rmax": 0.991707, "test_tmax": 0.2, "lr": 6.024222850222043e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.005723001497440026, "train_cur_epoch_loss": 0.6899261389226368, "train_cur_epoch_avg_loss": 0.00013870650159280995, "train_cur_epoch_time": 1090.7033891677856, "train_cur_epoch_avg_time": 0.21928093871487447, "epoch": 20, "step": 99480} ################################################## Training, Epoch: 0021, Batch: 000120, Sample Num: 1920, Cur Loss: 0.00001991, Cur Avg Loss: 0.00014113, Log Avg loss: 0.00019844, Global Avg Loss: 0.00571628, Time: 0.2164 Steps: 99600, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 000320, Sample Num: 5120, Cur Loss: 0.00000902, Cur Avg Loss: 0.00010747, Log Avg loss: 0.00008728, Global Avg Loss: 0.00570500, Time: 0.2515 Steps: 99800, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 000520, Sample Num: 8320, Cur Loss: 0.00003963, Cur Avg Loss: 0.00014380, Log Avg loss: 0.00020192, Global Avg Loss: 0.00569399, Time: 0.2165 Steps: 100000, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 000720, Sample Num: 11520, Cur Loss: 0.00000426, Cur Avg Loss: 0.00013850, Log Avg loss: 0.00012474, Global Avg Loss: 0.00568287, Time: 0.1107 Steps: 100200, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 000920, Sample Num: 14720, Cur Loss: 0.00000159, Cur Avg Loss: 0.00014255, Log Avg loss: 0.00015709, Global Avg Loss: 0.00567187, Time: 0.0853 Steps: 100400, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 001120, Sample Num: 17920, Cur Loss: 0.00000045, Cur Avg Loss: 0.00013067, Log Avg loss: 0.00007603, Global Avg Loss: 0.00566074, Time: 0.2529 Steps: 100600, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 001320, Sample Num: 21120, Cur Loss: 0.00001951, Cur Avg Loss: 0.00012425, Log Avg loss: 0.00008829, Global Avg Loss: 0.00564968, Time: 0.3377 Steps: 100800, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 001520, Sample Num: 24320, Cur Loss: 0.00000185, Cur Avg Loss: 0.00011361, Log Avg loss: 0.00004342, Global Avg Loss: 0.00563858, Time: 0.2211 Steps: 101000, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 001720, Sample Num: 27520, Cur Loss: 0.00001623, Cur Avg Loss: 0.00011290, Log Avg loss: 0.00010753, Global Avg Loss: 0.00562765, Time: 0.2187 Steps: 101200, Updated lr: 0.000060 Training, Epoch: 0021, Batch: 001920, Sample Num: 30720, Cur Loss: 0.00000147, Cur Avg Loss: 0.00011119, Log Avg loss: 0.00009645, Global Avg Loss: 0.00561674, Time: 0.2569 Steps: 101400, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 002120, Sample Num: 33920, Cur Loss: 0.00006509, Cur Avg Loss: 0.00011575, Log Avg loss: 0.00015953, Global Avg Loss: 0.00560600, Time: 0.2205 Steps: 101600, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 002320, Sample Num: 37120, Cur Loss: 0.00000290, Cur Avg Loss: 0.00012498, Log Avg loss: 0.00022283, Global Avg Loss: 0.00559542, Time: 0.2187 Steps: 101800, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 002520, Sample Num: 40320, Cur Loss: 0.00000417, Cur Avg Loss: 0.00011739, Log Avg loss: 0.00002937, Global Avg Loss: 0.00558451, Time: 0.2198 Steps: 102000, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 002720, Sample Num: 43520, Cur Loss: 0.00000328, Cur Avg Loss: 0.00011996, Log Avg loss: 0.00015225, Global Avg Loss: 0.00557388, Time: 0.2535 Steps: 102200, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 002920, Sample Num: 46720, Cur Loss: 0.00012683, Cur Avg Loss: 0.00012676, Log Avg loss: 0.00021928, Global Avg Loss: 0.00556342, Time: 0.2210 Steps: 102400, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 003120, Sample Num: 49920, Cur Loss: 0.00221429, Cur Avg Loss: 0.00012876, Log Avg loss: 0.00015798, Global Avg Loss: 0.00555288, Time: 0.2227 Steps: 102600, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 003320, Sample Num: 53120, Cur Loss: 0.00000624, Cur Avg Loss: 0.00012611, Log Avg loss: 0.00008482, Global Avg Loss: 0.00554225, Time: 0.2188 Steps: 102800, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 003520, Sample Num: 56320, Cur Loss: 0.00000110, Cur Avg Loss: 0.00012808, Log Avg loss: 0.00016077, Global Avg Loss: 0.00553180, Time: 0.2577 Steps: 103000, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 003720, Sample Num: 59520, Cur Loss: 0.00000078, Cur Avg Loss: 0.00012539, Log Avg loss: 0.00007795, Global Avg Loss: 0.00552123, Time: 0.2199 Steps: 103200, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 003920, Sample Num: 62720, Cur Loss: 0.00000123, Cur Avg Loss: 0.00012156, Log Avg loss: 0.00005041, Global Avg Loss: 0.00551065, Time: 0.2104 Steps: 103400, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 004120, Sample Num: 65920, Cur Loss: 0.00000199, Cur Avg Loss: 0.00012069, Log Avg loss: 0.00010360, Global Avg Loss: 0.00550021, Time: 0.2189 Steps: 103600, Updated lr: 0.000059 Training, Epoch: 0021, Batch: 004320, Sample Num: 69120, Cur Loss: 0.00000188, Cur Avg Loss: 0.00011976, Log Avg loss: 0.00010060, Global Avg Loss: 0.00548980, Time: 0.2500 Steps: 103800, Updated lr: 0.000058 Training, Epoch: 0021, Batch: 004520, Sample Num: 72320, Cur Loss: 0.00000430, Cur Avg Loss: 0.00012249, Log Avg loss: 0.00018141, Global Avg Loss: 0.00547959, Time: 0.3935 Steps: 104000, Updated lr: 0.000058 Training, Epoch: 0021, Batch: 004720, Sample Num: 75520, Cur Loss: 0.00000521, Cur Avg Loss: 0.00012386, Log Avg loss: 0.00015487, Global Avg Loss: 0.00546937, Time: 0.2202 Steps: 104200, Updated lr: 0.000058 Training, Epoch: 0021, Batch: 004920, Sample Num: 78720, Cur Loss: 0.00000259, Cur Avg Loss: 0.00012389, Log Avg loss: 0.00012448, Global Avg Loss: 0.00545914, Time: 0.2204 Steps: 104400, Updated lr: 0.000058 ***** Running evaluation checkpoint-104454 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-104454 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1086.117694, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001836, "eval_total_loss": 1.957682, "eval_acc": 0.999719, "eval_jaccard": 0.988518, "eval_prec": 0.98966, "eval_recall": 0.98999, "eval_f1": 0.989479, "eval_pr_auc": 0.995229, "eval_roc_auc": 0.999367, "eval_fmax": 0.994734, "eval_pmax": 0.996719, "eval_rmax": 0.992758, "eval_tmax": 0.04, "update_flag": false, "test_avg_loss": 0.001964, "test_total_loss": 2.093213, "test_acc": 0.99975, "test_jaccard": 0.98923, "test_prec": 0.990223, "test_recall": 0.990598, "test_f1": 0.990093, "test_pr_auc": 0.995176, "test_roc_auc": 0.999124, "test_fmax": 0.994886, "test_pmax": 0.998096, "test_rmax": 0.991697, "test_tmax": 0.2, "lr": 5.8234154218813086e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.005456495325243135, "train_cur_epoch_loss": 0.6285737376129643, "train_cur_epoch_avg_loss": 0.00012637188130538085, "train_cur_epoch_time": 1086.1176943778992, "train_cur_epoch_avg_time": 0.2183590057052471, "epoch": 21, "step": 104454} ################################################## Training, Epoch: 0022, Batch: 000146, Sample Num: 2336, Cur Loss: 0.00000603, Cur Avg Loss: 0.00009937, Log Avg loss: 0.00016782, Global Avg Loss: 0.00544902, Time: 0.2203 Steps: 104600, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 000346, Sample Num: 5536, Cur Loss: 0.00001690, Cur Avg Loss: 0.00010346, Log Avg loss: 0.00010645, Global Avg Loss: 0.00543882, Time: 0.2233 Steps: 104800, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 000546, Sample Num: 8736, Cur Loss: 0.00000316, Cur Avg Loss: 0.00013291, Log Avg loss: 0.00018385, Global Avg Loss: 0.00542881, Time: 0.2209 Steps: 105000, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 000746, Sample Num: 11936, Cur Loss: 0.00004982, Cur Avg Loss: 0.00014933, Log Avg loss: 0.00019416, Global Avg Loss: 0.00541886, Time: 0.2198 Steps: 105200, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 000946, Sample Num: 15136, Cur Loss: 0.00000193, Cur Avg Loss: 0.00014319, Log Avg loss: 0.00012030, Global Avg Loss: 0.00540881, Time: 0.2206 Steps: 105400, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 001146, Sample Num: 18336, Cur Loss: 0.00000403, Cur Avg Loss: 0.00014178, Log Avg loss: 0.00013512, Global Avg Loss: 0.00539882, Time: 0.2193 Steps: 105600, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 001346, Sample Num: 21536, Cur Loss: 0.00000660, Cur Avg Loss: 0.00013255, Log Avg loss: 0.00007960, Global Avg Loss: 0.00538876, Time: 0.2229 Steps: 105800, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 001546, Sample Num: 24736, Cur Loss: 0.00041624, Cur Avg Loss: 0.00012341, Log Avg loss: 0.00006191, Global Avg Loss: 0.00537871, Time: 0.2131 Steps: 106000, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 001746, Sample Num: 27936, Cur Loss: 0.00000118, Cur Avg Loss: 0.00012582, Log Avg loss: 0.00014449, Global Avg Loss: 0.00536886, Time: 0.2226 Steps: 106200, Updated lr: 0.000058 Training, Epoch: 0022, Batch: 001946, Sample Num: 31136, Cur Loss: 0.00000006, Cur Avg Loss: 0.00012018, Log Avg loss: 0.00007094, Global Avg Loss: 0.00535890, Time: 0.2176 Steps: 106400, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 002146, Sample Num: 34336, Cur Loss: 0.00002195, Cur Avg Loss: 0.00012471, Log Avg loss: 0.00016876, Global Avg Loss: 0.00534916, Time: 0.2281 Steps: 106600, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 002346, Sample Num: 37536, Cur Loss: 0.00001146, Cur Avg Loss: 0.00012901, Log Avg loss: 0.00017517, Global Avg Loss: 0.00533947, Time: 0.2331 Steps: 106800, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 002546, Sample Num: 40736, Cur Loss: 0.00004553, Cur Avg Loss: 0.00012161, Log Avg loss: 0.00003485, Global Avg Loss: 0.00532956, Time: 0.2171 Steps: 107000, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 002746, Sample Num: 43936, Cur Loss: 0.00001325, Cur Avg Loss: 0.00012932, Log Avg loss: 0.00022742, Global Avg Loss: 0.00532004, Time: 0.3754 Steps: 107200, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 002946, Sample Num: 47136, Cur Loss: 0.00243386, Cur Avg Loss: 0.00013236, Log Avg loss: 0.00017412, Global Avg Loss: 0.00531045, Time: 0.2275 Steps: 107400, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 003146, Sample Num: 50336, Cur Loss: 0.00000188, Cur Avg Loss: 0.00013372, Log Avg loss: 0.00015370, Global Avg Loss: 0.00530087, Time: 0.2168 Steps: 107600, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 003346, Sample Num: 53536, Cur Loss: 0.00001933, Cur Avg Loss: 0.00013291, Log Avg loss: 0.00012025, Global Avg Loss: 0.00529126, Time: 0.2203 Steps: 107800, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 003546, Sample Num: 56736, Cur Loss: 0.00001119, Cur Avg Loss: 0.00013239, Log Avg loss: 0.00012358, Global Avg Loss: 0.00528169, Time: 0.2180 Steps: 108000, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 003746, Sample Num: 59936, Cur Loss: 0.00000377, Cur Avg Loss: 0.00012981, Log Avg loss: 0.00008418, Global Avg Loss: 0.00527208, Time: 0.2249 Steps: 108200, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 003946, Sample Num: 63136, Cur Loss: 0.00007671, Cur Avg Loss: 0.00012512, Log Avg loss: 0.00003726, Global Avg Loss: 0.00526242, Time: 0.2181 Steps: 108400, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 004146, Sample Num: 66336, Cur Loss: 0.00001795, Cur Avg Loss: 0.00012513, Log Avg loss: 0.00012538, Global Avg Loss: 0.00525296, Time: 0.2210 Steps: 108600, Updated lr: 0.000057 Training, Epoch: 0022, Batch: 004346, Sample Num: 69536, Cur Loss: 0.00000040, Cur Avg Loss: 0.00012401, Log Avg loss: 0.00010077, Global Avg Loss: 0.00524349, Time: 0.2108 Steps: 108800, Updated lr: 0.000056 Training, Epoch: 0022, Batch: 004546, Sample Num: 72736, Cur Loss: 0.00002813, Cur Avg Loss: 0.00012661, Log Avg loss: 0.00018310, Global Avg Loss: 0.00523421, Time: 0.2237 Steps: 109000, Updated lr: 0.000056 Training, Epoch: 0022, Batch: 004746, Sample Num: 75936, Cur Loss: 0.00000907, Cur Avg Loss: 0.00012718, Log Avg loss: 0.00014016, Global Avg Loss: 0.00522488, Time: 0.2192 Steps: 109200, Updated lr: 0.000056 Training, Epoch: 0022, Batch: 004946, Sample Num: 79136, Cur Loss: 0.00003548, Cur Avg Loss: 0.00012572, Log Avg loss: 0.00009104, Global Avg Loss: 0.00521549, Time: 0.2138 Steps: 109400, Updated lr: 0.000056 ***** Running evaluation checkpoint-109428 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-109428 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1091.679741, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001768, "eval_total_loss": 1.884516, "eval_acc": 0.999744, "eval_jaccard": 0.989093, "eval_prec": 0.989777, "eval_recall": 0.990743, "eval_f1": 0.98995, "eval_pr_auc": 0.995288, "eval_roc_auc": 0.999393, "eval_fmax": 0.994842, "eval_pmax": 0.99753, "eval_rmax": 0.99217, "eval_tmax": 0.13, "update_flag": true, "test_avg_loss": 0.001927, "test_total_loss": 2.054589, "test_acc": 0.999758, "test_jaccard": 0.989297, "test_prec": 0.990013, "test_recall": 0.990833, "test_f1": 0.990124, "test_pr_auc": 0.995121, "test_roc_auc": 0.999102, "test_fmax": 0.994828, "test_pmax": 0.997, "test_rmax": 0.992665, "test_tmax": 0.08, "lr": 5.622607993540574e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.005214247861265154, "train_cur_epoch_loss": 0.6319522595764617, "train_cur_epoch_avg_loss": 0.00012705111772747522, "train_cur_epoch_time": 1091.6797409057617, "train_cur_epoch_avg_time": 0.2194772297759875, "epoch": 22, "step": 109428} ################################################## Training, Epoch: 0023, Batch: 000172, Sample Num: 2752, Cur Loss: 0.00000660, Cur Avg Loss: 0.00013347, Log Avg loss: 0.00016544, Global Avg Loss: 0.00520627, Time: 0.2210 Steps: 109600, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 000372, Sample Num: 5952, Cur Loss: 0.00000280, Cur Avg Loss: 0.00012090, Log Avg loss: 0.00011009, Global Avg Loss: 0.00519699, Time: 0.2195 Steps: 109800, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 000572, Sample Num: 9152, Cur Loss: 0.00001067, Cur Avg Loss: 0.00014045, Log Avg loss: 0.00017680, Global Avg Loss: 0.00518786, Time: 0.2211 Steps: 110000, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 000772, Sample Num: 12352, Cur Loss: 0.00000241, Cur Avg Loss: 0.00015459, Log Avg loss: 0.00019502, Global Avg Loss: 0.00517880, Time: 0.2202 Steps: 110200, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 000972, Sample Num: 15552, Cur Loss: 0.00002807, Cur Avg Loss: 0.00014251, Log Avg loss: 0.00009592, Global Avg Loss: 0.00516959, Time: 0.1089 Steps: 110400, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 001172, Sample Num: 18752, Cur Loss: 0.00000696, Cur Avg Loss: 0.00013716, Log Avg loss: 0.00011116, Global Avg Loss: 0.00516045, Time: 0.2240 Steps: 110600, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 001372, Sample Num: 21952, Cur Loss: 0.00001682, Cur Avg Loss: 0.00012811, Log Avg loss: 0.00007504, Global Avg Loss: 0.00515127, Time: 0.2179 Steps: 110800, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 001572, Sample Num: 25152, Cur Loss: 0.00000920, Cur Avg Loss: 0.00011770, Log Avg loss: 0.00004630, Global Avg Loss: 0.00514207, Time: 0.2474 Steps: 111000, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 001772, Sample Num: 28352, Cur Loss: 0.00000609, Cur Avg Loss: 0.00011905, Log Avg loss: 0.00012966, Global Avg Loss: 0.00513305, Time: 0.2211 Steps: 111200, Updated lr: 0.000056 Training, Epoch: 0023, Batch: 001972, Sample Num: 31552, Cur Loss: 0.00000126, Cur Avg Loss: 0.00011912, Log Avg loss: 0.00011978, Global Avg Loss: 0.00512405, Time: 0.2211 Steps: 111400, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 002172, Sample Num: 34752, Cur Loss: 0.00000686, Cur Avg Loss: 0.00012659, Log Avg loss: 0.00020022, Global Avg Loss: 0.00511523, Time: 0.2180 Steps: 111600, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 002372, Sample Num: 37952, Cur Loss: 0.00000394, Cur Avg Loss: 0.00013043, Log Avg loss: 0.00017215, Global Avg Loss: 0.00510639, Time: 0.1130 Steps: 111800, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 002572, Sample Num: 41152, Cur Loss: 0.00000296, Cur Avg Loss: 0.00012278, Log Avg loss: 0.00003204, Global Avg Loss: 0.00509733, Time: 0.2207 Steps: 112000, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 002772, Sample Num: 44352, Cur Loss: 0.00002792, Cur Avg Loss: 0.00012976, Log Avg loss: 0.00021957, Global Avg Loss: 0.00508863, Time: 0.2222 Steps: 112200, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 002972, Sample Num: 47552, Cur Loss: 0.00000510, Cur Avg Loss: 0.00013445, Log Avg loss: 0.00019939, Global Avg Loss: 0.00507993, Time: 0.2220 Steps: 112400, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 003172, Sample Num: 50752, Cur Loss: 0.00000738, Cur Avg Loss: 0.00013197, Log Avg loss: 0.00009509, Global Avg Loss: 0.00507108, Time: 0.2210 Steps: 112600, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 003372, Sample Num: 53952, Cur Loss: 0.00000122, Cur Avg Loss: 0.00013096, Log Avg loss: 0.00011501, Global Avg Loss: 0.00506229, Time: 0.2216 Steps: 112800, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 003572, Sample Num: 57152, Cur Loss: 0.00004794, Cur Avg Loss: 0.00013199, Log Avg loss: 0.00014932, Global Avg Loss: 0.00505359, Time: 0.2124 Steps: 113000, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 003772, Sample Num: 60352, Cur Loss: 0.00000201, Cur Avg Loss: 0.00012790, Log Avg loss: 0.00005490, Global Avg Loss: 0.00504476, Time: 0.2175 Steps: 113200, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 003972, Sample Num: 63552, Cur Loss: 0.00000080, Cur Avg Loss: 0.00012520, Log Avg loss: 0.00007434, Global Avg Loss: 0.00503600, Time: 0.2188 Steps: 113400, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 004172, Sample Num: 66752, Cur Loss: 0.00001502, Cur Avg Loss: 0.00012222, Log Avg loss: 0.00006298, Global Avg Loss: 0.00502724, Time: 0.2230 Steps: 113600, Updated lr: 0.000055 Training, Epoch: 0023, Batch: 004372, Sample Num: 69952, Cur Loss: 0.00002304, Cur Avg Loss: 0.00012035, Log Avg loss: 0.00008134, Global Avg Loss: 0.00501855, Time: 0.2189 Steps: 113800, Updated lr: 0.000054 Training, Epoch: 0023, Batch: 004572, Sample Num: 73152, Cur Loss: 0.00005804, Cur Avg Loss: 0.00012137, Log Avg loss: 0.00014368, Global Avg Loss: 0.00501000, Time: 0.2004 Steps: 114000, Updated lr: 0.000054 Training, Epoch: 0023, Batch: 004772, Sample Num: 76352, Cur Loss: 0.00000937, Cur Avg Loss: 0.00012347, Log Avg loss: 0.00017147, Global Avg Loss: 0.00500152, Time: 0.2296 Steps: 114200, Updated lr: 0.000054 Training, Epoch: 0023, Batch: 004972, Sample Num: 79552, Cur Loss: 0.00000744, Cur Avg Loss: 0.00012395, Log Avg loss: 0.00013549, Global Avg Loss: 0.00499302, Time: 0.2175 Steps: 114400, Updated lr: 0.000054 ***** Running evaluation checkpoint-114402 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-114402 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1090.515763, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001806, "eval_total_loss": 1.924919, "eval_acc": 0.999742, "eval_jaccard": 0.98909, "eval_prec": 0.989845, "eval_recall": 0.990611, "eval_f1": 0.989922, "eval_pr_auc": 0.995367, "eval_roc_auc": 0.999392, "eval_fmax": 0.994897, "eval_pmax": 0.997193, "eval_rmax": 0.992611, "eval_tmax": 0.08, "update_flag": false, "test_avg_loss": 0.001931, "test_total_loss": 2.058158, "test_acc": 0.99975, "test_jaccard": 0.989119, "test_prec": 0.990023, "test_recall": 0.990642, "test_f1": 0.990001, "test_pr_auc": 0.995235, "test_roc_auc": 0.999119, "test_fmax": 0.994898, "test_pmax": 0.996825, "test_rmax": 0.992978, "test_tmax": 0.07, "lr": 5.421800565199838e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0049929286630677845, "train_cur_epoch_loss": 0.6163099497581701, "train_cur_epoch_avg_loss": 0.00012390630272580822, "train_cur_epoch_time": 1090.5157630443573, "train_cur_epoch_avg_time": 0.21924321733903443, "epoch": 23, "step": 114402} ################################################## Training, Epoch: 0024, Batch: 000198, Sample Num: 3168, Cur Loss: 0.00000889, Cur Avg Loss: 0.00012315, Log Avg loss: 0.00012195, Global Avg Loss: 0.00498451, Time: 0.2247 Steps: 114600, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 000398, Sample Num: 6368, Cur Loss: 0.00003664, Cur Avg Loss: 0.00010713, Log Avg loss: 0.00009127, Global Avg Loss: 0.00497599, Time: 0.2230 Steps: 114800, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 000598, Sample Num: 9568, Cur Loss: 0.00000377, Cur Avg Loss: 0.00013043, Log Avg loss: 0.00017678, Global Avg Loss: 0.00496764, Time: 0.2209 Steps: 115000, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 000798, Sample Num: 12768, Cur Loss: 0.00000038, Cur Avg Loss: 0.00013778, Log Avg loss: 0.00015976, Global Avg Loss: 0.00495930, Time: 0.0773 Steps: 115200, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 000998, Sample Num: 15968, Cur Loss: 0.00000632, Cur Avg Loss: 0.00012584, Log Avg loss: 0.00007821, Global Avg Loss: 0.00495084, Time: 0.2212 Steps: 115400, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 001198, Sample Num: 19168, Cur Loss: 0.00000429, Cur Avg Loss: 0.00012599, Log Avg loss: 0.00012675, Global Avg Loss: 0.00494249, Time: 0.2190 Steps: 115600, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 001398, Sample Num: 22368, Cur Loss: 0.00000064, Cur Avg Loss: 0.00011409, Log Avg loss: 0.00004280, Global Avg Loss: 0.00493403, Time: 0.2191 Steps: 115800, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 001598, Sample Num: 25568, Cur Loss: 0.00000182, Cur Avg Loss: 0.00010579, Log Avg loss: 0.00004776, Global Avg Loss: 0.00492560, Time: 0.2274 Steps: 116000, Updated lr: 0.000054 Training, Epoch: 0024, Batch: 001798, Sample Num: 28768, Cur Loss: 0.00000325, Cur Avg Loss: 0.00010397, Log Avg loss: 0.00008944, Global Avg Loss: 0.00491728, Time: 0.2195 Steps: 116200, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 001998, Sample Num: 31968, Cur Loss: 0.00000310, Cur Avg Loss: 0.00009945, Log Avg loss: 0.00005876, Global Avg Loss: 0.00490893, Time: 0.2198 Steps: 116400, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 002198, Sample Num: 35168, Cur Loss: 0.00000105, Cur Avg Loss: 0.00011385, Log Avg loss: 0.00025776, Global Avg Loss: 0.00490095, Time: 0.2200 Steps: 116600, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 002398, Sample Num: 38368, Cur Loss: 0.00001590, Cur Avg Loss: 0.00011160, Log Avg loss: 0.00008688, Global Avg Loss: 0.00489271, Time: 0.0880 Steps: 116800, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 002598, Sample Num: 41568, Cur Loss: 0.00001733, Cur Avg Loss: 0.00010928, Log Avg loss: 0.00008148, Global Avg Loss: 0.00488449, Time: 0.2198 Steps: 117000, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 002798, Sample Num: 44768, Cur Loss: 0.00000080, Cur Avg Loss: 0.00011188, Log Avg loss: 0.00014564, Global Avg Loss: 0.00487640, Time: 0.0770 Steps: 117200, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 002998, Sample Num: 47968, Cur Loss: 0.00001811, Cur Avg Loss: 0.00011675, Log Avg loss: 0.00018493, Global Avg Loss: 0.00486841, Time: 0.2248 Steps: 117400, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 003198, Sample Num: 51168, Cur Loss: 0.00000691, Cur Avg Loss: 0.00011227, Log Avg loss: 0.00004498, Global Avg Loss: 0.00486020, Time: 0.1314 Steps: 117600, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 003398, Sample Num: 54368, Cur Loss: 0.00000250, Cur Avg Loss: 0.00011254, Log Avg loss: 0.00011694, Global Avg Loss: 0.00485215, Time: 0.2209 Steps: 117800, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 003598, Sample Num: 57568, Cur Loss: 0.00000080, Cur Avg Loss: 0.00011167, Log Avg loss: 0.00009685, Global Avg Loss: 0.00484409, Time: 0.2198 Steps: 118000, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 003798, Sample Num: 60768, Cur Loss: 0.00000617, Cur Avg Loss: 0.00010925, Log Avg loss: 0.00006574, Global Avg Loss: 0.00483601, Time: 0.2200 Steps: 118200, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 003998, Sample Num: 63968, Cur Loss: 0.00000161, Cur Avg Loss: 0.00010885, Log Avg loss: 0.00010130, Global Avg Loss: 0.00482801, Time: 0.2203 Steps: 118400, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 004198, Sample Num: 67168, Cur Loss: 0.00003903, Cur Avg Loss: 0.00010674, Log Avg loss: 0.00006454, Global Avg Loss: 0.00481998, Time: 0.2181 Steps: 118600, Updated lr: 0.000053 Training, Epoch: 0024, Batch: 004398, Sample Num: 70368, Cur Loss: 0.00001045, Cur Avg Loss: 0.00010538, Log Avg loss: 0.00007685, Global Avg Loss: 0.00481199, Time: 0.2223 Steps: 118800, Updated lr: 0.000052 Training, Epoch: 0024, Batch: 004598, Sample Num: 73568, Cur Loss: 0.00000071, Cur Avg Loss: 0.00010743, Log Avg loss: 0.00015240, Global Avg Loss: 0.00480416, Time: 0.3939 Steps: 119000, Updated lr: 0.000052 Training, Epoch: 0024, Batch: 004798, Sample Num: 76768, Cur Loss: 0.00018361, Cur Avg Loss: 0.00011040, Log Avg loss: 0.00017865, Global Avg Loss: 0.00479640, Time: 0.7048 Steps: 119200, Updated lr: 0.000052 ***** Running evaluation checkpoint-119376 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-119376 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1093.630895, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001867, "eval_total_loss": 1.990723, "eval_acc": 0.999738, "eval_jaccard": 0.988797, "eval_prec": 0.989635, "eval_recall": 0.990295, "eval_f1": 0.989654, "eval_pr_auc": 0.995216, "eval_roc_auc": 0.999371, "eval_fmax": 0.994835, "eval_pmax": 0.997652, "eval_rmax": 0.992035, "eval_tmax": 0.12, "update_flag": false, "test_avg_loss": 0.002047, "test_total_loss": 2.182022, "test_acc": 0.999751, "test_jaccard": 0.988993, "test_prec": 0.989871, "test_recall": 0.99031, "test_f1": 0.989791, "test_pr_auc": 0.995114, "test_roc_auc": 0.999089, "test_fmax": 0.994908, "test_pmax": 0.997674, "test_rmax": 0.992157, "test_tmax": 0.13, "lr": 5.220993136859104e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.004789547400885836, "train_cur_epoch_loss": 0.5559856158658789, "train_cur_epoch_avg_loss": 0.00011177837070082004, "train_cur_epoch_time": 1093.6308951377869, "train_cur_epoch_avg_time": 0.21986950042979228, "epoch": 24, "step": 119376} ################################################## Training, Epoch: 0025, Batch: 000024, Sample Num: 384, Cur Loss: 0.00001000, Cur Avg Loss: 0.00002604, Log Avg loss: 0.00013465, Global Avg Loss: 0.00478859, Time: 0.2218 Steps: 119400, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 000224, Sample Num: 3584, Cur Loss: 0.00001069, Cur Avg Loss: 0.00008805, Log Avg loss: 0.00009550, Global Avg Loss: 0.00478074, Time: 0.2634 Steps: 119600, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 000424, Sample Num: 6784, Cur Loss: 0.00000340, Cur Avg Loss: 0.00010953, Log Avg loss: 0.00013359, Global Avg Loss: 0.00477298, Time: 0.2208 Steps: 119800, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 000624, Sample Num: 9984, Cur Loss: 0.00000135, Cur Avg Loss: 0.00010801, Log Avg loss: 0.00010479, Global Avg Loss: 0.00476520, Time: 0.2221 Steps: 120000, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 000824, Sample Num: 13184, Cur Loss: 0.00002550, Cur Avg Loss: 0.00012071, Log Avg loss: 0.00016034, Global Avg Loss: 0.00475754, Time: 0.2197 Steps: 120200, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 001024, Sample Num: 16384, Cur Loss: 0.00000015, Cur Avg Loss: 0.00011036, Log Avg loss: 0.00006772, Global Avg Loss: 0.00474975, Time: 0.2451 Steps: 120400, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 001224, Sample Num: 19584, Cur Loss: 0.00000544, Cur Avg Loss: 0.00010958, Log Avg loss: 0.00010557, Global Avg Loss: 0.00474205, Time: 0.2204 Steps: 120600, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 001424, Sample Num: 22784, Cur Loss: 0.00023610, Cur Avg Loss: 0.00009779, Log Avg loss: 0.00002566, Global Avg Loss: 0.00473424, Time: 0.2235 Steps: 120800, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 001624, Sample Num: 25984, Cur Loss: 0.00000036, Cur Avg Loss: 0.00009675, Log Avg loss: 0.00008934, Global Avg Loss: 0.00472656, Time: 0.2187 Steps: 121000, Updated lr: 0.000052 Training, Epoch: 0025, Batch: 001824, Sample Num: 29184, Cur Loss: 0.00000080, Cur Avg Loss: 0.00009425, Log Avg loss: 0.00007396, Global Avg Loss: 0.00471889, Time: 0.2241 Steps: 121200, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 002024, Sample Num: 32384, Cur Loss: 0.00000394, Cur Avg Loss: 0.00009214, Log Avg loss: 0.00007287, Global Avg Loss: 0.00471123, Time: 0.2131 Steps: 121400, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 002224, Sample Num: 35584, Cur Loss: 0.00000262, Cur Avg Loss: 0.00010638, Log Avg loss: 0.00025053, Global Avg Loss: 0.00470389, Time: 0.2205 Steps: 121600, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 002424, Sample Num: 38784, Cur Loss: 0.00000760, Cur Avg Loss: 0.00010898, Log Avg loss: 0.00013784, Global Avg Loss: 0.00469640, Time: 0.2206 Steps: 121800, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 002624, Sample Num: 41984, Cur Loss: 0.00000016, Cur Avg Loss: 0.00011434, Log Avg loss: 0.00017932, Global Avg Loss: 0.00468899, Time: 0.2380 Steps: 122000, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 002824, Sample Num: 45184, Cur Loss: 0.00000122, Cur Avg Loss: 0.00011543, Log Avg loss: 0.00012978, Global Avg Loss: 0.00468153, Time: 0.2178 Steps: 122200, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 003024, Sample Num: 48384, Cur Loss: 0.00000907, Cur Avg Loss: 0.00011538, Log Avg loss: 0.00011455, Global Avg Loss: 0.00467407, Time: 0.1480 Steps: 122400, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 003224, Sample Num: 51584, Cur Loss: 0.00000460, Cur Avg Loss: 0.00011151, Log Avg loss: 0.00005300, Global Avg Loss: 0.00466653, Time: 0.2192 Steps: 122600, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 003424, Sample Num: 54784, Cur Loss: 0.00000019, Cur Avg Loss: 0.00011205, Log Avg loss: 0.00012075, Global Avg Loss: 0.00465913, Time: 0.2604 Steps: 122800, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 003624, Sample Num: 57984, Cur Loss: 0.00001133, Cur Avg Loss: 0.00011288, Log Avg loss: 0.00012709, Global Avg Loss: 0.00465176, Time: 0.2201 Steps: 123000, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 003824, Sample Num: 61184, Cur Loss: 0.00000025, Cur Avg Loss: 0.00010886, Log Avg loss: 0.00003601, Global Avg Loss: 0.00464426, Time: 0.2222 Steps: 123200, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 004024, Sample Num: 64384, Cur Loss: 0.00002524, Cur Avg Loss: 0.00010732, Log Avg loss: 0.00007791, Global Avg Loss: 0.00463686, Time: 0.2216 Steps: 123400, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 004224, Sample Num: 67584, Cur Loss: 0.00004372, Cur Avg Loss: 0.00010574, Log Avg loss: 0.00007404, Global Avg Loss: 0.00462948, Time: 0.2540 Steps: 123600, Updated lr: 0.000051 Training, Epoch: 0025, Batch: 004424, Sample Num: 70784, Cur Loss: 0.00000006, Cur Avg Loss: 0.00010378, Log Avg loss: 0.00006236, Global Avg Loss: 0.00462210, Time: 0.2261 Steps: 123800, Updated lr: 0.000050 Training, Epoch: 0025, Batch: 004624, Sample Num: 73984, Cur Loss: 0.00001187, Cur Avg Loss: 0.00010525, Log Avg loss: 0.00013762, Global Avg Loss: 0.00461487, Time: 0.2195 Steps: 124000, Updated lr: 0.000050 Training, Epoch: 0025, Batch: 004824, Sample Num: 77184, Cur Loss: 0.00003706, Cur Avg Loss: 0.00010910, Log Avg loss: 0.00019825, Global Avg Loss: 0.00460776, Time: 0.1695 Steps: 124200, Updated lr: 0.000050 ***** Running evaluation checkpoint-124350 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-124350 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1094.526140, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001846, "eval_total_loss": 1.968072, "eval_acc": 0.999737, "eval_jaccard": 0.989077, "eval_prec": 0.989875, "eval_recall": 0.990725, "eval_f1": 0.989968, "eval_pr_auc": 0.995427, "eval_roc_auc": 0.999376, "eval_fmax": 0.994837, "eval_pmax": 0.997476, "eval_rmax": 0.992211, "eval_tmax": 0.12, "update_flag": true, "test_avg_loss": 0.001963, "test_total_loss": 2.092087, "test_acc": 0.999748, "test_jaccard": 0.989236, "test_prec": 0.990042, "test_recall": 0.990833, "test_f1": 0.990107, "test_pr_auc": 0.995359, "test_roc_auc": 0.999116, "test_fmax": 0.994952, "test_pmax": 0.997704, "test_rmax": 0.992215, "test_tmax": 0.22, "lr": 5.020185708518369e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.00460231751366396, "train_cur_epoch_loss": 0.5411722959649947, "train_cur_epoch_avg_loss": 0.0001088002203387605, "train_cur_epoch_time": 1094.5261402130127, "train_cur_epoch_avg_time": 0.2200494853665084, "epoch": 25, "step": 124350} ################################################## Training, Epoch: 0026, Batch: 000050, Sample Num: 800, Cur Loss: 0.00000133, Cur Avg Loss: 0.00001911, Log Avg loss: 0.00007912, Global Avg Loss: 0.00460048, Time: 0.2176 Steps: 124400, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 000250, Sample Num: 4000, Cur Loss: 0.00001042, Cur Avg Loss: 0.00007654, Log Avg loss: 0.00009089, Global Avg Loss: 0.00459324, Time: 0.2151 Steps: 124600, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 000450, Sample Num: 7200, Cur Loss: 0.00001116, Cur Avg Loss: 0.00009416, Log Avg loss: 0.00011620, Global Avg Loss: 0.00458606, Time: 0.2302 Steps: 124800, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 000650, Sample Num: 10400, Cur Loss: 0.00002532, Cur Avg Loss: 0.00010212, Log Avg loss: 0.00012002, Global Avg Loss: 0.00457892, Time: 0.0853 Steps: 125000, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 000850, Sample Num: 13600, Cur Loss: 0.00000060, Cur Avg Loss: 0.00011765, Log Avg loss: 0.00016812, Global Avg Loss: 0.00457187, Time: 0.2218 Steps: 125200, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 001050, Sample Num: 16800, Cur Loss: 0.00000049, Cur Avg Loss: 0.00010762, Log Avg loss: 0.00006500, Global Avg Loss: 0.00456468, Time: 0.2205 Steps: 125400, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 001250, Sample Num: 20000, Cur Loss: 0.00000381, Cur Avg Loss: 0.00010757, Log Avg loss: 0.00010729, Global Avg Loss: 0.00455758, Time: 0.2230 Steps: 125600, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 001450, Sample Num: 23200, Cur Loss: 0.00000341, Cur Avg Loss: 0.00009817, Log Avg loss: 0.00003944, Global Avg Loss: 0.00455040, Time: 0.2219 Steps: 125800, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 001650, Sample Num: 26400, Cur Loss: 0.00000634, Cur Avg Loss: 0.00009443, Log Avg loss: 0.00006726, Global Avg Loss: 0.00454329, Time: 0.2272 Steps: 126000, Updated lr: 0.000050 Training, Epoch: 0026, Batch: 001850, Sample Num: 29600, Cur Loss: 0.00000413, Cur Avg Loss: 0.00009402, Log Avg loss: 0.00009068, Global Avg Loss: 0.00453623, Time: 0.2219 Steps: 126200, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 002050, Sample Num: 32800, Cur Loss: 0.00000181, Cur Avg Loss: 0.00009539, Log Avg loss: 0.00010807, Global Avg Loss: 0.00452922, Time: 0.0885 Steps: 126400, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 002250, Sample Num: 36000, Cur Loss: 0.00000691, Cur Avg Loss: 0.00010141, Log Avg loss: 0.00016309, Global Avg Loss: 0.00452233, Time: 0.2390 Steps: 126600, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 002450, Sample Num: 39200, Cur Loss: 0.00000023, Cur Avg Loss: 0.00010257, Log Avg loss: 0.00011558, Global Avg Loss: 0.00451537, Time: 0.2198 Steps: 126800, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 002650, Sample Num: 42400, Cur Loss: 0.00003541, Cur Avg Loss: 0.00010419, Log Avg loss: 0.00012412, Global Avg Loss: 0.00450846, Time: 0.7066 Steps: 127000, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 002850, Sample Num: 45600, Cur Loss: 0.00000188, Cur Avg Loss: 0.00010659, Log Avg loss: 0.00013840, Global Avg Loss: 0.00450159, Time: 0.1097 Steps: 127200, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 003050, Sample Num: 48800, Cur Loss: 0.00000115, Cur Avg Loss: 0.00010854, Log Avg loss: 0.00013623, Global Avg Loss: 0.00449473, Time: 0.1316 Steps: 127400, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 003250, Sample Num: 52000, Cur Loss: 0.00001279, Cur Avg Loss: 0.00010462, Log Avg loss: 0.00004488, Global Avg Loss: 0.00448776, Time: 0.3449 Steps: 127600, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 003450, Sample Num: 55200, Cur Loss: 0.00000175, Cur Avg Loss: 0.00010538, Log Avg loss: 0.00011777, Global Avg Loss: 0.00448092, Time: 0.2218 Steps: 127800, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 003650, Sample Num: 58400, Cur Loss: 0.00000079, Cur Avg Loss: 0.00010616, Log Avg loss: 0.00011957, Global Avg Loss: 0.00447411, Time: 0.2285 Steps: 128000, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 003850, Sample Num: 61600, Cur Loss: 0.00000728, Cur Avg Loss: 0.00010323, Log Avg loss: 0.00004972, Global Avg Loss: 0.00446720, Time: 0.2168 Steps: 128200, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 004050, Sample Num: 64800, Cur Loss: 0.00000330, Cur Avg Loss: 0.00010270, Log Avg loss: 0.00009255, Global Avg Loss: 0.00446039, Time: 0.2182 Steps: 128400, Updated lr: 0.000049 Training, Epoch: 0026, Batch: 004250, Sample Num: 68000, Cur Loss: 0.00000024, Cur Avg Loss: 0.00010182, Log Avg loss: 0.00008401, Global Avg Loss: 0.00445358, Time: 0.0842 Steps: 128600, Updated lr: 0.000048 Training, Epoch: 0026, Batch: 004450, Sample Num: 71200, Cur Loss: 0.00336422, Cur Avg Loss: 0.00009954, Log Avg loss: 0.00005111, Global Avg Loss: 0.00444675, Time: 0.2196 Steps: 128800, Updated lr: 0.000048 Training, Epoch: 0026, Batch: 004650, Sample Num: 74400, Cur Loss: 0.00002343, Cur Avg Loss: 0.00010046, Log Avg loss: 0.00012097, Global Avg Loss: 0.00444004, Time: 0.2212 Steps: 129000, Updated lr: 0.000048 Training, Epoch: 0026, Batch: 004850, Sample Num: 77600, Cur Loss: 0.00000724, Cur Avg Loss: 0.00010393, Log Avg loss: 0.00018444, Global Avg Loss: 0.00443345, Time: 0.2175 Steps: 129200, Updated lr: 0.000048 ***** Running evaluation checkpoint-129324 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-129324 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1096.746949, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.00184, "eval_total_loss": 1.961172, "eval_acc": 0.999748, "eval_jaccard": 0.989227, "eval_prec": 0.990065, "eval_recall": 0.990637, "eval_f1": 0.990051, "eval_pr_auc": 0.995416, "eval_roc_auc": 0.999357, "eval_fmax": 0.994822, "eval_pmax": 0.997417, "eval_rmax": 0.99224, "eval_tmax": 0.11, "update_flag": true, "test_avg_loss": 0.001982, "test_total_loss": 2.113219, "test_acc": 0.999758, "test_jaccard": 0.989236, "test_prec": 0.99015, "test_recall": 0.990691, "test_f1": 0.990109, "test_pr_auc": 0.99526, "test_roc_auc": 0.999106, "test_fmax": 0.99489, "test_pmax": 0.997777, "test_rmax": 0.99202, "test_tmax": 0.15, "lr": 4.8193782801776346e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.004429352771686358, "train_cur_epoch_loss": 0.5234350214550361, "train_cur_epoch_avg_loss": 0.00010523422224669001, "train_cur_epoch_time": 1096.7469487190247, "train_cur_epoch_avg_time": 0.22049596878146857, "epoch": 26, "step": 129324} ################################################## Training, Epoch: 0027, Batch: 000076, Sample Num: 1216, Cur Loss: 0.00000169, Cur Avg Loss: 0.00014557, Log Avg loss: 0.00015231, Global Avg Loss: 0.00442684, Time: 0.2193 Steps: 129400, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 000276, Sample Num: 4416, Cur Loss: 0.00005000, Cur Avg Loss: 0.00007924, Log Avg loss: 0.00005404, Global Avg Loss: 0.00442009, Time: 0.2214 Steps: 129600, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 000476, Sample Num: 7616, Cur Loss: 0.00000926, Cur Avg Loss: 0.00009599, Log Avg loss: 0.00011910, Global Avg Loss: 0.00441346, Time: 0.2182 Steps: 129800, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 000676, Sample Num: 10816, Cur Loss: 0.00000197, Cur Avg Loss: 0.00009377, Log Avg loss: 0.00008850, Global Avg Loss: 0.00440681, Time: 0.1188 Steps: 130000, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 000876, Sample Num: 14016, Cur Loss: 0.00000297, Cur Avg Loss: 0.00010594, Log Avg loss: 0.00014704, Global Avg Loss: 0.00440026, Time: 0.2195 Steps: 130200, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 001076, Sample Num: 17216, Cur Loss: 0.00000369, Cur Avg Loss: 0.00009319, Log Avg loss: 0.00003738, Global Avg Loss: 0.00439357, Time: 0.2170 Steps: 130400, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 001276, Sample Num: 20416, Cur Loss: 0.00000128, Cur Avg Loss: 0.00009139, Log Avg loss: 0.00008170, Global Avg Loss: 0.00438697, Time: 0.1324 Steps: 130600, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 001476, Sample Num: 23616, Cur Loss: 0.00000137, Cur Avg Loss: 0.00008789, Log Avg loss: 0.00006553, Global Avg Loss: 0.00438036, Time: 0.2170 Steps: 130800, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 001676, Sample Num: 26816, Cur Loss: 0.00000042, Cur Avg Loss: 0.00008521, Log Avg loss: 0.00006544, Global Avg Loss: 0.00437377, Time: 0.2259 Steps: 131000, Updated lr: 0.000048 Training, Epoch: 0027, Batch: 001876, Sample Num: 30016, Cur Loss: 0.00000189, Cur Avg Loss: 0.00008338, Log Avg loss: 0.00006802, Global Avg Loss: 0.00436721, Time: 0.2291 Steps: 131200, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 002076, Sample Num: 33216, Cur Loss: 0.00000234, Cur Avg Loss: 0.00009016, Log Avg loss: 0.00015380, Global Avg Loss: 0.00436080, Time: 0.2199 Steps: 131400, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 002276, Sample Num: 36416, Cur Loss: 0.00001174, Cur Avg Loss: 0.00009339, Log Avg loss: 0.00012696, Global Avg Loss: 0.00435436, Time: 0.2200 Steps: 131600, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 002476, Sample Num: 39616, Cur Loss: 0.00001360, Cur Avg Loss: 0.00009310, Log Avg loss: 0.00008974, Global Avg Loss: 0.00434789, Time: 0.3419 Steps: 131800, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 002676, Sample Num: 42816, Cur Loss: 0.00013449, Cur Avg Loss: 0.00009606, Log Avg loss: 0.00013277, Global Avg Loss: 0.00434151, Time: 0.4387 Steps: 132000, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 002876, Sample Num: 46016, Cur Loss: 0.00002366, Cur Avg Loss: 0.00009880, Log Avg loss: 0.00013547, Global Avg Loss: 0.00433514, Time: 0.2185 Steps: 132200, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 003076, Sample Num: 49216, Cur Loss: 0.00001530, Cur Avg Loss: 0.00009990, Log Avg loss: 0.00011560, Global Avg Loss: 0.00432877, Time: 0.3957 Steps: 132400, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 003276, Sample Num: 52416, Cur Loss: 0.00000616, Cur Avg Loss: 0.00009773, Log Avg loss: 0.00006447, Global Avg Loss: 0.00432234, Time: 0.2209 Steps: 132600, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 003476, Sample Num: 55616, Cur Loss: 0.00000819, Cur Avg Loss: 0.00009810, Log Avg loss: 0.00010417, Global Avg Loss: 0.00431598, Time: 0.2161 Steps: 132800, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 003676, Sample Num: 58816, Cur Loss: 0.00000531, Cur Avg Loss: 0.00009732, Log Avg loss: 0.00008372, Global Avg Loss: 0.00430962, Time: 0.3940 Steps: 133000, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 003876, Sample Num: 62016, Cur Loss: 0.00000125, Cur Avg Loss: 0.00009322, Log Avg loss: 0.00001782, Global Avg Loss: 0.00430318, Time: 0.2182 Steps: 133200, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 004076, Sample Num: 65216, Cur Loss: 0.00000436, Cur Avg Loss: 0.00009380, Log Avg loss: 0.00010514, Global Avg Loss: 0.00429688, Time: 0.2188 Steps: 133400, Updated lr: 0.000047 Training, Epoch: 0027, Batch: 004276, Sample Num: 68416, Cur Loss: 0.00001063, Cur Avg Loss: 0.00009389, Log Avg loss: 0.00009574, Global Avg Loss: 0.00429059, Time: 0.2288 Steps: 133600, Updated lr: 0.000046 Training, Epoch: 0027, Batch: 004476, Sample Num: 71616, Cur Loss: 0.00000373, Cur Avg Loss: 0.00009339, Log Avg loss: 0.00008257, Global Avg Loss: 0.00428430, Time: 0.2212 Steps: 133800, Updated lr: 0.000046 Training, Epoch: 0027, Batch: 004676, Sample Num: 74816, Cur Loss: 0.00000588, Cur Avg Loss: 0.00009244, Log Avg loss: 0.00007123, Global Avg Loss: 0.00427801, Time: 0.2226 Steps: 134000, Updated lr: 0.000046 Training, Epoch: 0027, Batch: 004876, Sample Num: 78016, Cur Loss: 0.00000930, Cur Avg Loss: 0.00009418, Log Avg loss: 0.00013483, Global Avg Loss: 0.00427184, Time: 0.2193 Steps: 134200, Updated lr: 0.000046 ***** Running evaluation checkpoint-134298 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-134298 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1094.700123, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001886, "eval_total_loss": 2.010126, "eval_acc": 0.999731, "eval_jaccard": 0.988513, "eval_prec": 0.989386, "eval_recall": 0.990129, "eval_f1": 0.989408, "eval_pr_auc": 0.995197, "eval_roc_auc": 0.999375, "eval_fmax": 0.994566, "eval_pmax": 0.997941, "eval_rmax": 0.991214, "eval_tmax": 0.26, "update_flag": false, "test_avg_loss": 0.002024, "test_total_loss": 2.157814, "test_acc": 0.999755, "test_jaccard": 0.98926, "test_prec": 0.990018, "test_recall": 0.990857, "test_f1": 0.990118, "test_pr_auc": 0.99521, "test_roc_auc": 0.999097, "test_fmax": 0.994772, "test_pmax": 0.997846, "test_rmax": 0.991717, "test_tmax": 0.22, "lr": 4.6185708518368994e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0042688518096080565, "train_cur_epoch_loss": 0.4766424811760954, "train_cur_epoch_avg_loss": 9.58267955721945e-05, "train_cur_epoch_time": 1094.700122833252, "train_cur_epoch_avg_time": 0.22008446377829755, "epoch": 27, "step": 134298} ################################################## Training, Epoch: 0028, Batch: 000102, Sample Num: 1632, Cur Loss: 0.00005809, Cur Avg Loss: 0.00009527, Log Avg loss: 0.00013571, Global Avg Loss: 0.00426568, Time: 0.2192 Steps: 134400, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 000302, Sample Num: 4832, Cur Loss: 0.00000198, Cur Avg Loss: 0.00006788, Log Avg loss: 0.00005391, Global Avg Loss: 0.00425943, Time: 0.2012 Steps: 134600, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 000502, Sample Num: 8032, Cur Loss: 0.00000259, Cur Avg Loss: 0.00010536, Log Avg loss: 0.00016196, Global Avg Loss: 0.00425335, Time: 0.0865 Steps: 134800, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 000702, Sample Num: 11232, Cur Loss: 0.00000256, Cur Avg Loss: 0.00010237, Log Avg loss: 0.00009485, Global Avg Loss: 0.00424719, Time: 0.2236 Steps: 135000, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 000902, Sample Num: 14432, Cur Loss: 0.00013690, Cur Avg Loss: 0.00011883, Log Avg loss: 0.00017660, Global Avg Loss: 0.00424116, Time: 0.2237 Steps: 135200, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 001102, Sample Num: 17632, Cur Loss: 0.00000048, Cur Avg Loss: 0.00011051, Log Avg loss: 0.00007298, Global Avg Loss: 0.00423501, Time: 0.2005 Steps: 135400, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 001302, Sample Num: 20832, Cur Loss: 0.00000034, Cur Avg Loss: 0.00010477, Log Avg loss: 0.00007313, Global Avg Loss: 0.00422887, Time: 0.3951 Steps: 135600, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 001502, Sample Num: 24032, Cur Loss: 0.00000631, Cur Avg Loss: 0.00009666, Log Avg loss: 0.00004393, Global Avg Loss: 0.00422271, Time: 0.2028 Steps: 135800, Updated lr: 0.000046 Training, Epoch: 0028, Batch: 001702, Sample Num: 27232, Cur Loss: 0.00000509, Cur Avg Loss: 0.00009770, Log Avg loss: 0.00010547, Global Avg Loss: 0.00421665, Time: 0.2222 Steps: 136000, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 001902, Sample Num: 30432, Cur Loss: 0.00000153, Cur Avg Loss: 0.00009020, Log Avg loss: 0.00002638, Global Avg Loss: 0.00421050, Time: 0.2208 Steps: 136200, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 002102, Sample Num: 33632, Cur Loss: 0.00000070, Cur Avg Loss: 0.00009650, Log Avg loss: 0.00015637, Global Avg Loss: 0.00420455, Time: 0.2227 Steps: 136400, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 002302, Sample Num: 36832, Cur Loss: 0.00011457, Cur Avg Loss: 0.00009879, Log Avg loss: 0.00012292, Global Avg Loss: 0.00419858, Time: 0.2203 Steps: 136600, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 002502, Sample Num: 40032, Cur Loss: 0.00000922, Cur Avg Loss: 0.00009343, Log Avg loss: 0.00003172, Global Avg Loss: 0.00419249, Time: 0.0857 Steps: 136800, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 002702, Sample Num: 43232, Cur Loss: 0.00000020, Cur Avg Loss: 0.00009551, Log Avg loss: 0.00012158, Global Avg Loss: 0.00418654, Time: 0.2231 Steps: 137000, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 002902, Sample Num: 46432, Cur Loss: 0.00008557, Cur Avg Loss: 0.00009951, Log Avg loss: 0.00015345, Global Avg Loss: 0.00418066, Time: 0.2193 Steps: 137200, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 003102, Sample Num: 49632, Cur Loss: 0.00000027, Cur Avg Loss: 0.00009816, Log Avg loss: 0.00007859, Global Avg Loss: 0.00417469, Time: 0.2895 Steps: 137400, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 003302, Sample Num: 52832, Cur Loss: 0.00000314, Cur Avg Loss: 0.00009517, Log Avg loss: 0.00004879, Global Avg Loss: 0.00416870, Time: 0.2125 Steps: 137600, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 003502, Sample Num: 56032, Cur Loss: 0.00001373, Cur Avg Loss: 0.00009752, Log Avg loss: 0.00013636, Global Avg Loss: 0.00416284, Time: 0.2046 Steps: 137800, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 003702, Sample Num: 59232, Cur Loss: 0.00000328, Cur Avg Loss: 0.00009617, Log Avg loss: 0.00007246, Global Avg Loss: 0.00415691, Time: 0.2143 Steps: 138000, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 003902, Sample Num: 62432, Cur Loss: 0.00000144, Cur Avg Loss: 0.00009389, Log Avg loss: 0.00005171, Global Avg Loss: 0.00415097, Time: 0.2123 Steps: 138200, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 004102, Sample Num: 65632, Cur Loss: 0.00012232, Cur Avg Loss: 0.00009180, Log Avg loss: 0.00005111, Global Avg Loss: 0.00414505, Time: 0.2282 Steps: 138400, Updated lr: 0.000045 Training, Epoch: 0028, Batch: 004302, Sample Num: 68832, Cur Loss: 0.00000360, Cur Avg Loss: 0.00009006, Log Avg loss: 0.00005426, Global Avg Loss: 0.00413915, Time: 0.2198 Steps: 138600, Updated lr: 0.000044 Training, Epoch: 0028, Batch: 004502, Sample Num: 72032, Cur Loss: 0.00001126, Cur Avg Loss: 0.00009182, Log Avg loss: 0.00012971, Global Avg Loss: 0.00413337, Time: 0.2216 Steps: 138800, Updated lr: 0.000044 Training, Epoch: 0028, Batch: 004702, Sample Num: 75232, Cur Loss: 0.00001735, Cur Avg Loss: 0.00008970, Log Avg loss: 0.00004190, Global Avg Loss: 0.00412748, Time: 0.2214 Steps: 139000, Updated lr: 0.000044 Training, Epoch: 0028, Batch: 004902, Sample Num: 78432, Cur Loss: 0.00000091, Cur Avg Loss: 0.00009528, Log Avg loss: 0.00022650, Global Avg Loss: 0.00412188, Time: 0.2222 Steps: 139200, Updated lr: 0.000044 ***** Running evaluation checkpoint-139272 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-139272 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1096.259225, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001864, "eval_total_loss": 1.987382, "eval_acc": 0.999746, "eval_jaccard": 0.988987, "eval_prec": 0.989787, "eval_recall": 0.99051, "eval_f1": 0.989829, "eval_pr_auc": 0.995514, "eval_roc_auc": 0.99937, "eval_fmax": 0.994866, "eval_pmax": 0.998139, "eval_rmax": 0.991614, "eval_tmax": 0.23, "update_flag": false, "test_avg_loss": 0.001984, "test_total_loss": 2.114524, "test_acc": 0.999753, "test_jaccard": 0.989383, "test_prec": 0.990229, "test_recall": 0.990935, "test_f1": 0.990256, "test_pr_auc": 0.995128, "test_roc_auc": 0.999113, "test_fmax": 0.99498, "test_pmax": 0.997423, "test_rmax": 0.992548, "test_tmax": 0.14, "lr": 4.417763423496165e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.004119896487591805, "train_cur_epoch_loss": 0.4879632931439897, "train_cur_epoch_avg_loss": 9.810279315319455e-05, "train_cur_epoch_time": 1096.2592251300812, "train_cur_epoch_avg_time": 0.22039791417975094, "epoch": 28, "step": 139272} ################################################## Training, Epoch: 0029, Batch: 000128, Sample Num: 2048, Cur Loss: 0.00001584, Cur Avg Loss: 0.00008253, Log Avg loss: 0.00015739, Global Avg Loss: 0.00411619, Time: 0.2733 Steps: 139400, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 000328, Sample Num: 5248, Cur Loss: 0.00000208, Cur Avg Loss: 0.00007241, Log Avg loss: 0.00006593, Global Avg Loss: 0.00411039, Time: 0.2193 Steps: 139600, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 000528, Sample Num: 8448, Cur Loss: 0.00000441, Cur Avg Loss: 0.00010218, Log Avg loss: 0.00015102, Global Avg Loss: 0.00410472, Time: 0.2193 Steps: 139800, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 000728, Sample Num: 11648, Cur Loss: 0.00185369, Cur Avg Loss: 0.00010043, Log Avg loss: 0.00009582, Global Avg Loss: 0.00409900, Time: 0.2215 Steps: 140000, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 000928, Sample Num: 14848, Cur Loss: 0.00000390, Cur Avg Loss: 0.00010396, Log Avg loss: 0.00011678, Global Avg Loss: 0.00409331, Time: 0.2391 Steps: 140200, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 001128, Sample Num: 18048, Cur Loss: 0.00000045, Cur Avg Loss: 0.00009906, Log Avg loss: 0.00007634, Global Avg Loss: 0.00408759, Time: 0.2181 Steps: 140400, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 001328, Sample Num: 21248, Cur Loss: 0.00000085, Cur Avg Loss: 0.00009421, Log Avg loss: 0.00006684, Global Avg Loss: 0.00408187, Time: 0.2201 Steps: 140600, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 001528, Sample Num: 24448, Cur Loss: 0.00000104, Cur Avg Loss: 0.00009016, Log Avg loss: 0.00006327, Global Avg Loss: 0.00407616, Time: 0.2222 Steps: 140800, Updated lr: 0.000044 Training, Epoch: 0029, Batch: 001728, Sample Num: 27648, Cur Loss: 0.00000266, Cur Avg Loss: 0.00008962, Log Avg loss: 0.00008550, Global Avg Loss: 0.00407050, Time: 0.2378 Steps: 141000, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 001928, Sample Num: 30848, Cur Loss: 0.00002920, Cur Avg Loss: 0.00008341, Log Avg loss: 0.00002978, Global Avg Loss: 0.00406478, Time: 0.2229 Steps: 141200, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 002128, Sample Num: 34048, Cur Loss: 0.00000077, Cur Avg Loss: 0.00008909, Log Avg loss: 0.00014379, Global Avg Loss: 0.00405923, Time: 0.2218 Steps: 141400, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 002328, Sample Num: 37248, Cur Loss: 0.00000012, Cur Avg Loss: 0.00009323, Log Avg loss: 0.00013731, Global Avg Loss: 0.00405370, Time: 0.2200 Steps: 141600, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 002528, Sample Num: 40448, Cur Loss: 0.00000090, Cur Avg Loss: 0.00008837, Log Avg loss: 0.00003183, Global Avg Loss: 0.00404802, Time: 0.2384 Steps: 141800, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 002728, Sample Num: 43648, Cur Loss: 0.00000097, Cur Avg Loss: 0.00009252, Log Avg loss: 0.00014494, Global Avg Loss: 0.00404253, Time: 0.2208 Steps: 142000, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 002928, Sample Num: 46848, Cur Loss: 0.00002571, Cur Avg Loss: 0.00009492, Log Avg loss: 0.00012767, Global Avg Loss: 0.00403702, Time: 0.2223 Steps: 142200, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 003128, Sample Num: 50048, Cur Loss: 0.00000876, Cur Avg Loss: 0.00009406, Log Avg loss: 0.00008151, Global Avg Loss: 0.00403146, Time: 0.2195 Steps: 142400, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 003328, Sample Num: 53248, Cur Loss: 0.00000052, Cur Avg Loss: 0.00009253, Log Avg loss: 0.00006848, Global Avg Loss: 0.00402591, Time: 0.2583 Steps: 142600, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 003528, Sample Num: 56448, Cur Loss: 0.00000385, Cur Avg Loss: 0.00009294, Log Avg loss: 0.00009981, Global Avg Loss: 0.00402041, Time: 0.0910 Steps: 142800, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 003728, Sample Num: 59648, Cur Loss: 0.00000647, Cur Avg Loss: 0.00009102, Log Avg loss: 0.00005713, Global Avg Loss: 0.00401486, Time: 0.2221 Steps: 143000, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 003928, Sample Num: 62848, Cur Loss: 0.00000701, Cur Avg Loss: 0.00008713, Log Avg loss: 0.00001459, Global Avg Loss: 0.00400928, Time: 0.2189 Steps: 143200, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 004128, Sample Num: 66048, Cur Loss: 0.00001958, Cur Avg Loss: 0.00008659, Log Avg loss: 0.00007609, Global Avg Loss: 0.00400379, Time: 0.2555 Steps: 143400, Updated lr: 0.000043 Training, Epoch: 0029, Batch: 004328, Sample Num: 69248, Cur Loss: 0.00000561, Cur Avg Loss: 0.00008664, Log Avg loss: 0.00008762, Global Avg Loss: 0.00399834, Time: 0.2152 Steps: 143600, Updated lr: 0.000042 Training, Epoch: 0029, Batch: 004528, Sample Num: 72448, Cur Loss: 0.00002330, Cur Avg Loss: 0.00008836, Log Avg loss: 0.00012555, Global Avg Loss: 0.00399295, Time: 0.2242 Steps: 143800, Updated lr: 0.000042 Training, Epoch: 0029, Batch: 004728, Sample Num: 75648, Cur Loss: 0.00000140, Cur Avg Loss: 0.00008945, Log Avg loss: 0.00011415, Global Avg Loss: 0.00398756, Time: 0.2123 Steps: 144000, Updated lr: 0.000042 Training, Epoch: 0029, Batch: 004928, Sample Num: 78848, Cur Loss: 0.00000189, Cur Avg Loss: 0.00008954, Log Avg loss: 0.00009166, Global Avg Loss: 0.00398216, Time: 0.2367 Steps: 144200, Updated lr: 0.000042 ***** Running evaluation checkpoint-144246 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-144246 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1093.125263, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001944, "eval_total_loss": 2.072725, "eval_acc": 0.999738, "eval_jaccard": 0.988884, "eval_prec": 0.989889, "eval_recall": 0.990187, "eval_f1": 0.989728, "eval_pr_auc": 0.995205, "eval_roc_auc": 0.999358, "eval_fmax": 0.994986, "eval_pmax": 0.997619, "eval_rmax": 0.992367, "eval_tmax": 0.09, "update_flag": false, "test_avg_loss": 0.002089, "test_total_loss": 2.226618, "test_acc": 0.999753, "test_jaccard": 0.989164, "test_prec": 0.99012, "test_recall": 0.990525, "test_f1": 0.990017, "test_pr_auc": 0.995134, "test_roc_auc": 0.999099, "test_fmax": 0.994921, "test_pmax": 0.997413, "test_rmax": 0.99244, "test_tmax": 0.08, "lr": 4.21695599515543e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0039810349544924396, "train_cur_epoch_loss": 0.46214442582865756, "train_cur_epoch_avg_loss": 9.291202770982259e-05, "train_cur_epoch_time": 1093.1252627372742, "train_cur_epoch_avg_time": 0.21976784534323968, "epoch": 29, "step": 144246} ################################################## Training, Epoch: 0030, Batch: 000154, Sample Num: 2464, Cur Loss: 0.00000267, Cur Avg Loss: 0.00009664, Log Avg loss: 0.00017891, Global Avg Loss: 0.00397689, Time: 0.2207 Steps: 144400, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 000354, Sample Num: 5664, Cur Loss: 0.00000023, Cur Avg Loss: 0.00006869, Log Avg loss: 0.00004717, Global Avg Loss: 0.00397146, Time: 0.2164 Steps: 144600, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 000554, Sample Num: 8864, Cur Loss: 0.00332874, Cur Avg Loss: 0.00010780, Log Avg loss: 0.00017701, Global Avg Loss: 0.00396622, Time: 0.2202 Steps: 144800, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 000754, Sample Num: 12064, Cur Loss: 0.00000122, Cur Avg Loss: 0.00011560, Log Avg loss: 0.00013721, Global Avg Loss: 0.00396093, Time: 0.2186 Steps: 145000, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 000954, Sample Num: 15264, Cur Loss: 0.00000089, Cur Avg Loss: 0.00010804, Log Avg loss: 0.00007953, Global Avg Loss: 0.00395559, Time: 0.2217 Steps: 145200, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 001154, Sample Num: 18464, Cur Loss: 0.00000309, Cur Avg Loss: 0.00010118, Log Avg loss: 0.00006845, Global Avg Loss: 0.00395024, Time: 0.2224 Steps: 145400, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 001354, Sample Num: 21664, Cur Loss: 0.00000142, Cur Avg Loss: 0.00009258, Log Avg loss: 0.00004301, Global Avg Loss: 0.00394487, Time: 0.2204 Steps: 145600, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 001554, Sample Num: 24864, Cur Loss: 0.00031370, Cur Avg Loss: 0.00008625, Log Avg loss: 0.00004337, Global Avg Loss: 0.00393952, Time: 0.2280 Steps: 145800, Updated lr: 0.000042 Training, Epoch: 0030, Batch: 001754, Sample Num: 28064, Cur Loss: 0.00000334, Cur Avg Loss: 0.00008696, Log Avg loss: 0.00009247, Global Avg Loss: 0.00393425, Time: 0.2216 Steps: 146000, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 001954, Sample Num: 31264, Cur Loss: 0.00000202, Cur Avg Loss: 0.00008354, Log Avg loss: 0.00005358, Global Avg Loss: 0.00392894, Time: 0.2230 Steps: 146200, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 002154, Sample Num: 34464, Cur Loss: 0.00000057, Cur Avg Loss: 0.00008827, Log Avg loss: 0.00013447, Global Avg Loss: 0.00392376, Time: 0.2213 Steps: 146400, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 002354, Sample Num: 37664, Cur Loss: 0.00000095, Cur Avg Loss: 0.00009723, Log Avg loss: 0.00019375, Global Avg Loss: 0.00391867, Time: 0.3941 Steps: 146600, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 002554, Sample Num: 40864, Cur Loss: 0.00000068, Cur Avg Loss: 0.00009101, Log Avg loss: 0.00001771, Global Avg Loss: 0.00391336, Time: 0.2247 Steps: 146800, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 002754, Sample Num: 44064, Cur Loss: 0.00000107, Cur Avg Loss: 0.00009771, Log Avg loss: 0.00018338, Global Avg Loss: 0.00390828, Time: 0.2212 Steps: 147000, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 002954, Sample Num: 47264, Cur Loss: 0.00628850, Cur Avg Loss: 0.00010405, Log Avg loss: 0.00019123, Global Avg Loss: 0.00390323, Time: 0.2155 Steps: 147200, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 003154, Sample Num: 50464, Cur Loss: 0.00000291, Cur Avg Loss: 0.00009855, Log Avg loss: 0.00001736, Global Avg Loss: 0.00389796, Time: 0.2212 Steps: 147400, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 003354, Sample Num: 53664, Cur Loss: 0.00000687, Cur Avg Loss: 0.00009840, Log Avg loss: 0.00009601, Global Avg Loss: 0.00389281, Time: 0.2182 Steps: 147600, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 003554, Sample Num: 56864, Cur Loss: 0.00000234, Cur Avg Loss: 0.00009540, Log Avg loss: 0.00004518, Global Avg Loss: 0.00388760, Time: 0.2208 Steps: 147800, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 003754, Sample Num: 60064, Cur Loss: 0.00000081, Cur Avg Loss: 0.00009258, Log Avg loss: 0.00004237, Global Avg Loss: 0.00388240, Time: 0.0855 Steps: 148000, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 003954, Sample Num: 63264, Cur Loss: 0.00000820, Cur Avg Loss: 0.00008978, Log Avg loss: 0.00003730, Global Avg Loss: 0.00387722, Time: 0.2164 Steps: 148200, Updated lr: 0.000041 Training, Epoch: 0030, Batch: 004154, Sample Num: 66464, Cur Loss: 0.00001217, Cur Avg Loss: 0.00008822, Log Avg loss: 0.00005732, Global Avg Loss: 0.00387207, Time: 0.1237 Steps: 148400, Updated lr: 0.000040 Training, Epoch: 0030, Batch: 004354, Sample Num: 69664, Cur Loss: 0.00002244, Cur Avg Loss: 0.00008763, Log Avg loss: 0.00007545, Global Avg Loss: 0.00386696, Time: 0.2209 Steps: 148600, Updated lr: 0.000040 Training, Epoch: 0030, Batch: 004554, Sample Num: 72864, Cur Loss: 0.00000050, Cur Avg Loss: 0.00008801, Log Avg loss: 0.00009632, Global Avg Loss: 0.00386189, Time: 0.2199 Steps: 148800, Updated lr: 0.000040 Training, Epoch: 0030, Batch: 004754, Sample Num: 76064, Cur Loss: 0.00000516, Cur Avg Loss: 0.00008854, Log Avg loss: 0.00010066, Global Avg Loss: 0.00385684, Time: 0.2192 Steps: 149000, Updated lr: 0.000040 Training, Epoch: 0030, Batch: 004954, Sample Num: 79264, Cur Loss: 0.00000177, Cur Avg Loss: 0.00009143, Log Avg loss: 0.00015993, Global Avg Loss: 0.00385189, Time: 0.2144 Steps: 149200, Updated lr: 0.000040 ***** Running evaluation checkpoint-149220 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-149220 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1096.558808, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001893, "eval_total_loss": 2.017789, "eval_acc": 0.999741, "eval_jaccard": 0.988663, "eval_prec": 0.989484, "eval_recall": 0.990148, "eval_f1": 0.989505, "eval_pr_auc": 0.995468, "eval_roc_auc": 0.999389, "eval_fmax": 0.994701, "eval_pmax": 0.997421, "eval_rmax": 0.991996, "eval_tmax": 0.09, "update_flag": false, "test_avg_loss": 0.002064, "test_total_loss": 2.199732, "test_acc": 0.999758, "test_jaccard": 0.989378, "test_prec": 0.990174, "test_recall": 0.990749, "test_f1": 0.990176, "test_pr_auc": 0.99524, "test_roc_auc": 0.999094, "test_fmax": 0.99498, "test_pmax": 0.997522, "test_rmax": 0.99245, "test_tmax": 0.11, "lr": 4.016148566814695e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.003851416909799677, "train_cur_epoch_loss": 0.46006323459033327, "train_cur_epoch_avg_loss": 9.24936137093553e-05, "train_cur_epoch_time": 1096.558807849884, "train_cur_epoch_avg_time": 0.22045814391835225, "epoch": 30, "step": 149220} ################################################## Training, Epoch: 0031, Batch: 000180, Sample Num: 2880, Cur Loss: 0.00002144, Cur Avg Loss: 0.00011265, Log Avg loss: 0.00013705, Global Avg Loss: 0.00384691, Time: 0.2230 Steps: 149400, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 000380, Sample Num: 6080, Cur Loss: 0.00000418, Cur Avg Loss: 0.00009003, Log Avg loss: 0.00006968, Global Avg Loss: 0.00384186, Time: 0.2196 Steps: 149600, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 000580, Sample Num: 9280, Cur Loss: 0.00000129, Cur Avg Loss: 0.00009157, Log Avg loss: 0.00009450, Global Avg Loss: 0.00383686, Time: 0.2198 Steps: 149800, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 000780, Sample Num: 12480, Cur Loss: 0.00000002, Cur Avg Loss: 0.00010460, Log Avg loss: 0.00014239, Global Avg Loss: 0.00383193, Time: 0.2123 Steps: 150000, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 000980, Sample Num: 15680, Cur Loss: 0.00000135, Cur Avg Loss: 0.00009709, Log Avg loss: 0.00006782, Global Avg Loss: 0.00382692, Time: 0.0864 Steps: 150200, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 001180, Sample Num: 18880, Cur Loss: 0.00000231, Cur Avg Loss: 0.00009203, Log Avg loss: 0.00006724, Global Avg Loss: 0.00382192, Time: 0.2177 Steps: 150400, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 001380, Sample Num: 22080, Cur Loss: 0.00000406, Cur Avg Loss: 0.00008536, Log Avg loss: 0.00004598, Global Avg Loss: 0.00381691, Time: 0.2231 Steps: 150600, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 001580, Sample Num: 25280, Cur Loss: 0.00000774, Cur Avg Loss: 0.00007845, Log Avg loss: 0.00003080, Global Avg Loss: 0.00381189, Time: 0.2211 Steps: 150800, Updated lr: 0.000040 Training, Epoch: 0031, Batch: 001780, Sample Num: 28480, Cur Loss: 0.00000162, Cur Avg Loss: 0.00008027, Log Avg loss: 0.00009465, Global Avg Loss: 0.00380696, Time: 0.2151 Steps: 151000, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 001980, Sample Num: 31680, Cur Loss: 0.00000073, Cur Avg Loss: 0.00007675, Log Avg loss: 0.00004536, Global Avg Loss: 0.00380199, Time: 0.2194 Steps: 151200, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 002180, Sample Num: 34880, Cur Loss: 0.00000942, Cur Avg Loss: 0.00008699, Log Avg loss: 0.00018843, Global Avg Loss: 0.00379721, Time: 0.2264 Steps: 151400, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 002380, Sample Num: 38080, Cur Loss: 0.00001236, Cur Avg Loss: 0.00009058, Log Avg loss: 0.00012966, Global Avg Loss: 0.00379237, Time: 0.2202 Steps: 151600, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 002580, Sample Num: 41280, Cur Loss: 0.00000061, Cur Avg Loss: 0.00008904, Log Avg loss: 0.00007081, Global Avg Loss: 0.00378747, Time: 0.2182 Steps: 151800, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 002780, Sample Num: 44480, Cur Loss: 0.00001781, Cur Avg Loss: 0.00008889, Log Avg loss: 0.00008687, Global Avg Loss: 0.00378260, Time: 0.2205 Steps: 152000, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 002980, Sample Num: 47680, Cur Loss: 0.00000109, Cur Avg Loss: 0.00009414, Log Avg loss: 0.00016713, Global Avg Loss: 0.00377785, Time: 0.2213 Steps: 152200, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 003180, Sample Num: 50880, Cur Loss: 0.00000404, Cur Avg Loss: 0.00009067, Log Avg loss: 0.00003894, Global Avg Loss: 0.00377294, Time: 0.0840 Steps: 152400, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 003380, Sample Num: 54080, Cur Loss: 0.00001037, Cur Avg Loss: 0.00009130, Log Avg loss: 0.00010142, Global Avg Loss: 0.00376813, Time: 0.2198 Steps: 152600, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 003580, Sample Num: 57280, Cur Loss: 0.00000228, Cur Avg Loss: 0.00008856, Log Avg loss: 0.00004219, Global Avg Loss: 0.00376326, Time: 0.2189 Steps: 152800, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 003780, Sample Num: 60480, Cur Loss: 0.00000028, Cur Avg Loss: 0.00008568, Log Avg loss: 0.00003405, Global Avg Loss: 0.00375838, Time: 0.2213 Steps: 153000, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 003980, Sample Num: 63680, Cur Loss: 0.00000087, Cur Avg Loss: 0.00008332, Log Avg loss: 0.00003883, Global Avg Loss: 0.00375353, Time: 0.0867 Steps: 153200, Updated lr: 0.000039 Training, Epoch: 0031, Batch: 004180, Sample Num: 66880, Cur Loss: 0.00000046, Cur Avg Loss: 0.00008133, Log Avg loss: 0.00004176, Global Avg Loss: 0.00374869, Time: 0.2212 Steps: 153400, Updated lr: 0.000038 Training, Epoch: 0031, Batch: 004380, Sample Num: 70080, Cur Loss: 0.00000067, Cur Avg Loss: 0.00008076, Log Avg loss: 0.00006876, Global Avg Loss: 0.00374389, Time: 0.2210 Steps: 153600, Updated lr: 0.000038 Training, Epoch: 0031, Batch: 004580, Sample Num: 73280, Cur Loss: 0.00000247, Cur Avg Loss: 0.00008086, Log Avg loss: 0.00008295, Global Avg Loss: 0.00373913, Time: 0.1153 Steps: 153800, Updated lr: 0.000038 Training, Epoch: 0031, Batch: 004780, Sample Num: 76480, Cur Loss: 0.00000268, Cur Avg Loss: 0.00008348, Log Avg loss: 0.00014367, Global Avg Loss: 0.00373446, Time: 0.2226 Steps: 154000, Updated lr: 0.000038 ***** Running evaluation checkpoint-154194 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-154194 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1091.869512, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.00188, "eval_total_loss": 2.003756, "eval_acc": 0.999744, "eval_jaccard": 0.988797, "eval_prec": 0.989489, "eval_recall": 0.990383, "eval_f1": 0.989627, "eval_pr_auc": 0.995368, "eval_roc_auc": 0.999393, "eval_fmax": 0.994753, "eval_pmax": 0.997863, "eval_rmax": 0.991663, "eval_tmax": 0.18, "update_flag": false, "test_avg_loss": 0.002036, "test_total_loss": 2.170487, "test_acc": 0.999762, "test_jaccard": 0.989418, "test_prec": 0.990208, "test_recall": 0.990749, "test_f1": 0.990201, "test_pr_auc": 0.995175, "test_roc_auc": 0.999104, "test_fmax": 0.994986, "test_pmax": 0.997822, "test_rmax": 0.992167, "test_tmax": 0.16, "lr": 3.8153411384739607e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.003729987499929546, "train_cur_epoch_loss": 0.43326128383048523, "train_cur_epoch_avg_loss": 8.71052038259922e-05, "train_cur_epoch_time": 1091.869511604309, "train_cur_epoch_avg_time": 0.21951538230886794, "epoch": 31, "step": 154194} ################################################## Training, Epoch: 0032, Batch: 000006, Sample Num: 96, Cur Loss: 0.00000285, Cur Avg Loss: 0.00000534, Log Avg loss: 0.00017120, Global Avg Loss: 0.00372984, Time: 0.2228 Steps: 154200, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 000206, Sample Num: 3296, Cur Loss: 0.00000307, Cur Avg Loss: 0.00007965, Log Avg loss: 0.00008188, Global Avg Loss: 0.00372512, Time: 0.2226 Steps: 154400, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 000406, Sample Num: 6496, Cur Loss: 0.00000099, Cur Avg Loss: 0.00009479, Log Avg loss: 0.00011038, Global Avg Loss: 0.00372044, Time: 0.2197 Steps: 154600, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 000606, Sample Num: 9696, Cur Loss: 0.00000036, Cur Avg Loss: 0.00009623, Log Avg loss: 0.00009917, Global Avg Loss: 0.00371576, Time: 0.2210 Steps: 154800, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 000806, Sample Num: 12896, Cur Loss: 0.00000250, Cur Avg Loss: 0.00010828, Log Avg loss: 0.00014478, Global Avg Loss: 0.00371115, Time: 0.2164 Steps: 155000, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 001006, Sample Num: 16096, Cur Loss: 0.00000611, Cur Avg Loss: 0.00009683, Log Avg loss: 0.00005071, Global Avg Loss: 0.00370644, Time: 0.2202 Steps: 155200, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 001206, Sample Num: 19296, Cur Loss: 0.00000194, Cur Avg Loss: 0.00009543, Log Avg loss: 0.00008838, Global Avg Loss: 0.00370178, Time: 0.2203 Steps: 155400, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 001406, Sample Num: 22496, Cur Loss: 0.00000031, Cur Avg Loss: 0.00008437, Log Avg loss: 0.00001769, Global Avg Loss: 0.00369705, Time: 0.2212 Steps: 155600, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 001606, Sample Num: 25696, Cur Loss: 0.00000031, Cur Avg Loss: 0.00007821, Log Avg loss: 0.00003486, Global Avg Loss: 0.00369234, Time: 0.2147 Steps: 155800, Updated lr: 0.000038 Training, Epoch: 0032, Batch: 001806, Sample Num: 28896, Cur Loss: 0.00000035, Cur Avg Loss: 0.00008012, Log Avg loss: 0.00009552, Global Avg Loss: 0.00368773, Time: 0.2120 Steps: 156000, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 002006, Sample Num: 32096, Cur Loss: 0.00000121, Cur Avg Loss: 0.00007805, Log Avg loss: 0.00005928, Global Avg Loss: 0.00368309, Time: 0.1508 Steps: 156200, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 002206, Sample Num: 35296, Cur Loss: 0.00000003, Cur Avg Loss: 0.00008291, Log Avg loss: 0.00013168, Global Avg Loss: 0.00367855, Time: 0.2187 Steps: 156400, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 002406, Sample Num: 38496, Cur Loss: 0.00004134, Cur Avg Loss: 0.00008354, Log Avg loss: 0.00009049, Global Avg Loss: 0.00367396, Time: 0.2192 Steps: 156600, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 002606, Sample Num: 41696, Cur Loss: 0.00000302, Cur Avg Loss: 0.00008567, Log Avg loss: 0.00011125, Global Avg Loss: 0.00366942, Time: 0.2214 Steps: 156800, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 002806, Sample Num: 44896, Cur Loss: 0.00000097, Cur Avg Loss: 0.00008451, Log Avg loss: 0.00006943, Global Avg Loss: 0.00366483, Time: 0.2178 Steps: 157000, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 003006, Sample Num: 48096, Cur Loss: 0.00000164, Cur Avg Loss: 0.00008805, Log Avg loss: 0.00013777, Global Avg Loss: 0.00366035, Time: 0.2187 Steps: 157200, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 003206, Sample Num: 51296, Cur Loss: 0.00000684, Cur Avg Loss: 0.00008418, Log Avg loss: 0.00002599, Global Avg Loss: 0.00365573, Time: 0.2208 Steps: 157400, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 003406, Sample Num: 54496, Cur Loss: 0.00001009, Cur Avg Loss: 0.00008430, Log Avg loss: 0.00008615, Global Avg Loss: 0.00365120, Time: 0.2188 Steps: 157600, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 003606, Sample Num: 57696, Cur Loss: 0.00000099, Cur Avg Loss: 0.00008388, Log Avg loss: 0.00007677, Global Avg Loss: 0.00364667, Time: 0.2062 Steps: 157800, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 003806, Sample Num: 60896, Cur Loss: 0.00000590, Cur Avg Loss: 0.00008089, Log Avg loss: 0.00002706, Global Avg Loss: 0.00364209, Time: 0.2179 Steps: 158000, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 004006, Sample Num: 64096, Cur Loss: 0.00000450, Cur Avg Loss: 0.00007952, Log Avg loss: 0.00005326, Global Avg Loss: 0.00363755, Time: 0.2207 Steps: 158200, Updated lr: 0.000037 Training, Epoch: 0032, Batch: 004206, Sample Num: 67296, Cur Loss: 0.00125748, Cur Avg Loss: 0.00007786, Log Avg loss: 0.00004469, Global Avg Loss: 0.00363301, Time: 0.2111 Steps: 158400, Updated lr: 0.000036 Training, Epoch: 0032, Batch: 004406, Sample Num: 70496, Cur Loss: 0.00000026, Cur Avg Loss: 0.00007602, Log Avg loss: 0.00003734, Global Avg Loss: 0.00362848, Time: 0.2198 Steps: 158600, Updated lr: 0.000036 Training, Epoch: 0032, Batch: 004606, Sample Num: 73696, Cur Loss: 0.00000087, Cur Avg Loss: 0.00007675, Log Avg loss: 0.00009273, Global Avg Loss: 0.00362403, Time: 0.2229 Steps: 158800, Updated lr: 0.000036 Training, Epoch: 0032, Batch: 004806, Sample Num: 76896, Cur Loss: 0.00006282, Cur Avg Loss: 0.00007971, Log Avg loss: 0.00014796, Global Avg Loss: 0.00361965, Time: 0.2148 Steps: 159000, Updated lr: 0.000036 ***** Running evaluation checkpoint-159168 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-159168 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1088.255284, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001937, "eval_total_loss": 2.06524, "eval_acc": 0.99974, "eval_jaccard": 0.988918, "eval_prec": 0.989738, "eval_recall": 0.990431, "eval_f1": 0.989771, "eval_pr_auc": 0.995323, "eval_roc_auc": 0.999365, "eval_fmax": 0.994817, "eval_pmax": 0.99717, "eval_rmax": 0.992474, "eval_tmax": 0.06, "update_flag": false, "test_avg_loss": 0.002065, "test_total_loss": 2.200912, "test_acc": 0.99976, "test_jaccard": 0.989178, "test_prec": 0.98993, "test_recall": 0.990554, "test_f1": 0.989965, "test_pr_auc": 0.995267, "test_roc_auc": 0.99911, "test_fmax": 0.994953, "test_pmax": 0.997924, "test_rmax": 0.992, "test_tmax": 0.14, "lr": 3.614533710133226e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0036159939713731708, "train_cur_epoch_loss": 0.40883587138370814, "train_cur_epoch_avg_loss": 8.219458612458949e-05, "train_cur_epoch_time": 1088.2552843093872, "train_cur_epoch_avg_time": 0.21878875840558648, "epoch": 32, "step": 159168} ################################################## Training, Epoch: 0033, Batch: 000032, Sample Num: 512, Cur Loss: 0.00018407, Cur Avg Loss: 0.00001058, Log Avg loss: 0.00013047, Global Avg Loss: 0.00361527, Time: 0.2597 Steps: 159200, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 000232, Sample Num: 3712, Cur Loss: 0.00000590, Cur Avg Loss: 0.00004507, Log Avg loss: 0.00005058, Global Avg Loss: 0.00361080, Time: 0.2200 Steps: 159400, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 000432, Sample Num: 6912, Cur Loss: 0.00006797, Cur Avg Loss: 0.00007735, Log Avg loss: 0.00011481, Global Avg Loss: 0.00360642, Time: 0.1326 Steps: 159600, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 000632, Sample Num: 10112, Cur Loss: 0.00000262, Cur Avg Loss: 0.00008095, Log Avg loss: 0.00008871, Global Avg Loss: 0.00360201, Time: 0.2173 Steps: 159800, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 000832, Sample Num: 13312, Cur Loss: 0.00000421, Cur Avg Loss: 0.00008903, Log Avg loss: 0.00011458, Global Avg Loss: 0.00359765, Time: 0.2481 Steps: 160000, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 001032, Sample Num: 16512, Cur Loss: 0.00000199, Cur Avg Loss: 0.00007916, Log Avg loss: 0.00003808, Global Avg Loss: 0.00359321, Time: 0.3374 Steps: 160200, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 001232, Sample Num: 19712, Cur Loss: 0.00005321, Cur Avg Loss: 0.00008648, Log Avg loss: 0.00012426, Global Avg Loss: 0.00358888, Time: 0.2268 Steps: 160400, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 001432, Sample Num: 22912, Cur Loss: 0.00000149, Cur Avg Loss: 0.00008305, Log Avg loss: 0.00006191, Global Avg Loss: 0.00358449, Time: 0.2127 Steps: 160600, Updated lr: 0.000036 Training, Epoch: 0033, Batch: 001632, Sample Num: 26112, Cur Loss: 0.00417550, Cur Avg Loss: 0.00007841, Log Avg loss: 0.00004523, Global Avg Loss: 0.00358009, Time: 0.1003 Steps: 160800, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 001832, Sample Num: 29312, Cur Loss: 0.00000047, Cur Avg Loss: 0.00007580, Log Avg loss: 0.00005445, Global Avg Loss: 0.00357571, Time: 0.2229 Steps: 161000, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 002032, Sample Num: 32512, Cur Loss: 0.00000154, Cur Avg Loss: 0.00007528, Log Avg loss: 0.00007052, Global Avg Loss: 0.00357136, Time: 0.2236 Steps: 161200, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 002232, Sample Num: 35712, Cur Loss: 0.00007144, Cur Avg Loss: 0.00008590, Log Avg loss: 0.00019384, Global Avg Loss: 0.00356718, Time: 0.2300 Steps: 161400, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 002432, Sample Num: 38912, Cur Loss: 0.00000504, Cur Avg Loss: 0.00008887, Log Avg loss: 0.00012202, Global Avg Loss: 0.00356291, Time: 0.2515 Steps: 161600, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 002632, Sample Num: 42112, Cur Loss: 0.00000184, Cur Avg Loss: 0.00008892, Log Avg loss: 0.00008948, Global Avg Loss: 0.00355862, Time: 0.2243 Steps: 161800, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 002832, Sample Num: 45312, Cur Loss: 0.00000096, Cur Avg Loss: 0.00008829, Log Avg loss: 0.00008002, Global Avg Loss: 0.00355432, Time: 0.2227 Steps: 162000, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 003032, Sample Num: 48512, Cur Loss: 0.00004091, Cur Avg Loss: 0.00009097, Log Avg loss: 0.00012885, Global Avg Loss: 0.00355010, Time: 0.2190 Steps: 162200, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 003232, Sample Num: 51712, Cur Loss: 0.00000940, Cur Avg Loss: 0.00008666, Log Avg loss: 0.00002139, Global Avg Loss: 0.00354576, Time: 0.2451 Steps: 162400, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 003432, Sample Num: 54912, Cur Loss: 0.00039211, Cur Avg Loss: 0.00008688, Log Avg loss: 0.00009043, Global Avg Loss: 0.00354150, Time: 0.2183 Steps: 162600, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 003632, Sample Num: 58112, Cur Loss: 0.00000034, Cur Avg Loss: 0.00008657, Log Avg loss: 0.00008118, Global Avg Loss: 0.00353725, Time: 0.2188 Steps: 162800, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 003832, Sample Num: 61312, Cur Loss: 0.00000064, Cur Avg Loss: 0.00008355, Log Avg loss: 0.00002878, Global Avg Loss: 0.00353295, Time: 0.2183 Steps: 163000, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 004032, Sample Num: 64512, Cur Loss: 0.00000032, Cur Avg Loss: 0.00008242, Log Avg loss: 0.00006067, Global Avg Loss: 0.00352869, Time: 0.2595 Steps: 163200, Updated lr: 0.000035 Training, Epoch: 0033, Batch: 004232, Sample Num: 67712, Cur Loss: 0.00000250, Cur Avg Loss: 0.00007995, Log Avg loss: 0.00003017, Global Avg Loss: 0.00352441, Time: 0.2181 Steps: 163400, Updated lr: 0.000034 Training, Epoch: 0033, Batch: 004432, Sample Num: 70912, Cur Loss: 0.00000191, Cur Avg Loss: 0.00007826, Log Avg loss: 0.00004248, Global Avg Loss: 0.00352015, Time: 0.2213 Steps: 163600, Updated lr: 0.000034 Training, Epoch: 0033, Batch: 004632, Sample Num: 74112, Cur Loss: 0.00000078, Cur Avg Loss: 0.00007852, Log Avg loss: 0.00008437, Global Avg Loss: 0.00351596, Time: 0.2173 Steps: 163800, Updated lr: 0.000034 Training, Epoch: 0033, Batch: 004832, Sample Num: 77312, Cur Loss: 0.00000798, Cur Avg Loss: 0.00008106, Log Avg loss: 0.00013989, Global Avg Loss: 0.00351184, Time: 0.2433 Steps: 164000, Updated lr: 0.000034 ***** Running evaluation checkpoint-164142 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-164142 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1087.336761, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001898, "eval_total_loss": 2.02283, "eval_acc": 0.999743, "eval_jaccard": 0.988816, "eval_prec": 0.989733, "eval_recall": 0.990393, "eval_f1": 0.989719, "eval_pr_auc": 0.995463, "eval_roc_auc": 0.999387, "eval_fmax": 0.994612, "eval_pmax": 0.996325, "eval_rmax": 0.992904, "eval_tmax": 0.03, "update_flag": false, "test_avg_loss": 0.002053, "test_total_loss": 2.188705, "test_acc": 0.999769, "test_jaccard": 0.989672, "test_prec": 0.990443, "test_recall": 0.991121, "test_f1": 0.990486, "test_pr_auc": 0.995276, "test_roc_auc": 0.999097, "test_fmax": 0.995092, "test_pmax": 0.997699, "test_rmax": 0.992499, "test_tmax": 0.12, "lr": 3.413726281792491e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0035089578954043137, "train_cur_epoch_loss": 0.4168384319297682, "train_cur_epoch_avg_loss": 8.3803464400838e-05, "train_cur_epoch_time": 1087.3367607593536, "train_cur_epoch_avg_time": 0.218604093437747, "epoch": 33, "step": 164142} ################################################## Training, Epoch: 0034, Batch: 000058, Sample Num: 928, Cur Loss: 0.00003260, Cur Avg Loss: 0.00006454, Log Avg loss: 0.00014451, Global Avg Loss: 0.00350774, Time: 0.2210 Steps: 164200, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 000258, Sample Num: 4128, Cur Loss: 0.00000171, Cur Avg Loss: 0.00006019, Log Avg loss: 0.00005893, Global Avg Loss: 0.00350355, Time: 0.2173 Steps: 164400, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 000458, Sample Num: 7328, Cur Loss: 0.00000074, Cur Avg Loss: 0.00007575, Log Avg loss: 0.00009581, Global Avg Loss: 0.00349940, Time: 0.2273 Steps: 164600, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 000658, Sample Num: 10528, Cur Loss: 0.00001472, Cur Avg Loss: 0.00009998, Log Avg loss: 0.00015548, Global Avg Loss: 0.00349535, Time: 0.2175 Steps: 164800, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 000858, Sample Num: 13728, Cur Loss: 0.00000956, Cur Avg Loss: 0.00009840, Log Avg loss: 0.00009321, Global Avg Loss: 0.00349122, Time: 0.1998 Steps: 165000, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 001058, Sample Num: 16928, Cur Loss: 0.00000417, Cur Avg Loss: 0.00009086, Log Avg loss: 0.00005849, Global Avg Loss: 0.00348707, Time: 0.2180 Steps: 165200, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 001258, Sample Num: 20128, Cur Loss: 0.00000003, Cur Avg Loss: 0.00009166, Log Avg loss: 0.00009593, Global Avg Loss: 0.00348297, Time: 0.2174 Steps: 165400, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 001458, Sample Num: 23328, Cur Loss: 0.00000174, Cur Avg Loss: 0.00008747, Log Avg loss: 0.00006110, Global Avg Loss: 0.00347883, Time: 0.2319 Steps: 165600, Updated lr: 0.000034 Training, Epoch: 0034, Batch: 001658, Sample Num: 26528, Cur Loss: 0.00000503, Cur Avg Loss: 0.00008388, Log Avg loss: 0.00005773, Global Avg Loss: 0.00347471, Time: 0.2218 Steps: 165800, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 001858, Sample Num: 29728, Cur Loss: 0.00000369, Cur Avg Loss: 0.00008403, Log Avg loss: 0.00008524, Global Avg Loss: 0.00347062, Time: 0.2147 Steps: 166000, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 002058, Sample Num: 32928, Cur Loss: 0.00002782, Cur Avg Loss: 0.00008392, Log Avg loss: 0.00008289, Global Avg Loss: 0.00346655, Time: 0.2164 Steps: 166200, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 002258, Sample Num: 36128, Cur Loss: 0.00001028, Cur Avg Loss: 0.00008736, Log Avg loss: 0.00012283, Global Avg Loss: 0.00346253, Time: 0.2198 Steps: 166400, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 002458, Sample Num: 39328, Cur Loss: 0.00000246, Cur Avg Loss: 0.00008607, Log Avg loss: 0.00007150, Global Avg Loss: 0.00345846, Time: 0.2138 Steps: 166600, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 002658, Sample Num: 42528, Cur Loss: 0.00000107, Cur Avg Loss: 0.00008854, Log Avg loss: 0.00011881, Global Avg Loss: 0.00345445, Time: 0.2177 Steps: 166800, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 002858, Sample Num: 45728, Cur Loss: 0.00000020, Cur Avg Loss: 0.00008870, Log Avg loss: 0.00009093, Global Avg Loss: 0.00345042, Time: 0.2189 Steps: 167000, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 003058, Sample Num: 48928, Cur Loss: 0.00000014, Cur Avg Loss: 0.00009080, Log Avg loss: 0.00012070, Global Avg Loss: 0.00344644, Time: 0.2233 Steps: 167200, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 003258, Sample Num: 52128, Cur Loss: 0.00000097, Cur Avg Loss: 0.00008687, Log Avg loss: 0.00002676, Global Avg Loss: 0.00344236, Time: 0.2179 Steps: 167400, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 003458, Sample Num: 55328, Cur Loss: 0.00011580, Cur Avg Loss: 0.00008745, Log Avg loss: 0.00009688, Global Avg Loss: 0.00343836, Time: 0.2189 Steps: 167600, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 003658, Sample Num: 58528, Cur Loss: 0.00000127, Cur Avg Loss: 0.00008540, Log Avg loss: 0.00005013, Global Avg Loss: 0.00343433, Time: 0.2162 Steps: 167800, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 003858, Sample Num: 61728, Cur Loss: 0.00000428, Cur Avg Loss: 0.00008177, Log Avg loss: 0.00001534, Global Avg Loss: 0.00343026, Time: 0.3414 Steps: 168000, Updated lr: 0.000033 Training, Epoch: 0034, Batch: 004058, Sample Num: 64928, Cur Loss: 0.00002611, Cur Avg Loss: 0.00008017, Log Avg loss: 0.00004921, Global Avg Loss: 0.00342623, Time: 0.2162 Steps: 168200, Updated lr: 0.000032 Training, Epoch: 0034, Batch: 004258, Sample Num: 68128, Cur Loss: 0.00000563, Cur Avg Loss: 0.00007898, Log Avg loss: 0.00005483, Global Avg Loss: 0.00342223, Time: 0.2341 Steps: 168400, Updated lr: 0.000032 Training, Epoch: 0034, Batch: 004458, Sample Num: 71328, Cur Loss: 0.00258863, Cur Avg Loss: 0.00007903, Log Avg loss: 0.00008012, Global Avg Loss: 0.00341827, Time: 0.2332 Steps: 168600, Updated lr: 0.000032 Training, Epoch: 0034, Batch: 004658, Sample Num: 74528, Cur Loss: 0.00001591, Cur Avg Loss: 0.00007737, Log Avg loss: 0.00004037, Global Avg Loss: 0.00341426, Time: 0.2178 Steps: 168800, Updated lr: 0.000032 Training, Epoch: 0034, Batch: 004858, Sample Num: 77728, Cur Loss: 0.00000175, Cur Avg Loss: 0.00008095, Log Avg loss: 0.00016446, Global Avg Loss: 0.00341042, Time: 0.2184 Steps: 169000, Updated lr: 0.000032 ***** Running evaluation checkpoint-169116 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-169116 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1086.845309, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001918, "eval_total_loss": 2.044602, "eval_acc": 0.999734, "eval_jaccard": 0.988482, "eval_prec": 0.989303, "eval_recall": 0.990031, "eval_f1": 0.989325, "eval_pr_auc": 0.995093, "eval_roc_auc": 0.999377, "eval_fmax": 0.994637, "eval_pmax": 0.9967, "eval_rmax": 0.992582, "eval_tmax": 0.05, "update_flag": false, "test_avg_loss": 0.002101, "test_total_loss": 2.239672, "test_acc": 0.999758, "test_jaccard": 0.989363, "test_prec": 0.990169, "test_recall": 0.990798, "test_f1": 0.990178, "test_pr_auc": 0.995185, "test_roc_auc": 0.999076, "test_fmax": 0.994854, "test_pmax": 0.997625, "test_rmax": 0.992098, "test_tmax": 0.11, "lr": 3.2129188534517564e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.003408172236267184, "train_cur_epoch_loss": 0.4090890411070396, "train_cur_epoch_avg_loss": 8.224548474206666e-05, "train_cur_epoch_time": 1086.845309495926, "train_cur_epoch_avg_time": 0.21850528940408642, "epoch": 34, "step": 169116} ################################################## Training, Epoch: 0035, Batch: 000084, Sample Num: 1344, Cur Loss: 0.00001186, Cur Avg Loss: 0.00009516, Log Avg loss: 0.00011902, Global Avg Loss: 0.00340653, Time: 0.2220 Steps: 169200, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 000284, Sample Num: 4544, Cur Loss: 0.00000071, Cur Avg Loss: 0.00006242, Log Avg loss: 0.00004867, Global Avg Loss: 0.00340256, Time: 0.2178 Steps: 169400, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 000484, Sample Num: 7744, Cur Loss: 0.00001163, Cur Avg Loss: 0.00007946, Log Avg loss: 0.00010366, Global Avg Loss: 0.00339867, Time: 0.2116 Steps: 169600, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 000684, Sample Num: 10944, Cur Loss: 0.00000767, Cur Avg Loss: 0.00008715, Log Avg loss: 0.00010578, Global Avg Loss: 0.00339479, Time: 0.2228 Steps: 169800, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 000884, Sample Num: 14144, Cur Loss: 0.00000168, Cur Avg Loss: 0.00009387, Log Avg loss: 0.00011683, Global Avg Loss: 0.00339094, Time: 0.2060 Steps: 170000, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 001084, Sample Num: 17344, Cur Loss: 0.00000486, Cur Avg Loss: 0.00008731, Log Avg loss: 0.00005832, Global Avg Loss: 0.00338702, Time: 0.2190 Steps: 170200, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 001284, Sample Num: 20544, Cur Loss: 0.00000580, Cur Avg Loss: 0.00008266, Log Avg loss: 0.00005744, Global Avg Loss: 0.00338311, Time: 0.2200 Steps: 170400, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 001484, Sample Num: 23744, Cur Loss: 0.00000569, Cur Avg Loss: 0.00007582, Log Avg loss: 0.00003188, Global Avg Loss: 0.00337919, Time: 0.2196 Steps: 170600, Updated lr: 0.000032 Training, Epoch: 0035, Batch: 001684, Sample Num: 26944, Cur Loss: 0.00000054, Cur Avg Loss: 0.00007228, Log Avg loss: 0.00004609, Global Avg Loss: 0.00337528, Time: 0.2179 Steps: 170800, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 001884, Sample Num: 30144, Cur Loss: 0.00000179, Cur Avg Loss: 0.00007041, Log Avg loss: 0.00005461, Global Avg Loss: 0.00337140, Time: 0.0898 Steps: 171000, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 002084, Sample Num: 33344, Cur Loss: 0.00000251, Cur Avg Loss: 0.00007509, Log Avg loss: 0.00011915, Global Avg Loss: 0.00336760, Time: 0.0849 Steps: 171200, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 002284, Sample Num: 36544, Cur Loss: 0.00000064, Cur Avg Loss: 0.00008031, Log Avg loss: 0.00013478, Global Avg Loss: 0.00336383, Time: 0.2210 Steps: 171400, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 002484, Sample Num: 39744, Cur Loss: 0.00000379, Cur Avg Loss: 0.00007615, Log Avg loss: 0.00002862, Global Avg Loss: 0.00335994, Time: 0.2232 Steps: 171600, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 002684, Sample Num: 42944, Cur Loss: 0.00000438, Cur Avg Loss: 0.00007810, Log Avg loss: 0.00010232, Global Avg Loss: 0.00335615, Time: 0.2189 Steps: 171800, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 002884, Sample Num: 46144, Cur Loss: 0.00000466, Cur Avg Loss: 0.00008147, Log Avg loss: 0.00012671, Global Avg Loss: 0.00335239, Time: 0.2058 Steps: 172000, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 003084, Sample Num: 49344, Cur Loss: 0.00003151, Cur Avg Loss: 0.00008184, Log Avg loss: 0.00008711, Global Avg Loss: 0.00334860, Time: 0.3928 Steps: 172200, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 003284, Sample Num: 52544, Cur Loss: 0.00000740, Cur Avg Loss: 0.00007828, Log Avg loss: 0.00002343, Global Avg Loss: 0.00334474, Time: 0.1533 Steps: 172400, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 003484, Sample Num: 55744, Cur Loss: 0.00000019, Cur Avg Loss: 0.00007843, Log Avg loss: 0.00008089, Global Avg Loss: 0.00334096, Time: 0.2174 Steps: 172600, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 003684, Sample Num: 58944, Cur Loss: 0.00000179, Cur Avg Loss: 0.00007651, Log Avg loss: 0.00004307, Global Avg Loss: 0.00333714, Time: 0.2142 Steps: 172800, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 003884, Sample Num: 62144, Cur Loss: 0.00000048, Cur Avg Loss: 0.00007304, Log Avg loss: 0.00000902, Global Avg Loss: 0.00333330, Time: 0.2177 Steps: 173000, Updated lr: 0.000031 Training, Epoch: 0035, Batch: 004084, Sample Num: 65344, Cur Loss: 0.00000019, Cur Avg Loss: 0.00007170, Log Avg loss: 0.00004567, Global Avg Loss: 0.00332950, Time: 0.2180 Steps: 173200, Updated lr: 0.000030 Training, Epoch: 0035, Batch: 004284, Sample Num: 68544, Cur Loss: 0.00000238, Cur Avg Loss: 0.00007122, Log Avg loss: 0.00006157, Global Avg Loss: 0.00332573, Time: 0.2180 Steps: 173400, Updated lr: 0.000030 Training, Epoch: 0035, Batch: 004484, Sample Num: 71744, Cur Loss: 0.00000149, Cur Avg Loss: 0.00007156, Log Avg loss: 0.00007881, Global Avg Loss: 0.00332199, Time: 0.2196 Steps: 173600, Updated lr: 0.000030 Training, Epoch: 0035, Batch: 004684, Sample Num: 74944, Cur Loss: 0.00003531, Cur Avg Loss: 0.00007143, Log Avg loss: 0.00006851, Global Avg Loss: 0.00331825, Time: 0.2180 Steps: 173800, Updated lr: 0.000030 Training, Epoch: 0035, Batch: 004884, Sample Num: 78144, Cur Loss: 0.00002380, Cur Avg Loss: 0.00007396, Log Avg loss: 0.00013331, Global Avg Loss: 0.00331458, Time: 0.2176 Steps: 174000, Updated lr: 0.000030 ***** Running evaluation checkpoint-174090 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-174090 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1085.506603, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.00192, "eval_total_loss": 2.046219, "eval_acc": 0.999743, "eval_jaccard": 0.988633, "eval_prec": 0.989406, "eval_recall": 0.99008, "eval_f1": 0.989436, "eval_pr_auc": 0.995117, "eval_roc_auc": 0.999364, "eval_fmax": 0.994722, "eval_pmax": 0.997118, "eval_rmax": 0.992338, "eval_tmax": 0.06, "update_flag": false, "test_avg_loss": 0.002088, "test_total_loss": 2.226298, "test_acc": 0.999762, "test_jaccard": 0.989319, "test_prec": 0.990096, "test_recall": 0.990701, "test_f1": 0.990106, "test_pr_auc": 0.995196, "test_roc_auc": 0.999099, "test_fmax": 0.995003, "test_pmax": 0.997885, "test_rmax": 0.992137, "test_tmax": 0.13, "lr": 3.0121114251110216e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.003312948284426897, "train_cur_epoch_loss": 0.3747109273178286, "train_cur_epoch_avg_loss": 7.533392185722328e-05, "train_cur_epoch_time": 1085.506602525711, "train_cur_epoch_avg_time": 0.21823614847722378, "epoch": 35, "step": 174090} ################################################## Training, Epoch: 0036, Batch: 000110, Sample Num: 1760, Cur Loss: 0.00000044, Cur Avg Loss: 0.00009174, Log Avg loss: 0.00011781, Global Avg Loss: 0.00331091, Time: 0.2175 Steps: 174200, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 000310, Sample Num: 4960, Cur Loss: 0.00000223, Cur Avg Loss: 0.00005587, Log Avg loss: 0.00003614, Global Avg Loss: 0.00330716, Time: 0.2178 Steps: 174400, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 000510, Sample Num: 8160, Cur Loss: 0.00000330, Cur Avg Loss: 0.00007967, Log Avg loss: 0.00011655, Global Avg Loss: 0.00330350, Time: 0.2170 Steps: 174600, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 000710, Sample Num: 11360, Cur Loss: 0.00000215, Cur Avg Loss: 0.00007539, Log Avg loss: 0.00006450, Global Avg Loss: 0.00329980, Time: 0.2222 Steps: 174800, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 000910, Sample Num: 14560, Cur Loss: 0.00000211, Cur Avg Loss: 0.00008555, Log Avg loss: 0.00012159, Global Avg Loss: 0.00329617, Time: 0.2165 Steps: 175000, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 001110, Sample Num: 17760, Cur Loss: 0.00000008, Cur Avg Loss: 0.00008076, Log Avg loss: 0.00005900, Global Avg Loss: 0.00329247, Time: 0.2180 Steps: 175200, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 001310, Sample Num: 20960, Cur Loss: 0.00000040, Cur Avg Loss: 0.00007722, Log Avg loss: 0.00005753, Global Avg Loss: 0.00328878, Time: 0.2181 Steps: 175400, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 001510, Sample Num: 24160, Cur Loss: 0.00000061, Cur Avg Loss: 0.00006978, Log Avg loss: 0.00002103, Global Avg Loss: 0.00328506, Time: 0.2138 Steps: 175600, Updated lr: 0.000030 Training, Epoch: 0036, Batch: 001710, Sample Num: 27360, Cur Loss: 0.00000282, Cur Avg Loss: 0.00006718, Log Avg loss: 0.00004758, Global Avg Loss: 0.00328138, Time: 0.2325 Steps: 175800, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 001910, Sample Num: 30560, Cur Loss: 0.00000986, Cur Avg Loss: 0.00006506, Log Avg loss: 0.00004695, Global Avg Loss: 0.00327770, Time: 0.2188 Steps: 176000, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 002110, Sample Num: 33760, Cur Loss: 0.00004247, Cur Avg Loss: 0.00006990, Log Avg loss: 0.00011610, Global Avg Loss: 0.00327411, Time: 0.2248 Steps: 176200, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 002310, Sample Num: 36960, Cur Loss: 0.00000014, Cur Avg Loss: 0.00007523, Log Avg loss: 0.00013150, Global Avg Loss: 0.00327055, Time: 0.2176 Steps: 176400, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 002510, Sample Num: 40160, Cur Loss: 0.00000067, Cur Avg Loss: 0.00007158, Log Avg loss: 0.00002943, Global Avg Loss: 0.00326688, Time: 0.2194 Steps: 176600, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 002710, Sample Num: 43360, Cur Loss: 0.00001127, Cur Avg Loss: 0.00007209, Log Avg loss: 0.00007849, Global Avg Loss: 0.00326327, Time: 0.2211 Steps: 176800, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 002910, Sample Num: 46560, Cur Loss: 0.00051780, Cur Avg Loss: 0.00007535, Log Avg loss: 0.00011951, Global Avg Loss: 0.00325972, Time: 0.2167 Steps: 177000, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 003110, Sample Num: 49760, Cur Loss: 0.00000710, Cur Avg Loss: 0.00007417, Log Avg loss: 0.00005694, Global Avg Loss: 0.00325611, Time: 0.2183 Steps: 177200, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 003310, Sample Num: 52960, Cur Loss: 0.00000410, Cur Avg Loss: 0.00007333, Log Avg loss: 0.00006034, Global Avg Loss: 0.00325250, Time: 0.2278 Steps: 177400, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 003510, Sample Num: 56160, Cur Loss: 0.00022624, Cur Avg Loss: 0.00007280, Log Avg loss: 0.00006402, Global Avg Loss: 0.00324891, Time: 0.3911 Steps: 177600, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 003710, Sample Num: 59360, Cur Loss: 0.00014529, Cur Avg Loss: 0.00007058, Log Avg loss: 0.00003152, Global Avg Loss: 0.00324529, Time: 0.1446 Steps: 177800, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 003910, Sample Num: 62560, Cur Loss: 0.00000109, Cur Avg Loss: 0.00006795, Log Avg loss: 0.00001927, Global Avg Loss: 0.00324167, Time: 0.0896 Steps: 178000, Updated lr: 0.000029 Training, Epoch: 0036, Batch: 004110, Sample Num: 65760, Cur Loss: 0.00000561, Cur Avg Loss: 0.00006583, Log Avg loss: 0.00002442, Global Avg Loss: 0.00323806, Time: 0.2226 Steps: 178200, Updated lr: 0.000028 Training, Epoch: 0036, Batch: 004310, Sample Num: 68960, Cur Loss: 0.00000460, Cur Avg Loss: 0.00006586, Log Avg loss: 0.00006652, Global Avg Loss: 0.00323450, Time: 0.2179 Steps: 178400, Updated lr: 0.000028 Training, Epoch: 0036, Batch: 004510, Sample Num: 72160, Cur Loss: 0.00000030, Cur Avg Loss: 0.00006672, Log Avg loss: 0.00008517, Global Avg Loss: 0.00323097, Time: 0.2180 Steps: 178600, Updated lr: 0.000028 Training, Epoch: 0036, Batch: 004710, Sample Num: 75360, Cur Loss: 0.00000150, Cur Avg Loss: 0.00006890, Log Avg loss: 0.00011798, Global Avg Loss: 0.00322749, Time: 0.2191 Steps: 178800, Updated lr: 0.000028 Training, Epoch: 0036, Batch: 004910, Sample Num: 78560, Cur Loss: 0.00000204, Cur Avg Loss: 0.00007007, Log Avg loss: 0.00009771, Global Avg Loss: 0.00322400, Time: 0.2244 Steps: 179000, Updated lr: 0.000028 ***** Running evaluation checkpoint-179064 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-179064 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1087.877085, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001953, "eval_total_loss": 2.082199, "eval_acc": 0.999739, "eval_jaccard": 0.988594, "eval_prec": 0.989425, "eval_recall": 0.990139, "eval_f1": 0.989434, "eval_pr_auc": 0.995175, "eval_roc_auc": 0.999377, "eval_fmax": 0.994763, "eval_pmax": 0.997062, "eval_rmax": 0.992474, "eval_tmax": 0.05, "update_flag": false, "test_avg_loss": 0.002126, "test_total_loss": 2.266505, "test_acc": 0.999757, "test_jaccard": 0.989187, "test_prec": 0.990013, "test_recall": 0.990661, "test_f1": 0.990014, "test_pr_auc": 0.995253, "test_roc_auc": 0.999105, "test_fmax": 0.995038, "test_pmax": 0.996876, "test_rmax": 0.993207, "test_tmax": 0.04, "lr": 2.811303996770287e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0032228977921784628, "train_cur_epoch_loss": 0.35380342276604937, "train_cur_epoch_avg_loss": 7.113056348332316e-05, "train_cur_epoch_time": 1087.8770847320557, "train_cur_epoch_avg_time": 0.21871272310656528, "epoch": 36, "step": 179064} ################################################## Training, Epoch: 0037, Batch: 000136, Sample Num: 2176, Cur Loss: 0.00004432, Cur Avg Loss: 0.00006961, Log Avg loss: 0.00009610, Global Avg Loss: 0.00322050, Time: 0.2101 Steps: 179200, Updated lr: 0.000028 Training, Epoch: 0037, Batch: 000336, Sample Num: 5376, Cur Loss: 0.00000346, Cur Avg Loss: 0.00004898, Log Avg loss: 0.00003495, Global Avg Loss: 0.00321695, Time: 0.1628 Steps: 179400, Updated lr: 0.000028 Training, Epoch: 0037, Batch: 000536, Sample Num: 8576, Cur Loss: 0.00000373, Cur Avg Loss: 0.00007676, Log Avg loss: 0.00012343, Global Avg Loss: 0.00321351, Time: 0.2184 Steps: 179600, Updated lr: 0.000028 Training, Epoch: 0037, Batch: 000736, Sample Num: 11776, Cur Loss: 0.00017677, Cur Avg Loss: 0.00007775, Log Avg loss: 0.00008042, Global Avg Loss: 0.00321002, Time: 0.2367 Steps: 179800, Updated lr: 0.000028 Training, Epoch: 0037, Batch: 000936, Sample Num: 14976, Cur Loss: 0.00000063, Cur Avg Loss: 0.00008004, Log Avg loss: 0.00008847, Global Avg Loss: 0.00320655, Time: 0.2191 Steps: 180000, Updated lr: 0.000028 Training, Epoch: 0037, Batch: 001136, Sample Num: 18176, Cur Loss: 0.00000611, Cur Avg Loss: 0.00007650, Log Avg loss: 0.00005992, Global Avg Loss: 0.00320306, Time: 0.2201 Steps: 180200, Updated lr: 0.000028 Training, Epoch: 0037, Batch: 001336, Sample Num: 21376, Cur Loss: 0.00000389, Cur Avg Loss: 0.00007492, Log Avg loss: 0.00006593, Global Avg Loss: 0.00319958, Time: 0.3367 Steps: 180400, Updated lr: 0.000028 Training, Epoch: 0037, Batch: 001536, Sample Num: 24576, Cur Loss: 0.00000079, Cur Avg Loss: 0.00006994, Log Avg loss: 0.00003672, Global Avg Loss: 0.00319608, Time: 0.2602 Steps: 180600, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 001736, Sample Num: 27776, Cur Loss: 0.00000055, Cur Avg Loss: 0.00007004, Log Avg loss: 0.00007077, Global Avg Loss: 0.00319262, Time: 0.3039 Steps: 180800, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 001936, Sample Num: 30976, Cur Loss: 0.00000094, Cur Avg Loss: 0.00006576, Log Avg loss: 0.00002857, Global Avg Loss: 0.00318913, Time: 0.3396 Steps: 181000, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 002136, Sample Num: 34176, Cur Loss: 0.00000410, Cur Avg Loss: 0.00007269, Log Avg loss: 0.00013980, Global Avg Loss: 0.00318576, Time: 0.2212 Steps: 181200, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 002336, Sample Num: 37376, Cur Loss: 0.00000070, Cur Avg Loss: 0.00008068, Log Avg loss: 0.00016597, Global Avg Loss: 0.00318243, Time: 0.2482 Steps: 181400, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 002536, Sample Num: 40576, Cur Loss: 0.00000055, Cur Avg Loss: 0.00007547, Log Avg loss: 0.00001470, Global Avg Loss: 0.00317894, Time: 0.2152 Steps: 181600, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 002736, Sample Num: 43776, Cur Loss: 0.00000140, Cur Avg Loss: 0.00007456, Log Avg loss: 0.00006302, Global Avg Loss: 0.00317552, Time: 0.2174 Steps: 181800, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 002936, Sample Num: 46976, Cur Loss: 0.00000019, Cur Avg Loss: 0.00007819, Log Avg loss: 0.00012776, Global Avg Loss: 0.00317217, Time: 0.2184 Steps: 182000, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 003136, Sample Num: 50176, Cur Loss: 0.00000962, Cur Avg Loss: 0.00007749, Log Avg loss: 0.00006721, Global Avg Loss: 0.00316876, Time: 0.2580 Steps: 182200, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 003336, Sample Num: 53376, Cur Loss: 0.00000381, Cur Avg Loss: 0.00007558, Log Avg loss: 0.00004560, Global Avg Loss: 0.00316533, Time: 0.2168 Steps: 182400, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 003536, Sample Num: 56576, Cur Loss: 0.00000044, Cur Avg Loss: 0.00007444, Log Avg loss: 0.00005558, Global Avg Loss: 0.00316193, Time: 0.2193 Steps: 182600, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 003736, Sample Num: 59776, Cur Loss: 0.00000274, Cur Avg Loss: 0.00007163, Log Avg loss: 0.00002184, Global Avg Loss: 0.00315849, Time: 0.2185 Steps: 182800, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 003936, Sample Num: 62976, Cur Loss: 0.00000032, Cur Avg Loss: 0.00006961, Log Avg loss: 0.00003195, Global Avg Loss: 0.00315508, Time: 0.1359 Steps: 183000, Updated lr: 0.000027 Training, Epoch: 0037, Batch: 004136, Sample Num: 66176, Cur Loss: 0.00000126, Cur Avg Loss: 0.00006727, Log Avg loss: 0.00002117, Global Avg Loss: 0.00315166, Time: 0.2177 Steps: 183200, Updated lr: 0.000026 Training, Epoch: 0037, Batch: 004336, Sample Num: 69376, Cur Loss: 0.00000159, Cur Avg Loss: 0.00006671, Log Avg loss: 0.00005520, Global Avg Loss: 0.00314828, Time: 0.2170 Steps: 183400, Updated lr: 0.000026 Training, Epoch: 0037, Batch: 004536, Sample Num: 72576, Cur Loss: 0.00000158, Cur Avg Loss: 0.00006652, Log Avg loss: 0.00006237, Global Avg Loss: 0.00314492, Time: 0.2138 Steps: 183600, Updated lr: 0.000026 Training, Epoch: 0037, Batch: 004736, Sample Num: 75776, Cur Loss: 0.00010837, Cur Avg Loss: 0.00006749, Log Avg loss: 0.00008943, Global Avg Loss: 0.00314159, Time: 0.2629 Steps: 183800, Updated lr: 0.000026 Training, Epoch: 0037, Batch: 004936, Sample Num: 78976, Cur Loss: 0.00000671, Cur Avg Loss: 0.00006903, Log Avg loss: 0.00010546, Global Avg Loss: 0.00313829, Time: 0.2177 Steps: 184000, Updated lr: 0.000026 ***** Running evaluation checkpoint-184038 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-184038 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1084.894389, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001934, "eval_total_loss": 2.06156, "eval_acc": 0.999734, "eval_jaccard": 0.988473, "eval_prec": 0.989318, "eval_recall": 0.990002, "eval_f1": 0.989333, "eval_pr_auc": 0.995239, "eval_roc_auc": 0.999375, "eval_fmax": 0.994703, "eval_pmax": 0.99636, "eval_rmax": 0.993051, "eval_tmax": 0.03, "update_flag": false, "test_avg_loss": 0.002086, "test_total_loss": 2.223205, "test_acc": 0.99976, "test_jaccard": 0.989252, "test_prec": 0.990042, "test_recall": 0.990691, "test_f1": 0.990066, "test_pr_auc": 0.995287, "test_roc_auc": 0.999114, "test_fmax": 0.994975, "test_pmax": 0.997423, "test_rmax": 0.992538, "test_tmax": 0.08, "lr": 2.610496568429552e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0031377111631360507, "train_cur_epoch_loss": 0.3531167825894195, "train_cur_epoch_avg_loss": 7.099251760945305e-05, "train_cur_epoch_time": 1084.8943886756897, "train_cur_epoch_avg_time": 0.21811306567665656, "epoch": 37, "step": 184038} ################################################## Training, Epoch: 0038, Batch: 000162, Sample Num: 2592, Cur Loss: 0.00001024, Cur Avg Loss: 0.00005739, Log Avg loss: 0.00010846, Global Avg Loss: 0.00313500, Time: 0.2154 Steps: 184200, Updated lr: 0.000026 Training, Epoch: 0038, Batch: 000362, Sample Num: 5792, Cur Loss: 0.00000088, Cur Avg Loss: 0.00005655, Log Avg loss: 0.00005586, Global Avg Loss: 0.00313166, Time: 0.2212 Steps: 184400, Updated lr: 0.000026 Training, Epoch: 0038, Batch: 000562, Sample Num: 8992, Cur Loss: 0.00000018, Cur Avg Loss: 0.00008019, Log Avg loss: 0.00012299, Global Avg Loss: 0.00312840, Time: 0.2215 Steps: 184600, Updated lr: 0.000026 Training, Epoch: 0038, Batch: 000762, Sample Num: 12192, Cur Loss: 0.00001491, Cur Avg Loss: 0.00008666, Log Avg loss: 0.00010483, Global Avg Loss: 0.00312513, Time: 0.2073 Steps: 184800, Updated lr: 0.000026 Training, Epoch: 0038, Batch: 000962, Sample Num: 15392, Cur Loss: 0.00000210, Cur Avg Loss: 0.00007985, Log Avg loss: 0.00005394, Global Avg Loss: 0.00312181, Time: 0.2182 Steps: 185000, Updated lr: 0.000026 Training, Epoch: 0038, Batch: 001162, Sample Num: 18592, Cur Loss: 0.00000399, Cur Avg Loss: 0.00007968, Log Avg loss: 0.00007887, Global Avg Loss: 0.00311852, Time: 0.2170 Steps: 185200, Updated lr: 0.000026 Training, Epoch: 0038, Batch: 001362, Sample Num: 21792, Cur Loss: 0.00000162, Cur Avg Loss: 0.00007615, Log Avg loss: 0.00005560, Global Avg Loss: 0.00311522, Time: 0.2185 Steps: 185400, Updated lr: 0.000026 Training, Epoch: 0038, Batch: 001562, Sample Num: 24992, Cur Loss: 0.00001254, Cur Avg Loss: 0.00006891, Log Avg loss: 0.00001962, Global Avg Loss: 0.00311188, Time: 0.2336 Steps: 185600, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 001762, Sample Num: 28192, Cur Loss: 0.00000905, Cur Avg Loss: 0.00006850, Log Avg loss: 0.00006525, Global Avg Loss: 0.00310860, Time: 0.2163 Steps: 185800, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 001962, Sample Num: 31392, Cur Loss: 0.00000003, Cur Avg Loss: 0.00006328, Log Avg loss: 0.00001728, Global Avg Loss: 0.00310528, Time: 0.2178 Steps: 186000, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 002162, Sample Num: 34592, Cur Loss: 0.00076896, Cur Avg Loss: 0.00006990, Log Avg loss: 0.00013489, Global Avg Loss: 0.00310209, Time: 0.2209 Steps: 186200, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 002362, Sample Num: 37792, Cur Loss: 0.00001693, Cur Avg Loss: 0.00007004, Log Avg loss: 0.00007158, Global Avg Loss: 0.00309884, Time: 0.2180 Steps: 186400, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 002562, Sample Num: 40992, Cur Loss: 0.00001208, Cur Avg Loss: 0.00006613, Log Avg loss: 0.00001995, Global Avg Loss: 0.00309554, Time: 0.2168 Steps: 186600, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 002762, Sample Num: 44192, Cur Loss: 0.00000809, Cur Avg Loss: 0.00007007, Log Avg loss: 0.00012048, Global Avg Loss: 0.00309235, Time: 0.2067 Steps: 186800, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 002962, Sample Num: 47392, Cur Loss: 0.00000077, Cur Avg Loss: 0.00007428, Log Avg loss: 0.00013241, Global Avg Loss: 0.00308919, Time: 0.2185 Steps: 187000, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 003162, Sample Num: 50592, Cur Loss: 0.00000047, Cur Avg Loss: 0.00007154, Log Avg loss: 0.00003104, Global Avg Loss: 0.00308592, Time: 0.0957 Steps: 187200, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 003362, Sample Num: 53792, Cur Loss: 0.00001006, Cur Avg Loss: 0.00007236, Log Avg loss: 0.00008530, Global Avg Loss: 0.00308272, Time: 0.2146 Steps: 187400, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 003562, Sample Num: 56992, Cur Loss: 0.00008252, Cur Avg Loss: 0.00007080, Log Avg loss: 0.00004454, Global Avg Loss: 0.00307948, Time: 0.2178 Steps: 187600, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 003762, Sample Num: 60192, Cur Loss: 0.00000264, Cur Avg Loss: 0.00006858, Log Avg loss: 0.00002902, Global Avg Loss: 0.00307623, Time: 0.0942 Steps: 187800, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 003962, Sample Num: 63392, Cur Loss: 0.00000170, Cur Avg Loss: 0.00006675, Log Avg loss: 0.00003247, Global Avg Loss: 0.00307299, Time: 0.2168 Steps: 188000, Updated lr: 0.000025 Training, Epoch: 0038, Batch: 004162, Sample Num: 66592, Cur Loss: 0.00000017, Cur Avg Loss: 0.00006460, Log Avg loss: 0.00002202, Global Avg Loss: 0.00306975, Time: 0.2152 Steps: 188200, Updated lr: 0.000024 Training, Epoch: 0038, Batch: 004362, Sample Num: 69792, Cur Loss: 0.00000120, Cur Avg Loss: 0.00006527, Log Avg loss: 0.00007904, Global Avg Loss: 0.00306658, Time: 0.2201 Steps: 188400, Updated lr: 0.000024 Training, Epoch: 0038, Batch: 004562, Sample Num: 72992, Cur Loss: 0.00000108, Cur Avg Loss: 0.00006622, Log Avg loss: 0.00008691, Global Avg Loss: 0.00306342, Time: 0.2079 Steps: 188600, Updated lr: 0.000024 Training, Epoch: 0038, Batch: 004762, Sample Num: 76192, Cur Loss: 0.00005121, Cur Avg Loss: 0.00006679, Log Avg loss: 0.00007998, Global Avg Loss: 0.00306026, Time: 0.3008 Steps: 188800, Updated lr: 0.000024 Training, Epoch: 0038, Batch: 004962, Sample Num: 79392, Cur Loss: 0.00000090, Cur Avg Loss: 0.00006867, Log Avg loss: 0.00011325, Global Avg Loss: 0.00305714, Time: 0.2169 Steps: 189000, Updated lr: 0.000024 ***** Running evaluation checkpoint-189012 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-189012 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1086.816495, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001938, "eval_total_loss": 2.066151, "eval_acc": 0.999739, "eval_jaccard": 0.988578, "eval_prec": 0.989371, "eval_recall": 0.990117, "eval_f1": 0.989423, "eval_pr_auc": 0.995373, "eval_roc_auc": 0.999386, "eval_fmax": 0.994686, "eval_pmax": 0.996818, "eval_rmax": 0.992562, "eval_tmax": 0.05, "update_flag": false, "test_avg_loss": 0.002086, "test_total_loss": 2.223927, "test_acc": 0.999761, "test_jaccard": 0.989187, "test_prec": 0.989974, "test_recall": 0.990525, "test_f1": 0.989969, "test_pr_auc": 0.995201, "test_roc_auc": 0.999109, "test_fmax": 0.994994, "test_pmax": 0.997748, "test_rmax": 0.992255, "test_tmax": 0.13, "lr": 2.4096891400888173e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0030569470764208306, "train_cur_epoch_loss": 0.34159376722301715, "train_cur_epoch_avg_loss": 6.867586795798495e-05, "train_cur_epoch_time": 1086.8164947032928, "train_cur_epoch_avg_time": 0.21849949632153054, "epoch": 38, "step": 189012} ################################################## Training, Epoch: 0039, Batch: 000188, Sample Num: 3008, Cur Loss: 0.00000003, Cur Avg Loss: 0.00006934, Log Avg loss: 0.00006955, Global Avg Loss: 0.00305398, Time: 0.2187 Steps: 189200, Updated lr: 0.000024 Training, Epoch: 0039, Batch: 000388, Sample Num: 6208, Cur Loss: 0.00000148, Cur Avg Loss: 0.00006814, Log Avg loss: 0.00006701, Global Avg Loss: 0.00305082, Time: 0.2168 Steps: 189400, Updated lr: 0.000024 Training, Epoch: 0039, Batch: 000588, Sample Num: 9408, Cur Loss: 0.00000076, Cur Avg Loss: 0.00007537, Log Avg loss: 0.00008941, Global Avg Loss: 0.00304770, Time: 0.2194 Steps: 189600, Updated lr: 0.000024 Training, Epoch: 0039, Batch: 000788, Sample Num: 12608, Cur Loss: 0.00000132, Cur Avg Loss: 0.00008518, Log Avg loss: 0.00011403, Global Avg Loss: 0.00304461, Time: 0.3928 Steps: 189800, Updated lr: 0.000024 Training, Epoch: 0039, Batch: 000988, Sample Num: 15808, Cur Loss: 0.00000068, Cur Avg Loss: 0.00007917, Log Avg loss: 0.00005546, Global Avg Loss: 0.00304146, Time: 0.2173 Steps: 190000, Updated lr: 0.000024 Training, Epoch: 0039, Batch: 001188, Sample Num: 19008, Cur Loss: 0.00000169, Cur Avg Loss: 0.00007837, Log Avg loss: 0.00007446, Global Avg Loss: 0.00303834, Time: 0.2266 Steps: 190200, Updated lr: 0.000024 Training, Epoch: 0039, Batch: 001388, Sample Num: 22208, Cur Loss: 0.00000539, Cur Avg Loss: 0.00007391, Log Avg loss: 0.00004738, Global Avg Loss: 0.00303520, Time: 0.2251 Steps: 190400, Updated lr: 0.000024 Training, Epoch: 0039, Batch: 001588, Sample Num: 25408, Cur Loss: 0.00000058, Cur Avg Loss: 0.00006748, Log Avg loss: 0.00002284, Global Avg Loss: 0.00303204, Time: 0.3421 Steps: 190600, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 001788, Sample Num: 28608, Cur Loss: 0.00000180, Cur Avg Loss: 0.00006809, Log Avg loss: 0.00007297, Global Avg Loss: 0.00302894, Time: 0.2139 Steps: 190800, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 001988, Sample Num: 31808, Cur Loss: 0.00000336, Cur Avg Loss: 0.00006497, Log Avg loss: 0.00003708, Global Avg Loss: 0.00302581, Time: 0.1451 Steps: 191000, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 002188, Sample Num: 35008, Cur Loss: 0.00000521, Cur Avg Loss: 0.00007075, Log Avg loss: 0.00012816, Global Avg Loss: 0.00302277, Time: 0.2290 Steps: 191200, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 002388, Sample Num: 38208, Cur Loss: 0.00000023, Cur Avg Loss: 0.00007166, Log Avg loss: 0.00008161, Global Avg Loss: 0.00301970, Time: 0.2105 Steps: 191400, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 002588, Sample Num: 41408, Cur Loss: 0.00000070, Cur Avg Loss: 0.00006963, Log Avg loss: 0.00004543, Global Avg Loss: 0.00301660, Time: 0.1437 Steps: 191600, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 002788, Sample Num: 44608, Cur Loss: 0.00001027, Cur Avg Loss: 0.00006915, Log Avg loss: 0.00006298, Global Avg Loss: 0.00301352, Time: 0.4110 Steps: 191800, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 002988, Sample Num: 47808, Cur Loss: 0.00004083, Cur Avg Loss: 0.00007526, Log Avg loss: 0.00016033, Global Avg Loss: 0.00301054, Time: 0.2305 Steps: 192000, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 003188, Sample Num: 51008, Cur Loss: 0.00000026, Cur Avg Loss: 0.00007147, Log Avg loss: 0.00001489, Global Avg Loss: 0.00300743, Time: 0.2250 Steps: 192200, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 003388, Sample Num: 54208, Cur Loss: 0.00000116, Cur Avg Loss: 0.00007114, Log Avg loss: 0.00006590, Global Avg Loss: 0.00300437, Time: 0.2120 Steps: 192400, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 003588, Sample Num: 57408, Cur Loss: 0.00000183, Cur Avg Loss: 0.00006839, Log Avg loss: 0.00002177, Global Avg Loss: 0.00300127, Time: 0.2194 Steps: 192600, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 003788, Sample Num: 60608, Cur Loss: 0.00000002, Cur Avg Loss: 0.00006661, Log Avg loss: 0.00003469, Global Avg Loss: 0.00299819, Time: 0.2180 Steps: 192800, Updated lr: 0.000023 Training, Epoch: 0039, Batch: 003988, Sample Num: 63808, Cur Loss: 0.00000050, Cur Avg Loss: 0.00006483, Log Avg loss: 0.00003116, Global Avg Loss: 0.00299512, Time: 0.2201 Steps: 193000, Updated lr: 0.000022 Training, Epoch: 0039, Batch: 004188, Sample Num: 67008, Cur Loss: 0.00000231, Cur Avg Loss: 0.00006223, Log Avg loss: 0.00001034, Global Avg Loss: 0.00299203, Time: 0.2189 Steps: 193200, Updated lr: 0.000022 Training, Epoch: 0039, Batch: 004388, Sample Num: 70208, Cur Loss: 0.00000061, Cur Avg Loss: 0.00006150, Log Avg loss: 0.00004632, Global Avg Loss: 0.00298898, Time: 0.2194 Steps: 193400, Updated lr: 0.000022 Training, Epoch: 0039, Batch: 004588, Sample Num: 73408, Cur Loss: 0.00000127, Cur Avg Loss: 0.00006125, Log Avg loss: 0.00005579, Global Avg Loss: 0.00298595, Time: 0.2184 Steps: 193600, Updated lr: 0.000022 Training, Epoch: 0039, Batch: 004788, Sample Num: 76608, Cur Loss: 0.00000082, Cur Avg Loss: 0.00006314, Log Avg loss: 0.00010648, Global Avg Loss: 0.00298298, Time: 0.2166 Steps: 193800, Updated lr: 0.000022 ***** Running evaluation checkpoint-193986 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-193986 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1092.356759, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001962, "eval_total_loss": 2.091219, "eval_acc": 0.999739, "eval_jaccard": 0.988682, "eval_prec": 0.989572, "eval_recall": 0.990315, "eval_f1": 0.989591, "eval_pr_auc": 0.995245, "eval_roc_auc": 0.999371, "eval_fmax": 0.994683, "eval_pmax": 0.997405, "eval_rmax": 0.991976, "eval_tmax": 0.09, "update_flag": false, "test_avg_loss": 0.002097, "test_total_loss": 2.235529, "test_acc": 0.99976, "test_jaccard": 0.98927, "test_prec": 0.990135, "test_recall": 0.990622, "test_f1": 0.99008, "test_pr_auc": 0.995327, "test_roc_auc": 0.999104, "test_fmax": 0.994927, "test_pmax": 0.997959, "test_rmax": 0.991912, "test_tmax": 0.17, "lr": 2.2088817117480825e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0029802934918773064, "train_cur_epoch_loss": 0.33553250685799174, "train_cur_epoch_avg_loss": 6.745727922356086e-05, "train_cur_epoch_time": 1092.3567588329315, "train_cur_epoch_avg_time": 0.219613341140517, "epoch": 39, "step": 193986} ################################################## Training, Epoch: 0040, Batch: 000014, Sample Num: 224, Cur Loss: 0.00001100, Cur Avg Loss: 0.00000474, Log Avg loss: 0.00016632, Global Avg Loss: 0.00298008, Time: 0.2149 Steps: 194000, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 000214, Sample Num: 3424, Cur Loss: 0.00000022, Cur Avg Loss: 0.00003704, Log Avg loss: 0.00003930, Global Avg Loss: 0.00297705, Time: 0.2070 Steps: 194200, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 000414, Sample Num: 6624, Cur Loss: 0.00000018, Cur Avg Loss: 0.00005897, Log Avg loss: 0.00008243, Global Avg Loss: 0.00297407, Time: 0.2217 Steps: 194400, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 000614, Sample Num: 9824, Cur Loss: 0.00000017, Cur Avg Loss: 0.00006448, Log Avg loss: 0.00007589, Global Avg Loss: 0.00297109, Time: 0.2119 Steps: 194600, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 000814, Sample Num: 13024, Cur Loss: 0.00001148, Cur Avg Loss: 0.00007132, Log Avg loss: 0.00009232, Global Avg Loss: 0.00296814, Time: 0.2189 Steps: 194800, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 001014, Sample Num: 16224, Cur Loss: 0.00002778, Cur Avg Loss: 0.00006904, Log Avg loss: 0.00005974, Global Avg Loss: 0.00296515, Time: 0.2479 Steps: 195000, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 001214, Sample Num: 19424, Cur Loss: 0.00000952, Cur Avg Loss: 0.00007002, Log Avg loss: 0.00007498, Global Avg Loss: 0.00296219, Time: 0.3386 Steps: 195200, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 001414, Sample Num: 22624, Cur Loss: 0.00000139, Cur Avg Loss: 0.00006549, Log Avg loss: 0.00003802, Global Avg Loss: 0.00295920, Time: 0.3918 Steps: 195400, Updated lr: 0.000022 Training, Epoch: 0040, Batch: 001614, Sample Num: 25824, Cur Loss: 0.00000018, Cur Avg Loss: 0.00005930, Log Avg loss: 0.00001555, Global Avg Loss: 0.00295619, Time: 0.2083 Steps: 195600, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 001814, Sample Num: 29024, Cur Loss: 0.00250736, Cur Avg Loss: 0.00006216, Log Avg loss: 0.00008521, Global Avg Loss: 0.00295326, Time: 0.2283 Steps: 195800, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 002014, Sample Num: 32224, Cur Loss: 0.00000118, Cur Avg Loss: 0.00006091, Log Avg loss: 0.00004955, Global Avg Loss: 0.00295030, Time: 0.2243 Steps: 196000, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 002214, Sample Num: 35424, Cur Loss: 0.00000203, Cur Avg Loss: 0.00006772, Log Avg loss: 0.00013632, Global Avg Loss: 0.00294743, Time: 0.2215 Steps: 196200, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 002414, Sample Num: 38624, Cur Loss: 0.00000854, Cur Avg Loss: 0.00006829, Log Avg loss: 0.00007467, Global Avg Loss: 0.00294450, Time: 0.2184 Steps: 196400, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 002614, Sample Num: 41824, Cur Loss: 0.00000006, Cur Avg Loss: 0.00006756, Log Avg loss: 0.00005868, Global Avg Loss: 0.00294157, Time: 0.2204 Steps: 196600, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 002814, Sample Num: 45024, Cur Loss: 0.00000021, Cur Avg Loss: 0.00006817, Log Avg loss: 0.00007610, Global Avg Loss: 0.00293865, Time: 0.2175 Steps: 196800, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 003014, Sample Num: 48224, Cur Loss: 0.00000146, Cur Avg Loss: 0.00007115, Log Avg loss: 0.00011312, Global Avg Loss: 0.00293579, Time: 0.2183 Steps: 197000, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 003214, Sample Num: 51424, Cur Loss: 0.00000273, Cur Avg Loss: 0.00006781, Log Avg loss: 0.00001751, Global Avg Loss: 0.00293283, Time: 0.2178 Steps: 197200, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 003414, Sample Num: 54624, Cur Loss: 0.00000024, Cur Avg Loss: 0.00006778, Log Avg loss: 0.00006723, Global Avg Loss: 0.00292992, Time: 0.2111 Steps: 197400, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 003614, Sample Num: 57824, Cur Loss: 0.00008175, Cur Avg Loss: 0.00006772, Log Avg loss: 0.00006668, Global Avg Loss: 0.00292702, Time: 0.2180 Steps: 197600, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 003814, Sample Num: 61024, Cur Loss: 0.00000124, Cur Avg Loss: 0.00006473, Log Avg loss: 0.00001085, Global Avg Loss: 0.00292408, Time: 0.2195 Steps: 197800, Updated lr: 0.000021 Training, Epoch: 0040, Batch: 004014, Sample Num: 64224, Cur Loss: 0.00000275, Cur Avg Loss: 0.00006336, Log Avg loss: 0.00003723, Global Avg Loss: 0.00292116, Time: 0.1177 Steps: 198000, Updated lr: 0.000020 Training, Epoch: 0040, Batch: 004214, Sample Num: 67424, Cur Loss: 0.00001831, Cur Avg Loss: 0.00006165, Log Avg loss: 0.00002728, Global Avg Loss: 0.00291824, Time: 0.1816 Steps: 198200, Updated lr: 0.000020 Training, Epoch: 0040, Batch: 004414, Sample Num: 70624, Cur Loss: 0.00000045, Cur Avg Loss: 0.00006086, Log Avg loss: 0.00004418, Global Avg Loss: 0.00291534, Time: 0.0850 Steps: 198400, Updated lr: 0.000020 Training, Epoch: 0040, Batch: 004614, Sample Num: 73824, Cur Loss: 0.00001179, Cur Avg Loss: 0.00006034, Log Avg loss: 0.00004882, Global Avg Loss: 0.00291246, Time: 0.3410 Steps: 198600, Updated lr: 0.000020 Training, Epoch: 0040, Batch: 004814, Sample Num: 77024, Cur Loss: 0.00002321, Cur Avg Loss: 0.00006306, Log Avg loss: 0.00012592, Global Avg Loss: 0.00290965, Time: 0.2131 Steps: 198800, Updated lr: 0.000020 ***** Running evaluation checkpoint-198960 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-198960 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1086.895526, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001963, "eval_total_loss": 2.092222, "eval_acc": 0.999735, "eval_jaccard": 0.98847, "eval_prec": 0.989313, "eval_recall": 0.990041, "eval_f1": 0.989351, "eval_pr_auc": 0.995158, "eval_roc_auc": 0.99936, "eval_fmax": 0.99473, "eval_pmax": 0.997415, "eval_rmax": 0.992059, "eval_tmax": 0.07, "update_flag": false, "test_avg_loss": 0.002102, "test_total_loss": 2.240328, "test_acc": 0.999762, "test_jaccard": 0.989251, "test_prec": 0.990101, "test_recall": 0.99052, "test_f1": 0.990029, "test_pr_auc": 0.995203, "test_roc_auc": 0.999096, "test_fmax": 0.99498, "test_pmax": 0.997371, "test_rmax": 0.992601, "test_tmax": 0.06, "lr": 2.0080742834073476e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.002907425329009124, "train_cur_epoch_loss": 0.3261301443423157, "train_cur_epoch_avg_loss": 6.556697714964127e-05, "train_cur_epoch_time": 1086.8955256938934, "train_cur_epoch_avg_time": 0.21851538514151456, "epoch": 40, "step": 198960} ################################################## Training, Epoch: 0041, Batch: 000040, Sample Num: 640, Cur Loss: 0.00002075, Cur Avg Loss: 0.00000577, Log Avg loss: 0.00011389, Global Avg Loss: 0.00290684, Time: 0.2184 Steps: 199000, Updated lr: 0.000020 Training, Epoch: 0041, Batch: 000240, Sample Num: 3840, Cur Loss: 0.00000149, Cur Avg Loss: 0.00003896, Log Avg loss: 0.00004559, Global Avg Loss: 0.00290397, Time: 0.2182 Steps: 199200, Updated lr: 0.000020 Training, Epoch: 0041, Batch: 000440, Sample Num: 7040, Cur Loss: 0.00000160, Cur Avg Loss: 0.00005586, Log Avg loss: 0.00007615, Global Avg Loss: 0.00290113, Time: 0.2145 Steps: 199400, Updated lr: 0.000020 Training, Epoch: 0041, Batch: 000640, Sample Num: 10240, Cur Loss: 0.00000309, Cur Avg Loss: 0.00005682, Log Avg loss: 0.00005893, Global Avg Loss: 0.00289829, Time: 0.2404 Steps: 199600, Updated lr: 0.000020 Training, Epoch: 0041, Batch: 000840, Sample Num: 13440, Cur Loss: 0.00000989, Cur Avg Loss: 0.00006231, Log Avg loss: 0.00007988, Global Avg Loss: 0.00289546, Time: 0.0850 Steps: 199800, Updated lr: 0.000020 Training, Epoch: 0041, Batch: 001040, Sample Num: 16640, Cur Loss: 0.00000022, Cur Avg Loss: 0.00005829, Log Avg loss: 0.00004143, Global Avg Loss: 0.00289261, Time: 0.2195 Steps: 200000, Updated lr: 0.000020 Training, Epoch: 0041, Batch: 001240, Sample Num: 19840, Cur Loss: 0.00000064, Cur Avg Loss: 0.00006151, Log Avg loss: 0.00007822, Global Avg Loss: 0.00288980, Time: 0.2182 Steps: 200200, Updated lr: 0.000020 Training, Epoch: 0041, Batch: 001440, Sample Num: 23040, Cur Loss: 0.00000185, Cur Avg Loss: 0.00005824, Log Avg loss: 0.00003801, Global Avg Loss: 0.00288695, Time: 0.4142 Steps: 200400, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 001640, Sample Num: 26240, Cur Loss: 0.00000161, Cur Avg Loss: 0.00005561, Log Avg loss: 0.00003661, Global Avg Loss: 0.00288411, Time: 0.0857 Steps: 200600, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 001840, Sample Num: 29440, Cur Loss: 0.00000186, Cur Avg Loss: 0.00005402, Log Avg loss: 0.00004103, Global Avg Loss: 0.00288128, Time: 0.2200 Steps: 200800, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 002040, Sample Num: 32640, Cur Loss: 0.00000426, Cur Avg Loss: 0.00005655, Log Avg loss: 0.00007980, Global Avg Loss: 0.00287849, Time: 0.1967 Steps: 201000, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 002240, Sample Num: 35840, Cur Loss: 0.00000109, Cur Avg Loss: 0.00006167, Log Avg loss: 0.00011388, Global Avg Loss: 0.00287574, Time: 0.2628 Steps: 201200, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 002440, Sample Num: 39040, Cur Loss: 0.00000018, Cur Avg Loss: 0.00006245, Log Avg loss: 0.00007126, Global Avg Loss: 0.00287296, Time: 0.1636 Steps: 201400, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 002640, Sample Num: 42240, Cur Loss: 0.00000512, Cur Avg Loss: 0.00006520, Log Avg loss: 0.00009872, Global Avg Loss: 0.00287021, Time: 0.1112 Steps: 201600, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 002840, Sample Num: 45440, Cur Loss: 0.00000041, Cur Avg Loss: 0.00006516, Log Avg loss: 0.00006462, Global Avg Loss: 0.00286743, Time: 0.2180 Steps: 201800, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 003040, Sample Num: 48640, Cur Loss: 0.00000397, Cur Avg Loss: 0.00006749, Log Avg loss: 0.00010062, Global Avg Loss: 0.00286469, Time: 0.2346 Steps: 202000, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 003240, Sample Num: 51840, Cur Loss: 0.00000004, Cur Avg Loss: 0.00006414, Log Avg loss: 0.00001325, Global Avg Loss: 0.00286187, Time: 0.2167 Steps: 202200, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 003440, Sample Num: 55040, Cur Loss: 0.00000815, Cur Avg Loss: 0.00006435, Log Avg loss: 0.00006767, Global Avg Loss: 0.00285910, Time: 0.2179 Steps: 202400, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 003640, Sample Num: 58240, Cur Loss: 0.00000072, Cur Avg Loss: 0.00006341, Log Avg loss: 0.00004722, Global Avg Loss: 0.00285633, Time: 0.2175 Steps: 202600, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 003840, Sample Num: 61440, Cur Loss: 0.00000682, Cur Avg Loss: 0.00006143, Log Avg loss: 0.00002550, Global Avg Loss: 0.00285354, Time: 0.2489 Steps: 202800, Updated lr: 0.000019 Training, Epoch: 0041, Batch: 004040, Sample Num: 64640, Cur Loss: 0.00001416, Cur Avg Loss: 0.00005982, Log Avg loss: 0.00002885, Global Avg Loss: 0.00285075, Time: 0.2177 Steps: 203000, Updated lr: 0.000018 Training, Epoch: 0041, Batch: 004240, Sample Num: 67840, Cur Loss: 0.00000118, Cur Avg Loss: 0.00005870, Log Avg loss: 0.00003614, Global Avg Loss: 0.00284798, Time: 0.2176 Steps: 203200, Updated lr: 0.000018 Training, Epoch: 0041, Batch: 004440, Sample Num: 71040, Cur Loss: 0.00000315, Cur Avg Loss: 0.00005723, Log Avg loss: 0.00002602, Global Avg Loss: 0.00284521, Time: 0.2225 Steps: 203400, Updated lr: 0.000018 Training, Epoch: 0041, Batch: 004640, Sample Num: 74240, Cur Loss: 0.00000316, Cur Avg Loss: 0.00005804, Log Avg loss: 0.00007598, Global Avg Loss: 0.00284249, Time: 0.2530 Steps: 203600, Updated lr: 0.000018 Training, Epoch: 0041, Batch: 004840, Sample Num: 77440, Cur Loss: 0.00000167, Cur Avg Loss: 0.00006023, Log Avg loss: 0.00011103, Global Avg Loss: 0.00283981, Time: 0.2194 Steps: 203800, Updated lr: 0.000018 ***** Running evaluation checkpoint-203934 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-203934 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1082.875226, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001994, "eval_total_loss": 2.125704, "eval_acc": 0.999737, "eval_jaccard": 0.988572, "eval_prec": 0.989425, "eval_recall": 0.99009, "eval_f1": 0.989416, "eval_pr_auc": 0.995187, "eval_roc_auc": 0.999352, "eval_fmax": 0.99478, "eval_pmax": 0.997214, "eval_rmax": 0.992357, "eval_tmax": 0.06, "update_flag": false, "test_avg_loss": 0.002126, "test_total_loss": 2.265995, "test_acc": 0.999758, "test_jaccard": 0.989226, "test_prec": 0.990081, "test_recall": 0.990574, "test_f1": 0.99003, "test_pr_auc": 0.995282, "test_roc_auc": 0.999109, "test_fmax": 0.994896, "test_pmax": 0.997462, "test_rmax": 0.992343, "test_tmax": 0.07, "lr": 1.807266855066613e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.002837986608971479, "train_cur_epoch_loss": 0.3006176543286967, "train_cur_epoch_avg_loss": 6.043780746455502e-05, "train_cur_epoch_time": 1082.8752264976501, "train_cur_epoch_avg_time": 0.21770712233567555, "epoch": 41, "step": 203934} ################################################## Training, Epoch: 0042, Batch: 000066, Sample Num: 1056, Cur Loss: 0.00008259, Cur Avg Loss: 0.00009238, Log Avg loss: 0.00007602, Global Avg Loss: 0.00283710, Time: 0.1064 Steps: 204000, Updated lr: 0.000018 Training, Epoch: 0042, Batch: 000266, Sample Num: 4256, Cur Loss: 0.00000107, Cur Avg Loss: 0.00005041, Log Avg loss: 0.00003656, Global Avg Loss: 0.00283436, Time: 0.2155 Steps: 204200, Updated lr: 0.000018 Training, Epoch: 0042, Batch: 000466, Sample Num: 7456, Cur Loss: 0.00097169, Cur Avg Loss: 0.00007123, Log Avg loss: 0.00009893, Global Avg Loss: 0.00283168, Time: 0.2201 Steps: 204400, Updated lr: 0.000018 Training, Epoch: 0042, Batch: 000666, Sample Num: 10656, Cur Loss: 0.00000301, Cur Avg Loss: 0.00006635, Log Avg loss: 0.00005499, Global Avg Loss: 0.00282896, Time: 0.3919 Steps: 204600, Updated lr: 0.000018 Training, Epoch: 0042, Batch: 000866, Sample Num: 13856, Cur Loss: 0.00108849, Cur Avg Loss: 0.00006995, Log Avg loss: 0.00008193, Global Avg Loss: 0.00282628, Time: 0.2113 Steps: 204800, Updated lr: 0.000018 Training, Epoch: 0042, Batch: 001066, Sample Num: 17056, Cur Loss: 0.00000251, Cur Avg Loss: 0.00006008, Log Avg loss: 0.00001736, Global Avg Loss: 0.00282354, Time: 0.2203 Steps: 205000, Updated lr: 0.000018 Training, Epoch: 0042, Batch: 001266, Sample Num: 20256, Cur Loss: 0.00013790, Cur Avg Loss: 0.00006788, Log Avg loss: 0.00010944, Global Avg Loss: 0.00282090, Time: 0.2167 Steps: 205200, Updated lr: 0.000018 Training, Epoch: 0042, Batch: 001466, Sample Num: 23456, Cur Loss: 0.00000319, Cur Avg Loss: 0.00006327, Log Avg loss: 0.00003409, Global Avg Loss: 0.00281818, Time: 0.2194 Steps: 205400, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 001666, Sample Num: 26656, Cur Loss: 0.00000869, Cur Avg Loss: 0.00005977, Log Avg loss: 0.00003415, Global Avg Loss: 0.00281547, Time: 0.2189 Steps: 205600, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 001866, Sample Num: 29856, Cur Loss: 0.00000604, Cur Avg Loss: 0.00005968, Log Avg loss: 0.00005887, Global Avg Loss: 0.00281280, Time: 0.2188 Steps: 205800, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 002066, Sample Num: 33056, Cur Loss: 0.00741091, Cur Avg Loss: 0.00006558, Log Avg loss: 0.00012066, Global Avg Loss: 0.00281018, Time: 0.3177 Steps: 206000, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 002266, Sample Num: 36256, Cur Loss: 0.00000076, Cur Avg Loss: 0.00006494, Log Avg loss: 0.00005830, Global Avg Loss: 0.00280751, Time: 0.2167 Steps: 206200, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 002466, Sample Num: 39456, Cur Loss: 0.00000087, Cur Avg Loss: 0.00006812, Log Avg loss: 0.00010419, Global Avg Loss: 0.00280489, Time: 0.0851 Steps: 206400, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 002666, Sample Num: 42656, Cur Loss: 0.00000005, Cur Avg Loss: 0.00006718, Log Avg loss: 0.00005556, Global Avg Loss: 0.00280223, Time: 0.2180 Steps: 206600, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 002866, Sample Num: 45856, Cur Loss: 0.00000170, Cur Avg Loss: 0.00006720, Log Avg loss: 0.00006750, Global Avg Loss: 0.00279959, Time: 0.2180 Steps: 206800, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 003066, Sample Num: 49056, Cur Loss: 0.00000042, Cur Avg Loss: 0.00006843, Log Avg loss: 0.00008602, Global Avg Loss: 0.00279697, Time: 0.2175 Steps: 207000, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 003266, Sample Num: 52256, Cur Loss: 0.00000033, Cur Avg Loss: 0.00006506, Log Avg loss: 0.00001346, Global Avg Loss: 0.00279428, Time: 0.2194 Steps: 207200, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 003466, Sample Num: 55456, Cur Loss: 0.00000257, Cur Avg Loss: 0.00006517, Log Avg loss: 0.00006699, Global Avg Loss: 0.00279165, Time: 0.2128 Steps: 207400, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 003666, Sample Num: 58656, Cur Loss: 0.00000007, Cur Avg Loss: 0.00006372, Log Avg loss: 0.00003860, Global Avg Loss: 0.00278900, Time: 0.1577 Steps: 207600, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 003866, Sample Num: 61856, Cur Loss: 0.00000185, Cur Avg Loss: 0.00006120, Log Avg loss: 0.00001501, Global Avg Loss: 0.00278633, Time: 0.2136 Steps: 207800, Updated lr: 0.000017 Training, Epoch: 0042, Batch: 004066, Sample Num: 65056, Cur Loss: 0.00000131, Cur Avg Loss: 0.00005965, Log Avg loss: 0.00002963, Global Avg Loss: 0.00278368, Time: 0.2142 Steps: 208000, Updated lr: 0.000016 Training, Epoch: 0042, Batch: 004266, Sample Num: 68256, Cur Loss: 0.00000151, Cur Avg Loss: 0.00005882, Log Avg loss: 0.00004195, Global Avg Loss: 0.00278104, Time: 0.2139 Steps: 208200, Updated lr: 0.000016 Training, Epoch: 0042, Batch: 004466, Sample Num: 71456, Cur Loss: 0.00000470, Cur Avg Loss: 0.00005838, Log Avg loss: 0.00004908, Global Avg Loss: 0.00277842, Time: 0.2179 Steps: 208400, Updated lr: 0.000016 Training, Epoch: 0042, Batch: 004666, Sample Num: 74656, Cur Loss: 0.00000044, Cur Avg Loss: 0.00005721, Log Avg loss: 0.00003101, Global Avg Loss: 0.00277579, Time: 0.3626 Steps: 208600, Updated lr: 0.000016 Training, Epoch: 0042, Batch: 004866, Sample Num: 77856, Cur Loss: 0.00000343, Cur Avg Loss: 0.00006067, Log Avg loss: 0.00014133, Global Avg Loss: 0.00277326, Time: 0.2183 Steps: 208800, Updated lr: 0.000016 ***** Running evaluation checkpoint-208908 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-208908 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1081.053221, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001967, "eval_total_loss": 2.09646, "eval_acc": 0.99974, "eval_jaccard": 0.988751, "eval_prec": 0.989445, "eval_recall": 0.99051, "eval_f1": 0.989634, "eval_pr_auc": 0.995377, "eval_roc_auc": 0.999368, "eval_fmax": 0.9948, "eval_pmax": 0.997155, "eval_rmax": 0.992455, "eval_tmax": 0.06, "update_flag": false, "test_avg_loss": 0.002078, "test_total_loss": 2.215228, "test_acc": 0.999762, "test_jaccard": 0.989304, "test_prec": 0.99014, "test_recall": 0.99071, "test_f1": 0.99012, "test_pr_auc": 0.995285, "test_roc_auc": 0.999112, "test_fmax": 0.994991, "test_pmax": 0.996545, "test_rmax": 0.993442, "test_tmax": 0.03, "lr": 1.6064594267258782e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.002771886412600924, "train_cur_epoch_loss": 0.30728556964337894, "train_cur_epoch_avg_loss": 6.177836140799738e-05, "train_cur_epoch_time": 1081.0532205104828, "train_cur_epoch_avg_time": 0.21734081634710148, "epoch": 42, "step": 208908} ################################################## Training, Epoch: 0043, Batch: 000092, Sample Num: 1472, Cur Loss: 0.00000173, Cur Avg Loss: 0.00008916, Log Avg loss: 0.00010136, Global Avg Loss: 0.00277071, Time: 0.2177 Steps: 209000, Updated lr: 0.000016 Training, Epoch: 0043, Batch: 000292, Sample Num: 4672, Cur Loss: 0.00000129, Cur Avg Loss: 0.00004622, Log Avg loss: 0.00002648, Global Avg Loss: 0.00276808, Time: 0.2203 Steps: 209200, Updated lr: 0.000016 Training, Epoch: 0043, Batch: 000492, Sample Num: 7872, Cur Loss: 0.00000470, Cur Avg Loss: 0.00006790, Log Avg loss: 0.00009955, Global Avg Loss: 0.00276553, Time: 0.2199 Steps: 209400, Updated lr: 0.000016 Training, Epoch: 0043, Batch: 000692, Sample Num: 11072, Cur Loss: 0.00000395, Cur Avg Loss: 0.00006053, Log Avg loss: 0.00004239, Global Avg Loss: 0.00276293, Time: 0.2215 Steps: 209600, Updated lr: 0.000016 Training, Epoch: 0043, Batch: 000892, Sample Num: 14272, Cur Loss: 0.00000092, Cur Avg Loss: 0.00006678, Log Avg loss: 0.00008841, Global Avg Loss: 0.00276039, Time: 0.2178 Steps: 209800, Updated lr: 0.000016 Training, Epoch: 0043, Batch: 001092, Sample Num: 17472, Cur Loss: 0.00000067, Cur Avg Loss: 0.00006550, Log Avg loss: 0.00005981, Global Avg Loss: 0.00275781, Time: 0.2231 Steps: 210000, Updated lr: 0.000016 Training, Epoch: 0043, Batch: 001292, Sample Num: 20672, Cur Loss: 0.00000644, Cur Avg Loss: 0.00006345, Log Avg loss: 0.00005225, Global Avg Loss: 0.00275524, Time: 0.2169 Steps: 210200, Updated lr: 0.000016 Training, Epoch: 0043, Batch: 001492, Sample Num: 23872, Cur Loss: 0.00002768, Cur Avg Loss: 0.00005806, Log Avg loss: 0.00002324, Global Avg Loss: 0.00275264, Time: 0.2217 Steps: 210400, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 001692, Sample Num: 27072, Cur Loss: 0.00001137, Cur Avg Loss: 0.00005581, Log Avg loss: 0.00003903, Global Avg Loss: 0.00275006, Time: 0.2205 Steps: 210600, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 001892, Sample Num: 30272, Cur Loss: 0.00000331, Cur Avg Loss: 0.00005603, Log Avg loss: 0.00005791, Global Avg Loss: 0.00274751, Time: 0.2233 Steps: 210800, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 002092, Sample Num: 33472, Cur Loss: 0.00000126, Cur Avg Loss: 0.00005970, Log Avg loss: 0.00009437, Global Avg Loss: 0.00274500, Time: 0.3912 Steps: 211000, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 002292, Sample Num: 36672, Cur Loss: 0.00000654, Cur Avg Loss: 0.00006415, Log Avg loss: 0.00011071, Global Avg Loss: 0.00274250, Time: 0.3395 Steps: 211200, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 002492, Sample Num: 39872, Cur Loss: 0.00000367, Cur Avg Loss: 0.00006071, Log Avg loss: 0.00002126, Global Avg Loss: 0.00273993, Time: 0.3041 Steps: 211400, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 002692, Sample Num: 43072, Cur Loss: 0.00000324, Cur Avg Loss: 0.00006136, Log Avg loss: 0.00006955, Global Avg Loss: 0.00273740, Time: 0.2161 Steps: 211600, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 002892, Sample Num: 46272, Cur Loss: 0.00000463, Cur Avg Loss: 0.00006168, Log Avg loss: 0.00006593, Global Avg Loss: 0.00273488, Time: 0.0721 Steps: 211800, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 003092, Sample Num: 49472, Cur Loss: 0.00000017, Cur Avg Loss: 0.00006261, Log Avg loss: 0.00007614, Global Avg Loss: 0.00273237, Time: 0.2168 Steps: 212000, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 003292, Sample Num: 52672, Cur Loss: 0.00000061, Cur Avg Loss: 0.00006025, Log Avg loss: 0.00002368, Global Avg Loss: 0.00272982, Time: 0.2208 Steps: 212200, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 003492, Sample Num: 55872, Cur Loss: 0.00000412, Cur Avg Loss: 0.00005981, Log Avg loss: 0.00005254, Global Avg Loss: 0.00272730, Time: 0.2200 Steps: 212400, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 003692, Sample Num: 59072, Cur Loss: 0.00000035, Cur Avg Loss: 0.00005866, Log Avg loss: 0.00003866, Global Avg Loss: 0.00272477, Time: 0.2244 Steps: 212600, Updated lr: 0.000015 Training, Epoch: 0043, Batch: 003892, Sample Num: 62272, Cur Loss: 0.00000152, Cur Avg Loss: 0.00005589, Log Avg loss: 0.00000468, Global Avg Loss: 0.00272221, Time: 0.2186 Steps: 212800, Updated lr: 0.000014 Training, Epoch: 0043, Batch: 004092, Sample Num: 65472, Cur Loss: 0.00001204, Cur Avg Loss: 0.00005467, Log Avg loss: 0.00003104, Global Avg Loss: 0.00271969, Time: 0.2211 Steps: 213000, Updated lr: 0.000014 Training, Epoch: 0043, Batch: 004292, Sample Num: 68672, Cur Loss: 0.00000023, Cur Avg Loss: 0.00005400, Log Avg loss: 0.00004028, Global Avg Loss: 0.00271717, Time: 0.2176 Steps: 213200, Updated lr: 0.000014 Training, Epoch: 0043, Batch: 004492, Sample Num: 71872, Cur Loss: 0.00000039, Cur Avg Loss: 0.00005318, Log Avg loss: 0.00003554, Global Avg Loss: 0.00271466, Time: 0.2097 Steps: 213400, Updated lr: 0.000014 Training, Epoch: 0043, Batch: 004692, Sample Num: 75072, Cur Loss: 0.00000027, Cur Avg Loss: 0.00005244, Log Avg loss: 0.00003579, Global Avg Loss: 0.00271215, Time: 0.1241 Steps: 213600, Updated lr: 0.000014 Training, Epoch: 0043, Batch: 004892, Sample Num: 78272, Cur Loss: 0.00000005, Cur Avg Loss: 0.00005423, Log Avg loss: 0.00009632, Global Avg Loss: 0.00270970, Time: 0.2175 Steps: 213800, Updated lr: 0.000014 ***** Running evaluation checkpoint-213882 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-213882 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1083.011860, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.002002, "eval_total_loss": 2.13383, "eval_acc": 0.999727, "eval_jaccard": 0.988282, "eval_prec": 0.989127, "eval_recall": 0.989928, "eval_f1": 0.989185, "eval_pr_auc": 0.995304, "eval_roc_auc": 0.999371, "eval_fmax": 0.994674, "eval_pmax": 0.996966, "eval_rmax": 0.992391, "eval_tmax": 0.05, "update_flag": false, "test_avg_loss": 0.002119, "test_total_loss": 2.258591, "test_acc": 0.999762, "test_jaccard": 0.989291, "test_prec": 0.990042, "test_recall": 0.99073, "test_f1": 0.99009, "test_pr_auc": 0.995234, "test_roc_auc": 0.999105, "test_fmax": 0.994896, "test_pmax": 0.997093, "test_rmax": 0.992709, "test_tmax": 0.05, "lr": 1.4056519983851435e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.002708723652037373, "train_cur_epoch_loss": 0.2779854614243654, "train_cur_epoch_avg_loss": 5.588770836838871e-05, "train_cur_epoch_time": 1083.0118601322174, "train_cur_epoch_avg_time": 0.21773459190434608, "epoch": 43, "step": 213882} ################################################## Training, Epoch: 0044, Batch: 000118, Sample Num: 1888, Cur Loss: 0.00000588, Cur Avg Loss: 0.00006358, Log Avg loss: 0.00010090, Global Avg Loss: 0.00270727, Time: 0.2176 Steps: 214000, Updated lr: 0.000014 Training, Epoch: 0044, Batch: 000318, Sample Num: 5088, Cur Loss: 0.00000001, Cur Avg Loss: 0.00003797, Log Avg loss: 0.00002286, Global Avg Loss: 0.00270476, Time: 0.2179 Steps: 214200, Updated lr: 0.000014 Training, Epoch: 0044, Batch: 000518, Sample Num: 8288, Cur Loss: 0.00000020, Cur Avg Loss: 0.00007128, Log Avg loss: 0.00012423, Global Avg Loss: 0.00270235, Time: 0.2215 Steps: 214400, Updated lr: 0.000014 Training, Epoch: 0044, Batch: 000718, Sample Num: 11488, Cur Loss: 0.00000010, Cur Avg Loss: 0.00006080, Log Avg loss: 0.00003368, Global Avg Loss: 0.00269986, Time: 0.2241 Steps: 214600, Updated lr: 0.000014 Training, Epoch: 0044, Batch: 000918, Sample Num: 14688, Cur Loss: 0.00004449, Cur Avg Loss: 0.00006591, Log Avg loss: 0.00008426, Global Avg Loss: 0.00269743, Time: 0.2230 Steps: 214800, Updated lr: 0.000014 Training, Epoch: 0044, Batch: 001118, Sample Num: 17888, Cur Loss: 0.00003069, Cur Avg Loss: 0.00006436, Log Avg loss: 0.00005724, Global Avg Loss: 0.00269497, Time: 0.2171 Steps: 215000, Updated lr: 0.000014 Training, Epoch: 0044, Batch: 001318, Sample Num: 21088, Cur Loss: 0.00000067, Cur Avg Loss: 0.00006283, Log Avg loss: 0.00005430, Global Avg Loss: 0.00269252, Time: 0.2309 Steps: 215200, Updated lr: 0.000014 Training, Epoch: 0044, Batch: 001518, Sample Num: 24288, Cur Loss: 0.00000435, Cur Avg Loss: 0.00005613, Log Avg loss: 0.00001194, Global Avg Loss: 0.00269003, Time: 0.2148 Steps: 215400, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 001718, Sample Num: 27488, Cur Loss: 0.00000011, Cur Avg Loss: 0.00005632, Log Avg loss: 0.00005776, Global Avg Loss: 0.00268759, Time: 0.2184 Steps: 215600, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 001918, Sample Num: 30688, Cur Loss: 0.00001953, Cur Avg Loss: 0.00005176, Log Avg loss: 0.00001265, Global Avg Loss: 0.00268511, Time: 0.2147 Steps: 215800, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 002118, Sample Num: 33888, Cur Loss: 0.00000362, Cur Avg Loss: 0.00005547, Log Avg loss: 0.00009104, Global Avg Loss: 0.00268271, Time: 0.2239 Steps: 216000, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 002318, Sample Num: 37088, Cur Loss: 0.00000013, Cur Avg Loss: 0.00005800, Log Avg loss: 0.00008471, Global Avg Loss: 0.00268030, Time: 0.2186 Steps: 216200, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 002518, Sample Num: 40288, Cur Loss: 0.00000192, Cur Avg Loss: 0.00005424, Log Avg loss: 0.00001070, Global Avg Loss: 0.00267784, Time: 0.3886 Steps: 216400, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 002718, Sample Num: 43488, Cur Loss: 0.00000164, Cur Avg Loss: 0.00005618, Log Avg loss: 0.00008056, Global Avg Loss: 0.00267544, Time: 0.2172 Steps: 216600, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 002918, Sample Num: 46688, Cur Loss: 0.00006880, Cur Avg Loss: 0.00005697, Log Avg loss: 0.00006777, Global Avg Loss: 0.00267303, Time: 0.0880 Steps: 216800, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 003118, Sample Num: 49888, Cur Loss: 0.00002513, Cur Avg Loss: 0.00005662, Log Avg loss: 0.00005151, Global Avg Loss: 0.00267062, Time: 0.2197 Steps: 217000, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 003318, Sample Num: 53088, Cur Loss: 0.00000075, Cur Avg Loss: 0.00005532, Log Avg loss: 0.00003502, Global Avg Loss: 0.00266819, Time: 0.2185 Steps: 217200, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 003518, Sample Num: 56288, Cur Loss: 0.00000238, Cur Avg Loss: 0.00005620, Log Avg loss: 0.00007091, Global Avg Loss: 0.00266580, Time: 0.2176 Steps: 217400, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 003718, Sample Num: 59488, Cur Loss: 0.00000215, Cur Avg Loss: 0.00005629, Log Avg loss: 0.00005776, Global Avg Loss: 0.00266340, Time: 0.2184 Steps: 217600, Updated lr: 0.000013 Training, Epoch: 0044, Batch: 003918, Sample Num: 62688, Cur Loss: 0.00000253, Cur Avg Loss: 0.00005615, Log Avg loss: 0.00005362, Global Avg Loss: 0.00266101, Time: 0.2250 Steps: 217800, Updated lr: 0.000012 Training, Epoch: 0044, Batch: 004118, Sample Num: 65888, Cur Loss: 0.00000057, Cur Avg Loss: 0.00005476, Log Avg loss: 0.00002744, Global Avg Loss: 0.00265859, Time: 0.2184 Steps: 218000, Updated lr: 0.000012 Training, Epoch: 0044, Batch: 004318, Sample Num: 69088, Cur Loss: 0.00000252, Cur Avg Loss: 0.00005473, Log Avg loss: 0.00005425, Global Avg Loss: 0.00265620, Time: 0.2167 Steps: 218200, Updated lr: 0.000012 Training, Epoch: 0044, Batch: 004518, Sample Num: 72288, Cur Loss: 0.00000544, Cur Avg Loss: 0.00005427, Log Avg loss: 0.00004425, Global Avg Loss: 0.00265381, Time: 0.2239 Steps: 218400, Updated lr: 0.000012 Training, Epoch: 0044, Batch: 004718, Sample Num: 75488, Cur Loss: 0.00000007, Cur Avg Loss: 0.00005473, Log Avg loss: 0.00006503, Global Avg Loss: 0.00265144, Time: 0.2177 Steps: 218600, Updated lr: 0.000012 Training, Epoch: 0044, Batch: 004918, Sample Num: 78688, Cur Loss: 0.00002065, Cur Avg Loss: 0.00005529, Log Avg loss: 0.00006868, Global Avg Loss: 0.00264908, Time: 0.2183 Steps: 218800, Updated lr: 0.000012 ***** Running evaluation checkpoint-218856 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-218856 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1089.562382, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001994, "eval_total_loss": 2.125303, "eval_acc": 0.999733, "eval_jaccard": 0.988482, "eval_prec": 0.989352, "eval_recall": 0.990099, "eval_f1": 0.989382, "eval_pr_auc": 0.995172, "eval_roc_auc": 0.999356, "eval_fmax": 0.994703, "eval_pmax": 0.997455, "eval_rmax": 0.991966, "eval_tmax": 0.09, "update_flag": false, "test_avg_loss": 0.002101, "test_total_loss": 2.239393, "test_acc": 0.999759, "test_jaccard": 0.989266, "test_prec": 0.990076, "test_recall": 0.990671, "test_f1": 0.990074, "test_pr_auc": 0.995285, "test_roc_auc": 0.999102, "test_fmax": 0.994983, "test_pmax": 0.997757, "test_rmax": 0.992225, "test_tmax": 0.11, "lr": 1.2048445700444087e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.002648436494612399, "train_cur_epoch_loss": 0.27898531983124686, "train_cur_epoch_avg_loss": 5.6088725338007006e-05, "train_cur_epoch_time": 1089.5623817443848, "train_cur_epoch_avg_time": 0.21905154437965113, "epoch": 44, "step": 218856} ################################################## Training, Epoch: 0045, Batch: 000144, Sample Num: 2304, Cur Loss: 0.00000005, Cur Avg Loss: 0.00006047, Log Avg loss: 0.00007880, Global Avg Loss: 0.00264673, Time: 0.2119 Steps: 219000, Updated lr: 0.000012 Training, Epoch: 0045, Batch: 000344, Sample Num: 5504, Cur Loss: 0.00029366, Cur Avg Loss: 0.00003605, Log Avg loss: 0.00001847, Global Avg Loss: 0.00264434, Time: 0.2182 Steps: 219200, Updated lr: 0.000012 Training, Epoch: 0045, Batch: 000544, Sample Num: 8704, Cur Loss: 0.00000565, Cur Avg Loss: 0.00005763, Log Avg loss: 0.00009474, Global Avg Loss: 0.00264201, Time: 0.2372 Steps: 219400, Updated lr: 0.000012 Training, Epoch: 0045, Batch: 000744, Sample Num: 11904, Cur Loss: 0.00024266, Cur Avg Loss: 0.00005602, Log Avg loss: 0.00005164, Global Avg Loss: 0.00263965, Time: 0.2387 Steps: 219600, Updated lr: 0.000012 Training, Epoch: 0045, Batch: 000944, Sample Num: 15104, Cur Loss: 0.00000229, Cur Avg Loss: 0.00005684, Log Avg loss: 0.00005989, Global Avg Loss: 0.00263731, Time: 0.2149 Steps: 219800, Updated lr: 0.000012 Training, Epoch: 0045, Batch: 001144, Sample Num: 18304, Cur Loss: 0.00420385, Cur Avg Loss: 0.00005844, Log Avg loss: 0.00006603, Global Avg Loss: 0.00263497, Time: 0.2261 Steps: 220000, Updated lr: 0.000012 Training, Epoch: 0045, Batch: 001344, Sample Num: 21504, Cur Loss: 0.00000108, Cur Avg Loss: 0.00005792, Log Avg loss: 0.00005494, Global Avg Loss: 0.00263263, Time: 0.2481 Steps: 220200, Updated lr: 0.000012 Training, Epoch: 0045, Batch: 001544, Sample Num: 24704, Cur Loss: 0.00000121, Cur Avg Loss: 0.00005315, Log Avg loss: 0.00002110, Global Avg Loss: 0.00263026, Time: 0.2245 Steps: 220400, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 001744, Sample Num: 27904, Cur Loss: 0.00000534, Cur Avg Loss: 0.00005368, Log Avg loss: 0.00005772, Global Avg Loss: 0.00262792, Time: 0.1048 Steps: 220600, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 001944, Sample Num: 31104, Cur Loss: 0.00001675, Cur Avg Loss: 0.00004933, Log Avg loss: 0.00001144, Global Avg Loss: 0.00262555, Time: 0.3927 Steps: 220800, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 002144, Sample Num: 34304, Cur Loss: 0.00000075, Cur Avg Loss: 0.00005359, Log Avg loss: 0.00009496, Global Avg Loss: 0.00262326, Time: 0.2547 Steps: 221000, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 002344, Sample Num: 37504, Cur Loss: 0.00014420, Cur Avg Loss: 0.00005594, Log Avg loss: 0.00008115, Global Avg Loss: 0.00262096, Time: 0.2183 Steps: 221200, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 002544, Sample Num: 40704, Cur Loss: 0.00000553, Cur Avg Loss: 0.00005344, Log Avg loss: 0.00002412, Global Avg Loss: 0.00261862, Time: 0.3925 Steps: 221400, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 002744, Sample Num: 43904, Cur Loss: 0.00000051, Cur Avg Loss: 0.00005647, Log Avg loss: 0.00009500, Global Avg Loss: 0.00261634, Time: 0.1914 Steps: 221600, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 002944, Sample Num: 47104, Cur Loss: 0.00004502, Cur Avg Loss: 0.00005471, Log Avg loss: 0.00003066, Global Avg Loss: 0.00261401, Time: 0.2586 Steps: 221800, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 003144, Sample Num: 50304, Cur Loss: 0.00000042, Cur Avg Loss: 0.00005464, Log Avg loss: 0.00005358, Global Avg Loss: 0.00261170, Time: 0.2193 Steps: 222000, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 003344, Sample Num: 53504, Cur Loss: 0.00000045, Cur Avg Loss: 0.00005448, Log Avg loss: 0.00005188, Global Avg Loss: 0.00260940, Time: 0.2099 Steps: 222200, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 003544, Sample Num: 56704, Cur Loss: 0.00001954, Cur Avg Loss: 0.00005245, Log Avg loss: 0.00001849, Global Avg Loss: 0.00260707, Time: 0.2162 Steps: 222400, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 003744, Sample Num: 59904, Cur Loss: 0.00000246, Cur Avg Loss: 0.00005122, Log Avg loss: 0.00002959, Global Avg Loss: 0.00260475, Time: 0.2411 Steps: 222600, Updated lr: 0.000011 Training, Epoch: 0045, Batch: 003944, Sample Num: 63104, Cur Loss: 0.00000511, Cur Avg Loss: 0.00004934, Log Avg loss: 0.00001407, Global Avg Loss: 0.00260243, Time: 0.4356 Steps: 222800, Updated lr: 0.000010 Training, Epoch: 0045, Batch: 004144, Sample Num: 66304, Cur Loss: 0.00011076, Cur Avg Loss: 0.00004821, Log Avg loss: 0.00002594, Global Avg Loss: 0.00260012, Time: 0.3946 Steps: 223000, Updated lr: 0.000010 Training, Epoch: 0045, Batch: 004344, Sample Num: 69504, Cur Loss: 0.00013756, Cur Avg Loss: 0.00004909, Log Avg loss: 0.00006738, Global Avg Loss: 0.00259785, Time: 0.2215 Steps: 223200, Updated lr: 0.000010 Training, Epoch: 0045, Batch: 004544, Sample Num: 72704, Cur Loss: 0.00000540, Cur Avg Loss: 0.00005037, Log Avg loss: 0.00007804, Global Avg Loss: 0.00259559, Time: 0.2502 Steps: 223400, Updated lr: 0.000010 Training, Epoch: 0045, Batch: 004744, Sample Num: 75904, Cur Loss: 0.00003405, Cur Avg Loss: 0.00005014, Log Avg loss: 0.00004486, Global Avg Loss: 0.00259331, Time: 0.2191 Steps: 223600, Updated lr: 0.000010 Training, Epoch: 0045, Batch: 004944, Sample Num: 79104, Cur Loss: 0.00007716, Cur Avg Loss: 0.00005089, Log Avg loss: 0.00006876, Global Avg Loss: 0.00259105, Time: 0.2202 Steps: 223800, Updated lr: 0.000010 ***** Running evaluation checkpoint-223830 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-223830 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1088.140393, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.002014, "eval_total_loss": 2.146808, "eval_acc": 0.999729, "eval_jaccard": 0.988345, "eval_prec": 0.989254, "eval_recall": 0.990002, "eval_f1": 0.989255, "eval_pr_auc": 0.995116, "eval_roc_auc": 0.999355, "eval_fmax": 0.994695, "eval_pmax": 0.997548, "eval_rmax": 0.991859, "eval_tmax": 0.09, "update_flag": false, "test_avg_loss": 0.002116, "test_total_loss": 2.25585, "test_acc": 0.99976, "test_jaccard": 0.989295, "test_prec": 0.99012, "test_recall": 0.99072, "test_f1": 0.990112, "test_pr_auc": 0.995265, "test_roc_auc": 0.999106, "test_fmax": 0.994945, "test_pmax": 0.997368, "test_rmax": 0.992533, "test_tmax": 0.06, "lr": 1.0040371417036738e-05, "cur_epoch_step": 4974, "train_global_avg_loss": 0.002590732024221397, "train_cur_epoch_loss": 0.257331516583734, "train_cur_epoch_avg_loss": 5.173532701723643e-05, "train_cur_epoch_time": 1088.140392780304, "train_cur_epoch_avg_time": 0.21876565998799838, "epoch": 45, "step": 223830} ################################################## Training, Epoch: 0046, Batch: 000170, Sample Num: 2720, Cur Loss: 0.00000199, Cur Avg Loss: 0.00006632, Log Avg loss: 0.00008506, Global Avg Loss: 0.00258882, Time: 0.2213 Steps: 224000, Updated lr: 0.000010 Training, Epoch: 0046, Batch: 000370, Sample Num: 5920, Cur Loss: 0.00003596, Cur Avg Loss: 0.00006256, Log Avg loss: 0.00005936, Global Avg Loss: 0.00258656, Time: 0.2175 Steps: 224200, Updated lr: 0.000010 Training, Epoch: 0046, Batch: 000570, Sample Num: 9120, Cur Loss: 0.00000487, Cur Avg Loss: 0.00006072, Log Avg loss: 0.00005732, Global Avg Loss: 0.00258431, Time: 0.2988 Steps: 224400, Updated lr: 0.000010 Training, Epoch: 0046, Batch: 000770, Sample Num: 12320, Cur Loss: 0.00000054, Cur Avg Loss: 0.00006625, Log Avg loss: 0.00008202, Global Avg Loss: 0.00258208, Time: 0.1042 Steps: 224600, Updated lr: 0.000010 Training, Epoch: 0046, Batch: 000970, Sample Num: 15520, Cur Loss: 0.00000917, Cur Avg Loss: 0.00006159, Log Avg loss: 0.00004364, Global Avg Loss: 0.00257982, Time: 0.2199 Steps: 224800, Updated lr: 0.000010 Training, Epoch: 0046, Batch: 001170, Sample Num: 18720, Cur Loss: 0.00000212, Cur Avg Loss: 0.00005948, Log Avg loss: 0.00004926, Global Avg Loss: 0.00257757, Time: 0.2180 Steps: 225000, Updated lr: 0.000010 Training, Epoch: 0046, Batch: 001370, Sample Num: 21920, Cur Loss: 0.00000388, Cur Avg Loss: 0.00005737, Log Avg loss: 0.00004503, Global Avg Loss: 0.00257532, Time: 0.2185 Steps: 225200, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 001570, Sample Num: 25120, Cur Loss: 0.00000704, Cur Avg Loss: 0.00005259, Log Avg loss: 0.00001981, Global Avg Loss: 0.00257305, Time: 0.3406 Steps: 225400, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 001770, Sample Num: 28320, Cur Loss: 0.00000060, Cur Avg Loss: 0.00005238, Log Avg loss: 0.00005078, Global Avg Loss: 0.00257082, Time: 0.2262 Steps: 225600, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 001970, Sample Num: 31520, Cur Loss: 0.00000485, Cur Avg Loss: 0.00004835, Log Avg loss: 0.00001267, Global Avg Loss: 0.00256855, Time: 0.2174 Steps: 225800, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 002170, Sample Num: 34720, Cur Loss: 0.00000091, Cur Avg Loss: 0.00005614, Log Avg loss: 0.00013281, Global Avg Loss: 0.00256640, Time: 0.2212 Steps: 226000, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 002370, Sample Num: 37920, Cur Loss: 0.00000016, Cur Avg Loss: 0.00005694, Log Avg loss: 0.00006569, Global Avg Loss: 0.00256418, Time: 0.0841 Steps: 226200, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 002570, Sample Num: 41120, Cur Loss: 0.00000009, Cur Avg Loss: 0.00005459, Log Avg loss: 0.00002677, Global Avg Loss: 0.00256194, Time: 0.1271 Steps: 226400, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 002770, Sample Num: 44320, Cur Loss: 0.00000057, Cur Avg Loss: 0.00005545, Log Avg loss: 0.00006650, Global Avg Loss: 0.00255974, Time: 0.2168 Steps: 226600, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 002970, Sample Num: 47520, Cur Loss: 0.00000107, Cur Avg Loss: 0.00005775, Log Avg loss: 0.00008955, Global Avg Loss: 0.00255756, Time: 0.0952 Steps: 226800, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 003170, Sample Num: 50720, Cur Loss: 0.00000468, Cur Avg Loss: 0.00005498, Log Avg loss: 0.00001379, Global Avg Loss: 0.00255532, Time: 0.2138 Steps: 227000, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 003370, Sample Num: 53920, Cur Loss: 0.00017506, Cur Avg Loss: 0.00005461, Log Avg loss: 0.00004874, Global Avg Loss: 0.00255311, Time: 0.3621 Steps: 227200, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 003570, Sample Num: 57120, Cur Loss: 0.00000651, Cur Avg Loss: 0.00005245, Log Avg loss: 0.00001611, Global Avg Loss: 0.00255088, Time: 0.2179 Steps: 227400, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 003770, Sample Num: 60320, Cur Loss: 0.00000015, Cur Avg Loss: 0.00005104, Log Avg loss: 0.00002592, Global Avg Loss: 0.00254866, Time: 0.2141 Steps: 227600, Updated lr: 0.000009 Training, Epoch: 0046, Batch: 003970, Sample Num: 63520, Cur Loss: 0.00000292, Cur Avg Loss: 0.00004968, Log Avg loss: 0.00002399, Global Avg Loss: 0.00254645, Time: 0.2161 Steps: 227800, Updated lr: 0.000008 Training, Epoch: 0046, Batch: 004170, Sample Num: 66720, Cur Loss: 0.00000145, Cur Avg Loss: 0.00004822, Log Avg loss: 0.00001921, Global Avg Loss: 0.00254423, Time: 0.2199 Steps: 228000, Updated lr: 0.000008 Training, Epoch: 0046, Batch: 004370, Sample Num: 69920, Cur Loss: 0.00000512, Cur Avg Loss: 0.00004824, Log Avg loss: 0.00004861, Global Avg Loss: 0.00254204, Time: 0.2188 Steps: 228200, Updated lr: 0.000008 Training, Epoch: 0046, Batch: 004570, Sample Num: 73120, Cur Loss: 0.00008340, Cur Avg Loss: 0.00004947, Log Avg loss: 0.00007635, Global Avg Loss: 0.00253988, Time: 0.0837 Steps: 228400, Updated lr: 0.000008 Training, Epoch: 0046, Batch: 004770, Sample Num: 76320, Cur Loss: 0.00000031, Cur Avg Loss: 0.00004915, Log Avg loss: 0.00004185, Global Avg Loss: 0.00253770, Time: 0.2200 Steps: 228600, Updated lr: 0.000008 Training, Epoch: 0046, Batch: 004970, Sample Num: 79520, Cur Loss: 0.00000013, Cur Avg Loss: 0.00005084, Log Avg loss: 0.00009125, Global Avg Loss: 0.00253556, Time: 0.2174 Steps: 228800, Updated lr: 0.000008 ***** Running evaluation checkpoint-228804 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-228804 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1086.169949, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.002007, "eval_total_loss": 2.139446, "eval_acc": 0.999734, "eval_jaccard": 0.988493, "eval_prec": 0.989371, "eval_recall": 0.990032, "eval_f1": 0.98937, "eval_pr_auc": 0.99519, "eval_roc_auc": 0.999353, "eval_fmax": 0.994663, "eval_pmax": 0.997592, "eval_rmax": 0.991751, "eval_tmax": 0.1, "update_flag": false, "test_avg_loss": 0.002109, "test_total_loss": 2.248278, "test_acc": 0.999764, "test_jaccard": 0.989407, "test_prec": 0.990213, "test_recall": 0.990808, "test_f1": 0.990209, "test_pr_auc": 0.995233, "test_roc_auc": 0.999103, "test_fmax": 0.994969, "test_pmax": 0.997535, "test_rmax": 0.992416, "test_tmax": 0.08, "lr": 8.032297133629391e-06, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0025355170610378433, "train_cur_epoch_loss": 0.25289665222832003, "train_cur_epoch_avg_loss": 5.084371777811018e-05, "train_cur_epoch_time": 1086.1699488162994, "train_cur_epoch_avg_time": 0.21836951122161227, "epoch": 46, "step": 228804} ################################################## Training, Epoch: 0047, Batch: 000196, Sample Num: 3136, Cur Loss: 0.00000496, Cur Avg Loss: 0.00003469, Log Avg loss: 0.00003506, Global Avg Loss: 0.00253338, Time: 0.2213 Steps: 229000, Updated lr: 0.000008 Training, Epoch: 0047, Batch: 000396, Sample Num: 6336, Cur Loss: 0.00000080, Cur Avg Loss: 0.00004400, Log Avg loss: 0.00005313, Global Avg Loss: 0.00253121, Time: 0.2192 Steps: 229200, Updated lr: 0.000008 Training, Epoch: 0047, Batch: 000596, Sample Num: 9536, Cur Loss: 0.00000042, Cur Avg Loss: 0.00004705, Log Avg loss: 0.00005309, Global Avg Loss: 0.00252905, Time: 0.2183 Steps: 229400, Updated lr: 0.000008 Training, Epoch: 0047, Batch: 000796, Sample Num: 12736, Cur Loss: 0.00000057, Cur Avg Loss: 0.00005531, Log Avg loss: 0.00007992, Global Avg Loss: 0.00252692, Time: 0.3925 Steps: 229600, Updated lr: 0.000008 Training, Epoch: 0047, Batch: 000996, Sample Num: 15936, Cur Loss: 0.00000065, Cur Avg Loss: 0.00005379, Log Avg loss: 0.00004772, Global Avg Loss: 0.00252476, Time: 0.2173 Steps: 229800, Updated lr: 0.000008 Training, Epoch: 0047, Batch: 001196, Sample Num: 19136, Cur Loss: 0.00000085, Cur Avg Loss: 0.00004871, Log Avg loss: 0.00002342, Global Avg Loss: 0.00252259, Time: 0.1918 Steps: 230000, Updated lr: 0.000008 Training, Epoch: 0047, Batch: 001396, Sample Num: 22336, Cur Loss: 0.00000075, Cur Avg Loss: 0.00004823, Log Avg loss: 0.00004536, Global Avg Loss: 0.00252043, Time: 0.2164 Steps: 230200, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 001596, Sample Num: 25536, Cur Loss: 0.00001040, Cur Avg Loss: 0.00004409, Log Avg loss: 0.00001519, Global Avg Loss: 0.00251826, Time: 0.2166 Steps: 230400, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 001796, Sample Num: 28736, Cur Loss: 0.00000393, Cur Avg Loss: 0.00004519, Log Avg loss: 0.00005399, Global Avg Loss: 0.00251612, Time: 0.2207 Steps: 230600, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 001996, Sample Num: 31936, Cur Loss: 0.00000031, Cur Avg Loss: 0.00004173, Log Avg loss: 0.00001062, Global Avg Loss: 0.00251395, Time: 0.3923 Steps: 230800, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 002196, Sample Num: 35136, Cur Loss: 0.00000013, Cur Avg Loss: 0.00005095, Log Avg loss: 0.00014300, Global Avg Loss: 0.00251190, Time: 0.2183 Steps: 231000, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 002396, Sample Num: 38336, Cur Loss: 0.00000415, Cur Avg Loss: 0.00004989, Log Avg loss: 0.00003822, Global Avg Loss: 0.00250976, Time: 0.2243 Steps: 231200, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 002596, Sample Num: 41536, Cur Loss: 0.00000038, Cur Avg Loss: 0.00004939, Log Avg loss: 0.00004343, Global Avg Loss: 0.00250763, Time: 0.2169 Steps: 231400, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 002796, Sample Num: 44736, Cur Loss: 0.00002328, Cur Avg Loss: 0.00005036, Log Avg loss: 0.00006297, Global Avg Loss: 0.00250551, Time: 0.2171 Steps: 231600, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 002996, Sample Num: 47936, Cur Loss: 0.00022699, Cur Avg Loss: 0.00005264, Log Avg loss: 0.00008444, Global Avg Loss: 0.00250343, Time: 0.2187 Steps: 231800, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 003196, Sample Num: 51136, Cur Loss: 0.00000050, Cur Avg Loss: 0.00005036, Log Avg loss: 0.00001629, Global Avg Loss: 0.00250128, Time: 0.2174 Steps: 232000, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 003396, Sample Num: 54336, Cur Loss: 0.00000050, Cur Avg Loss: 0.00005031, Log Avg loss: 0.00004951, Global Avg Loss: 0.00249917, Time: 0.2174 Steps: 232200, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 003596, Sample Num: 57536, Cur Loss: 0.00000031, Cur Avg Loss: 0.00004932, Log Avg loss: 0.00003249, Global Avg Loss: 0.00249705, Time: 0.2140 Steps: 232400, Updated lr: 0.000007 Training, Epoch: 0047, Batch: 003796, Sample Num: 60736, Cur Loss: 0.00000029, Cur Avg Loss: 0.00004742, Log Avg loss: 0.00001321, Global Avg Loss: 0.00249491, Time: 0.2180 Steps: 232600, Updated lr: 0.000006 Training, Epoch: 0047, Batch: 003996, Sample Num: 63936, Cur Loss: 0.00000050, Cur Avg Loss: 0.00004735, Log Avg loss: 0.00004611, Global Avg Loss: 0.00249281, Time: 0.2252 Steps: 232800, Updated lr: 0.000006 Training, Epoch: 0047, Batch: 004196, Sample Num: 67136, Cur Loss: 0.00000015, Cur Avg Loss: 0.00004579, Log Avg loss: 0.00001449, Global Avg Loss: 0.00249068, Time: 0.2284 Steps: 233000, Updated lr: 0.000006 Training, Epoch: 0047, Batch: 004396, Sample Num: 70336, Cur Loss: 0.00001290, Cur Avg Loss: 0.00004510, Log Avg loss: 0.00003059, Global Avg Loss: 0.00248857, Time: 0.2164 Steps: 233200, Updated lr: 0.000006 Training, Epoch: 0047, Batch: 004596, Sample Num: 73536, Cur Loss: 0.00002875, Cur Avg Loss: 0.00004514, Log Avg loss: 0.00004622, Global Avg Loss: 0.00248648, Time: 0.3933 Steps: 233400, Updated lr: 0.000006 Training, Epoch: 0047, Batch: 004796, Sample Num: 76736, Cur Loss: 0.00013107, Cur Avg Loss: 0.00004611, Log Avg loss: 0.00006825, Global Avg Loss: 0.00248441, Time: 0.7047 Steps: 233600, Updated lr: 0.000006 ***** Running evaluation checkpoint-233778 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-233778 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1083.310848, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.002002, "eval_total_loss": 2.133848, "eval_acc": 0.999731, "eval_jaccard": 0.988355, "eval_prec": 0.989244, "eval_recall": 0.990021, "eval_f1": 0.989281, "eval_pr_auc": 0.995271, "eval_roc_auc": 0.999357, "eval_fmax": 0.994624, "eval_pmax": 0.997356, "eval_rmax": 0.991908, "eval_tmax": 0.09, "update_flag": false, "test_avg_loss": 0.002112, "test_total_loss": 2.251705, "test_acc": 0.999761, "test_jaccard": 0.989255, "test_prec": 0.990032, "test_recall": 0.990652, "test_f1": 0.990046, "test_pr_auc": 0.995258, "test_roc_auc": 0.999105, "test_fmax": 0.994973, "test_pmax": 0.997845, "test_rmax": 0.992118, "test_tmax": 0.13, "lr": 6.024222850222043e-06, "cur_epoch_step": 4974, "train_global_avg_loss": 0.002482582964936429, "train_cur_epoch_loss": 0.23683474320696174, "train_cur_epoch_avg_loss": 4.7614544271604696e-05, "train_cur_epoch_time": 1083.3108477592468, "train_cur_epoch_avg_time": 0.21779470200226111, "epoch": 47, "step": 233778} ################################################## Training, Epoch: 0048, Batch: 000022, Sample Num: 352, Cur Loss: 0.00000784, Cur Avg Loss: 0.00004556, Log Avg loss: 0.00008353, Global Avg Loss: 0.00248235, Time: 0.2247 Steps: 233800, Updated lr: 0.000006 Training, Epoch: 0048, Batch: 000222, Sample Num: 3552, Cur Loss: 0.00000093, Cur Avg Loss: 0.00005220, Log Avg loss: 0.00005294, Global Avg Loss: 0.00248028, Time: 0.2207 Steps: 234000, Updated lr: 0.000006 Training, Epoch: 0048, Batch: 000422, Sample Num: 6752, Cur Loss: 0.00000111, Cur Avg Loss: 0.00005484, Log Avg loss: 0.00005777, Global Avg Loss: 0.00247821, Time: 0.2121 Steps: 234200, Updated lr: 0.000006 Training, Epoch: 0048, Batch: 000622, Sample Num: 9952, Cur Loss: 0.00000001, Cur Avg Loss: 0.00005575, Log Avg loss: 0.00005767, Global Avg Loss: 0.00247614, Time: 0.2193 Steps: 234400, Updated lr: 0.000006 Training, Epoch: 0048, Batch: 000822, Sample Num: 13152, Cur Loss: 0.00000056, Cur Avg Loss: 0.00006189, Log Avg loss: 0.00008097, Global Avg Loss: 0.00247410, Time: 0.2239 Steps: 234600, Updated lr: 0.000006 Training, Epoch: 0048, Batch: 001022, Sample Num: 16352, Cur Loss: 0.00000031, Cur Avg Loss: 0.00005692, Log Avg loss: 0.00003649, Global Avg Loss: 0.00247202, Time: 0.2184 Steps: 234800, Updated lr: 0.000006 Training, Epoch: 0048, Batch: 001222, Sample Num: 19552, Cur Loss: 0.00000222, Cur Avg Loss: 0.00005898, Log Avg loss: 0.00006955, Global Avg Loss: 0.00246998, Time: 0.2225 Steps: 235000, Updated lr: 0.000006 Training, Epoch: 0048, Batch: 001422, Sample Num: 22752, Cur Loss: 0.00000110, Cur Avg Loss: 0.00005392, Log Avg loss: 0.00002296, Global Avg Loss: 0.00246790, Time: 0.0971 Steps: 235200, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 001622, Sample Num: 25952, Cur Loss: 0.00000012, Cur Avg Loss: 0.00005074, Log Avg loss: 0.00002817, Global Avg Loss: 0.00246583, Time: 0.2190 Steps: 235400, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 001822, Sample Num: 29152, Cur Loss: 0.00000035, Cur Avg Loss: 0.00004825, Log Avg loss: 0.00002804, Global Avg Loss: 0.00246376, Time: 0.2197 Steps: 235600, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 002022, Sample Num: 32352, Cur Loss: 0.00000044, Cur Avg Loss: 0.00004817, Log Avg loss: 0.00004738, Global Avg Loss: 0.00246171, Time: 0.0853 Steps: 235800, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 002222, Sample Num: 35552, Cur Loss: 0.00021835, Cur Avg Loss: 0.00005426, Log Avg loss: 0.00011588, Global Avg Loss: 0.00245972, Time: 0.2314 Steps: 236000, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 002422, Sample Num: 38752, Cur Loss: 0.00034639, Cur Avg Loss: 0.00005535, Log Avg loss: 0.00006749, Global Avg Loss: 0.00245769, Time: 0.6550 Steps: 236200, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 002622, Sample Num: 41952, Cur Loss: 0.00000030, Cur Avg Loss: 0.00005660, Log Avg loss: 0.00007170, Global Avg Loss: 0.00245568, Time: 0.2100 Steps: 236400, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 002822, Sample Num: 45152, Cur Loss: 0.00000116, Cur Avg Loss: 0.00005550, Log Avg loss: 0.00004103, Global Avg Loss: 0.00245363, Time: 0.2185 Steps: 236600, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 003022, Sample Num: 48352, Cur Loss: 0.00000132, Cur Avg Loss: 0.00005647, Log Avg loss: 0.00007019, Global Avg Loss: 0.00245162, Time: 0.2172 Steps: 236800, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 003222, Sample Num: 51552, Cur Loss: 0.00000268, Cur Avg Loss: 0.00005356, Log Avg loss: 0.00000968, Global Avg Loss: 0.00244956, Time: 0.2188 Steps: 237000, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 003422, Sample Num: 54752, Cur Loss: 0.00000003, Cur Avg Loss: 0.00005349, Log Avg loss: 0.00005228, Global Avg Loss: 0.00244754, Time: 0.2172 Steps: 237200, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 003622, Sample Num: 57952, Cur Loss: 0.00000202, Cur Avg Loss: 0.00005275, Log Avg loss: 0.00004019, Global Avg Loss: 0.00244551, Time: 0.2221 Steps: 237400, Updated lr: 0.000005 Training, Epoch: 0048, Batch: 003822, Sample Num: 61152, Cur Loss: 0.00000650, Cur Avg Loss: 0.00005225, Log Avg loss: 0.00004304, Global Avg Loss: 0.00244349, Time: 0.2191 Steps: 237600, Updated lr: 0.000004 Training, Epoch: 0048, Batch: 004022, Sample Num: 64352, Cur Loss: 0.00000146, Cur Avg Loss: 0.00005103, Log Avg loss: 0.00002774, Global Avg Loss: 0.00244146, Time: 0.2187 Steps: 237800, Updated lr: 0.000004 Training, Epoch: 0048, Batch: 004222, Sample Num: 67552, Cur Loss: 0.00000053, Cur Avg Loss: 0.00004947, Log Avg loss: 0.00001812, Global Avg Loss: 0.00243942, Time: 0.2168 Steps: 238000, Updated lr: 0.000004 Training, Epoch: 0048, Batch: 004422, Sample Num: 70752, Cur Loss: 0.00000136, Cur Avg Loss: 0.00004890, Log Avg loss: 0.00003692, Global Avg Loss: 0.00243740, Time: 0.2228 Steps: 238200, Updated lr: 0.000004 Training, Epoch: 0048, Batch: 004622, Sample Num: 73952, Cur Loss: 0.00002693, Cur Avg Loss: 0.00004909, Log Avg loss: 0.00005324, Global Avg Loss: 0.00243540, Time: 0.2161 Steps: 238400, Updated lr: 0.000004 Training, Epoch: 0048, Batch: 004822, Sample Num: 77152, Cur Loss: 0.00000276, Cur Avg Loss: 0.00004998, Log Avg loss: 0.00007066, Global Avg Loss: 0.00243342, Time: 0.2164 Steps: 238600, Updated lr: 0.000004 ***** Running evaluation checkpoint-238752 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-238752 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1086.248034, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.001997, "eval_total_loss": 2.129281, "eval_acc": 0.999733, "eval_jaccard": 0.988453, "eval_prec": 0.989323, "eval_recall": 0.990041, "eval_f1": 0.989343, "eval_pr_auc": 0.995282, "eval_roc_auc": 0.999354, "eval_fmax": 0.994678, "eval_pmax": 0.997366, "eval_rmax": 0.992005, "eval_tmax": 0.08, "update_flag": false, "test_avg_loss": 0.002103, "test_total_loss": 2.241996, "test_acc": 0.999762, "test_jaccard": 0.989265, "test_prec": 0.990042, "test_recall": 0.990652, "test_f1": 0.990053, "test_pr_auc": 0.99527, "test_roc_auc": 0.999106, "test_fmax": 0.995015, "test_pmax": 0.997826, "test_rmax": 0.99222, "test_tmax": 0.11, "lr": 4.0161485668146955e-06, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0024319072452379112, "train_cur_epoch_loss": 0.24943823813236743, "train_cur_epoch_avg_loss": 5.014841940739192e-05, "train_cur_epoch_time": 1086.2480335235596, "train_cur_epoch_avg_time": 0.2183852097956493, "epoch": 48, "step": 238752} ################################################## Training, Epoch: 0049, Batch: 000048, Sample Num: 768, Cur Loss: 0.00000112, Cur Avg Loss: 0.00001197, Log Avg loss: 0.00004497, Global Avg Loss: 0.00243142, Time: 0.2861 Steps: 238800, Updated lr: 0.000004 Training, Epoch: 0049, Batch: 000248, Sample Num: 3968, Cur Loss: 0.00000464, Cur Avg Loss: 0.00003352, Log Avg loss: 0.00003869, Global Avg Loss: 0.00242942, Time: 0.2177 Steps: 239000, Updated lr: 0.000004 Training, Epoch: 0049, Batch: 000448, Sample Num: 7168, Cur Loss: 0.00000553, Cur Avg Loss: 0.00004435, Log Avg loss: 0.00005778, Global Avg Loss: 0.00242744, Time: 0.2600 Steps: 239200, Updated lr: 0.000004 Training, Epoch: 0049, Batch: 000648, Sample Num: 10368, Cur Loss: 0.00003564, Cur Avg Loss: 0.00004692, Log Avg loss: 0.00005266, Global Avg Loss: 0.00242545, Time: 0.2262 Steps: 239400, Updated lr: 0.000004 Training, Epoch: 0049, Batch: 000848, Sample Num: 13568, Cur Loss: 0.00020587, Cur Avg Loss: 0.00005803, Log Avg loss: 0.00009404, Global Avg Loss: 0.00242351, Time: 0.2240 Steps: 239600, Updated lr: 0.000004 Training, Epoch: 0049, Batch: 001048, Sample Num: 16768, Cur Loss: 0.00001304, Cur Avg Loss: 0.00004950, Log Avg loss: 0.00001332, Global Avg Loss: 0.00242150, Time: 0.2204 Steps: 239800, Updated lr: 0.000004 Training, Epoch: 0049, Batch: 001248, Sample Num: 19968, Cur Loss: 0.00000039, Cur Avg Loss: 0.00005235, Log Avg loss: 0.00006732, Global Avg Loss: 0.00241953, Time: 0.2427 Steps: 240000, Updated lr: 0.000004 Training, Epoch: 0049, Batch: 001448, Sample Num: 23168, Cur Loss: 0.00006795, Cur Avg Loss: 0.00005013, Log Avg loss: 0.00003627, Global Avg Loss: 0.00241755, Time: 0.2308 Steps: 240200, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 001648, Sample Num: 26368, Cur Loss: 0.00000040, Cur Avg Loss: 0.00004806, Log Avg loss: 0.00003302, Global Avg Loss: 0.00241557, Time: 0.1304 Steps: 240400, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 001848, Sample Num: 29568, Cur Loss: 0.00000035, Cur Avg Loss: 0.00004500, Log Avg loss: 0.00001983, Global Avg Loss: 0.00241357, Time: 0.3926 Steps: 240600, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 002048, Sample Num: 32768, Cur Loss: 0.00000056, Cur Avg Loss: 0.00004678, Log Avg loss: 0.00006324, Global Avg Loss: 0.00241162, Time: 0.2193 Steps: 240800, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 002248, Sample Num: 35968, Cur Loss: 0.00001233, Cur Avg Loss: 0.00005213, Log Avg loss: 0.00010685, Global Avg Loss: 0.00240971, Time: 0.2172 Steps: 241000, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 002448, Sample Num: 39168, Cur Loss: 0.00000181, Cur Avg Loss: 0.00005211, Log Avg loss: 0.00005188, Global Avg Loss: 0.00240775, Time: 0.2173 Steps: 241200, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 002648, Sample Num: 42368, Cur Loss: 0.00039042, Cur Avg Loss: 0.00005204, Log Avg loss: 0.00005119, Global Avg Loss: 0.00240580, Time: 0.5057 Steps: 241400, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 002848, Sample Num: 45568, Cur Loss: 0.00000123, Cur Avg Loss: 0.00005153, Log Avg loss: 0.00004480, Global Avg Loss: 0.00240385, Time: 0.2589 Steps: 241600, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 003048, Sample Num: 48768, Cur Loss: 0.00000033, Cur Avg Loss: 0.00005346, Log Avg loss: 0.00008090, Global Avg Loss: 0.00240193, Time: 0.3940 Steps: 241800, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 003248, Sample Num: 51968, Cur Loss: 0.00000239, Cur Avg Loss: 0.00005089, Log Avg loss: 0.00001174, Global Avg Loss: 0.00239995, Time: 0.2237 Steps: 242000, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 003448, Sample Num: 55168, Cur Loss: 0.00000102, Cur Avg Loss: 0.00005052, Log Avg loss: 0.00004460, Global Avg Loss: 0.00239801, Time: 0.2199 Steps: 242200, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 003648, Sample Num: 58368, Cur Loss: 0.00000139, Cur Avg Loss: 0.00004874, Log Avg loss: 0.00001794, Global Avg Loss: 0.00239604, Time: 0.3391 Steps: 242400, Updated lr: 0.000003 Training, Epoch: 0049, Batch: 003848, Sample Num: 61568, Cur Loss: 0.00000345, Cur Avg Loss: 0.00004665, Log Avg loss: 0.00000867, Global Avg Loss: 0.00239407, Time: 0.2153 Steps: 242600, Updated lr: 0.000002 Training, Epoch: 0049, Batch: 004048, Sample Num: 64768, Cur Loss: 0.00000196, Cur Avg Loss: 0.00004537, Log Avg loss: 0.00002060, Global Avg Loss: 0.00239212, Time: 0.2184 Steps: 242800, Updated lr: 0.000002 Training, Epoch: 0049, Batch: 004248, Sample Num: 67968, Cur Loss: 0.00000100, Cur Avg Loss: 0.00004527, Log Avg loss: 0.00004329, Global Avg Loss: 0.00239019, Time: 0.2202 Steps: 243000, Updated lr: 0.000002 Training, Epoch: 0049, Batch: 004448, Sample Num: 71168, Cur Loss: 0.00000062, Cur Avg Loss: 0.00004375, Log Avg loss: 0.00001141, Global Avg Loss: 0.00238823, Time: 0.2575 Steps: 243200, Updated lr: 0.000002 Training, Epoch: 0049, Batch: 004648, Sample Num: 74368, Cur Loss: 0.00000017, Cur Avg Loss: 0.00004390, Log Avg loss: 0.00004727, Global Avg Loss: 0.00238631, Time: 0.2205 Steps: 243400, Updated lr: 0.000002 Training, Epoch: 0049, Batch: 004848, Sample Num: 77568, Cur Loss: 0.00000075, Cur Avg Loss: 0.00004586, Log Avg loss: 0.00009157, Global Avg Loss: 0.00238442, Time: 0.2198 Steps: 243600, Updated lr: 0.000002 ***** Running evaluation checkpoint-243726 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-243726 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1088.190175, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.002008, "eval_total_loss": 2.140029, "eval_acc": 0.999737, "eval_jaccard": 0.98857, "eval_prec": 0.98942, "eval_recall": 0.99006, "eval_f1": 0.989418, "eval_pr_auc": 0.995215, "eval_roc_auc": 0.999349, "eval_fmax": 0.994743, "eval_pmax": 0.99716, "eval_rmax": 0.992338, "eval_tmax": 0.06, "update_flag": false, "test_avg_loss": 0.002106, "test_total_loss": 2.244844, "test_acc": 0.999764, "test_jaccard": 0.989324, "test_prec": 0.990101, "test_recall": 0.990652, "test_f1": 0.990092, "test_pr_auc": 0.995345, "test_roc_auc": 0.99911, "test_fmax": 0.994942, "test_pmax": 0.997796, "test_rmax": 0.992103, "test_tmax": 0.11, "lr": 2.0080742834073478e-06, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0023832240777453472, "train_cur_epoch_loss": 0.23095295752091816, "train_cur_epoch_avg_loss": 4.6432038102315674e-05, "train_cur_epoch_time": 1088.1901745796204, "train_cur_epoch_avg_time": 0.21877566839156018, "epoch": 49, "step": 243726} ################################################## Training, Epoch: 0050, Batch: 000074, Sample Num: 1184, Cur Loss: 0.00035142, Cur Avg Loss: 0.00003233, Log Avg loss: 0.00005498, Global Avg Loss: 0.00238251, Time: 0.2207 Steps: 243800, Updated lr: 0.000002 Training, Epoch: 0050, Batch: 000274, Sample Num: 4384, Cur Loss: 0.00000050, Cur Avg Loss: 0.00003925, Log Avg loss: 0.00004181, Global Avg Loss: 0.00238059, Time: 0.1983 Steps: 244000, Updated lr: 0.000002 Training, Epoch: 0050, Batch: 000474, Sample Num: 7584, Cur Loss: 0.00000119, Cur Avg Loss: 0.00005149, Log Avg loss: 0.00006825, Global Avg Loss: 0.00237870, Time: 0.1145 Steps: 244200, Updated lr: 0.000002 Training, Epoch: 0050, Batch: 000674, Sample Num: 10784, Cur Loss: 0.00000025, Cur Avg Loss: 0.00005243, Log Avg loss: 0.00005468, Global Avg Loss: 0.00237680, Time: 0.2180 Steps: 244400, Updated lr: 0.000002 Training, Epoch: 0050, Batch: 000874, Sample Num: 13984, Cur Loss: 0.00000024, Cur Avg Loss: 0.00005614, Log Avg loss: 0.00006865, Global Avg Loss: 0.00237491, Time: 0.2202 Steps: 244600, Updated lr: 0.000002 Training, Epoch: 0050, Batch: 001074, Sample Num: 17184, Cur Loss: 0.00000301, Cur Avg Loss: 0.00004796, Log Avg loss: 0.00001217, Global Avg Loss: 0.00237298, Time: 0.2187 Steps: 244800, Updated lr: 0.000002 Training, Epoch: 0050, Batch: 001274, Sample Num: 20384, Cur Loss: 0.00000005, Cur Avg Loss: 0.00005324, Log Avg loss: 0.00008165, Global Avg Loss: 0.00237111, Time: 0.2185 Steps: 245000, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 001474, Sample Num: 23584, Cur Loss: 0.00000509, Cur Avg Loss: 0.00004799, Log Avg loss: 0.00001453, Global Avg Loss: 0.00236919, Time: 0.2147 Steps: 245200, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 001674, Sample Num: 26784, Cur Loss: 0.00000069, Cur Avg Loss: 0.00004564, Log Avg loss: 0.00002829, Global Avg Loss: 0.00236728, Time: 0.2177 Steps: 245400, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 001874, Sample Num: 29984, Cur Loss: 0.00000006, Cur Avg Loss: 0.00004216, Log Avg loss: 0.00001301, Global Avg Loss: 0.00236536, Time: 0.0897 Steps: 245600, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 002074, Sample Num: 33184, Cur Loss: 0.00000046, Cur Avg Loss: 0.00004542, Log Avg loss: 0.00007598, Global Avg Loss: 0.00236350, Time: 0.2209 Steps: 245800, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 002274, Sample Num: 36384, Cur Loss: 0.00000377, Cur Avg Loss: 0.00004553, Log Avg loss: 0.00004667, Global Avg Loss: 0.00236161, Time: 0.3404 Steps: 246000, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 002474, Sample Num: 39584, Cur Loss: 0.00001030, Cur Avg Loss: 0.00004516, Log Avg loss: 0.00004096, Global Avg Loss: 0.00235973, Time: 0.1432 Steps: 246200, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 002674, Sample Num: 42784, Cur Loss: 0.00005668, Cur Avg Loss: 0.00004673, Log Avg loss: 0.00006612, Global Avg Loss: 0.00235787, Time: 0.2188 Steps: 246400, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 002874, Sample Num: 45984, Cur Loss: 0.00000216, Cur Avg Loss: 0.00004862, Log Avg loss: 0.00007397, Global Avg Loss: 0.00235602, Time: 0.2109 Steps: 246600, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 003074, Sample Num: 49184, Cur Loss: 0.00000423, Cur Avg Loss: 0.00004915, Log Avg loss: 0.00005666, Global Avg Loss: 0.00235415, Time: 0.2174 Steps: 246800, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 003274, Sample Num: 52384, Cur Loss: 0.00000069, Cur Avg Loss: 0.00004666, Log Avg loss: 0.00000842, Global Avg Loss: 0.00235225, Time: 0.2202 Steps: 247000, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 003474, Sample Num: 55584, Cur Loss: 0.00000670, Cur Avg Loss: 0.00004781, Log Avg loss: 0.00006665, Global Avg Loss: 0.00235040, Time: 0.2177 Steps: 247200, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 003674, Sample Num: 58784, Cur Loss: 0.00000459, Cur Avg Loss: 0.00004610, Log Avg loss: 0.00001651, Global Avg Loss: 0.00234852, Time: 0.2240 Steps: 247400, Updated lr: 0.000001 Training, Epoch: 0050, Batch: 003874, Sample Num: 61984, Cur Loss: 0.00000132, Cur Avg Loss: 0.00004402, Log Avg loss: 0.00000572, Global Avg Loss: 0.00234662, Time: 0.2126 Steps: 247600, Updated lr: 0.000000 Training, Epoch: 0050, Batch: 004074, Sample Num: 65184, Cur Loss: 0.00000114, Cur Avg Loss: 0.00004396, Log Avg loss: 0.00004283, Global Avg Loss: 0.00234477, Time: 0.2886 Steps: 247800, Updated lr: 0.000000 Training, Epoch: 0050, Batch: 004274, Sample Num: 68384, Cur Loss: 0.00000204, Cur Avg Loss: 0.00004397, Log Avg loss: 0.00004415, Global Avg Loss: 0.00234291, Time: 0.2255 Steps: 248000, Updated lr: 0.000000 Training, Epoch: 0050, Batch: 004474, Sample Num: 71584, Cur Loss: 0.00236905, Cur Avg Loss: 0.00004321, Log Avg loss: 0.00002707, Global Avg Loss: 0.00234104, Time: 0.2331 Steps: 248200, Updated lr: 0.000000 Training, Epoch: 0050, Batch: 004674, Sample Num: 74784, Cur Loss: 0.00000353, Cur Avg Loss: 0.00004320, Log Avg loss: 0.00004285, Global Avg Loss: 0.00233919, Time: 0.2273 Steps: 248400, Updated lr: 0.000000 Training, Epoch: 0050, Batch: 004874, Sample Num: 77984, Cur Loss: 0.00000054, Cur Avg Loss: 0.00004443, Log Avg loss: 0.00007315, Global Avg Loss: 0.00233737, Time: 0.2177 Steps: 248600, Updated lr: 0.000000 ***** Running evaluation checkpoint-248700 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## ***** Running testing checkpoint-248700 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## Epoch Time: 1089.888311, Avg time per batch (s): 0.220000 {"eval_avg_loss": 0.002009, "eval_total_loss": 2.141177, "eval_acc": 0.999738, "eval_jaccard": 0.988589, "eval_prec": 0.98941, "eval_recall": 0.99009, "eval_f1": 0.989429, "eval_pr_auc": 0.995167, "eval_roc_auc": 0.99935, "eval_fmax": 0.994811, "eval_pmax": 0.997416, "eval_rmax": 0.99222, "eval_tmax": 0.07, "update_flag": false, "test_avg_loss": 0.002111, "test_total_loss": 2.250413, "test_acc": 0.999765, "test_jaccard": 0.989363, "test_prec": 0.99014, "test_recall": 0.990652, "test_f1": 0.990116, "test_pr_auc": 0.995345, "test_roc_auc": 0.999109, "test_fmax": 0.994956, "test_pmax": 0.997233, "test_rmax": 0.992689, "test_tmax": 0.05, "lr": 0.0, "cur_epoch_step": 4974, "train_global_avg_loss": 0.0023364546773159732, "train_cur_epoch_loss": 0.22260667591989614, "train_cur_epoch_avg_loss": 4.475405627661764e-05, "train_cur_epoch_time": 1089.8883113861084, "train_cur_epoch_avg_time": 0.21911707104666434, "epoch": 50, "step": 248700} ################################################## #########################Best Metric######################### {"epoch": 26, "global_step": 129324, "eval_avg_loss": 0.00184, "eval_total_loss": 1.961172, "eval_acc": 0.999748, "eval_jaccard": 0.989227, "eval_prec": 0.990065, "eval_recall": 0.990637, "eval_f1": 0.990051, "eval_pr_auc": 0.995416, "eval_roc_auc": 0.999357, "eval_fmax": 0.994822, "eval_pmax": 0.997417, "eval_rmax": 0.99224, "eval_tmax": 0.11, "update_flag": true, "test_avg_loss": 0.001982, "test_total_loss": 2.113219, "test_acc": 0.999758, "test_jaccard": 0.989236, "test_prec": 0.99015, "test_recall": 0.990691, "test_f1": 0.990109, "test_pr_auc": 0.99526, "test_roc_auc": 0.999106, "test_fmax": 0.99489, "test_pmax": 0.997777, "test_rmax": 0.99202, "test_tmax": 0.15} ################################################## Total Time: 393157.431728, Avg time per epoch(50 epochs): 7863.150000 ++++++++++++Validation+++++++++++++ best f1 global step: 129324 checkpoint path: ../models/VirusEC4/protein/multi_label/luca_base/matrix/20250508171231/checkpoint-129324 ***** Running evaluation checkpoint-129324 ***** Dev Dataset Instantaneous batch size per GPU = 16 Dev Dataset Num examples = 17053 ################################################## {"evaluation_avg_loss_129324": 0.00184, "evaluation_total_loss_129324": 1.961172, "evaluation_acc_129324": 0.999748, "evaluation_jaccard_129324": 0.989227, "evaluation_prec_129324": 0.990065, "evaluation_recall_129324": 0.990637, "evaluation_f1_129324": 0.990051, "evaluation_pr_auc_129324": 0.995416, "evaluation_roc_auc_129324": 0.999357, "evaluation_fmax_129324": 0.994822, "evaluation_pmax_129324": 0.997417, "evaluation_rmax_129324": 0.99224, "evaluation_tmax_129324": 0.11} ++++++++++++Testing+++++++++++++ best f1 global step: 129324 checkpoint path: ../models/VirusEC4/protein/multi_label/luca_base/matrix/20250508171231/checkpoint-129324 ***** Running testing checkpoint-129324 ***** Test Dataset Instantaneous batch size per GPU = 16 Test Dataset Num examples = [17053] ################################################## {"evaluation_avg_loss_129324": 0.001982, "evaluation_total_loss_129324": 2.113219, "evaluation_acc_129324": 0.999758, "evaluation_jaccard_129324": 0.989236, "evaluation_prec_129324": 0.99015, "evaluation_recall_129324": 0.990691, "evaluation_f1_129324": 0.990109, "evaluation_pr_auc_129324": 0.99526, "evaluation_roc_auc_129324": 0.999106, "evaluation_fmax_129324": 0.99489, "evaluation_pmax_129324": 0.997777, "evaluation_rmax_129324": 0.99202, "evaluation_tmax_129324": 0.15}