diff --git "a/Task500_Glacier_zonefronts/fold_1/training_log_2022_4_11_16_28_44.txt" "b/Task500_Glacier_zonefronts/fold_1/training_log_2022_4_11_16_28_44.txt" new file mode 100644--- /dev/null +++ "b/Task500_Glacier_zonefronts/fold_1/training_log_2022_4_11_16_28_44.txt" @@ -0,0 +1,7776 @@ +Starting... +2022-04-11 16:28:44.757984: Using splits from existing split file: /home/woody/iwi5/iwi5039h/nnUNet_data/nnUNet_preprocessed/Task500_Glacier_zonefronts/splits_final.pkl +2022-04-11 16:28:44.794761: The split file contains 5 splits. +2022-04-11 16:28:44.795399: Desired fold for training: 1 +2022-04-11 16:28:44.796404: This split has 447 training and 112 validation cases. +2022-04-11 16:28:45.327678: TRAINING KEYS: + odict_keys(['Crane_2002-11-09_ERS_20_2_061', 'Crane_2002-11-18_ERS_20_3_195', 'Crane_2003-01-18_ERS_20_1_061', 'Crane_2006-11-27_ERS_20_1_195', 'Crane_2007-03-31_ENVISAT_20_3_467', 'Crane_2007-05-05_ENVISAT_20_1_467', 'Crane_2007-06-09_ENVISAT_20_1_467', 'Crane_2007-07-14_ENVISAT_20_1_467', 'Crane_2007-09-22_ENVISAT_20_1_467', 'Crane_2007-09-24_PALSAR_17_1_137', 'Crane_2007-10-27_ENVISAT_20_2_467', 'Crane_2007-11-21_PALSAR_17_2_135', 'Crane_2008-01-05_ENVISAT_20_2_467', 'Crane_2008-10-13_TSX_7_3_034', 'Crane_2008-10-24_TSX_7_2_034', 'Crane_2008-11-04_TSX_7_1_034', 'Crane_2010-05-29_ENVISAT_20_3_467', 'Crane_2010-09-15_PALSAR_17_4_138', 'Crane_2010-10-14_PALSAR_17_2_135', 'Crane_2010-11-05_TSX_7_2_110', 'Crane_2010-11-16_TSX_7_2_110', 'Crane_2010-11-27_TSX_7_1_110', 'Crane_2010-11-29_PALSAR_17_1_135', 'Crane_2010-12-08_TSX_7_1_110', 'Crane_2010-12-14_TSX_7_1_034', 'Crane_2010-12-25_TSX_7_1_034', 'Crane_2010-12-30_TSX_7_1_110', 'Crane_2011-06-02_TSX_7_2_110', 'Crane_2011-06-24_TSX_7_1_110', 'Crane_2011-07-16_TSX_7_1_110', 'Crane_2011-08-13_TSX_7_1_034', 'Crane_2011-10-18_TSX_7_1_034', 'Crane_2012-05-14_TSX_7_2_034', 'Crane_2012-06-16_TSX_7_2_034', 'Crane_2012-06-27_TSX_7_2_034', 'Crane_2012-07-03_TSX_7_1_125', 'Crane_2012-07-25_TSX_7_2_125', 'Crane_2012-07-30_TSX_7_2_034', 'Crane_2012-08-10_TSX_7_3_034', 'Crane_2012-08-21_TSX_7_3_034', 'Crane_2012-10-04_TSX_7_3_034', 'Crane_2012-11-06_TSX_7_2_034', 'Crane_2013-02-24_TSX_7_2_034', 'Crane_2013-03-07_TSX_7_2_034', 'Crane_2013-03-29_TSX_7_2_034', 'Crane_2013-06-30_TSX_7_2_110', 'Crane_2013-07-22_TSX_7_3_110', 'Crane_2013-08-02_TSX_7_3_110', 'Crane_2013-11-04_TSX_7_4_034', 'Crane_2013-11-15_TSX_7_3_034', 'Crane_2013-11-26_TSX_7_4_034', 'Crane_2013-12-07_TSX_7_5_034', 'Crane_2013-12-18_TSX_7_4_034', 'Crane_2014-02-11_TSX_7_3_034', 'DBE_1995-11-16_ERS_20_1_152', 'DBE_1996-01-25_ERS_20_1_152', 'DBE_1997-01-29_ERS_20_2_424', 'DBE_1997-03-05_ERS_20_1_424', 'DBE_1999-11-10_ERS_20_4_424', 'DBE_2000-09-10_RSAT_20_3_282', 'DBE_2000-10-04_RSAT_20_4_282', 'DBE_2000-10-06_RSAT_12_3_306', 'DBE_2000-10-13_RSAT_20_4_063', 'DBE_2002-02-27_ERS_20_1_424', 'DBE_2002-11-16_ERS_20_3_161', 'DBE_2002-12-21_ERS_20_4_161', 'DBE_2003-01-08_ERS_20_4_424', 'DBE_2003-02-12_ERS_20_3_424', 'DBE_2004-03-03_ERS_20_3_424', 'DBE_2004-04-07_ENVISAT_20_3_424', 'DBE_2004-05-12_ENVISAT_20_3_424', 'DBE_2004-06-16_ENVISAT_20_3_424', 'DBE_2004-07-21_ENVISAT_20_3_424', 'DBE_2004-08-25_ENVISAT_20_3_424', 'DBE_2004-09-13_RSAT_12_3_282', 'DBE_2004-09-29_ENVISAT_20_3_424', 'DBE_2004-10-07_RSAT_12_4_282', 'DBE_2004-10-16_RSAT_12_3_063', 'DBE_2004-11-03_ENVISAT_20_4_424', 'DBE_2004-11-09_RSAT_12_3_063', 'DBE_2004-11-24_RSAT_12_3_282', 'DBE_2004-12-08_ENVISAT_20_5_424', 'DBE_2004-12-08_ERS_20_5_424', 'DBE_2004-12-18_RSAT_12_3_282', 'DBE_2005-02-16_ENVISAT_20_1_424', 'DBE_2005-03-23_ENVISAT_20_1_424', 'DBE_2005-04-27_ENVISAT_20_2_424', 'DBE_2005-05-11_RSAT_12_3_282', 'DBE_2005-06-01_ENVISAT_20_2_424', 'DBE_2005-07-06_ENVISAT_20_3_424', 'DBE_2005-08-10_ENVISAT_20_2_424', 'DBE_2006-01-06_RSAT_12_4_282', 'DBE_2006-02-23_RSAT_12_3_282', 'DBE_2006-03-08_ENVISAT_20_1_424', 'DBE_2006-03-19_RSAT_12_4_282', 'DBE_2006-04-12_ENVISAT_20_1_424', 'DBE_2006-04-12_RSAT_12_1_282', 'DBE_2006-05-06_RSAT_12_3_282', 'DBE_2006-05-18_PALSAR_17_2_135', 'DBE_2006-06-21_ENVISAT_20_2_424', 'DBE_2006-06-21_PALSAR_17_2_137', 'DBE_2006-07-03_PALSAR_17_3_135', 'DBE_2006-07-26_ENVISAT_20_3_424', 'DBE_2006-08-10_RSAT_20_4_282', 'DBE_2006-08-30_ENVISAT_20_2_424', 'DBE_2006-09-03_RSAT_20_5_282', 'DBE_2006-10-04_ENVISAT_20_3_424', 'DBE_2006-11-08_ENVISAT_20_3_424', 'DBE_2007-01-25_RSAT_12_5_282', 'DBE_2007-02-18_RSAT_12_4_282', 'DBE_2007-03-14_RSAT_12_3_282', 'DBE_2007-08-15_ENVISAT_20_5_424', 'DBE_2007-09-19_ENVISAT_20_6_424', 'DBE_2007-12-27_PALSAR_17_5_129', 'DBE_2007-12-27_RSAT_12_5_282', 'DBE_2008-01-20_RSAT_12_4_282', 'DBE_2008-03-30_TSX_7_3_049', 'DBE_2008-09-28_PALSAR_17_5_129', 'DBE_2008-10-14_TSX_7_3_049', 'DBE_2008-10-25_TSX_7_2_049', 'DBE_2008-11-05_TSX_7_2_049', 'DBE_2008-11-13_PALSAR_17_5_129', 'DBE_2008-12-29_PALSAR_17_5_129', 'DBE_2009-02-13_PALSAR_17_5_129', 'DBE_2009-10-23_TSX_7_2_049', 'DBE_2009-11-03_TSX_7_2_049', 'DBE_2010-01-18_PALSAR_17_5_132', 'DBE_2010-01-22_ERS_20_6_152', 'DBE_2010-03-17_ENVISAT_20_6_424', 'DBE_2010-04-21_ENVISAT_20_6_424', 'DBE_2010-10-16_PALSAR_17_4_127', 'DBE_2010-11-12_TSX_7_4_049', 'DBE_2010-11-19_PALSAR_17_4_129', 'DBE_2010-12-06_PALSAR_17_3_132', 'DBE_2011-01-04_PALSAR_17_4_129', 'DBE_2011-01-16_PALSAR_17_4_127', 'DBE_2011-01-17_TSX_7_4_049', 'DBE_2011-07-27_TSX_7_1_110', 'DBE_2011-09-09_TSX_7_1_110', 'DBE_2011-09-20_TSX_7_1_110', 'DBE_2011-10-23_TSX_7_1_110', 'DBE_2011-11-14_TSX_7_1_110', 'DBE_2011-12-28_TSX_7_2_110', 'DBE_2012-03-14_TSX_7_1_110', 'DBE_2012-04-05_TSX_7_1_110', 'DBE_2012-05-08_TSX_7_1_110', 'DBE_2012-06-10_TSX_7_1_110', 'DBE_2012-06-21_TSX_7_1_110', 'DBE_2012-08-04_TSX_7_1_110', 'DBE_2012-08-15_TSX_7_1_110', 'DBE_2012-10-31_TSX_7_1_110', 'DBE_2012-11-11_TSX_7_1_110', 'DBE_2013-01-16_TSX_7_5_110', 'DBE_2013-03-12_TSX_7_1_110', 'DBE_2013-04-03_TSX_7_1_110', 'DBE_2013-05-28_TSX_7_1_110', 'DBE_2013-06-19_TSX_7_1_110', 'DBE_2013-07-11_TSX_7_1_110', 'DBE_2013-09-04_TSX_7_1_110', 'DBE_2013-10-07_TSX_7_1_110', 'DBE_2013-11-09_TSX_7_1_110', 'DBE_2013-12-12_TSX_7_3_110', 'DBE_2013-12-23_TSX_7_2_110', 'DBE_2014-01-03_TSX_7_5_110', 'DBE_2014-02-05_TSX_7_1_110', 'JAC_2009-04-16_TSX_6_1_005', 'JAC_2009-04-27_TSX_6_1_005', 'JAC_2009-05-19_TSX_6_1_005', 'JAC_2009-06-10_TSX_6_1_005', 'JAC_2009-06-21_TSX_6_1_005', 'JAC_2009-07-02_TSX_6_1_005', 'JAC_2009-07-13_TSX_6_1_005', 'JAC_2009-07-24_TSX_6_1_005', 'JAC_2009-08-04_TSX_6_1_005', 'JAC_2009-08-15_TSX_6_1_005', 'JAC_2009-08-26_TSX_6_1_005', 'JAC_2009-09-28_TSX_6_1_005', 'JAC_2009-10-09_TSX_6_1_005', 'JAC_2009-10-31_TSX_6_1_005', 'JAC_2009-11-22_TSX_6_1_005', 'JAC_2009-12-03_TSX_6_1_005', 'JAC_2009-12-14_TSX_6_1_005', 'JAC_2009-12-25_TSX_6_1_005', 'JAC_2010-01-05_TSX_6_1_005', 'JAC_2010-01-16_TSX_6_1_005', 'JAC_2010-02-07_TSX_6_1_005', 'JAC_2010-02-18_TSX_6_1_005', 'JAC_2010-03-12_TSX_6_1_005', 'JAC_2010-03-23_TSX_6_1_005', 'JAC_2010-04-14_TSX_6_1_005', 'JAC_2010-04-25_TSX_6_1_005', 'JAC_2010-05-17_TSX_6_1_005', 'JAC_2010-06-19_TSX_6_1_005', 'JAC_2010-06-30_TSX_6_1_005', 'JAC_2010-07-11_TSX_6_1_005', 'JAC_2010-09-15_TSX_6_1_005', 'JAC_2010-10-07_TSX_6_1_005', 'JAC_2010-10-18_TSX_6_1_005', 'JAC_2010-11-20_TSX_6_1_005', 'JAC_2010-12-01_TSX_6_1_005', 'JAC_2010-12-23_TSX_6_1_005', 'JAC_2011-01-03_TSX_6_1_005', 'JAC_2011-01-25_TSX_6_1_005', 'JAC_2011-03-10_TSX_6_1_005', 'JAC_2011-05-15_TSX_6_1_005', 'JAC_2011-05-26_TSX_6_1_005', 'JAC_2011-06-17_TSX_6_1_005', 'JAC_2011-07-20_TSX_6_1_005', 'JAC_2011-08-23_TSX_6_1_027', 'JAC_2011-09-03_TSX_6_1_027', 'JAC_2011-09-14_TSX_6_1_027', 'JAC_2011-10-05_TSX_6_1_005', 'JAC_2011-10-16_TSX_6_1_005', 'JAC_2011-11-08_TSX_6_1_027', 'JAC_2011-11-18_TSX_6_1_005', 'JAC_2011-12-21_TSX_6_1_005', 'JAC_2012-01-13_TSX_6_1_027', 'JAC_2012-01-23_TSX_6_1_005', 'JAC_2012-02-25_TSX_6_1_005', 'JAC_2012-03-07_TSX_6_1_005', 'JAC_2012-03-29_TSX_6_1_005', 'JAC_2012-04-20_TSX_6_1_005', 'JAC_2012-05-24_TSX_6_1_027', 'JAC_2012-06-14_TSX_6_1_005', 'JAC_2012-07-07_TSX_6_1_027', 'JAC_2012-07-18_TSX_6_1_027', 'JAC_2012-08-09_TSX_6_1_027', 'JAC_2012-08-20_TSX_6_1_027', 'JAC_2012-09-11_TSX_6_1_027', 'JAC_2012-09-21_TSX_6_1_005', 'JAC_2012-10-03_TSX_6_1_027', 'JAC_2012-10-14_TSX_6_1_027', 'JAC_2012-11-04_TSX_6_1_005', 'JAC_2012-11-15_TSX_6_1_005', 'JAC_2012-12-07_TSX_6_1_005', 'JAC_2012-12-19_TSX_6_1_027', 'JAC_2013-01-09_TSX_6_1_005', 'JAC_2013-02-22_TSX_6_1_005', 'JAC_2013-03-06_TSX_6_1_027', 'JAC_2013-04-07_TSX_6_1_005', 'JAC_2013-04-18_TSX_6_1_005', 'JAC_2013-05-22_TSX_6_1_027', 'JAC_2013-06-13_TSX_6_1_027', 'JAC_2013-06-24_TSX_6_1_027', 'JAC_2013-07-05_TSX_6_1_027', 'JAC_2013-07-27_TSX_6_1_027', 'JAC_2013-08-28_TSX_6_1_005', 'JAC_2013-09-08_TSX_6_1_005', 'JAC_2013-09-20_TSX_6_1_027', 'JAC_2013-10-12_TSX_6_1_027', 'JAC_2013-10-23_TSX_6_1_027', 'JAC_2013-12-06_TSX_6_1_027', 'JAC_2013-12-17_TSX_6_1_027', 'JAC_2014-01-29_TSX_6_1_005', 'JAC_2014-03-03_TSX_6_1_005', 'JAC_2014-03-14_TSX_6_1_005', 'JAC_2014-04-05_TSX_6_1_005', 'JAC_2014-04-16_TSX_6_1_005', 'JAC_2014-06-10_TSX_6_1_005', 'JAC_2014-06-21_TSX_6_1_005', 'JAC_2014-07-02_TSX_6_1_005', 'JAC_2014-07-03_TSX_6_1_027', 'JAC_2014-07-14_TSX_6_1_027', 'JAC_2014-07-25_TSX_6_1_027', 'JAC_2014-08-04_TSX_6_1_005', 'JAC_2014-09-28_TSX_6_1_005', 'JAC_2014-10-09_TSX_6_1_005', 'JAC_2014-10-20_TSX_6_1_005', 'JAC_2014-12-03_TSX_6_1_005', 'JAC_2014-12-14_TSX_6_1_005', 'JAC_2014-12-25_TSX_6_1_005', 'JAC_2015-01-05_TSX_6_1_005', 'JAC_2015-01-16_TSX_6_1_005', 'JAC_2015-01-27_TSX_6_1_005', 'JAC_2015-02-07_TSX_6_1_005', 'JAC_2015-02-19_TSX_6_1_027', 'JAC_2015-04-14_TSX_6_1_005', 'JAC_2015-04-25_TSX_6_1_005', 'JAC_2015-05-06_TSX_6_1_005', 'JAC_2015-05-17_TSX_6_1_005', 'JAC_2015-06-08_TSX_6_1_005', 'JAC_2015-07-11_TSX_6_1_005', 'JAC_2015-09-04_TSX_6_1_005', 'JAC_2015-09-26_TSX_6_1_005', 'JAC_2015-11-09_TSX_6_1_005', 'JAC_2015-11-20_TSX_6_1_005', 'JAC_2015-12-12_TSX_6_1_005', 'JAC_2015-12-23_TSX_6_1_005', 'Jorum_2003-01-27_ERS_20_1_195', 'Jorum_2007-02-05_ERS_20_1_195', 'Jorum_2007-03-31_ENVISAT_20_2_467', 'Jorum_2007-05-05_ENVISAT_20_2_467', 'Jorum_2007-06-09_ENVISAT_20_2_467', 'Jorum_2007-07-14_ENVISAT_20_2_467', 'Jorum_2007-10-27_ENVISAT_20_3_467', 'Jorum_2007-11-21_PALSAR_17_2_135', 'Jorum_2008-01-05_ENVISAT_20_2_467', 'Jorum_2008-11-23_PALSAR_17_2_135', 'Jorum_2009-12-25_PALSAR_17_1_136', 'Jorum_2010-03-20_ENVISAT_20_2_467', 'Jorum_2010-05-29_ENVISAT_20_2_467', 'Jorum_2010-07-03_ENVISAT_20_2_467', 'Jorum_2010-09-15_PALSAR_17_2_138', 'Jorum_2010-10-14_PALSAR_17_2_135', 'Jorum_2010-11-05_TSX_7_1_110', 'Jorum_2010-11-16_TSX_7_2_110', 'Jorum_2010-11-29_PALSAR_17_1_135', 'Jorum_2010-12-08_TSX_7_2_110', 'Jorum_2010-12-19_TSX_7_1_110', 'Jorum_2010-12-25_TSX_7_1_034', 'Jorum_2010-12-30_TSX_7_1_110', 'Jorum_2011-06-02_TSX_7_3_110', 'Jorum_2011-06-24_TSX_7_2_110', 'Jorum_2011-08-13_TSX_7_2_034', 'Jorum_2011-09-04_TSX_7_4_034', 'Jorum_2011-10-18_TSX_7_3_034', 'Jorum_2011-12-01_TSX_7_1_034', 'Jorum_2011-12-12_TSX_7_3_034', 'Jorum_2012-05-14_TSX_7_2_034', 'Jorum_2012-06-16_TSX_7_2_034', 'Jorum_2012-06-27_TSX_7_2_034', 'Jorum_2012-07-03_TSX_7_2_125', 'Jorum_2012-07-30_TSX_7_2_034', 'Jorum_2012-08-10_TSX_7_2_034', 'Jorum_2012-09-01_TSX_7_2_034', 'Jorum_2012-10-04_TSX_7_2_034', 'Jorum_2012-11-06_TSX_7_2_034', 'Jorum_2013-02-24_TSX_7_3_034', 'Jorum_2013-03-07_TSX_7_2_034', 'Jorum_2013-03-29_TSX_7_2_034', 'Jorum_2013-06-30_TSX_7_2_110', 'Jorum_2013-07-22_TSX_7_1_110', 'Jorum_2013-08-02_TSX_7_2_110', 'Jorum_2013-08-19_TSX_7_2_034', 'Jorum_2013-11-04_TSX_7_2_034', 'Jorum_2013-11-15_TSX_7_2_034', 'Jorum_2013-11-26_TSX_7_5_034', 'Jorum_2013-12-07_TSX_7_5_034', 'Jorum_2013-12-18_TSX_7_5_034', 'Jorum_2013-12-29_TSX_7_4_034', 'Jorum_2014-02-11_TSX_7_2_034', 'Jorum_2016-12-30_S1_20_2_009', 'Jorum_2017-06-16_S1_20_2_009', 'Jorum_2017-09-08_S1_20_2_009', 'Jorum_2018-01-06_S1_20_3_009', 'Jorum_2018-03-07_S1_20_3_009', 'Jorum_2018-09-03_S1_20_3_009', 'Jorum_2019-01-01_S1_20_3_009', 'Jorum_2019-03-02_S1_20_3_009', 'Jorum_2019-09-22_S1_20_3_009', 'Jorum_2020-01-08_S1_20_3_009', 'Jorum_2020-03-08_S1_20_3_009', 'Jorum_2020-06-12_S1_20_3_009', 'SI_1995-11-14_ERS_20_4_118', 'SI_1995-11-15_ERS_20_3_118', 'SI_1995-11-16_ERS_20_4_152', 'SI_1996-01-22_ERS_20_3_109', 'SI_1996-01-23_ERS_20_5_109', 'SI_1996-01-24_ERS_20_4_118', 'SI_1996-02-12_ERS_20_2_390', 'SI_1996-02-29_ERS_20_2_152', 'SI_1996-03-18_ERS_20_2_390', 'SI_1997-01-27_ERS_20_4_390', 'SI_1997-03-03_ERS_20_4_390', 'SI_1999-11-07_ERS_20_2_381', 'SI_2000-02-20_ERS_20_3_381', 'SI_2000-09-10_RSAT_20_5_282', 'SI_2000-09-19_RSAT_20_5_063', 'SI_2000-10-04_RSAT_20_5_282', 'SI_2000-10-13_RSAT_20_5_063', 'SI_2002-01-21_ERS_20_2_390', 'SI_2003-01-25_ERS_20_3_161', 'SI_2003-12-05_ENVISAT_20_4_152', 'SI_2004-02-13_ENVISAT_20_4_152', 'SI_2004-07-02_ENVISAT_20_4_152', 'SI_2004-08-06_ENVISAT_20_4_152', 'SI_2004-10-15_ERS_20_5_152', 'SI_2004-11-09_RSAT_12_5_063', 'SI_2004-12-03_RSAT_12_5_063', 'SI_2004-12-18_RSAT_12_4_282', 'SI_2004-12-24_ENVISAT_20_5_152', 'SI_2004-12-24_ERS_20_5_152', 'SI_2005-02-28_RSAT_12_5_282', 'SI_2005-03-04_ENVISAT_20_3_152', 'SI_2005-04-08_ENVISAT_20_2_152', 'SI_2005-04-17_RSAT_12_3_282', 'SI_2005-05-11_RSAT_12_2_282', 'SI_2005-07-22_ENVISAT_20_2_152', 'SI_2005-08-26_ENVISAT_20_2_152', 'SI_2006-01-06_RSAT_12_2_282', 'SI_2006-01-29_ENVISAT_20_3_381', 'SI_2006-03-05_ENVISAT_20_2_381', 'SI_2006-03-19_RSAT_12_2_282', 'SI_2006-05-06_RSAT_12_3_282', 'SI_2006-05-18_PALSAR_17_2_135', 'SI_2006-07-03_PALSAR_17_3_135', 'SI_2006-07-23_ENVISAT_20_4_381', 'SI_2006-08-10_RSAT_20_3_282', 'SI_2006-08-27_ENVISAT_20_3_381', 'SI_2006-09-03_RSAT_12_2_282', 'SI_2006-10-18_ERS_20_4_118', 'SI_2006-11-21_ERS_20_3_109', 'SI_2007-01-25_RSAT_12_1_282', 'SI_2007-03-14_RSAT_12_2_282', 'SI_2007-03-25_ENVISAT_20_3_381', 'SI_2007-05-01_RSAT_12_2_282', 'SI_2007-06-03_ENVISAT_20_3_381', 'SI_2007-10-21_ENVISAT_20_3_381', 'SI_2007-12-03_RSAT_12_4_282', 'SI_2007-12-27_RSAT_12_2_282', 'SI_2008-01-20_RSAT_12_2_282', 'SI_2008-04-13_ENVISAT_20_2_381', 'SI_2008-05-18_ENVISAT_20_3_381', 'SI_2008-07-27_ENVISAT_20_4_381', 'SI_2008-08-31_ENVISAT_20_4_381', 'SI_2009-02-22_ENVISAT_20_2_381', 'SI_2009-07-12_ENVISAT_20_3_381', 'SI_2010-01-22_ERS_20_3_152', 'SI_2010-02-26_ERS_20_2_152', 'SI_2010-04-02_ERS_20_2_152', 'SI_2010-10-16_PALSAR_17_3_127', 'SI_2010-12-01_PALSAR_17_3_127', 'SI_2011-01-16_PALSAR_17_2_127', 'SI_2011-03-03_PALSAR_17_2_127', 'SI_2011-07-06_TSX_7_1_125', 'SI_2011-07-28_TSX_7_1_125', 'SI_2011-08-19_TSX_7_1_125', 'SI_2011-09-21_TSX_7_1_125', 'SI_2011-11-26_TSX_7_2_125', 'SI_2011-12-29_TSX_7_1_125', 'SI_2012-03-04_TSX_7_1_125', 'SI_2012-03-15_TSX_7_1_125', 'SI_2012-03-26_TSX_7_1_125', 'SI_2012-04-28_TSX_7_1_125', 'SI_2012-05-20_TSX_7_1_125', 'SI_2012-06-22_TSX_7_1_125', 'SI_2012-07-03_TSX_7_1_125', 'SI_2013-08-14_TSX_7_1_125', 'SI_2013-09-16_TSX_7_1_125', 'SI_2013-10-19_TSX_7_1_125', 'SI_2013-10-28_TSX_7_1_104', 'SI_2013-11-08_TSX_7_2_104', 'SI_2013-12-02_TSX_7_1_125', 'SI_2013-12-13_TSX_7_2_125', 'SI_2014-03-22_TSX_7_1_125', 'SI_2014-12-11_TSX_7_4_125', 'SI_2014-12-22_TSX_7_2_125']) +2022-04-11 16:28:45.328726: VALIDATION KEYS: + odict_keys(['Crane_2007-02-05_ERS_20_1_195', 'Crane_2008-11-23_PALSAR_17_2_135', 'Crane_2009-09-29_PALSAR_17_1_137', 'Crane_2009-11-02_TSX_7_3_034', 'Crane_2009-11-13_TSX_7_1_034', 'Crane_2010-07-03_ENVISAT_20_2_467', 'Crane_2010-12-19_TSX_7_1_110', 'Crane_2011-09-04_TSX_7_1_034', 'Crane_2011-12-01_TSX_7_1_034', 'Crane_2011-12-12_TSX_7_2_034', 'Crane_2012-09-01_TSX_7_1_034', 'Crane_2013-07-17_TSX_7_3_034', 'Crane_2013-08-19_TSX_7_4_034', 'Crane_2013-12-29_TSX_7_4_034', 'Crane_2014-01-20_TSX_7_2_034', 'DBE_1998-02-18_ERS_20_1_424', 'DBE_2003-11-19_ERS_20_3_424', 'DBE_2004-03-03_ENVISAT_20_3_424', 'DBE_2004-12-11_RSAT_12_4_182', 'DBE_2005-01-12_ERS_20_6_424', 'DBE_2006-08-06_PALSAR_17_2_137', 'DBE_2006-11-06_PALSAR_17_3_137', 'DBE_2007-01-01_RSAT_12_5_282', 'DBE_2007-05-01_RSAT_12_4_282', 'DBE_2007-11-11_PALSAR_17_4_129', 'DBE_2007-12-03_RSAT_12_4_282', 'DBE_2009-12-03_PALSAR_17_5_132', 'DBE_2010-10-10_TSX_7_2_049', 'DBE_2010-10-21_PALSAR_17_4_132', 'DBE_2010-11-23_TSX_7_5_049', 'DBE_2010-12-01_PALSAR_17_4_127', 'DBE_2011-01-28_TSX_7_2_049', 'DBE_2011-07-05_TSX_7_1_110', 'DBE_2011-12-06_TSX_7_5_110', 'DBE_2013-03-23_TSX_7_1_110', 'DBE_2013-12-01_TSX_7_2_110', 'DBE_2014-03-21_TSX_7_2_110', 'JAC_2009-05-08_TSX_6_1_005', 'JAC_2009-05-30_TSX_6_1_005', 'JAC_2009-09-06_TSX_6_1_005', 'JAC_2009-09-17_TSX_6_1_005', 'JAC_2009-10-20_TSX_6_1_005', 'JAC_2009-11-11_TSX_6_1_005', 'JAC_2010-05-06_TSX_6_1_005', 'JAC_2010-07-22_TSX_6_1_005', 'JAC_2010-09-04_TSX_6_1_005', 'JAC_2010-11-09_TSX_6_1_005', 'JAC_2011-02-16_TSX_6_1_005', 'JAC_2011-03-21_TSX_6_1_005', 'JAC_2011-04-12_TSX_6_1_005', 'JAC_2011-04-23_TSX_6_1_005', 'JAC_2011-12-10_TSX_6_1_005', 'JAC_2012-02-14_TSX_6_1_005', 'JAC_2012-06-03_TSX_6_1_005', 'JAC_2013-01-20_TSX_6_1_005', 'JAC_2013-02-11_TSX_6_1_005', 'JAC_2013-03-17_TSX_6_1_027', 'JAC_2013-05-10_TSX_6_1_005', 'JAC_2013-08-17_TSX_6_1_005', 'JAC_2013-11-14_TSX_6_1_027', 'JAC_2013-11-25_TSX_6_1_027', 'JAC_2014-03-25_TSX_6_1_005', 'JAC_2014-04-27_TSX_6_1_005', 'JAC_2014-09-18_TSX_6_1_027', 'JAC_2014-11-11_TSX_6_1_005', 'JAC_2015-03-12_TSX_6_1_005', 'JAC_2015-03-23_TSX_6_1_005', 'JAC_2015-06-19_TSX_6_1_005', 'JAC_2015-07-22_TSX_6_1_005', 'JAC_2015-08-02_TSX_6_1_005', 'JAC_2015-08-13_TSX_6_1_005', 'JAC_2015-10-18_TSX_6_1_005', 'JAC_2015-10-29_TSX_6_1_005', 'Jorum_2006-11-27_ERS_20_3_195', 'Jorum_2007-09-22_ENVISAT_20_3_467', 'Jorum_2010-11-27_TSX_7_2_110', 'Jorum_2010-12-14_TSX_7_1_034', 'Jorum_2011-07-16_TSX_7_3_110', 'Jorum_2012-07-25_TSX_7_2_125', 'Jorum_2012-08-21_TSX_7_2_034', 'Jorum_2013-07-17_TSX_7_1_034', 'Jorum_2014-01-20_TSX_7_2_034', 'Jorum_2016-07-03_S1_20_2_009', 'Jorum_2016-09-01_S1_20_2_009', 'Jorum_2018-06-11_S1_20_2_009', 'SI_1995-11-13_ERS_20_3_109', 'SI_2002-02-25_ERS_20_3_390', 'SI_2002-11-12_ERS_20_4_109', 'SI_2002-12-21_ERS_20_3_161', 'SI_2004-09-13_RSAT_12_5_282', 'SI_2004-11-19_ERS_20_4_152', 'SI_2004-11-24_RSAT_12_4_282', 'SI_2005-01-28_ERS_20_5_152', 'SI_2006-02-23_RSAT_12_1_282', 'SI_2006-04-12_RSAT_12_1_282', 'SI_2006-09-03_RSAT_20_2_282', 'SI_2007-01-01_RSAT_12_3_282', 'SI_2007-01-14_ENVISAT_20_3_381', 'SI_2007-07-08_ENVISAT_20_3_381', 'SI_2007-09-16_ENVISAT_20_3_381', 'SI_2009-03-29_ENVISAT_20_4_381', 'SI_2009-08-16_ENVISAT_20_3_381', 'SI_2010-01-13_PALSAR_17_2_127', 'SI_2011-06-14_TSX_7_1_125', 'SI_2011-09-10_TSX_7_1_125', 'SI_2012-01-09_TSX_7_1_125', 'SI_2012-08-05_TSX_7_1_125', 'SI_2012-08-16_TSX_7_1_125', 'SI_2013-07-12_TSX_7_1_125', 'SI_2013-11-21_TSX_7_1_125', 'SI_2014-01-04_TSX_7_2_125', 'SI_2014-04-02_TSX_7_1_125']) +2022-04-11 16:29:02.291081: lr: 0.01 +2022-04-11 16:29:49.335207: Unable to plot network architecture: +2022-04-11 16:29:49.564789: No module named 'hiddenlayer' +2022-04-11 16:29:50.311340: +printing the network instead: + +2022-04-11 16:29:50.739761: Generic_UNet( + (conv_blocks_localization): ModuleList( + (0): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(960, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (1): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(960, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (2): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(960, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (3): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(960, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (4): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(512, 256, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(256, 256, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (5): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(256, 128, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(128, 128, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (6): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(128, 64, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(64, 64, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + (7): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(64, 32, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(32, 32, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + ) + (conv_blocks_context): ModuleList( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(1, 32, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(32, 32, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(32, 64, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(64, 64, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (2): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(64, 128, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(128, 128, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (3): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(128, 256, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(256, 256, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (4): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(256, 480, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (5): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (6): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (7): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + (1): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (8): Sequential( + (0): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=[2, 2], padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + (1): StackedConvLayers( + (blocks): Sequential( + (0): ConvDropoutNormNonlin( + (conv): Conv2d(480, 480, kernel_size=[3, 3], stride=(1, 1), padding=[1, 1]) + (instnorm): InstanceNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False) + (lrelu): LeakyReLU(negative_slope=0.01, inplace=True) + ) + ) + ) + ) + ) + (td): ModuleList() + (tu): ModuleList( + (0): ConvTranspose2d(480, 480, kernel_size=[2, 2], stride=[2, 2], bias=False) + (1): ConvTranspose2d(480, 480, kernel_size=[2, 2], stride=[2, 2], bias=False) + (2): ConvTranspose2d(480, 480, kernel_size=[2, 2], stride=[2, 2], bias=False) + (3): ConvTranspose2d(480, 480, kernel_size=[2, 2], stride=[2, 2], bias=False) + (4): ConvTranspose2d(480, 256, kernel_size=[2, 2], stride=[2, 2], bias=False) + (5): ConvTranspose2d(256, 128, kernel_size=[2, 2], stride=[2, 2], bias=False) + (6): ConvTranspose2d(128, 64, kernel_size=[2, 2], stride=[2, 2], bias=False) + (7): ConvTranspose2d(64, 32, kernel_size=[2, 2], stride=[2, 2], bias=False) + ) + (seg_outputs): ModuleList( + (0): Conv2d(480, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + (1): Conv2d(480, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + (2): Conv2d(480, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + (3): Conv2d(480, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + (4): Conv2d(256, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + (5): Conv2d(128, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + (6): Conv2d(64, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + (7): Conv2d(32, 5, kernel_size=(1, 1), stride=(1, 1), bias=False) + ) +) +2022-04-11 16:29:51.112449: + +2022-04-11 16:29:51.561174: svmem(total=99932160000, available=85085331456, percent=14.9, used=11392286720, free=80409554944, active=11379920896, inactive=6379593728, buffers=11878400, cached=8118439936, shared=1669828608, slab=688058368) +2022-04-11 16:29:52.360780: +epoch: 0 +2022-04-11 16:32:22.851044: train loss : 0.8387 +2022-04-11 16:32:38.870098: validation loss: 0.6249 +2022-04-11 16:32:38.891727: Average global foreground Dice: [0.0003, 0.6868, 0.1581, 0.0001] +2022-04-11 16:32:38.913346: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:32:39.783700: lr: 0.009982 +2022-04-11 16:32:39.793674: saving scheduled checkpoint file... +2022-04-11 16:32:40.004084: saving checkpoint... +2022-04-11 16:32:44.319499: done, saving took 4.52 seconds +2022-04-11 16:32:44.360532: done +2022-04-11 16:32:44.370920: This epoch took 171.216346 s + +2022-04-11 16:32:44.376782: svmem(total=99932160000, available=81716068352, percent=18.2, used=14222749696, free=62264999936, active=23440875520, inactive=12403032064, buffers=12124160, cached=23432286208, shared=2190729216, slab=730554368) +2022-04-11 16:32:44.384186: +epoch: 1 +2022-04-11 16:34:33.576086: train loss : 0.5688 +2022-04-11 16:34:45.237858: validation loss: 0.5661 +2022-04-11 16:34:45.244386: Average global foreground Dice: [0.2513, 0.6382, 0.3453, 0.0003] +2022-04-11 16:34:45.245377: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:34:46.037022: lr: 0.009964 +2022-04-11 16:34:46.178297: saving scheduled checkpoint file... +2022-04-11 16:34:46.424321: saving checkpoint... +2022-04-11 16:34:53.008708: done, saving took 6.71 seconds +2022-04-11 16:34:53.023137: done +2022-04-11 16:34:53.069706: saving checkpoint... +2022-04-11 16:34:57.363526: done, saving took 4.34 seconds +2022-04-11 16:34:57.396667: This epoch took 133.004647 s + +2022-04-11 16:34:57.406317: svmem(total=99932160000, available=81501237248, percent=18.4, used=14432690176, free=57373626368, active=29730091008, inactive=11002667008, buffers=12156928, cached=28113686528, shared=2191028224, slab=740278272) +2022-04-11 16:34:57.413803: +epoch: 2 +2022-04-11 16:36:48.088983: train loss : 0.4660 +2022-04-11 16:37:00.191470: validation loss: 0.3796 +2022-04-11 16:37:00.205987: Average global foreground Dice: [0.2257, 0.716, 0.5192, 0.0] +2022-04-11 16:37:00.208762: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:37:01.313987: lr: 0.009946 +2022-04-11 16:37:01.456285: saving scheduled checkpoint file... +2022-04-11 16:37:01.734924: saving checkpoint... +2022-04-11 16:37:07.904109: done, saving took 6.32 seconds +2022-04-11 16:37:07.925874: done +2022-04-11 16:37:07.975958: saving checkpoint... +2022-04-11 16:37:12.349180: done, saving took 4.42 seconds +2022-04-11 16:37:12.365805: This epoch took 134.942971 s + +2022-04-11 16:37:12.367403: svmem(total=99932160000, available=81381396480, percent=18.6, used=14549848064, free=55819485184, active=33234403328, inactive=9045590016, buffers=12308480, cached=29550518272, shared=2190807040, slab=745578496) +2022-04-11 16:37:12.368121: +epoch: 3 +2022-04-11 16:39:01.986212: train loss : 0.4435 +2022-04-11 16:39:13.533003: validation loss: 0.4070 +2022-04-11 16:39:13.678184: Average global foreground Dice: [0.2512, 0.7192, 0.609, 0.0] +2022-04-11 16:39:13.820138: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:39:14.837539: lr: 0.009928 +2022-04-11 16:39:14.962598: saving scheduled checkpoint file... +2022-04-11 16:39:15.285748: saving checkpoint... +2022-04-11 16:39:23.010463: done, saving took 7.86 seconds +2022-04-11 16:39:23.148024: done +2022-04-11 16:39:23.302097: saving checkpoint... +2022-04-11 16:39:27.721234: done, saving took 4.57 seconds +2022-04-11 16:39:27.730822: This epoch took 135.361876 s + +2022-04-11 16:39:27.732061: svmem(total=99932160000, available=81227829248, percent=18.7, used=14704582656, free=54899195904, active=35305279488, inactive=7898091520, buffers=12341248, cached=30316040192, shared=2190729216, slab=747560960) +2022-04-11 16:39:27.733003: +epoch: 4 +2022-04-11 16:41:15.822732: train loss : 0.3951 +2022-04-11 16:41:28.019509: validation loss: 0.5156 +2022-04-11 16:41:28.153995: Average global foreground Dice: [0.1016, 0.6758, 0.6367, 0.0] +2022-04-11 16:41:28.304846: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:41:29.127619: lr: 0.00991 +2022-04-11 16:41:29.251398: saving scheduled checkpoint file... +2022-04-11 16:41:29.532953: saving checkpoint... +2022-04-11 16:41:37.014208: done, saving took 7.61 seconds +2022-04-11 16:41:37.345384: done +2022-04-11 16:41:37.500910: saving checkpoint... +2022-04-11 16:41:41.906652: done, saving took 4.45 seconds +2022-04-11 16:41:41.916638: This epoch took 134.182872 s + +2022-04-11 16:41:41.917740: svmem(total=99932160000, available=81042239488, percent=18.9, used=14888292352, free=54242410496, active=36560154624, inactive=7304122368, buffers=12374016, cached=30789083136, shared=2190540800, slab=748752896) +2022-04-11 16:41:41.918553: +epoch: 5 +2022-04-11 16:43:31.329178: train loss : 0.4305 +2022-04-11 16:43:42.930660: validation loss: 0.4195 +2022-04-11 16:43:43.053982: Average global foreground Dice: [0.2473, 0.6809, 0.665, 0.0] +2022-04-11 16:43:43.186478: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:43:44.312053: lr: 0.009892 +2022-04-11 16:43:44.463362: saving scheduled checkpoint file... +2022-04-11 16:43:44.733538: saving checkpoint... +2022-04-11 16:43:52.556399: done, saving took 7.94 seconds +2022-04-11 16:43:52.711126: done +2022-04-11 16:43:52.852572: saving checkpoint... +2022-04-11 16:43:57.275541: done, saving took 4.56 seconds +2022-04-11 16:43:57.297131: This epoch took 135.377998 s + +2022-04-11 16:43:57.298205: svmem(total=99932160000, available=81070395392, percent=18.9, used=14859681792, free=54018875392, active=37152407552, inactive=6932336640, buffers=12406784, cached=31041196032, shared=2190884864, slab=748417024) +2022-04-11 16:43:57.299167: +epoch: 6 +2022-04-11 16:45:46.887656: train loss : 0.3397 +2022-04-11 16:45:59.590245: validation loss: 0.3012 +2022-04-11 16:45:59.731947: Average global foreground Dice: [0.5297, 0.6697, 0.7007, 0.0] +2022-04-11 16:45:59.874517: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:46:00.770111: lr: 0.009874 +2022-04-11 16:46:00.792947: saving scheduled checkpoint file... +2022-04-11 16:46:00.938152: saving checkpoint... +2022-04-11 16:46:08.806324: done, saving took 8.01 seconds +2022-04-11 16:46:08.828189: done +2022-04-11 16:46:08.958582: saving checkpoint... +2022-04-11 16:46:13.340025: done, saving took 4.51 seconds +2022-04-11 16:46:13.370847: This epoch took 136.070919 s + +2022-04-11 16:46:13.375800: svmem(total=99932160000, available=80597118976, percent=19.3, used=15333089280, free=53395849216, active=38130290688, inactive=6582841344, buffers=12484608, cached=31190736896, shared=2191114240, slab=748720128) +2022-04-11 16:46:13.376669: +epoch: 7 +2022-04-11 16:48:02.791476: train loss : 0.2677 +2022-04-11 16:48:15.062535: validation loss: 0.2341 +2022-04-11 16:48:15.195285: Average global foreground Dice: [0.4551, 0.7249, 0.67, 0.0] +2022-04-11 16:48:15.319233: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:48:16.248610: lr: 0.009856 +2022-04-11 16:48:16.462167: saving scheduled checkpoint file... +2022-04-11 16:48:16.718826: saving checkpoint... +2022-04-11 16:48:23.239424: done, saving took 6.64 seconds +2022-04-11 16:48:23.614866: done +2022-04-11 16:48:23.893689: saving checkpoint... +2022-04-11 16:48:29.332078: done, saving took 5.57 seconds +2022-04-11 16:48:29.349108: This epoch took 135.971718 s + +2022-04-11 16:48:29.350209: svmem(total=99932160000, available=80698380288, percent=19.2, used=15231954944, free=53484863488, active=38557114368, inactive=6064390144, buffers=12402688, cached=31202938880, shared=2190651392, slab=749850624) +2022-04-11 16:48:29.351063: +epoch: 8 +2022-04-11 16:50:19.376632: train loss : 0.2881 +2022-04-11 16:50:31.194967: validation loss: 0.2800 +2022-04-11 16:50:31.333372: Average global foreground Dice: [0.4484, 0.7409, 0.6366, 0.0] +2022-04-11 16:50:31.475135: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:50:32.485145: lr: 0.009838 +2022-04-11 16:50:32.609708: saving scheduled checkpoint file... +2022-04-11 16:50:32.858352: saving checkpoint... +2022-04-11 16:50:39.430165: done, saving took 6.70 seconds +2022-04-11 16:50:39.778511: done +2022-04-11 16:50:40.051229: saving checkpoint... +2022-04-11 16:50:45.548341: done, saving took 5.64 seconds +2022-04-11 16:50:45.562904: This epoch took 136.211067 s + +2022-04-11 16:50:45.563752: svmem(total=99932160000, available=80629350400, percent=19.3, used=15301009408, free=53517463552, active=39213604864, inactive=5372723200, buffers=12443648, cached=31101243392, shared=2190729216, slab=749125632) +2022-04-11 16:50:45.564384: +epoch: 9 +2022-04-11 16:52:34.911842: train loss : 0.2454 +2022-04-11 16:52:46.998659: validation loss: 0.2224 +2022-04-11 16:52:47.130115: Average global foreground Dice: [0.5031, 0.6704, 0.7371, 0.0] +2022-04-11 16:52:47.254200: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:52:48.138784: lr: 0.00982 +2022-04-11 16:52:48.263493: saving scheduled checkpoint file... +2022-04-11 16:52:48.511663: saving checkpoint... +2022-04-11 16:52:55.072227: done, saving took 6.68 seconds +2022-04-11 16:52:55.429927: done +2022-04-11 16:52:55.688440: saving checkpoint... +2022-04-11 16:53:01.178425: done, saving took 5.62 seconds +2022-04-11 16:53:01.199085: This epoch took 135.633986 s + +2022-04-11 16:53:01.200294: svmem(total=99932160000, available=80432734208, percent=19.5, used=15487258624, free=53307142144, active=39651336192, inactive=5150101504, buffers=12476416, cached=31125282816, shared=2201104384, slab=748285952) +2022-04-11 16:53:01.201119: +epoch: 10 +2022-04-11 16:54:52.288086: train loss : 0.2175 +2022-04-11 16:55:05.530443: validation loss: 0.1670 +2022-04-11 16:55:05.683029: Average global foreground Dice: [0.5061, 0.7509, 0.7257, 0.0] +2022-04-11 16:55:05.807375: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:55:06.611601: lr: 0.009802 +2022-04-11 16:55:06.744511: saving scheduled checkpoint file... +2022-04-11 16:55:07.013744: saving checkpoint... +2022-04-11 16:55:14.356552: done, saving took 7.46 seconds +2022-04-11 16:55:14.759122: done +2022-04-11 16:55:15.032290: saving checkpoint... +2022-04-11 16:55:19.582328: done, saving took 4.69 seconds +2022-04-11 16:55:19.607991: This epoch took 138.406222 s + +2022-04-11 16:55:19.609390: svmem(total=99932160000, available=80327901184, percent=19.6, used=15597633536, free=53356470272, active=39909793792, inactive=4837277696, buffers=12550144, cached=30965506048, shared=2195386368, slab=749256704) +2022-04-11 16:55:19.610164: +epoch: 11 +2022-04-11 16:57:10.041878: train loss : 0.2134 +2022-04-11 16:57:23.295912: validation loss: 0.2031 +2022-04-11 16:57:23.456900: Average global foreground Dice: [0.4384, 0.7592, 0.7788, 0.0] +2022-04-11 16:57:23.598804: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:57:24.483875: lr: 0.009784 +2022-04-11 16:57:24.590530: saving scheduled checkpoint file... +2022-04-11 16:57:24.679921: saving checkpoint... +2022-04-11 16:57:29.040233: done, saving took 4.45 seconds +2022-04-11 16:57:29.055886: done +2022-04-11 16:57:29.102710: saving checkpoint... +2022-04-11 16:57:33.517781: done, saving took 4.46 seconds +2022-04-11 16:57:33.531358: This epoch took 133.920230 s + +2022-04-11 16:57:33.532493: svmem(total=99932160000, available=80157368320, percent=19.8, used=15772889088, free=53258186752, active=39972315136, inactive=4877398016, buffers=12570624, cached=30888513536, shared=2190618624, slab=749481984) +2022-04-11 16:57:33.533245: +epoch: 12 +2022-04-11 16:59:23.207253: train loss : 0.1841 +2022-04-11 16:59:34.562366: validation loss: 0.2138 +2022-04-11 16:59:34.689645: Average global foreground Dice: [0.3436, 0.7392, 0.753, 0.0] +2022-04-11 16:59:34.814550: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 16:59:35.953537: lr: 0.009766 +2022-04-11 16:59:35.988711: saving scheduled checkpoint file... +2022-04-11 16:59:36.100141: saving checkpoint... +2022-04-11 16:59:40.449815: done, saving took 4.46 seconds +2022-04-11 16:59:40.462604: done +2022-04-11 16:59:40.509611: saving checkpoint... +2022-04-11 16:59:44.895900: done, saving took 4.43 seconds +2022-04-11 16:59:44.904498: This epoch took 131.370515 s + +2022-04-11 16:59:44.905599: svmem(total=99932160000, available=79851614208, percent=20.1, used=16078573568, free=53148225536, active=39960424448, inactive=4994113536, buffers=9449472, cached=30695911424, shared=2190729216, slab=749780992) +2022-04-11 16:59:44.906320: +epoch: 13 +2022-04-11 17:01:32.895899: train loss : 0.1683 +2022-04-11 17:01:43.713175: validation loss: 0.1473 +2022-04-11 17:01:43.855180: Average global foreground Dice: [0.5711, 0.7245, 0.7792, 0.0] +2022-04-11 17:01:43.997682: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:01:45.043135: lr: 0.009748 +2022-04-11 17:01:45.185077: saving scheduled checkpoint file... +2022-04-11 17:01:45.440285: saving checkpoint... +2022-04-11 17:01:49.992150: done, saving took 4.64 seconds +2022-04-11 17:01:50.009696: done +2022-04-11 17:01:50.088047: saving checkpoint... +2022-04-11 17:01:54.495333: done, saving took 4.48 seconds +2022-04-11 17:01:54.507156: This epoch took 129.600284 s + +2022-04-11 17:01:54.508392: svmem(total=99932160000, available=79992508416, percent=20.0, used=15927296000, free=53521293312, active=39739547648, inactive=4840153088, buffers=9510912, cached=30474059776, shared=2201137152, slab=750379008) +2022-04-11 17:01:54.509224: +epoch: 14 +2022-04-11 17:03:42.135279: train loss : 0.0696 +2022-04-11 17:03:53.811982: validation loss: 0.1164 +2022-04-11 17:03:53.813984: Average global foreground Dice: [0.4726, 0.7706, 0.7887, 0.0] +2022-04-11 17:03:53.814980: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:03:54.859366: lr: 0.00973 +2022-04-11 17:03:55.000770: saving scheduled checkpoint file... +2022-04-11 17:03:55.247236: saving checkpoint... +2022-04-11 17:04:02.047794: done, saving took 6.92 seconds +2022-04-11 17:04:02.070296: done +2022-04-11 17:04:02.162562: saving checkpoint... +2022-04-11 17:04:06.621084: done, saving took 4.55 seconds +2022-04-11 17:04:06.639367: This epoch took 132.129495 s + +2022-04-11 17:04:06.640777: svmem(total=99932160000, available=79884840960, percent=20.1, used=16045510656, free=53371854848, active=39987769344, inactive=4742721536, buffers=9543680, cached=30505250816, shared=2190573568, slab=749756416) +2022-04-11 17:04:06.641752: +epoch: 15 +2022-04-11 17:05:55.548264: train loss : 0.1155 +2022-04-11 17:06:07.258955: validation loss: 0.1197 +2022-04-11 17:06:07.386089: Average global foreground Dice: [0.5137, 0.7876, 0.7882, 0.0] +2022-04-11 17:06:07.518574: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:06:08.356221: lr: 0.009712 +2022-04-11 17:06:08.357409: saving scheduled checkpoint file... +2022-04-11 17:06:08.487832: saving checkpoint... +2022-04-11 17:06:16.064886: done, saving took 7.71 seconds +2022-04-11 17:06:16.084080: done +2022-04-11 17:06:16.193575: saving checkpoint... +2022-04-11 17:06:20.536005: done, saving took 4.45 seconds +2022-04-11 17:06:20.555851: This epoch took 133.913117 s + +2022-04-11 17:06:20.556893: svmem(total=99932160000, available=79928729600, percent=20.0, used=16001540096, free=53372096512, active=40031084544, inactive=4704571392, buffers=9625600, cached=30548897792, shared=2190540800, slab=749907968) +2022-04-11 17:06:20.557589: +epoch: 16 +2022-04-11 17:08:11.121404: train loss : 0.0730 +2022-04-11 17:08:23.535895: validation loss: -0.0058 +2022-04-11 17:08:23.679373: Average global foreground Dice: [0.6032, 0.7903, 0.8718, 0.0] +2022-04-11 17:08:23.821649: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:08:24.750670: lr: 0.009693 +2022-04-11 17:08:24.893004: saving scheduled checkpoint file... +2022-04-11 17:08:25.134312: saving checkpoint... +2022-04-11 17:08:30.668805: done, saving took 5.65 seconds +2022-04-11 17:08:30.689987: done +2022-04-11 17:08:30.817981: saving checkpoint... +2022-04-11 17:08:35.215347: done, saving took 4.52 seconds +2022-04-11 17:08:35.241390: This epoch took 134.683070 s + +2022-04-11 17:08:35.242638: svmem(total=99932160000, available=79883542528, percent=20.1, used=16046637056, free=53304385536, active=40126320640, inactive=4670767104, buffers=9678848, cached=30571458560, shared=2190618624, slab=749801472) +2022-04-11 17:08:35.243460: +epoch: 17 +2022-04-11 17:10:26.750651: train loss : 0.0166 +2022-04-11 17:10:38.105085: validation loss: 0.0268 +2022-04-11 17:10:38.249484: Average global foreground Dice: [0.5516, 0.8211, 0.814, 0.0] +2022-04-11 17:10:38.391842: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:10:39.606342: lr: 0.009675 +2022-04-11 17:10:39.747667: saving scheduled checkpoint file... +2022-04-11 17:10:39.981538: saving checkpoint... +2022-04-11 17:10:44.647743: done, saving took 4.76 seconds +2022-04-11 17:10:44.663821: done +2022-04-11 17:10:44.710446: saving checkpoint... +2022-04-11 17:10:48.970498: done, saving took 4.31 seconds +2022-04-11 17:10:48.984980: This epoch took 133.740801 s + +2022-04-11 17:10:48.986310: svmem(total=99932160000, available=79872315392, percent=20.1, used=16057634816, free=53339045888, active=39872430080, inactive=4901421056, buffers=9719808, cached=30525759488, shared=2190729216, slab=750510080) +2022-04-11 17:10:48.987100: +epoch: 18 +2022-04-11 17:12:37.568877: train loss : 0.0536 +2022-04-11 17:12:49.037714: validation loss: 0.0605 +2022-04-11 17:12:49.179592: Average global foreground Dice: [0.4022, 0.7914, 0.8262, 0.0] +2022-04-11 17:12:49.304262: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:12:50.376974: lr: 0.009657 +2022-04-11 17:12:50.491098: saving scheduled checkpoint file... +2022-04-11 17:12:50.724334: saving checkpoint... +2022-04-11 17:12:55.707940: done, saving took 5.07 seconds +2022-04-11 17:12:55.723735: done +2022-04-11 17:12:55.770345: saving checkpoint... +2022-04-11 17:13:00.170143: done, saving took 4.45 seconds +2022-04-11 17:13:00.185284: This epoch took 131.197330 s + +2022-04-11 17:13:00.186476: svmem(total=99932160000, available=79995830272, percent=19.9, used=15934251008, free=53479305216, active=39846043648, inactive=4777394176, buffers=9752576, cached=30508851200, shared=2190729216, slab=750194688) +2022-04-11 17:13:00.187264: +epoch: 19 +2022-04-11 17:14:48.074859: train loss : 0.0176 +2022-04-11 17:14:59.386159: validation loss: 0.0566 +2022-04-11 17:14:59.516736: Average global foreground Dice: [0.5729, 0.8213, 0.7878, 0.0] +2022-04-11 17:14:59.640723: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:15:00.444138: lr: 0.009639 +2022-04-11 17:15:00.445119: saving scheduled checkpoint file... +2022-04-11 17:15:00.526103: saving checkpoint... +2022-04-11 17:15:07.959501: done, saving took 7.51 seconds +2022-04-11 17:15:07.974283: done +2022-04-11 17:15:08.022269: saving checkpoint... +2022-04-11 17:15:12.414402: done, saving took 4.44 seconds +2022-04-11 17:15:12.424569: This epoch took 132.236482 s + +2022-04-11 17:15:12.425778: svmem(total=99932160000, available=79948500992, percent=20.0, used=15981162496, free=53413572608, active=40046829568, inactive=4646469632, buffers=9826304, cached=30527598592, shared=2191224832, slab=748703744) +2022-04-11 17:15:12.426633: +epoch: 20 +2022-04-11 17:17:00.203952: train loss : -0.0253 +2022-04-11 17:17:10.876648: validation loss: -0.0205 +2022-04-11 17:17:10.878384: Average global foreground Dice: [0.559, 0.8187, 0.8637, 0.0] +2022-04-11 17:17:10.879371: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:17:12.031835: lr: 0.009621 +2022-04-11 17:17:12.155728: saving scheduled checkpoint file... +2022-04-11 17:17:12.383047: saving checkpoint... +2022-04-11 17:17:18.942645: done, saving took 6.64 seconds +2022-04-11 17:17:19.299971: done +2022-04-11 17:17:19.531894: saving checkpoint... +2022-04-11 17:17:25.949845: done, saving took 6.50 seconds +2022-04-11 17:17:25.966974: This epoch took 133.539698 s + +2022-04-11 17:17:25.968039: svmem(total=99932160000, available=80046346240, percent=19.9, used=15883948032, free=53500788736, active=39970701312, inactive=4633370624, buffers=9883648, cached=30537539584, shared=2190618624, slab=748797952) +2022-04-11 17:17:25.968899: +epoch: 21 +2022-04-11 17:19:14.150337: train loss : -0.0751 +2022-04-11 17:19:26.054693: validation loss: -0.0483 +2022-04-11 17:19:26.056540: Average global foreground Dice: [0.5355, 0.8407, 0.8751, 0.0] +2022-04-11 17:19:26.058020: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:19:26.751179: lr: 0.009603 +2022-04-11 17:19:26.752102: saving scheduled checkpoint file... +2022-04-11 17:19:26.859299: saving checkpoint... +2022-04-11 17:19:34.496387: done, saving took 7.74 seconds +2022-04-11 17:19:34.845655: done +2022-04-11 17:19:35.045925: saving checkpoint... +2022-04-11 17:19:42.681576: done, saving took 7.73 seconds +2022-04-11 17:19:42.933918: This epoch took 136.964347 s + +2022-04-11 17:19:42.935645: svmem(total=99932160000, available=79956418560, percent=20.0, used=15987712000, free=53388333056, active=40090107904, inactive=4617244672, buffers=9924608, cached=30546190336, shared=2176741376, slab=762195968) +2022-04-11 17:19:42.936418: +epoch: 22 +2022-04-11 17:21:31.142153: train loss : -0.0183 +2022-04-11 17:21:43.232691: validation loss: -0.0582 +2022-04-11 17:21:43.235595: Average global foreground Dice: [0.5938, 0.8468, 0.8795, 0.0] +2022-04-11 17:21:43.236730: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:21:43.881331: lr: 0.009585 +2022-04-11 17:21:43.882460: saving scheduled checkpoint file... +2022-04-11 17:21:44.025737: saving checkpoint... +2022-04-11 17:21:49.757233: done, saving took 5.87 seconds +2022-04-11 17:21:50.124083: done +2022-04-11 17:21:50.357560: saving checkpoint... +2022-04-11 17:21:56.880948: done, saving took 6.62 seconds +2022-04-11 17:21:57.220686: This epoch took 134.283559 s + +2022-04-11 17:21:57.345821: svmem(total=99932160000, available=79968542720, percent=20.0, used=15975563264, free=53382504448, active=40094187520, inactive=4616089600, buffers=10354688, cached=30563737600, shared=2176741376, slab=761405440) +2022-04-11 17:21:57.487880: +epoch: 23 +2022-04-11 17:23:45.583662: train loss : -0.0845 +2022-04-11 17:23:57.220658: validation loss: 0.0042 +2022-04-11 17:23:57.222530: Average global foreground Dice: [0.5697, 0.7844, 0.8574, 0.0] +2022-04-11 17:23:57.223525: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:23:58.048913: lr: 0.009567 +2022-04-11 17:23:58.049975: saving scheduled checkpoint file... +2022-04-11 17:23:58.130538: saving checkpoint... +2022-04-11 17:24:02.429746: done, saving took 4.38 seconds +2022-04-11 17:24:02.441990: done +2022-04-11 17:24:02.490974: saving checkpoint... +2022-04-11 17:24:08.567537: done, saving took 6.12 seconds +2022-04-11 17:24:08.890129: This epoch took 131.277332 s + +2022-04-11 17:24:09.036785: svmem(total=99932160000, available=80014585856, percent=19.9, used=15929520128, free=53431631872, active=39929016320, inactive=4726775808, buffers=10387456, cached=30560620544, shared=2176741376, slab=761380864) +2022-04-11 17:24:09.185233: +epoch: 24 +2022-04-11 17:25:56.064388: train loss : -0.0253 +2022-04-11 17:26:07.930057: validation loss: -0.0863 +2022-04-11 17:26:07.939054: Average global foreground Dice: [0.5968, 0.8391, 0.8754, 0.0] +2022-04-11 17:26:07.940025: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:26:08.741969: lr: 0.009549 +2022-04-11 17:26:08.742932: saving scheduled checkpoint file... +2022-04-11 17:26:08.835108: saving checkpoint... +2022-04-11 17:26:13.282431: done, saving took 4.54 seconds +2022-04-11 17:26:13.296289: done +2022-04-11 17:26:13.346963: saving checkpoint... +2022-04-11 17:26:17.781211: done, saving took 4.48 seconds +2022-04-11 17:26:17.796355: This epoch took 128.483035 s + +2022-04-11 17:26:17.797413: svmem(total=99932160000, available=80064364544, percent=19.9, used=15879720960, free=53461266432, active=39973408768, inactive=4677042176, buffers=10457088, cached=30580715520, shared=2176741376, slab=748228608) +2022-04-11 17:26:17.798189: +epoch: 25 +2022-04-11 17:28:05.469283: train loss : -0.0488 +2022-04-11 17:28:17.340185: validation loss: -0.1425 +2022-04-11 17:28:17.354318: Average global foreground Dice: [0.6195, 0.8537, 0.9039, 0.0] +2022-04-11 17:28:17.355084: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:28:18.207303: lr: 0.009531 +2022-04-11 17:28:18.208273: saving scheduled checkpoint file... +2022-04-11 17:28:18.296295: saving checkpoint... +2022-04-11 17:28:22.690125: done, saving took 4.48 seconds +2022-04-11 17:28:22.703032: done +2022-04-11 17:28:22.750637: saving checkpoint... +2022-04-11 17:28:27.125170: done, saving took 4.42 seconds +2022-04-11 17:28:27.139674: This epoch took 129.340670 s + +2022-04-11 17:28:27.140710: svmem(total=99932160000, available=80250793984, percent=19.7, used=15782645760, free=53652955136, active=39904473088, inactive=4549443584, buffers=10489856, cached=30486069248, shared=2087383040, slab=749895680) +2022-04-11 17:28:27.141341: +epoch: 26 +2022-04-11 17:30:15.793894: train loss : -0.0854 +2022-04-11 17:30:27.499657: validation loss: -0.1165 +2022-04-11 17:30:27.503930: Average global foreground Dice: [0.6238, 0.8494, 0.8905, 0.0] +2022-04-11 17:30:27.504991: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:30:28.313657: lr: 0.009513 +2022-04-11 17:30:28.314649: saving scheduled checkpoint file... +2022-04-11 17:30:28.393093: saving checkpoint... +2022-04-11 17:30:32.824408: done, saving took 4.51 seconds +2022-04-11 17:30:32.847316: done +2022-04-11 17:30:32.929892: saving checkpoint... +2022-04-11 17:30:37.335801: done, saving took 4.49 seconds +2022-04-11 17:30:37.353215: This epoch took 130.211130 s + +2022-04-11 17:30:37.354412: svmem(total=99932160000, available=80184475648, percent=19.8, used=15815438336, free=53570605056, active=39923884032, inactive=4611268608, buffers=10530816, cached=30535585792, shared=2120855552, slab=750723072) +2022-04-11 17:30:37.355394: +epoch: 27 +2022-04-11 17:32:24.984377: train loss : -0.0981 +2022-04-11 17:32:37.226871: validation loss: 0.0052 +2022-04-11 17:32:37.244283: Average global foreground Dice: [0.6367, 0.8143, 0.8297, 0.0] +2022-04-11 17:32:37.245360: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:32:38.029291: lr: 0.009495 +2022-04-11 17:32:38.030328: saving scheduled checkpoint file... +2022-04-11 17:32:38.124317: saving checkpoint... +2022-04-11 17:32:42.597221: done, saving took 4.57 seconds +2022-04-11 17:32:42.701996: done +2022-04-11 17:32:42.796120: saving checkpoint... +2022-04-11 17:32:47.145940: done, saving took 4.44 seconds +2022-04-11 17:32:47.165051: This epoch took 129.808845 s + +2022-04-11 17:32:47.166292: svmem(total=99932160000, available=80167751680, percent=19.8, used=15853305856, free=53536571392, active=40007401472, inactive=4563984384, buffers=10563584, cached=30531719168, shared=2099818496, slab=750927872) +2022-04-11 17:32:47.167197: +epoch: 28 +2022-04-11 17:34:34.945234: train loss : -0.1183 +2022-04-11 17:34:46.256294: validation loss: -0.0854 +2022-04-11 17:34:46.264015: Average global foreground Dice: [0.651, 0.8667, 0.8354, 0.0] +2022-04-11 17:34:46.277095: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:34:47.356097: lr: 0.009476 +2022-04-11 17:34:47.356996: saving scheduled checkpoint file... +2022-04-11 17:34:47.494672: saving checkpoint... +2022-04-11 17:34:51.980916: done, saving took 4.62 seconds +2022-04-11 17:34:52.004422: done +2022-04-11 17:34:52.111709: saving checkpoint... +2022-04-11 17:34:56.478456: done, saving took 4.47 seconds +2022-04-11 17:34:56.498975: This epoch took 129.330931 s + +2022-04-11 17:34:56.500472: svmem(total=99932160000, available=80196612096, percent=19.7, used=15831199744, free=53556273152, active=39986520064, inactive=4564660224, buffers=10596352, cached=30534090752, shared=2093010944, slab=749010944) +2022-04-11 17:34:56.501181: +epoch: 29 +2022-04-11 17:36:45.484087: train loss : -0.0812 +2022-04-11 17:36:57.778585: validation loss: -0.0810 +2022-04-11 17:36:57.786262: Average global foreground Dice: [0.6247, 0.8367, 0.8677, 0.0] +2022-04-11 17:36:57.787382: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:36:58.796384: lr: 0.009458 +2022-04-11 17:36:58.797262: saving scheduled checkpoint file... +2022-04-11 17:36:58.895005: saving checkpoint... +2022-04-11 17:37:03.172105: done, saving took 4.37 seconds +2022-04-11 17:37:03.189180: done +2022-04-11 17:37:03.236404: saving checkpoint... +2022-04-11 17:37:07.515288: done, saving took 4.33 seconds +2022-04-11 17:37:07.529709: This epoch took 131.027804 s + +2022-04-11 17:37:07.530873: svmem(total=99932160000, available=80156704768, percent=19.8, used=15836737536, free=53507248128, active=39930531840, inactive=4674371584, buffers=10657792, cached=30577516544, shared=2127265792, slab=748810240) +2022-04-11 17:37:07.531859: +epoch: 30 +2022-04-11 17:38:56.968721: train loss : -0.0249 +2022-04-11 17:39:08.438950: validation loss: -0.0605 +2022-04-11 17:39:08.443893: Average global foreground Dice: [0.6649, 0.8214, 0.8715, 0.0] +2022-04-11 17:39:08.447927: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:39:09.472925: lr: 0.00944 +2022-04-11 17:39:09.474063: saving scheduled checkpoint file... +2022-04-11 17:39:09.561104: saving checkpoint... +2022-04-11 17:39:13.861527: done, saving took 4.39 seconds +2022-04-11 17:39:13.876024: done +2022-04-11 17:39:13.922826: saving checkpoint... +2022-04-11 17:39:18.171802: done, saving took 4.29 seconds +2022-04-11 17:39:18.184045: This epoch took 130.651321 s + +2022-04-11 17:39:18.185442: svmem(total=99932160000, available=80121585664, percent=19.8, used=15822454784, free=53465366528, active=39930003456, inactive=4716900352, buffers=10706944, cached=30633631744, shared=2176741376, slab=749748224) +2022-04-11 17:39:18.186152: +epoch: 31 +2022-04-11 17:41:06.478267: train loss : -0.1562 +2022-04-11 17:41:18.897004: validation loss: -0.1713 +2022-04-11 17:41:18.902093: Average global foreground Dice: [0.6262, 0.8552, 0.9096, 0.0] +2022-04-11 17:41:18.903070: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:41:19.597995: lr: 0.009422 +2022-04-11 17:41:19.599060: saving scheduled checkpoint file... +2022-04-11 17:41:19.675421: saving checkpoint... +2022-04-11 17:41:24.036710: done, saving took 4.44 seconds +2022-04-11 17:41:24.048397: done +2022-04-11 17:41:24.095170: saving checkpoint... +2022-04-11 17:41:28.561822: done, saving took 4.51 seconds +2022-04-11 17:41:28.574304: This epoch took 130.387415 s + +2022-04-11 17:41:28.575528: svmem(total=99932160000, available=80121520128, percent=19.8, used=15847534592, free=53447680000, active=39985901568, inactive=4672688128, buffers=10747904, cached=30626197504, shared=2151661568, slab=750006272) +2022-04-11 17:41:28.576248: +epoch: 32 +2022-04-11 17:43:15.588984: train loss : -0.1150 +2022-04-11 17:43:27.276668: validation loss: -0.0811 +2022-04-11 17:43:27.278647: Average global foreground Dice: [0.6387, 0.8426, 0.8914, 0.0041] +2022-04-11 17:43:27.279752: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:43:28.158787: lr: 0.009404 +2022-04-11 17:43:28.159920: saving scheduled checkpoint file... +2022-04-11 17:43:28.254954: saving checkpoint... +2022-04-11 17:43:33.196594: done, saving took 5.04 seconds +2022-04-11 17:43:33.212607: done +2022-04-11 17:43:33.259587: saving checkpoint... +2022-04-11 17:43:37.851285: done, saving took 4.64 seconds +2022-04-11 17:43:37.860687: This epoch took 129.283698 s + +2022-04-11 17:43:37.861816: svmem(total=99932160000, available=80112807936, percent=19.8, used=15904423936, free=53429972992, active=40074362880, inactive=4603568128, buffers=10780672, cached=30586982400, shared=2103472128, slab=749314048) +2022-04-11 17:43:37.862711: +epoch: 33 +2022-04-11 17:45:26.846800: train loss : -0.1331 +2022-04-11 17:45:37.714204: validation loss: -0.1813 +2022-04-11 17:45:37.721594: Average global foreground Dice: [0.6122, 0.8861, 0.8924, 0.0747] +2022-04-11 17:45:37.729244: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:45:38.417727: lr: 0.009386 +2022-04-11 17:45:38.418621: saving scheduled checkpoint file... +2022-04-11 17:45:38.524434: saving checkpoint... +2022-04-11 17:45:42.954038: done, saving took 4.53 seconds +2022-04-11 17:45:42.971263: done +2022-04-11 17:45:43.046872: saving checkpoint... +2022-04-11 17:45:47.432857: done, saving took 4.46 seconds +2022-04-11 17:45:47.450191: This epoch took 129.586821 s + +2022-04-11 17:45:47.451417: svmem(total=99932160000, available=80254427136, percent=19.7, used=15804039168, free=53597306880, active=39943073792, inactive=4569231360, buffers=10846208, cached=30519967744, shared=2062307328, slab=748105728) +2022-04-11 17:45:47.452400: +epoch: 34 +2022-04-11 17:47:35.187714: train loss : -0.2039 +2022-04-11 17:47:45.796936: validation loss: -0.1929 +2022-04-11 17:47:45.798983: Average global foreground Dice: [0.6425, 0.8583, 0.9167, 0.0803] +2022-04-11 17:47:45.799863: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:47:46.511437: lr: 0.009368 +2022-04-11 17:47:46.512580: saving scheduled checkpoint file... +2022-04-11 17:47:46.630364: saving checkpoint... +2022-04-11 17:47:51.123539: done, saving took 4.61 seconds +2022-04-11 17:47:51.141738: done +2022-04-11 17:47:51.232926: saving checkpoint... +2022-04-11 17:47:55.623554: done, saving took 4.48 seconds +2022-04-11 17:47:55.642289: This epoch took 128.189119 s + +2022-04-11 17:47:55.643426: svmem(total=99932160000, available=80136728576, percent=19.8, used=15793242112, free=53462179840, active=39948500992, inactive=4697804800, buffers=10878976, cached=30665859072, shared=2190618624, slab=750194688) +2022-04-11 17:47:55.644427: +epoch: 35 +2022-04-11 17:49:42.181471: train loss : -0.1300 +2022-04-11 17:49:53.896109: validation loss: -0.1547 +2022-04-11 17:49:53.899812: Average global foreground Dice: [0.6528, 0.8639, 0.8978, 0.0888] +2022-04-11 17:49:53.901026: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:49:54.924771: lr: 0.00935 +2022-04-11 17:49:54.925952: saving scheduled checkpoint file... +2022-04-11 17:49:55.014790: saving checkpoint... +2022-04-11 17:49:59.338350: done, saving took 4.41 seconds +2022-04-11 17:49:59.352764: done +2022-04-11 17:49:59.399311: saving checkpoint... +2022-04-11 17:50:03.718788: done, saving took 4.37 seconds +2022-04-11 17:50:03.729506: This epoch took 128.084347 s + +2022-04-11 17:50:03.730877: svmem(total=99932160000, available=79998984192, percent=19.9, used=15934091264, free=53321658368, active=40057524224, inactive=4729991168, buffers=10919936, cached=30665490432, shared=2187370496, slab=749174784) +2022-04-11 17:50:03.731827: +epoch: 36 +2022-04-11 17:51:50.874010: train loss : -0.1478 +2022-04-11 17:52:02.101088: validation loss: -0.1084 +2022-04-11 17:52:02.103339: Average global foreground Dice: [0.6484, 0.8609, 0.8638, 0.0863] +2022-04-11 17:52:02.104435: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:52:03.025158: lr: 0.009331 +2022-04-11 17:52:03.026610: saving scheduled checkpoint file... +2022-04-11 17:52:03.103683: saving checkpoint... +2022-04-11 17:52:07.492370: done, saving took 4.46 seconds +2022-04-11 17:52:07.506637: done +2022-04-11 17:52:07.552712: saving checkpoint... +2022-04-11 17:52:11.864844: done, saving took 4.36 seconds +2022-04-11 17:52:11.878592: This epoch took 128.145841 s + +2022-04-11 17:52:11.879877: svmem(total=99932160000, available=79716806656, percent=20.2, used=16213315584, free=53022056448, active=40370335744, inactive=4714475520, buffers=10952704, cached=30685835264, shared=2190389248, slab=749203456) +2022-04-11 17:52:11.880601: +epoch: 37 +2022-04-11 17:54:00.367808: train loss : -0.2224 +2022-04-11 17:54:11.563936: validation loss: -0.2004 +2022-04-11 17:54:11.571210: Average global foreground Dice: [0.6596, 0.856, 0.8989, 0.1625] +2022-04-11 17:54:11.572264: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:54:12.467371: lr: 0.009313 +2022-04-11 17:54:12.468963: saving scheduled checkpoint file... +2022-04-11 17:54:12.561061: saving checkpoint... +2022-04-11 17:54:16.932199: done, saving took 4.46 seconds +2022-04-11 17:54:16.944921: done +2022-04-11 17:54:16.992296: saving checkpoint... +2022-04-11 17:54:21.411434: done, saving took 4.47 seconds +2022-04-11 17:54:21.425746: This epoch took 129.544464 s + +2022-04-11 17:54:21.427104: svmem(total=99932160000, available=79874519040, percent=20.1, used=16060874752, free=53179363328, active=40182800384, inactive=4743360512, buffers=10932224, cached=30680989696, shared=2185166848, slab=749993984) +2022-04-11 17:54:21.427852: +epoch: 38 +2022-04-11 17:56:10.315157: train loss : -0.2424 +2022-04-11 17:56:21.696187: validation loss: -0.1931 +2022-04-11 17:56:21.698082: Average global foreground Dice: [0.5886, 0.8692, 0.9039, 0.1155] +2022-04-11 17:56:21.704952: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:56:22.568598: lr: 0.009295 +2022-04-11 17:56:22.569916: saving scheduled checkpoint file... +2022-04-11 17:56:22.663056: saving checkpoint... +2022-04-11 17:56:27.119580: done, saving took 4.55 seconds +2022-04-11 17:56:27.137936: done +2022-04-11 17:56:27.217574: saving checkpoint... +2022-04-11 17:56:31.777857: done, saving took 4.64 seconds +2022-04-11 17:56:31.821416: This epoch took 130.392703 s + +2022-04-11 17:56:31.825818: svmem(total=99932160000, available=79831855104, percent=20.1, used=16097882112, free=53129072640, active=40261435392, inactive=4713730048, buffers=10985472, cached=30694219776, shared=2190729216, slab=749047808) +2022-04-11 17:56:31.829817: +epoch: 39 +2022-04-11 17:58:18.780224: train loss : -0.1966 +2022-04-11 17:58:30.369636: validation loss: -0.1424 +2022-04-11 17:58:30.379353: Average global foreground Dice: [0.6455, 0.8571, 0.9001, 0.1116] +2022-04-11 17:58:30.380665: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 17:58:31.250788: lr: 0.009277 +2022-04-11 17:58:31.252389: saving scheduled checkpoint file... +2022-04-11 17:58:31.381985: saving checkpoint... +2022-04-11 17:58:36.141691: done, saving took 4.89 seconds +2022-04-11 17:58:36.206371: done +2022-04-11 17:58:36.310702: saving checkpoint... +2022-04-11 17:58:40.750131: done, saving took 4.53 seconds +2022-04-11 17:58:40.772186: This epoch took 128.940755 s + +2022-04-11 17:58:40.773714: svmem(total=99932160000, available=79802822656, percent=20.1, used=16126332928, free=53082427392, active=40291536896, inactive=4727783424, buffers=11018240, cached=30712381440, shared=2191413248, slab=748519424) +2022-04-11 17:58:40.774528: +epoch: 40 +2022-04-11 18:00:28.926029: train loss : -0.2266 +2022-04-11 18:00:41.979446: validation loss: -0.2372 +2022-04-11 18:00:41.997477: Average global foreground Dice: [0.6462, 0.8763, 0.9208, 0.1622] +2022-04-11 18:00:41.998914: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:00:42.779712: lr: 0.009259 +2022-04-11 18:00:42.780979: saving scheduled checkpoint file... +2022-04-11 18:00:42.908139: saving checkpoint... +2022-04-11 18:00:47.427544: done, saving took 4.65 seconds +2022-04-11 18:00:47.447417: done +2022-04-11 18:00:47.559368: saving checkpoint... +2022-04-11 18:00:51.958736: done, saving took 4.51 seconds +2022-04-11 18:00:51.981052: This epoch took 131.205657 s + +2022-04-11 18:00:51.982415: svmem(total=99932160000, available=79847108608, percent=20.1, used=16082599936, free=53109276672, active=40255238144, inactive=4743913472, buffers=11059200, cached=30729224192, shared=2190729216, slab=749203456) +2022-04-11 18:00:51.983417: +epoch: 41 +2022-04-11 18:02:40.610247: train loss : -0.2471 +2022-04-11 18:02:52.474400: validation loss: -0.2569 +2022-04-11 18:02:52.476890: Average global foreground Dice: [0.652, 0.8848, 0.9267, 0.1774] +2022-04-11 18:02:52.477800: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:02:53.412390: lr: 0.009241 +2022-04-11 18:02:53.413574: saving scheduled checkpoint file... +2022-04-11 18:02:53.513599: saving checkpoint... +2022-04-11 18:02:57.802774: done, saving took 4.39 seconds +2022-04-11 18:02:57.816841: done +2022-04-11 18:02:57.868782: saving checkpoint... +2022-04-11 18:03:02.213350: done, saving took 4.40 seconds +2022-04-11 18:03:02.243841: This epoch took 130.259707 s + +2022-04-11 18:03:02.245152: svmem(total=99932160000, available=79960014848, percent=20.0, used=15969529856, free=53204684800, active=40143077376, inactive=4761051136, buffers=11091968, cached=30746853376, shared=2190807040, slab=749420544) +2022-04-11 18:03:02.246212: +epoch: 42 +2022-04-11 18:04:50.217796: train loss : -0.2743 +2022-04-11 18:05:02.327107: validation loss: -0.3133 +2022-04-11 18:05:02.329196: Average global foreground Dice: [0.6923, 0.8878, 0.9382, 0.2045] +2022-04-11 18:05:02.330083: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:05:03.222821: lr: 0.009223 +2022-04-11 18:05:03.224155: saving scheduled checkpoint file... +2022-04-11 18:05:03.305999: saving checkpoint... +2022-04-11 18:05:07.612870: done, saving took 4.39 seconds +2022-04-11 18:05:07.626517: done +2022-04-11 18:05:07.673589: saving checkpoint... +2022-04-11 18:05:12.010874: done, saving took 4.38 seconds +2022-04-11 18:05:12.024194: This epoch took 129.777169 s + +2022-04-11 18:05:12.025417: svmem(total=99932160000, available=79998513152, percent=19.9, used=15930933248, free=53235453952, active=40105148416, inactive=4768591872, buffers=11149312, cached=30754623488, shared=2190729216, slab=748523520) +2022-04-11 18:05:12.026548: +epoch: 43 +2022-04-11 18:07:00.292531: train loss : -0.2798 +2022-04-11 18:07:11.967903: validation loss: -0.3081 +2022-04-11 18:07:11.985798: Average global foreground Dice: [0.6298, 0.91, 0.9312, 0.1966] +2022-04-11 18:07:11.988901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:07:12.736760: lr: 0.009204 +2022-04-11 18:07:12.737885: saving scheduled checkpoint file... +2022-04-11 18:07:12.814425: saving checkpoint... +2022-04-11 18:07:17.290234: done, saving took 4.55 seconds +2022-04-11 18:07:17.305427: done +2022-04-11 18:07:17.355064: saving checkpoint... +2022-04-11 18:07:21.878549: done, saving took 4.57 seconds +2022-04-11 18:07:21.891829: This epoch took 129.864220 s + +2022-04-11 18:07:21.893109: svmem(total=99932160000, available=80058363904, percent=19.9, used=15871229952, free=53277663232, active=40043102208, inactive=4786212864, buffers=11206656, cached=30772060160, shared=2190807040, slab=749527040) +2022-04-11 18:07:21.893856: +epoch: 44 +2022-04-11 18:09:08.404257: train loss : -0.2894 +2022-04-11 18:09:19.529491: validation loss: -0.3076 +2022-04-11 18:09:19.535299: Average global foreground Dice: [0.707, 0.9046, 0.9422, 0.1959] +2022-04-11 18:09:19.536465: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:09:20.368123: lr: 0.009186 +2022-04-11 18:09:20.369346: saving scheduled checkpoint file... +2022-04-11 18:09:20.460459: saving checkpoint... +2022-04-11 18:09:24.991950: done, saving took 4.62 seconds +2022-04-11 18:09:25.014746: done +2022-04-11 18:09:25.061482: saving checkpoint... +2022-04-11 18:09:29.577350: done, saving took 4.56 seconds +2022-04-11 18:09:29.617404: This epoch took 127.722498 s + +2022-04-11 18:09:29.619040: svmem(total=99932160000, available=79999811584, percent=19.9, used=15929442304, free=53201850368, active=40103038976, inactive=4803518464, buffers=11239424, cached=30789627904, shared=2190917632, slab=749641728) +2022-04-11 18:09:29.619793: +epoch: 45 +2022-04-11 18:11:18.033771: train loss : -0.3334 +2022-04-11 18:11:30.120179: validation loss: -0.2743 +2022-04-11 18:11:30.126402: Average global foreground Dice: [0.6416, 0.8934, 0.9145, 0.2048] +2022-04-11 18:11:30.127946: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:11:31.051908: lr: 0.009168 +2022-04-11 18:11:31.053169: saving scheduled checkpoint file... +2022-04-11 18:11:31.159516: saving checkpoint... +2022-04-11 18:11:36.010849: done, saving took 4.96 seconds +2022-04-11 18:11:36.047310: done +2022-04-11 18:11:36.125302: saving checkpoint... +2022-04-11 18:11:40.612027: done, saving took 4.56 seconds +2022-04-11 18:11:40.648087: This epoch took 131.027546 s + +2022-04-11 18:11:40.649440: svmem(total=99932160000, available=79891197952, percent=20.1, used=16038498304, free=53085319168, active=40214286336, inactive=4810903552, buffers=11280384, cached=30797062144, shared=2190508032, slab=748642304) +2022-04-11 18:11:40.650341: +epoch: 46 +2022-04-11 18:13:29.329735: train loss : -0.2921 +2022-04-11 18:13:41.136032: validation loss: -0.2721 +2022-04-11 18:13:41.141715: Average global foreground Dice: [0.6482, 0.878, 0.9179, 0.2396] +2022-04-11 18:13:41.142783: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:13:41.954724: lr: 0.00915 +2022-04-11 18:13:41.956707: saving scheduled checkpoint file... +2022-04-11 18:13:42.066681: saving checkpoint... +2022-04-11 18:13:46.578130: done, saving took 4.62 seconds +2022-04-11 18:13:46.596924: done +2022-04-11 18:13:46.690796: saving checkpoint... +2022-04-11 18:13:51.216652: done, saving took 4.62 seconds +2022-04-11 18:13:51.231924: This epoch took 130.580829 s + +2022-04-11 18:13:51.233376: svmem(total=99932160000, available=79899873280, percent=20.0, used=16029646848, free=53076484096, active=40203948032, inactive=4828585984, buffers=11313152, cached=30814715904, shared=2190729216, slab=748638208) +2022-04-11 18:13:51.234137: +epoch: 47 +2022-04-11 18:15:38.929541: train loss : -0.3196 +2022-04-11 18:15:50.858725: validation loss: -0.2542 +2022-04-11 18:15:50.867640: Average global foreground Dice: [0.7189, 0.8781, 0.9181, 0.2254] +2022-04-11 18:15:50.868906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:15:51.941463: lr: 0.009132 +2022-04-11 18:15:51.942729: saving scheduled checkpoint file... +2022-04-11 18:15:52.049011: saving checkpoint... +2022-04-11 18:15:56.481559: done, saving took 4.54 seconds +2022-04-11 18:15:56.490836: done +2022-04-11 18:15:56.541285: saving checkpoint... +2022-04-11 18:16:01.095071: done, saving took 4.60 seconds +2022-04-11 18:16:01.138860: This epoch took 129.903735 s + +2022-04-11 18:16:01.140392: svmem(total=99932160000, available=79838257152, percent=20.1, used=16091000832, free=52997435392, active=40264306688, inactive=4845674496, buffers=11374592, cached=30832349184, shared=2190917632, slab=748240896) +2022-04-11 18:16:01.141210: +epoch: 48 +2022-04-11 18:17:49.161601: train loss : -0.2772 +2022-04-11 18:18:01.480206: validation loss: -0.2007 +2022-04-11 18:18:01.502256: Average global foreground Dice: [0.6854, 0.8816, 0.8831, 0.1994] +2022-04-11 18:18:01.504744: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:18:02.271499: lr: 0.009114 +2022-04-11 18:18:02.272797: saving scheduled checkpoint file... +2022-04-11 18:18:02.349257: saving checkpoint... +2022-04-11 18:18:06.768642: done, saving took 4.50 seconds +2022-04-11 18:18:06.801664: done +2022-04-11 18:18:06.849250: saving checkpoint... +2022-04-11 18:18:11.258104: done, saving took 4.46 seconds +2022-04-11 18:18:11.271933: This epoch took 130.129531 s + +2022-04-11 18:18:11.272980: svmem(total=99932160000, available=79687278592, percent=20.3, used=16241721344, free=52838457344, active=40412504064, inactive=4853452800, buffers=11423744, cached=30840557568, shared=2191224832, slab=749953024) +2022-04-11 18:18:11.273716: +epoch: 49 +2022-04-11 18:20:00.462749: train loss : -0.2621 +2022-04-11 18:20:12.183484: validation loss: -0.3268 +2022-04-11 18:20:12.200800: Average global foreground Dice: [0.6929, 0.9069, 0.9333, 0.2234] +2022-04-11 18:20:12.216692: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:20:13.020898: lr: 0.009095 +2022-04-11 18:20:13.022086: saving scheduled checkpoint file... +2022-04-11 18:20:13.110605: saving checkpoint... +2022-04-11 18:20:17.613127: done, saving took 4.59 seconds +2022-04-11 18:20:17.628222: done +2022-04-11 18:20:17.675289: saving checkpoint... +2022-04-11 18:20:22.200685: done, saving took 4.57 seconds +2022-04-11 18:20:22.216693: This epoch took 130.942171 s + +2022-04-11 18:20:22.218319: svmem(total=99932160000, available=79882919936, percent=20.1, used=16059076608, free=53016522752, active=40229244928, inactive=4857274368, buffers=11481088, cached=30845079552, shared=2178150400, slab=749522944) +2022-04-11 18:20:22.219205: +epoch: 50 +2022-04-11 18:22:09.087894: train loss : -0.2573 +2022-04-11 18:22:19.043933: validation loss: -0.2996 +2022-04-11 18:22:19.046983: Average global foreground Dice: [0.6997, 0.8849, 0.923, 0.2316] +2022-04-11 18:22:19.048253: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:22:19.992514: lr: 0.009077 +2022-04-11 18:22:19.994200: saving scheduled checkpoint file... +2022-04-11 18:22:20.089626: saving checkpoint... +2022-04-11 18:22:24.690033: done, saving took 4.69 seconds +2022-04-11 18:22:24.702682: done +2022-04-11 18:22:24.754424: saving checkpoint... +2022-04-11 18:22:29.317696: done, saving took 4.61 seconds +2022-04-11 18:22:29.329918: This epoch took 127.109942 s + +2022-04-11 18:22:29.331295: svmem(total=99932160000, available=79751589888, percent=20.2, used=16177811456, free=52868825088, active=40348946432, inactive=4885753856, buffers=11522048, cached=30874001408, shared=2190729216, slab=751034368) +2022-04-11 18:22:29.332280: +epoch: 51 +2022-04-11 18:24:17.828316: train loss : -0.2880 +2022-04-11 18:24:29.550500: validation loss: -0.3107 +2022-04-11 18:24:29.552274: Average global foreground Dice: [0.6514, 0.9049, 0.9276, 0.2224] +2022-04-11 18:24:29.553266: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:24:30.593558: lr: 0.009059 +2022-04-11 18:24:30.594757: saving scheduled checkpoint file... +2022-04-11 18:24:30.699214: saving checkpoint... +2022-04-11 18:24:36.139499: done, saving took 5.54 seconds +2022-04-11 18:24:36.153137: done +2022-04-11 18:24:36.238715: saving checkpoint... +2022-04-11 18:24:40.757931: done, saving took 4.60 seconds +2022-04-11 18:24:40.775576: This epoch took 131.442413 s + +2022-04-11 18:24:40.776865: svmem(total=99932160000, available=79928934400, percent=20.0, used=16001617920, free=53027717120, active=40176930816, inactive=4903055360, buffers=11554816, cached=30891270144, shared=2189549568, slab=751419392) +2022-04-11 18:24:40.777730: +epoch: 52 +2022-04-11 18:26:28.588221: train loss : -0.3045 +2022-04-11 18:26:39.930453: validation loss: -0.3015 +2022-04-11 18:26:39.932774: Average global foreground Dice: [0.6609, 0.8905, 0.9309, 0.2311] +2022-04-11 18:26:39.933923: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:26:40.793746: lr: 0.009041 +2022-04-11 18:26:40.795345: saving scheduled checkpoint file... +2022-04-11 18:26:40.893187: saving checkpoint... +2022-04-11 18:26:45.557642: done, saving took 4.76 seconds +2022-04-11 18:26:46.018341: done +2022-04-11 18:26:46.065079: saving checkpoint... +2022-04-11 18:26:50.620729: done, saving took 4.60 seconds +2022-04-11 18:26:50.637231: This epoch took 129.858834 s + +2022-04-11 18:26:50.638591: svmem(total=99932160000, available=79890599936, percent=20.1, used=16039161856, free=52981735424, active=40201601024, inactive=4911673344, buffers=11603968, cached=30899658752, shared=2190540800, slab=751603712) +2022-04-11 18:26:50.639516: +epoch: 53 +2022-04-11 18:28:39.283472: train loss : -0.3359 +2022-04-11 18:28:50.311458: validation loss: -0.3549 +2022-04-11 18:28:50.326032: Average global foreground Dice: [0.7258, 0.9122, 0.9392, 0.2527] +2022-04-11 18:28:50.327275: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:28:51.433218: lr: 0.009023 +2022-04-11 18:28:51.434426: saving scheduled checkpoint file... +2022-04-11 18:28:51.546445: saving checkpoint... +2022-04-11 18:28:56.024525: done, saving took 4.59 seconds +2022-04-11 18:28:56.068498: done +2022-04-11 18:28:56.115406: saving checkpoint... +2022-04-11 18:29:00.743500: done, saving took 4.67 seconds +2022-04-11 18:29:00.758475: This epoch took 130.118295 s + +2022-04-11 18:29:00.760850: svmem(total=99932160000, available=79856898048, percent=20.1, used=16062214144, free=52933582848, active=40206303232, inactive=4963119104, buffers=11636736, cached=30924726272, shared=2200977408, slab=751820800) +2022-04-11 18:29:00.761757: +epoch: 54 +2022-04-11 18:30:49.981861: train loss : -0.3162 +2022-04-11 18:31:02.068327: validation loss: -0.2645 +2022-04-11 18:31:02.078906: Average global foreground Dice: [0.7033, 0.8836, 0.9144, 0.2336] +2022-04-11 18:31:02.080133: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:31:02.736315: lr: 0.009004 +2022-04-11 18:31:02.737853: saving scheduled checkpoint file... +2022-04-11 18:31:02.826494: saving checkpoint... +2022-04-11 18:31:07.431070: done, saving took 4.69 seconds +2022-04-11 18:31:07.445638: done +2022-04-11 18:31:07.493996: saving checkpoint... +2022-04-11 18:31:12.731832: done, saving took 5.29 seconds +2022-04-11 18:31:12.741673: This epoch took 131.978979 s + +2022-04-11 18:31:12.743318: svmem(total=99932160000, available=79828979712, percent=20.1, used=16100413440, free=52887662592, active=40259379200, inactive=4952776704, buffers=11694080, cached=30932389888, shared=2190540800, slab=751419392) +2022-04-11 18:31:12.744461: +epoch: 55 +2022-04-11 18:33:01.029867: train loss : -0.2969 +2022-04-11 18:33:12.820279: validation loss: -0.3049 +2022-04-11 18:33:12.822241: Average global foreground Dice: [0.6846, 0.9013, 0.9028, 0.2134] +2022-04-11 18:33:12.823099: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:33:13.703179: lr: 0.008986 +2022-04-11 18:33:13.704431: saving scheduled checkpoint file... +2022-04-11 18:33:13.791871: saving checkpoint... +2022-04-11 18:33:18.977715: done, saving took 5.27 seconds +2022-04-11 18:33:18.997957: done +2022-04-11 18:33:19.052258: saving checkpoint... +2022-04-11 18:33:24.501472: done, saving took 5.49 seconds +2022-04-11 18:33:24.510255: This epoch took 131.764664 s + +2022-04-11 18:33:24.511401: svmem(total=99932160000, available=79669469184, percent=20.3, used=16259911680, free=52720386048, active=40435167232, inactive=4951736320, buffers=11726848, cached=30940135424, shared=2190618624, slab=750514176) +2022-04-11 18:33:24.512099: +epoch: 56 +2022-04-11 18:35:12.357006: train loss : -0.3277 +2022-04-11 18:35:23.781758: validation loss: -0.2709 +2022-04-11 18:35:23.789647: Average global foreground Dice: [0.7017, 0.8636, 0.9115, 0.2595] +2022-04-11 18:35:23.790977: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:35:24.835479: lr: 0.008968 +2022-04-11 18:35:24.836912: saving scheduled checkpoint file... +2022-04-11 18:35:24.933187: saving checkpoint... +2022-04-11 18:35:29.376529: done, saving took 4.54 seconds +2022-04-11 18:35:29.394568: done +2022-04-11 18:35:29.441164: saving checkpoint... +2022-04-11 18:35:33.893470: done, saving took 4.50 seconds +2022-04-11 18:35:33.908277: This epoch took 129.395142 s + +2022-04-11 18:35:33.909456: svmem(total=99932160000, available=79689658368, percent=20.3, used=16253636608, free=52723081216, active=40427851776, inactive=4955533312, buffers=11784192, cached=30943657984, shared=2176606208, slab=750608384) +2022-04-11 18:35:33.910080: +epoch: 57 +2022-04-11 18:37:20.846544: train loss : -0.3363 +2022-04-11 18:37:32.947740: validation loss: -0.3798 +2022-04-11 18:37:32.949564: Average global foreground Dice: [0.7668, 0.8957, 0.9379, 0.2506] +2022-04-11 18:37:32.950574: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:37:33.712429: lr: 0.00895 +2022-04-11 18:37:33.713809: saving scheduled checkpoint file... +2022-04-11 18:37:33.802059: saving checkpoint... +2022-04-11 18:37:38.625063: done, saving took 4.91 seconds +2022-04-11 18:37:38.672687: done +2022-04-11 18:37:38.751064: saving checkpoint... +2022-04-11 18:37:43.286377: done, saving took 4.61 seconds +2022-04-11 18:37:43.302986: This epoch took 129.392255 s + +2022-04-11 18:37:43.304449: svmem(total=99932160000, available=79704244224, percent=20.2, used=16224940032, free=52720152576, active=40397361152, inactive=4986564608, buffers=11927552, cached=30975139840, shared=2190729216, slab=750260224) +2022-04-11 18:37:43.308390: +epoch: 58 +2022-04-11 18:39:31.453988: train loss : -0.3465 +2022-04-11 18:39:43.690816: validation loss: -0.1831 +2022-04-11 18:39:43.698542: Average global foreground Dice: [0.6676, 0.8648, 0.8693, 0.2232] +2022-04-11 18:39:43.699481: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:39:44.249891: lr: 0.008931 +2022-04-11 18:39:44.251048: saving scheduled checkpoint file... +2022-04-11 18:39:44.343344: saving checkpoint... +2022-04-11 18:39:48.738937: done, saving took 4.49 seconds +2022-04-11 18:39:48.759314: done +2022-04-11 18:39:48.760542: This epoch took 125.451239 s + +2022-04-11 18:39:48.761700: svmem(total=99932160000, available=79839764480, percent=20.1, used=16089575424, free=52848758784, active=40266752000, inactive=4993052672, buffers=11960320, cached=30981865472, shared=2190508032, slab=750366720) +2022-04-11 18:39:48.762445: +epoch: 59 +2022-04-11 18:41:36.039095: train loss : -0.2509 +2022-04-11 18:41:48.025295: validation loss: -0.2919 +2022-04-11 18:41:48.027207: Average global foreground Dice: [0.7037, 0.8926, 0.9192, 0.2172] +2022-04-11 18:41:48.028253: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:41:49.004244: lr: 0.008913 +2022-04-11 18:41:49.005326: saving scheduled checkpoint file... +2022-04-11 18:41:49.100497: saving checkpoint... +2022-04-11 18:41:53.416007: done, saving took 4.41 seconds +2022-04-11 18:41:53.439719: done +2022-04-11 18:41:53.493681: saving checkpoint... +2022-04-11 18:41:57.828144: done, saving took 4.39 seconds +2022-04-11 18:41:57.865937: This epoch took 129.102794 s + +2022-04-11 18:41:57.867299: svmem(total=99932160000, available=79675420672, percent=20.3, used=16253861888, free=52667092992, active=40426512384, inactive=5010419712, buffers=12001280, cached=30999203840, shared=2190540800, slab=750235648) +2022-04-11 18:41:57.867997: +epoch: 60 +2022-04-11 18:43:44.508491: train loss : -0.3445 +2022-04-11 18:43:56.766967: validation loss: -0.3684 +2022-04-11 18:43:56.769137: Average global foreground Dice: [0.6767, 0.9077, 0.9449, 0.2724] +2022-04-11 18:43:56.770443: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:43:57.620376: lr: 0.008895 +2022-04-11 18:43:57.621810: saving scheduled checkpoint file... +2022-04-11 18:43:57.672901: saving checkpoint... +2022-04-11 18:44:02.041486: done, saving took 4.42 seconds +2022-04-11 18:44:02.056488: done +2022-04-11 18:44:02.102826: saving checkpoint... +2022-04-11 18:44:06.393725: done, saving took 4.34 seconds +2022-04-11 18:44:06.416152: This epoch took 128.547467 s + +2022-04-11 18:44:06.417955: svmem(total=99932160000, available=79842263040, percent=20.1, used=16076660736, free=52816191488, active=40246398976, inactive=5038489600, buffers=12034048, cached=31027273728, shared=2200948736, slab=750649344) +2022-04-11 18:44:06.418802: +epoch: 61 +2022-04-11 18:45:55.245812: train loss : -0.3608 +2022-04-11 18:46:07.339430: validation loss: -0.3673 +2022-04-11 18:46:07.353806: Average global foreground Dice: [0.713, 0.8996, 0.9411, 0.2703] +2022-04-11 18:46:07.355172: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:46:08.023125: lr: 0.008877 +2022-04-11 18:46:08.024216: saving scheduled checkpoint file... +2022-04-11 18:46:08.100655: saving checkpoint... +2022-04-11 18:46:12.661531: done, saving took 4.64 seconds +2022-04-11 18:46:12.707824: done +2022-04-11 18:46:12.754632: saving checkpoint... +2022-04-11 18:46:17.288408: done, saving took 4.58 seconds +2022-04-11 18:46:17.302933: This epoch took 130.883468 s + +2022-04-11 18:46:17.304345: svmem(total=99932160000, available=79728783360, percent=20.2, used=16199823360, free=52706447360, active=40332054528, inactive=5068689408, buffers=12091392, cached=31013797888, shared=2191192064, slab=751235072) +2022-04-11 18:46:17.305130: +epoch: 62 +2022-04-11 18:48:06.186156: train loss : -0.3073 +2022-04-11 18:48:18.241692: validation loss: -0.3519 +2022-04-11 18:48:18.258788: Average global foreground Dice: [0.6872, 0.9074, 0.9366, 0.2481] +2022-04-11 18:48:18.260775: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:48:19.123618: lr: 0.008859 +2022-04-11 18:48:19.124872: saving scheduled checkpoint file... +2022-04-11 18:48:19.227350: saving checkpoint... +2022-04-11 18:48:23.715449: done, saving took 4.59 seconds +2022-04-11 18:48:23.752705: done +2022-04-11 18:48:23.831868: saving checkpoint... +2022-04-11 18:48:28.240651: done, saving took 4.49 seconds +2022-04-11 18:48:28.258410: This epoch took 130.952303 s + +2022-04-11 18:48:28.259748: svmem(total=99932160000, available=79797866496, percent=20.1, used=16121507840, free=52775608320, active=40206192640, inactive=5124734976, buffers=12124160, cached=31022919680, shared=2200301568, slab=749809664) +2022-04-11 18:48:28.260546: +epoch: 63 +2022-04-11 18:50:14.725969: train loss : -0.3310 +2022-04-11 18:50:26.573640: validation loss: -0.3598 +2022-04-11 18:50:26.581588: Average global foreground Dice: [0.6782, 0.917, 0.9367, 0.2581] +2022-04-11 18:50:26.582515: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:50:27.272270: lr: 0.00884 +2022-04-11 18:50:27.273177: saving scheduled checkpoint file... +2022-04-11 18:50:27.378560: saving checkpoint... +2022-04-11 18:50:31.809543: done, saving took 4.54 seconds +2022-04-11 18:50:31.829110: done +2022-04-11 18:50:31.922627: saving checkpoint... +2022-04-11 18:50:36.424579: done, saving took 4.59 seconds +2022-04-11 18:50:36.443985: This epoch took 128.182570 s + +2022-04-11 18:50:36.445191: svmem(total=99932160000, available=79849414656, percent=20.1, used=16079499264, free=52794949632, active=40245878784, inactive=5065129984, buffers=12165120, cached=31045545984, shared=2190729216, slab=750493696) +2022-04-11 18:50:36.446449: +epoch: 64 +2022-04-11 18:52:26.544470: train loss : -0.3448 +2022-04-11 18:52:36.946631: validation loss: -0.4032 +2022-04-11 18:52:36.949323: Average global foreground Dice: [0.7194, 0.9216, 0.9465, 0.2919] +2022-04-11 18:52:36.950454: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:52:37.770449: lr: 0.008822 +2022-04-11 18:52:37.771758: saving scheduled checkpoint file... +2022-04-11 18:52:37.878870: saving checkpoint... +2022-04-11 18:52:42.555212: done, saving took 4.78 seconds +2022-04-11 18:52:42.598064: done +2022-04-11 18:52:42.708325: saving checkpoint... +2022-04-11 18:52:47.195959: done, saving took 4.60 seconds +2022-04-11 18:52:47.214044: This epoch took 130.766780 s + +2022-04-11 18:52:47.215568: svmem(total=99932160000, available=79882067968, percent=20.1, used=16047415296, free=52807589888, active=40214933504, inactive=5080432640, buffers=12197888, cached=31064956928, shared=2190278656, slab=750538752) +2022-04-11 18:52:47.216522: +epoch: 65 +2022-04-11 18:54:35.447657: train loss : -0.3733 +2022-04-11 18:54:47.132319: validation loss: -0.3569 +2022-04-11 18:54:47.142302: Average global foreground Dice: [0.7083, 0.8877, 0.9406, 0.2793] +2022-04-11 18:54:47.143321: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:54:48.157160: lr: 0.008804 +2022-04-11 18:54:48.158366: saving scheduled checkpoint file... +2022-04-11 18:54:48.261988: saving checkpoint... +2022-04-11 18:54:52.728981: done, saving took 4.57 seconds +2022-04-11 18:54:52.743543: done +2022-04-11 18:54:52.790498: saving checkpoint... +2022-04-11 18:54:57.693454: done, saving took 4.95 seconds +2022-04-11 18:54:57.707921: This epoch took 130.490453 s + +2022-04-11 18:54:57.709388: svmem(total=99932160000, available=79766511616, percent=20.2, used=16162455552, free=52684296192, active=40335228928, inactive=5088567296, buffers=12230656, cached=31073177600, shared=2190729216, slab=751677440) +2022-04-11 18:54:57.710178: +epoch: 66 +2022-04-11 18:56:46.869076: train loss : -0.3483 +2022-04-11 18:56:57.842350: validation loss: -0.3477 +2022-04-11 18:56:57.848307: Average global foreground Dice: [0.6916, 0.8933, 0.943, 0.2599] +2022-04-11 18:56:57.849331: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:57:00.232454: lr: 0.008785 +2022-04-11 18:57:00.233800: saving scheduled checkpoint file... +2022-04-11 18:57:00.311422: saving checkpoint... +2022-04-11 18:57:04.605660: done, saving took 4.37 seconds +2022-04-11 18:57:04.619759: done +2022-04-11 18:57:04.666849: saving checkpoint... +2022-04-11 18:57:08.998761: done, saving took 4.38 seconds +2022-04-11 18:57:09.011714: This epoch took 131.300400 s + +2022-04-11 18:57:09.012858: svmem(total=99932160000, available=79702872064, percent=20.2, used=16226619392, free=52603305984, active=40387805184, inactive=5104828416, buffers=12275712, cached=31089958912, shared=2190413824, slab=750657536) +2022-04-11 18:57:09.013632: +epoch: 67 +2022-04-11 18:58:57.515632: train loss : -0.3959 +2022-04-11 18:59:08.368708: validation loss: -0.3602 +2022-04-11 18:59:08.371648: Average global foreground Dice: [0.6874, 0.9093, 0.9417, 0.2719] +2022-04-11 18:59:08.372674: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 18:59:09.420907: lr: 0.008767 +2022-04-11 18:59:09.422143: saving scheduled checkpoint file... +2022-04-11 18:59:09.513619: saving checkpoint... +2022-04-11 18:59:14.221631: done, saving took 4.80 seconds +2022-04-11 18:59:14.235804: done +2022-04-11 18:59:14.282759: saving checkpoint... +2022-04-11 18:59:18.849514: done, saving took 4.61 seconds +2022-04-11 18:59:18.874124: This epoch took 129.859389 s + +2022-04-11 18:59:18.875499: svmem(total=99932160000, available=79897321472, percent=20.0, used=16044593152, free=52780232704, active=40220934144, inactive=5105483776, buffers=12324864, cached=31095009280, shared=2177843200, slab=750526464) +2022-04-11 18:59:18.876263: +epoch: 68 +2022-04-11 19:01:06.075528: train loss : -0.4116 +2022-04-11 19:01:18.249496: validation loss: -0.4321 +2022-04-11 19:01:18.259380: Average global foreground Dice: [0.7542, 0.9086, 0.9596, 0.298] +2022-04-11 19:01:18.260560: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:01:19.246301: lr: 0.008749 +2022-04-11 19:01:19.247619: saving scheduled checkpoint file... +2022-04-11 19:01:19.325382: saving checkpoint... +2022-04-11 19:01:23.699148: done, saving took 4.45 seconds +2022-04-11 19:01:23.716146: done +2022-04-11 19:01:23.796409: saving checkpoint... +2022-04-11 19:01:28.953350: done, saving took 5.24 seconds +2022-04-11 19:01:28.973464: This epoch took 130.096078 s + +2022-04-11 19:01:28.975249: svmem(total=99932160000, available=79753191424, percent=20.2, used=16175374336, free=52628242432, active=40347279360, inactive=5126938624, buffers=12365824, cached=31116177408, shared=2191114240, slab=751435776) +2022-04-11 19:01:28.976106: +epoch: 69 +2022-04-11 19:03:16.362791: train loss : -0.3749 +2022-04-11 19:03:28.637330: validation loss: -0.3055 +2022-04-11 19:03:28.648227: Average global foreground Dice: [0.7622, 0.8765, 0.8994, 0.2684] +2022-04-11 19:03:28.649256: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:03:29.519485: lr: 0.008731 +2022-04-11 19:03:29.521194: saving scheduled checkpoint file... +2022-04-11 19:03:29.644855: saving checkpoint... +2022-04-11 19:03:34.152767: done, saving took 4.63 seconds +2022-04-11 19:03:34.167604: done +2022-04-11 19:03:34.253452: saving checkpoint... +2022-04-11 19:03:38.736091: done, saving took 4.57 seconds +2022-04-11 19:03:38.753500: This epoch took 129.776583 s + +2022-04-11 19:03:38.754980: svmem(total=99932160000, available=79996698624, percent=19.9, used=15932280832, free=52854312960, active=40109731840, inactive=5143982080, buffers=12398592, cached=31133167616, shared=2190729216, slab=751452160) +2022-04-11 19:03:38.755773: +epoch: 70 +2022-04-11 19:05:26.563096: train loss : -0.3669 +2022-04-11 19:05:37.630366: validation loss: -0.3610 +2022-04-11 19:05:37.642543: Average global foreground Dice: [0.7311, 0.9127, 0.9354, 0.2775] +2022-04-11 19:05:37.644116: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:05:38.598649: lr: 0.008712 +2022-04-11 19:05:38.600179: saving scheduled checkpoint file... +2022-04-11 19:05:38.747385: saving checkpoint... +2022-04-11 19:05:43.444025: done, saving took 4.84 seconds +2022-04-11 19:05:43.498688: done +2022-04-11 19:05:43.610085: saving checkpoint... +2022-04-11 19:05:48.137256: done, saving took 4.64 seconds +2022-04-11 19:05:48.157374: This epoch took 129.400798 s + +2022-04-11 19:05:48.159145: svmem(total=99932160000, available=79818727424, percent=20.1, used=16110239744, free=52658823168, active=40281792512, inactive=5161418752, buffers=12447744, cached=31150649344, shared=2190729216, slab=751833088) +2022-04-11 19:05:48.160228: +epoch: 71 +2022-04-11 19:07:38.341651: train loss : -0.3879 +2022-04-11 19:07:50.076204: validation loss: -0.2858 +2022-04-11 19:07:50.087454: Average global foreground Dice: [0.6766, 0.8765, 0.9208, 0.2793] +2022-04-11 19:07:50.088646: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:07:50.785676: lr: 0.008694 +2022-04-11 19:07:50.787000: saving scheduled checkpoint file... +2022-04-11 19:07:50.874712: saving checkpoint... +2022-04-11 19:07:55.164136: done, saving took 4.38 seconds +2022-04-11 19:07:55.173231: done +2022-04-11 19:07:55.174506: This epoch took 127.013355 s + +2022-04-11 19:07:55.175445: svmem(total=99932160000, available=79779704832, percent=20.2, used=16149123072, free=52615266304, active=40283955200, inactive=5211807744, buffers=12492800, cached=31155277824, shared=2190729216, slab=750477312) +2022-04-11 19:07:55.176593: +epoch: 72 +2022-04-11 19:09:43.511403: train loss : -0.3931 +2022-04-11 19:09:55.440154: validation loss: -0.4142 +2022-04-11 19:09:55.447896: Average global foreground Dice: [0.7467, 0.9061, 0.9537, 0.3064] +2022-04-11 19:09:55.449448: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:09:56.376261: lr: 0.008676 +2022-04-11 19:09:56.377632: saving scheduled checkpoint file... +2022-04-11 19:09:56.476723: saving checkpoint... +2022-04-11 19:10:00.869930: done, saving took 4.49 seconds +2022-04-11 19:10:00.884620: done +2022-04-11 19:10:00.932486: saving checkpoint... +2022-04-11 19:10:05.294529: done, saving took 4.41 seconds +2022-04-11 19:10:05.308827: This epoch took 130.131480 s + +2022-04-11 19:10:05.310227: svmem(total=99932160000, available=79835578368, percent=20.1, used=16082767872, free=52705775616, active=40099454976, inactive=5299789824, buffers=12533760, cached=31131082752, shared=2201325568, slab=750813184) +2022-04-11 19:10:05.311243: +epoch: 73 +2022-04-11 19:11:53.085191: train loss : -0.4352 +2022-04-11 19:12:05.293985: validation loss: -0.3920 +2022-04-11 19:12:05.296457: Average global foreground Dice: [0.7371, 0.9246, 0.9488, 0.2666] +2022-04-11 19:12:05.297647: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:12:06.357589: lr: 0.008658 +2022-04-11 19:12:06.359140: saving scheduled checkpoint file... +2022-04-11 19:12:06.438239: saving checkpoint... +2022-04-11 19:12:10.863297: done, saving took 4.50 seconds +2022-04-11 19:12:10.916741: done +2022-04-11 19:12:10.964251: saving checkpoint... +2022-04-11 19:12:15.441360: done, saving took 4.52 seconds +2022-04-11 19:12:15.524531: This epoch took 130.212611 s + +2022-04-11 19:12:15.534634: svmem(total=99932160000, available=79676116992, percent=20.3, used=16252649472, free=52532797440, active=40324947968, inactive=5250326528, buffers=12582912, cached=31134130176, shared=2190872576, slab=751837184) +2022-04-11 19:12:15.550505: +epoch: 74 +2022-04-11 19:14:02.490041: train loss : -0.4561 +2022-04-11 19:14:13.587905: validation loss: -0.4314 +2022-04-11 19:14:13.590308: Average global foreground Dice: [0.7147, 0.9173, 0.9568, 0.3144] +2022-04-11 19:14:13.591080: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:14:14.513557: lr: 0.008639 +2022-04-11 19:14:14.515014: saving scheduled checkpoint file... +2022-04-11 19:14:14.620781: saving checkpoint... +2022-04-11 19:14:19.069839: done, saving took 4.55 seconds +2022-04-11 19:14:19.086912: done +2022-04-11 19:14:19.182809: saving checkpoint... +2022-04-11 19:14:23.717630: done, saving took 4.63 seconds +2022-04-11 19:14:23.735856: This epoch took 128.181486 s + +2022-04-11 19:14:23.737129: svmem(total=99932160000, available=79801634816, percent=20.1, used=16127221760, free=52640747520, active=40265003008, inactive=5199896576, buffers=12615680, cached=31151575040, shared=2190729216, slab=751185920) +2022-04-11 19:14:23.738115: +epoch: 75 +2022-04-11 19:16:12.544352: train loss : -0.4260 +2022-04-11 19:16:23.623897: validation loss: -0.4548 +2022-04-11 19:16:23.639842: Average global foreground Dice: [0.7596, 0.9282, 0.9571, 0.3242] +2022-04-11 19:16:23.643587: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:16:24.591120: lr: 0.008621 +2022-04-11 19:16:24.592878: saving scheduled checkpoint file... +2022-04-11 19:16:24.729525: saving checkpoint... +2022-04-11 19:16:29.189019: done, saving took 4.60 seconds +2022-04-11 19:16:29.239116: done +2022-04-11 19:16:29.341109: saving checkpoint... +2022-04-11 19:16:33.860571: done, saving took 4.62 seconds +2022-04-11 19:16:33.876927: This epoch took 130.137795 s + +2022-04-11 19:16:33.878405: svmem(total=99932160000, available=79898329088, percent=20.0, used=16028512256, free=52728557568, active=40193171456, inactive=5183918080, buffers=12664832, cached=31162425344, shared=2192711680, slab=750862336) +2022-04-11 19:16:33.879293: +epoch: 76 +2022-04-11 19:18:24.447434: train loss : -0.3967 +2022-04-11 19:18:37.210924: validation loss: -0.3878 +2022-04-11 19:18:37.219648: Average global foreground Dice: [0.7039, 0.9065, 0.9449, 0.2982] +2022-04-11 19:18:37.220773: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:18:38.118721: lr: 0.008603 +2022-04-11 19:18:38.120346: saving scheduled checkpoint file... +2022-04-11 19:18:38.267461: saving checkpoint... +2022-04-11 19:18:42.848627: done, saving took 4.73 seconds +2022-04-11 19:18:42.902080: done +2022-04-11 19:18:43.017860: saving checkpoint... +2022-04-11 19:18:47.464572: done, saving took 4.56 seconds +2022-04-11 19:18:47.490098: This epoch took 133.610060 s + +2022-04-11 19:18:47.491426: svmem(total=99932160000, available=79859310592, percent=20.1, used=16069816320, free=52672102400, active=40239677440, inactive=5192527872, buffers=12701696, cached=31177539584, shared=2190389248, slab=750202880) +2022-04-11 19:18:47.492169: +epoch: 77 +2022-04-11 19:20:36.795068: train loss : -0.4182 +2022-04-11 19:20:48.895961: validation loss: -0.4298 +2022-04-11 19:20:48.899080: Average global foreground Dice: [0.7169, 0.9219, 0.9662, 0.3097] +2022-04-11 19:20:48.900397: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:20:49.748127: lr: 0.008584 +2022-04-11 19:20:49.749409: saving scheduled checkpoint file... +2022-04-11 19:20:49.870121: saving checkpoint... +2022-04-11 19:20:54.272557: done, saving took 4.52 seconds +2022-04-11 19:20:54.347307: done +2022-04-11 19:20:54.394802: saving checkpoint... +2022-04-11 19:20:59.041762: done, saving took 4.69 seconds +2022-04-11 19:20:59.054786: This epoch took 131.561682 s + +2022-04-11 19:20:59.055620: svmem(total=99932160000, available=79885111296, percent=20.1, used=16037093376, free=52680568832, active=40213319680, inactive=5214638080, buffers=12742656, cached=31201755136, shared=2197172224, slab=751939584) +2022-04-11 19:20:59.056519: +epoch: 78 +2022-04-11 19:22:47.162479: train loss : -0.4484 +2022-04-11 19:22:58.112589: validation loss: -0.4560 +2022-04-11 19:22:58.115435: Average global foreground Dice: [0.7581, 0.9297, 0.9631, 0.2989] +2022-04-11 19:22:58.116273: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:22:58.756389: lr: 0.008566 +2022-04-11 19:22:58.757811: saving scheduled checkpoint file... +2022-04-11 19:22:58.836745: saving checkpoint... +2022-04-11 19:23:03.412780: done, saving took 4.65 seconds +2022-04-11 19:23:03.426757: done +2022-04-11 19:23:03.474584: saving checkpoint... +2022-04-11 19:23:07.875460: done, saving took 4.45 seconds +2022-04-11 19:23:07.892689: This epoch took 128.835431 s + +2022-04-11 19:23:07.894103: svmem(total=99932160000, available=79696265216, percent=20.2, used=16232374272, free=52480958464, active=40402427904, inactive=5216083968, buffers=12783616, cached=31206043648, shared=2190807040, slab=751124480) +2022-04-11 19:23:07.895312: +epoch: 79 +2022-04-11 19:24:55.741223: train loss : -0.4677 +2022-04-11 19:25:07.410757: validation loss: -0.4640 +2022-04-11 19:25:07.427805: Average global foreground Dice: [0.7594, 0.9253, 0.9602, 0.2945] +2022-04-11 19:25:07.429159: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:25:08.257298: lr: 0.008548 +2022-04-11 19:25:08.258808: saving scheduled checkpoint file... +2022-04-11 19:25:08.344839: saving checkpoint... +2022-04-11 19:25:12.739618: done, saving took 4.48 seconds +2022-04-11 19:25:12.795807: done +2022-04-11 19:25:12.842814: saving checkpoint... +2022-04-11 19:25:17.193079: done, saving took 4.40 seconds +2022-04-11 19:25:17.206915: This epoch took 129.310618 s + +2022-04-11 19:25:17.208107: svmem(total=99932160000, available=79741988864, percent=20.2, used=16186941440, free=52509306880, active=40360443904, inactive=5233168384, buffers=12840960, cached=31223070720, shared=2190508032, slab=749731840) +2022-04-11 19:25:17.208808: +epoch: 80 +2022-04-11 19:27:06.409382: train loss : -0.3908 +2022-04-11 19:27:17.230137: validation loss: -0.3468 +2022-04-11 19:27:17.241741: Average global foreground Dice: [0.6271, 0.892, 0.9356, 0.2953] +2022-04-11 19:27:17.243052: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:27:17.976284: lr: 0.008529 +2022-04-11 19:27:17.977603: saving scheduled checkpoint file... +2022-04-11 19:27:18.068660: saving checkpoint... +2022-04-11 19:27:22.149351: done, saving took 4.17 seconds +2022-04-11 19:27:22.224005: done +2022-04-11 19:27:22.225187: This epoch took 125.015712 s + +2022-04-11 19:27:22.226417: svmem(total=99932160000, available=79768760320, percent=20.2, used=16159924224, free=52518559744, active=40335994880, inactive=5250793472, buffers=12881920, cached=31240794112, shared=2190729216, slab=751255552) +2022-04-11 19:27:22.227356: +epoch: 81 +2022-04-11 19:29:09.863700: train loss : -0.3910 +2022-04-11 19:29:20.707150: validation loss: -0.4403 +2022-04-11 19:29:20.710915: Average global foreground Dice: [0.7991, 0.9222, 0.9542, 0.3066] +2022-04-11 19:29:20.712058: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:29:21.591520: lr: 0.008511 +2022-04-11 19:29:21.592742: saving scheduled checkpoint file... +2022-04-11 19:29:21.694085: saving checkpoint... +2022-04-11 19:29:26.184731: done, saving took 4.59 seconds +2022-04-11 19:29:26.194649: done +2022-04-11 19:29:26.243786: saving checkpoint... +2022-04-11 19:29:30.761778: done, saving took 4.57 seconds +2022-04-11 19:29:30.776899: This epoch took 128.548619 s + +2022-04-11 19:29:30.778390: svmem(total=99932160000, available=79642931200, percent=20.3, used=16288862208, free=52384993280, active=40463798272, inactive=5255159808, buffers=12914688, cached=31245389824, shared=2187468800, slab=750780416) +2022-04-11 19:29:30.779156: +epoch: 82 +2022-04-11 19:31:20.661741: train loss : -0.4159 +2022-04-11 19:31:32.041285: validation loss: -0.4254 +2022-04-11 19:31:32.043363: Average global foreground Dice: [0.7665, 0.9235, 0.9529, 0.2863] +2022-04-11 19:31:32.044589: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:31:32.878782: lr: 0.008493 +2022-04-11 19:31:32.879964: saving scheduled checkpoint file... +2022-04-11 19:31:32.976323: saving checkpoint... +2022-04-11 19:31:37.500487: done, saving took 4.62 seconds +2022-04-11 19:31:37.520452: done +2022-04-11 19:31:37.602209: saving checkpoint... +2022-04-11 19:31:42.025412: done, saving took 4.50 seconds +2022-04-11 19:31:42.045388: This epoch took 131.265494 s + +2022-04-11 19:31:42.047092: svmem(total=99932160000, available=79664242688, percent=20.3, used=16264323072, free=52393009152, active=40441798656, inactive=5271764992, buffers=12955648, cached=31261872128, shared=2190807040, slab=750387200) +2022-04-11 19:31:42.048165: +epoch: 83 +2022-04-11 19:33:29.665313: train loss : -0.4645 +2022-04-11 19:33:41.283932: validation loss: -0.4296 +2022-04-11 19:33:41.298159: Average global foreground Dice: [0.7724, 0.9203, 0.9444, 0.3033] +2022-04-11 19:33:41.299664: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:33:42.269911: lr: 0.008474 +2022-04-11 19:33:42.271636: saving scheduled checkpoint file... +2022-04-11 19:33:42.373670: saving checkpoint... +2022-04-11 19:33:47.081060: done, saving took 4.81 seconds +2022-04-11 19:33:47.093937: done +2022-04-11 19:33:47.141786: saving checkpoint... +2022-04-11 19:33:51.495783: done, saving took 4.40 seconds +2022-04-11 19:33:51.509133: This epoch took 129.459791 s + +2022-04-11 19:33:51.510523: svmem(total=99932160000, available=79878852608, percent=20.1, used=16039526400, free=52590206976, active=40214544384, inactive=5299109888, buffers=12988416, cached=31289438208, shared=2200813568, slab=752267264) +2022-04-11 19:33:51.511582: +epoch: 84 +2022-04-11 19:35:40.246417: train loss : -0.4490 +2022-04-11 19:35:51.641781: validation loss: -0.4178 +2022-04-11 19:35:51.647440: Average global foreground Dice: [0.7732, 0.9018, 0.9525, 0.3194] +2022-04-11 19:35:51.648721: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:35:52.654099: lr: 0.008456 +2022-04-11 19:35:52.655277: saving scheduled checkpoint file... +2022-04-11 19:35:52.740784: saving checkpoint... +2022-04-11 19:35:57.095400: done, saving took 4.44 seconds +2022-04-11 19:35:57.110557: done +2022-04-11 19:35:57.157864: saving checkpoint... +2022-04-11 19:36:01.690311: done, saving took 4.58 seconds +2022-04-11 19:36:01.735493: This epoch took 130.223023 s + +2022-04-11 19:36:01.737422: svmem(total=99932160000, available=79862706176, percent=20.1, used=16055259136, free=52556386304, active=40231124992, inactive=5316988928, buffers=13037568, cached=31307476992, shared=2201247744, slab=751960064) +2022-04-11 19:36:01.738812: +epoch: 85 +2022-04-11 19:37:51.428868: train loss : -0.4696 +2022-04-11 19:38:02.956184: validation loss: -0.4803 +2022-04-11 19:38:02.959296: Average global foreground Dice: [0.7839, 0.9259, 0.9641, 0.3258] +2022-04-11 19:38:02.960653: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:38:03.899186: lr: 0.008438 +2022-04-11 19:38:03.900622: saving scheduled checkpoint file... +2022-04-11 19:38:03.986154: saving checkpoint... +2022-04-11 19:38:08.812686: done, saving took 4.91 seconds +2022-04-11 19:38:08.827627: done +2022-04-11 19:38:08.876489: saving checkpoint... +2022-04-11 19:38:13.537353: done, saving took 4.71 seconds +2022-04-11 19:38:13.551795: This epoch took 131.811968 s + +2022-04-11 19:38:13.553194: svmem(total=99932160000, available=79914672128, percent=20.0, used=16027361280, free=52600520704, active=40200966144, inactive=5300703232, buffers=13099008, cached=31291179008, shared=2177228800, slab=752893952) +2022-04-11 19:38:13.554153: +epoch: 86 +2022-04-11 19:40:01.802183: train loss : -0.3850 +2022-04-11 19:40:13.884734: validation loss: -0.3379 +2022-04-11 19:40:13.890552: Average global foreground Dice: [0.7049, 0.9045, 0.9296, 0.3038] +2022-04-11 19:40:13.891687: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:40:14.883258: lr: 0.008419 +2022-04-11 19:40:14.884866: saving scheduled checkpoint file... +2022-04-11 19:40:14.980566: saving checkpoint... +2022-04-11 19:40:19.986335: done, saving took 5.10 seconds +2022-04-11 19:40:20.006526: done +2022-04-11 19:40:20.007947: This epoch took 126.453031 s + +2022-04-11 19:40:20.009496: svmem(total=99932160000, available=79838539776, percent=20.1, used=16079572992, free=52506898432, active=40257691648, inactive=5342105600, buffers=13139968, cached=31332548608, shared=2201137152, slab=751771648) +2022-04-11 19:40:20.010617: +epoch: 87 +2022-04-11 19:42:08.617539: train loss : -0.3989 +2022-04-11 19:42:20.788828: validation loss: -0.4400 +2022-04-11 19:42:20.797498: Average global foreground Dice: [0.7343, 0.924, 0.9572, 0.31] +2022-04-11 19:42:20.798343: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:42:21.665926: lr: 0.008401 +2022-04-11 19:42:21.667256: saving scheduled checkpoint file... +2022-04-11 19:42:21.787039: saving checkpoint... +2022-04-11 19:42:26.496973: done, saving took 4.83 seconds +2022-04-11 19:42:26.540568: done +2022-04-11 19:42:26.541937: This epoch took 126.530579 s + +2022-04-11 19:42:26.543595: svmem(total=99932160000, available=80024231936, percent=19.9, used=15904751616, free=52675936256, active=40081747968, inactive=5347590144, buffers=13172736, cached=31338299392, shared=2190123008, slab=753938432) +2022-04-11 19:42:26.544728: +epoch: 88 +2022-04-11 19:44:14.353142: train loss : -0.3752 +2022-04-11 19:44:25.894989: validation loss: -0.3654 +2022-04-11 19:44:25.902107: Average global foreground Dice: [0.7258, 0.8998, 0.9275, 0.2993] +2022-04-11 19:44:25.903195: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:44:26.786378: lr: 0.008383 +2022-04-11 19:44:26.800165: saving scheduled checkpoint file... +2022-04-11 19:44:26.928071: saving checkpoint... +2022-04-11 19:44:31.637829: done, saving took 4.83 seconds +2022-04-11 19:44:31.671587: done +2022-04-11 19:44:31.672854: This epoch took 125.127039 s + +2022-04-11 19:44:31.673948: svmem(total=99932160000, available=79842074624, percent=20.1, used=16086233088, free=52486111232, active=40266342400, inactive=5355859968, buffers=13205504, cached=31346610176, shared=2190807040, slab=752893952) +2022-04-11 19:44:31.674781: +epoch: 89 +2022-04-11 19:46:20.182520: train loss : -0.4354 +2022-04-11 19:46:32.011469: validation loss: -0.4660 +2022-04-11 19:46:32.013554: Average global foreground Dice: [0.7585, 0.9275, 0.9679, 0.2998] +2022-04-11 19:46:32.014390: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:46:32.982958: lr: 0.008364 +2022-04-11 19:46:32.984604: saving scheduled checkpoint file... +2022-04-11 19:46:33.087950: saving checkpoint... +2022-04-11 19:46:38.169844: done, saving took 5.18 seconds +2022-04-11 19:46:38.208046: done +2022-04-11 19:46:38.257920: saving checkpoint... +2022-04-11 19:46:42.576129: done, saving took 4.37 seconds +2022-04-11 19:46:42.600294: This epoch took 130.924766 s + +2022-04-11 19:46:42.601631: svmem(total=99932160000, available=79911759872, percent=20.0, used=16016486400, free=52539195392, active=40197922816, inactive=5372477440, buffers=13250560, cached=31363227648, shared=2190864384, slab=751321088) +2022-04-11 19:46:42.602395: +epoch: 90 +2022-04-11 19:48:31.836580: train loss : -0.4533 +2022-04-11 19:48:43.416851: validation loss: -0.4314 +2022-04-11 19:48:43.423199: Average global foreground Dice: [0.7635, 0.915, 0.9516, 0.3088] +2022-04-11 19:48:43.424306: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:48:44.319192: lr: 0.008346 +2022-04-11 19:48:44.320176: saving scheduled checkpoint file... +2022-04-11 19:48:44.417774: saving checkpoint... +2022-04-11 19:48:49.010467: done, saving took 4.69 seconds +2022-04-11 19:48:49.024982: done +2022-04-11 19:48:49.072820: saving checkpoint... +2022-04-11 19:48:53.440278: done, saving took 4.41 seconds +2022-04-11 19:48:53.453603: This epoch took 130.850118 s + +2022-04-11 19:48:53.455092: svmem(total=99932160000, available=79748165632, percent=20.2, used=16180109312, free=52358242304, active=40357933056, inactive=5389647872, buffers=13283328, cached=31380525056, shared=2190729216, slab=753532928) +2022-04-11 19:48:53.455923: +epoch: 91 +2022-04-11 19:50:42.051522: train loss : -0.4590 +2022-04-11 19:50:53.322737: validation loss: -0.4585 +2022-04-11 19:50:53.345944: Average global foreground Dice: [0.7213, 0.9287, 0.9649, 0.3126] +2022-04-11 19:50:53.347014: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:50:54.351455: lr: 0.008328 +2022-04-11 19:50:54.352870: saving scheduled checkpoint file... +2022-04-11 19:50:54.443802: saving checkpoint... +2022-04-11 19:50:59.082914: done, saving took 4.73 seconds +2022-04-11 19:50:59.095361: done +2022-04-11 19:50:59.143024: saving checkpoint... +2022-04-11 19:51:03.732670: done, saving took 4.64 seconds +2022-04-11 19:51:03.777081: This epoch took 130.320165 s + +2022-04-11 19:51:03.778488: svmem(total=99932160000, available=79793479680, percent=20.2, used=16134860800, free=52395687936, active=40310972416, inactive=5397266432, buffers=13324288, cached=31388286976, shared=2190618624, slab=751624192) +2022-04-11 19:51:03.779484: +epoch: 92 +2022-04-11 19:52:51.690961: train loss : -0.4358 +2022-04-11 19:53:02.202614: validation loss: -0.4027 +2022-04-11 19:53:02.204683: Average global foreground Dice: [0.7469, 0.9068, 0.9495, 0.2942] +2022-04-11 19:53:02.205572: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:53:03.180001: lr: 0.008309 +2022-04-11 19:53:03.181572: saving scheduled checkpoint file... +2022-04-11 19:53:03.291664: saving checkpoint... +2022-04-11 19:53:08.004106: done, saving took 4.82 seconds +2022-04-11 19:53:08.021950: done +2022-04-11 19:53:08.023160: This epoch took 124.242956 s + +2022-04-11 19:53:08.024156: svmem(total=99932160000, available=79898083328, percent=20.0, used=16030224384, free=52482785280, active=40207048704, inactive=5414813696, buffers=13357056, cached=31405793280, shared=2190729216, slab=751960064) +2022-04-11 19:53:08.025281: +epoch: 93 +2022-04-11 19:54:55.823580: train loss : -0.4465 +2022-04-11 19:55:07.310613: validation loss: -0.4747 +2022-04-11 19:55:07.312649: Average global foreground Dice: [0.7651, 0.92, 0.9647, 0.3141] +2022-04-11 19:55:07.314017: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:55:08.201754: lr: 0.008291 +2022-04-11 19:55:08.203441: saving scheduled checkpoint file... +2022-04-11 19:55:08.326880: saving checkpoint... +2022-04-11 19:55:12.984848: done, saving took 4.78 seconds +2022-04-11 19:55:13.004427: done +2022-04-11 19:55:13.107886: saving checkpoint... +2022-04-11 19:55:17.684939: done, saving took 4.68 seconds +2022-04-11 19:55:17.709018: This epoch took 129.683005 s + +2022-04-11 19:55:17.710113: svmem(total=99932160000, available=79836954624, percent=20.1, used=16091369472, free=52404338688, active=40266645504, inactive=5432000512, buffers=13414400, cached=31423037440, shared=2190585856, slab=751247360) +2022-04-11 19:55:17.710909: +epoch: 94 +2022-04-11 19:57:07.349931: train loss : -0.4802 +2022-04-11 19:57:19.091652: validation loss: -0.4863 +2022-04-11 19:57:19.093633: Average global foreground Dice: [0.7867, 0.9298, 0.9603, 0.3465] +2022-04-11 19:57:19.094884: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:57:19.940768: lr: 0.008272 +2022-04-11 19:57:19.942166: saving scheduled checkpoint file... +2022-04-11 19:57:20.089638: saving checkpoint... +2022-04-11 19:57:24.628466: done, saving took 4.69 seconds +2022-04-11 19:57:24.688016: done +2022-04-11 19:57:24.803847: saving checkpoint... +2022-04-11 19:57:29.515782: done, saving took 4.83 seconds +2022-04-11 19:57:29.538985: This epoch took 131.827413 s + +2022-04-11 19:57:29.540678: svmem(total=99932160000, available=80119861248, percent=19.8, used=15808483328, free=52679507968, active=39978135552, inactive=5439750144, buffers=13455360, cached=31430713344, shared=2190618624, slab=752099328) +2022-04-11 19:57:29.541531: +epoch: 95 +2022-04-11 19:59:19.014756: train loss : -0.4674 +2022-04-11 19:59:31.588081: validation loss: -0.2720 +2022-04-11 19:59:31.602788: Average global foreground Dice: [0.7467, 0.8875, 0.8799, 0.2939] +2022-04-11 19:59:31.609369: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 19:59:32.516207: lr: 0.008254 +2022-04-11 19:59:32.517506: saving scheduled checkpoint file... +2022-04-11 19:59:32.611369: saving checkpoint... +2022-04-11 19:59:37.199674: done, saving took 4.68 seconds +2022-04-11 19:59:37.237712: done +2022-04-11 19:59:37.238922: This epoch took 127.696675 s + +2022-04-11 19:59:37.239911: svmem(total=99932160000, available=79824834560, percent=20.1, used=16103309312, free=52366954496, active=40281387008, inactive=5457154048, buffers=13488128, cached=31448408064, shared=2190729216, slab=752377856) +2022-04-11 19:59:37.240742: +epoch: 96 +2022-04-11 20:01:24.923025: train loss : -0.3961 +2022-04-11 20:01:37.089884: validation loss: -0.4574 +2022-04-11 20:01:37.665590: Average global foreground Dice: [0.698, 0.9203, 0.9666, 0.3084] +2022-04-11 20:01:37.668552: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:01:38.449270: lr: 0.008236 +2022-04-11 20:01:38.450478: saving scheduled checkpoint file... +2022-04-11 20:01:38.541123: saving checkpoint... +2022-04-11 20:01:43.073060: done, saving took 4.62 seconds +2022-04-11 20:01:43.085453: done +2022-04-11 20:01:43.086832: This epoch took 125.845351 s + +2022-04-11 20:01:43.088143: svmem(total=99932160000, available=79705055232, percent=20.2, used=16212492288, free=52230868992, active=40388157440, inactive=5484011520, buffers=13529088, cached=31475269632, shared=2201325568, slab=752750592) +2022-04-11 20:01:43.089074: +epoch: 97 +2022-04-11 20:03:31.848931: train loss : -0.4244 +2022-04-11 20:03:43.807447: validation loss: -0.4259 +2022-04-11 20:03:43.810994: Average global foreground Dice: [0.7857, 0.914, 0.9475, 0.3186] +2022-04-11 20:03:43.812083: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:03:44.793998: lr: 0.008217 +2022-04-11 20:03:44.795400: saving scheduled checkpoint file... +2022-04-11 20:03:44.848425: saving checkpoint... +2022-04-11 20:03:49.571049: done, saving took 4.77 seconds +2022-04-11 20:03:49.593417: done +2022-04-11 20:03:49.594351: This epoch took 126.504376 s + +2022-04-11 20:03:49.595389: svmem(total=99932160000, available=79746314240, percent=20.2, used=16171638784, free=52264226816, active=40352169984, inactive=5491384320, buffers=13561856, cached=31482732544, shared=2200940544, slab=752197632) +2022-04-11 20:03:49.600711: +epoch: 98 +2022-04-11 20:05:37.384757: train loss : -0.4333 +2022-04-11 20:05:49.164663: validation loss: -0.4502 +2022-04-11 20:05:49.291724: Average global foreground Dice: [0.7385, 0.9188, 0.9563, 0.325] +2022-04-11 20:05:49.442578: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:05:50.237129: lr: 0.008199 +2022-04-11 20:05:50.238806: saving scheduled checkpoint file... +2022-04-11 20:05:50.324512: saving checkpoint... +2022-04-11 20:05:54.848052: done, saving took 4.61 seconds +2022-04-11 20:05:54.891246: done +2022-04-11 20:05:54.892361: This epoch took 125.290608 s + +2022-04-11 20:05:54.893534: svmem(total=99932160000, available=79803461632, percent=20.1, used=16124633088, free=52310556672, active=40251645952, inactive=5536067584, buffers=13606912, cached=31483363328, shared=2190729216, slab=752066560) +2022-04-11 20:05:54.894414: +epoch: 99 +2022-04-11 20:07:44.087059: train loss : -0.4667 +2022-04-11 20:07:56.141561: validation loss: -0.4666 +2022-04-11 20:07:56.143795: Average global foreground Dice: [0.7797, 0.9317, 0.9631, 0.3143] +2022-04-11 20:07:56.145411: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:07:57.030022: lr: 0.008181 +2022-04-11 20:07:57.031923: saving scheduled checkpoint file... +2022-04-11 20:07:57.143838: saving checkpoint... +2022-04-11 20:08:02.062042: done, saving took 5.03 seconds +2022-04-11 20:08:02.081877: done +2022-04-11 20:08:02.176327: saving checkpoint... +2022-04-11 20:08:06.755875: done, saving took 4.67 seconds +2022-04-11 20:08:06.774969: This epoch took 131.879678 s + +2022-04-11 20:08:06.776398: svmem(total=99932160000, available=79871942656, percent=20.1, used=16056582144, free=52361650176, active=40226725888, inactive=5511962624, buffers=13651968, cached=31500275712, shared=2190352384, slab=751636480) +2022-04-11 20:08:06.777309: +epoch: 100 +2022-04-11 20:09:55.154272: train loss : -0.4874 +2022-04-11 20:10:06.213759: validation loss: -0.4811 +2022-04-11 20:10:06.215537: Average global foreground Dice: [0.7834, 0.9213, 0.9609, 0.3294] +2022-04-11 20:10:06.216871: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:10:07.080530: lr: 0.008162 +2022-04-11 20:10:07.082076: saving scheduled checkpoint file... +2022-04-11 20:10:07.215639: saving checkpoint... +2022-04-11 20:10:12.356652: done, saving took 5.27 seconds +2022-04-11 20:10:12.393930: done +2022-04-11 20:10:12.504639: saving checkpoint... +2022-04-11 20:10:17.091472: done, saving took 4.70 seconds +2022-04-11 20:10:17.113410: This epoch took 130.335062 s + +2022-04-11 20:10:17.114577: svmem(total=99932160000, available=79817056256, percent=20.1, used=16111071232, free=52289224704, active=40282185728, inactive=5526958080, buffers=13709312, cached=31518154752, shared=2190729216, slab=751579136) +2022-04-11 20:10:17.115694: +epoch: 101 +2022-04-11 20:12:07.609580: train loss : -0.4860 +2022-04-11 20:12:20.418274: validation loss: -0.4759 +2022-04-11 20:12:20.424563: Average global foreground Dice: [0.7344, 0.9399, 0.951, 0.3268] +2022-04-11 20:12:20.425873: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:12:21.466294: lr: 0.008144 +2022-04-11 20:12:21.467682: saving scheduled checkpoint file... +2022-04-11 20:12:21.571564: saving checkpoint... +2022-04-11 20:12:25.952471: done, saving took 4.48 seconds +2022-04-11 20:12:25.968833: done +2022-04-11 20:12:26.019163: saving checkpoint... +2022-04-11 20:12:30.387316: done, saving took 4.42 seconds +2022-04-11 20:12:30.401979: This epoch took 133.285460 s + +2022-04-11 20:12:30.403386: svmem(total=99932160000, available=79820582912, percent=20.1, used=16107565056, free=52307869696, active=40274665472, inactive=5522456576, buffers=13742080, cached=31502983168, shared=2190618624, slab=752164864) +2022-04-11 20:12:30.404132: +epoch: 102 +2022-04-11 20:14:18.159738: train loss : -0.5018 +2022-04-11 20:14:29.251460: validation loss: -0.5161 +2022-04-11 20:14:29.260822: Average global foreground Dice: [0.7864, 0.9368, 0.9553, 0.3622] +2022-04-11 20:14:29.262338: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:14:29.882040: lr: 0.008125 +2022-04-11 20:14:29.883636: saving scheduled checkpoint file... +2022-04-11 20:14:29.975288: saving checkpoint... +2022-04-11 20:14:34.390020: done, saving took 4.51 seconds +2022-04-11 20:14:34.404187: done +2022-04-11 20:14:34.453299: saving checkpoint... +2022-04-11 20:14:38.827426: done, saving took 4.42 seconds +2022-04-11 20:14:38.839952: This epoch took 128.435045 s + +2022-04-11 20:14:38.841362: svmem(total=99932160000, available=79810691072, percent=20.1, used=16117067776, free=52280655872, active=40293531648, inactive=5528956928, buffers=13774848, cached=31520661504, shared=2190917632, slab=752410624) +2022-04-11 20:14:38.842112: +epoch: 103 +2022-04-11 20:16:26.779030: train loss : -0.5075 +2022-04-11 20:16:38.524334: validation loss: -0.5314 +2022-04-11 20:16:38.534017: Average global foreground Dice: [0.8045, 0.9391, 0.9739, 0.3513] +2022-04-11 20:16:38.535493: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:16:39.384684: lr: 0.008107 +2022-04-11 20:16:39.386263: saving scheduled checkpoint file... +2022-04-11 20:16:39.478128: saving checkpoint... +2022-04-11 20:16:43.980380: done, saving took 4.59 seconds +2022-04-11 20:16:44.026322: done +2022-04-11 20:16:44.077008: saving checkpoint... +2022-04-11 20:16:48.640165: done, saving took 4.61 seconds +2022-04-11 20:16:48.657060: This epoch took 129.813884 s + +2022-04-11 20:16:48.658327: svmem(total=99932160000, available=79722299392, percent=20.2, used=16205107200, free=52122136576, active=40383946752, inactive=5593276416, buffers=15802368, cached=31589113856, shared=2190729216, slab=753971200) +2022-04-11 20:16:48.659233: +epoch: 104 +2022-04-11 20:18:36.429466: train loss : -0.5230 +2022-04-11 20:18:48.231116: validation loss: -0.5064 +2022-04-11 20:18:48.232991: Average global foreground Dice: [0.8133, 0.9283, 0.9673, 0.3237] +2022-04-11 20:18:48.234264: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:18:49.106088: lr: 0.008088 +2022-04-11 20:18:49.107472: saving scheduled checkpoint file... +2022-04-11 20:18:49.206403: saving checkpoint... +2022-04-11 20:18:53.807105: done, saving took 4.70 seconds +2022-04-11 20:18:53.825313: done +2022-04-11 20:18:53.904717: saving checkpoint... +2022-04-11 20:18:58.458887: done, saving took 4.63 seconds +2022-04-11 20:18:58.475392: This epoch took 129.814878 s + +2022-04-11 20:18:58.476912: svmem(total=99932160000, available=79689318400, percent=20.3, used=16238108672, free=52081451008, active=40420413440, inactive=5600964608, buffers=15847424, cached=31596752896, shared=2190729216, slab=754335744) +2022-04-11 20:18:58.477918: +epoch: 105 +2022-04-11 20:20:47.036443: train loss : -0.5270 +2022-04-11 20:20:59.068136: validation loss: -0.5255 +2022-04-11 20:20:59.079223: Average global foreground Dice: [0.7736, 0.9384, 0.9671, 0.3293] +2022-04-11 20:20:59.080990: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:20:59.864659: lr: 0.00807 +2022-04-11 20:20:59.866062: saving scheduled checkpoint file... +2022-04-11 20:20:59.961494: saving checkpoint... +2022-04-11 20:21:04.635602: done, saving took 4.77 seconds +2022-04-11 20:21:04.655157: done +2022-04-11 20:21:04.748056: saving checkpoint... +2022-04-11 20:21:09.311886: done, saving took 4.66 seconds +2022-04-11 20:21:09.333816: This epoch took 130.855243 s + +2022-04-11 20:21:09.335021: svmem(total=99932160000, available=79744888832, percent=20.2, used=16192962560, free=52153446400, active=40152502272, inactive=5791264768, buffers=15888384, cached=31569862656, shared=2180210688, slab=754196480) +2022-04-11 20:21:09.335867: +epoch: 106 +2022-04-11 20:22:58.005316: train loss : -0.5204 +2022-04-11 20:23:09.881851: validation loss: -0.4996 +2022-04-11 20:23:09.890060: Average global foreground Dice: [0.7963, 0.9322, 0.9709, 0.3276] +2022-04-11 20:23:09.893464: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:23:10.751470: lr: 0.008052 +2022-04-11 20:23:10.752899: saving scheduled checkpoint file... +2022-04-11 20:23:10.890725: saving checkpoint... +2022-04-11 20:23:15.452967: done, saving took 4.70 seconds +2022-04-11 20:23:15.507003: done +2022-04-11 20:23:15.628566: saving checkpoint... +2022-04-11 20:23:20.172951: done, saving took 4.67 seconds +2022-04-11 20:23:20.194452: This epoch took 130.857719 s + +2022-04-11 20:23:20.196018: svmem(total=99932160000, available=79925891072, percent=20.0, used=16001515520, free=52317032448, active=40100990976, inactive=5682515968, buffers=15945728, cached=31597666304, shared=2190741504, slab=752648192) +2022-04-11 20:23:20.196922: +epoch: 107 +2022-04-11 20:25:10.996327: train loss : -0.5123 +2022-04-11 20:25:24.101255: validation loss: -0.4861 +2022-04-11 20:25:24.104193: Average global foreground Dice: [0.7992, 0.9287, 0.9646, 0.3235] +2022-04-11 20:25:24.105290: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:25:25.070838: lr: 0.008033 +2022-04-11 20:25:25.072222: saving scheduled checkpoint file... +2022-04-11 20:25:25.161526: saving checkpoint... +2022-04-11 20:25:29.562442: done, saving took 4.49 seconds +2022-04-11 20:25:29.585837: done +2022-04-11 20:25:29.633017: saving checkpoint... +2022-04-11 20:25:33.950480: done, saving took 4.36 seconds +2022-04-11 20:25:33.963571: This epoch took 133.765771 s + +2022-04-11 20:25:33.965265: svmem(total=99932160000, available=79782907904, percent=20.2, used=16134459392, free=52166275072, active=40258912256, inactive=5675425792, buffers=16003072, cached=31615422464, shared=2200641536, slab=753610752) +2022-04-11 20:25:33.966340: +epoch: 108 +2022-04-11 20:27:21.586562: train loss : -0.5135 +2022-04-11 20:27:32.415116: validation loss: -0.5089 +2022-04-11 20:27:32.417706: Average global foreground Dice: [0.7904, 0.9347, 0.9665, 0.3133] +2022-04-11 20:27:32.418763: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:27:33.436061: lr: 0.008015 +2022-04-11 20:27:33.437449: saving scheduled checkpoint file... +2022-04-11 20:27:33.541123: saving checkpoint... +2022-04-11 20:27:38.156873: done, saving took 4.72 seconds +2022-04-11 20:27:38.189678: done +2022-04-11 20:27:38.236683: saving checkpoint... +2022-04-11 20:27:42.608717: done, saving took 4.42 seconds +2022-04-11 20:27:42.643118: This epoch took 128.675428 s + +2022-04-11 20:27:42.644455: svmem(total=99932160000, available=79744565248, percent=20.2, used=16176214016, free=52110450688, active=40343392256, inactive=5647249408, buffers=16044032, cached=31629451264, shared=2197327872, slab=754483200) +2022-04-11 20:27:42.645317: +epoch: 109 +2022-04-11 20:29:31.793057: train loss : -0.5244 +2022-04-11 20:29:43.312592: validation loss: -0.5211 +2022-04-11 20:29:43.333909: Average global foreground Dice: [0.8012, 0.929, 0.9709, 0.3455] +2022-04-11 20:29:43.334986: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:29:44.193080: lr: 0.007996 +2022-04-11 20:29:44.194746: saving scheduled checkpoint file... +2022-04-11 20:29:44.274408: saving checkpoint... +2022-04-11 20:29:48.607821: done, saving took 4.41 seconds +2022-04-11 20:29:48.718233: done +2022-04-11 20:29:48.766506: saving checkpoint... +2022-04-11 20:29:53.231300: done, saving took 4.51 seconds +2022-04-11 20:29:53.240979: This epoch took 130.594806 s + +2022-04-11 20:29:53.242524: svmem(total=99932160000, available=79848079360, percent=20.1, used=16092962816, free=52210565120, active=40107671552, inactive=5781336064, buffers=15876096, cached=31612755968, shared=2177044480, slab=753811456) +2022-04-11 20:29:53.243754: +epoch: 110 +2022-04-11 20:31:41.929628: train loss : -0.5257 +2022-04-11 20:31:53.187560: validation loss: -0.4360 +2022-04-11 20:31:53.197493: Average global foreground Dice: [0.8149, 0.9223, 0.9332, 0.3448] +2022-04-11 20:31:53.199192: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:31:54.329628: lr: 0.007978 +2022-04-11 20:31:54.330806: saving scheduled checkpoint file... +2022-04-11 20:31:54.392241: saving checkpoint... +2022-04-11 20:31:58.842092: done, saving took 4.51 seconds +2022-04-11 20:31:58.854068: done +2022-04-11 20:31:58.904373: saving checkpoint... +2022-04-11 20:32:03.328433: done, saving took 4.47 seconds +2022-04-11 20:32:03.368668: This epoch took 130.124204 s + +2022-04-11 20:32:03.370006: svmem(total=99932160000, available=79937331200, percent=20.0, used=16002461696, free=52282580992, active=40100474880, inactive=5722832896, buffers=15917056, cached=31631200256, shared=2178031616, slab=755838976) +2022-04-11 20:32:03.371133: +epoch: 111 +2022-04-11 20:33:50.819420: train loss : -0.4648 +2022-04-11 20:34:03.185796: validation loss: -0.5021 +2022-04-11 20:34:03.196951: Average global foreground Dice: [0.7764, 0.9377, 0.9614, 0.3342] +2022-04-11 20:34:03.198227: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:34:03.772651: lr: 0.007959 +2022-04-11 20:34:03.774251: saving scheduled checkpoint file... +2022-04-11 20:34:03.869068: saving checkpoint... +2022-04-11 20:34:08.288106: done, saving took 4.51 seconds +2022-04-11 20:34:08.350340: done +2022-04-11 20:34:08.435703: saving checkpoint... +2022-04-11 20:34:13.067000: done, saving took 4.72 seconds +2022-04-11 20:34:13.085708: This epoch took 129.713920 s + +2022-04-11 20:34:13.087172: svmem(total=99932160000, available=79751049216, percent=20.2, used=16176234496, free=52033159168, active=40376373248, inactive=5689344000, buffers=15949824, cached=31706816512, shared=2190729216, slab=754692096) +2022-04-11 20:34:13.088015: +epoch: 112 +2022-04-11 20:36:01.539911: train loss : -0.4731 +2022-04-11 20:36:13.689190: validation loss: -0.4433 +2022-04-11 20:36:13.692290: Average global foreground Dice: [0.7661, 0.9207, 0.9502, 0.3401] +2022-04-11 20:36:13.693364: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:36:14.575728: lr: 0.007941 +2022-04-11 20:36:14.577152: saving scheduled checkpoint file... +2022-04-11 20:36:14.717507: saving checkpoint... +2022-04-11 20:36:19.352405: done, saving took 4.77 seconds +2022-04-11 20:36:19.423310: done +2022-04-11 20:36:19.424709: This epoch took 126.335925 s + +2022-04-11 20:36:19.425805: svmem(total=99932160000, available=79867428864, percent=20.1, used=16059834368, free=52133126144, active=40268972032, inactive=5705670656, buffers=16015360, cached=31723184128, shared=2190729216, slab=753123328) +2022-04-11 20:36:19.426678: +epoch: 113 +2022-04-11 20:38:09.056328: train loss : -0.4781 +2022-04-11 20:38:21.512498: validation loss: -0.4778 +2022-04-11 20:38:21.514844: Average global foreground Dice: [0.7829, 0.9353, 0.9439, 0.3389] +2022-04-11 20:38:21.516744: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:38:22.442187: lr: 0.007922 +2022-04-11 20:38:22.443586: saving scheduled checkpoint file... +2022-04-11 20:38:22.533526: saving checkpoint... +2022-04-11 20:38:26.867827: done, saving took 4.42 seconds +2022-04-11 20:38:26.937171: done +2022-04-11 20:38:26.938661: This epoch took 127.511199 s + +2022-04-11 20:38:26.940053: svmem(total=99932160000, available=79825235968, percent=20.1, used=16101994496, free=52073447424, active=40314171392, inactive=5716811776, buffers=16060416, cached=31740657664, shared=2190729216, slab=752967680) +2022-04-11 20:38:26.941242: +epoch: 114 +2022-04-11 20:40:16.133185: train loss : -0.4934 +2022-04-11 20:40:27.205410: validation loss: -0.4947 +2022-04-11 20:40:27.208138: Average global foreground Dice: [0.7689, 0.9339, 0.9687, 0.3573] +2022-04-11 20:40:27.209294: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:40:28.009453: lr: 0.007904 +2022-04-11 20:40:28.010685: saving scheduled checkpoint file... +2022-04-11 20:40:28.099188: saving checkpoint... +2022-04-11 20:40:32.282608: done, saving took 4.27 seconds +2022-04-11 20:40:32.296364: done +2022-04-11 20:40:32.345836: saving checkpoint... +2022-04-11 20:40:36.717616: done, saving took 4.42 seconds +2022-04-11 20:40:36.731709: This epoch took 129.789574 s + +2022-04-11 20:40:36.733022: svmem(total=99932160000, available=79711617024, percent=20.2, used=16225927168, free=51952099328, active=40434655232, inactive=5714108416, buffers=16101376, cached=31738032128, shared=2180399104, slab=754540544) +2022-04-11 20:40:36.733724: +epoch: 115 +2022-04-11 20:42:25.113825: train loss : -0.5153 +2022-04-11 20:42:36.428518: validation loss: -0.5206 +2022-04-11 20:42:36.444216: Average global foreground Dice: [0.8143, 0.9352, 0.969, 0.3533] +2022-04-11 20:42:36.447750: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:42:37.137437: lr: 0.007885 +2022-04-11 20:42:37.138496: saving scheduled checkpoint file... +2022-04-11 20:42:37.214881: saving checkpoint... +2022-04-11 20:42:42.448489: done, saving took 5.31 seconds +2022-04-11 20:42:42.531518: done +2022-04-11 20:42:42.579278: saving checkpoint... +2022-04-11 20:42:47.176923: done, saving took 4.64 seconds +2022-04-11 20:42:47.190166: This epoch took 130.455755 s + +2022-04-11 20:42:47.191656: svmem(total=99932160000, available=79781097472, percent=20.2, used=16145969152, free=52004106240, active=40356421632, inactive=5741752320, buffers=16134144, cached=31765950464, shared=2190729216, slab=754802688) +2022-04-11 20:42:47.192535: +epoch: 116 +2022-04-11 20:44:35.357269: train loss : -0.5343 +2022-04-11 20:44:46.904178: validation loss: -0.5445 +2022-04-11 20:44:46.924642: Average global foreground Dice: [0.7998, 0.9407, 0.9739, 0.3864] +2022-04-11 20:44:46.935903: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:44:47.911217: lr: 0.007867 +2022-04-11 20:44:47.914850: saving scheduled checkpoint file... +2022-04-11 20:44:48.006645: saving checkpoint... +2022-04-11 20:44:52.786377: done, saving took 4.87 seconds +2022-04-11 20:44:52.819280: done +2022-04-11 20:44:52.872462: saving checkpoint... +2022-04-11 20:44:57.651332: done, saving took 4.83 seconds +2022-04-11 20:44:57.665149: This epoch took 130.471431 s + +2022-04-11 20:44:57.666429: svmem(total=99932160000, available=79833006080, percent=20.1, used=16093880320, free=52038615040, active=40302764032, inactive=5759463424, buffers=16166912, cached=31783497728, shared=2190925824, slab=755179520) +2022-04-11 20:44:57.667144: +epoch: 117 +2022-04-11 20:46:45.330514: train loss : -0.5368 +2022-04-11 20:46:57.174769: validation loss: -0.5684 +2022-04-11 20:46:57.177905: Average global foreground Dice: [0.8031, 0.9492, 0.9731, 0.3623] +2022-04-11 20:46:57.178959: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:46:58.030337: lr: 0.007848 +2022-04-11 20:46:58.031623: saving scheduled checkpoint file... +2022-04-11 20:46:58.133105: saving checkpoint... +2022-04-11 20:47:02.539031: done, saving took 4.51 seconds +2022-04-11 20:47:02.558263: done +2022-04-11 20:47:02.639423: saving checkpoint... +2022-04-11 20:47:07.230918: done, saving took 4.67 seconds +2022-04-11 20:47:07.248644: This epoch took 129.580611 s + +2022-04-11 20:47:07.250155: svmem(total=99932160000, available=79807348736, percent=20.1, used=16109092864, free=52005171200, active=40314101760, inactive=5775298560, buffers=16211968, cached=31801683968, shared=2201325568, slab=754188288) +2022-04-11 20:47:07.251393: +epoch: 118 +2022-04-11 20:48:55.361721: train loss : -0.5289 +2022-04-11 20:49:06.687862: validation loss: -0.4849 +2022-04-11 20:49:06.694956: Average global foreground Dice: [0.7713, 0.9388, 0.965, 0.3551] +2022-04-11 20:49:06.695887: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:49:07.573007: lr: 0.00783 +2022-04-11 20:49:07.574558: saving scheduled checkpoint file... +2022-04-11 20:49:07.699975: saving checkpoint... +2022-04-11 20:49:12.318908: done, saving took 4.74 seconds +2022-04-11 20:49:12.338459: done +2022-04-11 20:49:12.434462: saving checkpoint... +2022-04-11 20:49:16.852087: done, saving took 4.51 seconds +2022-04-11 20:49:16.892689: This epoch took 129.640408 s + +2022-04-11 20:49:16.893975: svmem(total=99932160000, available=79745753088, percent=20.2, used=16180981760, free=51926069248, active=40397959168, inactive=5782675456, buffers=16261120, cached=31808847872, shared=2191114240, slab=754253824) +2022-04-11 20:49:16.894744: +epoch: 119 +2022-04-11 20:51:08.457278: train loss : -0.3925 +2022-04-11 20:51:21.169477: validation loss: -0.3985 +2022-04-11 20:51:21.171840: Average global foreground Dice: [0.7399, 0.9094, 0.9433, 0.3251] +2022-04-11 20:51:21.172988: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:51:22.048461: lr: 0.007811 +2022-04-11 20:51:22.049818: saving scheduled checkpoint file... +2022-04-11 20:51:22.141988: saving checkpoint... +2022-04-11 20:51:26.595623: done, saving took 4.54 seconds +2022-04-11 20:51:26.611858: done +2022-04-11 20:51:26.612836: This epoch took 129.717025 s + +2022-04-11 20:51:26.613874: svmem(total=99932160000, available=79826644992, percent=20.1, used=16100646912, free=51990970368, active=40285483008, inactive=5828669440, buffers=16302080, cached=31824240640, shared=2190422016, slab=753823744) +2022-04-11 20:51:26.614624: +epoch: 120 +2022-04-11 20:53:15.781915: train loss : -0.4528 +2022-04-11 20:53:27.426202: validation loss: -0.4494 +2022-04-11 20:53:27.428487: Average global foreground Dice: [0.7916, 0.9138, 0.9447, 0.3273] +2022-04-11 20:53:27.429772: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:53:28.266723: lr: 0.007793 +2022-04-11 20:53:28.268634: saving scheduled checkpoint file... +2022-04-11 20:53:28.364748: saving checkpoint... +2022-04-11 20:53:32.988655: done, saving took 4.72 seconds +2022-04-11 20:53:33.030528: done +2022-04-11 20:53:33.032038: This epoch took 126.416506 s + +2022-04-11 20:53:33.033282: svmem(total=99932160000, available=79674425344, percent=20.3, used=16252534784, free=51831001088, active=40458854400, inactive=5808660480, buffers=16334848, cached=31832289280, shared=2190729216, slab=755965952) +2022-04-11 20:53:33.034569: +epoch: 121 +2022-04-11 20:55:21.346689: train loss : -0.4671 +2022-04-11 20:55:33.423384: validation loss: -0.4994 +2022-04-11 20:55:33.449405: Average global foreground Dice: [0.7708, 0.9365, 0.9666, 0.3232] +2022-04-11 20:55:33.450835: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:55:34.348699: lr: 0.007774 +2022-04-11 20:55:34.350250: saving scheduled checkpoint file... +2022-04-11 20:55:34.441463: saving checkpoint... +2022-04-11 20:55:39.222114: done, saving took 4.87 seconds +2022-04-11 20:55:39.236961: done +2022-04-11 20:55:39.238311: This epoch took 126.202705 s + +2022-04-11 20:55:39.239436: svmem(total=99932160000, available=79846084608, percent=20.1, used=16080904192, free=52014411776, active=40113713152, inactive=5974675456, buffers=16400384, cached=31820443648, shared=2190729216, slab=756240384) +2022-04-11 20:55:39.240718: +epoch: 122 +2022-04-11 20:57:27.388025: train loss : -0.4766 +2022-04-11 20:57:39.382072: validation loss: -0.4407 +2022-04-11 20:57:39.384354: Average global foreground Dice: [0.7839, 0.9195, 0.9428, 0.3423] +2022-04-11 20:57:39.385220: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:57:40.312700: lr: 0.007756 +2022-04-11 20:57:40.314121: saving scheduled checkpoint file... +2022-04-11 20:57:40.417484: saving checkpoint... +2022-04-11 20:57:44.982258: done, saving took 4.67 seconds +2022-04-11 20:57:44.995435: done +2022-04-11 20:57:44.996778: This epoch took 125.755200 s + +2022-04-11 20:57:44.998077: svmem(total=99932160000, available=79909814272, percent=20.0, used=16017203200, free=52060684288, active=40196780032, inactive=5845757952, buffers=16433152, cached=31837839360, shared=2190729216, slab=755638272) +2022-04-11 20:57:44.998928: +epoch: 123 +2022-04-11 20:59:33.530352: train loss : -0.4940 +2022-04-11 20:59:46.016521: validation loss: -0.4898 +2022-04-11 20:59:46.032565: Average global foreground Dice: [0.7937, 0.924, 0.9614, 0.3466] +2022-04-11 20:59:46.033606: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 20:59:46.935035: lr: 0.007737 +2022-04-11 20:59:46.936831: saving scheduled checkpoint file... +2022-04-11 20:59:47.017862: saving checkpoint... +2022-04-11 20:59:51.743607: done, saving took 4.81 seconds +2022-04-11 20:59:51.821707: done +2022-04-11 20:59:51.822937: This epoch took 126.823220 s + +2022-04-11 20:59:51.823793: svmem(total=99932160000, available=79837749248, percent=20.1, used=16089108480, free=51980918784, active=40298352640, inactive=5820305408, buffers=16465920, cached=31845666816, shared=2190729216, slab=756830208) +2022-04-11 20:59:51.824433: +epoch: 124 +2022-04-11 21:01:41.185494: train loss : -0.4887 +2022-04-11 21:01:52.657605: validation loss: -0.5046 +2022-04-11 21:01:52.668145: Average global foreground Dice: [0.749, 0.9378, 0.9625, 0.3394] +2022-04-11 21:01:52.670094: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:01:53.546990: lr: 0.007719 +2022-04-11 21:01:53.548740: saving scheduled checkpoint file... +2022-04-11 21:01:53.676068: saving checkpoint... +2022-04-11 21:01:58.266808: done, saving took 4.72 seconds +2022-04-11 21:01:58.312476: done +2022-04-11 21:01:58.313631: This epoch took 126.488620 s + +2022-04-11 21:01:58.314607: svmem(total=99932160000, available=79817433088, percent=20.1, used=16109371392, free=51943112704, active=40318926848, inactive=5837713408, buffers=16506880, cached=31863169024, shared=2190729216, slab=755793920) +2022-04-11 21:01:58.315325: +epoch: 125 +2022-04-11 21:03:47.509071: train loss : -0.5225 +2022-04-11 21:03:59.756276: validation loss: -0.5162 +2022-04-11 21:03:59.761287: Average global foreground Dice: [0.7894, 0.9414, 0.9683, 0.3353] +2022-04-11 21:03:59.762550: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:04:00.771543: lr: 0.0077 +2022-04-11 21:04:00.773143: saving scheduled checkpoint file... +2022-04-11 21:04:00.866435: saving checkpoint... +2022-04-11 21:04:05.323301: done, saving took 4.55 seconds +2022-04-11 21:04:05.337180: done +2022-04-11 21:04:05.338415: This epoch took 127.022232 s + +2022-04-11 21:04:05.339406: svmem(total=99932160000, available=79772971008, percent=20.2, used=16144445440, free=51881136128, active=40360767488, inactive=5864583168, buffers=16539648, cached=31890038784, shared=2200145920, slab=755765248) +2022-04-11 21:04:05.340163: +epoch: 126 +2022-04-11 21:05:53.205372: train loss : -0.5243 +2022-04-11 21:06:04.884766: validation loss: -0.5429 +2022-04-11 21:06:04.887148: Average global foreground Dice: [0.8093, 0.9384, 0.9723, 0.3718] +2022-04-11 21:06:04.888539: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:06:05.519148: lr: 0.007682 +2022-04-11 21:06:05.520709: saving scheduled checkpoint file... +2022-04-11 21:06:05.596951: saving checkpoint... +2022-04-11 21:06:09.926372: done, saving took 4.40 seconds +2022-04-11 21:06:09.940844: done +2022-04-11 21:06:09.942132: This epoch took 124.601198 s + +2022-04-11 21:06:09.943142: svmem(total=99932160000, available=79778197504, percent=20.2, used=16148774912, free=51879526400, active=40357232640, inactive=5860368384, buffers=16588800, cached=31887269888, shared=2190618624, slab=755179520) +2022-04-11 21:06:09.943892: +epoch: 127 +2022-04-11 21:07:58.307423: train loss : -0.5226 +2022-04-11 21:08:10.019718: validation loss: -0.5417 +2022-04-11 21:08:10.041397: Average global foreground Dice: [0.7944, 0.931, 0.975, 0.3691] +2022-04-11 21:08:10.043231: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:08:10.930993: lr: 0.007663 +2022-04-11 21:08:10.932154: saving scheduled checkpoint file... +2022-04-11 21:08:11.020101: saving checkpoint... +2022-04-11 21:08:16.272469: done, saving took 5.34 seconds +2022-04-11 21:08:16.309774: done +2022-04-11 21:08:16.361489: saving checkpoint... +2022-04-11 21:08:20.969343: done, saving took 4.66 seconds +2022-04-11 21:08:21.031491: This epoch took 131.086894 s + +2022-04-11 21:08:21.032956: svmem(total=99932160000, available=79698108416, percent=20.2, used=16228818944, free=51801808896, active=40150364160, inactive=6148308992, buffers=16654336, cached=31884877824, shared=2190540800, slab=755666944) +2022-04-11 21:08:21.034122: +epoch: 128 +2022-04-11 21:10:08.975370: train loss : -0.5466 +2022-04-11 21:10:20.682709: validation loss: -0.5406 +2022-04-11 21:10:20.696498: Average global foreground Dice: [0.832, 0.9431, 0.9689, 0.3354] +2022-04-11 21:10:20.697939: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:10:21.744307: lr: 0.007645 +2022-04-11 21:10:21.746002: saving scheduled checkpoint file... +2022-04-11 21:10:21.846258: saving checkpoint... +2022-04-11 21:10:26.338006: done, saving took 4.59 seconds +2022-04-11 21:10:26.352495: done +2022-04-11 21:10:26.402341: saving checkpoint... +2022-04-11 21:10:30.980094: done, saving took 4.63 seconds +2022-04-11 21:10:30.995639: This epoch took 129.960739 s + +2022-04-11 21:10:30.997144: svmem(total=99932160000, available=79690649600, percent=20.3, used=16231165952, free=51730726912, active=40408334336, inactive=5958602752, buffers=16695296, cached=31953571840, shared=2195763200, slab=757030912) +2022-04-11 21:10:30.997902: +epoch: 129 +2022-04-11 21:12:19.567940: train loss : -0.5187 +2022-04-11 21:12:30.981887: validation loss: -0.4964 +2022-04-11 21:12:30.993392: Average global foreground Dice: [0.7541, 0.9286, 0.96, 0.3517] +2022-04-11 21:12:30.995261: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:12:31.845932: lr: 0.007626 +2022-04-11 21:12:31.847189: saving scheduled checkpoint file... +2022-04-11 21:12:31.925296: saving checkpoint... +2022-04-11 21:12:36.407003: done, saving took 4.56 seconds +2022-04-11 21:12:36.424383: done +2022-04-11 21:12:36.425404: This epoch took 125.426335 s + +2022-04-11 21:12:36.426835: svmem(total=99932160000, available=79860703232, percent=20.1, used=16066236416, free=51882151936, active=40258633728, inactive=5958361088, buffers=16728064, cached=31967043584, shared=2190548992, slab=757342208) +2022-04-11 21:12:36.427770: +epoch: 130 +2022-04-11 21:14:24.647490: train loss : -0.5114 +2022-04-11 21:14:36.020666: validation loss: -0.5252 +2022-04-11 21:14:36.022817: Average global foreground Dice: [0.7804, 0.9399, 0.9633, 0.3421] +2022-04-11 21:14:36.024178: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:14:37.229099: lr: 0.007608 +2022-04-11 21:14:37.230237: saving scheduled checkpoint file... +2022-04-11 21:14:37.353237: saving checkpoint... +2022-04-11 21:14:43.492948: done, saving took 6.26 seconds +2022-04-11 21:14:43.512667: done +2022-04-11 21:14:43.513909: This epoch took 127.085097 s + +2022-04-11 21:14:43.514992: svmem(total=99932160000, available=79792496640, percent=20.2, used=16133988352, free=51806027776, active=40330567680, inactive=5962391552, buffers=16760832, cached=31975383040, shared=2190995456, slab=756641792) +2022-04-11 21:14:43.515820: +epoch: 131 +2022-04-11 21:16:31.114635: train loss : -0.5462 +2022-04-11 21:16:43.303090: validation loss: -0.5676 +2022-04-11 21:16:43.314077: Average global foreground Dice: [0.8241, 0.9499, 0.9716, 0.3432] +2022-04-11 21:16:43.315500: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:16:44.410017: lr: 0.007589 +2022-04-11 21:16:44.411329: saving scheduled checkpoint file... +2022-04-11 21:16:44.511898: saving checkpoint... +2022-04-11 21:16:50.688790: done, saving took 6.28 seconds +2022-04-11 21:16:50.731567: done +2022-04-11 21:16:50.779109: saving checkpoint... +2022-04-11 21:16:55.918911: done, saving took 5.19 seconds +2022-04-11 21:16:55.932741: This epoch took 132.416155 s + +2022-04-11 21:16:55.934178: svmem(total=99932160000, available=79767506944, percent=20.2, used=16171446272, free=51763769344, active=40383369216, inactive=5956009984, buffers=16805888, cached=31980138496, shared=2178338816, slab=756113408) +2022-04-11 21:16:55.935044: +epoch: 132 +2022-04-11 21:18:42.839818: train loss : -0.5650 +2022-04-11 21:18:54.713657: validation loss: -0.5682 +2022-04-11 21:18:54.716144: Average global foreground Dice: [0.8084, 0.948, 0.9754, 0.3561] +2022-04-11 21:18:54.717062: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:18:55.759691: lr: 0.007571 +2022-04-11 21:18:55.761560: saving scheduled checkpoint file... +2022-04-11 21:18:55.861741: saving checkpoint... +2022-04-11 21:19:00.225384: done, saving took 4.46 seconds +2022-04-11 21:19:00.240366: done +2022-04-11 21:19:00.286987: saving checkpoint... +2022-04-11 21:19:04.596990: done, saving took 4.36 seconds +2022-04-11 21:19:04.611216: This epoch took 128.675150 s + +2022-04-11 21:19:04.612736: svmem(total=99932160000, available=79982370816, percent=20.0, used=15944769536, free=51961237504, active=40162840576, inactive=5981057024, buffers=16846848, cached=32009306112, shared=2190123008, slab=756858880) +2022-04-11 21:19:04.613890: +epoch: 133 +2022-04-11 21:20:52.092764: train loss : -0.5651 +2022-04-11 21:21:03.694431: validation loss: -0.5248 +2022-04-11 21:21:03.696703: Average global foreground Dice: [0.8031, 0.9364, 0.9645, 0.3539] +2022-04-11 21:21:03.697647: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:21:04.712862: lr: 0.007552 +2022-04-11 21:21:04.714556: saving scheduled checkpoint file... +2022-04-11 21:21:04.820223: saving checkpoint... +2022-04-11 21:21:09.402359: done, saving took 4.69 seconds +2022-04-11 21:21:09.417708: done +2022-04-11 21:21:09.464844: saving checkpoint... +2022-04-11 21:21:13.895623: done, saving took 4.48 seconds +2022-04-11 21:21:13.972281: This epoch took 129.357655 s + +2022-04-11 21:21:13.973461: svmem(total=99932160000, available=79824539648, percent=20.1, used=16102342656, free=51795472384, active=40314748928, inactive=5989150720, buffers=16904192, cached=32017440768, shared=2190422016, slab=757395456) +2022-04-11 21:21:13.974265: +epoch: 134 +2022-04-11 21:23:01.698261: train loss : -0.5306 +2022-04-11 21:23:13.905738: validation loss: -0.5272 +2022-04-11 21:23:13.914461: Average global foreground Dice: [0.7979, 0.9349, 0.9696, 0.3642] +2022-04-11 21:23:13.915860: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:23:14.688705: lr: 0.007533 +2022-04-11 21:23:14.690254: saving scheduled checkpoint file... +2022-04-11 21:23:14.771438: saving checkpoint... +2022-04-11 21:23:19.433878: done, saving took 4.74 seconds +2022-04-11 21:23:19.451841: done +2022-04-11 21:23:19.499076: saving checkpoint... +2022-04-11 21:23:24.087941: done, saving took 4.64 seconds +2022-04-11 21:23:24.102557: This epoch took 130.127509 s + +2022-04-11 21:23:24.104188: svmem(total=99932160000, available=79824515072, percent=20.1, used=16102252544, free=51777949696, active=40314826752, inactive=6006652928, buffers=16945152, cached=32035012608, shared=2190651392, slab=757846016) +2022-04-11 21:23:24.105103: +epoch: 135 +2022-04-11 21:25:13.126320: train loss : -0.5024 +2022-04-11 21:25:24.946579: validation loss: -0.5027 +2022-04-11 21:25:24.953707: Average global foreground Dice: [0.77, 0.9387, 0.9665, 0.3195] +2022-04-11 21:25:24.954811: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:25:25.862030: lr: 0.007515 +2022-04-11 21:25:25.863508: saving scheduled checkpoint file... +2022-04-11 21:25:25.967792: saving checkpoint... +2022-04-11 21:25:30.491401: done, saving took 4.63 seconds +2022-04-11 21:25:30.516736: done +2022-04-11 21:25:30.517967: This epoch took 126.411572 s + +2022-04-11 21:25:30.519060: svmem(total=99932160000, available=79975350272, percent=20.0, used=15940677632, free=51911303168, active=40153489408, inactive=6034644992, buffers=16986112, cached=32063193088, shared=2201325568, slab=757248000) +2022-04-11 21:25:30.519757: +epoch: 136 +2022-04-11 21:27:17.980317: train loss : -0.4747 +2022-04-11 21:27:29.984966: validation loss: -0.4489 +2022-04-11 21:27:29.987432: Average global foreground Dice: [0.7835, 0.9194, 0.9531, 0.3178] +2022-04-11 21:27:29.989210: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:27:30.702381: lr: 0.007496 +2022-04-11 21:27:30.703908: saving scheduled checkpoint file... +2022-04-11 21:27:30.812718: saving checkpoint... +2022-04-11 21:27:35.585744: done, saving took 4.88 seconds +2022-04-11 21:27:35.630603: done +2022-04-11 21:27:35.631701: This epoch took 125.111201 s + +2022-04-11 21:27:35.632631: svmem(total=99932160000, available=79912402944, percent=20.0, used=16003846144, free=51841679360, active=40218890240, inactive=6041051136, buffers=17022976, cached=32069611520, shared=2201026560, slab=757387264) +2022-04-11 21:27:35.633449: +epoch: 137 +2022-04-11 21:29:24.681167: train loss : -0.5181 +2022-04-11 21:29:35.947389: validation loss: -0.5371 +2022-04-11 21:29:35.957697: Average global foreground Dice: [0.807, 0.9458, 0.9703, 0.3525] +2022-04-11 21:29:35.959298: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:29:36.830333: lr: 0.007478 +2022-04-11 21:29:36.832027: saving scheduled checkpoint file... +2022-04-11 21:29:36.935519: saving checkpoint... +2022-04-11 21:29:41.503219: done, saving took 4.67 seconds +2022-04-11 21:29:41.519502: done +2022-04-11 21:29:41.520834: This epoch took 125.886187 s + +2022-04-11 21:29:41.521755: svmem(total=99932160000, available=79921238016, percent=20.0, used=16005849088, free=51833040896, active=40221085696, inactive=6047596544, buffers=17055744, cached=32076214272, shared=2190233600, slab=757682176) +2022-04-11 21:29:41.522408: +epoch: 138 +2022-04-11 21:31:29.524716: train loss : -0.4510 +2022-04-11 21:31:40.868189: validation loss: -0.3801 +2022-04-11 21:31:40.901039: Average global foreground Dice: [0.7949, 0.8918, 0.9065, 0.33] +2022-04-11 21:31:40.903100: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:31:41.721913: lr: 0.007459 +2022-04-11 21:31:41.725179: saving scheduled checkpoint file... +2022-04-11 21:31:41.816979: saving checkpoint... +2022-04-11 21:31:46.507831: done, saving took 4.78 seconds +2022-04-11 21:31:46.523321: done +2022-04-11 21:31:46.524701: This epoch took 125.001619 s + +2022-04-11 21:31:46.525765: svmem(total=99932160000, available=79546896384, percent=20.4, used=16379486208, free=51441348608, active=40590479360, inactive=6065459200, buffers=17096704, cached=32094228480, shared=2190807040, slab=757276672) +2022-04-11 21:31:46.526700: +epoch: 139 +2022-04-11 21:33:35.100183: train loss : -0.3819 +2022-04-11 21:33:47.577918: validation loss: -0.3753 +2022-04-11 21:33:47.580001: Average global foreground Dice: [0.736, 0.8833, 0.9444, 0.3013] +2022-04-11 21:33:47.581486: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:33:48.384636: lr: 0.00744 +2022-04-11 21:33:48.386192: saving scheduled checkpoint file... +2022-04-11 21:33:48.438089: saving checkpoint... +2022-04-11 21:33:53.096934: done, saving took 4.71 seconds +2022-04-11 21:33:53.172809: done +2022-04-11 21:33:53.174040: This epoch took 126.646008 s + +2022-04-11 21:33:53.174907: svmem(total=99932160000, available=79773868032, percent=20.2, used=16152555520, free=51660460032, active=40369168384, inactive=6073159680, buffers=17129472, cached=32102014976, shared=2190729216, slab=756236288) +2022-04-11 21:33:53.175925: +epoch: 140 +2022-04-11 21:35:40.010709: train loss : -0.4645 +2022-04-11 21:35:50.216168: validation loss: -0.5014 +2022-04-11 21:35:50.403498: Average global foreground Dice: [0.8042, 0.9295, 0.9645, 0.3192] +2022-04-11 21:35:50.517882: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:35:51.449842: lr: 0.007422 +2022-04-11 21:35:51.451593: saving scheduled checkpoint file... +2022-04-11 21:35:51.527930: saving checkpoint... +2022-04-11 21:35:59.111430: done, saving took 7.66 seconds +2022-04-11 21:35:59.152019: done +2022-04-11 21:35:59.153138: This epoch took 125.976548 s + +2022-04-11 21:35:59.154329: svmem(total=99932160000, available=79923646464, percent=20.0, used=15992459264, free=51793940480, active=40204369920, inactive=6099763200, buffers=17178624, cached=32128581632, shared=2201026560, slab=758153216) +2022-04-11 21:35:59.155774: +epoch: 141 +2022-04-11 21:37:47.753274: train loss : -0.4576 +2022-04-11 21:37:59.352544: validation loss: -0.0287 +2022-04-11 21:37:59.354598: Average global foreground Dice: [0.6442, 0.8123, 0.8594, 0.2908] +2022-04-11 21:37:59.355477: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:38:00.061199: lr: 0.007403 +2022-04-11 21:38:00.063031: saving scheduled checkpoint file... +2022-04-11 21:38:00.169471: saving checkpoint... +2022-04-11 21:38:07.902773: done, saving took 7.84 seconds +2022-04-11 21:38:08.296554: done +2022-04-11 21:38:08.416930: This epoch took 129.260356 s + +2022-04-11 21:38:08.446126: svmem(total=99932160000, available=79839232000, percent=20.1, used=16101228544, free=51691737088, active=40295489536, inactive=6092980224, buffers=17367040, cached=32121827328, shared=2176741376, slab=770998272) +2022-04-11 21:38:08.447021: +epoch: 142 +2022-04-11 21:39:55.635230: train loss : -0.2260 +2022-04-11 21:40:07.195573: validation loss: -0.3243 +2022-04-11 21:40:07.201645: Average global foreground Dice: [0.7166, 0.8914, 0.9312, 0.2743] +2022-04-11 21:40:07.206395: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:40:08.044576: lr: 0.007385 +2022-04-11 21:40:08.045990: saving scheduled checkpoint file... +2022-04-11 21:40:08.147435: saving checkpoint... +2022-04-11 21:40:12.748660: done, saving took 4.70 seconds +2022-04-11 21:40:12.767423: done +2022-04-11 21:40:12.768608: This epoch took 124.320751 s + +2022-04-11 21:40:12.769968: svmem(total=99932160000, available=80286715904, percent=19.7, used=15738179584, free=52131319808, active=39953616896, inactive=6016376832, buffers=17424384, cached=32045236224, shared=2092261376, slab=757735424) +2022-04-11 21:40:12.770811: +epoch: 143 +2022-04-11 21:42:00.479840: train loss : -0.3942 +2022-04-11 21:42:12.680914: validation loss: -0.4296 +2022-04-11 21:42:12.683100: Average global foreground Dice: [0.751, 0.9205, 0.9489, 0.304] +2022-04-11 21:42:12.684492: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:42:13.640208: lr: 0.007366 +2022-04-11 21:42:13.641641: saving scheduled checkpoint file... +2022-04-11 21:42:13.744266: saving checkpoint... +2022-04-11 21:42:18.259212: done, saving took 4.62 seconds +2022-04-11 21:42:18.299254: done +2022-04-11 21:42:18.300579: This epoch took 125.528951 s + +2022-04-11 21:42:18.301585: svmem(total=99932160000, available=79954305024, percent=20.0, used=15975485440, free=51781517312, active=40190058496, inactive=6128652288, buffers=17457152, cached=32157700096, shared=2187337728, slab=757907456) +2022-04-11 21:42:18.302827: +epoch: 144 +2022-04-11 21:44:07.780540: train loss : -0.4653 +2022-04-11 21:44:18.826331: validation loss: -0.4935 +2022-04-11 21:44:18.828845: Average global foreground Dice: [0.8029, 0.9309, 0.9661, 0.3212] +2022-04-11 21:44:18.830553: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:44:19.894531: lr: 0.007347 +2022-04-11 21:44:19.896322: saving scheduled checkpoint file... +2022-04-11 21:44:19.988803: saving checkpoint... +2022-04-11 21:44:24.373431: done, saving took 4.48 seconds +2022-04-11 21:44:24.388402: done +2022-04-11 21:44:24.389155: This epoch took 126.085559 s + +2022-04-11 21:44:24.390047: svmem(total=99932160000, available=79885959168, percent=20.1, used=16040169472, free=51695800320, active=40255590400, inactive=6149320704, buffers=17489920, cached=32178700288, shared=2190823424, slab=757702656) +2022-04-11 21:44:24.390774: +epoch: 145 +2022-04-11 21:46:12.035900: train loss : -0.4996 +2022-04-11 21:46:23.496224: validation loss: -0.4906 +2022-04-11 21:46:23.498451: Average global foreground Dice: [0.7947, 0.9268, 0.9593, 0.3477] +2022-04-11 21:46:23.499836: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:46:24.371293: lr: 0.007329 +2022-04-11 21:46:24.373079: saving scheduled checkpoint file... +2022-04-11 21:46:24.465700: saving checkpoint... +2022-04-11 21:46:29.133083: done, saving took 4.76 seconds +2022-04-11 21:46:29.181508: done +2022-04-11 21:46:29.182694: This epoch took 124.791179 s + +2022-04-11 21:46:29.184010: svmem(total=99932160000, available=79864479744, percent=20.1, used=16061857792, free=51667320832, active=40271335424, inactive=6156034048, buffers=17539072, cached=32185442304, shared=2190618624, slab=757104640) +2022-04-11 21:46:29.184701: +epoch: 146 +2022-04-11 21:48:16.937805: train loss : -0.4813 +2022-04-11 21:48:29.612370: validation loss: -0.4852 +2022-04-11 21:48:29.631222: Average global foreground Dice: [0.8103, 0.9335, 0.9554, 0.3237] +2022-04-11 21:48:29.633528: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:48:30.416830: lr: 0.00731 +2022-04-11 21:48:30.418432: saving scheduled checkpoint file... +2022-04-11 21:48:30.517916: saving checkpoint... +2022-04-11 21:48:35.175270: done, saving took 4.76 seconds +2022-04-11 21:48:35.190619: done +2022-04-11 21:48:35.192056: This epoch took 126.006353 s + +2022-04-11 21:48:35.193251: svmem(total=99932160000, available=79774081024, percent=20.2, used=16141721600, free=51559469056, active=40359391232, inactive=6183915520, buffers=17571840, cached=32213397504, shared=2201137152, slab=757817344) +2022-04-11 21:48:35.194070: +epoch: 147 +2022-04-11 21:50:22.899457: train loss : -0.5115 +2022-04-11 21:50:35.109645: validation loss: -0.5393 +2022-04-11 21:50:35.115769: Average global foreground Dice: [0.8185, 0.9339, 0.9729, 0.3747] +2022-04-11 21:50:35.116902: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:50:36.124082: lr: 0.007291 +2022-04-11 21:50:36.125492: saving scheduled checkpoint file... +2022-04-11 21:50:36.228222: saving checkpoint... +2022-04-11 21:50:40.699794: done, saving took 4.57 seconds +2022-04-11 21:50:40.725084: done +2022-04-11 21:50:40.726270: This epoch took 125.530883 s + +2022-04-11 21:50:40.727750: svmem(total=99932160000, available=79801237504, percent=20.1, used=16124743680, free=51569233920, active=40337842176, inactive=6190964736, buffers=17612800, cached=32220569600, shared=2190917632, slab=759164928) +2022-04-11 21:50:40.728985: +epoch: 148 +2022-04-11 21:52:29.284969: train loss : -0.5464 +2022-04-11 21:52:40.750693: validation loss: -0.5167 +2022-04-11 21:52:40.752686: Average global foreground Dice: [0.8025, 0.9351, 0.9655, 0.3627] +2022-04-11 21:52:40.753560: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:52:41.362635: lr: 0.007273 +2022-04-11 21:52:41.364096: saving scheduled checkpoint file... +2022-04-11 21:52:41.475087: saving checkpoint... +2022-04-11 21:52:45.956404: done, saving took 4.59 seconds +2022-04-11 21:52:45.972447: done +2022-04-11 21:52:45.973646: This epoch took 125.243750 s + +2022-04-11 21:52:45.974733: svmem(total=99932160000, available=80025591808, percent=19.9, used=15900487680, free=51786833920, active=40116899840, inactive=6197444608, buffers=17645568, cached=32227192832, shared=2190729216, slab=757456896) +2022-04-11 21:52:45.975593: +epoch: 149 +2022-04-11 21:54:35.229982: train loss : -0.5202 +2022-04-11 21:54:46.423778: validation loss: -0.5234 +2022-04-11 21:54:46.427048: Average global foreground Dice: [0.7706, 0.9409, 0.9672, 0.3429] +2022-04-11 21:54:46.427712: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:54:47.347523: lr: 0.007254 +2022-04-11 21:54:47.348908: saving scheduled checkpoint file... +2022-04-11 21:54:47.450845: saving checkpoint... +2022-04-11 21:54:51.964226: done, saving took 4.61 seconds +2022-04-11 21:54:52.014609: done +2022-04-11 21:54:52.015811: This epoch took 126.038991 s + +2022-04-11 21:54:52.016760: svmem(total=99932160000, available=79934775296, percent=20.0, used=15984361472, free=51678621696, active=40204427264, inactive=6221758464, buffers=17678336, cached=32251498496, shared=2197667840, slab=756101120) +2022-04-11 21:54:52.017943: +epoch: 150 +2022-04-11 21:56:40.283781: train loss : -0.4949 +2022-04-11 21:56:51.521945: validation loss: -0.4735 +2022-04-11 21:56:51.534085: Average global foreground Dice: [0.7809, 0.9236, 0.9462, 0.3613] +2022-04-11 21:56:51.535022: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:56:52.538101: lr: 0.007236 +2022-04-11 21:56:52.539627: saving scheduled checkpoint file... +2022-04-11 21:56:52.633138: saving checkpoint... +2022-04-11 21:56:57.158038: done, saving took 4.62 seconds +2022-04-11 21:56:57.174253: done +2022-04-11 21:56:57.175768: This epoch took 125.157034 s + +2022-04-11 21:56:57.177204: svmem(total=99932160000, available=79769661440, percent=20.2, used=16156344320, free=51496071168, active=40377200640, inactive=6232289280, buffers=17723392, cached=32262021120, shared=2190807040, slab=756862976) +2022-04-11 21:56:57.178431: +epoch: 151 +2022-04-11 21:58:44.708771: train loss : -0.5229 +2022-04-11 21:58:56.410976: validation loss: -0.5306 +2022-04-11 21:58:56.413164: Average global foreground Dice: [0.7743, 0.9438, 0.9695, 0.3377] +2022-04-11 21:58:56.414335: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 21:58:57.189194: lr: 0.007217 +2022-04-11 21:58:57.190645: saving scheduled checkpoint file... +2022-04-11 21:58:57.282595: saving checkpoint... +2022-04-11 21:59:01.973434: done, saving took 4.78 seconds +2022-04-11 21:59:02.027620: done +2022-04-11 21:59:02.028474: This epoch took 124.849161 s + +2022-04-11 21:59:02.029954: svmem(total=99932160000, available=79939244032, percent=20.0, used=15986946048, free=51648335872, active=40205668352, inactive=6249439232, buffers=17756160, cached=32279121920, shared=2190618624, slab=757538816) +2022-04-11 21:59:02.030770: +epoch: 152 +2022-04-11 22:00:49.695665: train loss : -0.5390 +2022-04-11 22:01:01.159581: validation loss: -0.5355 +2022-04-11 22:01:01.162816: Average global foreground Dice: [0.8187, 0.9404, 0.9675, 0.3458] +2022-04-11 22:01:01.163682: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:01:02.168259: lr: 0.007198 +2022-04-11 22:01:02.169822: saving scheduled checkpoint file... +2022-04-11 22:01:02.272351: saving checkpoint... +2022-04-11 22:01:06.995079: done, saving took 4.82 seconds +2022-04-11 22:01:07.012897: done +2022-04-11 22:01:07.014215: This epoch took 124.982555 s + +2022-04-11 22:01:07.015254: svmem(total=99932160000, available=79983411200, percent=20.0, used=15939235840, free=51685756928, active=40150675456, inactive=6259654656, buffers=17797120, cached=32289370112, shared=2194165760, slab=758685696) +2022-04-11 22:01:07.016432: +epoch: 153 +2022-04-11 22:02:55.167114: train loss : -0.5465 +2022-04-11 22:03:06.941916: validation loss: -0.5296 +2022-04-11 22:03:06.944233: Average global foreground Dice: [0.8198, 0.9374, 0.9635, 0.3407] +2022-04-11 22:03:06.945312: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:03:07.788965: lr: 0.00718 +2022-04-11 22:03:07.790520: saving scheduled checkpoint file... +2022-04-11 22:03:07.895026: saving checkpoint... +2022-04-11 22:03:12.589262: done, saving took 4.80 seconds +2022-04-11 22:03:12.606898: done +2022-04-11 22:03:12.607709: This epoch took 125.590535 s + +2022-04-11 22:03:12.608499: svmem(total=99932160000, available=79787302912, percent=20.2, used=16127569920, free=51472138240, active=40343482368, inactive=6284660736, buffers=17846272, cached=32314605568, shared=2201878528, slab=757059584) +2022-04-11 22:03:12.609471: +epoch: 154 +2022-04-11 22:05:01.346236: train loss : -0.5281 +2022-04-11 22:05:13.515411: validation loss: -0.5613 +2022-04-11 22:05:13.520426: Average global foreground Dice: [0.8229, 0.9462, 0.9728, 0.3777] +2022-04-11 22:05:13.521401: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:05:14.261509: lr: 0.007161 +2022-04-11 22:05:14.262825: saving scheduled checkpoint file... +2022-04-11 22:05:14.362633: saving checkpoint... +2022-04-11 22:05:19.272662: done, saving took 5.01 seconds +2022-04-11 22:05:19.292637: done +2022-04-11 22:05:19.294060: This epoch took 126.683963 s + +2022-04-11 22:05:19.295563: svmem(total=99932160000, available=79836463104, percent=20.1, used=16089694208, free=51503493120, active=40307068928, inactive=6290731008, buffers=18313216, cached=32320659456, shared=2190540800, slab=758775808) +2022-04-11 22:05:19.296414: +epoch: 155 +2022-04-11 22:07:07.341180: train loss : -0.5163 +2022-04-11 22:07:19.245635: validation loss: -0.5295 +2022-04-11 22:07:19.249544: Average global foreground Dice: [0.8172, 0.9233, 0.9629, 0.3777] +2022-04-11 22:07:19.251081: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:07:20.142361: lr: 0.007142 +2022-04-11 22:07:20.144107: saving scheduled checkpoint file... +2022-04-11 22:07:20.247154: saving checkpoint... +2022-04-11 22:07:24.662328: done, saving took 4.52 seconds +2022-04-11 22:07:24.676820: done +2022-04-11 22:07:24.677624: This epoch took 125.380390 s + +2022-04-11 22:07:24.678459: svmem(total=99932160000, available=79798321152, percent=20.1, used=16127528960, free=51457396736, active=40349061120, inactive=6298820608, buffers=18370560, cached=32328863744, shared=2190872576, slab=757706752) +2022-04-11 22:07:24.679217: +epoch: 156 +2022-04-11 22:09:12.874329: train loss : -0.5373 +2022-04-11 22:09:24.445862: validation loss: -0.5520 +2022-04-11 22:09:24.449238: Average global foreground Dice: [0.7985, 0.9467, 0.9698, 0.3458] +2022-04-11 22:09:24.450099: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:09:25.108667: lr: 0.007123 +2022-04-11 22:09:25.109887: saving scheduled checkpoint file... +2022-04-11 22:09:25.201091: saving checkpoint... +2022-04-11 22:09:31.632523: done, saving took 6.52 seconds +2022-04-11 22:09:31.668952: done +2022-04-11 22:09:31.670244: This epoch took 126.990330 s + +2022-04-11 22:09:31.671369: svmem(total=99932160000, available=79960698880, percent=20.0, used=15965798400, free=51602440192, active=40184156160, inactive=6315802624, buffers=18403328, cached=32345518080, shared=2190430208, slab=758603776) +2022-04-11 22:09:31.672425: +epoch: 157 +2022-04-11 22:11:19.894009: train loss : -0.5834 +2022-04-11 22:11:31.892329: validation loss: -0.5637 +2022-04-11 22:11:31.914053: Average global foreground Dice: [0.8292, 0.9418, 0.9719, 0.3763] +2022-04-11 22:11:31.915247: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:11:32.797641: lr: 0.007105 +2022-04-11 22:11:32.799300: saving scheduled checkpoint file... +2022-04-11 22:11:32.890010: saving checkpoint... +2022-04-11 22:11:37.542593: done, saving took 4.74 seconds +2022-04-11 22:11:37.552663: done +2022-04-11 22:11:37.553729: This epoch took 125.880265 s + +2022-04-11 22:11:37.554803: svmem(total=99932160000, available=79889846272, percent=20.1, used=16035799040, free=51510935552, active=40299876352, inactive=6291832832, buffers=19128320, cached=32366297088, shared=2190540800, slab=759959552) +2022-04-11 22:11:37.555472: +epoch: 158 +2022-04-11 22:13:27.463519: train loss : -0.5694 +2022-04-11 22:13:38.695833: validation loss: -0.5391 +2022-04-11 22:13:38.713940: Average global foreground Dice: [0.8342, 0.9412, 0.9681, 0.3285] +2022-04-11 22:13:38.714732: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:13:39.480015: lr: 0.007086 +2022-04-11 22:13:39.481486: saving scheduled checkpoint file... +2022-04-11 22:13:39.566732: saving checkpoint... +2022-04-11 22:13:44.095135: done, saving took 4.61 seconds +2022-04-11 22:13:44.109823: done +2022-04-11 22:13:44.159573: saving checkpoint... +2022-04-11 22:13:48.625741: done, saving took 4.51 seconds +2022-04-11 22:13:48.733307: This epoch took 131.177153 s + +2022-04-11 22:13:48.734980: svmem(total=99932160000, available=79888113664, percent=20.1, used=16037617664, free=51503005696, active=40158814208, inactive=6432497664, buffers=19542016, cached=32371994624, shared=2190442496, slab=760016896) +2022-04-11 22:13:48.736245: +epoch: 159 +2022-04-11 22:15:37.284425: train loss : -0.5400 +2022-04-11 22:15:49.057536: validation loss: -0.4191 +2022-04-11 22:15:49.060828: Average global foreground Dice: [0.7813, 0.9095, 0.9303, 0.3475] +2022-04-11 22:15:49.061717: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:15:50.040820: lr: 0.007067 +2022-04-11 22:15:50.042556: saving scheduled checkpoint file... +2022-04-11 22:15:50.160746: saving checkpoint... +2022-04-11 22:15:54.721399: done, saving took 4.68 seconds +2022-04-11 22:15:54.741020: done +2022-04-11 22:15:54.742306: This epoch took 126.005113 s + +2022-04-11 22:15:54.743720: svmem(total=99932160000, available=79946514432, percent=20.0, used=15978749952, free=51543707648, active=40219959296, inactive=6330916864, buffers=19718144, cached=32389984256, shared=2191028224, slab=758394880) +2022-04-11 22:15:54.744932: +epoch: 160 +2022-04-11 22:17:43.577647: train loss : -0.4619 +2022-04-11 22:17:55.134669: validation loss: -0.4923 +2022-04-11 22:17:55.147614: Average global foreground Dice: [0.766, 0.9423, 0.9648, 0.3163] +2022-04-11 22:17:55.148929: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:17:56.069575: lr: 0.007049 +2022-04-11 22:17:56.071227: saving scheduled checkpoint file... +2022-04-11 22:17:56.204407: saving checkpoint... +2022-04-11 22:18:00.729551: done, saving took 4.66 seconds +2022-04-11 22:18:00.749364: done +2022-04-11 22:18:00.750766: This epoch took 126.005013 s + +2022-04-11 22:18:00.751762: svmem(total=99932160000, available=80053526528, percent=19.9, used=15871688704, free=51633209344, active=40133955584, inactive=6333956096, buffers=19755008, cached=32407506944, shared=2191003648, slab=757129216) +2022-04-11 22:18:00.752669: +epoch: 161 +2022-04-11 22:19:50.233546: train loss : -0.5147 +2022-04-11 22:20:01.775558: validation loss: -0.4566 +2022-04-11 22:20:01.793706: Average global foreground Dice: [0.8019, 0.9429, 0.9247, 0.3592] +2022-04-11 22:20:01.794918: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:20:02.921880: lr: 0.00703 +2022-04-11 22:20:02.923426: saving scheduled checkpoint file... +2022-04-11 22:20:03.006549: saving checkpoint... +2022-04-11 22:20:07.316089: done, saving took 4.39 seconds +2022-04-11 22:20:07.356400: done +2022-04-11 22:20:07.357670: This epoch took 126.604143 s + +2022-04-11 22:20:07.358754: svmem(total=99932160000, available=79831961600, percent=20.1, used=16093433856, free=51403853824, active=40357650432, inactive=6340825088, buffers=19795968, cached=32415076352, shared=2190778368, slab=756649984) +2022-04-11 22:20:07.359688: +epoch: 162 +2022-04-11 22:21:56.277783: train loss : -0.5223 +2022-04-11 22:22:07.176382: validation loss: -0.5353 +2022-04-11 22:22:07.189642: Average global foreground Dice: [0.7991, 0.9354, 0.9648, 0.3654] +2022-04-11 22:22:07.190536: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:22:07.933075: lr: 0.007011 +2022-04-11 22:22:07.934873: saving scheduled checkpoint file... +2022-04-11 22:22:07.986082: saving checkpoint... +2022-04-11 22:22:12.613451: done, saving took 4.68 seconds +2022-04-11 22:22:12.628253: done +2022-04-11 22:22:12.629622: This epoch took 125.268850 s + +2022-04-11 22:22:12.630455: svmem(total=99932160000, available=79880196096, percent=20.1, used=16040898560, free=51435749376, active=40305426432, inactive=6361190400, buffers=19861504, cached=32435650560, shared=2195189760, slab=756801536) +2022-04-11 22:22:12.631177: +epoch: 163 +2022-04-11 22:24:00.126077: train loss : -0.4849 +2022-04-11 22:24:12.051640: validation loss: -0.5284 +2022-04-11 22:24:12.053983: Average global foreground Dice: [0.8103, 0.9397, 0.9663, 0.3468] +2022-04-11 22:24:12.056883: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:24:12.974863: lr: 0.006992 +2022-04-11 22:24:12.976395: saving scheduled checkpoint file... +2022-04-11 22:24:13.075961: saving checkpoint... +2022-04-11 22:24:17.847870: done, saving took 4.87 seconds +2022-04-11 22:24:17.897925: done +2022-04-11 22:24:17.898799: This epoch took 125.266423 s + +2022-04-11 22:24:17.899622: svmem(total=99932160000, available=79833632768, percent=20.1, used=16091549696, free=51371765760, active=40351576064, inactive=6374170624, buffers=19894272, cached=32448950272, shared=2190917632, slab=757932032) +2022-04-11 22:24:17.900335: +epoch: 164 +2022-04-11 22:26:05.917237: train loss : -0.5396 +2022-04-11 22:26:17.091361: validation loss: -0.5672 +2022-04-11 22:26:17.105900: Average global foreground Dice: [0.8404, 0.9507, 0.9745, 0.3468] +2022-04-11 22:26:17.107087: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:26:18.157919: lr: 0.006974 +2022-04-11 22:26:18.159625: saving scheduled checkpoint file... +2022-04-11 22:26:18.256554: saving checkpoint... +2022-04-11 22:26:22.883296: done, saving took 4.72 seconds +2022-04-11 22:26:22.899276: done +2022-04-11 22:26:22.948329: saving checkpoint... +2022-04-11 22:26:27.498603: done, saving took 4.60 seconds +2022-04-11 22:26:27.511485: This epoch took 129.610339 s + +2022-04-11 22:26:27.513080: svmem(total=99932160000, available=79897792512, percent=20.0, used=16017043456, free=51428081664, active=40276041728, inactive=6392229888, buffers=20078592, cached=32466956288, shared=2201280512, slab=758366208) +2022-04-11 22:26:27.514294: +epoch: 165 +2022-04-11 22:28:15.935628: train loss : -0.5549 +2022-04-11 22:28:27.964964: validation loss: -0.5236 +2022-04-11 22:28:27.981512: Average global foreground Dice: [0.8116, 0.9394, 0.9627, 0.3712] +2022-04-11 22:28:27.982659: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:28:28.932021: lr: 0.006955 +2022-04-11 22:28:28.933568: saving scheduled checkpoint file... +2022-04-11 22:28:29.018871: saving checkpoint... +2022-04-11 22:28:33.488590: done, saving took 4.55 seconds +2022-04-11 22:28:33.505489: done +2022-04-11 22:28:33.592145: saving checkpoint... +2022-04-11 22:28:38.076463: done, saving took 4.57 seconds +2022-04-11 22:28:38.094526: This epoch took 130.578938 s + +2022-04-11 22:28:38.096261: svmem(total=99932160000, available=79696416768, percent=20.2, used=16228986880, free=51226898432, active=40270258176, inactive=6602690560, buffers=20111360, cached=32456163328, shared=2190729216, slab=757415936) +2022-04-11 22:28:38.096987: +epoch: 166 +2022-04-11 22:30:24.579854: train loss : -0.5565 +2022-04-11 22:30:35.557540: validation loss: -0.5658 +2022-04-11 22:30:35.565600: Average global foreground Dice: [0.8401, 0.9426, 0.9721, 0.3697] +2022-04-11 22:30:35.566749: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:30:36.365955: lr: 0.006936 +2022-04-11 22:30:36.367568: saving scheduled checkpoint file... +2022-04-11 22:30:36.476501: saving checkpoint... +2022-04-11 22:30:41.039960: done, saving took 4.67 seconds +2022-04-11 22:30:41.111797: done +2022-04-11 22:30:41.211794: saving checkpoint... +2022-04-11 22:30:45.872464: done, saving took 4.76 seconds +2022-04-11 22:30:45.893575: This epoch took 127.795802 s + +2022-04-11 22:30:45.894895: svmem(total=99932160000, available=79714807808, percent=20.2, used=16220446720, free=51212861440, active=40390934528, inactive=6493773824, buffers=20152320, cached=32478699520, shared=2180816896, slab=756989952) +2022-04-11 22:30:45.895544: +epoch: 167 +2022-04-11 22:32:34.785051: train loss : -0.5681 +2022-04-11 22:32:45.903262: validation loss: -0.5893 +2022-04-11 22:32:45.905558: Average global foreground Dice: [0.8354, 0.9545, 0.9776, 0.3593] +2022-04-11 22:32:45.906882: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:32:46.901184: lr: 0.006918 +2022-04-11 22:32:46.902334: saving scheduled checkpoint file... +2022-04-11 22:32:46.998488: saving checkpoint... +2022-04-11 22:32:51.407670: done, saving took 4.50 seconds +2022-04-11 22:32:51.421583: done +2022-04-11 22:32:51.469658: saving checkpoint... +2022-04-11 22:32:55.821306: done, saving took 4.40 seconds +2022-04-11 22:32:55.836045: This epoch took 129.939889 s + +2022-04-11 22:32:55.837022: svmem(total=99932160000, available=79618027520, percent=20.3, used=16318488576, free=51108311040, active=40555663360, inactive=6437396480, buffers=20185088, cached=32485175296, shared=2179518464, slab=758063104) +2022-04-11 22:32:55.838089: +epoch: 168 +2022-04-11 22:34:42.919347: train loss : -0.5274 +2022-04-11 22:34:54.405205: validation loss: -0.4796 +2022-04-11 22:34:54.409760: Average global foreground Dice: [0.8069, 0.9319, 0.9502, 0.3473] +2022-04-11 22:34:54.411025: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:34:55.439341: lr: 0.006899 +2022-04-11 22:34:55.441220: saving scheduled checkpoint file... +2022-04-11 22:34:55.531131: saving checkpoint... +2022-04-11 22:35:00.368304: done, saving took 4.93 seconds +2022-04-11 22:35:00.394378: done +2022-04-11 22:35:00.396176: This epoch took 124.557326 s + +2022-04-11 22:35:00.397346: svmem(total=99932160000, available=79748419584, percent=20.2, used=16176881664, free=51221274624, active=40435142656, inactive=6447009792, buffers=20217856, cached=32513785856, shared=2190729216, slab=759287808) +2022-04-11 22:35:00.398464: +epoch: 169 +2022-04-11 22:36:48.779794: train loss : -0.5358 +2022-04-11 22:37:00.481377: validation loss: -0.4321 +2022-04-11 22:37:00.483890: Average global foreground Dice: [0.8123, 0.9203, 0.9279, 0.3443] +2022-04-11 22:37:00.485506: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:37:01.858908: lr: 0.00688 +2022-04-11 22:37:01.886560: saving scheduled checkpoint file... +2022-04-11 22:37:01.963650: saving checkpoint... +2022-04-11 22:37:06.502567: done, saving took 4.61 seconds +2022-04-11 22:37:06.538090: done +2022-04-11 22:37:06.539865: This epoch took 126.140580 s + +2022-04-11 22:37:06.541317: svmem(total=99932160000, available=79847579648, percent=20.1, used=16078299136, free=51303903232, active=40329998336, inactive=6454693888, buffers=20344832, cached=32529612800, shared=2190233600, slab=759332864) +2022-04-11 22:37:06.542517: +epoch: 170 +2022-04-11 22:38:55.384640: train loss : -0.5430 +2022-04-11 22:39:07.426768: validation loss: -0.5706 +2022-04-11 22:39:07.437544: Average global foreground Dice: [0.8281, 0.9477, 0.9749, 0.3694] +2022-04-11 22:39:07.441832: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:39:08.123271: lr: 0.006861 +2022-04-11 22:39:08.124672: saving scheduled checkpoint file... +2022-04-11 22:39:08.203095: saving checkpoint... +2022-04-11 22:39:12.800106: done, saving took 4.67 seconds +2022-04-11 22:39:12.810413: done +2022-04-11 22:39:12.812039: This epoch took 126.267532 s + +2022-04-11 22:39:12.813131: svmem(total=99932160000, available=79687356416, percent=20.3, used=16231534592, free=51135803392, active=40493584384, inactive=6469595136, buffers=20393984, cached=32544428032, shared=2197250048, slab=759570432) +2022-04-11 22:39:12.814333: +epoch: 171 +2022-04-11 22:41:00.873872: train loss : -0.5525 +2022-04-11 22:41:12.210852: validation loss: -0.5482 +2022-04-11 22:41:12.219314: Average global foreground Dice: [0.8326, 0.9371, 0.9696, 0.3431] +2022-04-11 22:41:12.223754: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:41:13.065002: lr: 0.006842 +2022-04-11 22:41:13.066410: saving scheduled checkpoint file... +2022-04-11 22:41:13.150715: saving checkpoint... +2022-04-11 22:41:17.568698: done, saving took 4.50 seconds +2022-04-11 22:41:17.642806: done +2022-04-11 22:41:17.723094: saving checkpoint... +2022-04-11 22:41:22.259853: done, saving took 4.62 seconds +2022-04-11 22:41:22.324455: This epoch took 129.509341 s + +2022-04-11 22:41:22.326522: svmem(total=99932160000, available=79691624448, percent=20.3, used=16223436800, free=51122651136, active=40484982784, inactive=6490558464, buffers=20434944, cached=32565637120, shared=2200944640, slab=759267328) +2022-04-11 22:41:22.327507: +epoch: 172 +2022-04-11 22:43:09.214434: train loss : -0.5756 +2022-04-11 22:43:20.963263: validation loss: -0.5955 +2022-04-11 22:43:20.965136: Average global foreground Dice: [0.8231, 0.9526, 0.9742, 0.3623] +2022-04-11 22:43:20.985018: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:43:21.848899: lr: 0.006824 +2022-04-11 22:43:21.850573: saving scheduled checkpoint file... +2022-04-11 22:43:21.979004: saving checkpoint... +2022-04-11 22:43:26.665274: done, saving took 4.81 seconds +2022-04-11 22:43:26.701726: done +2022-04-11 22:43:26.798426: saving checkpoint... +2022-04-11 22:43:31.475561: done, saving took 4.77 seconds +2022-04-11 22:43:31.493155: This epoch took 129.164773 s + +2022-04-11 22:43:31.495094: svmem(total=99932160000, available=79748820992, percent=20.2, used=16176263168, free=51162406912, active=40432398336, inactive=6497898496, buffers=20467712, cached=32573022208, shared=2190917632, slab=759926784) +2022-04-11 22:43:31.496503: +epoch: 173 +2022-04-11 22:45:19.449238: train loss : -0.5482 +2022-04-11 22:45:30.716166: validation loss: -0.5877 +2022-04-11 22:45:30.720838: Average global foreground Dice: [0.8272, 0.9485, 0.9773, 0.3922] +2022-04-11 22:45:30.722580: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:45:31.684291: lr: 0.006805 +2022-04-11 22:45:31.685746: saving scheduled checkpoint file... +2022-04-11 22:45:31.778590: saving checkpoint... +2022-04-11 22:45:36.761122: done, saving took 5.07 seconds +2022-04-11 22:45:36.807727: done +2022-04-11 22:45:36.858448: saving checkpoint... +2022-04-11 22:45:41.623799: done, saving took 4.81 seconds +2022-04-11 22:45:41.638238: This epoch took 130.140543 s + +2022-04-11 22:45:41.639980: svmem(total=99932160000, available=79791378432, percent=20.2, used=16133718016, free=51187593216, active=40388222976, inactive=6515015680, buffers=20512768, cached=32590336000, shared=2190729216, slab=760045568) +2022-04-11 22:45:41.640955: +epoch: 174 +2022-04-11 22:47:29.379114: train loss : -0.5453 +2022-04-11 22:47:41.087460: validation loss: -0.5390 +2022-04-11 22:47:41.090132: Average global foreground Dice: [0.8036, 0.933, 0.97, 0.3826] +2022-04-11 22:47:41.091020: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:47:41.988122: lr: 0.006786 +2022-04-11 22:47:41.989783: saving scheduled checkpoint file... +2022-04-11 22:47:42.081303: saving checkpoint... +2022-04-11 22:47:46.189019: done, saving took 4.20 seconds +2022-04-11 22:47:46.269182: done +2022-04-11 22:47:46.315801: saving checkpoint... +2022-04-11 22:47:50.644789: done, saving took 4.37 seconds +2022-04-11 22:47:50.665029: This epoch took 129.023102 s + +2022-04-11 22:47:50.667107: svmem(total=99932160000, available=79755591680, percent=20.2, used=16169365504, free=51143974912, active=40430501888, inactive=6522765312, buffers=20680704, cached=32598138880, shared=2190839808, slab=758722560) +2022-04-11 22:47:50.667997: +epoch: 175 +2022-04-11 22:49:38.045120: train loss : -0.5759 +2022-04-11 22:49:50.271006: validation loss: -0.5654 +2022-04-11 22:49:50.274939: Average global foreground Dice: [0.8206, 0.9469, 0.9737, 0.3504] +2022-04-11 22:49:50.276396: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:49:51.096717: lr: 0.006767 +2022-04-11 22:49:51.098031: saving scheduled checkpoint file... +2022-04-11 22:49:51.188581: saving checkpoint... +2022-04-11 22:49:55.658264: done, saving took 4.56 seconds +2022-04-11 22:49:55.667511: done +2022-04-11 22:49:55.715111: saving checkpoint... +2022-04-11 22:50:00.357729: done, saving took 4.69 seconds +2022-04-11 22:50:00.372189: This epoch took 129.703195 s + +2022-04-11 22:50:00.373803: svmem(total=99932160000, available=79888936960, percent=20.1, used=16029265920, free=51259711488, active=40291028992, inactive=6547058688, buffers=20713472, cached=32622469120, shared=2197667840, slab=757760000) +2022-04-11 22:50:00.374620: +epoch: 176 +2022-04-11 22:51:48.584084: train loss : -0.5801 +2022-04-11 22:52:00.635492: validation loss: -0.5646 +2022-04-11 22:52:00.646538: Average global foreground Dice: [0.8345, 0.9408, 0.9653, 0.3814] +2022-04-11 22:52:00.647816: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:52:01.289296: lr: 0.006749 +2022-04-11 22:52:01.290959: saving scheduled checkpoint file... +2022-04-11 22:52:01.376214: saving checkpoint... +2022-04-11 22:52:06.500438: done, saving took 5.21 seconds +2022-04-11 22:52:06.513613: done +2022-04-11 22:52:06.561297: saving checkpoint... +2022-04-11 22:52:11.412155: done, saving took 4.90 seconds +2022-04-11 22:52:11.451127: This epoch took 131.075688 s + +2022-04-11 22:52:11.455632: svmem(total=99932160000, available=79939895296, percent=20.0, used=15985324032, free=51293077504, active=40237162496, inactive=6557663232, buffers=20770816, cached=32632987648, shared=2190729216, slab=758157312) +2022-04-11 22:52:11.458633: +epoch: 177 +2022-04-11 22:53:59.982776: train loss : -0.5893 +2022-04-11 22:54:11.816523: validation loss: -0.5534 +2022-04-11 22:54:11.821487: Average global foreground Dice: [0.8263, 0.9458, 0.9698, 0.3722] +2022-04-11 22:54:11.822953: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:54:12.726115: lr: 0.00673 +2022-04-11 22:54:12.727579: saving scheduled checkpoint file... +2022-04-11 22:54:12.787352: saving checkpoint... +2022-04-11 22:54:17.470256: done, saving took 4.74 seconds +2022-04-11 22:54:17.488565: done +2022-04-11 22:54:17.568759: saving checkpoint... +2022-04-11 22:54:22.272171: done, saving took 4.78 seconds +2022-04-11 22:54:22.290461: This epoch took 130.826270 s + +2022-04-11 22:54:22.292378: svmem(total=99932160000, available=79776866304, percent=20.2, used=16138371072, free=51122307072, active=40395198464, inactive=6575460352, buffers=20803584, cached=32650678272, shared=2200764416, slab=759074816) +2022-04-11 22:54:22.293705: +epoch: 178 +2022-04-11 22:56:09.238039: train loss : -0.5582 +2022-04-11 22:56:21.132097: validation loss: -0.3384 +2022-04-11 22:56:21.153787: Average global foreground Dice: [0.7942, 0.8914, 0.874, 0.3602] +2022-04-11 22:56:21.156303: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:56:21.898084: lr: 0.006711 +2022-04-11 22:56:21.899777: saving scheduled checkpoint file... +2022-04-11 22:56:22.000243: saving checkpoint... +2022-04-11 22:56:26.537565: done, saving took 4.64 seconds +2022-04-11 22:56:26.562324: done +2022-04-11 22:56:26.563505: This epoch took 124.268902 s + +2022-04-11 22:56:26.564904: svmem(total=99932160000, available=79753977856, percent=20.2, used=16171151360, free=51082129408, active=40436314112, inactive=6582542336, buffers=20852736, cached=32658026496, shared=2190651392, slab=758845440) +2022-04-11 22:56:26.566095: +epoch: 179 +2022-04-11 22:58:15.405078: train loss : -0.4267 +2022-04-11 22:58:27.331038: validation loss: -0.4926 +2022-04-11 22:58:27.339982: Average global foreground Dice: [0.77, 0.9305, 0.9561, 0.3602] +2022-04-11 22:58:27.341779: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 22:58:28.342693: lr: 0.006692 +2022-04-11 22:58:28.344279: saving scheduled checkpoint file... +2022-04-11 22:58:28.447644: saving checkpoint... +2022-04-11 22:58:33.243463: done, saving took 4.90 seconds +2022-04-11 22:58:33.288682: done +2022-04-11 22:58:33.289912: This epoch took 126.722930 s + +2022-04-11 22:58:33.291040: svmem(total=99932160000, available=79629877248, percent=20.3, used=16289906688, free=50941513728, active=40559198208, inactive=6604427264, buffers=20885504, cached=32679854080, shared=2196070400, slab=758870016) +2022-04-11 22:58:33.292248: +epoch: 180 +2022-04-11 23:00:22.028975: train loss : -0.4926 +2022-04-11 23:00:33.252373: validation loss: -0.5157 +2022-04-11 23:00:33.254438: Average global foreground Dice: [0.7941, 0.9459, 0.9612, 0.352] +2022-04-11 23:00:33.255308: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:00:34.085655: lr: 0.006673 +2022-04-11 23:00:34.087111: saving scheduled checkpoint file... +2022-04-11 23:00:34.176966: saving checkpoint... +2022-04-11 23:00:38.742568: done, saving took 4.65 seconds +2022-04-11 23:00:38.752164: done +2022-04-11 23:00:38.753510: This epoch took 125.460555 s + +2022-04-11 23:00:38.755036: svmem(total=99932160000, available=79719571456, percent=20.2, used=16205713408, free=51023425536, active=40470818816, inactive=6606532608, buffers=20926464, cached=32682094592, shared=2190536704, slab=758796288) +2022-04-11 23:00:38.755965: +epoch: 181 +2022-04-11 23:02:26.594521: train loss : -0.5334 +2022-04-11 23:02:37.077895: validation loss: -0.5274 +2022-04-11 23:02:37.085528: Average global foreground Dice: [0.7899, 0.9322, 0.9713, 0.3724] +2022-04-11 23:02:37.086942: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:02:38.252915: lr: 0.006654 +2022-04-11 23:02:38.254277: saving scheduled checkpoint file... +2022-04-11 23:02:38.344573: saving checkpoint... +2022-04-11 23:02:43.260018: done, saving took 5.00 seconds +2022-04-11 23:02:43.274444: done +2022-04-11 23:02:43.275408: This epoch took 124.518523 s + +2022-04-11 23:02:43.276458: svmem(total=99932160000, available=79861514240, percent=20.1, used=16063381504, free=51146682368, active=40328687616, inactive=6624043008, buffers=21688320, cached=32700407808, shared=2190540800, slab=759939072) +2022-04-11 23:02:43.277585: +epoch: 182 +2022-04-11 23:04:32.668093: train loss : -0.5750 +2022-04-11 23:04:44.122659: validation loss: -0.5512 +2022-04-11 23:04:44.275725: Average global foreground Dice: [0.8334, 0.9476, 0.9675, 0.3408] +2022-04-11 23:04:44.460339: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:04:45.523410: lr: 0.006636 +2022-04-11 23:04:45.525416: saving scheduled checkpoint file... +2022-04-11 23:04:45.635274: saving checkpoint... +2022-04-11 23:04:50.658245: done, saving took 5.13 seconds +2022-04-11 23:04:50.670093: done +2022-04-11 23:04:50.671029: This epoch took 127.392621 s + +2022-04-11 23:04:50.672102: svmem(total=99932160000, available=79875440640, percent=20.1, used=16049348608, free=51143266304, active=40315265024, inactive=6641971200, buffers=21721088, cached=32717824000, shared=2190680064, slab=760598528) +2022-04-11 23:04:50.672930: +epoch: 183 +2022-04-11 23:06:38.770791: train loss : -0.5623 +2022-04-11 23:06:50.598977: validation loss: -0.5663 +2022-04-11 23:06:50.790482: Average global foreground Dice: [0.8409, 0.9544, 0.9772, 0.3742] +2022-04-11 23:06:50.953090: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:06:51.928452: lr: 0.006617 +2022-04-11 23:06:52.047035: saving scheduled checkpoint file... +2022-04-11 23:06:52.246907: saving checkpoint... +2022-04-11 23:06:57.874634: done, saving took 5.75 seconds +2022-04-11 23:06:57.893831: done +2022-04-11 23:06:57.894895: This epoch took 127.220720 s + +2022-04-11 23:06:57.895890: svmem(total=99932160000, available=79871991808, percent=20.1, used=16052682752, free=51132932096, active=40316289024, inactive=6648299520, buffers=21774336, cached=32724770816, shared=2190761984, slab=759914496) +2022-04-11 23:06:57.896801: +epoch: 184 +2022-04-11 23:08:45.435091: train loss : -0.5426 +2022-04-11 23:08:57.999454: validation loss: -0.5250 +2022-04-11 23:08:58.003702: Average global foreground Dice: [0.827, 0.9386, 0.9623, 0.3628] +2022-04-11 23:08:58.004516: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:08:58.753264: lr: 0.006598 +2022-04-11 23:08:58.877284: saving scheduled checkpoint file... +2022-04-11 23:08:59.109495: saving checkpoint... +2022-04-11 23:09:06.229976: done, saving took 7.23 seconds +2022-04-11 23:09:06.266607: done +2022-04-11 23:09:06.268120: This epoch took 128.370607 s + +2022-04-11 23:09:06.269703: svmem(total=99932160000, available=79962247168, percent=20.0, used=15955468288, free=51205722112, active=40211349504, inactive=6672613376, buffers=21811200, cached=32749158400, shared=2197667840, slab=761135104) +2022-04-11 23:09:06.270672: +epoch: 185 +2022-04-11 23:10:56.097113: train loss : -0.5583 +2022-04-11 23:11:08.223756: validation loss: -0.5648 +2022-04-11 23:11:08.354115: Average global foreground Dice: [0.8099, 0.9413, 0.9748, 0.3697] +2022-04-11 23:11:08.460745: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:11:09.353227: lr: 0.006579 +2022-04-11 23:11:09.354695: saving scheduled checkpoint file... +2022-04-11 23:11:09.443814: saving checkpoint... +2022-04-11 23:11:16.964535: done, saving took 7.61 seconds +2022-04-11 23:11:17.014838: done +2022-04-11 23:11:17.016420: This epoch took 130.744923 s + +2022-04-11 23:11:17.017329: svmem(total=99932160000, available=79888793600, percent=20.1, used=16035897344, free=51115069440, active=40299859968, inactive=6682832896, buffers=21860352, cached=32759332864, shared=2190708736, slab=761651200) +2022-04-11 23:11:17.018090: +epoch: 186 +2022-04-11 23:13:05.290071: train loss : -0.5747 +2022-04-11 23:13:16.870252: validation loss: -0.5445 +2022-04-11 23:13:16.874313: Average global foreground Dice: [0.8257, 0.941, 0.9587, 0.3482] +2022-04-11 23:13:16.876180: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:13:17.711463: lr: 0.00656 +2022-04-11 23:13:17.712958: saving scheduled checkpoint file... +2022-04-11 23:13:17.794027: saving checkpoint... +2022-04-11 23:13:22.912239: done, saving took 5.20 seconds +2022-04-11 23:13:23.234438: done +2022-04-11 23:13:23.368366: This epoch took 126.349592 s + +2022-04-11 23:13:23.485132: svmem(total=99932160000, available=79793606656, percent=20.2, used=16145092608, free=51011936256, active=40397160448, inactive=6676774912, buffers=21893120, cached=32753238016, shared=2176741376, slab=774635520) +2022-04-11 23:13:23.617553: +epoch: 187 +2022-04-11 23:15:11.679918: train loss : -0.5833 +2022-04-11 23:15:23.416504: validation loss: -0.5742 +2022-04-11 23:15:23.418904: Average global foreground Dice: [0.856, 0.9429, 0.9728, 0.3553] +2022-04-11 23:15:23.420430: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:15:24.289330: lr: 0.006541 +2022-04-11 23:15:24.292245: saving scheduled checkpoint file... +2022-04-11 23:15:24.343449: saving checkpoint... +2022-04-11 23:15:28.934419: done, saving took 4.64 seconds +2022-04-11 23:15:28.948432: done +2022-04-11 23:15:28.949607: This epoch took 125.206925 s + +2022-04-11 23:15:28.950837: svmem(total=99932160000, available=80070447104, percent=19.9, used=15892537344, free=51271258112, active=40160407552, inactive=6669815808, buffers=21934080, cached=32746430464, shared=2152378368, slab=761200640) +2022-04-11 23:15:28.951659: +epoch: 188 +2022-04-11 23:17:16.083990: train loss : -0.5555 +2022-04-11 23:17:27.570617: validation loss: -0.5436 +2022-04-11 23:17:27.576665: Average global foreground Dice: [0.8035, 0.9516, 0.9665, 0.3251] +2022-04-11 23:17:27.577523: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:17:28.446568: lr: 0.006522 +2022-04-11 23:17:28.447631: saving scheduled checkpoint file... +2022-04-11 23:17:28.551086: saving checkpoint... +2022-04-11 23:17:33.400595: done, saving took 4.95 seconds +2022-04-11 23:17:33.469009: done +2022-04-11 23:17:33.470812: This epoch took 124.517637 s + +2022-04-11 23:17:33.471898: svmem(total=99932160000, available=79853277184, percent=20.1, used=16077836288, free=51036782592, active=40340201472, inactive=6718746624, buffers=21975040, cached=32795566080, shared=2184482816, slab=761552896) +2022-04-11 23:17:33.472897: +epoch: 189 +2022-04-11 23:19:21.813250: train loss : -0.5572 +2022-04-11 23:19:32.766563: validation loss: -0.5251 +2022-04-11 23:19:33.321151: Average global foreground Dice: [0.8138, 0.9382, 0.9708, 0.3626] +2022-04-11 23:19:33.322963: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:19:33.927094: lr: 0.006504 +2022-04-11 23:19:33.928705: saving scheduled checkpoint file... +2022-04-11 23:19:34.002368: saving checkpoint... +2022-04-11 23:19:38.539902: done, saving took 4.61 seconds +2022-04-11 23:19:38.560063: done +2022-04-11 23:19:38.561126: This epoch took 125.086894 s + +2022-04-11 23:19:38.562346: svmem(total=99932160000, available=80054136832, percent=19.9, used=15860047872, free=51230785536, active=40125177856, inactive=6742560768, buffers=22016000, cached=32819310592, shared=2201137152, slab=760877056) +2022-04-11 23:19:38.563285: +epoch: 190 +2022-04-11 23:21:25.656339: train loss : -0.5667 +2022-04-11 23:21:37.411779: validation loss: -0.5493 +2022-04-11 23:21:37.413957: Average global foreground Dice: [0.8332, 0.9451, 0.9621, 0.3372] +2022-04-11 23:21:37.421280: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:21:38.354250: lr: 0.006485 +2022-04-11 23:21:38.355822: saving scheduled checkpoint file... +2022-04-11 23:21:38.484969: saving checkpoint... +2022-04-11 23:21:43.217275: done, saving took 4.86 seconds +2022-04-11 23:21:43.236460: done +2022-04-11 23:21:43.237867: This epoch took 124.673752 s + +2022-04-11 23:21:43.238961: svmem(total=99932160000, available=80046522368, percent=19.9, used=15877857280, free=51205791744, active=40138465280, inactive=6749519872, buffers=22073344, cached=32826437632, shared=2190839808, slab=760377344) +2022-04-11 23:21:43.239932: +epoch: 191 +2022-04-11 23:23:32.935145: train loss : -0.5777 +2022-04-11 23:23:45.428031: validation loss: -0.5791 +2022-04-11 23:23:45.432593: Average global foreground Dice: [0.8526, 0.9493, 0.969, 0.3348] +2022-04-11 23:23:45.433552: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:23:46.265136: lr: 0.006466 +2022-04-11 23:23:46.266978: saving scheduled checkpoint file... +2022-04-11 23:23:46.363771: saving checkpoint... +2022-04-11 23:23:50.733979: done, saving took 4.47 seconds +2022-04-11 23:23:50.748633: done +2022-04-11 23:23:50.750452: This epoch took 127.509643 s + +2022-04-11 23:23:50.751780: svmem(total=99932160000, available=80043311104, percent=19.9, used=15881441280, free=51185049600, active=40146362368, inactive=6766505984, buffers=22106112, cached=32843563008, shared=2190454784, slab=760090624) +2022-04-11 23:23:50.752642: +epoch: 192 +2022-04-11 23:25:38.321228: train loss : -0.5997 +2022-04-11 23:25:50.001873: validation loss: -0.5877 +2022-04-11 23:25:50.004734: Average global foreground Dice: [0.8309, 0.9543, 0.9752, 0.3626] +2022-04-11 23:25:50.006681: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:25:50.915499: lr: 0.006447 +2022-04-11 23:25:50.919012: saving scheduled checkpoint file... +2022-04-11 23:25:51.013432: saving checkpoint... +2022-04-11 23:25:55.751878: done, saving took 4.83 seconds +2022-04-11 23:25:55.819729: done +2022-04-11 23:25:55.820567: This epoch took 125.067196 s + +2022-04-11 23:25:55.821913: svmem(total=99932160000, available=79896801280, percent=20.0, used=16041644032, free=51031797760, active=40308760576, inactive=6759432192, buffers=22167552, cached=32836550656, shared=2176663552, slab=759783424) +2022-04-11 23:25:55.822594: +epoch: 193 +2022-04-11 23:27:42.745920: train loss : -0.5952 +2022-04-11 23:27:54.188924: validation loss: -0.6004 +2022-04-11 23:27:54.193193: Average global foreground Dice: [0.8091, 0.9501, 0.9811, 0.3983] +2022-04-11 23:27:54.194471: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:27:55.097483: lr: 0.006428 +2022-04-11 23:27:55.099255: saving scheduled checkpoint file... +2022-04-11 23:27:55.193472: saving checkpoint... +2022-04-11 23:27:59.813965: done, saving took 4.71 seconds +2022-04-11 23:27:59.828375: done +2022-04-11 23:27:59.881421: saving checkpoint... +2022-04-11 23:28:04.501865: done, saving took 4.67 seconds +2022-04-11 23:28:04.515111: This epoch took 128.691883 s + +2022-04-11 23:28:04.516723: svmem(total=99932160000, available=79860793344, percent=20.1, used=16063909888, free=50978271232, active=40333848576, inactive=6790750208, buffers=22200320, cached=32867778560, shared=2190536704, slab=761053184) +2022-04-11 23:28:04.517590: +epoch: 194 +2022-04-11 23:29:52.240149: train loss : -0.5956 +2022-04-11 23:30:03.896774: validation loss: -0.5862 +2022-04-11 23:30:03.917967: Average global foreground Dice: [0.8291, 0.9539, 0.975, 0.3739] +2022-04-11 23:30:03.919472: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:30:05.040895: lr: 0.006409 +2022-04-11 23:30:05.042677: saving scheduled checkpoint file... +2022-04-11 23:30:05.136015: saving checkpoint... +2022-04-11 23:30:09.612947: done, saving took 4.57 seconds +2022-04-11 23:30:09.627426: done +2022-04-11 23:30:09.674609: saving checkpoint... +2022-04-11 23:30:14.117743: done, saving took 4.49 seconds +2022-04-11 23:30:14.136192: This epoch took 129.617671 s + +2022-04-11 23:30:14.138612: svmem(total=99932160000, available=80102510592, percent=19.8, used=15821357056, free=51202502656, active=40083034112, inactive=6808907776, buffers=22257664, cached=32886042624, shared=2191269888, slab=760926208) +2022-04-11 23:30:14.140403: +epoch: 195 +2022-04-11 23:32:03.103889: train loss : -0.5818 +2022-04-11 23:32:14.752826: validation loss: -0.5873 +2022-04-11 23:32:14.762978: Average global foreground Dice: [0.8354, 0.9535, 0.9705, 0.3377] +2022-04-11 23:32:14.766511: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:32:15.715348: lr: 0.00639 +2022-04-11 23:32:15.717327: saving scheduled checkpoint file... +2022-04-11 23:32:15.825065: saving checkpoint... +2022-04-11 23:32:20.431417: done, saving took 4.71 seconds +2022-04-11 23:32:20.450718: done +2022-04-11 23:32:20.532273: saving checkpoint... +2022-04-11 23:32:25.464749: done, saving took 5.01 seconds +2022-04-11 23:32:25.482119: This epoch took 131.340417 s + +2022-04-11 23:32:25.483600: svmem(total=99932160000, available=80158904320, percent=19.8, used=15813853184, free=51241553920, active=40079302656, inactive=6777204736, buffers=22290432, cached=32854462464, shared=2142277632, slab=761446400) +2022-04-11 23:32:25.484397: +epoch: 196 +2022-04-11 23:34:13.942765: train loss : -0.5847 +2022-04-11 23:34:24.365925: validation loss: -0.5851 +2022-04-11 23:34:24.367762: Average global foreground Dice: [0.8182, 0.9551, 0.971, 0.36] +2022-04-11 23:34:24.368712: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:34:25.349876: lr: 0.006371 +2022-04-11 23:34:25.351735: saving scheduled checkpoint file... +2022-04-11 23:34:25.471970: saving checkpoint... +2022-04-11 23:34:30.155524: done, saving took 4.80 seconds +2022-04-11 23:34:30.175534: done +2022-04-11 23:34:30.271586: saving checkpoint... +2022-04-11 23:34:34.743408: done, saving took 4.57 seconds +2022-04-11 23:34:34.760741: This epoch took 129.275510 s + +2022-04-11 23:34:34.762778: svmem(total=99932160000, available=80191836160, percent=19.8, used=15747239936, free=51266555904, active=40014389248, inactive=6818791424, buffers=22323200, cached=32896040960, shared=2175954944, slab=760205312) +2022-04-11 23:34:34.763964: +epoch: 197 +2022-04-11 23:36:21.564703: train loss : -0.5980 +2022-04-11 23:36:32.958031: validation loss: -0.5847 +2022-04-11 23:36:32.960493: Average global foreground Dice: [0.8267, 0.9544, 0.9756, 0.3714] +2022-04-11 23:36:32.965538: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:36:33.656419: lr: 0.006352 +2022-04-11 23:36:33.658280: saving scheduled checkpoint file... +2022-04-11 23:36:33.748771: saving checkpoint... +2022-04-11 23:36:38.429733: done, saving took 4.77 seconds +2022-04-11 23:36:38.444470: done +2022-04-11 23:36:38.491870: saving checkpoint... +2022-04-11 23:36:43.171377: done, saving took 4.73 seconds +2022-04-11 23:36:43.185005: This epoch took 128.420102 s + +2022-04-11 23:36:43.186150: svmem(total=99932160000, available=80028311552, percent=19.9, used=15909785600, free=51085582336, active=40174125056, inactive=6836748288, buffers=22380544, cached=32914411520, shared=2176741376, slab=759832576) +2022-04-11 23:36:43.187042: +epoch: 198 +2022-04-11 23:38:31.530949: train loss : -0.5841 +2022-04-11 23:38:43.249017: validation loss: -0.5909 +2022-04-11 23:38:43.253134: Average global foreground Dice: [0.8389, 0.9563, 0.9784, 0.3274] +2022-04-11 23:38:43.255289: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:38:44.189079: lr: 0.006333 +2022-04-11 23:38:44.190132: saving scheduled checkpoint file... +2022-04-11 23:38:44.284307: saving checkpoint... +2022-04-11 23:38:48.915104: done, saving took 4.72 seconds +2022-04-11 23:38:48.954463: done +2022-04-11 23:38:49.003728: saving checkpoint... +2022-04-11 23:38:53.969619: done, saving took 5.01 seconds +2022-04-11 23:38:53.984111: This epoch took 130.796279 s + +2022-04-11 23:38:53.985935: svmem(total=99932160000, available=79777079296, percent=20.2, used=16161124352, free=50908508160, active=40421335040, inactive=6762602496, buffers=22417408, cached=32840110080, shared=2176741376, slab=769855488) +2022-04-11 23:38:53.986748: +epoch: 199 +2022-04-11 23:40:41.883605: train loss : -0.6029 +2022-04-11 23:40:52.802619: validation loss: -0.5678 +2022-04-11 23:40:52.805362: Average global foreground Dice: [0.8421, 0.9455, 0.9697, 0.3481] +2022-04-11 23:40:52.806493: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:40:53.654647: lr: 0.006314 +2022-04-11 23:40:53.656190: saving scheduled checkpoint file... +2022-04-11 23:40:53.749395: saving checkpoint... +2022-04-11 23:40:58.455316: done, saving took 4.80 seconds +2022-04-11 23:40:58.479677: done +2022-04-11 23:40:58.526403: saving checkpoint... +2022-04-11 23:41:03.531995: done, saving took 5.05 seconds +2022-04-11 23:41:03.612176: This epoch took 129.624514 s + +2022-04-11 23:41:03.613731: svmem(total=99932160000, available=80023531520, percent=19.9, used=15914708992, free=51055484928, active=40183771136, inactive=6861938688, buffers=22458368, cached=32939507712, shared=2176741376, slab=759218176) +2022-04-11 23:41:03.615057: +epoch: 200 +2022-04-11 23:42:51.927370: train loss : -0.5889 +2022-04-11 23:43:03.155850: validation loss: -0.5965 +2022-04-11 23:43:03.158812: Average global foreground Dice: [0.8386, 0.9503, 0.9708, 0.3592] +2022-04-11 23:43:03.162202: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:43:04.021024: lr: 0.006296 +2022-04-11 23:43:04.022254: saving scheduled checkpoint file... +2022-04-11 23:43:04.118677: saving checkpoint... +2022-04-11 23:43:08.904212: done, saving took 4.88 seconds +2022-04-11 23:43:08.915472: done +2022-04-11 23:43:08.962815: saving checkpoint... +2022-04-11 23:43:14.912585: done, saving took 6.00 seconds +2022-04-11 23:43:14.928179: This epoch took 131.312218 s + +2022-04-11 23:43:14.933694: svmem(total=99932160000, available=79969046528, percent=20.0, used=15969046528, free=51124838400, active=40235573248, inactive=6737993728, buffers=22507520, cached=32815767552, shared=2176741376, slab=761393152) +2022-04-11 23:43:14.934882: +epoch: 201 +2022-04-11 23:45:04.189786: train loss : -0.5908 +2022-04-11 23:45:15.395288: validation loss: -0.6101 +2022-04-11 23:45:15.426407: Average global foreground Dice: [0.8347, 0.9603, 0.978, 0.3582] +2022-04-11 23:45:15.447959: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:45:16.358194: lr: 0.006277 +2022-04-11 23:45:16.360079: saving scheduled checkpoint file... +2022-04-11 23:45:16.470636: saving checkpoint... +2022-04-11 23:45:22.290969: done, saving took 5.93 seconds +2022-04-11 23:45:22.390919: done +2022-04-11 23:45:22.488368: saving checkpoint... +2022-04-11 23:45:31.589837: done, saving took 9.20 seconds +2022-04-11 23:45:31.608602: This epoch took 136.655529 s + +2022-04-11 23:45:31.610343: svmem(total=99932160000, available=80032817152, percent=19.9, used=15905415168, free=51028856832, active=40166510592, inactive=6897762304, buffers=22548480, cached=32975339520, shared=2176741376, slab=762671104) +2022-04-11 23:45:31.611789: +epoch: 202 +2022-04-11 23:47:20.638429: train loss : -0.6101 +2022-04-11 23:47:31.914502: validation loss: -0.5935 +2022-04-11 23:47:31.916401: Average global foreground Dice: [0.8384, 0.9467, 0.9773, 0.3811] +2022-04-11 23:47:31.917349: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:47:32.856068: lr: 0.006258 +2022-04-11 23:47:32.857135: saving scheduled checkpoint file... +2022-04-11 23:47:32.976908: saving checkpoint... +2022-04-11 23:47:41.576593: done, saving took 8.72 seconds +2022-04-11 23:47:41.596214: done +2022-04-11 23:47:41.697265: saving checkpoint... +2022-04-11 23:47:46.425853: done, saving took 4.83 seconds +2022-04-11 23:47:46.451684: This epoch took 134.839056 s + +2022-04-11 23:47:46.453199: svmem(total=99932160000, available=79815405568, percent=20.1, used=16108605440, free=50794102784, active=40369025024, inactive=6929051648, buffers=22585344, cached=33006866432, shared=2190807040, slab=761819136) +2022-04-11 23:47:46.454155: +epoch: 203 +2022-04-11 23:49:36.212389: train loss : -0.6039 +2022-04-11 23:49:48.680018: validation loss: -0.5958 +2022-04-11 23:49:48.857051: Average global foreground Dice: [0.8298, 0.9571, 0.9789, 0.3704] +2022-04-11 23:49:49.005158: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:49:50.084332: lr: 0.006239 +2022-04-11 23:49:50.227217: saving scheduled checkpoint file... +2022-04-11 23:49:50.459364: saving checkpoint... +2022-04-11 23:49:56.047287: done, saving took 5.68 seconds +2022-04-11 23:49:56.063240: done +2022-04-11 23:49:56.113315: saving checkpoint... +2022-04-11 23:50:00.945365: done, saving took 4.88 seconds +2022-04-11 23:50:00.987172: This epoch took 134.532113 s + +2022-04-11 23:50:00.991602: svmem(total=99932160000, available=79788507136, percent=20.2, used=16135598080, free=50759274496, active=40401358848, inactive=6936780800, buffers=22618112, cached=33014669312, shared=2190729216, slab=761454592) +2022-04-11 23:50:00.993426: +epoch: 204 +2022-04-11 23:51:48.625535: train loss : -0.6104 +2022-04-11 23:52:00.892514: validation loss: -0.6146 +2022-04-11 23:52:01.048296: Average global foreground Dice: [0.8648, 0.9613, 0.9769, 0.4004] +2022-04-11 23:52:01.198566: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:52:02.098725: lr: 0.00622 +2022-04-11 23:52:02.196781: saving scheduled checkpoint file... +2022-04-11 23:52:02.284996: saving checkpoint... +2022-04-11 23:52:10.979339: done, saving took 8.78 seconds +2022-04-11 23:52:11.030245: done +2022-04-11 23:52:11.078574: saving checkpoint... +2022-04-11 23:52:15.945044: done, saving took 4.91 seconds +2022-04-11 23:52:15.960409: This epoch took 134.965476 s + +2022-04-11 23:52:15.961991: svmem(total=99932160000, available=79793328128, percent=20.2, used=16130670592, free=50746703872, active=40395329536, inactive=6954102784, buffers=22675456, cached=33032110080, shared=2190729216, slab=762740736) +2022-04-11 23:52:15.962971: +epoch: 205 +2022-04-11 23:54:03.660512: train loss : -0.6184 +2022-04-11 23:54:15.211968: validation loss: -0.6250 +2022-04-11 23:54:15.351584: Average global foreground Dice: [0.8587, 0.9501, 0.9826, 0.3753] +2022-04-11 23:54:15.511345: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:54:16.580778: lr: 0.006201 +2022-04-11 23:54:16.723521: saving scheduled checkpoint file... +2022-04-11 23:54:16.924129: saving checkpoint... +2022-04-11 23:54:22.743173: done, saving took 5.90 seconds +2022-04-11 23:54:22.772933: done +2022-04-11 23:54:22.820684: saving checkpoint... +2022-04-11 23:54:28.686054: done, saving took 5.91 seconds +2022-04-11 23:54:28.703801: This epoch took 132.740089 s + +2022-04-11 23:54:28.717477: svmem(total=99932160000, available=79728115712, percent=20.2, used=16196378624, free=50662903808, active=40460754944, inactive=6972514304, buffers=22708224, cached=33050169344, shared=2190614528, slab=761765888) +2022-04-11 23:54:28.724531: +epoch: 206 +2022-04-11 23:56:17.134836: train loss : -0.6114 +2022-04-11 23:56:28.725536: validation loss: -0.6056 +2022-04-11 23:56:28.735464: Average global foreground Dice: [0.8615, 0.9542, 0.9775, 0.3714] +2022-04-11 23:56:28.736959: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:56:29.711911: lr: 0.006182 +2022-04-11 23:56:29.713724: saving scheduled checkpoint file... +2022-04-11 23:56:29.812745: saving checkpoint... +2022-04-11 23:56:34.701720: done, saving took 4.99 seconds +2022-04-11 23:56:34.721740: done +2022-04-11 23:56:34.790991: saving checkpoint... +2022-04-11 23:56:39.566016: done, saving took 4.84 seconds +2022-04-11 23:56:39.584208: This epoch took 130.850833 s + +2022-04-11 23:56:39.585709: svmem(total=99932160000, available=79633809408, percent=20.3, used=16290607104, free=50560790528, active=40547553280, inactive=6979813376, buffers=22757376, cached=33058004992, shared=2190233600, slab=762253312) +2022-04-11 23:56:39.586522: +epoch: 207 +2022-04-11 23:58:27.550048: train loss : -0.6199 +2022-04-11 23:58:39.026330: validation loss: -0.5908 +2022-04-11 23:58:39.029929: Average global foreground Dice: [0.8413, 0.9519, 0.9749, 0.384] +2022-04-11 23:58:39.031535: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-11 23:58:40.011329: lr: 0.006163 +2022-04-11 23:58:40.013111: saving scheduled checkpoint file... +2022-04-11 23:58:40.130483: saving checkpoint... +2022-04-11 23:58:45.091761: done, saving took 5.08 seconds +2022-04-11 23:58:45.111460: done +2022-04-11 23:58:45.214700: saving checkpoint... +2022-04-11 23:58:50.142475: done, saving took 5.03 seconds +2022-04-11 23:58:50.229015: This epoch took 130.641672 s + +2022-04-11 23:58:50.230650: svmem(total=99932160000, available=79707488256, percent=20.2, used=16205975552, free=50629251072, active=40449277952, inactive=7010439168, buffers=22790144, cached=33074143232, shared=2201247744, slab=761331712) +2022-04-11 23:58:50.231439: +epoch: 208 +2022-04-12 00:00:40.218446: train loss : -0.5971 +2022-04-12 00:00:52.161571: validation loss: -0.5597 +2022-04-12 00:00:52.170392: Average global foreground Dice: [0.8494, 0.948, 0.9678, 0.3455] +2022-04-12 00:00:52.172287: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:00:53.285740: lr: 0.006144 +2022-04-12 00:00:53.287570: saving scheduled checkpoint file... +2022-04-12 00:00:53.416479: saving checkpoint... +2022-04-12 00:00:58.028407: done, saving took 4.74 seconds +2022-04-12 00:00:58.047919: done +2022-04-12 00:00:58.048809: This epoch took 127.816203 s + +2022-04-12 00:00:58.050558: svmem(total=99932160000, available=79873077248, percent=20.1, used=16048566272, free=50753499136, active=40309366784, inactive=7019335680, buffers=29790208, cached=33100304384, shared=2187149312, slab=774524928) +2022-04-12 00:00:58.051783: +epoch: 209 +2022-04-12 00:02:47.884727: train loss : -0.5741 +2022-04-12 00:03:00.125896: validation loss: -0.5896 +2022-04-12 00:03:00.136106: Average global foreground Dice: [0.8528, 0.9589, 0.9714, 0.3574] +2022-04-12 00:03:00.144685: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:03:00.975711: lr: 0.006125 +2022-04-12 00:03:00.977155: saving scheduled checkpoint file... +2022-04-12 00:03:01.065704: saving checkpoint... +2022-04-12 00:03:05.564289: done, saving took 4.59 seconds +2022-04-12 00:03:05.579188: done +2022-04-12 00:03:05.580093: This epoch took 127.527596 s + +2022-04-12 00:03:05.581017: svmem(total=99932160000, available=79797559296, percent=20.1, used=16120958976, free=50670125056, active=40388456448, inactive=7019237376, buffers=29822976, cached=33111252992, shared=2190508032, slab=774410240) +2022-04-12 00:03:05.581801: +epoch: 210 +2022-04-12 00:04:53.346934: train loss : -0.5859 +2022-04-12 00:05:05.186311: validation loss: -0.5852 +2022-04-12 00:05:05.188740: Average global foreground Dice: [0.8488, 0.9578, 0.9731, 0.3333] +2022-04-12 00:05:05.190037: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:05:06.064183: lr: 0.006106 +2022-04-12 00:05:06.065895: saving scheduled checkpoint file... +2022-04-12 00:05:06.163100: saving checkpoint... +2022-04-12 00:05:10.667744: done, saving took 4.60 seconds +2022-04-12 00:05:10.682689: done +2022-04-12 00:05:10.684060: This epoch took 125.101502 s + +2022-04-12 00:05:10.685548: svmem(total=99932160000, available=79871660032, percent=20.1, used=16046669824, free=50726789120, active=40317276160, inactive=7036571648, buffers=29872128, cached=33128828928, shared=2190540800, slab=774766592) +2022-04-12 00:05:10.686412: +epoch: 211 +2022-04-12 00:06:59.250582: train loss : -0.6089 +2022-04-12 00:07:10.482001: validation loss: -0.6102 +2022-04-12 00:07:10.484684: Average global foreground Dice: [0.857, 0.9578, 0.9752, 0.3639] +2022-04-12 00:07:10.486128: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:07:11.282444: lr: 0.006087 +2022-04-12 00:07:11.283724: saving scheduled checkpoint file... +2022-04-12 00:07:11.365317: saving checkpoint... +2022-04-12 00:07:16.125605: done, saving took 4.84 seconds +2022-04-12 00:07:16.143262: done +2022-04-12 00:07:16.144915: This epoch took 125.457276 s + +2022-04-12 00:07:16.146367: svmem(total=99932160000, available=79961538560, percent=20.0, used=15956611072, free=50799132672, active=40234393600, inactive=7054172160, buffers=29933568, cached=33146482688, shared=2190729216, slab=773623808) +2022-04-12 00:07:16.147442: +epoch: 212 +2022-04-12 00:09:04.691146: train loss : -0.5446 +2022-04-12 00:09:16.517854: validation loss: -0.5433 +2022-04-12 00:09:16.528544: Average global foreground Dice: [0.8193, 0.9442, 0.9662, 0.378] +2022-04-12 00:09:16.530053: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:09:17.423980: lr: 0.006068 +2022-04-12 00:09:17.425722: saving scheduled checkpoint file... +2022-04-12 00:09:17.513417: saving checkpoint... +2022-04-12 00:09:22.124733: done, saving took 4.70 seconds +2022-04-12 00:09:22.174946: done +2022-04-12 00:09:22.176383: This epoch took 126.027930 s + +2022-04-12 00:09:22.177844: svmem(total=99932160000, available=79572475904, percent=20.4, used=16345735168, free=50392600576, active=40621809664, inactive=7071547392, buffers=29966336, cached=33163857920, shared=2190729216, slab=772718592) +2022-04-12 00:09:22.178883: +epoch: 213 +2022-04-12 00:11:10.024232: train loss : -0.5488 +2022-04-12 00:11:21.708279: validation loss: -0.5579 +2022-04-12 00:11:21.924181: Average global foreground Dice: [0.8352, 0.9516, 0.9686, 0.3466] +2022-04-12 00:11:22.098637: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:11:23.024386: lr: 0.006049 +2022-04-12 00:11:23.166897: saving scheduled checkpoint file... +2022-04-12 00:11:23.376338: saving checkpoint... +2022-04-12 00:11:29.473325: done, saving took 6.18 seconds +2022-04-12 00:11:29.501798: done +2022-04-12 00:11:29.502770: This epoch took 127.322999 s + +2022-04-12 00:11:29.503988: svmem(total=99932160000, available=79893491712, percent=20.1, used=16024133632, free=50706817024, active=40299847680, inactive=7078219776, buffers=29999104, cached=33171210240, shared=2191224832, slab=773554176) +2022-04-12 00:11:29.504898: +epoch: 214 +2022-04-12 00:13:16.874756: train loss : -0.5863 +2022-04-12 00:13:28.482322: validation loss: -0.5694 +2022-04-12 00:13:28.490007: Average global foreground Dice: [0.825, 0.9435, 0.9739, 0.3506] +2022-04-12 00:13:28.491373: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:13:29.228432: lr: 0.00603 +2022-04-12 00:13:29.230546: saving scheduled checkpoint file... +2022-04-12 00:13:29.341312: saving checkpoint... +2022-04-12 00:13:36.311668: done, saving took 7.08 seconds +2022-04-12 00:13:36.723209: done +2022-04-12 00:13:36.886696: This epoch took 127.380617 s + +2022-04-12 00:13:37.011750: svmem(total=99932160000, available=79806832640, percent=20.1, used=16125308928, free=50602729472, active=40386342912, inactive=7081680896, buffers=30031872, cached=33174089728, shared=2176741376, slab=785895424) +2022-04-12 00:13:37.214074: +epoch: 215 +2022-04-12 00:15:26.226055: train loss : -0.6001 +2022-04-12 00:15:37.938784: validation loss: -0.5807 +2022-04-12 00:15:37.952579: Average global foreground Dice: [0.8555, 0.9534, 0.9739, 0.3506] +2022-04-12 00:15:37.953874: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:15:38.902976: lr: 0.006011 +2022-04-12 00:15:38.905238: saving scheduled checkpoint file... +2022-04-12 00:15:38.992069: saving checkpoint... +2022-04-12 00:15:43.473341: done, saving took 4.57 seconds +2022-04-12 00:15:43.488287: done +2022-04-12 00:15:43.489532: This epoch took 126.142100 s + +2022-04-12 00:15:43.490911: svmem(total=99932160000, available=80276357120, percent=19.7, used=15722639360, free=51054841856, active=40002125824, inactive=7032311808, buffers=30081024, cached=33124597760, shared=2109841408, slab=773013504) +2022-04-12 00:15:43.491685: +epoch: 216 +2022-04-12 00:17:32.909854: train loss : -0.5852 +2022-04-12 00:17:45.042141: validation loss: -0.5946 +2022-04-12 00:17:45.044868: Average global foreground Dice: [0.8516, 0.9584, 0.9693, 0.3738] +2022-04-12 00:17:45.045937: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:17:45.969299: lr: 0.005991 +2022-04-12 00:17:45.970932: saving scheduled checkpoint file... +2022-04-12 00:17:46.046641: saving checkpoint... +2022-04-12 00:17:50.560601: done, saving took 4.59 seconds +2022-04-12 00:17:50.575446: done +2022-04-12 00:17:50.576373: This epoch took 127.083965 s + +2022-04-12 00:17:50.577457: svmem(total=99932160000, available=79811424256, percent=20.1, used=16100397056, free=50582257664, active=40380760064, inactive=7127232512, buffers=30117888, cached=33219387392, shared=2196865024, slab=773545984) +2022-04-12 00:17:50.578323: +epoch: 217 +2022-04-12 00:19:39.443565: train loss : -0.5533 +2022-04-12 00:19:50.905300: validation loss: -0.5570 +2022-04-12 00:19:50.907694: Average global foreground Dice: [0.815, 0.9465, 0.9667, 0.3414] +2022-04-12 00:19:50.909295: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:19:52.001080: lr: 0.005972 +2022-04-12 00:19:52.002570: saving scheduled checkpoint file... +2022-04-12 00:19:52.098055: saving checkpoint... +2022-04-12 00:19:56.739403: done, saving took 4.74 seconds +2022-04-12 00:19:56.802431: done +2022-04-12 00:19:56.803930: This epoch took 126.224382 s + +2022-04-12 00:19:56.805825: svmem(total=99932160000, available=79833763840, percent=20.1, used=16084287488, free=50586972160, active=40357715968, inactive=7137792000, buffers=30150656, cached=33230749696, shared=2190544896, slab=775753728) +2022-04-12 00:19:56.807413: +epoch: 218 +2022-04-12 00:21:43.322281: train loss : -0.5605 +2022-04-12 00:21:54.731110: validation loss: -0.5430 +2022-04-12 00:21:54.773621: Average global foreground Dice: [0.8544, 0.9416, 0.9651, 0.349] +2022-04-12 00:21:54.795581: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:21:55.733203: lr: 0.005953 +2022-04-12 00:21:55.735107: saving scheduled checkpoint file... +2022-04-12 00:21:55.791189: saving checkpoint... +2022-04-12 00:22:00.479914: done, saving took 4.74 seconds +2022-04-12 00:22:00.495426: done +2022-04-12 00:22:00.496341: This epoch took 123.688090 s + +2022-04-12 00:22:00.497365: svmem(total=99932160000, available=79650684928, percent=20.3, used=16267251712, free=50387415040, active=40538574848, inactive=7154311168, buffers=30208000, cached=33247285248, shared=2190729216, slab=775487488) +2022-04-12 00:22:00.498107: +epoch: 219 +2022-04-12 00:23:48.825056: train loss : -0.5248 +2022-04-12 00:24:01.166646: validation loss: -0.5709 +2022-04-12 00:24:01.169423: Average global foreground Dice: [0.8355, 0.9491, 0.969, 0.3689] +2022-04-12 00:24:01.170541: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:24:02.029858: lr: 0.005934 +2022-04-12 00:24:02.031340: saving scheduled checkpoint file... +2022-04-12 00:24:02.116456: saving checkpoint... +2022-04-12 00:24:06.585439: done, saving took 4.55 seconds +2022-04-12 00:24:06.599894: done +2022-04-12 00:24:06.601021: This epoch took 126.101988 s + +2022-04-12 00:24:06.601929: svmem(total=99932160000, available=79659257856, percent=20.3, used=16258072576, free=50388185088, active=40528777216, inactive=7162494976, buffers=30240768, cached=33255661568, shared=2191224832, slab=776609792) +2022-04-12 00:24:06.602622: +epoch: 220 +2022-04-12 00:25:56.725542: train loss : -0.5070 +2022-04-12 00:26:08.996440: validation loss: -0.5362 +2022-04-12 00:26:09.000564: Average global foreground Dice: [0.8083, 0.9442, 0.9639, 0.3387] +2022-04-12 00:26:09.001952: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:26:09.882712: lr: 0.005915 +2022-04-12 00:26:09.884163: saving scheduled checkpoint file... +2022-04-12 00:26:10.022225: saving checkpoint... +2022-04-12 00:26:14.626279: done, saving took 4.74 seconds +2022-04-12 00:26:14.647755: done +2022-04-12 00:26:14.649216: This epoch took 128.045820 s + +2022-04-12 00:26:14.650828: svmem(total=99932160000, available=79634452480, percent=20.3, used=16283328512, free=50361769984, active=40543305728, inactive=7179403264, buffers=30306304, cached=33256755200, shared=2190729216, slab=774086656) +2022-04-12 00:26:14.651824: +epoch: 221 +2022-04-12 00:28:04.457165: train loss : -0.3985 +2022-04-12 00:28:16.771239: validation loss: -0.4739 +2022-04-12 00:28:16.783089: Average global foreground Dice: [0.7876, 0.9311, 0.9527, 0.3259] +2022-04-12 00:28:16.784555: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:28:17.770227: lr: 0.005896 +2022-04-12 00:28:17.772096: saving scheduled checkpoint file... +2022-04-12 00:28:17.866817: saving checkpoint... +2022-04-12 00:28:22.643380: done, saving took 4.87 seconds +2022-04-12 00:28:22.657499: done +2022-04-12 00:28:22.659262: This epoch took 128.006133 s + +2022-04-12 00:28:22.660314: svmem(total=99932160000, available=79744253952, percent=20.2, used=16173461504, free=50454208512, active=40449396736, inactive=7180861440, buffers=30339072, cached=33274150912, shared=2190729216, slab=773955584) +2022-04-12 00:28:22.661184: +epoch: 222 +2022-04-12 00:30:10.148399: train loss : -0.5228 +2022-04-12 00:30:21.770523: validation loss: -0.5213 +2022-04-12 00:30:21.772793: Average global foreground Dice: [0.8375, 0.9397, 0.9703, 0.3615] +2022-04-12 00:30:21.774479: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:30:22.488712: lr: 0.005877 +2022-04-12 00:30:22.489744: saving scheduled checkpoint file... +2022-04-12 00:30:22.577604: saving checkpoint... +2022-04-12 00:30:27.362145: done, saving took 4.87 seconds +2022-04-12 00:30:27.423902: done +2022-04-12 00:30:27.424998: This epoch took 124.762970 s + +2022-04-12 00:30:27.426360: svmem(total=99932160000, available=79737589760, percent=20.2, used=16180297728, free=50443821056, active=40439795712, inactive=7200849920, buffers=30408704, cached=33277632512, shared=2190618624, slab=773959680) +2022-04-12 00:30:27.427050: +epoch: 223 +2022-04-12 00:32:15.061607: train loss : -0.5644 +2022-04-12 00:32:26.610527: validation loss: -0.5824 +2022-04-12 00:32:26.638172: Average global foreground Dice: [0.8302, 0.957, 0.9722, 0.3503] +2022-04-12 00:32:26.659893: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:32:27.602403: lr: 0.005858 +2022-04-12 00:32:27.620857: saving scheduled checkpoint file... +2022-04-12 00:32:27.719336: saving checkpoint... +2022-04-12 00:32:32.367710: done, saving took 4.74 seconds +2022-04-12 00:32:32.432527: done +2022-04-12 00:32:32.435405: This epoch took 125.007727 s + +2022-04-12 00:32:32.437381: svmem(total=99932160000, available=79671939072, percent=20.3, used=16245866496, free=50393403392, active=40344395776, inactive=7342329856, buffers=30441472, cached=33262448640, shared=2190729216, slab=774512640) +2022-04-12 00:32:32.438415: +epoch: 224 +2022-04-12 00:34:20.738856: train loss : -0.5899 +2022-04-12 00:34:32.841190: validation loss: -0.6005 +2022-04-12 00:34:32.843601: Average global foreground Dice: [0.8327, 0.9577, 0.9793, 0.3614] +2022-04-12 00:34:32.845114: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:34:33.643696: lr: 0.005839 +2022-04-12 00:34:33.645184: saving scheduled checkpoint file... +2022-04-12 00:34:33.753644: saving checkpoint... +2022-04-12 00:34:38.331806: done, saving took 4.69 seconds +2022-04-12 00:34:38.350563: done +2022-04-12 00:34:38.351618: This epoch took 125.912199 s + +2022-04-12 00:34:38.352966: svmem(total=99932160000, available=79956168704, percent=20.0, used=15961518080, free=50660220928, active=40155398144, inactive=7270215680, buffers=30474240, cached=33279946752, shared=2190729216, slab=773513216) +2022-04-12 00:34:38.354209: +epoch: 225 +2022-04-12 00:36:27.226334: train loss : -0.6040 +2022-04-12 00:36:38.396106: validation loss: -0.6195 +2022-04-12 00:36:38.402417: Average global foreground Dice: [0.8635, 0.9543, 0.9748, 0.353] +2022-04-12 00:36:38.403974: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:36:39.298418: lr: 0.00582 +2022-04-12 00:36:39.299908: saving scheduled checkpoint file... +2022-04-12 00:36:39.424775: saving checkpoint... +2022-04-12 00:36:44.094336: done, saving took 4.79 seconds +2022-04-12 00:36:44.110435: done +2022-04-12 00:36:44.112270: This epoch took 125.757126 s + +2022-04-12 00:36:44.115669: svmem(total=99932160000, available=79910404096, percent=20.0, used=15996895232, free=50606706688, active=40263176192, inactive=7213518848, buffers=30527488, cached=33298030592, shared=2201104384, slab=774242304) +2022-04-12 00:36:44.116583: +epoch: 226 +2022-04-12 00:38:31.418702: train loss : -0.5837 +2022-04-12 00:38:43.484290: validation loss: -0.5972 +2022-04-12 00:38:43.489239: Average global foreground Dice: [0.8257, 0.9538, 0.9752, 0.3685] +2022-04-12 00:38:43.493306: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:38:44.109598: lr: 0.005801 +2022-04-12 00:38:44.111035: saving scheduled checkpoint file... +2022-04-12 00:38:44.214450: saving checkpoint... +2022-04-12 00:38:49.021139: done, saving took 4.91 seconds +2022-04-12 00:38:49.066511: done +2022-04-12 00:38:49.068343: This epoch took 124.950340 s + +2022-04-12 00:38:49.070176: svmem(total=99932160000, available=79813406720, percent=20.1, used=16104431616, free=50493280256, active=40378867712, inactive=7216852992, buffers=30564352, cached=33303883776, shared=2190696448, slab=774176768) +2022-04-12 00:38:49.071985: +epoch: 227 +2022-04-12 00:40:36.880682: train loss : -0.5977 +2022-04-12 00:40:48.082450: validation loss: -0.5993 +2022-04-12 00:40:48.084990: Average global foreground Dice: [0.8551, 0.9543, 0.9762, 0.3669] +2022-04-12 00:40:48.086915: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:40:48.981251: lr: 0.005781 +2022-04-12 00:40:48.982954: saving scheduled checkpoint file... +2022-04-12 00:40:49.071928: saving checkpoint... +2022-04-12 00:40:53.499829: done, saving took 4.52 seconds +2022-04-12 00:40:53.516850: done +2022-04-12 00:40:53.518866: This epoch took 124.445822 s + +2022-04-12 00:40:53.523999: svmem(total=99932160000, available=79715835904, percent=20.2, used=16191762432, free=50378297344, active=40465776640, inactive=7244435456, buffers=30605312, cached=33331494912, shared=2200948736, slab=774189056) +2022-04-12 00:40:53.524965: +epoch: 228 +2022-04-12 00:42:40.931325: train loss : -0.6095 +2022-04-12 00:42:52.460306: validation loss: -0.5816 +2022-04-12 00:42:52.471109: Average global foreground Dice: [0.847, 0.9555, 0.9742, 0.3461] +2022-04-12 00:42:52.472317: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:42:53.390026: lr: 0.005762 +2022-04-12 00:42:53.391868: saving scheduled checkpoint file... +2022-04-12 00:42:53.482447: saving checkpoint... +2022-04-12 00:42:58.124079: done, saving took 4.73 seconds +2022-04-12 00:42:58.137577: done +2022-04-12 00:42:58.138892: This epoch took 124.613014 s + +2022-04-12 00:42:58.139897: svmem(total=99932160000, available=79836774400, percent=20.1, used=16080969728, free=50491510784, active=40353218560, inactive=7241859072, buffers=30638080, cached=33329041408, shared=2190729216, slab=773357568) +2022-04-12 00:42:58.140659: +epoch: 229 +2022-04-12 00:44:46.898444: train loss : -0.5963 +2022-04-12 00:44:58.210459: validation loss: -0.5900 +2022-04-12 00:44:58.225871: Average global foreground Dice: [0.8456, 0.9526, 0.9719, 0.3788] +2022-04-12 00:44:58.227556: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:44:59.209975: lr: 0.005743 +2022-04-12 00:44:59.211739: saving scheduled checkpoint file... +2022-04-12 00:44:59.288871: saving checkpoint... +2022-04-12 00:45:04.870494: done, saving took 5.66 seconds +2022-04-12 00:45:04.884618: done +2022-04-12 00:45:04.885496: This epoch took 126.744109 s + +2022-04-12 00:45:04.886683: svmem(total=99932160000, available=79870365696, percent=20.1, used=16037462016, free=50508652544, active=40309587968, inactive=7268130816, buffers=30679040, cached=33355366400, shared=2200641536, slab=774402048) +2022-04-12 00:45:04.887421: +epoch: 230 +2022-04-12 00:46:53.760727: train loss : -0.5775 +2022-04-12 00:47:05.346975: validation loss: -0.5856 +2022-04-12 00:47:05.360952: Average global foreground Dice: [0.8252, 0.953, 0.9681, 0.3616] +2022-04-12 00:47:05.364657: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:47:05.919325: lr: 0.005724 +2022-04-12 00:47:05.920675: saving scheduled checkpoint file... +2022-04-12 00:47:06.005669: saving checkpoint... +2022-04-12 00:47:10.979721: done, saving took 5.06 seconds +2022-04-12 00:47:11.078126: done +2022-04-12 00:47:11.080600: This epoch took 126.191989 s + +2022-04-12 00:47:11.082019: svmem(total=99932160000, available=79575805952, percent=20.4, used=16356057088, free=50196553728, active=40630915072, inactive=7255400448, buffers=30752768, cached=33348796416, shared=2176516096, slab=774844416) +2022-04-12 00:47:11.083392: +epoch: 231 +2022-04-12 00:48:59.001174: train loss : -0.6030 +2022-04-12 00:49:11.509497: validation loss: -0.5932 +2022-04-12 00:49:11.522797: Average global foreground Dice: [0.8598, 0.9492, 0.9745, 0.3657] +2022-04-12 00:49:11.531368: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:49:12.207339: lr: 0.005705 +2022-04-12 00:49:12.208863: saving scheduled checkpoint file... +2022-04-12 00:49:12.316439: saving checkpoint... +2022-04-12 00:49:17.555352: done, saving took 5.35 seconds +2022-04-12 00:49:17.610889: done +2022-04-12 00:49:17.615144: This epoch took 126.517247 s + +2022-04-12 00:49:17.619857: svmem(total=99932160000, available=79857524736, percent=20.1, used=16059920384, free=50470469632, active=40334065664, inactive=7277109248, buffers=30785536, cached=33370984448, shared=2190995456, slab=775901184) +2022-04-12 00:49:17.627772: +epoch: 232 +2022-04-12 00:51:07.261586: train loss : -0.5736 +2022-04-12 00:51:18.433936: validation loss: -0.5970 +2022-04-12 00:51:18.449846: Average global foreground Dice: [0.8444, 0.9556, 0.9751, 0.3936] +2022-04-12 00:51:18.452673: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:51:19.328641: lr: 0.005686 +2022-04-12 00:51:19.331202: saving scheduled checkpoint file... +2022-04-12 00:51:19.459230: saving checkpoint... +2022-04-12 00:51:26.582079: done, saving took 7.25 seconds +2022-04-12 00:51:26.607826: done +2022-04-12 00:51:26.610337: This epoch took 128.972164 s + +2022-04-12 00:51:26.615438: svmem(total=99932160000, available=79842533376, percent=20.1, used=16075153408, free=50438004736, active=40349970432, inactive=7294111744, buffers=30826496, cached=33388175360, shared=2190618624, slab=775045120) +2022-04-12 00:51:26.616481: +epoch: 233 +2022-04-12 00:53:14.771733: train loss : -0.5841 +2022-04-12 00:53:26.847670: validation loss: -0.5967 +2022-04-12 00:53:26.863251: Average global foreground Dice: [0.8707, 0.9568, 0.9674, 0.3627] +2022-04-12 00:53:26.875970: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:53:27.899110: lr: 0.005667 +2022-04-12 00:53:27.909232: saving scheduled checkpoint file... +2022-04-12 00:53:28.043499: saving checkpoint... +2022-04-12 00:53:34.522713: done, saving took 6.58 seconds +2022-04-12 00:53:34.566663: done +2022-04-12 00:53:34.594719: This epoch took 127.977006 s + +2022-04-12 00:53:34.608944: svmem(total=99932160000, available=79935754240, percent=20.0, used=15981723648, free=50513879040, active=40250937344, inactive=7311360000, buffers=30859264, cached=33405698048, shared=2190618624, slab=776216576) +2022-04-12 00:53:34.617344: +epoch: 234 +2022-04-12 00:55:22.533900: train loss : -0.5734 +2022-04-12 00:55:34.777456: validation loss: -0.5564 +2022-04-12 00:55:34.795993: Average global foreground Dice: [0.8474, 0.9451, 0.9689, 0.3561] +2022-04-12 00:55:34.805352: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:55:35.702740: lr: 0.005647 +2022-04-12 00:55:35.709140: saving scheduled checkpoint file... +2022-04-12 00:55:35.795786: saving checkpoint... +2022-04-12 00:55:40.841238: done, saving took 5.13 seconds +2022-04-12 00:55:40.855828: done +2022-04-12 00:55:40.857388: This epoch took 126.192941 s + +2022-04-12 00:55:40.861043: svmem(total=99932160000, available=79743774720, percent=20.2, used=16173518848, free=50313940992, active=40455090176, inactive=7319379968, buffers=30908416, cached=33413791744, shared=2190807040, slab=774217728) +2022-04-12 00:55:40.867386: +epoch: 235 +2022-04-12 00:57:27.660053: train loss : -0.6028 +2022-04-12 00:57:40.071604: validation loss: -0.5948 +2022-04-12 00:57:40.085870: Average global foreground Dice: [0.838, 0.9515, 0.9789, 0.3616] +2022-04-12 00:57:40.141828: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:57:41.021494: lr: 0.005628 +2022-04-12 00:57:41.037854: saving scheduled checkpoint file... +2022-04-12 00:57:41.133833: saving checkpoint... +2022-04-12 00:57:45.800924: done, saving took 4.76 seconds +2022-04-12 00:57:45.835354: done +2022-04-12 00:57:45.836700: This epoch took 124.951946 s + +2022-04-12 00:57:45.838032: svmem(total=99932160000, available=79732981760, percent=20.2, used=16177610752, free=50286780416, active=40456048640, inactive=7342518272, buffers=30941184, cached=33436827648, shared=2197557248, slab=774643712) +2022-04-12 00:57:45.839087: +epoch: 236 +2022-04-12 00:59:33.800314: train loss : -0.6084 +2022-04-12 00:59:45.215816: validation loss: -0.5461 +2022-04-12 00:59:45.264320: Average global foreground Dice: [0.8616, 0.939, 0.9576, 0.3443] +2022-04-12 00:59:45.282091: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 00:59:46.130491: lr: 0.005609 +2022-04-12 00:59:46.134145: saving scheduled checkpoint file... +2022-04-12 00:59:46.210876: saving checkpoint... +2022-04-12 00:59:50.809874: done, saving took 4.67 seconds +2022-04-12 00:59:50.837689: done +2022-04-12 00:59:50.846572: This epoch took 125.006830 s + +2022-04-12 00:59:50.847804: svmem(total=99932160000, available=79757123584, percent=20.2, used=16160161792, free=50293473280, active=40437575680, inactive=7352983552, buffers=30973952, cached=33447550976, shared=2190729216, slab=775278592) +2022-04-12 00:59:50.850008: +epoch: 237 +2022-04-12 01:01:40.647049: train loss : -0.5962 +2022-04-12 01:01:52.110685: validation loss: -0.6207 +2022-04-12 01:01:52.125922: Average global foreground Dice: [0.8591, 0.9589, 0.9803, 0.3817] +2022-04-12 01:01:52.126935: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:01:52.722223: lr: 0.00559 +2022-04-12 01:01:52.723661: saving scheduled checkpoint file... +2022-04-12 01:01:52.808859: saving checkpoint... +2022-04-12 01:01:57.351229: done, saving took 4.63 seconds +2022-04-12 01:01:57.384500: done +2022-04-12 01:01:57.386065: This epoch took 126.533989 s + +2022-04-12 01:01:57.387450: svmem(total=99932160000, available=79889874944, percent=20.1, used=16027533312, free=50418298880, active=40306245632, inactive=7360802816, buffers=31014912, cached=33455312896, shared=2190729216, slab=774483968) +2022-04-12 01:01:57.388395: +epoch: 238 +2022-04-12 01:03:46.452372: train loss : -0.5973 +2022-04-12 01:03:58.299706: validation loss: -0.5974 +2022-04-12 01:03:58.313781: Average global foreground Dice: [0.8765, 0.9532, 0.9742, 0.3834] +2022-04-12 01:03:58.314933: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:03:59.254150: lr: 0.005571 +2022-04-12 01:03:59.255736: saving scheduled checkpoint file... +2022-04-12 01:03:59.357661: saving checkpoint... +2022-04-12 01:04:04.051145: done, saving took 4.79 seconds +2022-04-12 01:04:04.139842: done +2022-04-12 01:04:04.235967: saving checkpoint... +2022-04-12 01:04:08.886668: done, saving took 4.75 seconds +2022-04-12 01:04:08.906434: This epoch took 131.517246 s + +2022-04-12 01:04:08.908640: svmem(total=99932160000, available=79733182464, percent=20.2, used=16184274944, free=50244288512, active=40449191936, inactive=7378046976, buffers=31047680, cached=33472548864, shared=2190618624, slab=775680000) +2022-04-12 01:04:08.911353: +epoch: 239 +2022-04-12 01:05:56.643265: train loss : -0.6184 +2022-04-12 01:06:08.228060: validation loss: -0.6255 +2022-04-12 01:06:08.241555: Average global foreground Dice: [0.8596, 0.9582, 0.976, 0.3583] +2022-04-12 01:06:08.242398: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:06:09.047018: lr: 0.005551 +2022-04-12 01:06:09.048751: saving scheduled checkpoint file... +2022-04-12 01:06:09.139838: saving checkpoint... +2022-04-12 01:06:13.952507: done, saving took 4.90 seconds +2022-04-12 01:06:13.967579: done +2022-04-12 01:06:14.019996: saving checkpoint... +2022-04-12 01:06:18.906012: done, saving took 4.93 seconds +2022-04-12 01:06:18.920054: This epoch took 130.007224 s + +2022-04-12 01:06:18.921267: svmem(total=99932160000, available=79855923200, percent=20.1, used=16061288448, free=50349494272, active=40340590592, inactive=7395491840, buffers=31113216, cached=33490264064, shared=2190729216, slab=775450624) +2022-04-12 01:06:18.922160: +epoch: 240 +2022-04-12 01:08:06.823649: train loss : -0.6266 +2022-04-12 01:08:17.740737: validation loss: -0.6246 +2022-04-12 01:08:17.744419: Average global foreground Dice: [0.8739, 0.9617, 0.9791, 0.3879] +2022-04-12 01:08:17.745787: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:08:18.762990: lr: 0.005532 +2022-04-12 01:08:18.764857: saving scheduled checkpoint file... +2022-04-12 01:08:18.857482: saving checkpoint... +2022-04-12 01:08:23.477117: done, saving took 4.71 seconds +2022-04-12 01:08:23.491915: done +2022-04-12 01:08:23.539235: saving checkpoint... +2022-04-12 01:08:28.058187: done, saving took 4.57 seconds +2022-04-12 01:08:28.072313: This epoch took 129.148923 s + +2022-04-12 01:08:28.073407: svmem(total=99932160000, available=79737032704, percent=20.2, used=16180322304, free=50213113856, active=40458686464, inactive=7412879360, buffers=31158272, cached=33507565568, shared=2190618624, slab=774758400) +2022-04-12 01:08:28.074162: +epoch: 241 +2022-04-12 01:10:15.842307: train loss : -0.6200 +2022-04-12 01:10:26.518160: validation loss: -0.6331 +2022-04-12 01:10:26.522527: Average global foreground Dice: [0.8835, 0.9579, 0.9804, 0.3907] +2022-04-12 01:10:26.528076: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:10:27.468802: lr: 0.005513 +2022-04-12 01:10:27.470539: saving scheduled checkpoint file... +2022-04-12 01:10:27.564389: saving checkpoint... +2022-04-12 01:10:32.212130: done, saving took 4.74 seconds +2022-04-12 01:10:32.239722: done +2022-04-12 01:10:32.287551: saving checkpoint... +2022-04-12 01:10:36.824345: done, saving took 4.58 seconds +2022-04-12 01:10:36.931812: This epoch took 128.856511 s + +2022-04-12 01:10:36.933607: svmem(total=99932160000, available=79682117632, percent=20.3, used=16235069440, free=50182283264, active=40349638656, inactive=7551770624, buffers=31199232, cached=33483608064, shared=2190729216, slab=774455296) +2022-04-12 01:10:36.934465: +epoch: 242 +2022-04-12 01:12:25.242786: train loss : -0.6360 +2022-04-12 01:12:36.042621: validation loss: -0.6276 +2022-04-12 01:12:36.045547: Average global foreground Dice: [0.8606, 0.9548, 0.9817, 0.3611] +2022-04-12 01:12:36.046903: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:12:36.861935: lr: 0.005494 +2022-04-12 01:12:36.863697: saving scheduled checkpoint file... +2022-04-12 01:12:36.922590: saving checkpoint... +2022-04-12 01:12:44.939353: done, saving took 8.07 seconds +2022-04-12 01:12:44.987693: done +2022-04-12 01:12:45.037016: saving checkpoint... +2022-04-12 01:12:52.173208: done, saving took 7.18 seconds +2022-04-12 01:12:52.343223: This epoch took 135.407842 s + +2022-04-12 01:12:52.348705: svmem(total=99932160000, available=79685189632, percent=20.3, used=16232042496, free=50168078336, active=40434425856, inactive=7481823232, buffers=31232000, cached=33500807168, shared=2190729216, slab=774955008) +2022-04-12 01:12:52.367321: +epoch: 243 +2022-04-12 01:14:41.255037: train loss : -0.6430 +2022-04-12 01:14:53.585932: validation loss: -0.6157 +2022-04-12 01:14:53.590587: Average global foreground Dice: [0.8751, 0.9535, 0.9743, 0.351] +2022-04-12 01:14:53.630917: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:14:54.475121: lr: 0.005474 +2022-04-12 01:14:54.479245: saving scheduled checkpoint file... +2022-04-12 01:14:54.576379: saving checkpoint... +2022-04-12 01:15:01.623682: done, saving took 7.14 seconds +2022-04-12 01:15:01.778891: done +2022-04-12 01:15:01.867721: saving checkpoint... +2022-04-12 01:15:07.860005: done, saving took 6.08 seconds +2022-04-12 01:15:07.900763: This epoch took 135.482305 s + +2022-04-12 01:15:07.902404: svmem(total=99932160000, available=79762554880, percent=20.2, used=16155205632, free=50227859456, active=40388644864, inactive=7465304064, buffers=31289344, cached=33517805568, shared=2190123008, slab=773898240) +2022-04-12 01:15:07.903111: +epoch: 244 +2022-04-12 01:16:55.334951: train loss : -0.6187 +2022-04-12 01:17:06.906952: validation loss: -0.6115 +2022-04-12 01:17:06.994589: Average global foreground Dice: [0.8485, 0.9588, 0.9684, 0.3813] +2022-04-12 01:17:07.030331: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:17:07.880510: lr: 0.005455 +2022-04-12 01:17:07.882297: saving scheduled checkpoint file... +2022-04-12 01:17:08.005045: saving checkpoint... +2022-04-12 01:17:12.596684: done, saving took 4.71 seconds +2022-04-12 01:17:12.618545: done +2022-04-12 01:17:12.718565: saving checkpoint... +2022-04-12 01:17:18.011943: done, saving took 5.39 seconds +2022-04-12 01:17:18.032928: This epoch took 130.129167 s + +2022-04-12 01:17:18.033928: svmem(total=99932160000, available=79969329152, percent=20.0, used=15953764352, free=50344013824, active=40199876608, inactive=7538049024, buffers=31346688, cached=33603035136, shared=2184781824, slab=774389760) +2022-04-12 01:17:18.035060: +epoch: 245 +2022-04-12 01:19:07.249230: train loss : -0.6314 +2022-04-12 01:19:19.970406: validation loss: -0.6232 +2022-04-12 01:19:19.993621: Average global foreground Dice: [0.8742, 0.9619, 0.9772, 0.37] +2022-04-12 01:19:19.995571: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:19:21.003142: lr: 0.005436 +2022-04-12 01:19:21.013011: saving scheduled checkpoint file... +2022-04-12 01:19:21.102970: saving checkpoint... +2022-04-12 01:19:25.809859: done, saving took 4.80 seconds +2022-04-12 01:19:25.823171: done +2022-04-12 01:19:25.872980: saving checkpoint... +2022-04-12 01:19:30.711860: done, saving took 4.89 seconds +2022-04-12 01:19:30.727197: This epoch took 132.691489 s + +2022-04-12 01:19:30.729280: svmem(total=99932160000, available=79881326592, percent=20.1, used=16036302848, free=50238697472, active=40297365504, inactive=7547117568, buffers=31383552, cached=33625776128, shared=2190155776, slab=773926912) +2022-04-12 01:19:30.730863: +epoch: 246 +2022-04-12 01:21:17.987043: train loss : -0.6301 +2022-04-12 01:21:29.886784: validation loss: -0.6388 +2022-04-12 01:21:29.895002: Average global foreground Dice: [0.8714, 0.9645, 0.9761, 0.3873] +2022-04-12 01:21:29.896559: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:21:30.580634: lr: 0.005417 +2022-04-12 01:21:30.582773: saving scheduled checkpoint file... +2022-04-12 01:21:30.675111: saving checkpoint... +2022-04-12 01:21:37.945354: done, saving took 7.36 seconds +2022-04-12 01:21:37.978203: done +2022-04-12 01:21:38.027891: saving checkpoint... +2022-04-12 01:21:43.189236: done, saving took 5.21 seconds +2022-04-12 01:21:43.203881: This epoch took 132.471911 s + +2022-04-12 01:21:43.205598: svmem(total=99932160000, available=79735275520, percent=20.2, used=16181608448, free=50075127808, active=40460500992, inactive=7550156800, buffers=31440896, cached=33643982848, shared=2190884864, slab=775970816) +2022-04-12 01:21:43.206604: +epoch: 247 +2022-04-12 01:23:31.472437: train loss : -0.6448 +2022-04-12 01:23:43.230109: validation loss: -0.6314 +2022-04-12 01:23:43.254693: Average global foreground Dice: [0.8722, 0.9595, 0.9781, 0.3774] +2022-04-12 01:23:43.268845: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:23:44.122327: lr: 0.005397 +2022-04-12 01:23:44.123952: saving scheduled checkpoint file... +2022-04-12 01:23:44.215008: saving checkpoint... +2022-04-12 01:23:49.008476: done, saving took 4.88 seconds +2022-04-12 01:23:49.052581: done +2022-04-12 01:23:49.124328: saving checkpoint... +2022-04-12 01:23:54.378753: done, saving took 5.30 seconds +2022-04-12 01:23:54.387911: This epoch took 131.180338 s + +2022-04-12 01:23:54.393086: svmem(total=99932160000, available=79784968192, percent=20.2, used=16121683968, free=50106224640, active=40398020608, inactive=7578828800, buffers=31473664, cached=33672777728, shared=2201026560, slab=776224768) +2022-04-12 01:23:54.394600: +epoch: 248 +2022-04-12 01:25:41.964169: train loss : -0.6390 +2022-04-12 01:25:54.191246: validation loss: -0.6491 +2022-04-12 01:25:54.233455: Average global foreground Dice: [0.8953, 0.9596, 0.982, 0.3586] +2022-04-12 01:25:54.234810: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:25:55.345665: lr: 0.005378 +2022-04-12 01:25:55.359236: saving scheduled checkpoint file... +2022-04-12 01:25:55.467808: saving checkpoint... +2022-04-12 01:26:03.651474: done, saving took 8.28 seconds +2022-04-12 01:26:03.695505: done +2022-04-12 01:26:03.760414: saving checkpoint... +2022-04-12 01:26:08.847431: done, saving took 5.14 seconds +2022-04-12 01:26:08.945014: This epoch took 134.548923 s + +2022-04-12 01:26:08.946555: svmem(total=99932160000, available=79538843648, percent=20.4, used=16378204160, free=49852231680, active=40651436032, inactive=7576264704, buffers=31522816, cached=33670201344, shared=2190729216, slab=775430144) +2022-04-12 01:26:08.947200: +epoch: 249 +2022-04-12 01:27:56.779870: train loss : -0.6494 +2022-04-12 01:28:08.494459: validation loss: -0.6243 +2022-04-12 01:28:08.526251: Average global foreground Dice: [0.8488, 0.9595, 0.9766, 0.3847] +2022-04-12 01:28:08.531228: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:28:09.217880: lr: 0.005359 +2022-04-12 01:28:09.246859: saving scheduled checkpoint file... +2022-04-12 01:28:09.383928: saving checkpoint... +2022-04-12 01:28:14.785655: done, saving took 5.51 seconds +2022-04-12 01:28:14.818705: done +2022-04-12 01:28:14.901726: saving checkpoint... +2022-04-12 01:28:23.490657: done, saving took 8.67 seconds +2022-04-12 01:28:23.508985: This epoch took 134.561136 s + +2022-04-12 01:28:23.510867: svmem(total=99932160000, available=79879319552, percent=20.1, used=16037916672, free=50175258624, active=40315944960, inactive=7592177664, buffers=31571968, cached=33687412736, shared=2190573568, slab=776224768) +2022-04-12 01:28:23.512206: +epoch: 250 +2022-04-12 01:30:12.040034: train loss : -0.6395 +2022-04-12 01:30:24.076734: validation loss: -0.6461 +2022-04-12 01:30:24.078628: Average global foreground Dice: [0.8644, 0.9625, 0.9802, 0.3729] +2022-04-12 01:30:24.079616: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:30:24.952123: lr: 0.00534 +2022-04-12 01:30:24.953663: saving scheduled checkpoint file... +2022-04-12 01:30:25.081604: saving checkpoint... +2022-04-12 01:30:30.175006: done, saving took 5.22 seconds +2022-04-12 01:30:30.195087: done +2022-04-12 01:30:30.291049: saving checkpoint... +2022-04-12 01:30:34.865947: done, saving took 4.67 seconds +2022-04-12 01:30:34.885839: This epoch took 131.372215 s + +2022-04-12 01:30:34.887189: svmem(total=99932160000, available=79984992256, percent=20.0, used=15931863040, free=50263232512, active=40204861440, inactive=7609675776, buffers=31612928, cached=33705451520, shared=2190729216, slab=777277440) +2022-04-12 01:30:34.888237: +epoch: 251 +2022-04-12 01:32:22.597718: train loss : -0.6474 +2022-04-12 01:32:34.273633: validation loss: -0.6278 +2022-04-12 01:32:34.276098: Average global foreground Dice: [0.8813, 0.9592, 0.9797, 0.3598] +2022-04-12 01:32:34.278037: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:32:35.299172: lr: 0.00532 +2022-04-12 01:32:35.300874: saving scheduled checkpoint file... +2022-04-12 01:32:35.386181: saving checkpoint... +2022-04-12 01:32:39.948356: done, saving took 4.65 seconds +2022-04-12 01:32:39.963848: done +2022-04-12 01:32:40.010616: saving checkpoint... +2022-04-12 01:32:44.613955: done, saving took 4.65 seconds +2022-04-12 01:32:44.628761: This epoch took 129.739854 s + +2022-04-12 01:32:44.629918: svmem(total=99932160000, available=79909629952, percent=20.0, used=16008810496, free=50180452352, active=40286642176, inactive=7615971328, buffers=31645696, cached=33711251456, shared=2189053952, slab=777068544) +2022-04-12 01:32:44.630744: +epoch: 252 +2022-04-12 01:34:33.323064: train loss : -0.6522 +2022-04-12 01:34:43.418005: validation loss: -0.6256 +2022-04-12 01:34:43.429957: Average global foreground Dice: [0.8621, 0.9624, 0.9795, 0.379] +2022-04-12 01:34:43.430883: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:34:44.237545: lr: 0.005301 +2022-04-12 01:34:44.238891: saving scheduled checkpoint file... +2022-04-12 01:34:44.330616: saving checkpoint... +2022-04-12 01:34:49.108682: done, saving took 4.87 seconds +2022-04-12 01:34:49.121962: done +2022-04-12 01:34:49.172323: saving checkpoint... +2022-04-12 01:34:53.698004: done, saving took 4.57 seconds +2022-04-12 01:34:53.740160: This epoch took 129.108587 s + +2022-04-12 01:34:53.742312: svmem(total=99932160000, available=79868080128, percent=20.1, used=16048926720, free=50121252864, active=40328302592, inactive=7634685952, buffers=31678464, cached=33730301952, shared=2190618624, slab=776269824) +2022-04-12 01:34:53.743340: +epoch: 253 +2022-04-12 01:36:42.568318: train loss : -0.6527 +2022-04-12 01:36:54.111129: validation loss: -0.6139 +2022-04-12 01:36:54.114025: Average global foreground Dice: [0.8812, 0.9523, 0.9731, 0.3621] +2022-04-12 01:36:54.115401: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:36:54.993103: lr: 0.005282 +2022-04-12 01:36:54.994860: saving scheduled checkpoint file... +2022-04-12 01:36:55.087555: saving checkpoint... +2022-04-12 01:36:59.646118: done, saving took 4.65 seconds +2022-04-12 01:36:59.658543: done +2022-04-12 01:36:59.659776: This epoch took 125.915499 s + +2022-04-12 01:36:59.660859: svmem(total=99932160000, available=79875649536, percent=20.1, used=16030900224, free=50119688192, active=40045588480, inactive=7916613632, buffers=31731712, cached=33749839872, shared=2200940544, slab=776097792) +2022-04-12 01:36:59.661743: +epoch: 254 +2022-04-12 01:38:47.599554: train loss : -0.6213 +2022-04-12 01:38:58.828093: validation loss: -0.6281 +2022-04-12 01:38:58.910892: Average global foreground Dice: [0.872, 0.9634, 0.977, 0.3862] +2022-04-12 01:38:58.912035: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:38:59.778232: lr: 0.005262 +2022-04-12 01:38:59.779959: saving scheduled checkpoint file... +2022-04-12 01:38:59.861745: saving checkpoint... +2022-04-12 01:39:04.849119: done, saving took 5.07 seconds +2022-04-12 01:39:04.862465: done +2022-04-12 01:39:04.917884: saving checkpoint... +2022-04-12 01:39:09.596375: done, saving took 4.73 seconds +2022-04-12 01:39:09.611574: This epoch took 129.948676 s + +2022-04-12 01:39:09.613153: svmem(total=99932160000, available=79860936704, percent=20.1, used=16080343040, free=50110763008, active=40152653824, inactive=7817453568, buffers=31768576, cached=33709285376, shared=2166255616, slab=776687616) +2022-04-12 01:39:09.614033: +epoch: 255 +2022-04-12 01:40:57.026991: train loss : -0.6228 +2022-04-12 01:41:08.566908: validation loss: -0.6047 +2022-04-12 01:41:08.578172: Average global foreground Dice: [0.8609, 0.9534, 0.9787, 0.3337] +2022-04-12 01:41:08.579581: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:41:09.454185: lr: 0.005243 +2022-04-12 01:41:09.456002: saving scheduled checkpoint file... +2022-04-12 01:41:09.558212: saving checkpoint... +2022-04-12 01:41:14.346147: done, saving took 4.89 seconds +2022-04-12 01:41:14.388235: done +2022-04-12 01:41:14.389714: This epoch took 124.774821 s + +2022-04-12 01:41:14.391144: svmem(total=99932160000, available=79941464064, percent=20.0, used=15986733056, free=50174779392, active=40187002880, inactive=7717621760, buffers=31825920, cached=33738821632, shared=2179330048, slab=776634368) +2022-04-12 01:41:14.391948: +epoch: 256 +2022-04-12 01:43:03.805986: train loss : -0.6264 +2022-04-12 01:43:16.131690: validation loss: -0.6504 +2022-04-12 01:43:16.142662: Average global foreground Dice: [0.8796, 0.9614, 0.9767, 0.3994] +2022-04-12 01:43:16.209959: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:43:17.072134: lr: 0.005224 +2022-04-12 01:43:17.079187: saving scheduled checkpoint file... +2022-04-12 01:43:17.203722: saving checkpoint... +2022-04-12 01:43:23.900250: done, saving took 6.81 seconds +2022-04-12 01:43:23.985034: done +2022-04-12 01:43:24.106681: saving checkpoint... +2022-04-12 01:43:28.973270: done, saving took 4.96 seconds +2022-04-12 01:43:29.060711: This epoch took 134.668016 s + +2022-04-12 01:43:29.063894: svmem(total=99932160000, available=80127725568, percent=19.8, used=15876317184, free=50356826112, active=40019726336, inactive=7706583040, buffers=31858688, cached=33667158016, shared=2103431168, slab=775000064) +2022-04-12 01:43:29.065590: +epoch: 257 +2022-04-12 01:45:17.817092: train loss : -0.6264 +2022-04-12 01:45:29.060188: validation loss: -0.6215 +2022-04-12 01:45:29.062333: Average global foreground Dice: [0.872, 0.9615, 0.9796, 0.3728] +2022-04-12 01:45:29.063322: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:45:30.056278: lr: 0.005204 +2022-04-12 01:45:30.062080: saving scheduled checkpoint file... +2022-04-12 01:45:30.155818: saving checkpoint... +2022-04-12 01:45:35.090546: done, saving took 5.03 seconds +2022-04-12 01:45:35.161679: done +2022-04-12 01:45:35.208636: saving checkpoint... +2022-04-12 01:45:40.971639: done, saving took 5.81 seconds +2022-04-12 01:45:40.987019: This epoch took 131.806210 s + +2022-04-12 01:45:40.988565: svmem(total=99932160000, available=79832379392, percent=20.1, used=16098254848, free=50043998208, active=40320917504, inactive=7720337408, buffers=31907840, cached=33757999104, shared=2176741376, slab=775880704) +2022-04-12 01:45:40.989319: +epoch: 258 +2022-04-12 01:47:29.409359: train loss : -0.6194 +2022-04-12 01:47:40.961527: validation loss: -0.5740 +2022-04-12 01:47:41.003032: Average global foreground Dice: [0.8309, 0.9475, 0.9754, 0.3633] +2022-04-12 01:47:41.023257: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:47:41.946558: lr: 0.005185 +2022-04-12 01:47:41.980726: saving scheduled checkpoint file... +2022-04-12 01:47:42.059349: saving checkpoint... +2022-04-12 01:47:46.998850: done, saving took 5.01 seconds +2022-04-12 01:47:47.009055: done +2022-04-12 01:47:47.010169: This epoch took 126.020070 s + +2022-04-12 01:47:47.011381: svmem(total=99932160000, available=79865475072, percent=20.1, used=16080564224, free=50068303872, active=40228900864, inactive=7791370240, buffers=31940608, cached=33751351296, shared=2161336320, slab=774864896) +2022-04-12 01:47:47.012216: +epoch: 259 +2022-04-12 01:49:34.499650: train loss : -0.6236 +2022-04-12 01:49:47.034407: validation loss: -0.5801 +2022-04-12 01:49:47.091233: Average global foreground Dice: [0.8585, 0.947, 0.9625, 0.3514] +2022-04-12 01:49:47.106396: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:49:47.957383: lr: 0.005166 +2022-04-12 01:49:47.963050: saving scheduled checkpoint file... +2022-04-12 01:49:48.040575: saving checkpoint... +2022-04-12 01:49:54.202917: done, saving took 6.22 seconds +2022-04-12 01:49:54.250988: done +2022-04-12 01:49:54.252387: This epoch took 127.239410 s + +2022-04-12 01:49:54.253848: svmem(total=99932160000, available=79966089216, percent=20.0, used=15950491648, free=50145533952, active=40173232128, inactive=7766974464, buffers=31973376, cached=33804161024, shared=2190618624, slab=773599232) +2022-04-12 01:49:54.254575: +epoch: 260 +2022-04-12 01:51:42.527432: train loss : -0.6296 +2022-04-12 01:51:54.830186: validation loss: -0.6588 +2022-04-12 01:51:54.868195: Average global foreground Dice: [0.8812, 0.9639, 0.9841, 0.4257] +2022-04-12 01:51:54.877696: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:51:55.833999: lr: 0.005146 +2022-04-12 01:51:55.876085: saving scheduled checkpoint file... +2022-04-12 01:51:55.980448: saving checkpoint... +2022-04-12 01:52:02.747057: done, saving took 6.85 seconds +2022-04-12 01:52:02.843550: done +2022-04-12 01:52:02.879999: This epoch took 128.624648 s + +2022-04-12 01:52:02.909473: svmem(total=99932160000, available=79672881152, percent=20.3, used=16243621888, free=49852862464, active=40318009344, inactive=7914999808, buffers=32014336, cached=33803661312, shared=2190729216, slab=774365184) +2022-04-12 01:52:02.918486: +epoch: 261 +2022-04-12 01:53:51.362512: train loss : -0.6383 +2022-04-12 01:54:02.983579: validation loss: -0.6290 +2022-04-12 01:54:03.019319: Average global foreground Dice: [0.8644, 0.9608, 0.9815, 0.3715] +2022-04-12 01:54:03.066762: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:54:04.053620: lr: 0.005127 +2022-04-12 01:54:04.082645: saving scheduled checkpoint file... +2022-04-12 01:54:04.193649: saving checkpoint... +2022-04-12 01:54:09.578367: done, saving took 5.48 seconds +2022-04-12 01:54:09.624074: done +2022-04-12 01:54:09.631139: This epoch took 126.708700 s + +2022-04-12 01:54:09.644097: svmem(total=99932160000, available=79478005760, percent=20.5, used=16448847872, free=49645817856, active=40512172032, inactive=7917154304, buffers=32047104, cached=33805447168, shared=2180431872, slab=774586368) +2022-04-12 01:54:09.668816: +epoch: 262 +2022-04-12 01:55:56.950029: train loss : -0.6433 +2022-04-12 01:56:08.273223: validation loss: -0.6362 +2022-04-12 01:56:08.287781: Average global foreground Dice: [0.8875, 0.9622, 0.9804, 0.3682] +2022-04-12 01:56:08.290739: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:56:09.114484: lr: 0.005107 +2022-04-12 01:56:09.182083: saving scheduled checkpoint file... +2022-04-12 01:56:09.306993: saving checkpoint... +2022-04-12 01:56:14.915943: done, saving took 5.73 seconds +2022-04-12 01:56:15.070525: done +2022-04-12 01:56:15.172286: saving checkpoint... +2022-04-12 01:56:21.349952: done, saving took 6.27 seconds +2022-04-12 01:56:21.438828: This epoch took 131.544664 s + +2022-04-12 01:56:21.474294: svmem(total=99932160000, available=79817297920, percent=20.1, used=16099631104, free=49943412736, active=40288649216, inactive=7842586624, buffers=32243712, cached=33856872448, shared=2190344192, slab=776757248) +2022-04-12 01:56:21.487511: +epoch: 263 +2022-04-12 01:58:10.028056: train loss : -0.6474 +2022-04-12 01:58:21.707535: validation loss: -0.6295 +2022-04-12 01:58:21.748494: Average global foreground Dice: [0.8841, 0.9684, 0.9831, 0.3518] +2022-04-12 01:58:21.762901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 01:58:22.755299: lr: 0.005088 +2022-04-12 01:58:22.776241: saving scheduled checkpoint file... +2022-04-12 01:58:22.897058: saving checkpoint... +2022-04-12 01:58:29.762529: done, saving took 6.97 seconds +2022-04-12 01:58:29.895148: done +2022-04-12 01:58:29.989035: saving checkpoint... +2022-04-12 01:58:36.084859: done, saving took 6.14 seconds +2022-04-12 01:58:36.166131: This epoch took 134.673177 s + +2022-04-12 01:58:36.191236: svmem(total=99932160000, available=79781531648, percent=20.2, used=16134733824, free=49941479424, active=40083230720, inactive=8058630144, buffers=32276480, cached=33823670272, shared=2190807040, slab=775917568) +2022-04-12 01:58:36.224824: +epoch: 264 +2022-04-12 02:00:23.832481: train loss : -0.6435 +2022-04-12 02:00:35.181497: validation loss: -0.6560 +2022-04-12 02:00:35.191462: Average global foreground Dice: [0.8917, 0.9631, 0.9832, 0.3897] +2022-04-12 02:00:35.194274: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:00:36.235883: lr: 0.005069 +2022-04-12 02:00:36.320238: saving scheduled checkpoint file... +2022-04-12 02:00:36.418257: saving checkpoint... +2022-04-12 02:00:42.126929: done, saving took 5.80 seconds +2022-04-12 02:00:42.232107: done +2022-04-12 02:00:42.298285: saving checkpoint... +2022-04-12 02:00:51.363984: done, saving took 9.11 seconds +2022-04-12 02:00:51.429657: This epoch took 135.195769 s + +2022-04-12 02:00:51.447474: svmem(total=99932160000, available=79646121984, percent=20.3, used=16270868480, free=49751277568, active=40390119424, inactive=7944650752, buffers=32317440, cached=33877696512, shared=2190233600, slab=776142848) +2022-04-12 02:00:51.495644: +epoch: 265 +2022-04-12 02:02:39.606662: train loss : -0.6495 +2022-04-12 02:02:50.809932: validation loss: -0.5953 +2022-04-12 02:02:50.876163: Average global foreground Dice: [0.8746, 0.9549, 0.9672, 0.3601] +2022-04-12 02:02:50.880292: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:02:51.908946: lr: 0.005049 +2022-04-12 02:02:51.980430: saving scheduled checkpoint file... +2022-04-12 02:02:52.031898: saving checkpoint... +2022-04-12 02:02:59.020088: done, saving took 7.04 seconds +2022-04-12 02:02:59.051882: done +2022-04-12 02:02:59.053129: This epoch took 127.535869 s + +2022-04-12 02:02:59.054197: svmem(total=99932160000, available=79632949248, percent=20.3, used=16283467776, free=49716850688, active=40521285632, inactive=7839817728, buffers=32350208, cached=33899491328, shared=2190807040, slab=777555968) +2022-04-12 02:02:59.055063: +epoch: 266 +2022-04-12 02:04:46.761834: train loss : -0.6182 +2022-04-12 02:04:57.758146: validation loss: -0.6225 +2022-04-12 02:04:57.803689: Average global foreground Dice: [0.8835, 0.9615, 0.9758, 0.368] +2022-04-12 02:04:57.808799: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:04:58.915203: lr: 0.00503 +2022-04-12 02:04:58.921572: saving scheduled checkpoint file... +2022-04-12 02:04:59.074719: saving checkpoint... +2022-04-12 02:05:06.576686: done, saving took 7.60 seconds +2022-04-12 02:05:06.705746: done +2022-04-12 02:05:06.755011: This epoch took 127.699107 s + +2022-04-12 02:05:06.758447: svmem(total=99932160000, available=79782232064, percent=20.2, used=16134463488, free=49848635392, active=40384610304, inactive=7843287040, buffers=32391168, cached=33916669952, shared=2190540800, slab=775819264) +2022-04-12 02:05:06.771628: +epoch: 267 +2022-04-12 02:06:55.298761: train loss : -0.6247 +2022-04-12 02:07:06.818097: validation loss: -0.6351 +2022-04-12 02:07:06.829429: Average global foreground Dice: [0.8798, 0.9583, 0.9752, 0.3667] +2022-04-12 02:07:06.843171: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:07:07.737034: lr: 0.00501 +2022-04-12 02:07:07.771700: saving scheduled checkpoint file... +2022-04-12 02:07:07.982096: saving checkpoint... +2022-04-12 02:07:15.631059: done, saving took 7.75 seconds +2022-04-12 02:07:15.704219: done +2022-04-12 02:07:15.759455: This epoch took 128.970209 s + +2022-04-12 02:07:15.769544: svmem(total=99932160000, available=79881515008, percent=20.1, used=16035139584, free=49928040448, active=40294375424, inactive=7855501312, buffers=32473088, cached=33936506880, shared=2190417920, slab=777043968) +2022-04-12 02:07:15.799598: +epoch: 268 +2022-04-12 02:09:03.597288: train loss : -0.6469 +2022-04-12 02:09:15.067140: validation loss: -0.6469 +2022-04-12 02:09:15.089373: Average global foreground Dice: [0.8925, 0.9617, 0.9776, 0.3902] +2022-04-12 02:09:15.157933: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:09:15.898090: lr: 0.004991 +2022-04-12 02:09:15.972888: saving scheduled checkpoint file... +2022-04-12 02:09:16.111256: saving checkpoint... +2022-04-12 02:09:24.990935: done, saving took 8.99 seconds +2022-04-12 02:09:25.079532: done +2022-04-12 02:09:25.193744: saving checkpoint... +2022-04-12 02:09:32.400495: done, saving took 7.31 seconds +2022-04-12 02:09:32.483541: This epoch took 136.635700 s + +2022-04-12 02:09:32.527383: svmem(total=99932160000, available=79740579840, percent=20.2, used=16175759360, free=49775857664, active=40443949056, inactive=7859974144, buffers=32505856, cached=33948037120, shared=2190807040, slab=777416704) +2022-04-12 02:09:32.530023: +epoch: 269 +2022-04-12 02:11:22.915982: train loss : -0.6560 +2022-04-12 02:11:35.442690: validation loss: -0.6548 +2022-04-12 02:11:35.445962: Average global foreground Dice: [0.8942, 0.9663, 0.9781, 0.3873] +2022-04-12 02:11:35.447673: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:11:36.264742: lr: 0.004971 +2022-04-12 02:11:36.266857: saving scheduled checkpoint file... +2022-04-12 02:11:36.359648: saving checkpoint... +2022-04-12 02:11:43.078735: done, saving took 6.81 seconds +2022-04-12 02:11:43.092972: done +2022-04-12 02:11:43.140005: saving checkpoint... +2022-04-12 02:11:49.503776: done, saving took 6.41 seconds +2022-04-12 02:11:49.527054: This epoch took 136.991721 s + +2022-04-12 02:11:49.528743: svmem(total=99932160000, available=79990591488, percent=20.0, used=15925825536, free=50008416256, active=40196431872, inactive=7871750144, buffers=32546816, cached=33965371392, shared=2190618624, slab=778371072) +2022-04-12 02:11:49.543161: +epoch: 270 +2022-04-12 02:13:37.822139: train loss : -0.6532 +2022-04-12 02:13:49.306508: validation loss: -0.6452 +2022-04-12 02:13:49.368310: Average global foreground Dice: [0.8966, 0.9671, 0.9798, 0.3724] +2022-04-12 02:13:49.373835: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:13:50.281996: lr: 0.004952 +2022-04-12 02:13:50.299627: saving scheduled checkpoint file... +2022-04-12 02:13:50.448653: saving checkpoint... +2022-04-12 02:13:56.004242: done, saving took 5.65 seconds +2022-04-12 02:13:56.043928: done +2022-04-12 02:13:56.098092: saving checkpoint... +2022-04-12 02:14:02.317302: done, saving took 6.27 seconds +2022-04-12 02:14:02.395725: This epoch took 132.819117 s + +2022-04-12 02:14:02.481098: svmem(total=99932160000, available=79901650944, percent=20.0, used=16014409728, free=49902014464, active=40290893824, inactive=7886962688, buffers=32579584, cached=33983156224, shared=2190917632, slab=777592832) +2022-04-12 02:14:02.519717: +epoch: 271 +2022-04-12 02:15:51.389760: train loss : -0.6574 +2022-04-12 02:16:03.048972: validation loss: -0.6387 +2022-04-12 02:16:03.117362: Average global foreground Dice: [0.8676, 0.9586, 0.9787, 0.3681] +2022-04-12 02:16:03.132841: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:16:04.124740: lr: 0.004933 +2022-04-12 02:16:04.159472: saving scheduled checkpoint file... +2022-04-12 02:16:04.271675: saving checkpoint... +2022-04-12 02:16:13.258485: done, saving took 9.07 seconds +2022-04-12 02:16:13.354047: done +2022-04-12 02:16:13.409614: This epoch took 130.858093 s + +2022-04-12 02:16:13.450482: svmem(total=99932160000, available=79755857920, percent=20.2, used=16149901312, free=49747795968, active=40426901504, inactive=7905681408, buffers=32645120, cached=34001817600, shared=2201182208, slab=777183232) +2022-04-12 02:16:13.487817: +epoch: 272 +2022-04-12 02:18:00.999104: train loss : -0.6598 +2022-04-12 02:18:12.182311: validation loss: -0.6468 +2022-04-12 02:18:12.258487: Average global foreground Dice: [0.8851, 0.9602, 0.9824, 0.3707] +2022-04-12 02:18:12.308395: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:18:13.336353: lr: 0.004913 +2022-04-12 02:18:13.348330: saving scheduled checkpoint file... +2022-04-12 02:18:13.439811: saving checkpoint... +2022-04-12 02:18:21.196804: done, saving took 7.81 seconds +2022-04-12 02:18:21.286978: done +2022-04-12 02:18:21.332265: This epoch took 127.842139 s + +2022-04-12 02:18:21.342961: svmem(total=99932160000, available=79652753408, percent=20.3, used=16263507968, free=49628155904, active=40536891392, inactive=7911686144, buffers=32698368, cached=34007797760, shared=2190807040, slab=777142272) +2022-04-12 02:18:21.365779: +epoch: 273 +2022-04-12 02:20:09.210201: train loss : -0.6618 +2022-04-12 02:20:20.431598: validation loss: -0.6277 +2022-04-12 02:20:20.515678: Average global foreground Dice: [0.8794, 0.9572, 0.9758, 0.3654] +2022-04-12 02:20:20.588551: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:20:21.488099: lr: 0.004894 +2022-04-12 02:20:21.560446: saving scheduled checkpoint file... +2022-04-12 02:20:21.697147: saving checkpoint... +2022-04-12 02:20:31.730167: done, saving took 10.11 seconds +2022-04-12 02:20:31.823408: done +2022-04-12 02:20:31.844547: This epoch took 130.459301 s + +2022-04-12 02:20:31.849711: svmem(total=99932160000, available=79969259520, percent=20.0, used=15947132928, free=49927307264, active=40224391168, inactive=7928721408, buffers=32739328, cached=34024980480, shared=2190540800, slab=776093696) +2022-04-12 02:20:31.874210: +epoch: 274 +2022-04-12 02:22:19.879729: train loss : -0.6654 +2022-04-12 02:22:31.164845: validation loss: -0.6617 +2022-04-12 02:22:31.280957: Average global foreground Dice: [0.887, 0.965, 0.9804, 0.3967] +2022-04-12 02:22:31.369238: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:22:32.269444: lr: 0.004874 +2022-04-12 02:22:32.392944: saving scheduled checkpoint file... +2022-04-12 02:22:32.495522: saving checkpoint... +2022-04-12 02:22:41.607801: done, saving took 9.21 seconds +2022-04-12 02:22:41.678344: done +2022-04-12 02:22:41.788513: saving checkpoint... +2022-04-12 02:22:47.210124: done, saving took 5.52 seconds +2022-04-12 02:22:47.229649: This epoch took 135.320040 s + +2022-04-12 02:22:47.231353: svmem(total=99932160000, available=79844298752, percent=20.1, used=16071897088, free=49783808000, active=40351580160, inactive=7947350016, buffers=32788480, cached=34043666432, shared=2190729216, slab=777396224) +2022-04-12 02:22:47.232406: +epoch: 275 +2022-04-12 02:24:36.314391: train loss : -0.6051 +2022-04-12 02:24:48.058411: validation loss: -0.6074 +2022-04-12 02:24:48.067997: Average global foreground Dice: [0.8842, 0.9578, 0.9692, 0.3858] +2022-04-12 02:24:48.069554: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:24:49.432690: lr: 0.004855 +2022-04-12 02:24:49.434979: saving scheduled checkpoint file... +2022-04-12 02:24:49.519763: saving checkpoint... +2022-04-12 02:24:53.942266: done, saving took 4.51 seconds +2022-04-12 02:24:53.951633: done +2022-04-12 02:24:53.999340: saving checkpoint... +2022-04-12 02:24:59.962617: done, saving took 6.01 seconds +2022-04-12 02:25:00.026966: This epoch took 132.793146 s + +2022-04-12 02:25:00.060329: svmem(total=99932160000, available=79785226240, percent=20.2, used=16131059712, free=49716850688, active=40409325568, inactive=7954956288, buffers=32821248, cached=34051428352, shared=2190573568, slab=776802304) +2022-04-12 02:25:00.143342: +epoch: 276 +2022-04-12 02:26:47.209231: train loss : -0.6018 +2022-04-12 02:26:58.583355: validation loss: -0.6277 +2022-04-12 02:26:58.589995: Average global foreground Dice: [0.8851, 0.9557, 0.9795, 0.3748] +2022-04-12 02:26:58.591621: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:26:59.548100: lr: 0.004835 +2022-04-12 02:26:59.549549: saving scheduled checkpoint file... +2022-04-12 02:26:59.644302: saving checkpoint... +2022-04-12 02:27:06.232373: done, saving took 6.68 seconds +2022-04-12 02:27:06.287514: done +2022-04-12 02:27:06.340071: saving checkpoint... +2022-04-12 02:27:13.868633: done, saving took 7.58 seconds +2022-04-12 02:27:13.923207: This epoch took 133.776519 s + +2022-04-12 02:27:13.924483: svmem(total=99932160000, available=79919067136, percent=20.0, used=15997423616, free=49833205760, active=40269438976, inactive=7972659200, buffers=32878592, cached=34068652032, shared=2190344192, slab=776400896) +2022-04-12 02:27:13.925274: +epoch: 277 +2022-04-12 02:29:01.784641: train loss : -0.6475 +2022-04-12 02:29:12.649037: validation loss: -0.6202 +2022-04-12 02:29:12.677171: Average global foreground Dice: [0.8684, 0.9582, 0.9721, 0.3604] +2022-04-12 02:29:12.702727: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:29:13.686198: lr: 0.004816 +2022-04-12 02:29:13.689180: saving scheduled checkpoint file... +2022-04-12 02:29:13.772221: saving checkpoint... +2022-04-12 02:29:18.374947: done, saving took 4.68 seconds +2022-04-12 02:29:18.385284: done +2022-04-12 02:29:18.386596: This epoch took 124.459978 s + +2022-04-12 02:29:18.387582: svmem(total=99932160000, available=79762071552, percent=20.2, used=16154116096, free=49658638336, active=40435417088, inactive=7990091776, buffers=32911360, cached=34086494208, shared=2190729216, slab=775995392) +2022-04-12 02:29:18.388459: +epoch: 278 +2022-04-12 02:31:06.134234: train loss : -0.6396 +2022-04-12 02:31:17.619841: validation loss: -0.6140 +2022-04-12 02:31:17.686169: Average global foreground Dice: [0.8696, 0.9651, 0.975, 0.3646] +2022-04-12 02:31:17.699166: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:31:18.663262: lr: 0.004796 +2022-04-12 02:31:18.709903: saving scheduled checkpoint file... +2022-04-12 02:31:18.850754: saving checkpoint... +2022-04-12 02:31:27.240229: done, saving took 8.49 seconds +2022-04-12 02:31:27.375631: done +2022-04-12 02:31:27.404340: This epoch took 129.015066 s + +2022-04-12 02:31:27.407728: svmem(total=99932160000, available=79799332864, percent=20.1, used=16107016192, free=49689067520, active=40385191936, inactive=8006451200, buffers=32952320, cached=34103123968, shared=2200453120, slab=776241152) +2022-04-12 02:31:27.412444: +epoch: 279 +2022-04-12 02:33:15.524088: train loss : -0.6598 +2022-04-12 02:33:27.135518: validation loss: -0.6357 +2022-04-12 02:33:27.241375: Average global foreground Dice: [0.8822, 0.9639, 0.9808, 0.3659] +2022-04-12 02:33:27.249692: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:33:28.233598: lr: 0.004776 +2022-04-12 02:33:28.241883: saving scheduled checkpoint file... +2022-04-12 02:33:28.376546: saving checkpoint... +2022-04-12 02:33:38.376907: done, saving took 10.12 seconds +2022-04-12 02:33:38.587896: done +2022-04-12 02:33:38.594618: This epoch took 131.173576 s + +2022-04-12 02:33:38.600444: svmem(total=99932160000, available=79855648768, percent=20.1, used=16060366848, free=49736765440, active=40288841728, inactive=8058970112, buffers=32985088, cached=34102042624, shared=2190729216, slab=776523776) +2022-04-12 02:33:38.666582: +epoch: 280 +2022-04-12 02:35:26.844504: train loss : -0.6588 +2022-04-12 02:35:37.854278: validation loss: -0.6557 +2022-04-12 02:35:37.917867: Average global foreground Dice: [0.8883, 0.9682, 0.9826, 0.3775] +2022-04-12 02:35:38.000482: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:35:38.922928: lr: 0.004757 +2022-04-12 02:35:38.988548: saving scheduled checkpoint file... +2022-04-12 02:35:39.189038: saving checkpoint... +2022-04-12 02:35:49.523470: done, saving took 10.47 seconds +2022-04-12 02:35:49.685880: done +2022-04-12 02:35:49.690755: This epoch took 131.010363 s + +2022-04-12 02:35:49.747896: svmem(total=99932160000, available=79622553600, percent=20.3, used=16294043648, free=49519759360, active=40395177984, inactive=8161624064, buffers=33034240, cached=34085322752, shared=2190336000, slab=776937472) +2022-04-12 02:35:49.837749: +epoch: 281 +2022-04-12 02:37:38.344573: train loss : -0.6659 +2022-04-12 02:37:49.431167: validation loss: -0.6398 +2022-04-12 02:37:49.461115: Average global foreground Dice: [0.8855, 0.9606, 0.9805, 0.3798] +2022-04-12 02:37:49.473690: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:37:50.469052: lr: 0.004737 +2022-04-12 02:37:50.471655: saving scheduled checkpoint file... +2022-04-12 02:37:50.557084: saving checkpoint... +2022-04-12 02:37:58.027806: done, saving took 7.55 seconds +2022-04-12 02:37:58.037263: done +2022-04-12 02:37:58.038563: This epoch took 128.191859 s + +2022-04-12 02:37:58.039518: svmem(total=99932160000, available=79943827456, percent=20.0, used=15972392960, free=49834401792, active=40182730752, inactive=8063586304, buffers=33083392, cached=34092281856, shared=2190729216, slab=777220096) +2022-04-12 02:37:58.040346: +epoch: 282 +2022-04-12 02:39:46.265814: train loss : -0.6427 +2022-04-12 02:39:58.173138: validation loss: -0.6187 +2022-04-12 02:39:58.247330: Average global foreground Dice: [0.8884, 0.9609, 0.9719, 0.3671] +2022-04-12 02:39:58.312639: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:39:59.266834: lr: 0.004718 +2022-04-12 02:39:59.295655: saving scheduled checkpoint file... +2022-04-12 02:39:59.394732: saving checkpoint... +2022-04-12 02:40:06.773607: done, saving took 7.46 seconds +2022-04-12 02:40:06.789159: done +2022-04-12 02:40:06.791085: This epoch took 128.749923 s + +2022-04-12 02:40:06.792760: svmem(total=99932160000, available=79809314816, percent=20.1, used=16099852288, free=49682378752, active=40370290688, inactive=8021913600, buffers=33124352, cached=34116804608, shared=2197667840, slab=777596928) +2022-04-12 02:40:06.793591: +epoch: 283 +2022-04-12 02:41:54.994434: train loss : -0.6587 +2022-04-12 02:42:06.918906: validation loss: -0.6316 +2022-04-12 02:42:06.985284: Average global foreground Dice: [0.8718, 0.9641, 0.9727, 0.3731] +2022-04-12 02:42:07.027754: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:42:08.001805: lr: 0.004698 +2022-04-12 02:42:08.005200: saving scheduled checkpoint file... +2022-04-12 02:42:08.117879: saving checkpoint... +2022-04-12 02:42:12.932290: done, saving took 4.91 seconds +2022-04-12 02:42:12.948392: done +2022-04-12 02:42:12.949891: This epoch took 126.155425 s + +2022-04-12 02:42:12.951296: svmem(total=99932160000, available=79683284992, percent=20.3, used=16232677376, free=49538965504, active=40512438272, inactive=8030740480, buffers=33173504, cached=34127343616, shared=2190917632, slab=776716288) +2022-04-12 02:42:12.952257: +epoch: 284 +2022-04-12 02:44:00.560697: train loss : -0.6585 +2022-04-12 02:44:12.530535: validation loss: -0.6427 +2022-04-12 02:44:12.608448: Average global foreground Dice: [0.8764, 0.9631, 0.9806, 0.3839] +2022-04-12 02:44:12.612065: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:44:13.356845: lr: 0.004679 +2022-04-12 02:44:13.407324: saving scheduled checkpoint file... +2022-04-12 02:44:13.552935: saving checkpoint... +2022-04-12 02:44:21.812483: done, saving took 8.35 seconds +2022-04-12 02:44:21.857311: done +2022-04-12 02:44:21.895747: This epoch took 128.942598 s + +2022-04-12 02:44:21.898681: svmem(total=99932160000, available=79852736512, percent=20.1, used=16053125120, free=49700704256, active=40331370496, inactive=8048582656, buffers=33206272, cached=34145124352, shared=2201026560, slab=776269824) +2022-04-12 02:44:21.956736: +epoch: 285 +2022-04-12 02:46:10.852490: train loss : -0.6570 +2022-04-12 02:46:22.236456: validation loss: -0.6443 +2022-04-12 02:46:22.241785: Average global foreground Dice: [0.8873, 0.9663, 0.9785, 0.3456] +2022-04-12 02:46:22.263235: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:46:23.260025: lr: 0.004659 +2022-04-12 02:46:23.356533: saving scheduled checkpoint file... +2022-04-12 02:46:23.556548: saving checkpoint... +2022-04-12 02:46:32.696720: done, saving took 9.26 seconds +2022-04-12 02:46:33.071140: done +2022-04-12 02:46:33.142063: This epoch took 131.165314 s + +2022-04-12 02:46:33.148436: svmem(total=99932160000, available=79760936960, percent=20.2, used=16155238400, free=49591349248, active=40431198208, inactive=8055611392, buffers=33251328, cached=34152321024, shared=2190651392, slab=777576448) +2022-04-12 02:46:33.155120: +epoch: 286 +2022-04-12 02:48:24.145887: train loss : -0.6619 +2022-04-12 02:48:36.176081: validation loss: -0.6533 +2022-04-12 02:48:36.234839: Average global foreground Dice: [0.897, 0.9632, 0.9814, 0.3596] +2022-04-12 02:48:36.262976: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:48:37.112346: lr: 0.004639 +2022-04-12 02:48:37.165543: saving scheduled checkpoint file... +2022-04-12 02:48:37.386722: saving checkpoint... +2022-04-12 02:48:44.787164: done, saving took 7.53 seconds +2022-04-12 02:48:44.920516: done +2022-04-12 02:48:44.942625: This epoch took 131.717814 s + +2022-04-12 02:48:44.978299: svmem(total=99932160000, available=79690039296, percent=20.3, used=16226025472, free=49503002624, active=40507125760, inactive=8072962048, buffers=33284096, cached=34169847808, shared=2190651392, slab=777211904) +2022-04-12 02:48:44.979592: +epoch: 287 +2022-04-12 02:50:35.213799: train loss : -0.6579 +2022-04-12 02:50:47.784644: validation loss: -0.6460 +2022-04-12 02:50:47.811979: Average global foreground Dice: [0.8951, 0.9595, 0.9807, 0.3848] +2022-04-12 02:50:47.818761: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:50:48.816232: lr: 0.00462 +2022-04-12 02:50:48.822372: saving scheduled checkpoint file... +2022-04-12 02:50:48.973221: saving checkpoint... +2022-04-12 02:50:58.841486: done, saving took 9.95 seconds +2022-04-12 02:50:59.072556: done +2022-04-12 02:50:59.131584: saving checkpoint... +2022-04-12 02:51:09.253289: done, saving took 10.17 seconds +2022-04-12 02:51:09.420940: This epoch took 144.437532 s + +2022-04-12 02:51:09.438040: svmem(total=99932160000, available=80043048960, percent=19.9, used=15873064960, free=49794703360, active=40145969152, inactive=8134299648, buffers=33325056, cached=34231066624, shared=2190729216, slab=775917568) +2022-04-12 02:51:09.500869: +epoch: 288 +2022-04-12 02:52:57.775733: train loss : -0.6834 +2022-04-12 02:53:09.014524: validation loss: -0.6574 +2022-04-12 02:53:09.022869: Average global foreground Dice: [0.8674, 0.9616, 0.9809, 0.4067] +2022-04-12 02:53:09.032990: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:53:10.131468: lr: 0.0046 +2022-04-12 02:53:10.175926: saving scheduled checkpoint file... +2022-04-12 02:53:10.304773: saving checkpoint... +2022-04-12 02:53:20.763771: done, saving took 10.55 seconds +2022-04-12 02:53:20.812166: done +2022-04-12 02:53:20.896244: saving checkpoint... +2022-04-12 02:53:27.508322: done, saving took 6.66 seconds +2022-04-12 02:53:27.589791: This epoch took 138.011887 s + +2022-04-12 02:53:27.612337: svmem(total=99932160000, available=79815946240, percent=20.1, used=16091635712, free=49559769088, active=40374755328, inactive=8150437888, buffers=33374208, cached=34247380992, shared=2199154688, slab=776667136) +2022-04-12 02:53:27.631174: +epoch: 289 +2022-04-12 02:55:16.064293: train loss : -0.6612 +2022-04-12 02:55:27.026304: validation loss: -0.6560 +2022-04-12 02:55:27.124466: Average global foreground Dice: [0.9025, 0.9619, 0.9805, 0.3848] +2022-04-12 02:55:27.134823: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:55:28.107592: lr: 0.004581 +2022-04-12 02:55:28.109412: saving scheduled checkpoint file... +2022-04-12 02:55:28.205526: saving checkpoint... +2022-04-12 02:55:33.230414: done, saving took 5.12 seconds +2022-04-12 02:55:33.244755: done +2022-04-12 02:55:33.295459: saving checkpoint... +2022-04-12 02:55:39.159646: done, saving took 5.91 seconds +2022-04-12 02:55:39.228338: This epoch took 131.590265 s + +2022-04-12 02:55:39.485924: svmem(total=99932160000, available=79774269440, percent=20.2, used=16141955072, free=49499598848, active=40423354368, inactive=8160403456, buffers=33439744, cached=34257166336, shared=2190761984, slab=777211904) +2022-04-12 02:55:39.521265: +epoch: 290 +2022-04-12 02:57:28.082993: train loss : -0.6635 +2022-04-12 02:57:38.921244: validation loss: -0.6144 +2022-04-12 02:57:38.942528: Average global foreground Dice: [0.8756, 0.9631, 0.9736, 0.3582] +2022-04-12 02:57:38.945876: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:57:40.106000: lr: 0.004561 +2022-04-12 02:57:40.108844: saving scheduled checkpoint file... +2022-04-12 02:57:40.209644: saving checkpoint... +2022-04-12 02:57:50.532243: done, saving took 10.42 seconds +2022-04-12 02:57:50.648402: done +2022-04-12 02:57:50.653090: This epoch took 131.097771 s + +2022-04-12 02:57:50.659425: svmem(total=99932160000, available=79789715456, percent=20.2, used=16126033920, free=49497579520, active=40407867392, inactive=8178106368, buffers=33472512, cached=34275074048, shared=2190958592, slab=776892416) +2022-04-12 02:57:50.694155: +epoch: 291 +2022-04-12 02:59:38.314857: train loss : -0.5779 +2022-04-12 02:59:50.752505: validation loss: -0.4898 +2022-04-12 02:59:50.768651: Average global foreground Dice: [0.8519, 0.9281, 0.9224, 0.3497] +2022-04-12 02:59:50.772427: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 02:59:51.723746: lr: 0.004541 +2022-04-12 02:59:51.773884: saving scheduled checkpoint file... +2022-04-12 02:59:51.866325: saving checkpoint... +2022-04-12 02:59:59.518079: done, saving took 7.72 seconds +2022-04-12 02:59:59.644044: done +2022-04-12 02:59:59.645885: This epoch took 128.924988 s + +2022-04-12 02:59:59.652582: svmem(total=99932160000, available=79571693568, percent=20.4, used=16347316224, free=49302102016, active=40401125376, inactive=8380952576, buffers=33505280, cached=34249236480, shared=2187644928, slab=777752576) +2022-04-12 02:59:59.699039: +epoch: 292 +2022-04-12 03:01:48.007423: train loss : -0.5418 +2022-04-12 03:02:00.335445: validation loss: -0.5630 +2022-04-12 03:02:00.361562: Average global foreground Dice: [0.8461, 0.9502, 0.9685, 0.3538] +2022-04-12 03:02:00.415354: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:02:01.417956: lr: 0.004522 +2022-04-12 03:02:01.479871: saving scheduled checkpoint file... +2022-04-12 03:02:01.680055: saving checkpoint... +2022-04-12 03:02:12.518074: done, saving took 10.97 seconds +2022-04-12 03:02:12.723576: done +2022-04-12 03:02:12.726855: This epoch took 133.001978 s + +2022-04-12 03:02:12.730109: svmem(total=99932160000, available=79829729280, percent=20.1, used=16085979136, free=49551949824, active=40294289408, inactive=8230567936, buffers=33562624, cached=34260668416, shared=2190929920, slab=777752576) +2022-04-12 03:02:12.740526: +epoch: 293 +2022-04-12 03:04:00.444710: train loss : -0.6197 +2022-04-12 03:04:12.437590: validation loss: -0.6172 +2022-04-12 03:04:12.543439: Average global foreground Dice: [0.8726, 0.9562, 0.9803, 0.3683] +2022-04-12 03:04:12.552068: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:04:13.593961: lr: 0.004502 +2022-04-12 03:04:13.597341: saving scheduled checkpoint file... +2022-04-12 03:04:13.758021: saving checkpoint... +2022-04-12 03:04:26.726024: done, saving took 13.07 seconds +2022-04-12 03:04:26.826678: done +2022-04-12 03:04:26.831023: This epoch took 134.024131 s + +2022-04-12 03:04:26.833807: svmem(total=99932160000, available=79706935296, percent=20.2, used=16209158144, free=49411842048, active=40464965632, inactive=8203239424, buffers=33595392, cached=34277564416, shared=2190475264, slab=777510912) +2022-04-12 03:04:26.835900: +epoch: 294 +2022-04-12 03:06:14.770678: train loss : -0.6435 +2022-04-12 03:06:26.960751: validation loss: -0.6440 +2022-04-12 03:06:27.369535: Average global foreground Dice: [0.8818, 0.9619, 0.978, 0.3681] +2022-04-12 03:06:27.374353: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:06:27.872223: lr: 0.004482 +2022-04-12 03:06:27.894960: saving scheduled checkpoint file... +2022-04-12 03:06:27.971676: saving checkpoint... +2022-04-12 03:06:36.367044: done, saving took 8.44 seconds +2022-04-12 03:06:36.522953: done +2022-04-12 03:06:36.527702: This epoch took 129.688525 s + +2022-04-12 03:06:36.570437: svmem(total=99932160000, available=79727329280, percent=20.2, used=16188325888, free=49414725632, active=40447868928, inactive=8218673152, buffers=33648640, cached=34295459840, shared=2190729216, slab=777097216) +2022-04-12 03:06:36.597379: +epoch: 295 +2022-04-12 03:08:24.488881: train loss : -0.5915 +2022-04-12 03:08:35.185744: validation loss: -0.5797 +2022-04-12 03:08:35.233768: Average global foreground Dice: [0.8335, 0.9519, 0.9746, 0.3625] +2022-04-12 03:08:35.314569: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:08:36.342538: lr: 0.004463 +2022-04-12 03:08:36.411072: saving scheduled checkpoint file... +2022-04-12 03:08:36.507654: saving checkpoint... +2022-04-12 03:08:46.273328: done, saving took 9.86 seconds +2022-04-12 03:08:46.393776: done +2022-04-12 03:08:46.459273: This epoch took 129.782428 s + +2022-04-12 03:08:46.469205: svmem(total=99932160000, available=79902621696, percent=20.0, used=16013119488, free=49582149632, active=40291995648, inactive=8206204928, buffers=33685504, cached=34303205376, shared=2190729216, slab=777383936) +2022-04-12 03:08:46.475534: +epoch: 296 +2022-04-12 03:10:35.022977: train loss : -0.5410 +2022-04-12 03:10:46.447490: validation loss: -0.5436 +2022-04-12 03:10:46.459782: Average global foreground Dice: [0.8071, 0.9371, 0.9689, 0.3587] +2022-04-12 03:10:46.514502: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:10:47.519051: lr: 0.004443 +2022-04-12 03:10:47.546006: saving scheduled checkpoint file... +2022-04-12 03:10:47.681799: saving checkpoint... +2022-04-12 03:10:58.860309: done, saving took 11.28 seconds +2022-04-12 03:10:58.969152: done +2022-04-12 03:10:58.972917: This epoch took 132.492465 s + +2022-04-12 03:10:58.975067: svmem(total=99932160000, available=79874564096, percent=20.1, used=16041750528, free=49536569344, active=40325292032, inactive=8222855168, buffers=33726464, cached=34320113664, shared=2190233600, slab=777375744) +2022-04-12 03:10:58.980783: +epoch: 297 +2022-04-12 03:12:46.702764: train loss : -0.6026 +2022-04-12 03:12:59.049366: validation loss: -0.6022 +2022-04-12 03:12:59.095304: Average global foreground Dice: [0.8564, 0.9615, 0.9723, 0.3451] +2022-04-12 03:12:59.101576: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:13:00.141629: lr: 0.004423 +2022-04-12 03:13:00.143989: saving scheduled checkpoint file... +2022-04-12 03:13:00.262650: saving checkpoint... +2022-04-12 03:13:08.500479: done, saving took 8.35 seconds +2022-04-12 03:13:08.683545: done +2022-04-12 03:13:08.685988: This epoch took 129.695521 s + +2022-04-12 03:13:08.688907: svmem(total=99932160000, available=79777173504, percent=20.2, used=16138240000, free=49421737984, active=40404774912, inactive=8241033216, buffers=33759232, cached=34338422784, shared=2191003648, slab=778383360) +2022-04-12 03:13:08.712628: +epoch: 298 +2022-04-12 03:14:56.635030: train loss : -0.6242 +2022-04-12 03:15:09.038908: validation loss: -0.6260 +2022-04-12 03:15:09.067130: Average global foreground Dice: [0.8646, 0.9595, 0.98, 0.3823] +2022-04-12 03:15:09.095076: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:15:10.080392: lr: 0.004404 +2022-04-12 03:15:10.106687: saving scheduled checkpoint file... +2022-04-12 03:15:10.256277: saving checkpoint... +2022-04-12 03:15:18.516206: done, saving took 8.38 seconds +2022-04-12 03:15:18.694545: done +2022-04-12 03:15:18.732802: This epoch took 129.970422 s + +2022-04-12 03:15:18.777381: svmem(total=99932160000, available=79755931648, percent=20.2, used=16159723520, free=49392689152, active=40436260864, inactive=8248328192, buffers=33808384, cached=34345938944, shared=2190839808, slab=779505664) +2022-04-12 03:15:18.781204: +epoch: 299 +2022-04-12 03:17:08.213995: train loss : -0.6539 +2022-04-12 03:17:19.140915: validation loss: -0.6348 +2022-04-12 03:17:19.277441: Average global foreground Dice: [0.8784, 0.9638, 0.979, 0.3883] +2022-04-12 03:17:19.285718: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:17:20.806407: lr: 0.004384 +2022-04-12 03:17:20.807946: saving scheduled checkpoint file... +2022-04-12 03:17:20.899862: saving checkpoint... +2022-04-12 03:17:28.551113: done, saving took 7.74 seconds +2022-04-12 03:17:28.702801: done +2022-04-12 03:17:28.759333: This epoch took 129.964422 s + +2022-04-12 03:17:28.848156: svmem(total=99932160000, available=79620423680, percent=20.3, used=16295424000, free=49239777280, active=40571043840, inactive=8265269248, buffers=33845248, cached=34363113472, shared=2190467072, slab=778768384) +2022-04-12 03:17:28.854272: +epoch: 300 +2022-04-12 03:19:16.541529: train loss : -0.6209 +2022-04-12 03:19:28.529984: validation loss: -0.5893 +2022-04-12 03:19:28.534321: Average global foreground Dice: [0.8483, 0.9572, 0.9709, 0.3755] +2022-04-12 03:19:28.536438: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:19:29.508114: lr: 0.004364 +2022-04-12 03:19:29.893768: saving scheduled checkpoint file... +2022-04-12 03:19:30.680883: saving checkpoint... +2022-04-12 03:19:53.544871: done, saving took 22.94 seconds +2022-04-12 03:19:53.700044: done +2022-04-12 03:19:53.765234: This epoch took 144.896707 s + +2022-04-12 03:19:53.771655: svmem(total=99932160000, available=79794036736, percent=20.2, used=16121602048, free=49394880512, active=40396541952, inactive=8283881472, buffers=33886208, cached=34381791232, shared=2190618624, slab=779370496) +2022-04-12 03:19:53.836426: +epoch: 301 +2022-04-12 03:21:41.763058: train loss : -0.6450 +2022-04-12 03:21:53.703645: validation loss: -0.6339 +2022-04-12 03:21:53.769048: Average global foreground Dice: [0.8685, 0.9604, 0.9824, 0.3829] +2022-04-12 03:21:53.832399: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:21:54.635751: lr: 0.004344 +2022-04-12 03:21:54.710200: saving scheduled checkpoint file... +2022-04-12 03:21:54.873151: saving checkpoint... +2022-04-12 03:22:08.258615: done, saving took 13.47 seconds +2022-04-12 03:22:09.493394: done +2022-04-12 03:22:09.919327: This epoch took 136.032286 s + +2022-04-12 03:22:10.275084: svmem(total=99932160000, available=79697158144, percent=20.2, used=16232337408, free=49280397312, active=40486293504, inactive=8287440896, buffers=33935360, cached=34385489920, shared=2176741376, slab=792612864) +2022-04-12 03:22:10.667025: +epoch: 302 +2022-04-12 03:23:58.120073: train loss : -0.6406 +2022-04-12 03:24:09.471628: validation loss: -0.6219 +2022-04-12 03:24:09.506356: Average global foreground Dice: [0.8815, 0.9662, 0.9828, 0.3743] +2022-04-12 03:24:09.508416: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:24:10.302979: lr: 0.004325 +2022-04-12 03:24:10.308053: saving scheduled checkpoint file... +2022-04-12 03:24:10.386980: saving checkpoint... +2022-04-12 03:24:16.026565: done, saving took 5.70 seconds +2022-04-12 03:24:16.041247: done +2022-04-12 03:24:16.042196: This epoch took 125.132935 s + +2022-04-12 03:24:16.043307: svmem(total=99932160000, available=80291061760, percent=19.7, used=15688531968, free=49866506240, active=39973236736, inactive=8245202944, buffers=33984512, cached=34343137280, shared=2126639104, slab=777723904) +2022-04-12 03:24:16.044204: +epoch: 303 +2022-04-12 03:26:04.641637: train loss : -0.6324 +2022-04-12 03:26:16.567111: validation loss: -0.6306 +2022-04-12 03:26:16.599002: Average global foreground Dice: [0.8867, 0.9613, 0.9789, 0.368] +2022-04-12 03:26:16.605685: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:26:17.497366: lr: 0.004305 +2022-04-12 03:26:17.534623: saving scheduled checkpoint file... +2022-04-12 03:26:17.636506: saving checkpoint... +2022-04-12 03:26:26.166809: done, saving took 8.63 seconds +2022-04-12 03:26:26.474248: done +2022-04-12 03:26:26.545635: This epoch took 130.500580 s + +2022-04-12 03:26:26.634785: svmem(total=99932160000, available=80061579264, percent=19.9, used=15902949376, free=49619353600, active=40185417728, inactive=8277442560, buffers=34033664, cached=34375823360, shared=2141585408, slab=778821632) +2022-04-12 03:26:26.722458: +epoch: 304 +2022-04-12 03:28:15.974668: train loss : -0.6437 +2022-04-12 03:28:27.078179: validation loss: -0.6203 +2022-04-12 03:28:27.111222: Average global foreground Dice: [0.8667, 0.9659, 0.9778, 0.3741] +2022-04-12 03:28:27.120714: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:28:28.247765: lr: 0.004285 +2022-04-12 03:28:28.355241: saving scheduled checkpoint file... +2022-04-12 03:28:28.496302: saving checkpoint... +2022-04-12 03:28:39.183721: done, saving took 10.82 seconds +2022-04-12 03:28:39.248264: done +2022-04-12 03:28:39.251634: This epoch took 132.424423 s + +2022-04-12 03:28:39.255668: svmem(total=99932160000, available=80328355840, percent=19.6, used=15723364352, free=49868787712, active=40004468736, inactive=8207908864, buffers=34066432, cached=34305941504, shared=2054483968, slab=777195520) +2022-04-12 03:28:39.275438: +epoch: 305 +2022-04-12 03:30:26.651116: train loss : -0.6511 +2022-04-12 03:30:37.896128: validation loss: -0.6495 +2022-04-12 03:30:37.924411: Average global foreground Dice: [0.8813, 0.9664, 0.9809, 0.3773] +2022-04-12 03:30:37.930512: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:30:39.048151: lr: 0.004265 +2022-04-12 03:30:39.051317: saving scheduled checkpoint file... +2022-04-12 03:30:39.156022: saving checkpoint... +2022-04-12 03:30:45.608719: done, saving took 6.55 seconds +2022-04-12 03:30:45.676897: done +2022-04-12 03:30:45.737399: This epoch took 126.455066 s + +2022-04-12 03:30:45.741656: svmem(total=99932160000, available=79943008256, percent=20.0, used=15973093376, free=49475735552, active=40254582784, inactive=8350900224, buffers=34107392, cached=34449223680, shared=2189967360, slab=778973184) +2022-04-12 03:30:45.754405: +epoch: 306 +2022-04-12 03:32:33.645913: train loss : -0.6603 +2022-04-12 03:32:44.574694: validation loss: -0.6455 +2022-04-12 03:32:44.580907: Average global foreground Dice: [0.8628, 0.9614, 0.9828, 0.3943] +2022-04-12 03:32:44.582450: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:32:45.519689: lr: 0.004245 +2022-04-12 03:32:45.568016: saving scheduled checkpoint file... +2022-04-12 03:32:45.635340: saving checkpoint... +2022-04-12 03:32:54.600883: done, saving took 9.03 seconds +2022-04-12 03:32:54.725785: done +2022-04-12 03:32:54.743204: This epoch took 128.901654 s + +2022-04-12 03:32:54.811746: svmem(total=99932160000, available=79818211328, percent=20.1, used=16100732928, free=49333383168, active=40382590976, inactive=8365436928, buffers=34140160, cached=34463903744, shared=2187149312, slab=778383360) +2022-04-12 03:32:54.816718: +epoch: 307 +2022-04-12 03:34:44.518113: train loss : -0.6603 +2022-04-12 03:34:56.126949: validation loss: -0.6213 +2022-04-12 03:34:56.181832: Average global foreground Dice: [0.8782, 0.9599, 0.9801, 0.3671] +2022-04-12 03:34:56.195190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:34:57.233734: lr: 0.004226 +2022-04-12 03:34:57.296479: saving scheduled checkpoint file... +2022-04-12 03:34:57.409610: saving checkpoint... +2022-04-12 03:35:08.429204: done, saving took 11.11 seconds +2022-04-12 03:35:08.585670: done +2022-04-12 03:35:08.590556: This epoch took 133.762287 s + +2022-04-12 03:35:08.647629: svmem(total=99932160000, available=79777075200, percent=20.2, used=16138215424, free=49274814464, active=40412536832, inactive=8386592768, buffers=34197504, cached=34484932608, shared=2190905344, slab=777842688) +2022-04-12 03:35:08.719344: +epoch: 308 +2022-04-12 03:36:55.877309: train loss : -0.6720 +2022-04-12 03:37:07.652438: validation loss: -0.6582 +2022-04-12 03:37:08.124953: Average global foreground Dice: [0.888, 0.9658, 0.9806, 0.3597] +2022-04-12 03:37:08.301734: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:37:09.150709: lr: 0.004206 +2022-04-12 03:37:09.160930: saving scheduled checkpoint file... +2022-04-12 03:37:09.313915: saving checkpoint... +2022-04-12 03:37:21.786769: done, saving took 12.56 seconds +2022-04-12 03:37:21.947249: done +2022-04-12 03:37:21.995134: This epoch took 133.266322 s + +2022-04-12 03:37:22.019929: svmem(total=99932160000, available=79820931072, percent=20.1, used=16094285824, free=49301135360, active=40380620800, inactive=8404008960, buffers=34246656, cached=34502492160, shared=2190917632, slab=778121216) +2022-04-12 03:37:22.049535: +epoch: 309 +2022-04-12 03:39:10.754668: train loss : -0.6671 +2022-04-12 03:39:22.035294: validation loss: -0.6733 +2022-04-12 03:39:22.061301: Average global foreground Dice: [0.9067, 0.9661, 0.9823, 0.3835] +2022-04-12 03:39:22.065363: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:39:22.913184: lr: 0.004186 +2022-04-12 03:39:22.916013: saving scheduled checkpoint file... +2022-04-12 03:39:22.994292: saving checkpoint... +2022-04-12 03:39:31.114463: done, saving took 8.20 seconds +2022-04-12 03:39:31.169805: done +2022-04-12 03:39:31.186799: This epoch took 129.099927 s + +2022-04-12 03:39:31.189456: svmem(total=99932160000, available=79796604928, percent=20.1, used=16108331008, free=49269084160, active=40385601536, inactive=8422248448, buffers=34279424, cached=34520465408, shared=2201214976, slab=777965568) +2022-04-12 03:39:31.195084: +epoch: 310 +2022-04-12 03:41:20.211285: train loss : -0.6103 +2022-04-12 03:41:32.903675: validation loss: -0.5931 +2022-04-12 03:41:32.920148: Average global foreground Dice: [0.8694, 0.9545, 0.974, 0.3587] +2022-04-12 03:41:32.938757: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:41:33.802641: lr: 0.004166 +2022-04-12 03:41:33.806426: saving scheduled checkpoint file... +2022-04-12 03:41:33.921896: saving checkpoint... +2022-04-12 03:41:40.353088: done, saving took 6.54 seconds +2022-04-12 03:41:40.468033: done +2022-04-12 03:41:40.471128: This epoch took 129.255396 s + +2022-04-12 03:41:40.474541: svmem(total=99932160000, available=79965126656, percent=20.0, used=15949832192, free=49420230656, active=40231055360, inactive=8429256704, buffers=34336768, cached=34527760384, shared=2191114240, slab=778477568) +2022-04-12 03:41:40.477935: +epoch: 311 +2022-04-12 03:43:29.385419: train loss : -0.6298 +2022-04-12 03:43:41.303121: validation loss: -0.5676 +2022-04-12 03:43:41.334284: Average global foreground Dice: [0.8884, 0.9484, 0.9629, 0.3387] +2022-04-12 03:43:41.336426: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:43:42.442296: lr: 0.004146 +2022-04-12 03:43:42.445477: saving scheduled checkpoint file... +2022-04-12 03:43:42.545574: saving checkpoint... +2022-04-12 03:43:49.339919: done, saving took 6.89 seconds +2022-04-12 03:43:49.522942: done +2022-04-12 03:43:49.547235: This epoch took 129.067391 s + +2022-04-12 03:43:49.591454: svmem(total=99932160000, available=79827349504, percent=20.1, used=16087887872, free=49264996352, active=40369872896, inactive=8446746624, buffers=34373632, cached=34544902144, shared=2190770176, slab=778506240) +2022-04-12 03:43:49.644601: +epoch: 312 +2022-04-12 03:45:37.954108: train loss : -0.6300 +2022-04-12 03:45:49.746444: validation loss: -0.6539 +2022-04-12 03:45:49.758471: Average global foreground Dice: [0.861, 0.9639, 0.9799, 0.4017] +2022-04-12 03:45:49.821294: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:45:50.860853: lr: 0.004127 +2022-04-12 03:45:50.867626: saving scheduled checkpoint file... +2022-04-12 03:45:51.039162: saving checkpoint... +2022-04-12 03:46:00.898980: done, saving took 9.95 seconds +2022-04-12 03:46:00.933480: done +2022-04-12 03:46:00.948188: This epoch took 131.281357 s + +2022-04-12 03:46:00.956716: svmem(total=99932160000, available=79824855040, percent=20.1, used=16090517504, free=49254572032, active=40370360320, inactive=8453824512, buffers=34414592, cached=34552655872, shared=2190540800, slab=778948608) +2022-04-12 03:46:00.996040: +epoch: 313 +2022-04-12 03:47:49.666197: train loss : -0.6437 +2022-04-12 03:48:02.097235: validation loss: -0.6493 +2022-04-12 03:48:02.141803: Average global foreground Dice: [0.9004, 0.9624, 0.9779, 0.3543] +2022-04-12 03:48:02.145692: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:48:03.051913: lr: 0.004107 +2022-04-12 03:48:03.068369: saving scheduled checkpoint file... +2022-04-12 03:48:03.224708: saving checkpoint... +2022-04-12 03:48:09.804672: done, saving took 6.66 seconds +2022-04-12 03:48:09.836576: done +2022-04-12 03:48:09.858865: This epoch took 128.854384 s + +2022-04-12 03:48:09.870088: svmem(total=99932160000, available=79843856384, percent=20.1, used=16071372800, free=49256222720, active=40344346624, inactive=8471343104, buffers=34455552, cached=34570108928, shared=2190729216, slab=780087296) +2022-04-12 03:48:09.886738: +epoch: 314 +2022-04-12 03:49:58.501705: train loss : -0.6308 +2022-04-12 03:50:09.403374: validation loss: -0.6305 +2022-04-12 03:50:09.442376: Average global foreground Dice: [0.8781, 0.9615, 0.9795, 0.3489] +2022-04-12 03:50:09.448578: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:50:10.418241: lr: 0.004087 +2022-04-12 03:50:10.425153: saving scheduled checkpoint file... +2022-04-12 03:50:10.562743: saving checkpoint... +2022-04-12 03:50:21.267974: done, saving took 10.80 seconds +2022-04-12 03:50:21.612156: done +2022-04-12 03:50:21.694946: This epoch took 131.795050 s + +2022-04-12 03:50:21.703076: svmem(total=99932160000, available=79692472320, percent=20.3, used=16222371840, free=49087205376, active=40502546432, inactive=8488972288, buffers=34504704, cached=34588078080, shared=2191028224, slab=779776000) +2022-04-12 03:50:21.732532: +epoch: 315 +2022-04-12 03:52:09.892301: train loss : -0.6639 +2022-04-12 03:52:21.484702: validation loss: -0.6406 +2022-04-12 03:52:21.528610: Average global foreground Dice: [0.879, 0.9656, 0.984, 0.3863] +2022-04-12 03:52:21.590621: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:52:22.606076: lr: 0.004067 +2022-04-12 03:52:22.655101: saving scheduled checkpoint file... +2022-04-12 03:52:22.805254: saving checkpoint... +2022-04-12 03:52:33.681275: done, saving took 10.99 seconds +2022-04-12 03:52:33.827128: done +2022-04-12 03:52:33.856809: This epoch took 132.066657 s + +2022-04-12 03:52:33.888438: svmem(total=99932160000, available=79873335296, percent=20.1, used=16041897984, free=49260363776, active=40322592768, inactive=8496451584, buffers=34537472, cached=34595360768, shared=2190696448, slab=778895360) +2022-04-12 03:52:33.908992: +epoch: 316 +2022-04-12 03:54:21.007928: train loss : -0.6496 +2022-04-12 03:54:32.478049: validation loss: -0.6292 +2022-04-12 03:54:32.493030: Average global foreground Dice: [0.8875, 0.9633, 0.9724, 0.3518] +2022-04-12 03:54:32.560742: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:54:33.538332: lr: 0.004047 +2022-04-12 03:54:33.598257: saving scheduled checkpoint file... +2022-04-12 03:54:33.786196: saving checkpoint... +2022-04-12 03:54:46.067704: done, saving took 12.40 seconds +2022-04-12 03:54:46.200677: done +2022-04-12 03:54:46.243022: This epoch took 132.298891 s + +2022-04-12 03:54:46.269520: svmem(total=99932160000, available=79882813440, percent=20.1, used=16032043008, free=49252265984, active=40312942592, inactive=8513843200, buffers=34570240, cached=34613280768, shared=2190807040, slab=779624448) +2022-04-12 03:54:46.300774: +epoch: 317 +2022-04-12 03:56:34.729240: train loss : -0.6261 +2022-04-12 03:56:46.691569: validation loss: -0.6174 +2022-04-12 03:56:46.753148: Average global foreground Dice: [0.8746, 0.9637, 0.9742, 0.374] +2022-04-12 03:56:46.787271: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:56:47.575205: lr: 0.004027 +2022-04-12 03:56:47.659637: saving scheduled checkpoint file... +2022-04-12 03:56:47.787332: saving checkpoint... +2022-04-12 03:56:56.529002: done, saving took 8.84 seconds +2022-04-12 03:56:56.694998: done +2022-04-12 03:56:56.752196: This epoch took 130.387179 s + +2022-04-12 03:56:56.781694: svmem(total=99932160000, available=80065462272, percent=19.9, used=15849717760, free=49417572352, active=40131084288, inactive=8531128320, buffers=34619392, cached=34630250496, shared=2190585856, slab=779567104) +2022-04-12 03:56:56.826325: +epoch: 318 +2022-04-12 03:58:44.807530: train loss : -0.6546 +2022-04-12 03:58:56.389319: validation loss: -0.6316 +2022-04-12 03:58:56.426096: Average global foreground Dice: [0.8786, 0.9617, 0.9786, 0.3749] +2022-04-12 03:58:56.431259: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 03:58:57.353835: lr: 0.004007 +2022-04-12 03:58:57.375856: saving scheduled checkpoint file... +2022-04-12 03:58:57.494487: saving checkpoint... +2022-04-12 03:59:03.881432: done, saving took 6.46 seconds +2022-04-12 03:59:03.970249: done +2022-04-12 03:59:03.986608: This epoch took 127.151978 s + +2022-04-12 03:59:03.989758: svmem(total=99932160000, available=79989407744, percent=20.0, used=15925579776, free=49324077056, active=40211353600, inactive=8548515840, buffers=34652160, cached=34647851008, shared=2190729216, slab=778723328) +2022-04-12 03:59:03.996136: +epoch: 319 +2022-04-12 04:00:52.053214: train loss : -0.6730 +2022-04-12 04:01:04.699427: validation loss: -0.6576 +2022-04-12 04:01:04.732315: Average global foreground Dice: [0.8848, 0.9648, 0.9812, 0.3746] +2022-04-12 04:01:04.737551: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:01:05.340239: lr: 0.003987 +2022-04-12 04:01:05.343193: saving scheduled checkpoint file... +2022-04-12 04:01:05.432493: saving checkpoint... +2022-04-12 04:01:13.532728: done, saving took 8.16 seconds +2022-04-12 04:01:13.698635: done +2022-04-12 04:01:13.724355: This epoch took 129.702265 s + +2022-04-12 04:01:13.788021: svmem(total=99932160000, available=79788482560, percent=20.2, used=16126599168, free=49115340800, active=40409976832, inactive=8556302336, buffers=34709504, cached=34655510528, shared=2190729216, slab=778555392) +2022-04-12 04:01:13.796490: +epoch: 320 +2022-04-12 04:03:02.651933: train loss : -0.6643 +2022-04-12 04:03:13.266718: validation loss: -0.6373 +2022-04-12 04:03:13.293411: Average global foreground Dice: [0.8851, 0.9618, 0.9791, 0.3602] +2022-04-12 04:03:13.294665: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:03:14.237764: lr: 0.003967 +2022-04-12 04:03:14.332134: saving scheduled checkpoint file... +2022-04-12 04:03:14.525361: saving checkpoint... +2022-04-12 04:03:25.836402: done, saving took 11.41 seconds +2022-04-12 04:03:25.864158: done +2022-04-12 04:03:25.885528: This epoch took 132.014561 s + +2022-04-12 04:03:25.925345: svmem(total=99932160000, available=79554846720, percent=20.4, used=16360624128, free=48864239616, active=40645087232, inactive=8573411328, buffers=34742272, cached=34672553984, shared=2190364672, slab=780398592) +2022-04-12 04:03:25.946399: +epoch: 321 +2022-04-12 04:05:13.765898: train loss : -0.6361 +2022-04-12 04:05:24.895035: validation loss: -0.6423 +2022-04-12 04:05:24.966153: Average global foreground Dice: [0.8768, 0.9643, 0.9819, 0.3655] +2022-04-12 04:05:25.048818: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:05:26.115950: lr: 0.003947 +2022-04-12 04:05:26.303568: saving scheduled checkpoint file... +2022-04-12 04:05:26.407752: saving checkpoint... +2022-04-12 04:05:35.949982: done, saving took 9.64 seconds +2022-04-12 04:05:36.035450: done +2022-04-12 04:05:36.038294: This epoch took 130.075511 s + +2022-04-12 04:05:36.045613: svmem(total=99932160000, available=79784468480, percent=20.2, used=16130453504, free=49101160448, active=40340275200, inactive=8633675776, buffers=34783232, cached=34665762816, shared=2190729216, slab=780316672) +2022-04-12 04:05:36.070973: +epoch: 322 +2022-04-12 04:07:23.549989: train loss : -0.6754 +2022-04-12 04:07:34.246951: validation loss: -0.6483 +2022-04-12 04:07:34.253212: Average global foreground Dice: [0.902, 0.959, 0.9799, 0.3799] +2022-04-12 04:07:34.259359: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:07:35.200356: lr: 0.003927 +2022-04-12 04:07:35.203171: saving scheduled checkpoint file... +2022-04-12 04:07:35.305831: saving checkpoint... +2022-04-12 04:07:47.213228: done, saving took 12.01 seconds +2022-04-12 04:07:47.374178: done +2022-04-12 04:07:47.415543: This epoch took 131.324137 s + +2022-04-12 04:07:47.469842: svmem(total=99932160000, available=79649202176, percent=20.3, used=16265363456, free=48958050304, active=40516644864, inactive=8605298688, buffers=34836480, cached=34673909760, shared=2191175680, slab=780574720) +2022-04-12 04:07:47.475745: +epoch: 323 +2022-04-12 04:09:36.117339: train loss : -0.6402 +2022-04-12 04:09:48.549037: validation loss: -0.6399 +2022-04-12 04:09:48.620426: Average global foreground Dice: [0.8779, 0.9631, 0.9765, 0.3719] +2022-04-12 04:09:48.679927: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:09:49.722758: lr: 0.003907 +2022-04-12 04:09:49.777937: saving scheduled checkpoint file... +2022-04-12 04:09:49.928162: saving checkpoint... +2022-04-12 04:10:03.324935: done, saving took 13.48 seconds +2022-04-12 04:10:03.490071: done +2022-04-12 04:10:03.510043: This epoch took 136.016178 s + +2022-04-12 04:10:03.547635: svmem(total=99932160000, available=79693283328, percent=20.3, used=16236056576, free=48984653824, active=40491610112, inactive=8599949312, buffers=34877440, cached=34676572160, shared=2176348160, slab=780455936) +2022-04-12 04:10:03.625131: +epoch: 324 +2022-04-12 04:11:50.981736: train loss : -0.6631 +2022-04-12 04:12:02.850040: validation loss: -0.6482 +2022-04-12 04:12:03.271993: Average global foreground Dice: [0.8828, 0.9644, 0.9812, 0.3529] +2022-04-12 04:12:03.968199: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:12:06.480954: lr: 0.003887 +2022-04-12 04:12:07.351857: saving scheduled checkpoint file... +2022-04-12 04:12:08.159136: saving checkpoint... +2022-04-12 04:12:20.811834: done, saving took 12.70 seconds +2022-04-12 04:12:20.901185: done +2022-04-12 04:12:20.906948: This epoch took 137.207886 s + +2022-04-12 04:12:20.913133: svmem(total=99932160000, available=79752867840, percent=20.2, used=16166895616, free=49025798144, active=40442195968, inactive=8607608832, buffers=34926592, cached=34704539648, shared=2185773056, slab=781647872) +2022-04-12 04:12:20.974075: +epoch: 325 +2022-04-12 04:14:09.787079: train loss : -0.6625 +2022-04-12 04:14:21.634230: validation loss: -0.6571 +2022-04-12 04:14:21.649351: Average global foreground Dice: [0.8956, 0.9677, 0.9813, 0.3919] +2022-04-12 04:14:21.686064: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:14:22.647672: lr: 0.003867 +2022-04-12 04:14:22.695082: saving scheduled checkpoint file... +2022-04-12 04:14:22.807058: saving checkpoint... +2022-04-12 04:14:30.308977: done, saving took 7.59 seconds +2022-04-12 04:14:30.567064: done +2022-04-12 04:14:30.592242: This epoch took 129.539494 s + +2022-04-12 04:14:30.658158: svmem(total=99932160000, available=79773192192, percent=20.2, used=16141778944, free=49028640768, active=40421777408, inactive=8627171328, buffers=34959360, cached=34726780928, shared=2190618624, slab=780980224) +2022-04-12 04:14:30.661446: +epoch: 326 +2022-04-12 04:16:18.928485: train loss : -0.6718 +2022-04-12 04:16:30.288275: validation loss: -0.6797 +2022-04-12 04:16:30.330433: Average global foreground Dice: [0.916, 0.9678, 0.9826, 0.375] +2022-04-12 04:16:30.333354: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:16:31.288856: lr: 0.003847 +2022-04-12 04:16:31.301495: saving scheduled checkpoint file... +2022-04-12 04:16:31.461975: saving checkpoint... +2022-04-12 04:16:42.973395: done, saving took 11.61 seconds +2022-04-12 04:16:42.998778: done +2022-04-12 04:16:43.009525: This epoch took 132.290094 s + +2022-04-12 04:16:43.016438: svmem(total=99932160000, available=79780020224, percent=20.2, used=16134881280, free=49027715072, active=40415600640, inactive=8634925056, buffers=35008512, cached=34734555136, shared=2190741504, slab=780673024) +2022-04-12 04:16:43.021212: +epoch: 327 +2022-04-12 04:18:30.034085: train loss : -0.6758 +2022-04-12 04:18:41.169837: validation loss: -0.6789 +2022-04-12 04:18:41.291875: Average global foreground Dice: [0.9137, 0.9675, 0.9845, 0.4004] +2022-04-12 04:18:41.375126: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:18:42.190515: lr: 0.003827 +2022-04-12 04:18:42.296456: saving scheduled checkpoint file... +2022-04-12 04:18:42.503633: saving checkpoint... +2022-04-12 04:18:48.000295: done, saving took 5.62 seconds +2022-04-12 04:18:48.019288: done +2022-04-12 04:18:48.102115: saving checkpoint... +2022-04-12 04:18:56.984014: done, saving took 8.96 seconds +2022-04-12 04:18:57.134147: This epoch took 134.084458 s + +2022-04-12 04:18:57.141693: svmem(total=99932160000, available=79741280256, percent=20.2, used=16164077568, free=48908005376, active=40444542976, inactive=8725254144, buffers=35045376, cached=34825031680, shared=2200145920, slab=782258176) +2022-04-12 04:18:57.225597: +epoch: 328 +2022-04-12 04:20:44.480990: train loss : -0.6772 +2022-04-12 04:20:55.949463: validation loss: -0.6752 +2022-04-12 04:20:56.034841: Average global foreground Dice: [0.8783, 0.9629, 0.9812, 0.4247] +2022-04-12 04:20:56.101772: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:20:57.112183: lr: 0.003807 +2022-04-12 04:20:57.151675: saving scheduled checkpoint file... +2022-04-12 04:20:57.387292: saving checkpoint... +2022-04-12 04:21:13.932594: done, saving took 16.67 seconds +2022-04-12 04:21:14.176828: done +2022-04-12 04:21:14.337665: saving checkpoint... +2022-04-12 04:21:31.812105: done, saving took 17.58 seconds +2022-04-12 04:21:31.907852: This epoch took 154.626230 s + +2022-04-12 04:21:31.911649: svmem(total=99932160000, available=79768461312, percent=20.2, used=16146198528, free=48916508672, active=40425410560, inactive=8734294016, buffers=35102720, cached=34834350080, shared=2190729216, slab=781959168) +2022-04-12 04:21:31.916259: +epoch: 329 +2022-04-12 04:23:20.151759: train loss : -0.6866 +2022-04-12 04:23:32.478093: validation loss: -0.6667 +2022-04-12 04:23:32.554352: Average global foreground Dice: [0.8988, 0.9713, 0.9823, 0.3948] +2022-04-12 04:23:32.576458: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:23:33.471457: lr: 0.003787 +2022-04-12 04:23:33.517695: saving scheduled checkpoint file... +2022-04-12 04:23:33.604033: saving checkpoint... +2022-04-12 04:23:44.065208: done, saving took 10.54 seconds +2022-04-12 04:23:44.206594: done +2022-04-12 04:23:44.262575: saving checkpoint... +2022-04-12 04:23:53.989320: done, saving took 9.77 seconds +2022-04-12 04:23:54.035049: This epoch took 142.108392 s + +2022-04-12 04:23:54.045969: svmem(total=99932160000, available=79915110400, percent=20.0, used=16010944512, free=49044930560, active=40281776128, inactive=8741117952, buffers=35151872, cached=34841133056, shared=2179330048, slab=781447168) +2022-04-12 04:23:54.106812: +epoch: 330 +2022-04-12 04:25:42.852246: train loss : -0.6668 +2022-04-12 04:25:54.726671: validation loss: -0.6502 +2022-04-12 04:25:54.887624: Average global foreground Dice: [0.8695, 0.9635, 0.9785, 0.3966] +2022-04-12 04:25:54.969243: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:25:55.964145: lr: 0.003767 +2022-04-12 04:25:56.004968: saving scheduled checkpoint file... +2022-04-12 04:25:56.193243: saving checkpoint... +2022-04-12 04:26:09.843987: done, saving took 13.73 seconds +2022-04-12 04:26:10.273935: done +2022-04-12 04:26:10.340741: This epoch took 136.226750 s + +2022-04-12 04:26:10.439617: svmem(total=99932160000, available=79633723392, percent=20.3, used=16281018368, free=48755482624, active=40554676224, inactive=8760512512, buffers=35201024, cached=34860457984, shared=2190729216, slab=781733888) +2022-04-12 04:26:10.536141: +epoch: 331 +2022-04-12 04:27:57.743596: train loss : -0.6542 +2022-04-12 04:28:09.288321: validation loss: -0.6445 +2022-04-12 04:28:09.452335: Average global foreground Dice: [0.8738, 0.9654, 0.9827, 0.373] +2022-04-12 04:28:09.530249: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:28:10.574701: lr: 0.003747 +2022-04-12 04:28:10.662446: saving scheduled checkpoint file... +2022-04-12 04:28:10.837675: saving checkpoint... +2022-04-12 04:28:24.148367: done, saving took 13.40 seconds +2022-04-12 04:28:24.341419: done +2022-04-12 04:28:24.350369: This epoch took 133.736741 s + +2022-04-12 04:28:24.399605: svmem(total=99932160000, available=79888306176, percent=20.1, used=16016289792, free=48992489472, active=40295227392, inactive=8788258816, buffers=35250176, cached=34888130560, shared=2200915968, slab=781389824) +2022-04-12 04:28:24.436460: +epoch: 332 +2022-04-12 04:30:12.169338: train loss : -0.6752 +2022-04-12 04:30:23.121863: validation loss: -0.6855 +2022-04-12 04:30:23.287049: Average global foreground Dice: [0.9062, 0.9709, 0.9814, 0.4014] +2022-04-12 04:30:23.354597: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:30:24.376983: lr: 0.003727 +2022-04-12 04:30:24.407747: saving scheduled checkpoint file... +2022-04-12 04:30:24.559393: saving checkpoint... +2022-04-12 04:30:34.715274: done, saving took 10.26 seconds +2022-04-12 04:30:34.845554: done +2022-04-12 04:30:34.960736: saving checkpoint... +2022-04-12 04:30:41.840426: done, saving took 6.93 seconds +2022-04-12 04:30:41.851260: This epoch took 137.346126 s + +2022-04-12 04:30:41.853091: svmem(total=99932160000, available=79760109568, percent=20.2, used=16154566656, free=48848384000, active=40055660544, inactive=9179291648, buffers=35581952, cached=34893627392, shared=2190729216, slab=780869632) +2022-04-12 04:30:41.853882: +epoch: 333 +2022-04-12 04:32:30.051856: train loss : -0.6652 +2022-04-12 04:32:41.590402: validation loss: -0.6499 +2022-04-12 04:32:41.623325: Average global foreground Dice: [0.899, 0.9668, 0.9821, 0.3802] +2022-04-12 04:32:41.672036: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:32:42.553348: lr: 0.003707 +2022-04-12 04:32:42.599885: saving scheduled checkpoint file... +2022-04-12 04:32:42.703723: saving checkpoint... +2022-04-12 04:32:50.136184: done, saving took 7.53 seconds +2022-04-12 04:32:50.276623: done +2022-04-12 04:32:50.361231: saving checkpoint... +2022-04-12 04:32:57.977453: done, saving took 7.70 seconds +2022-04-12 04:32:58.184896: This epoch took 136.329880 s + +2022-04-12 04:32:58.238903: svmem(total=99932160000, available=79906185216, percent=20.0, used=16008699904, free=48986771456, active=40188456960, inactive=8901730304, buffers=35614720, cached=34901073920, shared=2190581760, slab=781459456) +2022-04-12 04:32:58.255788: +epoch: 334 +2022-04-12 04:34:45.254016: train loss : -0.6748 +2022-04-12 04:34:56.660501: validation loss: -0.6574 +2022-04-12 04:34:56.766222: Average global foreground Dice: [0.9096, 0.9643, 0.9793, 0.3825] +2022-04-12 04:34:56.768683: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:34:57.885912: lr: 0.003687 +2022-04-12 04:34:57.889319: saving scheduled checkpoint file... +2022-04-12 04:34:58.010456: saving checkpoint... +2022-04-12 04:35:05.485511: done, saving took 7.59 seconds +2022-04-12 04:35:05.573880: done +2022-04-12 04:35:05.707094: saving checkpoint... +2022-04-12 04:35:11.459540: done, saving took 5.85 seconds +2022-04-12 04:35:11.532028: This epoch took 133.201391 s + +2022-04-12 04:35:11.549037: svmem(total=99932160000, available=79708659712, percent=20.2, used=16195731456, free=48771792896, active=40471629824, inactive=8830701568, buffers=35672064, cached=34928963584, shared=2201059328, slab=782000128) +2022-04-12 04:35:11.560313: +epoch: 335 +2022-04-12 04:36:59.392965: train loss : -0.6798 +2022-04-12 04:37:11.022851: validation loss: -0.6588 +2022-04-12 04:37:11.038462: Average global foreground Dice: [0.8935, 0.9697, 0.982, 0.362] +2022-04-12 04:37:11.052720: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:37:12.164124: lr: 0.003667 +2022-04-12 04:37:12.169861: saving scheduled checkpoint file... +2022-04-12 04:37:12.320426: saving checkpoint... +2022-04-12 04:37:20.028795: done, saving took 7.81 seconds +2022-04-12 04:37:20.179818: done +2022-04-12 04:37:20.189739: This epoch took 128.606929 s + +2022-04-12 04:37:20.252805: svmem(total=99932160000, available=79848878080, percent=20.1, used=16055246848, free=48894550016, active=40330465280, inactive=8848162816, buffers=35741696, cached=34946621440, shared=2201227264, slab=782802944) +2022-04-12 04:37:20.258713: +epoch: 336 +2022-04-12 04:39:09.465961: train loss : -0.6190 +2022-04-12 04:39:20.574212: validation loss: -0.5995 +2022-04-12 04:39:20.618943: Average global foreground Dice: [0.8571, 0.9615, 0.9657, 0.3668] +2022-04-12 04:39:20.665449: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:39:21.913819: lr: 0.003647 +2022-04-12 04:39:21.920124: saving scheduled checkpoint file... +2022-04-12 04:39:22.093367: saving checkpoint... +2022-04-12 04:39:32.763420: done, saving took 10.75 seconds +2022-04-12 04:39:32.830345: done +2022-04-12 04:39:32.906997: This epoch took 132.553337 s + +2022-04-12 04:39:32.911472: svmem(total=99932160000, available=79823425536, percent=20.1, used=16105000960, free=48851533824, active=40384720896, inactive=8839344128, buffers=35774464, cached=34939850752, shared=2176851968, slab=782729216) +2022-04-12 04:39:32.933850: +epoch: 337 +2022-04-12 04:41:19.895369: train loss : -0.6134 +2022-04-12 04:41:31.537158: validation loss: -0.6254 +2022-04-12 04:41:31.687169: Average global foreground Dice: [0.8762, 0.9622, 0.9768, 0.381] +2022-04-12 04:41:31.703674: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:41:32.743173: lr: 0.003627 +2022-04-12 04:41:32.779599: saving scheduled checkpoint file... +2022-04-12 04:41:32.944529: saving checkpoint... +2022-04-12 04:41:44.198884: done, saving took 11.34 seconds +2022-04-12 04:41:44.365392: done +2022-04-12 04:41:44.422351: This epoch took 131.440522 s + +2022-04-12 04:41:44.428978: svmem(total=99932160000, available=79910957056, percent=20.0, used=16006856704, free=48931340288, active=40290172928, inactive=8857071616, buffers=35815424, cached=34958147584, shared=2187448320, slab=781418496) +2022-04-12 04:41:44.497405: +epoch: 338 +2022-04-12 04:43:33.293803: train loss : -0.6303 +2022-04-12 04:43:44.277524: validation loss: -0.6519 +2022-04-12 04:43:44.308659: Average global foreground Dice: [0.8855, 0.961, 0.9789, 0.3796] +2022-04-12 04:43:44.370908: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:43:45.353025: lr: 0.003606 +2022-04-12 04:43:45.364727: saving scheduled checkpoint file... +2022-04-12 04:43:45.501269: saving checkpoint... +2022-04-12 04:43:57.160920: done, saving took 11.72 seconds +2022-04-12 04:43:57.381424: done +2022-04-12 04:43:57.392711: This epoch took 132.892937 s + +2022-04-12 04:43:57.453434: svmem(total=99932160000, available=79700324352, percent=20.2, used=16224534528, free=48702189568, active=40511717376, inactive=8868888576, buffers=35848192, cached=34969587712, shared=2180321280, slab=779182080) +2022-04-12 04:43:57.482472: +epoch: 339 +2022-04-12 04:45:46.827556: train loss : -0.6446 +2022-04-12 04:45:57.636061: validation loss: -0.6365 +2022-04-12 04:45:57.663129: Average global foreground Dice: [0.8813, 0.9606, 0.9778, 0.3903] +2022-04-12 04:45:57.667584: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:45:58.552085: lr: 0.003586 +2022-04-12 04:45:58.558512: saving scheduled checkpoint file... +2022-04-12 04:45:58.652538: saving checkpoint... +2022-04-12 04:46:06.283488: done, saving took 7.72 seconds +2022-04-12 04:46:06.301589: done +2022-04-12 04:46:06.302906: This epoch took 128.767354 s + +2022-04-12 04:46:06.304216: svmem(total=99932160000, available=79716593664, percent=20.2, used=16195174400, free=48704540672, active=40426631168, inactive=8937582592, buffers=35917824, cached=34996527104, shared=2193395712, slab=781574144) +2022-04-12 04:46:06.305210: +epoch: 340 +2022-04-12 04:47:53.898024: train loss : -0.6490 +2022-04-12 04:48:06.586267: validation loss: -0.6366 +2022-04-12 04:48:06.625265: Average global foreground Dice: [0.9013, 0.9675, 0.9768, 0.3668] +2022-04-12 04:48:06.690635: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:48:07.707642: lr: 0.003566 +2022-04-12 04:48:07.806252: saving scheduled checkpoint file... +2022-04-12 04:48:07.996140: saving checkpoint... +2022-04-12 04:48:18.704930: done, saving took 10.83 seconds +2022-04-12 04:48:18.801219: done +2022-04-12 04:48:18.824682: This epoch took 132.518717 s + +2022-04-12 04:48:18.878963: svmem(total=99932160000, available=79731085312, percent=20.2, used=16183619584, free=48711258112, active=40457801728, inactive=8906383360, buffers=35966976, cached=35001315328, shared=2190540800, slab=781643776) +2022-04-12 04:48:18.925722: +epoch: 341 +2022-04-12 04:50:07.355829: train loss : -0.6733 +2022-04-12 04:50:18.435042: validation loss: -0.6583 +2022-04-12 04:50:18.503628: Average global foreground Dice: [0.9141, 0.9676, 0.9818, 0.3614] +2022-04-12 04:50:18.597627: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:50:19.636938: lr: 0.003546 +2022-04-12 04:50:19.823421: saving scheduled checkpoint file... +2022-04-12 04:50:20.048451: saving checkpoint... +2022-04-12 04:50:32.578841: done, saving took 12.63 seconds +2022-04-12 04:50:32.590047: done +2022-04-12 04:50:32.592011: This epoch took 133.621429 s + +2022-04-12 04:50:32.593612: svmem(total=99932160000, available=79741759488, percent=20.2, used=16172769280, free=48704450560, active=40445915136, inactive=8922177536, buffers=36007936, cached=35018932224, shared=2190540800, slab=781582336) +2022-04-12 04:50:32.595164: +epoch: 342 +2022-04-12 04:52:21.045602: train loss : -0.6653 +2022-04-12 04:52:32.712310: validation loss: -0.6533 +2022-04-12 04:52:32.787606: Average global foreground Dice: [0.8983, 0.9651, 0.9757, 0.3847] +2022-04-12 04:52:32.848455: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:52:34.208538: lr: 0.003526 +2022-04-12 04:52:34.638692: saving scheduled checkpoint file... +2022-04-12 04:52:35.172430: saving checkpoint... +2022-04-12 04:52:51.054547: done, saving took 15.95 seconds +2022-04-12 04:52:51.199308: done +2022-04-12 04:52:51.282156: This epoch took 138.681190 s + +2022-04-12 04:52:51.342343: svmem(total=99932160000, available=79841263616, percent=20.1, used=16073170944, free=48786558976, active=40349310976, inactive=8939708416, buffers=36040704, cached=35036389376, shared=2190729216, slab=782233600) +2022-04-12 04:52:51.416462: +epoch: 343 +2022-04-12 04:54:39.437516: train loss : -0.6622 +2022-04-12 04:54:51.023513: validation loss: -0.6264 +2022-04-12 04:54:51.076502: Average global foreground Dice: [0.8854, 0.9626, 0.9645, 0.3541] +2022-04-12 04:54:51.140965: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:54:52.107740: lr: 0.003505 +2022-04-12 04:54:52.193092: saving scheduled checkpoint file... +2022-04-12 04:54:52.315343: saving checkpoint... +2022-04-12 04:55:11.746248: done, saving took 19.49 seconds +2022-04-12 04:55:13.844107: done +2022-04-12 04:55:14.310145: This epoch took 142.863349 s + +2022-04-12 04:55:15.441951: svmem(total=99932160000, available=79694962688, percent=20.3, used=16233377792, free=48621600768, active=40484671488, inactive=8944267264, buffers=36098048, cached=35041083392, shared=2176741376, slab=795127808) +2022-04-12 04:55:15.840970: +epoch: 344 +2022-04-12 04:57:03.319534: train loss : -0.6684 +2022-04-12 04:57:15.602710: validation loss: -0.6433 +2022-04-12 04:57:15.736006: Average global foreground Dice: [0.8871, 0.9651, 0.9823, 0.3983] +2022-04-12 04:57:15.742943: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:57:16.682079: lr: 0.003485 +2022-04-12 04:57:16.739969: saving scheduled checkpoint file... +2022-04-12 04:57:16.911491: saving checkpoint... +2022-04-12 04:57:26.724421: done, saving took 9.89 seconds +2022-04-12 04:57:27.758565: done +2022-04-12 04:57:28.242708: This epoch took 132.177757 s + +2022-04-12 04:57:28.723917: svmem(total=99932160000, available=79837052928, percent=20.1, used=16091205632, free=48755916800, active=40357875712, inactive=8951955456, buffers=36139008, cached=35048898560, shared=2176741376, slab=794464256) +2022-04-12 04:57:29.184683: +epoch: 345 +2022-04-12 04:59:19.218665: train loss : -0.6418 +2022-04-12 04:59:30.181717: validation loss: -0.6305 +2022-04-12 04:59:30.224280: Average global foreground Dice: [0.8697, 0.9636, 0.9762, 0.36] +2022-04-12 04:59:30.273190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 04:59:31.184501: lr: 0.003465 +2022-04-12 04:59:31.264127: saving scheduled checkpoint file... +2022-04-12 04:59:31.373091: saving checkpoint... +2022-04-12 04:59:43.303526: done, saving took 12.04 seconds +2022-04-12 04:59:44.321106: done +2022-04-12 04:59:44.744156: This epoch took 135.075381 s + +2022-04-12 04:59:44.998280: svmem(total=99932160000, available=80027947008, percent=19.9, used=15900356608, free=48929275904, active=40170082304, inactive=8962805760, buffers=36171776, cached=35066355712, shared=2176741376, slab=794873856) +2022-04-12 04:59:45.415763: +epoch: 346 +2022-04-12 05:01:33.748551: train loss : -0.6560 +2022-04-12 05:01:45.485479: validation loss: -0.6570 +2022-04-12 05:01:45.495137: Average global foreground Dice: [0.9067, 0.9655, 0.9814, 0.3802] +2022-04-12 05:01:45.502906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:01:46.561334: lr: 0.003445 +2022-04-12 05:01:46.566962: saving scheduled checkpoint file... +2022-04-12 05:01:46.757202: saving checkpoint... +2022-04-12 05:01:55.422052: done, saving took 8.80 seconds +2022-04-12 05:01:56.158069: done +2022-04-12 05:01:56.388195: This epoch took 130.288327 s + +2022-04-12 05:01:56.658827: svmem(total=99932160000, available=79907627008, percent=20.0, used=16020623360, free=48791584768, active=40293203968, inactive=8980135936, buffers=36212736, cached=35083739136, shared=2176741376, slab=793796608) +2022-04-12 05:01:57.082536: +epoch: 347 +2022-04-12 05:03:45.462533: train loss : -0.6652 +2022-04-12 05:03:56.911568: validation loss: -0.6495 +2022-04-12 05:03:56.968479: Average global foreground Dice: [0.8849, 0.9658, 0.9785, 0.3828] +2022-04-12 05:03:57.030703: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:03:58.091131: lr: 0.003424 +2022-04-12 05:03:58.096990: saving scheduled checkpoint file... +2022-04-12 05:03:58.189083: saving checkpoint... +2022-04-12 05:04:08.846278: done, saving took 10.74 seconds +2022-04-12 05:04:08.980741: done +2022-04-12 05:04:08.985339: This epoch took 131.567575 s + +2022-04-12 05:04:08.996019: svmem(total=99932160000, available=79961305088, percent=20.0, used=15966867456, free=48994291712, active=40244621312, inactive=8831053824, buffers=36245504, cached=34934755328, shared=2176741376, slab=787288064) +2022-04-12 05:04:09.051567: +epoch: 348 +2022-04-12 05:05:56.865473: train loss : -0.6915 +2022-04-12 05:06:08.471727: validation loss: -0.6729 +2022-04-12 05:06:08.552258: Average global foreground Dice: [0.8843, 0.9694, 0.9813, 0.3953] +2022-04-12 05:06:08.646405: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:06:09.642531: lr: 0.003404 +2022-04-12 05:06:09.648680: saving scheduled checkpoint file... +2022-04-12 05:06:09.765076: saving checkpoint... +2022-04-12 05:06:20.789622: done, saving took 11.11 seconds +2022-04-12 05:06:20.922424: done +2022-04-12 05:06:20.937701: This epoch took 131.808720 s + +2022-04-12 05:06:20.988879: svmem(total=99932160000, available=79993368576, percent=20.0, used=15959302144, free=48850837504, active=40245239808, inactive=8981983232, buffers=36306944, cached=35085713408, shared=2152267776, slab=779935744) +2022-04-12 05:06:21.048967: +epoch: 349 +2022-04-12 05:08:08.054031: train loss : -0.6936 +2022-04-12 05:08:21.159625: validation loss: -0.6770 +2022-04-12 05:08:21.273512: Average global foreground Dice: [0.8982, 0.9684, 0.9798, 0.3954] +2022-04-12 05:08:21.334345: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:08:22.391127: lr: 0.003384 +2022-04-12 05:08:22.440271: saving scheduled checkpoint file... +2022-04-12 05:08:22.540970: saving checkpoint... +2022-04-12 05:08:30.723124: done, saving took 8.27 seconds +2022-04-12 05:08:30.743661: done +2022-04-12 05:08:30.745432: This epoch took 129.684953 s + +2022-04-12 05:08:30.747627: svmem(total=99932160000, available=79865954304, percent=20.1, used=16048300032, free=48705986560, active=40332636160, inactive=9037656064, buffers=36352000, cached=35141521408, shared=2190618624, slab=780800000) +2022-04-12 05:08:30.749384: +epoch: 350 +2022-04-12 05:10:19.047902: train loss : -0.6903 +2022-04-12 05:10:31.552259: validation loss: -0.6708 +2022-04-12 05:10:31.634830: Average global foreground Dice: [0.9162, 0.9681, 0.9822, 0.3926] +2022-04-12 05:10:31.639561: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:10:32.657784: lr: 0.003364 +2022-04-12 05:10:32.692921: saving scheduled checkpoint file... +2022-04-12 05:10:32.817019: saving checkpoint... +2022-04-12 05:10:41.336637: done, saving took 8.57 seconds +2022-04-12 05:10:41.451886: done +2022-04-12 05:10:41.492036: This epoch took 130.741633 s + +2022-04-12 05:10:41.508364: svmem(total=99932160000, available=79801057280, percent=20.1, used=16112992256, free=48656187392, active=40303079424, inactive=9113133056, buffers=36392960, cached=35126587392, shared=2190729216, slab=783060992) +2022-04-12 05:10:41.546674: +epoch: 351 +2022-04-12 05:12:29.415937: train loss : -0.6952 +2022-04-12 05:12:40.676468: validation loss: -0.6536 +2022-04-12 05:12:40.764279: Average global foreground Dice: [0.8965, 0.9623, 0.9778, 0.3775] +2022-04-12 05:12:40.819903: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:12:41.806890: lr: 0.003343 +2022-04-12 05:12:41.857055: saving scheduled checkpoint file... +2022-04-12 05:12:41.964509: saving checkpoint... +2022-04-12 05:12:57.836459: done, saving took 15.98 seconds +2022-04-12 05:12:57.921036: done +2022-04-12 05:12:57.936635: This epoch took 136.373876 s + +2022-04-12 05:12:58.046168: svmem(total=99932160000, available=79748935680, percent=20.2, used=16165023744, free=48596299776, active=40425254912, inactive=9057361920, buffers=36425728, cached=35134410752, shared=2190729216, slab=781914112) +2022-04-12 05:12:58.049105: +epoch: 352 +2022-04-12 05:14:45.430094: train loss : -0.6955 +2022-04-12 05:14:57.822576: validation loss: -0.6497 +2022-04-12 05:14:57.898700: Average global foreground Dice: [0.8976, 0.9657, 0.9792, 0.3825] +2022-04-12 05:14:57.955648: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:14:58.857401: lr: 0.003323 +2022-04-12 05:14:58.927156: saving scheduled checkpoint file... +2022-04-12 05:14:59.063260: saving checkpoint... +2022-04-12 05:15:11.796853: done, saving took 12.86 seconds +2022-04-12 05:15:12.249817: done +2022-04-12 05:15:12.302260: This epoch took 134.217519 s + +2022-04-12 05:15:12.339172: svmem(total=99932160000, available=79758983168, percent=20.2, used=16156037120, free=48588767232, active=40420614144, inactive=9062633472, buffers=36483072, cached=35150872576, shared=2189815808, slab=781651968) +2022-04-12 05:15:12.375349: +epoch: 353 +2022-04-12 05:17:01.049687: train loss : -0.6910 +2022-04-12 05:17:12.447896: validation loss: -0.6645 +2022-04-12 05:17:12.533562: Average global foreground Dice: [0.9139, 0.9701, 0.9796, 0.3942] +2022-04-12 05:17:12.625148: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:17:13.766278: lr: 0.003303 +2022-04-12 05:17:13.768186: saving scheduled checkpoint file... +2022-04-12 05:17:13.883804: saving checkpoint... +2022-04-12 05:17:25.807297: done, saving took 12.04 seconds +2022-04-12 05:17:25.946406: done +2022-04-12 05:17:26.044583: saving checkpoint... +2022-04-12 05:17:34.319244: done, saving took 8.32 seconds +2022-04-12 05:17:34.356333: This epoch took 141.940809 s + +2022-04-12 05:17:34.360050: svmem(total=99932160000, available=79840940032, percent=20.1, used=16066363392, free=48616239104, active=40346382336, inactive=9116172288, buffers=36524032, cached=35213033472, shared=2197409792, slab=782299136) +2022-04-12 05:17:34.362388: +epoch: 354 +2022-04-12 05:19:22.557800: train loss : -0.6918 +2022-04-12 05:19:35.097436: validation loss: -0.6656 +2022-04-12 05:19:35.134071: Average global foreground Dice: [0.898, 0.9695, 0.9818, 0.3843] +2022-04-12 05:19:35.144195: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:19:35.750636: lr: 0.003282 +2022-04-12 05:19:35.772893: saving scheduled checkpoint file... +2022-04-12 05:19:35.911005: saving checkpoint... +2022-04-12 05:19:44.564682: done, saving took 8.75 seconds +2022-04-12 05:19:44.706482: done +2022-04-12 05:19:44.759016: saving checkpoint... +2022-04-12 05:19:51.251233: done, saving took 6.54 seconds +2022-04-12 05:19:51.280367: This epoch took 136.916689 s + +2022-04-12 05:19:51.304102: svmem(total=99932160000, available=80265736192, percent=19.7, used=15798161408, free=49023393792, active=40072171520, inactive=8976674816, buffers=36564992, cached=35074039808, shared=2040778752, slab=781979648) +2022-04-12 05:19:51.308801: +epoch: 355 +2022-04-12 05:21:38.985426: train loss : -0.6966 +2022-04-12 05:21:51.567219: validation loss: -0.6989 +2022-04-12 05:21:51.597917: Average global foreground Dice: [0.9186, 0.9696, 0.9829, 0.4101] +2022-04-12 05:21:51.669077: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:21:52.535642: lr: 0.003262 +2022-04-12 05:21:52.541554: saving scheduled checkpoint file... +2022-04-12 05:21:52.607501: saving checkpoint... +2022-04-12 05:21:59.428291: done, saving took 6.88 seconds +2022-04-12 05:21:59.612468: done +2022-04-12 05:21:59.663979: saving checkpoint... +2022-04-12 05:22:12.348457: done, saving took 12.73 seconds +2022-04-12 05:22:12.542260: This epoch took 141.230901 s + +2022-04-12 05:22:12.548028: svmem(total=99932160000, available=79944859648, percent=20.0, used=15982911488, free=48693620736, active=40261947392, inactive=9118732288, buffers=36630528, cached=35218997248, shared=2176741376, slab=782286848) +2022-04-12 05:22:12.613465: +epoch: 356 +2022-04-12 05:24:01.324477: train loss : -0.6761 +2022-04-12 05:24:13.966870: validation loss: -0.6865 +2022-04-12 05:24:14.000767: Average global foreground Dice: [0.9079, 0.97, 0.9823, 0.3918] +2022-04-12 05:24:14.052641: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:24:14.713700: lr: 0.003241 +2022-04-12 05:24:14.772637: saving scheduled checkpoint file... +2022-04-12 05:24:14.916565: saving checkpoint... +2022-04-12 05:24:24.436011: done, saving took 9.61 seconds +2022-04-12 05:24:24.627820: done +2022-04-12 05:24:24.679855: saving checkpoint... +2022-04-12 05:24:45.305808: done, saving took 20.67 seconds +2022-04-12 05:24:45.350616: This epoch took 152.735072 s + +2022-04-12 05:24:45.353610: svmem(total=99932160000, available=79749644288, percent=20.2, used=16178290688, free=48479830016, active=40445640704, inactive=9137319936, buffers=36663296, cached=35237376000, shared=2176741376, slab=791527424) +2022-04-12 05:24:45.356345: +epoch: 357 +2022-04-12 05:26:33.225728: train loss : -0.6711 +2022-04-12 05:26:45.260428: validation loss: -0.6234 +2022-04-12 05:26:45.344620: Average global foreground Dice: [0.9005, 0.9622, 0.9752, 0.3817] +2022-04-12 05:26:45.411135: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:26:46.420630: lr: 0.003221 +2022-04-12 05:26:46.487203: saving scheduled checkpoint file... +2022-04-12 05:26:46.627184: saving checkpoint... +2022-04-12 05:27:06.750755: done, saving took 20.23 seconds +2022-04-12 05:27:08.686729: done +2022-04-12 05:27:09.515620: This epoch took 144.155812 s + +2022-04-12 05:27:10.261240: svmem(total=99932160000, available=79959601152, percent=20.0, used=15968276480, free=48671358976, active=40223866880, inactive=9155723264, buffers=36708352, cached=35255816192, shared=2176741376, slab=795025408) +2022-04-12 05:27:11.039500: +epoch: 358 +2022-04-12 05:29:00.096667: train loss : -0.6654 +2022-04-12 05:29:12.028850: validation loss: -0.6593 +2022-04-12 05:29:12.106844: Average global foreground Dice: [0.895, 0.9677, 0.9754, 0.3828] +2022-04-12 05:29:12.174592: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:29:13.365733: lr: 0.003201 +2022-04-12 05:29:13.938355: saving scheduled checkpoint file... +2022-04-12 05:29:14.078589: saving checkpoint... +2022-04-12 05:29:25.833346: done, saving took 11.86 seconds +2022-04-12 05:29:26.666735: done +2022-04-12 05:29:27.038152: This epoch took 135.126283 s + +2022-04-12 05:29:27.475729: svmem(total=99932160000, available=79931400192, percent=20.0, used=15996481536, free=48625664000, active=40264003584, inactive=9173131264, buffers=36757504, cached=35273256960, shared=2176741376, slab=794927104) +2022-04-12 05:29:27.880271: +epoch: 359 +2022-04-12 05:31:16.711227: train loss : -0.6768 +2022-04-12 05:31:29.251367: validation loss: -0.6802 +2022-04-12 05:31:29.288044: Average global foreground Dice: [0.9017, 0.9696, 0.9814, 0.3883] +2022-04-12 05:31:29.293946: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:31:30.550110: lr: 0.00318 +2022-04-12 05:31:30.604838: saving scheduled checkpoint file... +2022-04-12 05:31:30.790025: saving checkpoint... +2022-04-12 05:31:38.716249: done, saving took 8.02 seconds +2022-04-12 05:31:38.769320: done +2022-04-12 05:31:38.847685: This epoch took 130.606343 s + +2022-04-12 05:31:38.853014: svmem(total=99932160000, available=79826489344, percent=20.1, used=16101240832, free=48778018816, active=40381722624, inactive=8915255296, buffers=36798464, cached=35016101888, shared=2176741376, slab=786358272) +2022-04-12 05:31:38.869680: +epoch: 360 +2022-04-12 05:33:27.420205: train loss : -0.6467 +2022-04-12 05:33:38.575317: validation loss: -0.6394 +2022-04-12 05:33:38.607224: Average global foreground Dice: [0.9078, 0.961, 0.9786, 0.3286] +2022-04-12 05:33:38.613157: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:33:39.803052: lr: 0.00316 +2022-04-12 05:33:39.827262: saving scheduled checkpoint file... +2022-04-12 05:33:39.950105: saving checkpoint... +2022-04-12 05:33:56.688226: done, saving took 16.83 seconds +2022-04-12 05:33:56.806197: done +2022-04-12 05:33:56.836068: This epoch took 137.959236 s + +2022-04-12 05:33:56.891840: svmem(total=99932160000, available=79641759744, percent=20.3, used=16286416896, free=48710238208, active=40568918016, inactive=8798257152, buffers=36831232, cached=34898673664, shared=2176741376, slab=782561280) +2022-04-12 05:33:56.914161: +epoch: 361 +2022-04-12 05:35:45.362022: train loss : -0.6639 +2022-04-12 05:35:56.618313: validation loss: -0.6439 +2022-04-12 05:35:56.674869: Average global foreground Dice: [0.8934, 0.965, 0.9788, 0.3804] +2022-04-12 05:35:56.749273: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:35:57.651461: lr: 0.003139 +2022-04-12 05:35:57.700921: saving scheduled checkpoint file... +2022-04-12 05:35:57.798837: saving checkpoint... +2022-04-12 05:36:04.713985: done, saving took 7.01 seconds +2022-04-12 05:36:04.729024: done +2022-04-12 05:36:04.730423: This epoch took 127.713954 s + +2022-04-12 05:36:04.731956: svmem(total=99932160000, available=80093999104, percent=19.9, used=15893409792, free=48758964224, active=40073854976, inactive=9245253632, buffers=36880384, cached=35242905600, shared=2117120000, slab=782479360) +2022-04-12 05:36:04.732965: +epoch: 362 +2022-04-12 05:37:52.806906: train loss : -0.6879 +2022-04-12 05:38:04.470442: validation loss: -0.6599 +2022-04-12 05:38:04.515652: Average global foreground Dice: [0.9048, 0.968, 0.9797, 0.3736] +2022-04-12 05:38:04.517114: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:38:05.521005: lr: 0.003119 +2022-04-12 05:38:05.535656: saving scheduled checkpoint file... +2022-04-12 05:38:05.622755: saving checkpoint... +2022-04-12 05:38:18.131210: done, saving took 12.56 seconds +2022-04-12 05:38:18.181720: done +2022-04-12 05:38:18.192675: This epoch took 133.458193 s + +2022-04-12 05:38:18.198961: svmem(total=99932160000, available=79937003520, percent=20.0, used=16053264384, free=48581353472, active=40312844288, inactive=9178906624, buffers=36941824, cached=35260600320, shared=2114105344, slab=783224832) +2022-04-12 05:38:18.206731: +epoch: 363 +2022-04-12 05:40:05.159046: train loss : -0.6827 +2022-04-12 05:40:16.703951: validation loss: -0.6404 +2022-04-12 05:40:16.708361: Average global foreground Dice: [0.9031, 0.9649, 0.9781, 0.3546] +2022-04-12 05:40:16.713745: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:40:17.597949: lr: 0.003098 +2022-04-12 05:40:17.602511: saving scheduled checkpoint file... +2022-04-12 05:40:17.710477: saving checkpoint... +2022-04-12 05:40:24.171366: done, saving took 6.57 seconds +2022-04-12 05:40:24.307376: done +2022-04-12 05:40:24.313035: This epoch took 126.092261 s + +2022-04-12 05:40:24.317283: svmem(total=99932160000, available=79744667648, percent=20.2, used=16169041920, free=48382177280, active=40449609728, inactive=9243455488, buffers=36982784, cached=35343958016, shared=2190729216, slab=781983744) +2022-04-12 05:40:24.327765: +epoch: 364 +2022-04-12 05:42:11.573644: train loss : -0.6976 +2022-04-12 05:42:23.738544: validation loss: -0.6803 +2022-04-12 05:42:23.776694: Average global foreground Dice: [0.9054, 0.9671, 0.9825, 0.4089] +2022-04-12 05:42:23.782629: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:42:24.645791: lr: 0.003078 +2022-04-12 05:42:24.654867: saving scheduled checkpoint file... +2022-04-12 05:42:24.756585: saving checkpoint... +2022-04-12 05:42:35.866932: done, saving took 11.21 seconds +2022-04-12 05:42:35.929197: done +2022-04-12 05:42:35.932902: This epoch took 131.597974 s + +2022-04-12 05:42:35.945032: svmem(total=99932160000, available=79891415040, percent=20.1, used=16022245376, free=48511307776, active=40303456256, inactive=9260728320, buffers=37015552, cached=35361591296, shared=2190729216, slab=781799424) +2022-04-12 05:42:35.950121: +epoch: 365 +2022-04-12 05:44:24.370641: train loss : -0.6897 +2022-04-12 05:44:36.386690: validation loss: -0.6653 +2022-04-12 05:44:36.465929: Average global foreground Dice: [0.9141, 0.9689, 0.9831, 0.386] +2022-04-12 05:44:36.578196: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:44:37.659995: lr: 0.003057 +2022-04-12 05:44:37.834770: saving scheduled checkpoint file... +2022-04-12 05:44:38.040196: saving checkpoint... +2022-04-12 05:44:44.140548: done, saving took 6.19 seconds +2022-04-12 05:44:44.312868: done +2022-04-12 05:44:44.316056: This epoch took 128.361293 s + +2022-04-12 05:44:44.318738: svmem(total=99932160000, available=79781765120, percent=20.2, used=16131780608, free=48384294912, active=40419233792, inactive=9277988864, buffers=37048320, cached=35379036160, shared=2190729216, slab=781123584) +2022-04-12 05:44:44.322467: +epoch: 366 +2022-04-12 05:46:31.717268: train loss : -0.7065 +2022-04-12 05:46:42.566413: validation loss: -0.6550 +2022-04-12 05:46:42.600531: Average global foreground Dice: [0.8962, 0.9678, 0.9768, 0.3539] +2022-04-12 05:46:42.627105: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:46:43.634085: lr: 0.003037 +2022-04-12 05:46:43.644568: saving scheduled checkpoint file... +2022-04-12 05:46:43.738344: saving checkpoint... +2022-04-12 05:46:53.087758: done, saving took 9.44 seconds +2022-04-12 05:46:53.157352: done +2022-04-12 05:46:53.162075: This epoch took 128.796176 s + +2022-04-12 05:46:53.172252: svmem(total=99932160000, available=79760572416, percent=20.2, used=16152870912, free=48355291136, active=40435912704, inactive=9283117056, buffers=37113856, cached=35386884096, shared=2190807040, slab=782692352) +2022-04-12 05:46:53.179266: +epoch: 367 +2022-04-12 05:48:41.229986: train loss : -0.7011 +2022-04-12 05:48:51.808574: validation loss: -0.6739 +2022-04-12 05:48:51.900903: Average global foreground Dice: [0.9182, 0.9722, 0.9786, 0.3733] +2022-04-12 05:48:51.919602: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:48:52.794143: lr: 0.003016 +2022-04-12 05:48:52.853396: saving scheduled checkpoint file... +2022-04-12 05:48:53.009886: saving checkpoint... +2022-04-12 05:49:00.172472: done, saving took 7.26 seconds +2022-04-12 05:49:00.243273: done +2022-04-12 05:49:00.288412: This epoch took 127.050961 s + +2022-04-12 05:49:00.322013: svmem(total=99932160000, available=79829196800, percent=20.1, used=16084361216, free=48406282240, active=40370307072, inactive=9300557824, buffers=37146624, cached=35404369920, shared=2190729216, slab=782991360) +2022-04-12 05:49:00.335688: +epoch: 368 +2022-04-12 05:50:49.844365: train loss : -0.6867 +2022-04-12 05:51:01.016385: validation loss: -0.6921 +2022-04-12 05:51:01.141302: Average global foreground Dice: [0.9114, 0.9687, 0.9814, 0.3975] +2022-04-12 05:51:01.154126: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:51:02.215888: lr: 0.002996 +2022-04-12 05:51:02.297147: saving scheduled checkpoint file... +2022-04-12 05:51:02.452204: saving checkpoint... +2022-04-12 05:51:13.476869: done, saving took 11.12 seconds +2022-04-12 05:51:13.601636: done +2022-04-12 05:51:13.608575: This epoch took 133.226602 s + +2022-04-12 05:51:13.621064: svmem(total=99932160000, available=80004706304, percent=19.9, used=15908626432, free=48562774016, active=40186855424, inactive=9318068224, buffers=37187584, cached=35423571968, shared=2190917632, slab=784011264) +2022-04-12 05:51:13.656182: +epoch: 369 +2022-04-12 05:53:01.395638: train loss : -0.6919 +2022-04-12 05:53:14.175152: validation loss: -0.6865 +2022-04-12 05:53:14.316968: Average global foreground Dice: [0.9185, 0.9735, 0.9829, 0.3924] +2022-04-12 05:53:14.413047: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:53:15.089344: lr: 0.002975 +2022-04-12 05:53:15.159821: saving scheduled checkpoint file... +2022-04-12 05:53:15.267243: saving checkpoint... +2022-04-12 05:53:25.038636: done, saving took 9.87 seconds +2022-04-12 05:53:25.152138: done +2022-04-12 05:53:25.253762: saving checkpoint... +2022-04-12 05:53:35.807902: done, saving took 10.64 seconds +2022-04-12 05:53:35.892056: This epoch took 142.215956 s + +2022-04-12 05:53:35.925188: svmem(total=99932160000, available=79632752640, percent=20.3, used=16280653824, free=48173895680, active=40563073024, inactive=9334734848, buffers=37236736, cached=35440373760, shared=2190696448, slab=784629760) +2022-04-12 05:53:35.931938: +epoch: 370 +2022-04-12 05:55:22.357492: train loss : -0.6950 +2022-04-12 05:55:34.342924: validation loss: -0.6748 +2022-04-12 05:55:34.424958: Average global foreground Dice: [0.9127, 0.9697, 0.981, 0.3789] +2022-04-12 05:55:34.483118: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:55:35.287596: lr: 0.002954 +2022-04-12 05:55:35.339252: saving scheduled checkpoint file... +2022-04-12 05:55:35.481661: saving checkpoint... +2022-04-12 05:55:47.136330: done, saving took 11.78 seconds +2022-04-12 05:55:47.397799: done +2022-04-12 05:55:47.565090: saving checkpoint... +2022-04-12 05:55:57.280561: done, saving took 9.81 seconds +2022-04-12 05:55:57.359457: This epoch took 141.421167 s + +2022-04-12 05:55:57.379811: svmem(total=99932160000, available=79823536128, percent=20.1, used=16090157056, free=48346320896, active=40374759424, inactive=9353113600, buffers=37285888, cached=35458396160, shared=2190729216, slab=783409152) +2022-04-12 05:55:57.392861: +epoch: 371 +2022-04-12 05:57:44.616812: train loss : -0.6958 +2022-04-12 05:57:57.258909: validation loss: -0.6828 +2022-04-12 05:57:57.359160: Average global foreground Dice: [0.9164, 0.9737, 0.9847, 0.3937] +2022-04-12 05:57:57.376884: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 05:57:58.307286: lr: 0.002934 +2022-04-12 05:57:58.385885: saving scheduled checkpoint file... +2022-04-12 05:57:58.553514: saving checkpoint... +2022-04-12 05:58:06.643771: done, saving took 8.18 seconds +2022-04-12 05:58:06.682042: done +2022-04-12 05:58:06.737909: saving checkpoint... +2022-04-12 05:58:18.934096: done, saving took 12.24 seconds +2022-04-12 05:58:19.048644: This epoch took 141.619321 s + +2022-04-12 05:58:19.059388: svmem(total=99932160000, available=79791226880, percent=20.2, used=16121503744, free=48296509440, active=40403189760, inactive=9370984448, buffers=37335040, cached=35476811776, shared=2191224832, slab=783605760) +2022-04-12 05:58:19.108938: +epoch: 372 +2022-04-12 06:00:07.560382: train loss : -0.6987 +2022-04-12 06:00:19.193052: validation loss: -0.6833 +2022-04-12 06:00:19.285245: Average global foreground Dice: [0.922, 0.97, 0.9805, 0.3938] +2022-04-12 06:00:19.356932: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:00:20.321307: lr: 0.002913 +2022-04-12 06:00:20.451095: saving scheduled checkpoint file... +2022-04-12 06:00:20.608028: saving checkpoint... +2022-04-12 06:00:34.936457: done, saving took 14.42 seconds +2022-04-12 06:00:35.083210: done +2022-04-12 06:00:35.176572: saving checkpoint... +2022-04-12 06:00:46.698292: done, saving took 11.57 seconds +2022-04-12 06:00:46.815253: This epoch took 147.642699 s + +2022-04-12 06:00:46.839148: svmem(total=99932160000, available=79807479808, percent=20.1, used=16109613056, free=48295116800, active=40390721536, inactive=9384144896, buffers=37376000, cached=35490054144, shared=2186928128, slab=783695872) +2022-04-12 06:00:46.876537: +epoch: 373 +2022-04-12 06:02:36.146111: train loss : -0.6972 +2022-04-12 06:02:47.133819: validation loss: -0.6569 +2022-04-12 06:02:47.281084: Average global foreground Dice: [0.9142, 0.9707, 0.9809, 0.3776] +2022-04-12 06:02:47.374392: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:02:48.631463: lr: 0.002892 +2022-04-12 06:02:48.786760: saving scheduled checkpoint file... +2022-04-12 06:02:48.936994: saving checkpoint... +2022-04-12 06:03:04.076481: done, saving took 15.21 seconds +2022-04-12 06:03:04.277136: done +2022-04-12 06:03:04.454246: saving checkpoint... +2022-04-12 06:03:13.872392: done, saving took 9.47 seconds +2022-04-12 06:03:13.908695: This epoch took 147.015439 s + +2022-04-12 06:03:13.935936: svmem(total=99932160000, available=79922229248, percent=20.0, used=16004947968, free=48391348224, active=40284536832, inactive=9392545792, buffers=37425152, cached=35498438656, shared=2176819200, slab=783876096) +2022-04-12 06:03:13.957347: +epoch: 374 +2022-04-12 06:05:01.270517: train loss : -0.6593 +2022-04-12 06:05:12.569116: validation loss: -0.6588 +2022-04-12 06:05:12.882501: Average global foreground Dice: [0.8876, 0.9656, 0.9804, 0.3912] +2022-04-12 06:05:12.886073: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:05:14.018341: lr: 0.002872 +2022-04-12 06:05:14.294636: saving scheduled checkpoint file... +2022-04-12 06:05:14.433694: saving checkpoint... +2022-04-12 06:05:27.073846: done, saving took 12.71 seconds +2022-04-12 06:05:27.350581: done +2022-04-12 06:05:27.409857: This epoch took 133.444908 s + +2022-04-12 06:05:27.514639: svmem(total=99932160000, available=79912058880, percent=20.0, used=16009105408, free=48373301248, active=40292704256, inactive=9406377984, buffers=37466112, cached=35512287232, shared=2182877184, slab=784179200) +2022-04-12 06:05:27.570379: +epoch: 375 +2022-04-12 06:07:16.359062: train loss : -0.6897 +2022-04-12 06:07:28.117161: validation loss: -0.6752 +2022-04-12 06:07:28.229297: Average global foreground Dice: [0.8972, 0.9714, 0.9829, 0.4079] +2022-04-12 06:07:28.304081: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:07:29.295353: lr: 0.002851 +2022-04-12 06:07:29.386087: saving scheduled checkpoint file... +2022-04-12 06:07:29.510644: saving checkpoint... +2022-04-12 06:07:46.199268: done, saving took 16.79 seconds +2022-04-12 06:07:46.495086: done +2022-04-12 06:07:46.663180: saving checkpoint... +2022-04-12 06:07:58.928813: done, saving took 12.35 seconds +2022-04-12 06:07:58.976289: This epoch took 151.340825 s + +2022-04-12 06:07:59.002860: svmem(total=99932160000, available=80150425600, percent=19.8, used=15825866752, free=48593256448, active=39960023040, inactive=9523937280, buffers=37519360, cached=35475517440, shared=2127724544, slab=783732736) +2022-04-12 06:07:59.006549: +epoch: 376 +2022-04-12 06:09:47.811434: train loss : -0.6802 +2022-04-12 06:09:59.595636: validation loss: -0.6834 +2022-04-12 06:09:59.693948: Average global foreground Dice: [0.9065, 0.969, 0.9844, 0.3962] +2022-04-12 06:09:59.762743: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:10:00.702186: lr: 0.00283 +2022-04-12 06:10:00.753138: saving scheduled checkpoint file... +2022-04-12 06:10:00.918012: saving checkpoint... +2022-04-12 06:10:14.243259: done, saving took 13.44 seconds +2022-04-12 06:10:14.419076: done +2022-04-12 06:10:14.546900: saving checkpoint... +2022-04-12 06:10:27.651149: done, saving took 13.20 seconds +2022-04-12 06:10:29.441329: This epoch took 150.422182 s + +2022-04-12 06:10:29.936177: svmem(total=99932160000, available=79959764992, percent=20.0, used=15967416320, free=48384143360, active=40155230208, inactive=9519153152, buffers=37576704, cached=35543023616, shared=2176741376, slab=797949952) +2022-04-12 06:10:30.531748: +epoch: 377 +2022-04-12 06:12:20.180569: train loss : -0.6937 +2022-04-12 06:12:32.039798: validation loss: -0.6379 +2022-04-12 06:12:32.145239: Average global foreground Dice: [0.9042, 0.9701, 0.9769, 0.3859] +2022-04-12 06:12:32.246798: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:12:33.291958: lr: 0.00281 +2022-04-12 06:12:33.380269: saving scheduled checkpoint file... +2022-04-12 06:12:33.556200: saving checkpoint... +2022-04-12 06:12:46.891694: done, saving took 13.42 seconds +2022-04-12 06:12:46.986406: done +2022-04-12 06:12:46.991846: This epoch took 135.891601 s + +2022-04-12 06:12:47.008388: svmem(total=99932160000, available=79791607808, percent=20.2, used=16121606144, free=48197300224, active=40365473792, inactive=9511833600, buffers=37609472, cached=35575644160, shared=2190729216, slab=784773120) +2022-04-12 06:12:47.037768: +epoch: 378 +2022-04-12 06:14:34.691205: train loss : -0.7099 +2022-04-12 06:14:47.203210: validation loss: -0.6657 +2022-04-12 06:14:47.340130: Average global foreground Dice: [0.893, 0.9717, 0.9815, 0.3808] +2022-04-12 06:14:47.370378: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:14:48.487162: lr: 0.002789 +2022-04-12 06:14:48.580456: saving scheduled checkpoint file... +2022-04-12 06:14:48.746927: saving checkpoint... +2022-04-12 06:15:01.061887: done, saving took 12.40 seconds +2022-04-12 06:15:01.213535: done +2022-04-12 06:15:01.273629: This epoch took 134.203128 s + +2022-04-12 06:15:01.304485: svmem(total=99932160000, available=79813373952, percent=20.1, used=16089513984, free=48211431424, active=40350265344, inactive=9512439808, buffers=37642240, cached=35593572352, shared=2201026560, slab=785756160) +2022-04-12 06:15:01.349021: +epoch: 379 +2022-04-12 06:16:50.091578: train loss : -0.6884 +2022-04-12 06:17:01.730477: validation loss: -0.6858 +2022-04-12 06:17:01.871768: Average global foreground Dice: [0.9194, 0.9713, 0.9847, 0.3988] +2022-04-12 06:17:01.954476: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:17:03.022988: lr: 0.002768 +2022-04-12 06:17:03.069997: saving scheduled checkpoint file... +2022-04-12 06:17:03.168739: saving checkpoint... +2022-04-12 06:17:15.067626: done, saving took 11.99 seconds +2022-04-12 06:17:15.267703: done +2022-04-12 06:17:15.345074: saving checkpoint... +2022-04-12 06:17:26.952280: done, saving took 11.66 seconds +2022-04-12 06:17:27.045428: This epoch took 145.640056 s + +2022-04-12 06:17:27.056117: svmem(total=99932160000, available=79729131520, percent=20.2, used=16183689216, free=48108638208, active=40452763648, inactive=9507438592, buffers=37703680, cached=35602128896, shared=2190917632, slab=784175104) +2022-04-12 06:17:27.078434: +epoch: 380 +2022-04-12 06:19:14.910826: train loss : -0.6950 +2022-04-12 06:19:25.766577: validation loss: -0.6765 +2022-04-12 06:19:25.895540: Average global foreground Dice: [0.9118, 0.9718, 0.9814, 0.362] +2022-04-12 06:19:25.942940: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:19:26.952224: lr: 0.002747 +2022-04-12 06:19:27.068933: saving scheduled checkpoint file... +2022-04-12 06:19:27.197586: saving checkpoint... +2022-04-12 06:19:39.573649: done, saving took 12.43 seconds +2022-04-12 06:19:39.691316: done +2022-04-12 06:19:39.744186: This epoch took 132.615818 s + +2022-04-12 06:19:39.812826: svmem(total=99932160000, available=79634108416, percent=20.3, used=16268533760, free=47996112896, active=40545943552, inactive=9535090688, buffers=37744640, cached=35629768704, shared=2201182208, slab=783876096) +2022-04-12 06:19:39.814585: +epoch: 381 +2022-04-12 06:21:26.724566: train loss : -0.6999 +2022-04-12 06:21:38.169069: validation loss: -0.6707 +2022-04-12 06:21:38.226525: Average global foreground Dice: [0.919, 0.972, 0.982, 0.3737] +2022-04-12 06:21:38.265036: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:21:39.311042: lr: 0.002727 +2022-04-12 06:21:39.432012: saving scheduled checkpoint file... +2022-04-12 06:21:39.540861: saving checkpoint... +2022-04-12 06:21:56.051121: done, saving took 16.62 seconds +2022-04-12 06:21:56.191710: done +2022-04-12 06:21:56.258820: This epoch took 136.328327 s + +2022-04-12 06:21:56.389233: svmem(total=99932160000, available=79652753408, percent=20.3, used=16260112384, free=48004128768, active=40472494080, inactive=9599356928, buffers=37793792, cached=35630125056, shared=2190917632, slab=784748544) +2022-04-12 06:21:56.535246: +epoch: 382 +2022-04-12 06:23:43.926252: train loss : -0.7050 +2022-04-12 06:23:55.701763: validation loss: -0.6766 +2022-04-12 06:23:55.789557: Average global foreground Dice: [0.9025, 0.9694, 0.983, 0.3984] +2022-04-12 06:23:55.832397: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:23:56.758854: lr: 0.002706 +2022-04-12 06:23:56.768798: saving scheduled checkpoint file... +2022-04-12 06:23:56.959665: saving checkpoint... +2022-04-12 06:24:11.020875: done, saving took 14.17 seconds +2022-04-12 06:24:11.310428: done +2022-04-12 06:24:11.376479: This epoch took 134.729392 s + +2022-04-12 06:24:11.381267: svmem(total=99932160000, available=79919095808, percent=20.0, used=15993921536, free=48260452352, active=40228405248, inactive=9578999808, buffers=37826560, cached=35639959552, shared=2190807040, slab=783810560) +2022-04-12 06:24:11.426924: +epoch: 383 +2022-04-12 06:25:59.403563: train loss : -0.7121 +2022-04-12 06:26:10.642330: validation loss: -0.6770 +2022-04-12 06:26:10.664762: Average global foreground Dice: [0.8997, 0.9706, 0.9826, 0.3968] +2022-04-12 06:26:10.688360: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:26:11.540946: lr: 0.002685 +2022-04-12 06:26:11.547449: saving scheduled checkpoint file... +2022-04-12 06:26:11.749455: saving checkpoint... +2022-04-12 06:26:25.543285: done, saving took 13.89 seconds +2022-04-12 06:26:25.567556: done +2022-04-12 06:26:25.633260: saving checkpoint... +2022-04-12 06:26:35.933225: done, saving took 10.35 seconds +2022-04-12 06:26:36.098617: This epoch took 144.622085 s + +2022-04-12 06:26:36.101492: svmem(total=99932160000, available=79610765312, percent=20.3, used=16302051328, free=47904337920, active=40579772416, inactive=9590398976, buffers=38428672, cached=35687342080, shared=2190729216, slab=783650816) +2022-04-12 06:26:36.129177: +epoch: 384 +2022-04-12 06:28:25.458706: train loss : -0.6935 +2022-04-12 06:28:36.824900: validation loss: -0.6685 +2022-04-12 06:28:36.969366: Average global foreground Dice: [0.8868, 0.9672, 0.982, 0.3888] +2022-04-12 06:28:37.085430: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:28:38.078169: lr: 0.002664 +2022-04-12 06:28:38.227986: saving scheduled checkpoint file... +2022-04-12 06:28:38.403887: saving checkpoint... +2022-04-12 06:28:49.535580: done, saving took 11.18 seconds +2022-04-12 06:28:49.839556: done +2022-04-12 06:28:49.890551: This epoch took 133.738588 s + +2022-04-12 06:28:49.894461: svmem(total=99932160000, available=79747887104, percent=20.2, used=16165093376, free=48024121344, active=40444665856, inactive=9606225920, buffers=38461440, cached=35704483840, shared=2190618624, slab=783126528) +2022-04-12 06:28:49.907843: +epoch: 385 +2022-04-12 06:30:38.112864: train loss : -0.6809 +2022-04-12 06:30:50.955182: validation loss: -0.6604 +2022-04-12 06:30:51.613350: Average global foreground Dice: [0.9008, 0.9711, 0.9802, 0.394] +2022-04-12 06:30:51.704280: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:30:52.512790: lr: 0.002643 +2022-04-12 06:30:52.639622: saving scheduled checkpoint file... +2022-04-12 06:30:52.788568: saving checkpoint... +2022-04-12 06:31:04.010992: done, saving took 11.30 seconds +2022-04-12 06:31:04.164098: done +2022-04-12 06:31:04.304323: This epoch took 134.367733 s + +2022-04-12 06:31:04.381311: svmem(total=99932160000, available=79857528832, percent=20.1, used=16055115776, free=48110538752, active=40335966208, inactive=9612406784, buffers=38526976, cached=35727978496, shared=2190917632, slab=784146432) +2022-04-12 06:31:04.451990: +epoch: 386 +2022-04-12 06:32:51.700593: train loss : -0.6886 +2022-04-12 06:33:02.540957: validation loss: -0.6779 +2022-04-12 06:33:02.658508: Average global foreground Dice: [0.9181, 0.9721, 0.9845, 0.3769] +2022-04-12 06:33:02.722482: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:33:03.758058: lr: 0.002622 +2022-04-12 06:33:03.883689: saving scheduled checkpoint file... +2022-04-12 06:33:03.996838: saving checkpoint... +2022-04-12 06:33:16.887678: done, saving took 12.99 seconds +2022-04-12 06:33:17.133806: done +2022-04-12 06:33:17.193085: This epoch took 132.735872 s + +2022-04-12 06:33:17.250278: svmem(total=99932160000, available=79750356992, percent=20.2, used=16152006656, free=47995793408, active=40443170816, inactive=9630298112, buffers=38584320, cached=35745775616, shared=2201214976, slab=783413248) +2022-04-12 06:33:17.312881: +epoch: 387 +2022-04-12 06:35:06.538954: train loss : -0.6929 +2022-04-12 06:35:18.702787: validation loss: -0.6856 +2022-04-12 06:35:18.722350: Average global foreground Dice: [0.9122, 0.9716, 0.9818, 0.4] +2022-04-12 06:35:18.774176: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:35:19.771763: lr: 0.002601 +2022-04-12 06:35:19.904408: saving scheduled checkpoint file... +2022-04-12 06:35:20.093216: saving checkpoint... +2022-04-12 06:35:40.661163: done, saving took 20.69 seconds +2022-04-12 06:35:40.817163: done +2022-04-12 06:35:41.009423: saving checkpoint... +2022-04-12 06:36:02.719439: done, saving took 21.82 seconds +2022-04-12 06:36:02.913304: This epoch took 165.538580 s + +2022-04-12 06:36:02.932390: svmem(total=99932160000, available=79834087424, percent=20.1, used=16091971584, free=48059555840, active=40387211264, inactive=9626419200, buffers=38645760, cached=35741986816, shared=2177568768, slab=783810560) +2022-04-12 06:36:02.945497: +epoch: 388 +2022-04-12 06:37:51.768583: train loss : -0.6921 +2022-04-12 06:38:03.169367: validation loss: -0.6610 +2022-04-12 06:38:03.213160: Average global foreground Dice: [0.9131, 0.9675, 0.9784, 0.3651] +2022-04-12 06:38:03.264672: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:38:04.115519: lr: 0.002581 +2022-04-12 06:38:04.151586: saving scheduled checkpoint file... +2022-04-12 06:38:04.310028: saving checkpoint... +2022-04-12 06:38:20.498955: done, saving took 16.30 seconds +2022-04-12 06:38:20.599875: done +2022-04-12 06:38:20.668650: This epoch took 137.697413 s + +2022-04-12 06:38:20.690711: svmem(total=99932160000, available=79737462784, percent=20.2, used=16175628288, free=47945465856, active=40468058112, inactive=9656905728, buffers=38707200, cached=35772358656, shared=2190540800, slab=783237120) +2022-04-12 06:38:20.733223: +epoch: 389 +2022-04-12 06:40:08.886186: train loss : -0.7004 +2022-04-12 06:40:21.727668: validation loss: -0.6679 +2022-04-12 06:40:21.844904: Average global foreground Dice: [0.9048, 0.969, 0.9829, 0.3632] +2022-04-12 06:40:21.902339: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:40:22.835158: lr: 0.00256 +2022-04-12 06:40:22.926959: saving scheduled checkpoint file... +2022-04-12 06:40:23.079543: saving checkpoint... +2022-04-12 06:40:35.706193: done, saving took 12.71 seconds +2022-04-12 06:40:35.851896: done +2022-04-12 06:40:35.929230: This epoch took 135.184168 s + +2022-04-12 06:40:35.992244: svmem(total=99932160000, available=79604338688, percent=20.3, used=16308645888, free=47794860032, active=40611262464, inactive=9674207232, buffers=38739968, cached=35789914112, shared=2190540800, slab=783233024) +2022-04-12 06:40:36.088068: +epoch: 390 +2022-04-12 06:42:24.699464: train loss : -0.7077 +2022-04-12 06:42:35.639917: validation loss: -0.6795 +2022-04-12 06:42:35.768363: Average global foreground Dice: [0.9038, 0.9715, 0.9832, 0.3843] +2022-04-12 06:42:35.831522: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:42:36.929511: lr: 0.002539 +2022-04-12 06:42:36.980083: saving scheduled checkpoint file... +2022-04-12 06:42:37.147691: saving checkpoint... +2022-04-12 06:42:47.402494: done, saving took 10.34 seconds +2022-04-12 06:42:47.503750: done +2022-04-12 06:42:47.534804: This epoch took 131.360812 s + +2022-04-12 06:42:47.578015: svmem(total=99932160000, available=79888654336, percent=20.1, used=16023838720, free=48070389760, active=40322494464, inactive=9683435520, buffers=38772736, cached=35799158784, shared=2191003648, slab=783536128) +2022-04-12 06:42:47.607769: +epoch: 391 +2022-04-12 06:44:34.964610: train loss : -0.7057 +2022-04-12 06:44:46.655451: validation loss: -0.6651 +2022-04-12 06:44:46.807380: Average global foreground Dice: [0.907, 0.9709, 0.9791, 0.3707] +2022-04-12 06:44:46.912652: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:44:48.364980: lr: 0.002518 +2022-04-12 06:44:48.459021: saving scheduled checkpoint file... +2022-04-12 06:44:48.578353: saving checkpoint... +2022-04-12 06:45:09.409937: done, saving took 20.89 seconds +2022-04-12 06:45:09.576912: done +2022-04-12 06:45:09.616932: This epoch took 141.954267 s + +2022-04-12 06:45:09.633802: svmem(total=99932160000, available=79596457984, percent=20.3, used=16316297216, free=47760773120, active=40605376512, inactive=9700511744, buffers=38830080, cached=35816259584, shared=2190729216, slab=783679488) +2022-04-12 06:45:09.717338: +epoch: 392 +2022-04-12 06:46:57.250494: train loss : -0.6960 +2022-04-12 06:47:08.522891: validation loss: -0.6705 +2022-04-12 06:47:08.681756: Average global foreground Dice: [0.9086, 0.9725, 0.9844, 0.41] +2022-04-12 06:47:08.735999: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:47:09.713507: lr: 0.002497 +2022-04-12 06:47:09.766593: saving scheduled checkpoint file... +2022-04-12 06:47:09.919790: saving checkpoint... +2022-04-12 06:47:21.497170: done, saving took 11.67 seconds +2022-04-12 06:47:21.774627: done +2022-04-12 06:47:21.854556: This epoch took 132.073402 s + +2022-04-12 06:47:21.861337: svmem(total=99932160000, available=79698595840, percent=20.2, used=16203931648, free=47845408768, active=40497430528, inactive=9728258048, buffers=38871040, cached=35843948544, shared=2201026560, slab=785518592) +2022-04-12 06:47:22.003329: +epoch: 393 +2022-04-12 06:49:10.819619: train loss : -0.7052 +2022-04-12 06:49:22.167113: validation loss: -0.6910 +2022-04-12 06:49:22.263983: Average global foreground Dice: [0.9203, 0.9705, 0.985, 0.3756] +2022-04-12 06:49:22.317424: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:49:23.385165: lr: 0.002476 +2022-04-12 06:49:23.455060: saving scheduled checkpoint file... +2022-04-12 06:49:23.638197: saving checkpoint... +2022-04-12 06:49:41.228223: done, saving took 17.70 seconds +2022-04-12 06:49:41.590567: done +2022-04-12 06:49:41.650438: This epoch took 139.640905 s + +2022-04-12 06:49:41.722531: svmem(total=99932160000, available=79558754304, percent=20.4, used=16354074624, free=47687045120, active=40646725632, inactive=9736249344, buffers=38903808, cached=35852136448, shared=2190618624, slab=786714624) +2022-04-12 06:49:41.782698: +epoch: 394 +2022-04-12 06:51:29.936174: train loss : -0.7116 +2022-04-12 06:51:42.495548: validation loss: -0.6688 +2022-04-12 06:51:42.554977: Average global foreground Dice: [0.9184, 0.9698, 0.9822, 0.3874] +2022-04-12 06:51:42.562689: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:51:43.561569: lr: 0.002455 +2022-04-12 06:51:43.583634: saving scheduled checkpoint file... +2022-04-12 06:51:43.755484: saving checkpoint... +2022-04-12 06:52:06.990276: done, saving took 23.35 seconds +2022-04-12 06:52:07.324229: done +2022-04-12 06:52:07.373122: This epoch took 145.523548 s + +2022-04-12 06:52:07.481037: svmem(total=99932160000, available=79785435136, percent=20.2, used=16127250432, free=47920758784, active=40402796544, inactive=9745338368, buffers=38981632, cached=35845169152, shared=2190729216, slab=785457152) +2022-04-12 06:52:07.598618: +epoch: 395 +2022-04-12 06:53:56.922993: train loss : -0.6948 +2022-04-12 06:54:08.443298: validation loss: -0.6793 +2022-04-12 06:54:08.614194: Average global foreground Dice: [0.9069, 0.9705, 0.9801, 0.4039] +2022-04-12 06:54:08.761759: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:54:09.707715: lr: 0.002434 +2022-04-12 06:54:09.766503: saving scheduled checkpoint file... +2022-04-12 06:54:09.946697: saving checkpoint... +2022-04-12 06:54:21.119849: done, saving took 11.26 seconds +2022-04-12 06:54:21.235767: done +2022-04-12 06:54:21.298294: saving checkpoint... +2022-04-12 06:54:36.698237: done, saving took 15.45 seconds +2022-04-12 06:54:36.928594: This epoch took 149.103408 s + +2022-04-12 06:54:36.972325: svmem(total=99932160000, available=79727693824, percent=20.2, used=16174555136, free=47845646336, active=40459046912, inactive=9766633472, buffers=39030784, cached=35872927744, shared=2201137152, slab=784773120) +2022-04-12 06:54:37.047319: +epoch: 396 +2022-04-12 06:56:24.792070: train loss : -0.6966 +2022-04-12 06:56:35.868771: validation loss: -0.6760 +2022-04-12 06:56:36.403951: Average global foreground Dice: [0.9116, 0.9742, 0.9813, 0.3962] +2022-04-12 06:56:36.938211: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:56:37.676157: lr: 0.002413 +2022-04-12 06:56:37.779523: saving scheduled checkpoint file... +2022-04-12 06:56:37.865342: saving checkpoint... +2022-04-12 06:56:52.523680: done, saving took 14.74 seconds +2022-04-12 06:56:52.651509: done +2022-04-12 06:56:52.718521: saving checkpoint... +2022-04-12 06:57:05.826822: done, saving took 13.16 seconds +2022-04-12 06:57:05.999114: This epoch took 148.924661 s + +2022-04-12 06:57:06.059590: svmem(total=99932160000, available=80351211520, percent=19.6, used=15739514880, free=48450609152, active=40024817664, inactive=9591250944, buffers=39079936, cached=35702956032, shared=2012647424, slab=785526784) +2022-04-12 06:57:06.069632: +epoch: 397 +2022-04-12 06:58:55.282214: train loss : -0.6998 +2022-04-12 06:59:07.255473: validation loss: -0.6915 +2022-04-12 06:59:07.317372: Average global foreground Dice: [0.9127, 0.9726, 0.9831, 0.4013] +2022-04-12 06:59:07.355995: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 06:59:08.422558: lr: 0.002391 +2022-04-12 06:59:08.484806: saving scheduled checkpoint file... +2022-04-12 06:59:08.576017: saving checkpoint... +2022-04-12 06:59:21.454648: done, saving took 12.96 seconds +2022-04-12 06:59:22.625057: done +2022-04-12 06:59:22.711050: saving checkpoint... +2022-04-12 06:59:32.941393: done, saving took 10.28 seconds +2022-04-12 06:59:34.217827: This epoch took 148.114552 s + +2022-04-12 06:59:34.851024: svmem(total=99932160000, available=79799250944, percent=20.1, used=16127250432, free=47880069120, active=40402202624, inactive=9771802624, buffers=39129088, cached=35885711360, shared=2176741376, slab=798748672) +2022-04-12 06:59:35.485658: +epoch: 398 +2022-04-12 07:01:24.774445: train loss : -0.7063 +2022-04-12 07:01:36.662560: validation loss: -0.6785 +2022-04-12 07:01:36.748091: Average global foreground Dice: [0.919, 0.9686, 0.9843, 0.4026] +2022-04-12 07:01:36.837480: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:01:37.581493: lr: 0.00237 +2022-04-12 07:01:37.620747: saving scheduled checkpoint file... +2022-04-12 07:01:37.738880: saving checkpoint... +2022-04-12 07:01:47.537920: done, saving took 9.87 seconds +2022-04-12 07:01:47.854703: done +2022-04-12 07:01:47.979428: saving checkpoint... +2022-04-12 07:01:58.501213: done, saving took 10.57 seconds +2022-04-12 07:01:59.344592: This epoch took 142.868555 s + +2022-04-12 07:01:59.623772: svmem(total=99932160000, available=79855144960, percent=20.1, used=16071327744, free=47931142144, active=40260153344, inactive=9865887744, buffers=39161856, cached=35890528256, shared=2176741376, slab=799830016) +2022-04-12 07:01:59.874173: +epoch: 399 +2022-04-12 07:03:49.984483: train loss : -0.6781 +2022-04-12 07:04:02.162607: validation loss: -0.6664 +2022-04-12 07:04:02.687656: Average global foreground Dice: [0.9096, 0.9678, 0.9834, 0.3764] +2022-04-12 07:04:03.180255: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:04:04.078783: lr: 0.002349 +2022-04-12 07:04:04.131951: saving scheduled checkpoint file... +2022-04-12 07:04:04.285763: saving checkpoint... +2022-04-12 07:04:28.878163: done, saving took 24.69 seconds +2022-04-12 07:04:30.942498: done +2022-04-12 07:04:31.801695: This epoch took 151.677172 s + +2022-04-12 07:04:32.371625: svmem(total=99932160000, available=79940546560, percent=20.0, used=15985889280, free=47994576896, active=40213266432, inactive=9843810304, buffers=39211008, cached=35912482816, shared=2176741376, slab=800948224) +2022-04-12 07:04:32.822365: +epoch: 400 +2022-04-12 07:06:21.921106: train loss : -0.6943 +2022-04-12 07:06:33.468083: validation loss: -0.6695 +2022-04-12 07:06:33.515553: Average global foreground Dice: [0.9095, 0.9668, 0.9829, 0.3866] +2022-04-12 07:06:33.537633: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:06:34.636720: lr: 0.002328 +2022-04-12 07:06:34.697258: saving scheduled checkpoint file... +2022-04-12 07:06:34.842905: saving checkpoint... +2022-04-12 07:06:56.102441: done, saving took 21.38 seconds +2022-04-12 07:06:56.359445: done +2022-04-12 07:06:56.442797: This epoch took 143.196974 s + +2022-04-12 07:06:56.494371: svmem(total=99932160000, available=79940333568, percent=20.0, used=15986016256, free=47976706048, active=40253808640, inactive=9841864704, buffers=39268352, cached=35930169344, shared=2176741376, slab=787750912) +2022-04-12 07:06:56.557076: +epoch: 401 +2022-04-12 07:08:47.877903: train loss : -0.7087 +2022-04-12 07:09:00.097125: validation loss: -0.6790 +2022-04-12 07:09:01.225802: Average global foreground Dice: [0.9195, 0.9661, 0.981, 0.3715] +2022-04-12 07:09:02.199962: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:09:05.245577: lr: 0.002307 +2022-04-12 07:09:06.421473: saving scheduled checkpoint file... +2022-04-12 07:09:07.493092: saving checkpoint... +2022-04-12 07:09:25.314354: done, saving took 17.92 seconds +2022-04-12 07:09:25.409134: done +2022-04-12 07:09:25.462787: This epoch took 148.850852 s + +2022-04-12 07:09:25.500635: svmem(total=99932160000, available=79858466816, percent=20.1, used=16043081728, free=47876272128, active=40319901696, inactive=9871572992, buffers=39321600, cached=35973484544, shared=2201513984, slab=786710528) +2022-04-12 07:09:25.563220: +epoch: 402 +2022-04-12 07:11:14.216275: train loss : -0.6951 +2022-04-12 07:11:25.923674: validation loss: -0.6763 +2022-04-12 07:11:25.990414: Average global foreground Dice: [0.9173, 0.971, 0.9807, 0.37] +2022-04-12 07:11:26.228338: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:11:27.964237: lr: 0.002286 +2022-04-12 07:11:28.693339: saving scheduled checkpoint file... +2022-04-12 07:11:29.288322: saving checkpoint... +2022-04-12 07:11:51.095121: done, saving took 21.86 seconds +2022-04-12 07:11:51.243319: done +2022-04-12 07:11:51.287728: This epoch took 145.686873 s + +2022-04-12 07:11:51.293074: svmem(total=99932160000, available=79720435712, percent=20.2, used=16205742080, free=47927730176, active=40490590208, inactive=9649713152, buffers=39362560, cached=35759325184, shared=2176741376, slab=793706496) +2022-04-12 07:11:51.338226: +epoch: 403 +2022-04-12 07:13:39.975068: train loss : -0.6949 +2022-04-12 07:13:51.697928: validation loss: -0.6707 +2022-04-12 07:13:51.728316: Average global foreground Dice: [0.9054, 0.9704, 0.9784, 0.4001] +2022-04-12 07:13:51.785116: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:13:52.819325: lr: 0.002264 +2022-04-12 07:13:52.873335: saving scheduled checkpoint file... +2022-04-12 07:13:52.979386: saving checkpoint... +2022-04-12 07:14:02.691700: done, saving took 9.81 seconds +2022-04-12 07:14:02.730115: done +2022-04-12 07:14:02.780067: This epoch took 131.422777 s + +2022-04-12 07:14:02.785701: svmem(total=99932160000, available=80135741440, percent=19.8, used=15947608064, free=48120639488, active=40246292480, inactive=9710858240, buffers=39395328, cached=35824517120, shared=2019643392, slab=786251776) +2022-04-12 07:14:02.795840: +epoch: 404 +2022-04-12 07:15:51.906802: train loss : -0.6974 +2022-04-12 07:16:03.736493: validation loss: -0.6692 +2022-04-12 07:16:03.899084: Average global foreground Dice: [0.8959, 0.972, 0.9835, 0.4113] +2022-04-12 07:16:04.123761: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:16:05.485573: lr: 0.002243 +2022-04-12 07:16:05.576414: saving scheduled checkpoint file... +2022-04-12 07:16:05.793470: saving checkpoint... +2022-04-12 07:16:20.136394: done, saving took 14.39 seconds +2022-04-12 07:16:20.243038: done +2022-04-12 07:16:20.293279: This epoch took 137.487098 s + +2022-04-12 07:16:20.341526: svmem(total=99932160000, available=79749464064, percent=20.2, used=16162570240, free=47715856384, active=40454025216, inactive=9900474368, buffers=39456768, cached=36014276608, shared=2191036416, slab=786735104) +2022-04-12 07:16:20.399964: +epoch: 405 +2022-04-12 07:18:08.978417: train loss : -0.7094 +2022-04-12 07:18:20.638734: validation loss: -0.6884 +2022-04-12 07:18:20.695832: Average global foreground Dice: [0.9245, 0.9724, 0.9846, 0.3839] +2022-04-12 07:18:20.765056: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:18:21.769427: lr: 0.002222 +2022-04-12 07:18:21.818334: saving scheduled checkpoint file... +2022-04-12 07:18:22.028027: saving checkpoint... +2022-04-12 07:18:33.286107: done, saving took 11.36 seconds +2022-04-12 07:18:33.525325: done +2022-04-12 07:18:33.550078: This epoch took 133.145514 s + +2022-04-12 07:18:33.649117: svmem(total=99932160000, available=79612882944, percent=20.3, used=16299204608, free=47618547712, active=40484102144, inactive=9970589696, buffers=39497728, cached=35974909952, shared=2190839808, slab=786386944) +2022-04-12 07:18:33.711789: +epoch: 406 +2022-04-12 07:20:21.311209: train loss : -0.7301 +2022-04-12 07:20:32.848167: validation loss: -0.6805 +2022-04-12 07:20:33.139800: Average global foreground Dice: [0.9098, 0.9718, 0.9828, 0.3889] +2022-04-12 07:20:33.166963: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:20:34.080718: lr: 0.002201 +2022-04-12 07:20:34.129067: saving scheduled checkpoint file... +2022-04-12 07:20:34.265040: saving checkpoint... +2022-04-12 07:20:42.364140: done, saving took 8.20 seconds +2022-04-12 07:20:42.699657: done +2022-04-12 07:20:42.714479: This epoch took 128.994681 s + +2022-04-12 07:20:42.784061: svmem(total=99932160000, available=79636680704, percent=20.3, used=16275320832, free=47628279808, active=40520970240, inactive=9926807552, buffers=39538688, cached=35989020672, shared=2191003648, slab=785858560) +2022-04-12 07:20:42.881188: +epoch: 407 +2022-04-12 07:22:31.672971: train loss : -0.7230 +2022-04-12 07:22:42.926694: validation loss: -0.6991 +2022-04-12 07:22:43.020688: Average global foreground Dice: [0.9259, 0.9749, 0.9837, 0.3871] +2022-04-12 07:22:43.091859: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:22:44.373379: lr: 0.002179 +2022-04-12 07:22:44.481883: saving scheduled checkpoint file... +2022-04-12 07:22:44.662375: saving checkpoint... +2022-04-12 07:22:58.679679: done, saving took 14.12 seconds +2022-04-12 07:22:58.737549: done +2022-04-12 07:22:58.792101: saving checkpoint... +2022-04-12 07:23:11.397810: done, saving took 12.66 seconds +2022-04-12 07:23:11.574257: This epoch took 148.565810 s + +2022-04-12 07:23:11.628054: svmem(total=99932160000, available=79702925312, percent=20.2, used=16209813504, free=47624404992, active=40489058304, inactive=9947480064, buffers=39596032, cached=36058345472, shared=2190311424, slab=786059264) +2022-04-12 07:23:11.645881: +epoch: 408 +2022-04-12 07:25:00.627128: train loss : -0.7197 +2022-04-12 07:25:12.235145: validation loss: -0.6903 +2022-04-12 07:25:12.321222: Average global foreground Dice: [0.9174, 0.9744, 0.9834, 0.3922] +2022-04-12 07:25:12.361490: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:25:13.435892: lr: 0.002158 +2022-04-12 07:25:13.450303: saving scheduled checkpoint file... +2022-04-12 07:25:13.583994: saving checkpoint... +2022-04-12 07:25:33.361754: done, saving took 19.87 seconds +2022-04-12 07:25:33.569646: done +2022-04-12 07:25:33.646861: saving checkpoint... +2022-04-12 07:25:54.226505: done, saving took 20.65 seconds +2022-04-12 07:25:54.405060: This epoch took 162.702097 s + +2022-04-12 07:25:54.521402: svmem(total=99932160000, available=79842955264, percent=20.1, used=16069279744, free=47744774144, active=40359059456, inactive=9964777472, buffers=39657472, cached=36078448640, shared=2190729216, slab=785596416) +2022-04-12 07:25:54.536889: +epoch: 409 +2022-04-12 07:27:43.291610: train loss : -0.7272 +2022-04-12 07:27:56.506981: validation loss: -0.6986 +2022-04-12 07:27:56.745596: Average global foreground Dice: [0.9317, 0.9778, 0.9833, 0.3804] +2022-04-12 07:27:56.934021: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:27:58.114607: lr: 0.002137 +2022-04-12 07:27:58.160750: saving scheduled checkpoint file... +2022-04-12 07:27:58.298607: saving checkpoint... +2022-04-12 07:28:15.872322: done, saving took 17.66 seconds +2022-04-12 07:28:16.188528: done +2022-04-12 07:28:16.337486: saving checkpoint... +2022-04-12 07:28:31.667305: done, saving took 15.39 seconds +2022-04-12 07:28:33.329012: This epoch took 158.665224 s + +2022-04-12 07:28:34.535628: svmem(total=99932160000, available=79801323520, percent=20.1, used=16124862464, free=47684730880, active=40406634496, inactive=9965883392, buffers=39706624, cached=36082860032, shared=2176741376, slab=799453184) +2022-04-12 07:28:35.687718: +epoch: 410 +2022-04-12 07:30:24.732808: train loss : -0.7175 +2022-04-12 07:30:36.747049: validation loss: -0.6853 +2022-04-12 07:30:36.865557: Average global foreground Dice: [0.9092, 0.9689, 0.984, 0.3894] +2022-04-12 07:30:36.926685: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:30:37.953222: lr: 0.002115 +2022-04-12 07:30:38.039660: saving scheduled checkpoint file... +2022-04-12 07:30:38.146124: saving checkpoint... +2022-04-12 07:30:54.906312: done, saving took 16.81 seconds +2022-04-12 07:30:55.120747: done +2022-04-12 07:30:55.183494: This epoch took 138.394486 s + +2022-04-12 07:30:55.232141: svmem(total=99932160000, available=79763922944, percent=20.2, used=16162361344, free=47650947072, active=40464019456, inactive=9962123264, buffers=39747584, cached=36079104000, shared=2176741376, slab=785125376) +2022-04-12 07:30:55.275630: +epoch: 411 +2022-04-12 07:32:44.901360: train loss : -0.7195 +2022-04-12 07:32:56.698977: validation loss: -0.7026 +2022-04-12 07:32:56.733315: Average global foreground Dice: [0.9242, 0.9721, 0.9813, 0.4053] +2022-04-12 07:32:56.753377: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:32:57.738949: lr: 0.002094 +2022-04-12 07:32:57.747903: saving scheduled checkpoint file... +2022-04-12 07:32:57.855337: saving checkpoint... +2022-04-12 07:33:10.923342: done, saving took 13.16 seconds +2022-04-12 07:33:11.252752: done +2022-04-12 07:33:11.386227: saving checkpoint... +2022-04-12 07:33:26.307181: done, saving took 15.00 seconds +2022-04-12 07:33:26.482316: This epoch took 151.154841 s + +2022-04-12 07:33:26.541326: svmem(total=99932160000, available=80189243392, percent=19.8, used=15914971136, free=48036474880, active=40012636160, inactive=10019622912, buffers=39796736, cached=35940917248, shared=1998864384, slab=784613376) +2022-04-12 07:33:26.630687: +epoch: 412 +2022-04-12 07:35:15.695379: train loss : -0.7186 +2022-04-12 07:35:27.435325: validation loss: -0.6919 +2022-04-12 07:35:27.521286: Average global foreground Dice: [0.9212, 0.9758, 0.9849, 0.3978] +2022-04-12 07:35:27.546679: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:35:28.421102: lr: 0.002072 +2022-04-12 07:35:28.479589: saving scheduled checkpoint file... +2022-04-12 07:35:28.671978: saving checkpoint... +2022-04-12 07:35:42.903940: done, saving took 14.36 seconds +2022-04-12 07:35:43.092305: done +2022-04-12 07:35:43.189960: saving checkpoint... +2022-04-12 07:35:57.871270: done, saving took 14.78 seconds +2022-04-12 07:35:57.974311: This epoch took 151.246705 s + +2022-04-12 07:35:58.015133: svmem(total=99932160000, available=80220217344, percent=19.7, used=15866613760, free=48067715072, active=40025169920, inactive=9976344576, buffers=39845888, cached=35957985280, shared=2016133120, slab=784068608) +2022-04-12 07:35:58.088341: +epoch: 413 +2022-04-12 07:37:45.977522: train loss : -0.7252 +2022-04-12 07:37:57.414108: validation loss: -0.7070 +2022-04-12 07:37:57.484395: Average global foreground Dice: [0.9344, 0.9758, 0.9852, 0.3911] +2022-04-12 07:37:57.544733: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:37:58.665223: lr: 0.002051 +2022-04-12 07:37:58.715936: saving scheduled checkpoint file... +2022-04-12 07:37:58.827795: saving checkpoint... +2022-04-12 07:38:06.798653: done, saving took 8.06 seconds +2022-04-12 07:38:06.994130: done +2022-04-12 07:38:07.186208: saving checkpoint... +2022-04-12 07:38:20.006267: done, saving took 12.87 seconds +2022-04-12 07:38:20.205873: This epoch took 142.081822 s + +2022-04-12 07:38:20.298821: svmem(total=99932160000, available=79904456704, percent=20.0, used=16011456512, free=47733284864, active=40261373952, inactive=10072920064, buffers=39911424, cached=36147507200, shared=2187071488, slab=784097280) +2022-04-12 07:38:20.304192: +epoch: 414 +2022-04-12 07:40:08.145141: train loss : -0.7123 +2022-04-12 07:40:19.662393: validation loss: -0.6915 +2022-04-12 07:40:19.750898: Average global foreground Dice: [0.9065, 0.9714, 0.9829, 0.4079] +2022-04-12 07:40:19.755150: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:40:20.830658: lr: 0.00203 +2022-04-12 07:40:20.860682: saving scheduled checkpoint file... +2022-04-12 07:40:20.972546: saving checkpoint... +2022-04-12 07:40:38.168311: done, saving took 17.28 seconds +2022-04-12 07:40:38.245437: done +2022-04-12 07:40:38.307327: saving checkpoint... +2022-04-12 07:40:46.959108: done, saving took 8.71 seconds +2022-04-12 07:40:47.059558: This epoch took 146.751108 s + +2022-04-12 07:40:47.102750: svmem(total=99932160000, available=80341581824, percent=19.6, used=15730569216, free=48143904768, active=40017752064, inactive=9914638336, buffers=39952384, cached=36017733632, shared=2030731264, slab=784474112) +2022-04-12 07:40:47.108072: +epoch: 415 +2022-04-12 07:42:36.705534: train loss : -0.7166 +2022-04-12 07:42:48.553426: validation loss: -0.6844 +2022-04-12 07:42:48.632296: Average global foreground Dice: [0.9175, 0.9735, 0.9796, 0.3675] +2022-04-12 07:42:48.699128: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:42:49.687556: lr: 0.002008 +2022-04-12 07:42:49.762547: saving scheduled checkpoint file... +2022-04-12 07:42:50.009332: saving checkpoint... +2022-04-12 07:43:04.819458: done, saving took 14.89 seconds +2022-04-12 07:43:05.002291: done +2022-04-12 07:43:05.071802: This epoch took 137.958845 s + +2022-04-12 07:43:05.231997: svmem(total=99932160000, available=80300351488, percent=19.6, used=15832059904, free=48084025344, active=40127520768, inactive=9861320704, buffers=39985152, cached=35976089600, shared=1970556928, slab=783675392) +2022-04-12 07:43:05.241086: +epoch: 416 +2022-04-12 07:44:53.991757: train loss : -0.7141 +2022-04-12 07:45:05.313861: validation loss: -0.6798 +2022-04-12 07:45:05.601784: Average global foreground Dice: [0.9225, 0.9731, 0.9846, 0.3711] +2022-04-12 07:45:05.960349: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:45:07.566883: lr: 0.001987 +2022-04-12 07:45:07.850590: saving scheduled checkpoint file... +2022-04-12 07:45:07.988723: saving checkpoint... +2022-04-12 07:45:18.071674: done, saving took 10.17 seconds +2022-04-12 07:45:18.273488: done +2022-04-12 07:45:18.331225: This epoch took 132.968547 s + +2022-04-12 07:45:18.340852: svmem(total=99932160000, available=79737106432, percent=20.2, used=16174948352, free=47513047040, active=40474558464, inactive=10086973440, buffers=40042496, cached=36204122112, shared=2190807040, slab=784863232) +2022-04-12 07:45:18.383080: +epoch: 417 +2022-04-12 07:47:05.014185: train loss : -0.7150 +2022-04-12 07:47:17.098503: validation loss: -0.6865 +2022-04-12 07:47:17.148800: Average global foreground Dice: [0.9174, 0.9729, 0.9821, 0.3752] +2022-04-12 07:47:17.302906: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:47:18.193933: lr: 0.001965 +2022-04-12 07:47:18.204832: saving scheduled checkpoint file... +2022-04-12 07:47:18.420810: saving checkpoint... +2022-04-12 07:47:32.717936: done, saving took 14.40 seconds +2022-04-12 07:47:32.813192: done +2022-04-12 07:47:32.876930: This epoch took 134.447072 s + +2022-04-12 07:47:32.939397: svmem(total=99932160000, available=79830740992, percent=20.1, used=16071274496, free=47589101568, active=40367181824, inactive=10114678784, buffers=40083456, cached=36231700480, shared=2200948736, slab=785219584) +2022-04-12 07:47:32.993121: +epoch: 418 +2022-04-12 07:49:22.713242: train loss : -0.7219 +2022-04-12 07:49:34.748927: validation loss: -0.6747 +2022-04-12 07:49:34.850378: Average global foreground Dice: [0.9156, 0.9766, 0.9835, 0.3887] +2022-04-12 07:49:34.913826: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:49:35.794021: lr: 0.001943 +2022-04-12 07:49:35.810722: saving scheduled checkpoint file... +2022-04-12 07:49:35.987314: saving checkpoint... +2022-04-12 07:49:48.852552: done, saving took 12.97 seconds +2022-04-12 07:49:48.996710: done +2022-04-12 07:49:48.998704: This epoch took 135.996104 s + +2022-04-12 07:49:49.076274: svmem(total=99932160000, available=79599640576, percent=20.3, used=16312492032, free=47339450368, active=40615243776, inactive=10122711040, buffers=40116224, cached=36240101376, shared=2190696448, slab=786173952) +2022-04-12 07:49:49.142300: +epoch: 419 +2022-04-12 07:51:36.979324: train loss : -0.7152 +2022-04-12 07:51:48.435895: validation loss: -0.6980 +2022-04-12 07:51:48.480540: Average global foreground Dice: [0.9299, 0.9721, 0.9819, 0.378] +2022-04-12 07:51:48.530419: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:51:49.648163: lr: 0.001922 +2022-04-12 07:51:49.725014: saving scheduled checkpoint file... +2022-04-12 07:51:49.871830: saving checkpoint... +2022-04-12 07:52:04.924344: done, saving took 15.15 seconds +2022-04-12 07:52:05.071557: done +2022-04-12 07:52:05.073088: This epoch took 135.864069 s + +2022-04-12 07:52:05.101573: svmem(total=99932160000, available=79831748608, percent=20.1, used=16080433152, free=47607169024, active=40302813184, inactive=10160070656, buffers=40157184, cached=36204400640, shared=2190618624, slab=786620416) +2022-04-12 07:52:05.137043: +epoch: 420 +2022-04-12 07:53:52.288207: train loss : -0.7262 +2022-04-12 07:54:04.494152: validation loss: -0.7016 +2022-04-12 07:54:04.616093: Average global foreground Dice: [0.9275, 0.9739, 0.9841, 0.3921] +2022-04-12 07:54:04.706045: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:54:05.819264: lr: 0.0019 +2022-04-12 07:54:05.865185: saving scheduled checkpoint file... +2022-04-12 07:54:05.961674: saving checkpoint... +2022-04-12 07:54:11.975355: done, saving took 6.10 seconds +2022-04-12 07:54:12.076335: done +2022-04-12 07:54:12.122731: This epoch took 126.931536 s + +2022-04-12 07:54:12.162964: svmem(total=99932160000, available=79748153344, percent=20.2, used=16163876864, free=47515787264, active=40443199488, inactive=10116227072, buffers=40206336, cached=36212289536, shared=2190696448, slab=785911808) +2022-04-12 07:54:12.235433: +epoch: 421 +2022-04-12 07:55:59.861450: train loss : -0.7175 +2022-04-12 07:56:11.500777: validation loss: -0.6893 +2022-04-12 07:56:11.584888: Average global foreground Dice: [0.9127, 0.9677, 0.9811, 0.3654] +2022-04-12 07:56:11.634760: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:56:12.465915: lr: 0.001879 +2022-04-12 07:56:12.481042: saving scheduled checkpoint file... +2022-04-12 07:56:12.576841: saving checkpoint... +2022-04-12 07:56:26.921933: done, saving took 14.44 seconds +2022-04-12 07:56:26.977685: done +2022-04-12 07:56:26.981770: This epoch took 134.696211 s + +2022-04-12 07:56:26.983785: svmem(total=99932160000, available=79757402112, percent=20.2, used=16154570752, free=47507599360, active=40451661824, inactive=10114052096, buffers=40251392, cached=36229738496, shared=2190729216, slab=786493440) +2022-04-12 07:56:26.985142: +epoch: 422 +2022-04-12 07:58:14.857113: train loss : -0.7308 +2022-04-12 07:58:26.634182: validation loss: -0.6798 +2022-04-12 07:58:26.742976: Average global foreground Dice: [0.9243, 0.9679, 0.9806, 0.3846] +2022-04-12 07:58:26.756618: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 07:58:27.663151: lr: 0.001857 +2022-04-12 07:58:27.665335: saving scheduled checkpoint file... +2022-04-12 07:58:27.757763: saving checkpoint... +2022-04-12 07:58:37.962676: done, saving took 10.30 seconds +2022-04-12 07:58:38.038420: done +2022-04-12 07:58:38.047129: This epoch took 131.060517 s + +2022-04-12 07:58:38.049682: svmem(total=99932160000, available=79743238144, percent=20.2, used=16169148416, free=47425024000, active=40468197376, inactive=10179977216, buffers=40284160, cached=36297703424, shared=2190233600, slab=787279872) +2022-04-12 07:58:38.053002: +epoch: 423 +2022-04-12 08:00:25.370766: train loss : -0.7365 +2022-04-12 08:00:36.631172: validation loss: -0.6711 +2022-04-12 08:00:36.718895: Average global foreground Dice: [0.9125, 0.9729, 0.9826, 0.3778] +2022-04-12 08:00:36.723561: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:00:37.615450: lr: 0.001835 +2022-04-12 08:00:37.677615: saving scheduled checkpoint file... +2022-04-12 08:00:37.804954: saving checkpoint... +2022-04-12 08:00:50.766578: done, saving took 13.02 seconds +2022-04-12 08:00:50.970654: done +2022-04-12 08:00:51.014558: This epoch took 132.956081 s + +2022-04-12 08:00:51.074409: svmem(total=99932160000, available=79725260800, percent=20.2, used=16177278976, free=47399284736, active=40478748672, inactive=10197594112, buffers=40325120, cached=36315271168, shared=2200145920, slab=787083264) +2022-04-12 08:00:51.130022: +epoch: 424 +2022-04-12 08:02:39.861238: train loss : -0.7306 +2022-04-12 08:02:51.455086: validation loss: -0.7040 +2022-04-12 08:02:51.692725: Average global foreground Dice: [0.9358, 0.9753, 0.9866, 0.3905] +2022-04-12 08:02:51.751134: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:02:52.921125: lr: 0.001813 +2022-04-12 08:02:52.984607: saving scheduled checkpoint file... +2022-04-12 08:02:53.179772: saving checkpoint... +2022-04-12 08:03:04.608837: done, saving took 11.56 seconds +2022-04-12 08:03:05.337065: done +2022-04-12 08:03:05.429485: This epoch took 134.263268 s + +2022-04-12 08:03:05.435838: svmem(total=99932160000, available=79998607360, percent=19.9, used=15922249728, free=47654084608, active=40216678400, inactive=10197192704, buffers=40357888, cached=36315467776, shared=2181808128, slab=787116032) +2022-04-12 08:03:05.459317: +epoch: 425 +2022-04-12 08:04:55.181365: train loss : -0.7002 +2022-04-12 08:05:08.684565: validation loss: -0.7058 +2022-04-12 08:05:08.720484: Average global foreground Dice: [0.9304, 0.9729, 0.9857, 0.3737] +2022-04-12 08:05:08.727545: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:05:09.714452: lr: 0.001792 +2022-04-12 08:05:09.731956: saving scheduled checkpoint file... +2022-04-12 08:05:09.838410: saving checkpoint... +2022-04-12 08:05:21.339484: done, saving took 11.59 seconds +2022-04-12 08:05:21.417852: done +2022-04-12 08:05:21.493235: This epoch took 135.999928 s + +2022-04-12 08:05:21.499543: svmem(total=99932160000, available=79647789056, percent=20.3, used=16277942272, free=47285657600, active=40575008768, inactive=10210041856, buffers=40415232, cached=36328144896, shared=2176851968, slab=787226624) +2022-04-12 08:05:21.547970: +epoch: 426 +2022-04-12 08:07:10.361130: train loss : -0.7197 +2022-04-12 08:07:22.622575: validation loss: -0.6898 +2022-04-12 08:07:22.796738: Average global foreground Dice: [0.9194, 0.9719, 0.9831, 0.3764] +2022-04-12 08:07:22.907054: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:07:23.840281: lr: 0.00177 +2022-04-12 08:07:23.914314: saving scheduled checkpoint file... +2022-04-12 08:07:24.027295: saving checkpoint... +2022-04-12 08:07:40.558622: done, saving took 16.58 seconds +2022-04-12 08:07:40.882250: done +2022-04-12 08:07:40.963889: This epoch took 139.348772 s + +2022-04-12 08:07:41.058096: svmem(total=99932160000, available=79890382848, percent=20.1, used=16021463040, free=47510880256, active=40318799872, inactive=10241269760, buffers=40472576, cached=36359344128, shared=2190729216, slab=787767296) +2022-04-12 08:07:41.163280: +epoch: 427 +2022-04-12 08:09:30.338973: train loss : -0.7127 +2022-04-12 08:09:42.123709: validation loss: -0.6995 +2022-04-12 08:09:42.209994: Average global foreground Dice: [0.9117, 0.9706, 0.9847, 0.4084] +2022-04-12 08:09:42.225169: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:09:43.247804: lr: 0.001748 +2022-04-12 08:09:43.256737: saving scheduled checkpoint file... +2022-04-12 08:09:43.420280: saving checkpoint... +2022-04-12 08:09:55.361892: done, saving took 12.04 seconds +2022-04-12 08:09:55.570250: done +2022-04-12 08:09:55.577465: This epoch took 134.347617 s + +2022-04-12 08:09:55.616168: svmem(total=99932160000, available=79824642048, percent=20.1, used=16087605248, free=47436349440, active=40387325952, inactive=10249670656, buffers=40505344, cached=36367699968, shared=2190082048, slab=787042304) +2022-04-12 08:09:55.667552: +epoch: 428 +2022-04-12 08:11:44.013767: train loss : -0.7229 +2022-04-12 08:11:55.256383: validation loss: -0.6746 +2022-04-12 08:11:55.845123: Average global foreground Dice: [0.9313, 0.9703, 0.9794, 0.3852] +2022-04-12 08:11:55.973152: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:11:57.185232: lr: 0.001726 +2022-04-12 08:11:57.231356: saving scheduled checkpoint file... +2022-04-12 08:11:57.380497: saving checkpoint... +2022-04-12 08:12:13.580316: done, saving took 16.28 seconds +2022-04-12 08:12:15.237583: done +2022-04-12 08:12:15.353849: This epoch took 139.683861 s + +2022-04-12 08:12:15.485327: svmem(total=99932160000, available=79618777088, percent=20.3, used=16293396480, free=47213150208, active=40590008320, inactive=10266701824, buffers=40562688, cached=36385050624, shared=2190233600, slab=786206720) +2022-04-12 08:12:15.610665: +epoch: 429 +2022-04-12 08:14:04.472803: train loss : -0.7244 +2022-04-12 08:14:16.167851: validation loss: -0.6881 +2022-04-12 08:14:16.339276: Average global foreground Dice: [0.9228, 0.9743, 0.9809, 0.3854] +2022-04-12 08:14:16.396679: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:14:17.643161: lr: 0.001704 +2022-04-12 08:14:17.699006: saving scheduled checkpoint file... +2022-04-12 08:14:17.806862: saving checkpoint... +2022-04-12 08:14:29.319267: done, saving took 11.58 seconds +2022-04-12 08:14:29.545586: done +2022-04-12 08:14:29.607275: This epoch took 133.812027 s + +2022-04-12 08:14:29.666448: svmem(total=99932160000, available=79676919808, percent=20.3, used=16241643520, free=47264550912, active=40406081536, inactive=10403340288, buffers=40595456, cached=36385370112, shared=2183983104, slab=785285120) +2022-04-12 08:14:29.728888: +epoch: 430 +2022-04-12 08:16:18.773756: train loss : -0.7204 +2022-04-12 08:16:30.393972: validation loss: -0.6878 +2022-04-12 08:16:30.462761: Average global foreground Dice: [0.9232, 0.9708, 0.986, 0.3794] +2022-04-12 08:16:30.514918: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:16:31.806579: lr: 0.001682 +2022-04-12 08:16:31.862438: saving scheduled checkpoint file... +2022-04-12 08:16:32.027119: saving checkpoint... +2022-04-12 08:16:51.191537: done, saving took 19.28 seconds +2022-04-12 08:16:51.811497: done +2022-04-12 08:16:51.856093: This epoch took 142.070765 s + +2022-04-12 08:16:52.002888: svmem(total=99932160000, available=79700180992, percent=20.2, used=16225308672, free=47273205760, active=40416407552, inactive=10381742080, buffers=40660992, cached=36392984576, shared=2176929792, slab=785879040) +2022-04-12 08:16:52.091719: +epoch: 431 +2022-04-12 08:18:41.257779: train loss : -0.7315 +2022-04-12 08:18:53.063924: validation loss: -0.6900 +2022-04-12 08:18:53.195994: Average global foreground Dice: [0.9104, 0.9757, 0.9854, 0.3713] +2022-04-12 08:18:53.347044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:18:54.648005: lr: 0.00166 +2022-04-12 08:18:54.860494: saving scheduled checkpoint file... +2022-04-12 08:18:55.018616: saving checkpoint... +2022-04-12 08:19:03.816678: done, saving took 8.89 seconds +2022-04-12 08:19:04.006267: done +2022-04-12 08:19:04.066444: This epoch took 131.925684 s + +2022-04-12 08:19:04.125979: svmem(total=99932160000, available=79589126144, percent=20.4, used=16312041472, free=47176433664, active=40449351680, inactive=10444619776, buffers=40701952, cached=36402982912, shared=2201325568, slab=784662528) +2022-04-12 08:19:04.186533: +epoch: 432 +2022-04-12 08:20:53.545113: train loss : -0.7176 +2022-04-12 08:21:04.949488: validation loss: -0.7077 +2022-04-12 08:21:04.978087: Average global foreground Dice: [0.925, 0.9736, 0.9841, 0.4037] +2022-04-12 08:21:05.141682: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:21:05.953574: lr: 0.001638 +2022-04-12 08:21:06.105273: saving scheduled checkpoint file... +2022-04-12 08:21:06.270391: saving checkpoint... +2022-04-12 08:21:22.040257: done, saving took 15.85 seconds +2022-04-12 08:21:22.231186: done +2022-04-12 08:21:22.246571: This epoch took 137.998195 s + +2022-04-12 08:21:22.324275: svmem(total=99932160000, available=79681486848, percent=20.3, used=16241217536, free=47262482432, active=40443699200, inactive=10361176064, buffers=40759296, cached=36387700736, shared=2179715072, slab=786104320) +2022-04-12 08:21:22.367450: +epoch: 433 +2022-04-12 08:23:10.717585: train loss : -0.7098 +2022-04-12 08:23:21.647166: validation loss: -0.6780 +2022-04-12 08:23:21.809194: Average global foreground Dice: [0.9311, 0.9696, 0.9842, 0.357] +2022-04-12 08:23:21.840688: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:23:22.982487: lr: 0.001616 +2022-04-12 08:23:23.174045: saving scheduled checkpoint file... +2022-04-12 08:23:23.316413: saving checkpoint... +2022-04-12 08:23:37.699299: done, saving took 14.46 seconds +2022-04-12 08:23:37.865872: done +2022-04-12 08:23:37.911627: This epoch took 135.454070 s + +2022-04-12 08:23:37.963516: svmem(total=99932160000, available=79465414656, percent=20.5, used=16445902848, free=47024627712, active=40694149120, inactive=10353733632, buffers=40800256, cached=36420829184, shared=2191114240, slab=786526208) +2022-04-12 08:23:38.007948: +epoch: 434 +2022-04-12 08:25:26.489882: train loss : -0.7169 +2022-04-12 08:25:38.724820: validation loss: -0.6926 +2022-04-12 08:25:38.874058: Average global foreground Dice: [0.9335, 0.9734, 0.9816, 0.3665] +2022-04-12 08:25:39.108821: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:25:40.193095: lr: 0.001594 +2022-04-12 08:25:40.315462: saving scheduled checkpoint file... +2022-04-12 08:25:40.381528: saving checkpoint... +2022-04-12 08:25:52.481863: done, saving took 12.15 seconds +2022-04-12 08:25:52.690092: done +2022-04-12 08:25:52.784146: This epoch took 134.727436 s + +2022-04-12 08:25:52.835341: svmem(total=99932160000, available=79751446528, percent=20.2, used=16163684352, free=47326871552, active=40024363008, inactive=10717962240, buffers=40845312, cached=36400758784, shared=2187259904, slab=786857984) +2022-04-12 08:25:52.896336: +epoch: 435 +2022-04-12 08:27:40.856405: train loss : -0.7270 +2022-04-12 08:27:55.734388: validation loss: -0.6634 +2022-04-12 08:27:56.272953: Average global foreground Dice: [0.9217, 0.9728, 0.9795, 0.3699] +2022-04-12 08:27:56.634064: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:27:57.543106: lr: 0.001572 +2022-04-12 08:27:57.598883: saving scheduled checkpoint file... +2022-04-12 08:27:57.689868: saving checkpoint... +2022-04-12 08:28:18.124561: done, saving took 20.52 seconds +2022-04-12 08:28:20.334107: done +2022-04-12 08:28:20.403029: This epoch took 147.457934 s + +2022-04-12 08:28:20.465788: svmem(total=99932160000, available=79866245120, percent=20.1, used=16059326464, free=47443525632, active=39930105856, inactive=10691743744, buffers=40894464, cached=36388413440, shared=2176741376, slab=786550784) +2022-04-12 08:28:20.595357: +epoch: 436 +2022-04-12 08:30:08.975519: train loss : -0.7281 +2022-04-12 08:30:19.785652: validation loss: -0.7162 +2022-04-12 08:30:21.522628: Average global foreground Dice: [0.9382, 0.978, 0.984, 0.386] +2022-04-12 08:30:21.913277: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:30:23.307349: lr: 0.00155 +2022-04-12 08:30:23.414488: saving scheduled checkpoint file... +2022-04-12 08:30:23.701079: saving checkpoint... +2022-04-12 08:30:41.250052: done, saving took 17.65 seconds +2022-04-12 08:30:42.250427: done +2022-04-12 08:30:42.303087: This epoch took 141.586975 s + +2022-04-12 08:30:42.348050: svmem(total=99932160000, available=79937511424, percent=20.0, used=15987990528, free=47493672960, active=40139456512, inactive=10435743744, buffers=40964096, cached=36409532416, shared=2176741376, slab=786411520) +2022-04-12 08:30:42.376420: +epoch: 437 +2022-04-12 08:32:31.543316: train loss : -0.7268 +2022-04-12 08:32:43.314179: validation loss: -0.6808 +2022-04-12 08:32:43.393879: Average global foreground Dice: [0.9198, 0.9742, 0.9831, 0.4016] +2022-04-12 08:32:43.507939: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:32:44.641158: lr: 0.001528 +2022-04-12 08:32:44.918563: saving scheduled checkpoint file... +2022-04-12 08:32:45.236420: saving checkpoint... +2022-04-12 08:33:03.230281: done, saving took 18.08 seconds +2022-04-12 08:33:03.389537: done +2022-04-12 08:33:03.508513: This epoch took 141.122031 s + +2022-04-12 08:33:03.581567: svmem(total=99932160000, available=80036642816, percent=19.9, used=16011243520, free=47608496128, active=40104505344, inactive=10365927424, buffers=40996864, cached=36271423488, shared=2054352896, slab=785035264) +2022-04-12 08:33:03.645028: +epoch: 438 +2022-04-12 08:34:52.456872: train loss : -0.7183 +2022-04-12 08:35:04.216339: validation loss: -0.6842 +2022-04-12 08:35:04.258781: Average global foreground Dice: [0.9191, 0.9729, 0.9828, 0.3862] +2022-04-12 08:35:04.291866: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:35:05.286119: lr: 0.001506 +2022-04-12 08:35:05.344935: saving scheduled checkpoint file... +2022-04-12 08:35:05.479911: saving checkpoint... +2022-04-12 08:35:23.800540: done, saving took 18.41 seconds +2022-04-12 08:35:25.890328: done +2022-04-12 08:35:26.052155: This epoch took 142.401468 s + +2022-04-12 08:35:26.196990: svmem(total=99932160000, available=80040267776, percent=19.9, used=15909621760, free=47563206656, active=40123752448, inactive=10390970368, buffers=41054208, cached=36418277376, shared=2152267776, slab=784838656) +2022-04-12 08:35:26.314341: +epoch: 439 +2022-04-12 08:37:14.435119: train loss : -0.7233 +2022-04-12 08:37:26.170506: validation loss: -0.6849 +2022-04-12 08:37:26.243360: Average global foreground Dice: [0.925, 0.9739, 0.9773, 0.3976] +2022-04-12 08:37:26.327409: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:37:27.688722: lr: 0.001483 +2022-04-12 08:37:28.164569: saving scheduled checkpoint file... +2022-04-12 08:37:28.285119: saving checkpoint... +2022-04-12 08:37:43.792852: done, saving took 15.56 seconds +2022-04-12 08:37:43.994915: done +2022-04-12 08:37:44.006380: This epoch took 137.461617 s + +2022-04-12 08:37:44.159996: svmem(total=99932160000, available=79766843392, percent=20.2, used=16144568320, free=47280930816, active=40401371136, inactive=10392432640, buffers=41111552, cached=36465549312, shared=2190729216, slab=784887808) +2022-04-12 08:37:44.172726: +epoch: 440 +2022-04-12 08:39:32.544471: train loss : -0.7230 +2022-04-12 08:39:44.129733: validation loss: -0.6901 +2022-04-12 08:39:44.225353: Average global foreground Dice: [0.9148, 0.9746, 0.9847, 0.4013] +2022-04-12 08:39:44.257129: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:39:45.435635: lr: 0.001461 +2022-04-12 08:39:45.465025: saving scheduled checkpoint file... +2022-04-12 08:39:45.565442: saving checkpoint... +2022-04-12 08:39:59.219440: done, saving took 13.73 seconds +2022-04-12 08:39:59.325891: done +2022-04-12 08:39:59.443572: saving checkpoint... +2022-04-12 08:40:16.866680: done, saving took 17.47 seconds +2022-04-12 08:40:17.161802: This epoch took 152.968208 s + +2022-04-12 08:40:17.276516: svmem(total=99932160000, available=80147730432, percent=19.8, used=15948685312, free=47556591616, active=40235393024, inactive=10279002112, buffers=41168896, cached=36385714176, shared=2005729280, slab=785203200) +2022-04-12 08:40:17.393573: +epoch: 441 +2022-04-12 08:42:05.166815: train loss : -0.7233 +2022-04-12 08:42:17.367778: validation loss: -0.6685 +2022-04-12 08:42:17.389918: Average global foreground Dice: [0.9273, 0.9755, 0.9826, 0.3674] +2022-04-12 08:42:17.422210: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:42:18.217542: lr: 0.001439 +2022-04-12 08:42:18.267280: saving scheduled checkpoint file... +2022-04-12 08:42:18.429892: saving checkpoint... +2022-04-12 08:42:35.589010: done, saving took 17.26 seconds +2022-04-12 08:42:36.041743: done +2022-04-12 08:42:36.103047: This epoch took 138.598187 s + +2022-04-12 08:42:36.252673: svmem(total=99932160000, available=80049741824, percent=19.9, used=15861923840, free=47441657856, active=40071430144, inactive=10559639552, buffers=41201664, cached=36587376640, shared=2190467072, slab=785137664) +2022-04-12 08:42:36.331067: +epoch: 442 +2022-04-12 08:44:25.591778: train loss : -0.7257 +2022-04-12 08:44:39.582342: validation loss: -0.7056 +2022-04-12 08:44:39.655904: Average global foreground Dice: [0.9326, 0.9748, 0.9843, 0.3951] +2022-04-12 08:44:39.672198: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:44:40.629745: lr: 0.001416 +2022-04-12 08:44:40.678826: saving scheduled checkpoint file... +2022-04-12 08:44:40.781516: saving checkpoint... +2022-04-12 08:44:51.266928: done, saving took 10.58 seconds +2022-04-12 08:44:51.403932: done +2022-04-12 08:44:51.519007: saving checkpoint... +2022-04-12 08:45:08.046058: done, saving took 16.63 seconds +2022-04-12 08:45:08.380508: This epoch took 151.975440 s + +2022-04-12 08:45:08.429617: svmem(total=99932160000, available=79922458624, percent=20.0, used=16055136256, free=47295877120, active=40298160128, inactive=10471583744, buffers=41259008, cached=36539887616, shared=2124423168, slab=786771968) +2022-04-12 08:45:08.472687: +epoch: 443 +2022-04-12 08:46:56.371021: train loss : -0.7354 +2022-04-12 08:47:08.050913: validation loss: -0.7013 +2022-04-12 08:47:08.070397: Average global foreground Dice: [0.9255, 0.9721, 0.9831, 0.3842] +2022-04-12 08:47:08.089706: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:47:09.298588: lr: 0.001394 +2022-04-12 08:47:09.360974: saving scheduled checkpoint file... +2022-04-12 08:47:09.469736: saving checkpoint... +2022-04-12 08:47:27.836253: done, saving took 18.46 seconds +2022-04-12 08:47:28.280470: done +2022-04-12 08:47:28.593950: This epoch took 140.064345 s + +2022-04-12 08:47:28.998663: svmem(total=99932160000, available=80183951360, percent=19.8, used=15860174848, free=47538860032, active=40154791936, inactive=10383945728, buffers=41295872, cached=36491829248, shared=2057834496, slab=786063360) +2022-04-12 08:47:29.355050: +epoch: 444 +2022-04-12 08:49:17.187819: train loss : -0.7389 +2022-04-12 08:49:27.996339: validation loss: -0.6970 +2022-04-12 08:49:28.118107: Average global foreground Dice: [0.9353, 0.9724, 0.9851, 0.3603] +2022-04-12 08:49:28.199029: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:49:29.267622: lr: 0.001372 +2022-04-12 08:49:29.325816: saving scheduled checkpoint file... +2022-04-12 08:49:29.482049: saving checkpoint... +2022-04-12 08:49:47.614193: done, saving took 18.22 seconds +2022-04-12 08:49:47.810483: done +2022-04-12 08:49:47.828161: This epoch took 138.254538 s + +2022-04-12 08:49:47.849990: svmem(total=99932160000, available=79656919040, percent=20.3, used=16254410752, free=47004041216, active=40552271872, inactive=10522120192, buffers=41328640, cached=36632379392, shared=2190729216, slab=784199680) +2022-04-12 08:49:47.911966: +epoch: 445 +2022-04-12 08:51:35.511397: train loss : -0.7367 +2022-04-12 08:51:46.541830: validation loss: -0.6839 +2022-04-12 08:51:46.627751: Average global foreground Dice: [0.9172, 0.9746, 0.986, 0.3931] +2022-04-12 08:51:46.761600: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:51:47.727566: lr: 0.001349 +2022-04-12 08:51:47.919466: saving scheduled checkpoint file... +2022-04-12 08:51:48.011412: saving checkpoint... +2022-04-12 08:52:03.774837: done, saving took 15.85 seconds +2022-04-12 08:52:04.004924: done +2022-04-12 08:52:04.051645: This epoch took 136.094650 s + +2022-04-12 08:52:04.055961: svmem(total=99932160000, available=79834402816, percent=20.1, used=16076652544, free=47164145664, active=40380637184, inactive=10531479552, buffers=41369600, cached=36649992192, shared=2190917632, slab=786161664) +2022-04-12 08:52:04.102163: +epoch: 446 +2022-04-12 08:53:53.631516: train loss : -0.7390 +2022-04-12 08:54:06.655726: validation loss: -0.6976 +2022-04-12 08:54:06.774283: Average global foreground Dice: [0.92, 0.9735, 0.9833, 0.38] +2022-04-12 08:54:06.963017: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:54:08.117269: lr: 0.001327 +2022-04-12 08:54:08.403259: saving scheduled checkpoint file... +2022-04-12 08:54:08.580970: saving checkpoint... +2022-04-12 08:54:24.585810: done, saving took 16.07 seconds +2022-04-12 08:54:24.759989: done +2022-04-12 08:54:24.814203: This epoch took 140.683693 s + +2022-04-12 08:54:24.854321: svmem(total=99932160000, available=79606857728, percent=20.3, used=16304635904, free=46953418752, active=40458436608, inactive=10657382400, buffers=41418752, cached=36632686592, shared=2190495744, slab=785653760) +2022-04-12 08:54:24.886887: +epoch: 447 +2022-04-12 08:56:12.693561: train loss : -0.7310 +2022-04-12 08:56:24.961493: validation loss: -0.6942 +2022-04-12 08:56:25.007762: Average global foreground Dice: [0.9248, 0.9747, 0.9827, 0.3764] +2022-04-12 08:56:25.056069: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:56:25.922059: lr: 0.001304 +2022-04-12 08:56:25.977344: saving scheduled checkpoint file... +2022-04-12 08:56:26.109830: saving checkpoint... +2022-04-12 08:56:43.771985: done, saving took 17.76 seconds +2022-04-12 08:56:44.238155: done +2022-04-12 08:56:44.307124: This epoch took 139.368330 s + +2022-04-12 08:56:44.365613: svmem(total=99932160000, available=79655800832, percent=20.3, used=16255901696, free=46949015552, active=40487780352, inactive=10633703424, buffers=41467904, cached=36685774848, shared=2190295040, slab=785420288) +2022-04-12 08:56:44.370378: +epoch: 448 +2022-04-12 08:58:32.217422: train loss : -0.7303 +2022-04-12 08:58:44.023277: validation loss: -0.7076 +2022-04-12 08:58:44.411528: Average global foreground Dice: [0.9353, 0.9761, 0.9843, 0.3869] +2022-04-12 08:58:44.488593: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 08:58:46.014128: lr: 0.001282 +2022-04-12 08:58:46.600569: saving scheduled checkpoint file... +2022-04-12 08:58:47.315852: saving checkpoint... +2022-04-12 08:59:06.118603: done, saving took 18.92 seconds +2022-04-12 08:59:06.489376: done +2022-04-12 08:59:06.543798: This epoch took 142.101217 s + +2022-04-12 08:59:06.608043: svmem(total=99932160000, available=79644143616, percent=20.3, used=16267010048, free=46929604608, active=40530489344, inactive=10610827264, buffers=41500672, cached=36694044672, shared=2190729216, slab=785006592) +2022-04-12 08:59:06.666579: +epoch: 449 +2022-04-12 09:00:56.878918: train loss : -0.7343 +2022-04-12 09:01:08.944535: validation loss: -0.6889 +2022-04-12 09:01:11.908110: Average global foreground Dice: [0.9284, 0.9709, 0.9854, 0.3734] +2022-04-12 09:01:12.245916: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:01:13.204132: lr: 0.001259 +2022-04-12 09:01:13.313251: saving scheduled checkpoint file... +2022-04-12 09:01:13.561228: saving checkpoint... +2022-04-12 09:01:28.800318: done, saving took 15.33 seconds +2022-04-12 09:01:29.007992: done +2022-04-12 09:01:29.100818: This epoch took 142.333442 s + +2022-04-12 09:01:29.143988: svmem(total=99932160000, available=79604785152, percent=20.3, used=16306941952, free=46871601152, active=40597422080, inactive=10599661568, buffers=41558016, cached=36712058880, shared=2190045184, slab=787308544) +2022-04-12 09:01:29.199944: +epoch: 450 +2022-04-12 09:03:18.164846: train loss : -0.7256 +2022-04-12 09:03:30.844413: validation loss: -0.7042 +2022-04-12 09:03:30.879326: Average global foreground Dice: [0.9219, 0.9769, 0.9843, 0.3835] +2022-04-12 09:03:30.926325: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:03:32.055070: lr: 0.001236 +2022-04-12 09:03:32.141151: saving scheduled checkpoint file... +2022-04-12 09:03:32.418767: saving checkpoint... +2022-04-12 09:03:50.183772: done, saving took 17.84 seconds +2022-04-12 09:03:50.351383: done +2022-04-12 09:03:50.369063: This epoch took 141.136461 s + +2022-04-12 09:03:50.405946: svmem(total=99932160000, available=79624728576, percent=20.3, used=16286236672, free=46874071040, active=40578408448, inactive=10615504896, buffers=41590784, cached=36730261504, shared=2190729216, slab=785924096) +2022-04-12 09:03:50.468088: +epoch: 451 +2022-04-12 09:05:38.770387: train loss : -0.7318 +2022-04-12 09:05:53.652678: validation loss: -0.7101 +2022-04-12 09:05:54.206403: Average global foreground Dice: [0.9293, 0.9749, 0.9848, 0.3795] +2022-04-12 09:05:54.772872: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:05:56.492234: lr: 0.001214 +2022-04-12 09:05:56.624809: saving scheduled checkpoint file... +2022-04-12 09:05:56.903767: saving checkpoint... +2022-04-12 09:06:13.392519: done, saving took 16.58 seconds +2022-04-12 09:06:13.874577: done +2022-04-12 09:06:13.937155: This epoch took 143.442773 s + +2022-04-12 09:06:13.986832: svmem(total=99932160000, available=79864881152, percent=20.1, used=16045858816, free=47095656448, active=40339804160, inactive=10629189632, buffers=41668608, cached=36748976128, shared=2190917632, slab=787374080) +2022-04-12 09:06:14.094108: +epoch: 452 +2022-04-12 09:08:02.222007: train loss : -0.7328 +2022-04-12 09:08:15.074170: validation loss: -0.6969 +2022-04-12 09:08:15.128278: Average global foreground Dice: [0.9231, 0.9729, 0.9852, 0.3775] +2022-04-12 09:08:15.324290: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:08:16.750391: lr: 0.001191 +2022-04-12 09:08:17.290938: saving scheduled checkpoint file... +2022-04-12 09:08:17.823234: saving checkpoint... +2022-04-12 09:08:40.625390: done, saving took 22.88 seconds +2022-04-12 09:08:40.875230: done +2022-04-12 09:08:41.001198: This epoch took 146.843380 s + +2022-04-12 09:08:41.069377: svmem(total=99932160000, available=79927300096, percent=20.0, used=15973298176, free=47149150208, active=40274251776, inactive=10648158208, buffers=41713664, cached=36767997952, shared=2201026560, slab=788230144) +2022-04-12 09:08:41.139261: +epoch: 453 +2022-04-12 09:10:31.334137: train loss : -0.7412 +2022-04-12 09:10:43.721759: validation loss: -0.6987 +2022-04-12 09:10:43.796532: Average global foreground Dice: [0.923, 0.9738, 0.983, 0.3807] +2022-04-12 09:10:43.939599: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:10:45.009248: lr: 0.001168 +2022-04-12 09:10:45.201491: saving scheduled checkpoint file... +2022-04-12 09:10:45.385512: saving checkpoint... +2022-04-12 09:11:11.554115: done, saving took 26.28 seconds +2022-04-12 09:11:11.828978: done +2022-04-12 09:11:11.874404: This epoch took 150.687236 s + +2022-04-12 09:11:11.937460: svmem(total=99932160000, available=79703134208, percent=20.2, used=16221810688, free=46945296384, active=40495210496, inactive=10618822656, buffers=41771008, cached=36723281920, shared=2176741376, slab=789041152) +2022-04-12 09:11:11.982329: +epoch: 454 +2022-04-12 09:13:01.737118: train loss : -0.7402 +2022-04-12 09:13:13.715553: validation loss: -0.6927 +2022-04-12 09:13:13.802878: Average global foreground Dice: [0.9249, 0.975, 0.9802, 0.3726] +2022-04-12 09:13:13.863816: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:13:15.019901: lr: 0.001145 +2022-04-12 09:13:15.086556: saving scheduled checkpoint file... +2022-04-12 09:13:15.278973: saving checkpoint... +2022-04-12 09:13:32.164489: done, saving took 17.01 seconds +2022-04-12 09:13:32.475356: done +2022-04-12 09:13:32.508482: This epoch took 140.477126 s + +2022-04-12 09:13:32.595265: svmem(total=99932160000, available=80252813312, percent=19.7, used=15825522688, free=47476449280, active=40113340416, inactive=10483851264, buffers=41803776, cached=36588384256, shared=2023305216, slab=789000192) +2022-04-12 09:13:32.647486: +epoch: 455 +2022-04-12 09:15:22.784153: train loss : -0.7373 +2022-04-12 09:15:34.766992: validation loss: -0.7032 +2022-04-12 09:15:34.838905: Average global foreground Dice: [0.9142, 0.9763, 0.9814, 0.4009] +2022-04-12 09:15:34.976323: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:15:36.063506: lr: 0.001122 +2022-04-12 09:15:36.084946: saving scheduled checkpoint file... +2022-04-12 09:15:36.224473: saving checkpoint... +2022-04-12 09:15:45.517072: done, saving took 9.41 seconds +2022-04-12 09:15:45.662070: done +2022-04-12 09:15:45.738317: This epoch took 133.087674 s + +2022-04-12 09:15:45.773900: svmem(total=99932160000, available=80170577920, percent=19.8, used=15879831552, free=47376789504, active=40161255424, inactive=10528669696, buffers=41852928, cached=36633686016, shared=2051280896, slab=787484672) +2022-04-12 09:15:45.817103: +epoch: 456 +2022-04-12 09:17:35.069255: train loss : -0.7421 +2022-04-12 09:17:46.857586: validation loss: -0.7011 +2022-04-12 09:17:46.955061: Average global foreground Dice: [0.9249, 0.9737, 0.982, 0.3798] +2022-04-12 09:17:47.002804: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:17:47.975049: lr: 0.001099 +2022-04-12 09:17:47.992807: saving scheduled checkpoint file... +2022-04-12 09:17:48.090048: saving checkpoint... +2022-04-12 09:18:00.670096: done, saving took 12.66 seconds +2022-04-12 09:18:00.976991: done +2022-04-12 09:18:01.021276: This epoch took 135.201528 s + +2022-04-12 09:18:01.162923: svmem(total=99932160000, available=80193335296, percent=19.8, used=15902502912, free=47391772672, active=40186097664, inactive=10491543552, buffers=41889792, cached=36595994624, shared=2005823488, slab=787763200) +2022-04-12 09:18:01.208427: +epoch: 457 +2022-04-12 09:19:49.343558: train loss : -0.7367 +2022-04-12 09:20:01.202070: validation loss: -0.7051 +2022-04-12 09:20:01.318398: Average global foreground Dice: [0.9356, 0.973, 0.984, 0.389] +2022-04-12 09:20:01.380131: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:20:02.531375: lr: 0.001076 +2022-04-12 09:20:02.697212: saving scheduled checkpoint file... +2022-04-12 09:20:02.831809: saving checkpoint... +2022-04-12 09:20:13.936018: done, saving took 11.19 seconds +2022-04-12 09:20:14.257618: done +2022-04-12 09:20:14.369001: This epoch took 133.053035 s + +2022-04-12 09:20:14.439727: svmem(total=99932160000, available=79806238720, percent=20.1, used=16094089216, free=46987104256, active=40387305472, inactive=10688815104, buffers=41947136, cached=36809019392, shared=2201214976, slab=786640896) +2022-04-12 09:20:14.520720: +epoch: 458 +2022-04-12 09:22:03.039557: train loss : -0.7428 +2022-04-12 09:22:14.521456: validation loss: -0.6888 +2022-04-12 09:22:14.601614: Average global foreground Dice: [0.9167, 0.975, 0.9848, 0.3713] +2022-04-12 09:22:14.619215: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:22:15.491307: lr: 0.001053 +2022-04-12 09:22:15.532635: saving scheduled checkpoint file... +2022-04-12 09:22:15.652465: saving checkpoint... +2022-04-12 09:22:28.899994: done, saving took 13.34 seconds +2022-04-12 09:22:29.098384: done +2022-04-12 09:22:29.170917: This epoch took 134.645924 s + +2022-04-12 09:22:29.197723: svmem(total=99932160000, available=79845957632, percent=20.1, used=16064835584, free=47008272384, active=40366436352, inactive=10696798208, buffers=41992192, cached=36817059840, shared=2190774272, slab=787574784) +2022-04-12 09:22:29.238900: +epoch: 459 +2022-04-12 09:24:18.127096: train loss : -0.7367 +2022-04-12 09:24:29.825262: validation loss: -0.7006 +2022-04-12 09:24:29.867113: Average global foreground Dice: [0.9271, 0.9732, 0.9845, 0.3703] +2022-04-12 09:24:29.923032: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:24:30.666177: lr: 0.00103 +2022-04-12 09:24:30.705104: saving scheduled checkpoint file... +2022-04-12 09:24:30.856155: saving checkpoint... +2022-04-12 09:24:45.177881: done, saving took 14.44 seconds +2022-04-12 09:24:45.295657: done +2022-04-12 09:24:45.363717: This epoch took 136.082098 s + +2022-04-12 09:24:45.410944: svmem(total=99932160000, available=79575326720, percent=20.4, used=16331567104, free=46720196608, active=40630525952, inactive=10718064640, buffers=42024960, cached=36838371328, shared=2194583552, slab=787484672) +2022-04-12 09:24:45.425780: +epoch: 460 +2022-04-12 09:26:33.848852: train loss : -0.7360 +2022-04-12 09:26:45.152451: validation loss: -0.6852 +2022-04-12 09:26:45.215055: Average global foreground Dice: [0.9346, 0.9716, 0.9821, 0.3816] +2022-04-12 09:26:45.301639: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:26:46.252449: lr: 0.001007 +2022-04-12 09:26:46.327096: saving scheduled checkpoint file... +2022-04-12 09:26:46.510577: saving checkpoint... +2022-04-12 09:26:58.231124: done, saving took 11.84 seconds +2022-04-12 09:26:58.534515: done +2022-04-12 09:26:58.606399: This epoch took 133.119119 s + +2022-04-12 09:26:58.666350: svmem(total=99932160000, available=79735185408, percent=20.2, used=16165371904, free=46872227840, active=40467996672, inactive=10731790336, buffers=42074112, cached=36852486144, shared=2200948736, slab=786575360) +2022-04-12 09:26:58.736692: +epoch: 461 +2022-04-12 09:28:46.838830: train loss : -0.7487 +2022-04-12 09:28:59.328028: validation loss: -0.6982 +2022-04-12 09:28:59.418627: Average global foreground Dice: [0.9287, 0.9781, 0.9855, 0.3955] +2022-04-12 09:28:59.463103: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:29:00.479142: lr: 0.000983 +2022-04-12 09:29:00.527284: saving scheduled checkpoint file... +2022-04-12 09:29:00.644387: saving checkpoint... +2022-04-12 09:29:19.548906: done, saving took 18.99 seconds +2022-04-12 09:29:19.742174: done +2022-04-12 09:29:19.916965: saving checkpoint... +2022-04-12 09:29:30.829734: done, saving took 10.98 seconds +2022-04-12 09:29:31.022625: This epoch took 152.278128 s + +2022-04-12 09:29:31.087177: svmem(total=99932160000, available=79635709952, percent=20.3, used=16273547264, free=46872899584, active=40265940992, inactive=10928218112, buffers=42123264, cached=36743589888, shared=2192105472, slab=788004864) +2022-04-12 09:29:31.140914: +epoch: 462 +2022-04-12 09:31:18.815420: train loss : -0.7310 +2022-04-12 09:31:31.513028: validation loss: -0.6962 +2022-04-12 09:31:31.689474: Average global foreground Dice: [0.9222, 0.9756, 0.9842, 0.3821] +2022-04-12 09:31:31.762525: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:31:32.731553: lr: 0.00096 +2022-04-12 09:31:32.945818: saving scheduled checkpoint file... +2022-04-12 09:31:33.197189: saving checkpoint... +2022-04-12 09:32:07.882769: done, saving took 34.78 seconds +2022-04-12 09:32:10.686741: done +2022-04-12 09:32:11.203780: This epoch took 160.000344 s + +2022-04-12 09:32:11.499561: svmem(total=99932160000, available=79817015296, percent=20.1, used=16108294144, free=47034597376, active=40300007424, inactive=10726395904, buffers=42180608, cached=36747087872, shared=2176176128, slab=787709952) +2022-04-12 09:32:11.779551: +epoch: 463 +2022-04-12 09:34:00.046601: train loss : -0.7366 +2022-04-12 09:34:10.779737: validation loss: -0.7029 +2022-04-12 09:34:10.849194: Average global foreground Dice: [0.9347, 0.9764, 0.986, 0.3958] +2022-04-12 09:34:10.870184: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:34:12.020931: lr: 0.000937 +2022-04-12 09:34:12.062848: saving scheduled checkpoint file... +2022-04-12 09:34:12.208583: saving checkpoint... +2022-04-12 09:34:29.134781: done, saving took 17.02 seconds +2022-04-12 09:34:29.351314: done +2022-04-12 09:34:29.480477: saving checkpoint... +2022-04-12 09:34:56.257911: done, saving took 26.83 seconds +2022-04-12 09:34:56.649997: This epoch took 164.687212 s + +2022-04-12 09:34:56.777584: svmem(total=99932160000, available=79757881344, percent=20.2, used=16166768640, free=46955786240, active=40414130176, inactive=10700423168, buffers=42213376, cached=36767391744, shared=2176741376, slab=789680128) +2022-04-12 09:34:56.904476: +epoch: 464 +2022-04-12 09:36:44.750952: train loss : -0.7423 +2022-04-12 09:36:56.394723: validation loss: -0.6669 +2022-04-12 09:36:56.485796: Average global foreground Dice: [0.9278, 0.9664, 0.9787, 0.3554] +2022-04-12 09:36:56.532600: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:36:57.422000: lr: 0.000913 +2022-04-12 09:36:57.449628: saving scheduled checkpoint file... +2022-04-12 09:36:57.517416: saving checkpoint... +2022-04-12 09:37:07.224658: done, saving took 9.76 seconds +2022-04-12 09:37:08.121897: done +2022-04-12 09:37:08.612024: This epoch took 131.629586 s + +2022-04-12 09:37:09.091739: svmem(total=99932160000, available=79773569024, percent=20.2, used=16151105536, free=46954004480, active=40418418688, inactive=10673221632, buffers=42278912, cached=36784771072, shared=2176741376, slab=802668544) +2022-04-12 09:37:09.533051: +epoch: 465 +2022-04-12 09:38:58.080812: train loss : -0.7432 +2022-04-12 09:39:09.694119: validation loss: -0.6988 +2022-04-12 09:39:09.775683: Average global foreground Dice: [0.934, 0.9767, 0.9829, 0.3715] +2022-04-12 09:39:09.782828: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:39:10.720940: lr: 0.00089 +2022-04-12 09:39:10.781856: saving scheduled checkpoint file... +2022-04-12 09:39:10.895436: saving checkpoint... +2022-04-12 09:39:22.823517: done, saving took 12.02 seconds +2022-04-12 09:39:24.619161: done +2022-04-12 09:39:25.253174: This epoch took 135.412664 s + +2022-04-12 09:39:26.401649: svmem(total=99932160000, available=79665508352, percent=20.3, used=16259100672, free=46869479424, active=40366854144, inactive=10818723840, buffers=42328064, cached=36761251840, shared=2176741376, slab=802230272) +2022-04-12 09:39:26.995077: +epoch: 466 +2022-04-12 09:41:15.732429: train loss : -0.7392 +2022-04-12 09:41:27.645167: validation loss: -0.7218 +2022-04-12 09:41:27.737601: Average global foreground Dice: [0.9355, 0.9783, 0.9866, 0.4073] +2022-04-12 09:41:27.825854: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:41:28.791775: lr: 0.000866 +2022-04-12 09:41:28.961257: saving scheduled checkpoint file... +2022-04-12 09:41:29.338713: saving checkpoint... +2022-04-12 09:41:48.269645: done, saving took 19.03 seconds +2022-04-12 09:41:49.840554: done +2022-04-12 09:41:50.348505: saving checkpoint... +2022-04-12 09:42:14.150907: done, saving took 23.91 seconds +2022-04-12 09:42:14.267843: This epoch took 166.517461 s + +2022-04-12 09:42:14.275953: svmem(total=99932160000, available=79967821824, percent=20.0, used=15941844992, free=47122300928, active=40167419904, inactive=10773168128, buffers=42385408, cached=36825628672, shared=2191609856, slab=788967424) +2022-04-12 09:42:14.302233: +epoch: 467 +2022-04-12 09:44:02.938777: train loss : -0.7351 +2022-04-12 09:44:15.312260: validation loss: -0.6737 +2022-04-12 09:44:15.412504: Average global foreground Dice: [0.9266, 0.9768, 0.9788, 0.3569] +2022-04-12 09:44:15.491070: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:44:16.657538: lr: 0.000842 +2022-04-12 09:44:17.135950: saving scheduled checkpoint file... +2022-04-12 09:44:17.550216: saving checkpoint... +2022-04-12 09:44:34.999296: done, saving took 17.53 seconds +2022-04-12 09:44:35.617982: done +2022-04-12 09:44:35.848272: This epoch took 141.519905 s + +2022-04-12 09:44:36.197208: svmem(total=99932160000, available=79963402240, percent=20.0, used=15947333632, free=47103684608, active=40146325504, inactive=10812755968, buffers=42418176, cached=36838723584, shared=2190540800, slab=788414464) +2022-04-12 09:44:36.409645: +epoch: 468 +2022-04-12 09:46:25.241096: train loss : -0.7442 +2022-04-12 09:46:36.933931: validation loss: -0.6882 +2022-04-12 09:46:37.008800: Average global foreground Dice: [0.9276, 0.9802, 0.9824, 0.3996] +2022-04-12 09:46:37.126180: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:46:38.772680: lr: 0.000819 +2022-04-12 09:46:39.554835: saving scheduled checkpoint file... +2022-04-12 09:46:40.286172: saving checkpoint... +2022-04-12 09:47:00.013381: done, saving took 19.81 seconds +2022-04-12 09:47:00.127767: done +2022-04-12 09:47:00.186604: This epoch took 143.515126 s + +2022-04-12 09:47:00.210307: svmem(total=99932160000, available=79858319360, percent=20.1, used=16052203520, free=46978613248, active=40303280128, inactive=10785517568, buffers=42463232, cached=36858880000, shared=2190807040, slab=789258240) +2022-04-12 09:47:00.267323: +epoch: 469 +2022-04-12 09:48:49.002424: train loss : -0.7463 +2022-04-12 09:49:01.025689: validation loss: -0.7087 +2022-04-12 09:49:01.184900: Average global foreground Dice: [0.9347, 0.9769, 0.9834, 0.3834] +2022-04-12 09:49:01.246450: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:49:02.255219: lr: 0.000795 +2022-04-12 09:49:02.308107: saving scheduled checkpoint file... +2022-04-12 09:49:02.401804: saving checkpoint... +2022-04-12 09:49:19.151637: done, saving took 16.80 seconds +2022-04-12 09:49:21.454796: done +2022-04-12 09:49:22.013386: saving checkpoint... +2022-04-12 09:49:35.632347: done, saving took 13.69 seconds +2022-04-12 09:49:35.787204: This epoch took 155.463608 s + +2022-04-12 09:49:35.841282: svmem(total=99932160000, available=79763218432, percent=20.2, used=16138293248, free=46862151680, active=40411500544, inactive=10788147200, buffers=42512384, cached=36889202688, shared=2199728128, slab=788586496) +2022-04-12 09:49:35.881344: +epoch: 470 +2022-04-12 09:51:23.611933: train loss : -0.7381 +2022-04-12 09:51:36.346544: validation loss: -0.6899 +2022-04-12 09:51:37.744406: Average global foreground Dice: [0.9221, 0.9753, 0.9849, 0.3927] +2022-04-12 09:51:38.320871: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:51:40.810279: lr: 0.000771 +2022-04-12 09:51:41.790398: saving scheduled checkpoint file... +2022-04-12 09:51:42.879165: saving checkpoint... +2022-04-12 09:51:55.929958: done, saving took 13.14 seconds +2022-04-12 09:51:56.111335: done +2022-04-12 09:51:56.274046: saving checkpoint... +2022-04-12 09:52:10.317871: done, saving took 14.09 seconds +2022-04-12 09:52:10.491709: This epoch took 154.544974 s + +2022-04-12 09:52:10.515835: svmem(total=99932160000, available=79782965248, percent=20.2, used=16127422464, free=46872997888, active=40407896064, inactive=10780348416, buffers=42569728, cached=36889169920, shared=2190729216, slab=788848640) +2022-04-12 09:52:10.558984: +epoch: 471 +2022-04-12 09:54:00.172946: train loss : -0.7330 +2022-04-12 09:54:12.167620: validation loss: -0.7060 +2022-04-12 09:54:12.211652: Average global foreground Dice: [0.9281, 0.9749, 0.9832, 0.4112] +2022-04-12 09:54:12.247219: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:54:13.344007: lr: 0.000747 +2022-04-12 09:54:13.376296: saving scheduled checkpoint file... +2022-04-12 09:54:13.521815: saving checkpoint... +2022-04-12 09:54:29.171783: done, saving took 15.76 seconds +2022-04-12 09:54:30.595712: done +2022-04-12 09:54:30.767685: saving checkpoint... +2022-04-12 09:54:42.564461: done, saving took 11.89 seconds +2022-04-12 09:54:42.852648: This epoch took 152.243882 s + +2022-04-12 09:54:42.888262: svmem(total=99932160000, available=79788335104, percent=20.2, used=16122052608, free=46858596352, active=40413302784, inactive=10797715456, buffers=42602496, cached=36908908544, shared=2190729216, slab=789102592) +2022-04-12 09:54:42.943046: +epoch: 472 +2022-04-12 09:56:31.605457: train loss : -0.7369 +2022-04-12 09:56:43.256921: validation loss: -0.7049 +2022-04-12 09:56:43.293949: Average global foreground Dice: [0.9312, 0.9739, 0.9846, 0.3902] +2022-04-12 09:56:43.343593: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:56:44.970546: lr: 0.000723 +2022-04-12 09:56:45.127042: saving scheduled checkpoint file... +2022-04-12 09:56:45.303110: saving checkpoint... +2022-04-12 09:57:03.217715: done, saving took 18.03 seconds +2022-04-12 09:57:03.511453: done +2022-04-12 09:57:03.733329: saving checkpoint... +2022-04-12 09:57:12.362587: done, saving took 8.76 seconds +2022-04-12 09:57:12.438269: This epoch took 149.490937 s + +2022-04-12 09:57:12.455076: svmem(total=99932160000, available=79612391424, percent=20.3, used=16297578496, free=46664175616, active=40587829248, inactive=10813693952, buffers=42663936, cached=36927741952, shared=2191114240, slab=787992576) +2022-04-12 09:57:12.537010: +epoch: 473 +2022-04-12 09:59:04.254466: train loss : -0.7465 +2022-04-12 09:59:16.441106: validation loss: -0.7030 +2022-04-12 09:59:16.534888: Average global foreground Dice: [0.9396, 0.976, 0.9829, 0.3694] +2022-04-12 09:59:16.601599: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 09:59:17.539724: lr: 0.000699 +2022-04-12 09:59:17.564472: saving scheduled checkpoint file... +2022-04-12 09:59:17.663966: saving checkpoint... +2022-04-12 09:59:27.788257: done, saving took 10.20 seconds +2022-04-12 09:59:28.314722: done +2022-04-12 09:59:28.524835: This epoch took 135.919983 s + +2022-04-12 09:59:28.693392: svmem(total=99932160000, available=79782563840, percent=20.2, used=16117391360, free=46816800768, active=40416653312, inactive=10840985600, buffers=42696704, cached=36955271168, shared=2201100288, slab=787312640) +2022-04-12 09:59:28.906220: +epoch: 474 +2022-04-12 10:01:17.678920: train loss : -0.7332 +2022-04-12 10:01:29.844708: validation loss: -0.7062 +2022-04-12 10:01:29.981261: Average global foreground Dice: [0.9308, 0.9754, 0.9839, 0.3925] +2022-04-12 10:01:30.313499: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:01:31.391450: lr: 0.000675 +2022-04-12 10:01:31.609329: saving scheduled checkpoint file... +2022-04-12 10:01:31.778847: saving checkpoint... +2022-04-12 10:01:47.649961: done, saving took 15.96 seconds +2022-04-12 10:01:47.894620: done +2022-04-12 10:01:47.954735: saving checkpoint... +2022-04-12 10:02:00.139400: done, saving took 12.24 seconds +2022-04-12 10:02:00.316693: This epoch took 151.128267 s + +2022-04-12 10:02:00.345130: svmem(total=99932160000, available=79716102144, percent=20.2, used=16193576960, free=46575128576, active=40487665664, inactive=11005816832, buffers=42737664, cached=37120716800, shared=2191224832, slab=789102592) +2022-04-12 10:02:00.387473: +epoch: 475 +2022-04-12 10:03:47.981184: train loss : -0.7463 +2022-04-12 10:03:58.607123: validation loss: -0.6950 +2022-04-12 10:03:58.847695: Average global foreground Dice: [0.9309, 0.9756, 0.985, 0.3937] +2022-04-12 10:03:58.871181: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:04:00.090149: lr: 0.00065 +2022-04-12 10:04:00.188896: saving scheduled checkpoint file... +2022-04-12 10:04:00.345877: saving checkpoint... +2022-04-12 10:04:12.804647: done, saving took 12.51 seconds +2022-04-12 10:04:12.926641: done +2022-04-12 10:04:13.035403: saving checkpoint... +2022-04-12 10:04:22.941888: done, saving took 9.95 seconds +2022-04-12 10:04:23.205093: This epoch took 142.759850 s + +2022-04-12 10:04:23.329469: svmem(total=99932160000, available=79897329664, percent=20.0, used=16013111296, free=46748704768, active=40286932992, inactive=11031298048, buffers=42786816, cached=37127557120, shared=2190422016, slab=789204992) +2022-04-12 10:04:23.447854: +epoch: 476 +2022-04-12 10:06:10.610745: train loss : -0.7585 +2022-04-12 10:06:21.314281: validation loss: -0.6994 +2022-04-12 10:06:21.395331: Average global foreground Dice: [0.9244, 0.9749, 0.985, 0.3824] +2022-04-12 10:06:21.454602: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:06:22.664136: lr: 0.000626 +2022-04-12 10:06:22.765563: saving scheduled checkpoint file... +2022-04-12 10:06:22.915753: saving checkpoint... +2022-04-12 10:06:40.404008: done, saving took 17.57 seconds +2022-04-12 10:06:40.747710: done +2022-04-12 10:06:40.836607: This epoch took 137.329355 s + +2022-04-12 10:06:40.898573: svmem(total=99932160000, available=79789105152, percent=20.2, used=16120922112, free=46628044800, active=40183861248, inactive=11261693952, buffers=42844160, cached=37140348928, shared=2190839808, slab=788525056) +2022-04-12 10:06:40.955886: +epoch: 477 +2022-04-12 10:08:28.591850: train loss : -0.7509 +2022-04-12 10:08:39.352434: validation loss: -0.7137 +2022-04-12 10:08:39.417345: Average global foreground Dice: [0.9367, 0.9763, 0.9874, 0.4143] +2022-04-12 10:08:39.567739: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:08:40.792587: lr: 0.000601 +2022-04-12 10:08:40.918940: saving scheduled checkpoint file... +2022-04-12 10:08:41.095564: saving checkpoint... +2022-04-12 10:08:52.791215: done, saving took 11.75 seconds +2022-04-12 10:08:53.257906: done +2022-04-12 10:08:53.446008: saving checkpoint... +2022-04-12 10:09:05.654671: done, saving took 12.29 seconds +2022-04-12 10:09:05.901064: This epoch took 144.931513 s + +2022-04-12 10:09:06.016277: svmem(total=99932160000, available=79718887424, percent=20.2, used=16180867072, free=46534823936, active=40401911808, inactive=11132645376, buffers=42885120, cached=37173583872, shared=2201137152, slab=789315584) +2022-04-12 10:09:06.033437: +epoch: 478 +2022-04-12 10:10:54.002884: train loss : -0.7488 +2022-04-12 10:11:06.075560: validation loss: -0.7093 +2022-04-12 10:11:06.174824: Average global foreground Dice: [0.9278, 0.9785, 0.987, 0.405] +2022-04-12 10:11:06.235771: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:11:07.344220: lr: 0.000577 +2022-04-12 10:11:07.468983: saving scheduled checkpoint file... +2022-04-12 10:11:07.645612: saving checkpoint... +2022-04-12 10:11:18.149521: done, saving took 10.61 seconds +2022-04-12 10:11:18.320412: done +2022-04-12 10:11:18.472951: saving checkpoint... +2022-04-12 10:11:31.432052: done, saving took 13.07 seconds +2022-04-12 10:11:31.611174: This epoch took 145.476477 s + +2022-04-12 10:11:31.629018: svmem(total=99932160000, available=79561175040, percent=20.4, used=16349351936, free=46464892928, active=40325607424, inactive=11279650816, buffers=42942464, cached=37074972672, shared=2190278656, slab=788467712) +2022-04-12 10:11:31.673451: +epoch: 479 +2022-04-12 10:13:19.855267: train loss : -0.7506 +2022-04-12 10:13:32.474228: validation loss: -0.7019 +2022-04-12 10:13:32.664232: Average global foreground Dice: [0.9309, 0.9758, 0.9819, 0.3838] +2022-04-12 10:13:32.908275: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:13:33.890926: lr: 0.000552 +2022-04-12 10:13:33.999448: saving scheduled checkpoint file... +2022-04-12 10:13:34.186428: saving checkpoint... +2022-04-12 10:13:56.460432: done, saving took 22.35 seconds +2022-04-12 10:13:56.913476: done +2022-04-12 10:13:57.019855: This epoch took 145.290813 s + +2022-04-12 10:13:57.266582: svmem(total=99932160000, available=79723614208, percent=20.2, used=16186474496, free=46619406336, active=40302804992, inactive=11146043392, buffers=42975232, cached=37083303936, shared=2190610432, slab=788340736) +2022-04-12 10:13:57.447622: +epoch: 480 +2022-04-12 10:15:46.337196: train loss : -0.7556 +2022-04-12 10:15:58.314723: validation loss: -0.7116 +2022-04-12 10:15:58.343890: Average global foreground Dice: [0.9343, 0.9754, 0.9827, 0.4157] +2022-04-12 10:15:58.366200: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:15:58.994522: lr: 0.000527 +2022-04-12 10:15:59.044405: saving scheduled checkpoint file... +2022-04-12 10:15:59.159997: saving checkpoint... +2022-04-12 10:16:10.234428: done, saving took 11.18 seconds +2022-04-12 10:16:10.365233: done +2022-04-12 10:16:10.430809: saving checkpoint... +2022-04-12 10:16:27.131238: done, saving took 16.75 seconds +2022-04-12 10:16:27.311283: This epoch took 149.794433 s + +2022-04-12 10:16:27.372758: svmem(total=99932160000, available=79682625536, percent=20.3, used=16217133056, free=46558928896, active=40422739968, inactive=11084107776, buffers=43036672, cached=37113061376, shared=2200948736, slab=788766720) +2022-04-12 10:16:27.441834: +epoch: 481 +2022-04-12 10:18:16.190065: train loss : -0.7555 +2022-04-12 10:18:28.392904: validation loss: -0.7117 +2022-04-12 10:18:28.440024: Average global foreground Dice: [0.9165, 0.9762, 0.9849, 0.4055] +2022-04-12 10:18:28.504277: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:18:29.458308: lr: 0.000502 +2022-04-12 10:18:29.516051: saving scheduled checkpoint file... +2022-04-12 10:18:29.627857: saving checkpoint... +2022-04-12 10:18:42.148544: done, saving took 12.57 seconds +2022-04-12 10:18:42.362233: done +2022-04-12 10:18:42.472661: saving checkpoint... +2022-04-12 10:18:57.745667: done, saving took 15.32 seconds +2022-04-12 10:18:58.018089: This epoch took 150.475963 s + +2022-04-12 10:18:58.071466: svmem(total=99932160000, available=79625191424, percent=20.3, used=16284811264, free=46482857984, active=40531656704, inactive=11050029056, buffers=43077632, cached=37121413120, shared=2190729216, slab=788590592) +2022-04-12 10:18:58.179750: +epoch: 482 +2022-04-12 10:20:46.790896: train loss : -0.7481 +2022-04-12 10:20:58.612752: validation loss: -0.6820 +2022-04-12 10:20:58.686007: Average global foreground Dice: [0.9313, 0.9749, 0.9827, 0.3963] +2022-04-12 10:20:58.749452: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:20:59.925689: lr: 0.000477 +2022-04-12 10:21:00.018955: saving scheduled checkpoint file... +2022-04-12 10:21:00.270334: saving checkpoint... +2022-04-12 10:21:13.883465: done, saving took 13.70 seconds +2022-04-12 10:21:14.074531: done +2022-04-12 10:21:14.128524: saving checkpoint... +2022-04-12 10:21:27.474963: done, saving took 13.40 seconds +2022-04-12 10:21:27.681296: This epoch took 149.442336 s + +2022-04-12 10:21:27.739522: svmem(total=99932160000, available=79701311488, percent=20.2, used=16208670720, free=46550360064, active=40404951040, inactive=11107225600, buffers=43134976, cached=37129994240, shared=2190618624, slab=790392832) +2022-04-12 10:21:27.800563: +epoch: 483 +2022-04-12 10:23:16.439002: train loss : -0.7454 +2022-04-12 10:23:27.468963: validation loss: -0.6970 +2022-04-12 10:23:27.566233: Average global foreground Dice: [0.9261, 0.9791, 0.9828, 0.4068] +2022-04-12 10:23:27.636442: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:23:28.642317: lr: 0.000451 +2022-04-12 10:23:28.762566: saving scheduled checkpoint file... +2022-04-12 10:23:28.827606: saving checkpoint... +2022-04-12 10:23:40.811262: done, saving took 12.04 seconds +2022-04-12 10:23:41.057934: done +2022-04-12 10:23:41.277287: saving checkpoint... +2022-04-12 10:24:00.020434: done, saving took 18.83 seconds +2022-04-12 10:24:00.467525: This epoch took 152.578160 s + +2022-04-12 10:24:00.614884: svmem(total=99932160000, available=80050376704, percent=19.9, used=15932489728, free=46880866304, active=40208953344, inactive=10980872192, buffers=43175936, cached=37075628032, shared=2117795840, slab=788873216) +2022-04-12 10:24:00.726238: +epoch: 484 +2022-04-12 10:25:48.353858: train loss : -0.7436 +2022-04-12 10:26:00.562175: validation loss: -0.7025 +2022-04-12 10:26:00.690465: Average global foreground Dice: [0.9217, 0.9751, 0.9861, 0.3965] +2022-04-12 10:26:00.742035: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:26:01.588000: lr: 0.000426 +2022-04-12 10:26:01.673179: saving scheduled checkpoint file... +2022-04-12 10:26:01.841744: saving checkpoint... +2022-04-12 10:26:16.374405: done, saving took 14.66 seconds +2022-04-12 10:26:16.583547: done +2022-04-12 10:26:16.627372: This epoch took 135.842601 s + +2022-04-12 10:26:16.646696: svmem(total=99932160000, available=80207790080, percent=19.7, used=15827963904, free=47094767616, active=39782928384, inactive=11183931392, buffers=43241472, cached=36966187008, shared=2064941056, slab=789508096) +2022-04-12 10:26:16.699067: +epoch: 485 +2022-04-12 10:28:05.586467: train loss : -0.7448 +2022-04-12 10:28:17.510793: validation loss: -0.6963 +2022-04-12 10:28:17.629350: Average global foreground Dice: [0.9289, 0.9776, 0.9854, 0.3746] +2022-04-12 10:28:17.691948: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:28:18.774007: lr: 0.0004 +2022-04-12 10:28:18.831432: saving scheduled checkpoint file... +2022-04-12 10:28:18.967674: saving checkpoint... +2022-04-12 10:28:29.186525: done, saving took 10.29 seconds +2022-04-12 10:28:29.682646: done +2022-04-12 10:28:29.816416: This epoch took 133.085232 s + +2022-04-12 10:28:29.877234: svmem(total=99932160000, available=80042749952, percent=19.9, used=15891648512, free=46912266240, active=40029040640, inactive=11131170816, buffers=43274240, cached=37084971008, shared=2166145024, slab=788799488) +2022-04-12 10:28:29.969545: +epoch: 486 +2022-04-12 10:30:20.219646: train loss : -0.7474 +2022-04-12 10:30:31.753947: validation loss: -0.6988 +2022-04-12 10:30:31.827201: Average global foreground Dice: [0.9393, 0.9748, 0.9822, 0.3802] +2022-04-12 10:30:31.945656: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:30:32.979974: lr: 0.000375 +2022-04-12 10:30:33.111117: saving scheduled checkpoint file... +2022-04-12 10:30:33.255758: saving checkpoint... +2022-04-12 10:30:46.884976: done, saving took 13.71 seconds +2022-04-12 10:30:47.037612: done +2022-04-12 10:30:47.173556: This epoch took 137.186457 s + +2022-04-12 10:30:47.192685: svmem(total=99932160000, available=80017416192, percent=19.9, used=15916789760, free=46868750336, active=40116142080, inactive=11088838656, buffers=43315200, cached=37103304704, shared=2166444032, slab=788537344) +2022-04-12 10:30:47.307941: +epoch: 487 +2022-04-12 10:32:35.221916: train loss : -0.7388 +2022-04-12 10:32:46.808824: validation loss: -0.7185 +2022-04-12 10:32:46.846366: Average global foreground Dice: [0.936, 0.9785, 0.9843, 0.4078] +2022-04-12 10:32:46.892820: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:32:47.671192: lr: 0.000348 +2022-04-12 10:32:47.691900: saving scheduled checkpoint file... +2022-04-12 10:32:47.785832: saving checkpoint... +2022-04-12 10:33:04.920347: done, saving took 17.19 seconds +2022-04-12 10:33:05.095506: done +2022-04-12 10:33:05.152125: This epoch took 137.777928 s + +2022-04-12 10:33:05.263467: svmem(total=99932160000, available=79675183104, percent=20.3, used=16234688512, free=46607634432, active=40137830400, inactive=11323125760, buffers=43347968, cached=37046489088, shared=2190807040, slab=789897216) +2022-04-12 10:33:05.317534: +epoch: 488 +2022-04-12 10:34:52.474168: train loss : -0.7561 +2022-04-12 10:35:03.457532: validation loss: -0.7113 +2022-04-12 10:35:03.479049: Average global foreground Dice: [0.9328, 0.9789, 0.9858, 0.4141] +2022-04-12 10:35:03.612405: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:35:04.927396: lr: 0.000322 +2022-04-12 10:35:05.073521: saving scheduled checkpoint file... +2022-04-12 10:35:05.236502: saving checkpoint... +2022-04-12 10:35:23.417684: done, saving took 18.27 seconds +2022-04-12 10:35:23.592499: done +2022-04-12 10:35:23.712747: saving checkpoint... +2022-04-12 10:35:39.460053: done, saving took 15.81 seconds +2022-04-12 10:35:39.475132: This epoch took 154.095050 s + +2022-04-12 10:35:39.476561: svmem(total=99932160000, available=80180273152, percent=19.8, used=15786680320, free=46824456192, active=39929901056, inactive=11316699136, buffers=43409408, cached=37277614080, shared=2133704704, slab=789897216) +2022-04-12 10:35:39.477922: +epoch: 489 +2022-04-12 10:37:27.508551: train loss : -0.7460 +2022-04-12 10:37:39.532874: validation loss: -0.7236 +2022-04-12 10:37:39.619708: Average global foreground Dice: [0.9324, 0.979, 0.9868, 0.4057] +2022-04-12 10:37:39.684128: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:37:40.621748: lr: 0.000296 +2022-04-12 10:37:40.690806: saving scheduled checkpoint file... +2022-04-12 10:37:40.840343: saving checkpoint... +2022-04-12 10:37:55.401921: done, saving took 14.67 seconds +2022-04-12 10:37:55.586967: done +2022-04-12 10:37:55.727437: saving checkpoint... +2022-04-12 10:38:10.798531: done, saving took 15.15 seconds +2022-04-12 10:38:11.051174: This epoch took 151.572305 s + +2022-04-12 10:38:11.134647: svmem(total=99932160000, available=79656755200, percent=20.3, used=16243281920, free=46286831616, active=40429064192, inactive=11342995456, buffers=43470848, cached=37358575616, shared=2200641536, slab=788590592) +2022-04-12 10:38:11.196257: +epoch: 490 +2022-04-12 10:39:59.659234: train loss : -0.7445 +2022-04-12 10:40:10.995596: validation loss: -0.7031 +2022-04-12 10:40:11.108670: Average global foreground Dice: [0.938, 0.9753, 0.9832, 0.3815] +2022-04-12 10:40:11.231402: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:40:12.305505: lr: 0.000269 +2022-04-12 10:40:12.367214: saving scheduled checkpoint file... +2022-04-12 10:40:12.522068: saving checkpoint... +2022-04-12 10:40:29.683735: done, saving took 17.27 seconds +2022-04-12 10:40:30.039565: done +2022-04-12 10:40:30.105506: This epoch took 138.849251 s + +2022-04-12 10:40:30.220414: svmem(total=99932160000, available=79939813376, percent=20.0, used=15970222080, free=46551158784, active=40236433408, inactive=11279003648, buffers=43511808, cached=37367267328, shared=2190548992, slab=789508096) +2022-04-12 10:40:30.376860: +epoch: 491 +2022-04-12 10:42:18.873247: train loss : -0.7567 +2022-04-12 10:42:30.022285: validation loss: -0.6984 +2022-04-12 10:42:30.056242: Average global foreground Dice: [0.936, 0.9773, 0.9838, 0.3584] +2022-04-12 10:42:30.109508: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:42:31.226286: lr: 0.000242 +2022-04-12 10:42:31.286335: saving scheduled checkpoint file... +2022-04-12 10:42:31.417727: saving checkpoint... +2022-04-12 10:42:41.191931: done, saving took 9.87 seconds +2022-04-12 10:42:41.371737: done +2022-04-12 10:42:41.417501: This epoch took 131.034061 s + +2022-04-12 10:42:41.463718: svmem(total=99932160000, available=79714181120, percent=20.2, used=16195514368, free=46308024320, active=40487989248, inactive=11274838016, buffers=43544576, cached=37385076736, shared=2190839808, slab=788987904) +2022-04-12 10:42:41.524216: +epoch: 492 +2022-04-12 10:44:28.487061: train loss : -0.7589 +2022-04-12 10:44:40.056155: validation loss: -0.7107 +2022-04-12 10:44:40.163029: Average global foreground Dice: [0.9222, 0.9794, 0.9885, 0.4172] +2022-04-12 10:44:40.206215: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:44:41.359624: lr: 0.000215 +2022-04-12 10:44:41.437848: saving scheduled checkpoint file... +2022-04-12 10:44:41.556901: saving checkpoint... +2022-04-12 10:44:55.848653: done, saving took 14.38 seconds +2022-04-12 10:44:56.075455: done +2022-04-12 10:44:56.143142: This epoch took 134.512360 s + +2022-04-12 10:44:56.190466: svmem(total=99932160000, available=79730827264, percent=20.2, used=16179380224, free=46316965888, active=40476762112, inactive=11274784768, buffers=43577344, cached=37392236544, shared=2190233600, slab=789979136) +2022-04-12 10:44:56.245937: +epoch: 493 +2022-04-12 10:46:45.227601: train loss : -0.7510 +2022-04-12 10:46:57.066272: validation loss: -0.7065 +2022-04-12 10:46:58.010572: Average global foreground Dice: [0.9263, 0.9776, 0.985, 0.4032] +2022-04-12 10:46:59.143368: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:47:00.777303: lr: 0.000187 +2022-04-12 10:47:01.045623: saving scheduled checkpoint file... +2022-04-12 10:47:01.350512: saving checkpoint... +2022-04-12 10:47:28.468549: done, saving took 27.17 seconds +2022-04-12 10:47:29.303013: done +2022-04-12 10:47:29.447194: This epoch took 153.147934 s + +2022-04-12 10:47:29.535241: svmem(total=99932160000, available=79577288704, percent=20.4, used=16323018752, free=46143774720, active=40617152512, inactive=11302535168, buffers=43642880, cached=37421723648, shared=2200145920, slab=791007232) +2022-04-12 10:47:29.597875: +epoch: 494 +2022-04-12 10:49:17.065576: train loss : -0.7537 +2022-04-12 10:50:17.237202: validation loss: -0.6794 +2022-04-12 10:50:17.458920: Average global foreground Dice: [0.9448, 0.9715, 0.9821, 0.3624] +2022-04-12 10:50:17.501997: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:50:24.119765: lr: 0.000158 +2022-04-12 10:50:32.920927: saving scheduled checkpoint file... +2022-04-12 10:50:54.130068: saving checkpoint... +2022-04-12 10:53:37.696169: done, saving took 163.67 seconds +2022-04-12 10:53:37.933341: done +2022-04-12 10:53:38.053044: This epoch took 368.412107 s + +2022-04-12 10:53:38.176798: svmem(total=99932160000, available=79765733376, percent=20.2, used=16143732736, free=46303096832, active=40422625280, inactive=11333427200, buffers=43749376, cached=37441581056, shared=2190807040, slab=791064576) +2022-04-12 10:53:38.245086: +epoch: 495 +2022-04-12 10:55:26.141762: train loss : -0.7523 +2022-04-12 10:55:38.445832: validation loss: -0.7009 +2022-04-12 10:55:38.519017: Average global foreground Dice: [0.9267, 0.9752, 0.9808, 0.3921] +2022-04-12 10:55:38.604202: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:55:39.578197: lr: 0.00013 +2022-04-12 10:55:39.693735: saving scheduled checkpoint file... +2022-04-12 10:55:39.848933: saving checkpoint... +2022-04-12 10:56:02.783547: done, saving took 23.02 seconds +2022-04-12 10:56:03.617265: done +2022-04-12 10:56:04.024285: This epoch took 145.660216 s + +2022-04-12 10:56:04.798144: svmem(total=99932160000, available=79683129344, percent=20.3, used=16240500736, free=46202834944, active=40513626112, inactive=11331371008, buffers=43794432, cached=37445029888, shared=2176741376, slab=803450880) +2022-04-12 10:56:05.909495: +epoch: 496 +2022-04-12 10:57:55.664013: train loss : -0.7543 +2022-04-12 10:58:07.515483: validation loss: -0.7158 +2022-04-12 10:58:07.572175: Average global foreground Dice: [0.924, 0.9733, 0.9872, 0.3902] +2022-04-12 10:58:07.620443: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 10:58:08.793699: lr: 0.0001 +2022-04-12 10:58:08.921727: saving scheduled checkpoint file... +2022-04-12 10:58:09.086082: saving checkpoint... +2022-04-12 10:58:27.335734: done, saving took 18.34 seconds +2022-04-12 10:58:27.749037: done +2022-04-12 10:58:27.824850: This epoch took 140.691015 s + +2022-04-12 10:58:27.848451: svmem(total=99932160000, available=80044494848, percent=19.9, used=15945564160, free=46544977920, active=40241164288, inactive=11279343616, buffers=43843584, cached=37397774336, shared=2110115840, slab=789671936) +2022-04-12 10:58:27.970867: +epoch: 497 +2022-04-12 11:00:16.750272: train loss : -0.7583 +2022-04-12 11:00:28.386005: validation loss: -0.7126 +2022-04-12 11:00:28.588941: Average global foreground Dice: [0.9291, 0.976, 0.9853, 0.4135] +2022-04-12 11:00:28.849877: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 11:00:30.098390: lr: 6.9e-05 +2022-04-12 11:00:30.148870: saving scheduled checkpoint file... +2022-04-12 11:00:30.296546: saving checkpoint... +2022-04-12 11:00:48.273590: done, saving took 18.06 seconds +2022-04-12 11:00:48.601301: done +2022-04-12 11:00:48.673311: This epoch took 140.631705 s + +2022-04-12 11:00:48.821504: svmem(total=99932160000, available=79938187264, percent=20.0, used=15961632768, free=46420045824, active=40264691712, inactive=11387645952, buffers=43884544, cached=37506596864, shared=2200334336, slab=790245376) +2022-04-12 11:00:48.974936: +epoch: 498 +2022-04-12 11:02:37.887730: train loss : -0.7430 +2022-04-12 11:02:50.446595: validation loss: -0.6785 +2022-04-12 11:02:50.525530: Average global foreground Dice: [0.9209, 0.9763, 0.9842, 0.3829] +2022-04-12 11:02:50.670074: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 11:02:51.334514: lr: 3.7e-05 +2022-04-12 11:02:51.402931: saving scheduled checkpoint file... +2022-04-12 11:02:51.672959: saving checkpoint... +2022-04-12 11:03:04.552525: done, saving took 12.97 seconds +2022-04-12 11:03:04.697676: done +2022-04-12 11:03:04.711412: This epoch took 135.651082 s + +2022-04-12 11:03:04.761385: svmem(total=99932160000, available=79945998336, percent=20.0, used=15963770880, free=46419644416, active=40260313088, inactive=11385700352, buffers=43950080, cached=37504794624, shared=2190467072, slab=789774336) +2022-04-12 11:03:04.768410: +epoch: 499 +2022-04-12 11:04:53.398350: train loss : -0.7518 +2022-04-12 11:05:04.554357: validation loss: -0.7002 +2022-04-12 11:05:04.615375: Average global foreground Dice: [0.9272, 0.9797, 0.9843, 0.4068] +2022-04-12 11:05:04.628510: (interpret this as an estimate for the Dice of the different classes. This is not exact.) +2022-04-12 11:05:05.608487: lr: 0.0 +2022-04-12 11:05:05.667055: saving scheduled checkpoint file... +2022-04-12 11:05:05.770279: saving checkpoint... +2022-04-12 11:05:21.323375: done, saving took 15.61 seconds +2022-04-12 11:05:21.533899: done +2022-04-12 11:05:21.590512: This epoch took 136.767951 s + +2022-04-12 11:05:21.775597: saving checkpoint... +2022-04-12 11:05:36.294306: done, saving took 14.57 seconds +2022-04-12 11:18:40.484149: finished prediction +2022-04-12 11:18:40.643665: evaluation of raw predictions